diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index d52b19c7..71c54885 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -61,6 +61,11 @@ Screenshots of EffectCraft itself, rendered headlessly, showing only procedurall - Procedural demo content (solids, shapes, gradients, text animations): `crates/engine/src/demo.rs`. - Text animator presets (Typewriter, Fade Up Characters, Bounce In Words, Tracking In, Scramble, Blur In, Jitter, Drop In Lines): `crates/engine/presets/text_animators.json` (sidecar `.attribution`). +- Sample scripts (File ▸ Scripts) and ScriptUI panel: `crates/engine/scripts/Rename Layers.jsx`, + `crates/engine/scripts/Sort Layers by In Point.jsx`, `crates/engine/scripts/Create Null at Selected Layers.jsx`, + `crates/engine/scripts/Render Queue Batch.jsx`, `crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx` + (sidecars `.attribution`). +- Example effect plug-in: `examples/plugins/posterize-bands` (plug-in API v1, WebAssembly). ## First-party brand marks diff --git a/Cargo.lock b/Cargo.lock index 87cd95d1..c3dcb41c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,6 +1197,7 @@ dependencies = [ "effectcraft-project", "effectcraft-raster", "effectcraft-track", + "log", "rayon", "serde", "serde_json", @@ -1237,6 +1238,7 @@ name = "effectcraft-export" version = "0.1.0" dependencies = [ "effectcraft-color", + "effectcraft-keyframe", "effectcraft-media", "effectcraft-opusenc", "effectcraft-project", @@ -1311,12 +1313,14 @@ dependencies = [ "effectcraft-expr", "effectcraft-media", "effectcraft-pdf", + "effectcraft-plugin", "effectcraft-project", "effectcraft-psd", "effectcraft-raster", "effectcraft-script", "effectcraft-svg", "effectcraft-time", + "rayon", "serde_json", ] @@ -1410,6 +1414,18 @@ dependencies = [ "miniz_oxide 0.8.9", ] +[[package]] +name = "effectcraft-plugin" +version = "0.1.0" +dependencies = [ + "effectcraft-effects", + "effectcraft-keyframe", + "effectcraft-project", + "effectcraft-raster", + "serde_json", + "wasmi", +] + [[package]] name = "effectcraft-project" version = "0.1.0" @@ -2114,6 +2130,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -2288,7 +2310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed4a1bb24121291d27230c1b1b44e07d6a9b28cefdb32fe1581dfb84e14f940a" dependencies = [ "bytemuck", - "foldhash", + "foldhash 0.2.0", "hashbrown 0.17.1", "log", "peniko", @@ -2428,6 +2450,15 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -2436,7 +2467,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -2447,7 +2478,7 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -2765,6 +2796,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lebe" version = "0.5.3" @@ -4444,6 +4481,12 @@ dependencies = [ "serde", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -4471,6 +4514,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +[[package]] +name = "string-interner" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23de088478b31c349c9ba67816fa55d9355232d63c3afea8bf513e31f0f1d2c0" +dependencies = [ + "hashbrown 0.15.5", + "serde", +] + [[package]] name = "strum" version = "0.28.0" @@ -5139,6 +5192,99 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2608e8bb6d67fd68f5a8d0eb1363d6e7bcbc1f8ded5a0bd3a1e382462b876b22" +dependencies = [ + "leb128fmt", + "wasmparser 0.261.0", +] + +[[package]] +name = "wasmi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78693fcdd618e0fc34af59c6b8efa9ac5d58c68df940beff4bedddb6acfe7c27" +dependencies = [ + "spin", + "wasmi_collections", + "wasmi_core", + "wasmi_ir", + "wasmparser 0.228.0", + "wat", +] + +[[package]] +name = "wasmi_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8be2aa467cf2d29e96ff759472c36eeb44a3c81c67fc9cb76c9a24c519c557" +dependencies = [ + "string-interner", +] + +[[package]] +name = "wasmi_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69372d5fda3ea3d1e0aa6603c7888110e0187e88ea17cd8fc2e2df0a0e1f37fa" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmi_ir" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f17b774caa13c618c7244f1ee51fe23c5e7b8538a471fa46d9949779758aed6" +dependencies = [ + "wasmi_core", +] + +[[package]] +name = "wasmparser" +version = "0.228.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abf1132c1fdf747d56bbc1bb52152400c70f336870f968b85e89ea422198ae3" +dependencies = [ + "bitflags 2.13.2", + "indexmap", +] + +[[package]] +name = "wasmparser" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8" +dependencies = [ + "bitflags 2.13.2", + "indexmap", +] + +[[package]] +name = "wast" +version = "261.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776443145731a4062e5b0d392892a2005909b6ab72d9fdc3cad53dd1a714e44a" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder", +] + +[[package]] +name = "wat" +version = "1.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b4d1a49ea73a8f3326e74e3a05db667001b16bd1035ed3356fc1a0ed05ca7f" +dependencies = [ + "wast", +] + [[package]] name = "wayland-backend" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index cda7c302..6641f2c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } effectcraft-opusenc = { path = "crates/opusenc" } effectcraft-script = { path = "crates/script" } +effectcraft-plugin = { path = "crates/plugin" } # foundation thiserror = "2" @@ -53,6 +54,8 @@ proptest = "1" # scripting (expressions) boa_engine = { version = "0.22", default-features = false } +# WebAssembly effect plug-ins (sandboxed interpreter; MIT/Apache-2.0, pure Rust) +wasmi = { version = "2", default-features = false, features = ["std", "validate", "wat", "deterministic", "portable-dispatch"] } # geometry / fonts kurbo = { version = "0.13", features = ["serde"] } diff --git a/README.md b/README.md index 3bff64fb..3a9cabbc 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,11 @@ tools, and Front, Top, Left and Custom views. ## Export A Render Queue like After Effects', with Render Settings and Output Modules: **H.264** MP4 and -**ProRes** MOV (Proxy to 4444 XQ with alpha) with audio, PNG, JPEG, TIFF and 32-bit EXR -sequences, and animated GIF. The same queue runs from the command line. Every encoder is +**ProRes** MOV (Proxy to 4444 XQ with alpha) with audio, **WebM** (VP9 with inter frames and +alpha, Opus), PNG, JPEG, TIFF and 32-bit EXR sequences, animated GIF and WAV/AIFF. Field +rendering with 3:2 pulldown, effect/solo/guide/depth overrides, crop, region of interest and +resize, Render Settings and Output Module templates with defaults, post-render actions, storage +overflow and render logs. The same queue runs from the command line. Every encoder is FilmCraft's pure-Rust code; there is no FFmpeg inside. **Lottie** goes both ways: File ▸ Export ▸ Lottie JSON… writes a composition (precomps, shape, diff --git a/ROADMAP.md b/ROADMAP.md index 7c7b337b..c02dd556 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,12 +25,14 @@ once. ## How far from full parity -Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026): +Measured feature by feature in [docs/parity.md](docs/parity.md) (3 October 2026): -- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of - professional daily-use features, 65% of the long tail); all 298 After Effects effects. -- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude - Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features. -- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API. +- **≈ 94% of After Effects' features, weighted by importance**, counting partial features as half + done (≈ 98% with per-feature fractions): 80 of 92 features done, 12 partial, none missing; all + 298 After Effects effects (42 still simplified). +- **≈ 19–22 agent-hours of work remain**: about **4–5 hours of wall-clock time** with five Claude + Opus 5.5 agents in parallel, ≈ 3 hours for the essentials and daily-use features. +- Hardest remaining pieces: the web app's threading, wider GPU effect coverage, Advanced 3D + completion, and learned-model quality for Roto Brush and face tracking. Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/apps/effectcraft-web/src/store.rs b/apps/effectcraft-web/src/store.rs index 0e4ea029..471d96b5 100644 --- a/apps/effectcraft-web/src/store.rs +++ b/apps/effectcraft-web/src/store.rs @@ -216,6 +216,18 @@ impl ConfigStore for WebConfig { self.store.changed(); Ok(()) } + fn list(&self, dir: &str) -> Vec { + let pre = format!("{CONFIG}{}/", dir.trim_end_matches('/')); + let mut v: Vec = self + .store + .lock() + .list(&pre) + .into_iter() + .filter_map(|(k, ..)| k.strip_prefix(&pre).filter(|n| !n.is_empty() && !n.contains('/')).map(str::to_string)) + .collect(); + v.sort(); + v + } fn dir(&self) -> Option { Some(PathBuf::from("/")) } diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index 157ffba2..bef701cb 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -66,6 +66,11 @@ fn main() -> eframe::Result { // Settings, shortcut presets and the crash-recovery sentinel live in the platform // config directory. Agent-driven runs (`--control`) skip crash recovery. if let Some(dir) = config_dir() { + // WebAssembly effect plug-ins in /Plug-ins load before the menus are built. + let plugins = dir.join("Plug-ins"); + if plugins.is_dir() { + let _ = session.execute("effect.plugins.load", json!({"folder": plugins.to_string_lossy()})); + } session.config = Some(std::sync::Arc::new(effectcraft_engine::config::DirConfig::new(dir))); } session.load_settings(); diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index 848d3ee0..9504094a 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -265,6 +265,32 @@ fn bridge_rejects_non_loopback() { assert!(Backend::bridge("localhost:9877").is_ok()); } +#[test] +fn script_ui_and_history_tools() { + let mut sess = Session::default(); + effectcraft_script::install(&mut sess); + let mut s = McpServer::new(Backend::headless(sess)); + let code = "var d = new Window('dialog', 'Ask'); var g = d.add('group'); g.add('button', undefined, 'OK', {name: 'ok'}); \ + if (d.show() == 1) app.project.items.addComp('Answered', 64, 64, 1, 1, 24); 'done'"; + let r = call_json(&mut s, "run_script", json!({"code": code})); + assert_eq!(r["waiting"], json!(true), "{r}"); + let list = call_json(&mut s, "script_ui", json!({})); + assert_eq!(list[0]["title"], "Ask"); + let tree = call_json(&mut s, "script_ui", json!({"action": "get"})); + assert_eq!(tree["root"]["children"][0]["children"][0]["name"], "ok"); + let r = call_json(&mut s, "script_ui", json!({"action": "click", "widget": "ok"})); + assert_eq!(r["result"], json!("done"), "{r}"); + // History: undo the comp, make another, and jump back to the first on its branch. + call_json(&mut s, "undo", json!({})); + call_json(&mut s, "execute_command", json!({"command": "comp.new", "params": {"name": "Other", "width": 64, "height": 64}})); + let h = call_json(&mut s, "history", json!({})); + let branch = h["states"].as_array().unwrap().iter().find(|n| n["depth"] == 1).unwrap()["index"].clone(); + let h = call_json(&mut s, "history", json!({"goto": branch})); + let cur = h["current"].as_u64().unwrap() as usize; + assert_eq!(h["states"][cur]["depth"], 0, "the jumped-to state is now the working line"); + assert!(call_json(&mut s, "get_project", json!({})).to_string().contains("Answered")); +} + #[test] fn run_script_round_trip() { let mut sess = Session::default(); diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 739c5524..9e1bfee5 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -315,6 +315,32 @@ fn redo(b: &mut Backend, a: &Value) -> Result { history(b, a, "edit.redo") } +fn script_ui_tool(b: &mut Backend, a: &Value) -> Result { + let action = get(a, "action").and_then(Value::as_str).unwrap_or("list"); + let id = match action { + "list" => "scriptui.list", + "get" => "scriptui.get", + "click" => "scriptui.click", + "set" => "scriptui.set", + "close" => "scriptui.close", + other => return Err(Error::BadArgs(format!("unknown action `{other}` (list | get | click | set | close)"))), + }; + let p = obj(&[("window", get(a, "window")), ("widget", get(a, "widget")), ("value", get(a, "value")), ("result", get(a, "result"))]); + json_reply(b.exec(id, p)?) +} + +fn history_tool(b: &mut Backend, a: &Value) -> Result { + if let Some(g) = get(a, "goto") { + let p = match g { + Value::String(id) if id.parse::().is_err() => json!({"id": id}), + Value::String(n) => json!({"index": n.parse::().unwrap_or(0)}), + v => json!({"index": v}), + }; + b.exec("edit.history.goto", p)?; + } + json_reply(b.exec("edit.history.list", json!({}))?) +} + // ------------------------------------------------------------------ bridge-only tools fn screenshot(b: &mut Backend, a: &Value) -> Result { @@ -535,6 +561,31 @@ static TOOLS: &[ToolDef] = &[ schema: || schema(json!({"steps": {"type": "integer", "minimum": 1}}), &[]), run: redo, }, + ToolDef { + name: "history", + description: "The branching undo history (History panel): without arguments lists every state {index, id, label, parent, depth (0 = working line, >0 = branch), current, future}; undoing and then editing keeps the undone states as a branch. With `goto` (an index or id from the listing) jumps to that state, on any branch, without losing the others.", + bridge_only: false, + schema: || schema(json!({"goto": {"type": ["integer", "string"], "description": "State index or id to jump to (edit.history.goto)."}}), &[]), + run: history_tool, + }, + ToolDef { + name: "script_ui", + description: "Drive ScriptUI windows that scripts opened (dialogs, palettes, dockable panels). action `list` (default): open windows; `get`: a window's control tree (type, name, text, value, checked, items, selection, bounds); `click`: press a button / toggle a checkbox / pick a radio button; `set`: type text, move a slider, pick a list item (index or text); `close`: close a window (`result` for a dialog's show(), default 2 = Cancel). Controls by id, \"#id\", properties.name or text. A dialog's script resumes when the dialog closes.", + bridge_only: false, + schema: || { + schema( + json!({ + "action": {"type": "string", "enum": ["list", "get", "click", "set", "close"]}, + "window": {"type": ["integer", "string"], "description": "Window id or title (optional when one is open)."}, + "widget": {"type": ["integer", "string"], "description": "Control id, \"#id\", properties.name or text."}, + "value": {"description": "For `set`: text, number, bool, item index or item text."}, + "result": {"type": "integer", "description": "For `close`: what the dialog's show() returns."} + }), + &[], + ) + }, + run: script_ui_tool, + }, ToolDef { name: "screenshot", description: "Screenshot of the live app window, or one panel (`Composition`, `Timeline`, `EffectControls`, `Project`, ...) or element id, as PNG (longest side `max_side`, default 1600). Use it to check what the UI looks like after an action.", diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index 965cdbd2..9b065360 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -19,4 +19,5 @@ effectcraft-track = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +log = { workspace = true } diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs index dd97b585..ff4e51fe 100644 --- a/crates/effects/src/camera_tracker.rs +++ b/crates/effects/src/camera_tracker.rs @@ -8,8 +8,12 @@ //! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing //! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. //! -//! The effect passes its input through; with **Render Track Points** on it draws the solved -//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! **Solve Lens Distortion** (Advanced; also on with Detailed Analysis) solves radial lens +//! distortion `k1, k2` jointly with the camera; **Undistort Footage** then renders the input +//! undistorted (the pinhole image the solved camera sees) so 3D layers line up with it. +//! +//! The effect passes its input through (undistorted when asked); with **Render Track Points** +//! on it draws the solved points (as the viewer shows them) into the frame. The viewer overlay, target and the Create //! commands live in the engine and the UI. use std::collections::HashMap; @@ -19,6 +23,7 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::{Image, Px}; use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; +use rayon::prelude::*; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -58,6 +63,8 @@ pub fn specs() -> Vec { p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/lensDistortion", "Solve Lens Distortion", Value::Bool(false), ParamUi::Checkbox), + p("advanced/undistort", "Undistort Footage", Value::Bool(false), ParamUi::Checkbox), p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), @@ -104,13 +111,15 @@ pub fn settings(params: &Params) -> SolveSettings { hfov: f(params, "horizontalAngleOfView"), method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), deleted: del, + lens_distortion: b(params, "advanced/lensDistortion") || b(params, "advanced/detailedAnalysis"), } } /// What a solve depends on besides the tracks. pub fn solve_key(st: &SolveSettings) -> String { let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; - format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) + let lens = if st.lens_distortion { "|lens" } else { "" }; + format!("{:?}|{hfov}|{:?}|{:?}{lens}", st.shot, st.method, st.deleted) } fn fnv(bytes: &[u8]) -> u64 { @@ -165,18 +174,49 @@ pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) } -/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +/// The solved points visible on frame `k`: (id, layer pixel position in the footage, camera +/// depth). pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + projected_with(solve, k, false) +} + +/// [`projected`] into the footage, or (`undistorted`) into the undistorted image. +pub fn projected_with(solve: &CameraSolve, k: usize, undistorted: bool) -> Vec<(u32, [f64; 2], f64)> { let Some(cam) = solve.frames.get(k) else { return vec![] }; solve .visible(k) .filter_map(|p| { let c = cam.to_cam(p.pos); - cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + solve.project(k, p.pos, undistorted).map(|q| (p.id, q, c[2])) }) .collect() } +/// Whether the instance renders its input undistorted (Undistort Footage with a solved +/// distortion). +pub fn undistorts(params: &Params, solve: &CameraSolve) -> bool { + b(params, "advanced/undistort") && solve.distortion.is_some_and(|d| !d.is_none()) +} + +/// The input resampled through the solved distortion: every output pixel shows the footage +/// at its distorted position (transparent where that falls outside the layer). +fn undistort(buf: &Buf, sol: &CameraSolve) -> Buf { + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let mut out = Image::new(buf.img.width, buf.img.height); + let size = sol.size; + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let p = [(x as f64 + 0.5 - buf.offset[0]) / sc, (y as f64 + 0.5 - buf.offset[1]) / sc]; + let q = sol.to_image(p); + if q[0] < 0.0 || q[1] < 0.0 || q[0] > size[0] || q[1] > size[1] { + continue; + } + *px = buf.img.sample_bilinear_clamped(q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]); + } + }); + Buf { img: out, offset: buf.offset, scale: buf.scale } +} + /// Median of the depths. pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { let mut z: Vec = v.iter().map(|x| x.2).collect(); @@ -188,15 +228,23 @@ pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { } fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { - if !b(ctx.params, "renderTrackPoints") { + let points = b(ctx.params, "renderTrackPoints"); + if !points && !b(ctx.params, "advanced/undistort") { return buf; } let Some(sol) = solve(ctx.params) else { return buf }; if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { return buf; } + let und = undistorts(ctx.params, &sol); + if und { + buf = undistort(&buf, &sol); + } + if !points { + return buf; + } let Some(k) = sol.frame_at(ctx.time) else { return buf }; - let pts = projected(&sol, k); + let pts = projected_with(&sol, k, und); let med = median_depth(&pts); let size = f(ctx.params, "trackPointSize"); let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; @@ -287,5 +335,37 @@ mod tests { let st = settings(&Params::default()); assert_eq!(st.shot, ShotType::FixedAngle); assert!(solve_key(&st).contains("FixedAngle")); + assert!(!st.lens_distortion && !solve_key(&st).contains("lens")); + } + + #[test] + fn undistort_footage_remaps_through_the_solved_lens() { + use effectcraft_track::camtrack::Distortion; + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + distortion: Some(Distortion { k1: -0.2, k2: 0.0, radius: 40.0 }), + ..Default::default() + }; + let mut params = params_with(&sol, false); + params.values.insert("advanced/undistort".into(), Value::Bool(true)); + assert!(undistorts(¶ms, &sol)); + // A horizontal ramp: undistortion keeps the centre and pulls edge content outwards. + let mut img = Image::new(64, 48); + for (i, px) in img.data.iter_mut().enumerate() { + let x = (i % 64) as f32 / 64.0; + *px = [x, x, x, 1.0]; + } + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let at = |im: &Image, x: u32, y: u32| im.data[im.idx(x, y)][0]; + assert!((at(&out.img, 32, 24) - at(&img, 32, 24)).abs() < 0.02); + // Barrel distortion compresses the edges: the ideal pixel at x = 56 shows footage nearer the centre. + assert!(at(&out.img, 56, 24) < at(&img, 56, 24) - 0.01, "{} vs {}", at(&out.img, 56, 24), at(&img, 56, 24)); + let sp = sol.to_image([56.0, 24.0]); + assert!(sp[0] < 56.0 && (sol.to_ideal(sp)[0] - 56.0).abs() < 1e-6); } } diff --git a/crates/effects/src/face_track.rs b/crates/effects/src/face_track.rs new file mode 100644 index 00000000..7b925165 --- /dev/null +++ b/crates/effects/src/face_track.rs @@ -0,0 +1,45 @@ +//! Face tracking results (Tracker panel ▸ Method ▸ Face Tracking (Detailed Features)): +//! +//! - **Face Track Points** holds one point control per facial landmark (eyebrows, eyes and +//! pupils, nose, nostrils, mouth, chin and jaw), keyed on every tracked frame by the engine's +//! mask tracker ([`effectcraft_track::face`]); expressions and other layers can follow them. +//! - **Face Measurements** holds the measurements derived from those points (head position, +//! scale and orientation, eye openness, eyebrow raise, mouth openness, width and offset), keyed +//! by Extract & Copy Face Measurements (`track.extractFaceMeasurements`). +//! +//! Both pass their input through unchanged: they are data, like expression controls. + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_track::face::{LANDMARKS, MEASUREMENTS}; + +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, num, p, slider}; + +/// Effect ids. +pub const POINTS_ID: &str = "ec.utility.facetrackpoints"; +pub const MEASUREMENTS_ID: &str = "ec.utility.facemeasurements"; + +fn passthrough(_: &EffectCtx, b: Buf) -> Buf { + b +} + +fn measurement_ui(id: &str) -> ParamUi { + match id { + "headPositionX" | "headPositionY" => slider(-100_000.0, 100_000.0, 0.0, 1920.0, 1), + "headOrientationX" | "headOrientationY" | "headOrientationZ" => slider(-180.0, 180.0, -90.0, 90.0, 1), + _ => slider(-100_000.0, 100_000.0, 0.0, 200.0, 1), + } +} + +pub fn specs() -> Vec { + let points: Vec = LANDMARKS + .iter() + // Point defaults are fractions of the layer size: the canonical face in the middle. + .map(|(id, name, q)| p(id, name, Value::Vec2([0.5 + 0.25 * q[0], 0.5 + 0.25 * q[1]]), ParamUi::Point)) + .collect(); + let measurements: Vec = MEASUREMENTS.iter().map(|(id, name)| p(id, name, num(0.0), measurement_ui(id))).collect(); + vec![ + EffectSpec { id: POINTS_ID, name: "Face Track Points", category: "Utility", params: points, render: passthrough, gpu: false, float: true }, + EffectSpec { id: MEASUREMENTS_ID, name: "Face Measurements", category: "Utility", params: measurements, render: passthrough, gpu: false, float: true }, + ] +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c4a47508..c81cb8db 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -22,6 +22,7 @@ mod distort; mod distort2; mod distort3; pub mod distort4; +pub mod face_track; mod generate; mod generate2; mod generate3; @@ -41,6 +42,7 @@ pub mod ocio_config; pub mod paint; mod perspective; mod perspective2; +pub mod plugin; pub mod psim; pub mod puppet; pub mod roto; @@ -412,6 +414,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(puppet::specs()); v.extend(warp_stab::specs()); v.extend(camera_tracker::specs()); + v.extend(face_track::specs()); v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { @@ -423,14 +426,37 @@ pub fn registry() -> &'static [EffectSpec] { }) } +/// A built-in effect or a registered plug-in by id. pub fn find(id: &str) -> Option<&'static EffectSpec> { - registry().iter().find(|s| s.id == id) + registry().iter().find(|s| s.id == id).or_else(|| plugin::plugins().into_iter().find(|s| s.id == id)) +} + +/// Every effect: the built-ins ([`registry`]) and the registered plug-ins +/// ([`plugin::register_plugin`]), sorted by category then name. +pub fn all() -> Vec<&'static EffectSpec> { + let plugins = plugin::plugins(); + let mut v: Vec<&'static EffectSpec> = registry().iter().collect(); + if !plugins.is_empty() { + v.extend(plugins); + v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); + } + v +} + +/// Effects & Presets categories: [`CATEGORIES`] then the plug-ins' own, alphabetically. +pub fn categories() -> Vec<&'static str> { + let mut v: Vec<&'static str> = CATEGORIES.to_vec(); + let mut extra: Vec<&'static str> = plugin::plugins().iter().map(|s| s.category).filter(|c| !CATEGORIES.contains(c)).collect(); + extra.sort_unstable(); + extra.dedup(); + v.extend(extra); + v } /// Find by id or (case-insensitive) display name. pub fn lookup(name_or_id: &str) -> Option<&'static EffectSpec> { find(name_or_id) - .or_else(|| registry().iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) + .or_else(|| all().into_iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) .or_else(|| { // After Effects' third-party display names we register under a generic name. keylight::KEYLIGHT_ALIASES.iter().any(|a| a.eq_ignore_ascii_case(name_or_id)).then(|| find("ec.keying.keylight")).flatten() diff --git a/crates/effects/src/plugin.rs b/crates/effects/src/plugin.rs new file mode 100644 index 00000000..242f52b9 --- /dev/null +++ b/crates/effects/src/plugin.rs @@ -0,0 +1,338 @@ +//! The effect plug-in API, version [`PLUGIN_API_VERSION`]. +//! +//! A plug-in is anything implementing [`EffectPlugin`]: a [`PluginManifest`] (stable id, +//! display name, Effects & Presets category, typed parameters) and a `render` that processes a +//! [`PluginFrame`] in place. [`register_plugin`] turns it into an ordinary [`EffectSpec`]: +//! instances are property groups like every built-in effect, so plug-in parameters animate, +//! take expressions, show in Effect Controls and are addressable by agents, and the effect is +//! applied by id (`effect.apply {"effect": "org.example.posterize"}`) like a built-in. +//! +//! Plug-ins come from Rust code (implement the trait and register it at start-up) or from +//! sandboxed WebAssembly modules (`effectcraft-plugin`, see `docs/plugins.md`), which implement +//! the same contract through a small C-like ABI. +//! +//! The contract (stable within an API version): +//! * pixels are premultiplied RGBA `f32` (0–1, may exceed 1 in 32 bpc projects), row-major, +//! in the layer's buffer, which may be scaled (preview resolution) and padded; +//! * parameters are flattened to `f64`s in declaration order: slider / angle / checkbox (0 or 1) +//! / popup (0-based index) take one value, point two (**frame pixels**, already mapped to the +//! buffer), colour four (RGBA 0–1); +//! * `time` is layer time in seconds; `scale` is the buffer's pixels per layer pixel (scale +//! pixel distances like blur radii by it); +//! * rendering must be deterministic (same inputs, same output) and may run on several threads +//! at once. + +use std::sync::{Arc, OnceLock, RwLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use serde::{Deserialize, Serialize}; + +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, RenderFn}; + +/// The plug-in API version this build implements. +pub const PLUGIN_API_VERSION: u32 = 1; + +/// The most plug-in effects one process can register. +pub const MAX_PLUGINS: usize = 64; + +/// A plug-in parameter's kind, default and range. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum PluginParamKind { + Slider { + default: f64, + #[serde(default)] + min: f64, + #[serde(default = "hundred")] + max: f64, + /// Slider range (defaults to min..max). + #[serde(default, rename = "sliderMin")] + slider_min: Option, + #[serde(default, rename = "sliderMax")] + slider_max: Option, + #[serde(default = "two")] + decimals: u8, + }, + Angle { + #[serde(default)] + default: f64, + }, + Checkbox { + #[serde(default)] + default: bool, + }, + Popup { + options: Vec, + /// 0-based index. + #[serde(default)] + default: u32, + }, + /// Default as fractions of the layer size (`[0.5, 0.5]` = centre). + Point { + #[serde(default = "centre")] + default: [f64; 2], + }, + /// RGBA, 0–1. + Color { + #[serde(default = "white")] + default: [f64; 4], + }, +} + +fn hundred() -> f64 { + 100.0 +} +fn two() -> u8 { + 2 +} +fn centre() -> [f64; 2] { + [0.5, 0.5] +} +fn white() -> [f64; 4] { + [1.0; 4] +} + +impl PluginParamKind { + /// How many `f64`s the parameter flattens to. + pub fn width(&self) -> usize { + match self { + PluginParamKind::Point { .. } => 2, + PluginParamKind::Color { .. } => 4, + _ => 1, + } + } +} + +/// One parameter of a plug-in effect. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PluginParam { + /// Stable id (the property's match name; letters, digits, `_`). + pub id: String, + pub name: String, + #[serde(flatten)] + pub kind: PluginParamKind, +} + +/// What a plug-in declares about itself. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PluginManifest { + /// The plug-in API version it was written for (must be [`PLUGIN_API_VERSION`]). + pub api: u32, + /// Stable, reverse-domain effect id (`org.example.posterize`); `ec.` is reserved. + pub id: String, + pub name: String, + /// Effects & Presets category: one of [`crate::CATEGORIES`] or a new one. + pub category: String, + #[serde(default)] + pub version: String, + #[serde(default)] + pub author: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub params: Vec, +} + +impl PluginManifest { + /// Check the manifest against this API version. + pub fn validate(&self) -> Result<(), String> { + if self.api != PLUGIN_API_VERSION { + return Err(format!("plug-in `{}` targets plug-in API {} but this build implements API {PLUGIN_API_VERSION}", self.id, self.api)); + } + let ok_id = |s: &str| !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')); + if !ok_id(&self.id) || self.id.starts_with("ec.") { + return Err(format!("bad plug-in id `{}` (reverse-domain, letters, digits, `.`, `_`, `-`; `ec.` is reserved)", self.id)); + } + if self.name.trim().is_empty() || self.category.trim().is_empty() { + return Err(format!("plug-in `{}` needs a name and a category", self.id)); + } + let mut seen = std::collections::HashSet::new(); + for p in &self.params { + if p.id.is_empty() || !p.id.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') || !seen.insert(p.id.as_str()) { + return Err(format!("plug-in `{}`: bad or duplicate parameter id `{}`", self.id, p.id)); + } + if let PluginParamKind::Popup { options, default } = &p.kind + && (options.is_empty() || *default as usize >= options.len()) + { + return Err(format!("plug-in `{}`: popup `{}` needs options and a default index in range", self.id, p.id)); + } + } + Ok(()) + } + + /// Total flattened parameter count. + pub fn flat_len(&self) -> usize { + self.params.iter().map(|p| p.kind.width()).sum() + } +} + +/// The pixels a plug-in renders, in place. +pub struct PluginFrame<'a> { + pub width: u32, + pub height: u32, + /// Premultiplied RGBA, row-major, `width * height` pixels. + pub pixels: &'a mut [[f32; 4]], + /// Buffer pixels per layer pixel (preview resolution). + pub scale: f64, +} + +/// Parameter values for one render, flattened (see the module docs) with typed accessors. +pub struct PluginParams<'a> { + pub manifest: &'a PluginManifest, + pub flat: &'a [f64], +} + +impl PluginParams<'_> { + fn slot(&self, id: &str) -> Option<&[f64]> { + let mut at = 0; + for p in &self.manifest.params { + let w = p.kind.width(); + if p.id == id { + return self.flat.get(at..at + w); + } + at += w; + } + None + } + /// Slider / angle / popup index / checkbox (0 or 1). + pub fn f(&self, id: &str) -> f64 { + self.slot(id).and_then(|s| s.first().copied()).unwrap_or(0.0) + } + pub fn b(&self, id: &str) -> bool { + self.f(id) != 0.0 + } + /// Point in frame pixels. + pub fn point(&self, id: &str) -> [f64; 2] { + self.slot(id).map(|s| [s[0], s[1]]).unwrap_or([0.0; 2]) + } + pub fn color(&self, id: &str) -> [f64; 4] { + self.slot(id).map(|s| [s[0], s[1], s[2], s[3]]).unwrap_or([1.0; 4]) + } +} + +/// An effect plug-in. +pub trait EffectPlugin: Send + Sync { + fn manifest(&self) -> &PluginManifest; + /// Process `frame` in place. `time` is layer time in seconds. An error leaves the frame as + /// it was (the error is logged once). + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String>; + /// Where the plug-in came from (a file path, `built-in`…), for listings. + fn source(&self) -> String { + "built-in".into() + } +} + +static SLOTS: [OnceLock>; MAX_PLUGINS] = [const { OnceLock::new() }; MAX_PLUGINS]; + +/// Registered plug-in effects, in registration order. +fn plugin_specs() -> &'static RwLock> { + static P: OnceLock>> = OnceLock::new(); + P.get_or_init(|| RwLock::new(vec![])) +} + +/// The registered plug-in effects. +pub fn plugins() -> Vec<&'static EffectSpec> { + plugin_specs().read().map(|v| v.clone()).unwrap_or_default() +} + +/// The plug-in behind a registered effect id. +pub fn plugin(id: &str) -> Option> { + SLOTS.iter().filter_map(|s| s.get()).find(|p| p.manifest().id == id).cloned() +} + +fn leak(s: &str) -> &'static str { + Box::leak(s.to_string().into_boxed_str()) +} + +fn param_spec(p: &PluginParam) -> ParamSpec { + let (default, ui) = match &p.kind { + PluginParamKind::Slider { default, min, max, slider_min, slider_max, decimals } => ( + Value::Scalar(*default), + ParamUi::Slider { min: *min, max: *max, slider_min: slider_min.unwrap_or(*min), slider_max: slider_max.unwrap_or(*max), decimals: *decimals }, + ), + PluginParamKind::Angle { default } => (Value::Scalar(*default), ParamUi::Angle), + PluginParamKind::Checkbox { default } => (Value::Bool(*default), ParamUi::Checkbox), + PluginParamKind::Popup { options, default } => (Value::Enum(*default), ParamUi::Popup { options: options.clone() }), + PluginParamKind::Point { default } => (Value::Vec2(*default), ParamUi::Point), + PluginParamKind::Color { default } => (Value::Color(*default), ParamUi::Color), + }; + ParamSpec { id: leak(&p.id), name: leak(&p.name), default, ui } +} + +/// Register a plug-in effect: it appears in Effects & Presets under its category and is +/// applied by id like a built-in. Fails for a bad manifest, a taken id or when +/// [`MAX_PLUGINS`] are registered. +pub fn register_plugin(plugin: Arc) -> Result<&'static EffectSpec, String> { + let m = plugin.manifest().clone(); + m.validate()?; + let mut specs = plugin_specs().write().map_err(|_| "plug-in registry poisoned".to_string())?; + if crate::registry().iter().any(|s| s.id == m.id) || specs.iter().any(|s| s.id == m.id) { + return Err(format!("an effect with id `{}` is already registered", m.id)); + } + let slot = SLOTS.iter().position(|s| s.get().is_none()).ok_or_else(|| format!("too many plug-ins (at most {MAX_PLUGINS})"))?; + SLOTS[slot].set(plugin).map_err(|_| "plug-in slot taken".to_string())?; + let spec: &'static EffectSpec = Box::leak(Box::new(EffectSpec { + id: leak(&m.id), + name: leak(&m.name), + category: leak(&m.category), + params: m.params.iter().map(param_spec).collect(), + render: TRAMPOLINES[slot], + gpu: false, + float: true, + })); + specs.push(spec); + Ok(spec) +} + +/// Each slot needs its own `fn` (effect render functions are plain function pointers). +fn trampoline(ctx: &EffectCtx, buf: Buf) -> Buf { + run_slot(I, ctx, buf) +} + +macro_rules! trampolines { + ($($i:literal)*) => { [$(trampoline::<$i> as RenderFn),*] }; +} + +const TRAMPOLINES: [RenderFn; MAX_PLUGINS] = trampolines!( + 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 +); + +/// Flatten an instance's evaluated parameters for the plug-in (points mapped to frame pixels). +pub fn flatten(m: &PluginManifest, ctx: &EffectCtx, buf: &Buf) -> Vec { + let mut out = Vec::with_capacity(m.flat_len()); + for p in &m.params { + let v = ctx.params.get(&p.id); + match &p.kind { + PluginParamKind::Point { .. } => { + let q = v.map(Value::as_vec2).unwrap_or([0.0; 2]); + let (x, y) = buf.to_px(q); + out.extend([x, y]); + } + PluginParamKind::Color { .. } => { + let c = v.map(Value::as_color).unwrap_or([1.0; 4]); + out.extend(c.iter().map(|x| *x as f64)); + } + PluginParamKind::Checkbox { .. } => out.push(if v.map(Value::as_bool).unwrap_or(false) { 1.0 } else { 0.0 }), + PluginParamKind::Popup { .. } => out.push(v.map(Value::as_enum).unwrap_or(0) as f64), + _ => out.push(v.map(Value::as_f64).unwrap_or(0.0)), + } + } + out +} + +fn run_slot(slot: usize, ctx: &EffectCtx, mut buf: Buf) -> Buf { + let Some(plugin) = SLOTS[slot].get() else { return buf }; + let m = plugin.manifest(); + let flat = flatten(m, ctx, &buf); + let params = PluginParams { manifest: m, flat: &flat }; + let (width, height) = (buf.img.width, buf.img.height); + let before = buf.img.data.clone(); + let mut frame = PluginFrame { width, height, pixels: &mut buf.img.data, scale: buf.scale }; + if let Err(e) = plugin.render(&mut frame, ¶ms, ctx.time) { + log::warn!("plug-in `{}`: {e}", m.id); + buf.img.data = before; + } + buf +} diff --git a/crates/effects/src/warp_stab.rs b/crates/effects/src/warp_stab.rs index c0bbced8..ff535c41 100644 --- a/crates/effects/src/warp_stab.rs +++ b/crates/effects/src/warp_stab.rs @@ -17,8 +17,11 @@ //! Edge Cropping ignores bad edge pixels of the input frames; //! - **Show Track Points** draws the analysed background features. //! -//! *Subspace Warp* is approximated by a per-frame perspective warp (no content-preserving -//! mesh warp yet), and *Rolling Shutter Ripple* is stored but has no effect. +//! *Subspace Warp* warps each frame with a content-preserving mesh fitted to the analysis' +//! subspace-smoothed feature trajectories ([`effectcraft_track::subspace`]); analyses made +//! before trajectories were stored fall back to a perspective warp. *Rolling Shutter Ripple* +//! picks the mesh: Automatic Reduction (12 × 8, stiffer) or Enhanced Reduction (twice the rows, +//! softer), which follows the row-wise wobble of rolling-shutter shake more closely. //! //! Without an analysis (or with one made for another layer size) the input passes through. @@ -30,7 +33,7 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::{Image, Px}; use effectcraft_track::Homography; -use effectcraft_track::stabilize::{Framing, Method, Plan, StabResult, StabSettings, WarpAnalysis, plan}; +use effectcraft_track::stabilize::{Framing, Method, Plan, Ripple, StabResult, StabSettings, WarpAnalysis, plan}; use rayon::prelude::*; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -145,6 +148,7 @@ pub fn settings(params: &Params, fps: f64) -> StabSettings { // AE's 0…100 slider (50 = balanced) onto the planner's −100…100. crop_less_smooth_more: (f(params, "advanced/cropLessSmoothMore") - 50.0) * 2.0, fps, + ripple: if e(params, "advanced/rollingShutterRipple") == 1 { Ripple::Enhanced } else { Ripple::Automatic }, } } @@ -175,7 +179,7 @@ fn settings_key(s: &StabSettings) -> u64 { for v in [s.smoothness, s.max_scale, s.action_safe, s.additional_scale, s.crop_less_smooth_more, s.fps] { h.write_u64(v.to_bits()); } - h.write(format!("{:?}{:?}{:?}{:?}", s.result, s.method, s.preserve_scale, s.framing).as_bytes()); + h.write(format!("{:?}{:?}{:?}{:?}{:?}", s.result, s.method, s.preserve_scale, s.framing, s.ripple).as_bytes()); h.finish() } @@ -275,6 +279,13 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { } let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; let Some(inv) = pl.warps[k].inverse() else { return buf }; + // Output point → source point of this frame (a mesh lookup for Subspace Warp). + let mesh = pl.meshes.get(k); + let finv = pl.framing.inverse().unwrap_or(Homography::IDENTITY); + let src_of = |p: [f64; 2]| match mesh { + Some(m) => m.sample(finv.apply(p)), + None => inv.apply(p), + }; let (ow, oh) = (((size[0] * sc).ceil() as u32).max(1), ((size[1] * sc).ceil() as u32).max(1)); let synth = matches!(settings(ctx.params, 30.0).framing, Framing::SynthesizeEdges); let crop = [ @@ -306,7 +317,7 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { { continue; } - let q = inv.apply(p); + let q = src_of(p); if !synth { *px = sample(src, q); continue; @@ -337,12 +348,12 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { }); let need = need_m.into_inner().unwrap_or_else(|e| e.into_inner()); if synth && need.iter().any(|w| *w > 0.0) { - synthesize(ctx, &a, k, &inv, &crop, sc, &mut out, &need); + synthesize(ctx, &a, k, &src_of, &crop, sc, &mut out, &need); } if b(ctx.params, "advanced/showTrackPoints") { let r = (2.5 * f(ctx.params, "advanced/trackPointSize") / 100.0 * sc).max(0.5); for q in &a.frames[k].points { - let o = pl.warps[k].apply([q[0] as f64, q[1] as f64]); + let Some(o) = pl.output_of(k, [q[0] as f64, q[1] as f64]) else { continue }; dot(&mut out, [o[0] * sc, o[1] * sc], r, [0.25, 0.75, 1.0, 1.0]); } } @@ -352,7 +363,16 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { /// Fill the pixels flagged in `need` (weight = how much synthesized content they take) from /// neighbouring frames, nearest first. #[allow(clippy::too_many_arguments)] -fn synthesize(ctx: &EffectCtx, a: &WarpAnalysis, k: usize, inv: &Homography, crop: &[f64; 4], sc: f64, out: &mut Image, need: &[f32]) { +fn synthesize( + ctx: &EffectCtx, + a: &WarpAnalysis, + k: usize, + src_of: &(dyn Fn([f64; 2]) -> [f64; 2] + Sync), + crop: &[f64; 4], + sc: f64, + out: &mut Image, + need: &[f32], +) { let Some(host) = ctx.env.host else { return }; let n = a.frames.len(); let range = ((f(ctx.params, "advanced/synthesizeInputRange").max(0.0) / a.frame_duration.max(1e-6)).round() as usize).min(60); @@ -376,7 +396,7 @@ fn synthesize(ctx: &EffectCtx, a: &WarpAnalysis, k: usize, inv: &Homography, cro .filter(|i| !filled[**i]) .filter_map(|i| { let (x, y) = (i % ow, i / ow); - let q = inv.apply([(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]); + let q = src_of([(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]); let qj = m.apply(q); inside(crop, qj).then(|| (*i, nb.img.sample_bilinear_clamped(qj[0] * nb.scale + nb.offset[0], qj[1] * nb.scale + nb.offset[1]))) }) @@ -412,7 +432,7 @@ mod tests { use effectcraft_track::stabilize::FrameMotion; fn analysis_json(n: usize, w: f64, h: f64, shift: impl Fn(usize) -> [f64; 2]) -> String { - let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 25.0, size: [w, h], detailed: false, frames: vec![] }; + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 25.0, size: [w, h], detailed: false, frames: vec![], tracks: vec![] }; for k in 0..n { let mut fm = FrameMotion::default(); if k > 0 { diff --git a/crates/engine/scripts/Create Null at Selected Layers.jsx b/crates/engine/scripts/Create Null at Selected Layers.jsx new file mode 100644 index 00000000..44442c05 --- /dev/null +++ b/crates/engine/scripts/Create Null at Selected Layers.jsx @@ -0,0 +1,41 @@ +// Create Null at Selected Layers — an EffectCraft sample script (original work, MIT OR +// Apache-2.0). +// +// Adds a null at the centroid of the selected layers' positions (at the current time) and +// parents the layers that have no parent to it (set PARENT below to false to skip that). +// One undo step. + +(function createNullAtCentroid() { + var PARENT = true; + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Create Null: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + alert("Create Null: select one or more layers."); + return; + } + var t = comp.time; + var sum = [0, 0, 0]; + for (var i = 0; i < layers.length; i++) { + var p = layers[i].transform.position.valueAtTime(t, false); + sum[0] += p[0]; + sum[1] += p[1]; + sum[2] += p.length > 2 ? p[2] : 0; + } + var c = [sum[0] / layers.length, sum[1] / layers.length, sum[2] / layers.length]; + + app.beginUndoGroup("Create Null at Selected Layers"); + var nul = comp.layers.addNull(); + nul.name = "Centroid"; + nul.transform.position.setValue(c); + if (PARENT) { + for (var k = 0; k < layers.length; k++) { + if (!layers[k].parent) layers[k].setParentWithJump(nul); + } + } + app.endUndoGroup(); + writeLn("Null at [" + c[0].toFixed(1) + ", " + c[1].toFixed(1) + "]"); +})(); diff --git a/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution b/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution new file mode 100644 index 00000000..ea166b70 --- /dev/null +++ b/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution @@ -0,0 +1,7 @@ +asset: Create Null at Selected Layers.jsx +title: EffectCraft sample script (Create Null at Selected Layers) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Rename Layers.jsx b/crates/engine/scripts/Rename Layers.jsx new file mode 100644 index 00000000..6de9d4d4 --- /dev/null +++ b/crates/engine/scripts/Rename Layers.jsx @@ -0,0 +1,71 @@ +// Rename Layers — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Renames the selected layers of the active composition (all layers when none are selected): +// find & replace, then an optional prefix, suffix and running number. One undo step. + +(function renameLayers() { + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Rename Layers: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + layers = []; + for (var i = 1; i <= comp.numLayers; i++) layers.push(comp.layer(i)); + } + + var dlg = new Window("dialog", "Rename Layers"); + dlg.orientation = "column"; + dlg.alignChildren = ["fill", "top"]; + + var find = dlg.add("panel", undefined, "Find and Replace"); + find.alignChildren = ["fill", "top"]; + var row1 = find.add("group"); + row1.add("statictext", undefined, "Find:"); + var findText = row1.add("edittext", undefined, ""); + findText.characters = 20; + var row2 = find.add("group"); + row2.add("statictext", undefined, "Replace:"); + var replaceText = row2.add("edittext", undefined, ""); + replaceText.characters = 20; + + var extra = dlg.add("panel", undefined, "Add"); + extra.alignChildren = ["fill", "top"]; + var row3 = extra.add("group"); + row3.add("statictext", undefined, "Prefix:"); + var prefix = row3.add("edittext", undefined, ""); + prefix.characters = 12; + row3.add("statictext", undefined, "Suffix:"); + var suffix = row3.add("edittext", undefined, ""); + suffix.characters = 12; + var number = extra.add("checkbox", undefined, "Number the layers"); + var row4 = extra.add("group"); + row4.add("statictext", undefined, "Start at:"); + var start = row4.add("edittext", undefined, "1"); + start.characters = 5; + start.enabled = false; + number.onClick = function () { + start.enabled = number.value; + }; + + var buttons = dlg.add("group"); + buttons.alignment = "right"; + buttons.add("button", undefined, "Cancel", { name: "cancel" }); + buttons.add("button", undefined, "OK", { name: "ok" }); + + if (dlg.show() !== 1) return; + + app.beginUndoGroup("Rename Layers"); + var n = parseInt(start.text, 10); + if (isNaN(n)) n = 1; + for (var k = 0; k < layers.length; k++) { + var name = layers[k].name; + if (findText.text !== "") name = name.split(findText.text).join(replaceText.text); + name = prefix.text + name + suffix.text; + if (number.value) name += " " + (n + k); + layers[k].name = name; + } + app.endUndoGroup(); + writeLn("Renamed " + layers.length + " layer(s)."); +})(); diff --git a/crates/engine/scripts/Rename Layers.jsx.attribution b/crates/engine/scripts/Rename Layers.jsx.attribution new file mode 100644 index 00000000..bc07b909 --- /dev/null +++ b/crates/engine/scripts/Rename Layers.jsx.attribution @@ -0,0 +1,7 @@ +asset: Rename Layers.jsx +title: EffectCraft sample script (Rename Layers) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Render Queue Batch.jsx b/crates/engine/scripts/Render Queue Batch.jsx new file mode 100644 index 00000000..1dc5ca4c --- /dev/null +++ b/crates/engine/scripts/Render Queue Batch.jsx @@ -0,0 +1,52 @@ +// Render Queue Batch — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Adds the compositions selected in the Project panel (every composition when none are +// selected) to the Render Queue, each with an output file named after the comp in a folder you +// choose, and optionally starts rendering. + +(function renderQueueBatch() { + var comps = []; + var sel = app.project.selection; + for (var i = 0; i < sel.length; i++) if (sel[i] instanceof CompItem) comps.push(sel[i]); + if (comps.length === 0) { + for (var j = 1; j <= app.project.numItems; j++) { + var it = app.project.item(j); + if (it instanceof CompItem) comps.push(it); + } + } + if (comps.length === 0) { + alert("Render Queue Batch: the project has no compositions."); + return; + } + + var dlg = new Window("dialog", "Render Queue Batch"); + dlg.alignChildren = ["fill", "top"]; + dlg.add("statictext", undefined, comps.length + " composition(s) will be queued."); + var row = dlg.add("group"); + row.add("statictext", undefined, "Output folder:"); + var folder = row.add("edittext", undefined, Folder.desktop ? Folder.desktop.fsName : ""); + folder.characters = 30; + var fmtRow = dlg.add("group"); + fmtRow.add("statictext", undefined, "Format:"); + var format = fmtRow.add("dropdownlist", undefined, ["H.264 (.mp4)", "PNG Sequence"]); + format.selection = 0; + var go = dlg.add("checkbox", undefined, "Start rendering"); + var buttons = dlg.add("group"); + buttons.alignment = "right"; + buttons.add("button", undefined, "Cancel", { name: "cancel" }); + buttons.add("button", undefined, "Queue", { name: "ok" }); + if (dlg.show() !== 1) return; + + var png = format.selection && format.selection.index === 1; + app.beginUndoGroup("Render Queue Batch"); + for (var k = 0; k < comps.length; k++) { + var rq = app.project.renderQueue.items.add(comps[k]); + var name = comps[k].name.replace(/[\\\/:*?"<>|]/g, "_"); + var om = rq.outputModule(1); + if (png) om.format = "png"; + om.file = new File(folder.text + "/" + name + (png ? "_[#####].png" : ".mp4")); + } + app.endUndoGroup(); + writeLn("Queued " + comps.length + " composition(s)."); + if (go.value) app.project.renderQueue.render(); +})(); diff --git a/crates/engine/scripts/Render Queue Batch.jsx.attribution b/crates/engine/scripts/Render Queue Batch.jsx.attribution new file mode 100644 index 00000000..5ab7be71 --- /dev/null +++ b/crates/engine/scripts/Render Queue Batch.jsx.attribution @@ -0,0 +1,7 @@ +asset: Render Queue Batch.jsx +title: EffectCraft sample script (Render Queue Batch) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx new file mode 100644 index 00000000..28991849 --- /dev/null +++ b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx @@ -0,0 +1,99 @@ +// Layer Tools — an EffectCraft sample ScriptUI panel (original work, MIT OR Apache-2.0). +// +// Installed in the ScriptUI Panels folder it docks like any panel (Window ▸ Layer Tools.jsx); +// run from File ▸ Scripts it opens as a floating palette. Buttons act on the selected layers of +// the active composition. + +(function layerTools(thisObj) { + var ui = thisObj instanceof Panel ? thisObj : new Window("palette", "Layer Tools", undefined, { resizeable: true }); + ui.orientation = "column"; + ui.alignChildren = ["fill", "top"]; + ui.spacing = 6; + + function comp() { + var c = app.project.activeItem; + if (!(c instanceof CompItem)) { + status.text = "Open a composition first."; + return null; + } + return c; + } + function selected(c) { + var l = c.selectedLayers; + if (l.length === 0) status.text = "Select some layers."; + return l; + } + + var labelRow = ui.add("group"); + labelRow.add("statictext", undefined, "Opacity:"); + var opacity = labelRow.add("slider", undefined, 100, 0, 100); + opacity.preferredSize.width = 120; + var opacityValue = labelRow.add("statictext", undefined, "100%"); + opacityValue.characters = 5; + opacity.onChanging = function () { + opacityValue.text = Math.round(opacity.value) + "%"; + }; + opacity.onChange = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Set Opacity"); + for (var i = 0; i < layers.length; i++) layers[i].transform.opacity.setValue(Math.round(opacity.value)); + app.endUndoGroup(); + status.text = "Opacity " + Math.round(opacity.value) + "% on " + layers.length + " layer(s)."; + }; + + var shy = ui.add("checkbox", undefined, "Shy"); + shy.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Shy Layers"); + for (var i = 0; i < layers.length; i++) layers[i].shy = shy.value; + app.endUndoGroup(); + }; + + var nameRow = ui.add("group"); + nameRow.alignChildren = ["fill", "center"]; + var newName = nameRow.add("edittext", undefined, "Layer"); + newName.characters = 14; + var renameBtn = nameRow.add("button", undefined, "Rename"); + renameBtn.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Rename Layers"); + for (var i = 0; i < layers.length; i++) layers[i].name = layers.length > 1 ? newName.text + " " + (i + 1) : newName.text; + app.endUndoGroup(); + status.text = "Renamed " + layers.length + " layer(s)."; + }; + + var nullBtn = ui.add("button", undefined, "Null at Centroid"); + nullBtn.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + if (layers.length === 0) return; + var x = 0, y = 0; + for (var i = 0; i < layers.length; i++) { + var p = layers[i].transform.position.value; + x += p[0]; + y += p[1]; + } + app.beginUndoGroup("Null at Centroid"); + var n = c.layers.addNull(); + n.transform.position.setValue([x / layers.length, y / layers.length]); + app.endUndoGroup(); + status.text = "Added " + n.name + "."; + }; + + var status = ui.add("statictext", undefined, "Select layers, then pick a tool."); + status.characters = 30; + + if (ui instanceof Window) { + ui.center(); + ui.show(); + } else { + ui.layout.layout(true); + } +})(this); diff --git a/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution new file mode 100644 index 00000000..6d4d572f --- /dev/null +++ b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution @@ -0,0 +1,7 @@ +asset: Layer Tools.jsx +title: EffectCraft sample script (Layer Tools) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Sort Layers by In Point.jsx b/crates/engine/scripts/Sort Layers by In Point.jsx new file mode 100644 index 00000000..c59473d7 --- /dev/null +++ b/crates/engine/scripts/Sort Layers by In Point.jsx @@ -0,0 +1,37 @@ +// Sort Layers by In Point — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Reorders the selected layers of the active composition (all layers when none are selected) +// so the layer that starts first is on top. Layers with the same in point keep their order. +// One undo step. + +(function sortLayersByInPoint() { + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Sort Layers by In Point: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + layers = []; + for (var i = 1; i <= comp.numLayers; i++) layers.push(comp.layer(i)); + } + if (layers.length < 2) return; + + // The slots the layers occupy now (stack indices, top first). + var slots = layers.map(function (l) { return l.index; }).sort(function (a, b) { return a - b; }); + var order = layers + .map(function (l, k) { return { layer: l, inPoint: l.inPoint, k: k }; }) + .sort(function (a, b) { return a.inPoint - b.inPoint || a.k - b.k; }); + + app.beginUndoGroup("Sort Layers by In Point"); + // Place each layer, earliest first, in the next slot from the top. + for (var j = 0; j < order.length; j++) { + var target = slots[j]; + var layer = order[j].layer; + if (layer.index === target) continue; + var anchor = comp.layer(target); + layer.moveBefore(anchor); + } + app.endUndoGroup(); + writeLn("Sorted " + layers.length + " layer(s) by in point."); +})(); diff --git a/crates/engine/scripts/Sort Layers by In Point.jsx.attribution b/crates/engine/scripts/Sort Layers by In Point.jsx.attribution new file mode 100644 index 00000000..e744d662 --- /dev/null +++ b/crates/engine/scripts/Sort Layers by In Point.jsx.attribution @@ -0,0 +1,7 @@ +asset: Sort Layers by In Point.jsx +title: EffectCraft sample script (Sort Layers by In Point) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs index 5d454aed..68cd14fb 100644 --- a/crates/engine/src/commands/camera_cmds.rs +++ b/crates/engine/src/commands/camera_cmds.rs @@ -23,7 +23,7 @@ pub fn specs() -> Vec { "Track Camera", ["Animation"], None, - "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?: solve k1/k2, undistort?: render undistorted, wait?}", can_track, track_camera ), @@ -140,6 +140,8 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { return Err(bad("track.camera", "aov: 1..170 degrees")); } let detailed = b_p(p, "detailed"); + let lens = b_p(p, "lensDistortion"); + let undistort = b_p(p, "undistort"); let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); let uid = match existing { Some(u) => u, @@ -148,7 +150,7 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? } }; - if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() || lens.is_some() || undistort.is_some() { s.edit("3D Camera Tracker Settings", None, |proj, _| { let g = proj .comp_mut(cid) @@ -176,6 +178,12 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { if let Some(d) = detailed { set("advanced/detailedAnalysis", KV::Bool(d)); } + if let Some(d) = lens { + set("advanced/lensDistortion", KV::Bool(d)); + } + if let Some(d) = undistort { + set("advanced/undistort", KV::Bool(d)); + } Ok(()) })?; } @@ -222,6 +230,7 @@ fn status(s: &mut Session, p: &Value) -> Result { out["shotType"] = json!(settings.shot.label()); out["solveMethod"] = json!(settings.method.label()); out["deleted"] = json!(settings.deleted.len()); + out["lensDistortionSolved"] = json!(settings.lens_distortion); let solve = ct::solve(¶ms); out["solved"] = json!(solve.is_some()); if let Some(sv) = &solve { @@ -233,6 +242,11 @@ fn status(s: &mut Session, p: &Value) -> Result { out["focalLength"] = json!(f); out["horizontalAngleOfView"] = json!(sv.hfov(f)); out["groundPlane"] = json!(sv.ground.is_some()); + out["lensDistortion"] = match &sv.distortion { + Some(d) => json!({"k1": d.k1, "k2": d.k2, "radius": d.radius}), + None => Value::Null, + }; + out["undistort"] = json!(ct::undistorts(¶ms, sv)); if let Some(pl) = placed(&s.project, cid, lid, uid) { let k = pl.frame_at(s.time_of(cid)); let (pos, ori, zoom) = pl.camera(k); diff --git a/crates/engine/src/commands/comp_more.rs b/crates/engine/src/commands/comp_more.rs index e3057f67..0f07fd67 100644 --- a/crates/engine/src/commands/comp_more.rs +++ b/crates/engine/src/commands/comp_more.rs @@ -82,7 +82,37 @@ pub(crate) fn encode_png(rgba: &[u8], w: u32, h: u32) -> std::result::Result Result { + use effectcraft_project::render_queue::{RenderQueueItem, TimeSpan}; + use effectcraft_project::render_templates::TemplateSlot; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or(s.time()); + let mut it = RenderQueueItem::new(0, cid); + it.settings = s.project.render_templates.default_render_settings(TemplateSlot::Still); + it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; + it.output = s.project.render_templates.default_output_module(TemplateSlot::Still); + if let Some(path) = str_p(p, "path") { + it.output.output = path.to_string(); + } else if !it.output.format.is_sequence() { + it.output.output = "[compName]_frame.[fileExtension]".into(); + } + let id = s.edit("Save Frame As", None, |proj, _| { + it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; + let id = it.id; + proj.render_queue.push(it); + Ok(id) + })?; + let path = s.project.render_queue.last().and_then(|i| s.resolve_output(i)); + Ok(json!({"item": id, "queued": true, "output": path, "time": t.seconds()})) +} + fn save_frame(s: &mut Session, p: &Value) -> Result { + if p.get("queue").and_then(Value::as_bool) == Some(true) { + return queue_frame(s, p); + } let path = str_p(p, "path").ok_or_else(|| bad("comp.saveFrameAs", "missing `path` (.png)"))?.to_string(); let cid = comp_id(s, p)?; let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; @@ -124,7 +154,15 @@ pub fn specs() -> Vec { vec![ cmd!("comp.cropToRegionOfInterest", "Crop Comp to Region of Interest", ["Composition"], None, "{comp?}", has_roi, crop_to_roi), cmd!("comp.cropToLayerBounds", "Crop Comp to Selected Layer(s) Bounds", ["Composition"], None, "{layers?}", has_layers, crop_to_layers), - cmd!("comp.saveFrameAs", "File...", ["Composition", "Save Frame As"], Some("Cmd+Alt+S"), "{path (.png), time?, scale?}", has_comp, save_frame), + cmd!( + "comp.saveFrameAs", + "File...", + ["Composition", "Save Frame As"], + Some("Cmd+Alt+S"), + "{path (.png; with queue: an output path or template), time?, scale?, queue?: bool (add a Render Queue item from the Frame Default templates instead of writing now)}", + has_comp, + save_frame + ), cmd!("comp.responsiveTime", "Responsive Design — Time", [], None, "{op: intro|outro|workArea}", has_comp, responsive_time), ] } diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index 44f49466..9b003d4d 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -591,8 +591,7 @@ fn edit_original(s: &mut Session, _: &Value) -> Result { } fn purge(s: &mut Session, _: &Value) -> Result { - s.history.undo.clear(); - s.history.redo.clear(); + s.history.clear(); s.toast("Purged undo history"); Ok(Value::Null) } @@ -637,9 +636,60 @@ pub fn specs() -> Vec { cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats), cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original), + crate::query!( + "edit.history.list", + "History", + "{} → {states: [{index, id, label, parent, depth, current, line, future}], current, branches}", + history_list + ), + cmd!( + "edit.history.goto", + "Go to History State", + [], + None, + "{index? (from edit.history.list) | id? | steps? (negative = back along the line)}", + always_ok, + history_goto + ), ] } +// ---------------------------------------------------------------- History panel + +/// `edit.history.list`: every state of the (branching) undo history. +fn history_list(s: &mut Session, _: &Value) -> Result { + let nodes = s.history_tree(); + let current = nodes.iter().position(|n| n.current); + Ok(json!({"states": nodes, "current": current, "branches": s.history.branches.len()})) +} + +/// `edit.history.goto`: jump to a state by `index` (from `edit.history.list`), `id`, or +/// `steps` along the working line (negative = back, like Undo). +fn history_goto(s: &mut Session, p: &Value) -> Result { + let c = "edit.history.goto"; + if let Some(n) = p.get("steps").and_then(Value::as_i64) { + for _ in 0..n.unsigned_abs() { + let moved = if n < 0 { s.undo() } else { s.redo() }; + if !moved { + break; + } + } + } else { + let nodes = s.history_tree(); + let id = match (p.get("id").and_then(Value::as_str), p.get("index").and_then(Value::as_u64)) { + (Some(id), _) => id.to_string(), + (None, Some(i)) => nodes.get(i as usize).map(|n| n.id.clone()).ok_or_else(|| super::bad(c, format!("no state {i} (0..{})", nodes.len())))?, + _ => return Err(super::bad(c, "give `index`, `id` or `steps`")), + }; + if !s.goto_history(&id) { + return Err(super::bad(c, format!("no history state `{id}`"))); + } + } + let nodes = s.history_tree(); + let cur = nodes.iter().find(|n| n.current).cloned(); + Ok(json!({"current": cur.as_ref().map(|n| n.index), "label": cur.map(|n| n.label)})) +} + fn layers_or_keys(s: &Session) -> std::result::Result<(), String> { if !s.state.selected_keys.is_empty() || !s.state.selected_vertices.is_empty() { super::has_comp(s) } else { has_layers(s) } } diff --git a/crates/engine/src/commands/effect.rs b/crates/engine/src/commands/effect.rs index b0fb8c31..6ce73ac3 100644 --- a/crates/engine/src/commands/effect.rs +++ b/crates/engine/src/commands/effect.rs @@ -7,6 +7,68 @@ use serde_json::{Value, json}; use super::{CommandSpec, b_p, bad, has_layers, layer_mut, layers_p, str_p}; use crate::{EngineError, Result, Session, cmd}; +/// `effect.plugins.list`: the registered plug-in effects (WebAssembly or Rust). +fn plugins_list(s: &mut Session, _: &Value) -> Result { + use effectcraft_effects::plugin; + let list: Vec = plugin::plugins() + .into_iter() + .map(|spec| { + let p = plugin::plugin(spec.id); + let m = p.as_ref().map(|p| p.manifest().clone()); + json!({ + "id": spec.id, + "name": spec.name, + "category": spec.category, + "version": m.as_ref().map(|m| m.version.clone()), + "author": m.as_ref().map(|m| m.author.clone()), + "description": m.as_ref().map(|m| m.description.clone()), + "source": p.map(|p| p.source()), + "params": spec.params.iter().map(|p| p.id).collect::>(), + }) + }) + .collect(); + Ok(json!({"api": plugin::PLUGIN_API_VERSION, "wasm": s.plugin_loader.is_some(), "plugins": list})) +} + +/// `effect.plugins.load`: load WebAssembly effect plug-ins (a file, or every `.wasm` in a +/// folder) through the host's loader ([`Session::plugin_loader`]). +fn plugins_load(s: &mut Session, p: &Value) -> Result { + let c = "effect.plugins.load"; + let loader = s.plugin_loader.ok_or_else(|| EngineError::Other("WebAssembly effect plug-ins are not available in this build".into()))?; + let files: Vec = match (str_p(p, "path"), str_p(p, "folder")) { + (Some(path), _) => vec![path.to_string()], + (None, Some(dir)) => { + let mut v: Vec = std::fs::read_dir(dir) + .map_err(|e| bad(c, format!("cannot list {dir}: {e}")))? + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x.eq_ignore_ascii_case("wasm"))) + .map(|p| p.to_string_lossy().to_string()) + .collect(); + v.sort(); + v + } + _ => return Err(bad(c, "give `path` or `folder`")), + }; + let (mut loaded, mut errors) = (vec![], vec![]); + for f in &files { + let r = s.services.read_file(f).map_err(|e| format!("cannot read {f}: {e}")).and_then(|bytes| loader(&bytes, f)); + match r { + Ok(v) => loaded.push(v), + Err(e) => errors.push(json!({"path": f, "error": e})), + } + } + if str_p(p, "path").is_some() + && let Some(e) = errors.first() + { + return Err(EngineError::Other(format!("{}: {}", files[0], e["error"].as_str().unwrap_or("failed")))); + } + if !loaded.is_empty() { + s.toast(format!("Loaded {} effect plug-in(s)", loaded.len())); + } + Ok(json!({"loaded": loaded, "errors": errors})) +} + fn apply(s: &mut Session, p: &Value) -> Result { let name = str_p(p, "effect").ok_or_else(|| bad("effect.apply", "missing `effect` (id or name)"))?; let spec = effectcraft_effects::lookup(name).ok_or_else(|| bad("effect.apply", format!("unknown effect `{name}`")))?; @@ -284,8 +346,8 @@ fn last(s: &mut Session, p: &Value) -> Result { fn list(_: &mut Session, p: &Value) -> Result { let filter = str_p(p, "filter").map(str::to_ascii_lowercase); - let v: Vec = effectcraft_effects::registry() - .iter() + let v: Vec = effectcraft_effects::all() + .into_iter() .filter(|e| filter.as_ref().is_none_or(|f| e.name.to_ascii_lowercase().contains(f) || e.id.contains(f.as_str()))) .map(|e| json!({"id": e.id, "name": e.name, "category": e.category, "params": e.params.iter().map(|p| json!({"id": p.id, "name": p.name, "default": p.default.to_json()})).collect::>()})) .collect(); @@ -305,5 +367,20 @@ pub fn specs() -> Vec { cmd!("effect.paste", "Paste Effects", [], None, "{layers?} — adds the copied effects to the layers", has_layers, paste), cmd!("effect.reset", "Reset Effect", [], None, "{layer?, effect}", has_layers, reset), crate::query!("effect.list", "List Effects", "{filter?}", list), + crate::query!( + "effect.plugins.list", + "List Effect Plug-ins", + "{} → {api, wasm, plugins: [{id, name, category, version, author, source, params}]}", + plugins_list + ), + cmd!( + "effect.plugins.load", + "Load Effect Plug-in...", + [], + None, + "{path (.wasm / .wat plug-in, API v1) | folder (loads every .wasm in it)}", + super::always, + plugins_load + ), ] } diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 6908bcd5..2041d400 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -364,6 +364,14 @@ fn collect_files(s: &mut Session, p: &Value) -> Result { /// array or JSON lines of `{"command": id, "params": {…}}` (also accepts the control channel's /// `{"method": "engine.execute", "params": {"id", "params"}}`). fn run_script(s: &mut Session, p: &Value) -> Result { + // File ▸ Scripts ▸ . + if let Some(name) = str_p(p, "name") + && p.get("path").is_none() + && p.get("steps").is_none() + { + let name = name.to_string(); + return super::scripts::run_named(s, &name); + } if let Some(path) = str_p(p, "path") && p.get("steps").is_none() && is_js_script(path) @@ -375,13 +383,7 @@ fn run_script(s: &mut Session, p: &Value) -> Result { let code = String::from_utf8_lossy(&bytes).to_string(); let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); let out = run_js(s, &code, path, false)?; - if let Some(e) = out.get("error").filter(|e| !e.is_null()) { - let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); - let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); - s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); - return Err(EngineError::Other(format!("{name}{line}: {msg}"))); - } - return Ok(out); + return report_js(s, &name, out); } let steps: Vec = match (p.get("steps"), str_p(p, "path")) { (Some(Value::Array(a)), _) => a.clone(), @@ -426,6 +428,23 @@ fn is_js_script(path: &str) -> bool { l.ends_with(".jsx") || l.ends_with(".js") || l.ends_with(".jsxbin") } +/// A script error becomes a toast and a command error. +fn report_js(s: &mut Session, name: &str, out: Value) -> Result { + if let Some(e) = out.get("error").filter(|e| !e.is_null()) { + let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); + let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); + s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); + return Err(EngineError::Other(format!("{name}{line}: {msg}"))); + } + Ok(out) +} + +/// Run a script's code under its file name (installed / sample scripts, ScriptUI panels). +pub(crate) fn run_js_named(s: &mut Session, code: &str, name: &str) -> Result { + let out = run_js(s, code, name, false)?; + report_js(s, name, out) +} + /// Run JavaScript through the session's scripting engine. fn run_js(s: &mut Session, code: &str, name: &str, console: bool) -> Result { let runner = s.script.ok_or_else(|| EngineError::Other("JavaScript scripting is not available in this build".into()))?; @@ -809,7 +828,7 @@ pub fn specs() -> Vec { "Run Script File...", ["File", "Scripts"], None, - "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}]}", + "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | name (an installed or sample script, see file.scripts.list) | steps: [{command, params}]}", always, run_script ), diff --git a/crates/engine/src/commands/mask_interp.rs b/crates/engine/src/commands/mask_interp.rs index 5df40c6d..a4548944 100644 --- a/crates/engine/src/commands/mask_interp.rs +++ b/crates/engine/src/commands/mask_interp.rs @@ -1,5 +1,6 @@ -//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode) and Smart Mask -//! Interpolation (Window ▸ Mask Interpolation). +//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode, including the Face +//! Tracking methods and Extract & Copy Face Measurements) and Smart Mask Interpolation +//! (Window ▸ Mask Interpolation). use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV}; use effectcraft_path::interp::{self, AddVertices, InterpOpts, Matching}; @@ -20,7 +21,7 @@ pub fn specs() -> Vec { "Track Mask", ["Animation"], None, - "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", + "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", has_mask, track_mask ), @@ -29,10 +30,19 @@ pub fn specs() -> Vec { "Mask Tracking Method", [], None, - "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective}", + "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed}", super::always, mask_method ), + cmd!( + "track.extractFaceMeasurements", + "Extract & Copy Face Measurements", + [], + None, + "{layer?} → keys a Face Measurements effect from the layer's Face Track Points (one key per tracked frame) and copies those keys", + has_face_points, + extract_face + ), cmd!( "mask.interpolate", "Apply Mask Interpolation", @@ -116,9 +126,12 @@ fn has_mask(s: &Session) -> std::result::Result<(), String> { fn method_p(p: &Value, cmd: &str) -> Result> { match str_p(p, "method") { - Some(m) => MaskMethod::from_name(m) - .map(Some) - .ok_or_else(|| bad(cmd, format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective)"))), + Some(m) => MaskMethod::from_name(m).map(Some).ok_or_else(|| { + bad( + cmd, + format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed)"), + ) + }), None => Ok(None), } } @@ -190,6 +203,112 @@ fn track_mask(s: &mut Session, p: &Value) -> Result { })) } +// ---------------------------------------------------------------- face measurements + +fn face_points_layer(s: &Session, p: &Value) -> Option<(effectcraft_project::ItemId, LayerId)> { + use effectcraft_effects::face_track::POINTS_ID; + let has = |l: &Layer| l.effects().is_some_and(|fx| fx.groups().any(|g| g.match_id == POINTS_ID)); + if p.get("layer").is_some() { + return layer_p(s, p, "track.extractFaceMeasurements").ok(); + } + let cid = s.active_comp_id()?; + let comp = s.project.comp(cid)?; + let sel = s.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| has(l)); + sel.or_else(|| comp.layers.iter().find(|l| has(l))).map(|l| (cid, l.id)) +} + +fn has_face_points(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + face_points_layer(s, &json!({})).map(|_| ()).ok_or_else(|| "no Face Track Points: track a mask with Face Tracking (Detailed Features) first".into()) +} + +/// Extract & Copy Face Measurements: measure the Face Track Points at every keyed time, key them +/// into the layer's Face Measurements effect and put those keys on the keyframe clipboard. +fn extract_face(s: &mut Session, p: &Value) -> Result { + use effectcraft_effects::face_track::{MEASUREMENTS_ID, POINTS_ID}; + use effectcraft_track::face::{LANDMARKS, MEASUREMENTS, N, face_frame, measure}; + const CMD: &str = "track.extractFaceMeasurements"; + let (cid, lid) = face_points_layer(s, p).ok_or_else(|| bad(CMD, "no Face Track Points: track a mask with Face Tracking (Detailed Features) first"))?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.effects().and_then(|fx| fx.groups().find(|g| g.match_id == POINTS_ID)).ok_or_else(|| bad(CMD, "the layer has no Face Track Points"))?; + let mut times: Vec = LANDMARKS.iter().filter_map(|l| g.get(l.0)).flat_map(|pr| pr.keys.iter().map(|k| k.time)).collect(); + times.sort(); + times.dedup(); + if times.is_empty() { + times.push(layer.layer_time(s.time_of(cid))); + } + let rows: Vec<(Tick, [f64; 14])> = { + let pts_at = |t: Tick| { + let mut l = [[0.0; 2]; N]; + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + if let Some(KV::Vec2(v)) = g.get(id).map(|pr| pr.value_at(t)) { + l[i] = v; + } + } + l + }; + let reference = face_frame(&pts_at(times[0])).1; + times.iter().map(|t| (*t, measure(&pts_at(*t), reference))).collect() + }; + let comp_times: Vec = times.iter().map(|t| layer.comp_time(*t)).collect(); + let fd = comp.frame_duration(); + let uid = s.edit("Extract Face Measurements", None, |proj, _| { + let uid = crate::mask_track::face_effect(proj, cid, lid, MEASUREMENTS_ID).ok_or_else(|| bad(CMD, "Face Measurements is not available"))?; + let g = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).ok_or(EngineError::NoComp)?; + for (k, (id, _)) in MEASUREMENTS.iter().enumerate() { + let Some(pr) = g.get_mut(id) else { continue }; + pr.keys.clear(); + for (t, m) in &rows { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*t, KV::Scalar(m[k]))); + } + } + Ok(uid) + })?; + // Copy the keys (Edit ▸ Paste pastes them at the CTI, like copied keyframes). + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let mg = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let earliest = comp_times.iter().copied().min().unwrap_or(Tick::ZERO); + let mut clip = vec![]; + for (id, _) in MEASUREMENTS { + let Some(pr) = mg.get(id) else { continue }; + let Some(path) = layer.props.match_path_of(pr.uid) else { continue }; + let keys = pr + .keys + .iter() + .map(|k| { + let mut k = k.clone(); + k.time = layer.comp_time(k.time) - earliest; + k + }) + .collect(); + clip.push(crate::KeyClip { layer: lid, path, keys }); + } + s.state.key_clipboard = clip; + s.state.clip_is_keys = true; + // The same data as text (tab-separated, one row per frame) for other applications. + let mut text = String::from("EffectCraft Face Measurements\n\nFrame\tTime"); + for (_, name) in MEASUREMENTS { + text.push('\t'); + text.push_str(name); + } + text.push('\n'); + let mut table = vec![]; + for ((_, m), ct) in rows.iter().zip(&comp_times) { + let frame = if fd > Tick::ZERO { (ct.seconds() / fd.seconds()).round() as i64 } else { 0 }; + text.push_str(&format!("{frame}\t{:.4}", ct.seconds())); + for v in m { + text.push_str(&format!("\t{v:.3}")); + } + text.push('\n'); + let obj: serde_json::Map = MEASUREMENTS.iter().zip(m).map(|((id, _), v)| (id.to_string(), json!(v))).collect(); + table.push(json!({"time": ct.seconds(), "frame": frame, "values": obj})); + } + s.bump(); + Ok(json!({"layer": lid.0, "effect": uid, "frames": rows.len(), "measurements": table, "clipboard": text})) +} + // ---------------------------------------------------------------- mask interpolation /// Mask Interpolation panel options (kept in the editor state, serde for agents). diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index d9ef2816..9ef10e09 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -48,6 +48,7 @@ mod render_queue; pub(crate) mod rig3d; pub mod roto_cmds; mod scene_detect; +pub mod scripts; mod settings; mod shape_stroke; mod stubs; @@ -175,6 +176,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(essential::specs()); v.extend(expr_tools::specs()); v.extend(proxy::specs()); + v.extend(scripts::specs()); v.extend(panels_cmds::specs()); v.extend(footage_panel::specs()); v.extend(autotrace::specs()); diff --git a/crates/engine/src/commands/proxy.rs b/crates/engine/src/commands/proxy.rs index 607dd5dd..3e1126a9 100644 --- a/crates/engine/src/commands/proxy.rs +++ b/crates/engine/src/commands/proxy.rs @@ -4,6 +4,7 @@ //! item's switch). use effectcraft_project::render_queue::{Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, TimeSpan}; +use effectcraft_project::render_templates::TemplateSlot; use effectcraft_project::{Footage, ItemId, ItemKind, Proxy}; use effectcraft_time::Tick; use serde_json::{Value, json}; @@ -159,18 +160,28 @@ fn create_proxy(s: &mut Session, p: &Value) -> Result { k => return Err(bad(C, format!("kind: still|movie, not `{k}`"))), }; let mut it = RenderQueueItem::new(0, cid); + // The Movie / Still Proxy Default templates, at draft quality and the proxy resolution. + let templates = &s.project.render_templates; + let slot = if still { TemplateSlot::StillProxy } else { TemplateSlot::MovieProxy }; + it.settings = templates.default_render_settings(slot); it.settings.quality = RenderQuality::Draft; it.settings.resolution = p.get("resolution").and_then(Value::as_f64).unwrap_or(0.5).clamp(0.05, 1.0); if still { // One frame at the current time (the comp's poster frame when not the active comp). let t = if s.active_comp_id() == Some(cid) { s.time() } else { comp.poster_time }; it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; - it.output = OutputModule::for_format(OutputFormat::PngSequence); - it.output.output = "[compName]_proxy_[#####].png".into(); + it.output = templates.default_output_module(slot); + if !it.output.format.is_sequence() { + it.output = OutputModule::for_format(OutputFormat::PngSequence); + } + it.output.output = "[compName]_proxy_[#####].[fileExtension]".into(); } else { it.settings.time_span = TimeSpan::LengthOfComp; - it.output = OutputModule::for_format(OutputFormat::ProRes); - it.output.prores_profile = ProResProfile::P4444; + it.output = templates.default_output_module(slot); + if !it.output.format.is_movie() { + it.output = OutputModule::for_format(OutputFormat::ProRes); + it.output.prores_profile = ProResProfile::P4444; + } it.output.output = "[compName]_proxy.[fileExtension]".into(); } it.output.channels = Channels::Rgba; diff --git a/crates/engine/src/commands/puppet.rs b/crates/engine/src/commands/puppet.rs index 4ada1dcd..890e1091 100644 --- a/crates/engine/src/commands/puppet.rs +++ b/crates/engine/src/commands/puppet.rs @@ -7,7 +7,7 @@ use effectcraft_effects::puppet::{self, MeshOpts, PinKind}; use effectcraft_effects::{Buf, Params}; -use effectcraft_keyframe::Value as KV; +use effectcraft_keyframe::{Keyframe, Value as KV}; use effectcraft_project::build::Ids; use effectcraft_project::{ItemId, LayerId, Node, PropGroup, Uid}; use effectcraft_render::{EvalCtx, RenderOpts, Renderer}; @@ -309,6 +309,122 @@ fn mesh_opts(s: &mut Session, p: &Value) -> Result { Ok(Value::Null) } +/// Parse recorded samples `[[t, x, y]…]` (t = seconds since the drag began, layer space). +fn samples(p: &Value, cmd: &str) -> Result> { + let arr = p + .get("samples") + .and_then(Value::as_array) + .ok_or_else(|| bad(cmd, "missing `samples`: [[t, x, y]…] (t = seconds since the drag began, layer space)"))?; + let mut rec = Vec::with_capacity(arr.len()); + for (i, v) in arr.iter().enumerate() { + let a: Vec = v.as_array().map(|a| a.iter().filter_map(Value::as_f64).collect()).unwrap_or_default(); + if a.len() < 3 || a.iter().any(|x| !x.is_finite()) { + return Err(bad(cmd, format!("sample {i}: expected [t, x, y]"))); + } + rec.push((a[0], [a[1], a[2]])); + } + if rec.is_empty() { + return Err(bad(cmd, "need at least one sample")); + } + rec.sort_by(|a, b| a.0.total_cmp(&b.0)); + Ok(rec) +} + +/// Puppet tool Record mode (⌘/Ctrl-drag a pin): the pin's drag, sampled in real time, becomes +/// Position keyframes at the comp frame rate from the current time on, replacing the keys in +/// the recorded span. Record Options: `speed` (%: 100 plays back at the speed of the drag, +/// 200 twice as slow) and `smoothing` (removes extraneous keys like the Smoother; 0 = a key +/// on every frame). The current time stays where recording began. +fn record_pin(s: &mut Session, p: &Value) -> Result { + let cmd = "puppet.recordPin"; + let (cid, lid) = layer_p(s, p, cmd)?; + let key = p.get("pin").cloned().ok_or_else(|| bad(cmd, "missing `pin` (uid or name)"))?; + let uid = pin_uid(s, cid, lid, &key, cmd)?; + let rec = samples(p, cmd)?; + let opts = &s.state.puppet; + let speed = f_p(p, "speed").unwrap_or(opts.record_speed).clamp(1.0, 10_000.0) / 100.0; + // Smoothing 10 (the default) keeps the path within 1 layer pixel of the drag. + let smoothing = f_p(p, "smoothing").unwrap_or(opts.record_smoothing).max(0.0) / 10.0; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let fr = comp.frame_rate; + let fd = fr.frame_duration().seconds(); + let start = match p.get("start").and_then(Value::as_f64) { + Some(t) => fr.snap_nearest(Tick::from_seconds_f64(t)), + None => s.time_of(cid), + }; + let f0 = fr.frame_at(start); + let end_frame = fr.frame_at(comp.duration - comp.frame_duration()); + let t_rec0 = rec[0].0; + let span = (rec[rec.len() - 1].0 - t_rec0) * speed; + let nframes = (span / fd).round().max(0.0) as i64; + let at = |secs: f64| { + let i = rec.partition_point(|r| r.0 <= secs).clamp(1, rec.len().max(2) - 1); + if rec.len() == 1 { + return rec[0].1; + } + let (a, b) = (rec[i - 1], rec[i]); + let f = if b.0 > a.0 { ((secs - a.0) / (b.0 - a.0)).clamp(0.0, 1.0) } else { 1.0 }; + [a.1[0] + (b.1[0] - a.1[0]) * f, a.1[1] + (b.1[1] - a.1[1]) * f] + }; + let n = s.edit("Record Puppet Pin", None, |proj, st| { + let l = layer_mut(proj, cid, lid)?; + let lc = l.clone(); + let g = l.props.find_group_mut(uid).ok_or_else(|| bad(cmd, "pin gone"))?; + let kind = PinKind::from_index(g.get("kind").map(|k| k.value.as_enum()).unwrap_or(0)); + if !kind.moves() { + return Err(bad(cmd, "only Position and Advanced pins record motion")); + } + let pos = g.get_mut("position").ok_or_else(|| bad(cmd, "the pin has no Position"))?; + let mut new: Vec = vec![]; + for k in 0..=nframes { + let f = f0 + k; + if f > end_frame { + break; + } + let q = at(t_rec0 + k as f64 * fd / speed); + new.push(Keyframe::new(lc.layer_time(fr.tick_of(f)), KV::Vec2(q))); + } + let (Some(t0), Some(t1)) = (new.first().map(|k| k.time), new.last().map(|k| k.time)) else { return Ok(0) }; + let mut keys: Vec = pos.keys.iter().filter(|k| k.time < t0 || k.time > t1).cloned().collect(); + keys.extend(new); + keys.sort_by_key(|k| k.time); + let (a, b) = (keys.iter().position(|k| k.time == t0).unwrap_or(0), keys.iter().position(|k| k.time == t1).unwrap_or(0)); + if smoothing > 0.0 && b > a + 1 { + let samples: Vec<(Tick, Vec)> = keys[a..=b].iter().map(|k| (k.time, k.value.components())).collect(); + keys = super::anim_tools::smooth_keys(&keys, a, b, &samples, smoothing, true); + } + pos.keys = keys; + pos.value = KV::Vec2(rec[0].1); + let puid = pos.uid; + st.selected_props = vec![(lid, uid)]; + st.selected_keys = pos.keys.iter().filter(|k| k.time >= t0 && k.time <= t1).map(|k| crate::KeyRef { layer: lid, prop: puid, time: k.time }).collect(); + Ok(st.selected_keys.len()) + })?; + // The current time returns to where recording began. + s.state.times.insert(cid, start); + let end = fr.tick_of((f0 + nframes).min(end_frame)); + Ok(json!({"keys": n, "start": start.seconds(), "end": end.seconds(), "frames": nframes + 1})) +} + +/// Record Options (the Puppet tool's Record Options… dialog): `speed`, `smoothing`, +/// `useDraftDeformation`, `showMesh`. Returns the options. +fn record_options(s: &mut Session, p: &Value) -> Result { + let o = &mut s.state.puppet; + if let Some(v) = f_p(p, "speed") { + o.record_speed = v.clamp(1.0, 10_000.0); + } + if let Some(v) = f_p(p, "smoothing") { + o.record_smoothing = v.clamp(0.0, 100.0); + } + if let Some(v) = p.get("useDraftDeformation").and_then(Value::as_bool) { + o.record_draft = v; + } + if let Some(v) = p.get("showMesh").and_then(Value::as_bool) { + o.record_show_mesh = v; + } + Ok(json!({"speed": o.record_speed, "smoothing": o.record_smoothing, "useDraftDeformation": o.record_draft, "showMesh": o.record_show_mesh})) +} + /// Meshes, pins and the deformed mesh at the current time (layer space). fn info(s: &mut Session, p: &Value) -> Result { let (cid, lid) = layer_p(s, p, "puppet.info")?; @@ -364,6 +480,24 @@ pub fn specs() -> Vec { mesh_opts ), crate::query!("puppet.info", "Puppet Mesh Info", "{layer?, time? | frame?}", info), + cmd!( + "puppet.recordPin", + "Record Puppet Pin", + [], + None, + "{layer?, pin: uid | name, samples: [[t, x, y]…] (t = seconds since the drag began, layer space), start? (s, default current time), speed? (%, Record Options), smoothing? (Record Options)}", + has_comp, + record_pin + ), + cmd!( + "puppet.recordOptions", + "Record Options...", + [], + None, + "{speed? (%, 100), smoothing? (0-100), useDraftDeformation?, showMesh?}", + super::always, + record_options + ), ] } @@ -376,11 +510,30 @@ pub struct PuppetOptions { pub triangles: f64, /// Show the mesh in the viewer. pub show_mesh: bool, + /// Record Options ▸ Speed (%): 100 plays a recording back at the speed of the drag. + pub record_speed: f64, + /// Record Options ▸ Smoothing: removes extraneous keys from recorded motion (0 = a key per + /// frame). + pub record_smoothing: f64, + /// Record Options ▸ Use Draft Deformation: the viewer shows the draft (unexpanded) mesh + /// while recording. + pub record_draft: bool, + /// Record Options ▸ Show Mesh while recording. + pub record_show_mesh: bool, } impl Default for PuppetOptions { fn default() -> Self { let d = MeshOpts::default(); - PuppetOptions { density: d.density, expansion: d.expansion, triangles: d.triangles, show_mesh: false } + PuppetOptions { + density: d.density, + expansion: d.expansion, + triangles: d.triangles, + show_mesh: false, + record_speed: 100.0, + record_smoothing: 10.0, + record_draft: false, + record_show_mesh: false, + } } } diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index bb2b8ce5..365d7e55 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -1,16 +1,20 @@ //! Render Queue: Composition ▸ Add to Render Queue, the queue's Render Settings / Output Module / -//! Output To, Render and Stop. +//! Output To / Log, Render and Stop, Notify, storage overflow folders, and the Render Settings / +//! Output Module templates (Edit ▸ Templates). //! //! Items are addressed by `item` (the stable id from `renderQueue.add` / `renderQueue.list`) or //! `index` (1-based, the # column). use effectcraft_project::render_queue::{ - AudioOutput, Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, RenderSettings, RenderStatus, TimeSpan, + AlphaMode, AudioFormat, AudioOutput, Channels, ColorDepth, CurrentOrOff, DEFAULT_SEQUENCE_TEMPLATE, DEFAULT_TEMPLATE, EffectsMode, FieldRender, + OutputFormat, OutputModule, PostRenderAction, ProResProfile, Pulldown, RESIZE_PRESETS, RenderLog, RenderQuality, RenderQueueItem, RenderSettings, + RenderStatus, ResizeQuality, SwitchOverride, TimeSpan, post_render_parse, }; +use effectcraft_project::render_templates::{RenderTemplates, TemplateKind, TemplateSlot}; use effectcraft_time::{FrameRate, Tick}; use serde_json::{Value, json}; -use super::{CommandSpec, b_p, bad, comp_id, f_p, has_comp, str_p}; +use super::{CommandSpec, always, b_p, bad, comp_id, f_p, has_comp, str_p}; use crate::{EngineError, Result, Session, cmd, query}; fn can_add(s: &Session) -> std::result::Result<(), String> { @@ -54,16 +58,32 @@ fn time_p(p: &Value, k: &str) -> Option { f_p(p, k).map(Tick::from_seconds_f64) } -/// Apply Render Settings parameters. -fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result { +/// Parse an enum parameter. +fn enum_p(p: &Value, k: &str, cmd: &str, parse: fn(&str) -> Option, help: &str) -> Result> { + match p.get(k) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(v)) => parse(v).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(Value::Bool(b)) => parse(if *b { "on" } else { "off" }).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(Value::Number(n)) => parse(&n.to_string()).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(_) => Err(bad(cmd, format!("{k}: {help}"))), + } +} + +/// Apply Render Settings parameters (`template` first, then the individual settings). Any +/// individual change makes the settings "Custom". +fn apply_settings(rs: &mut RenderSettings, templates: &RenderTemplates, p: &Value, cmd: &str) -> Result { let mut any = false; + if let Some(t) = str_p(p, "template") { + *rs = templates.render_settings(t).ok_or_else(|| bad(cmd, format!("no Render Settings template “{t}”")))?; + any = true; + } + let before = rs.clone(); if let Some(q) = str_p(p, "quality") { rs.quality = match q.to_ascii_lowercase().as_str() { "best" => RenderQuality::Best, "draft" => RenderQuality::Draft, _ => return Err(bad(cmd, "quality: best|draft")), }; - any = true; } match p.get("resolution") { Some(Value::String(r)) => { @@ -74,15 +94,21 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result "quarter" => 0.25, _ => return Err(bad(cmd, "resolution: full|half|third|quarter|")), }; - any = true; } Some(Value::Number(n)) => { rs.resolution = n.as_f64().unwrap_or(1.0).clamp(0.01, 4.0); - any = true; } _ => {} } - let (start, end) = (time_p(p, "start"), time_p(p, "end")); + let (start, mut end) = (time_p(p, "start"), time_p(p, "end")); + if let Some(d) = time_p(p, "duration") { + // Time Span ▸ Custom ▸ Duration: from the given (or current custom) start. + let a = start.unwrap_or(match rs.time_span { + TimeSpan::Custom { start, .. } => start, + _ => Tick::ZERO, + }); + end = Some(a + d); + } if let Some(ts) = str_p(p, "timeSpan") { rs.time_span = match ts.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { "workarea" | "workareaonly" => TimeSpan::WorkArea, @@ -90,14 +116,12 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result "custom" => TimeSpan::Custom { start: start.unwrap_or(Tick::ZERO), end: end.unwrap_or(Tick::from_seconds_f64(1.0)) }, _ => return Err(bad(cmd, "timeSpan: workArea|comp|custom")), }; - any = true; } else if start.is_some() || end.is_some() { let (a, b) = match rs.time_span { TimeSpan::Custom { start, end } => (start, end), _ => (Tick::ZERO, Tick(i64::MAX / 4)), }; rs.time_span = TimeSpan::Custom { start: start.unwrap_or(a), end: end.unwrap_or(b) }; - any = true; } if let TimeSpan::Custom { start, end } = rs.time_span && end <= start @@ -106,61 +130,193 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result } if let Some(u) = str_p(p, "proxyUse") { rs.proxy_use = effectcraft_project::render_queue::ProxyUse::parse(u).ok_or_else(|| bad(cmd, "proxyUse: current|all|comp|none"))?; - any = true; } match p.get("frameRate") { Some(Value::Null) => { rs.frame_rate = None; - any = true; } Some(v) => { let f = v.as_f64().filter(|f| *f > 0.0 && *f <= 1000.0).ok_or_else(|| bad(cmd, "frameRate: fps > 0 or null (comp rate)"))?; rs.frame_rate = Some(FrameRate::from_f64(f)); - any = true; } None => {} } - if let Some(b) = b_p(p, "motionBlur") { - rs.motion_blur = b; - any = true; + match p.get("motionBlur") { + Some(Value::Bool(b)) => { + rs.motion_blur = *b; + if *b && rs.motion_blur_mode == SwitchOverride::OffForAll { + rs.motion_blur_mode = SwitchOverride::Current; + } + } + Some(_) => { + let m = enum_p(p, "motionBlur", cmd, SwitchOverride::parse, "current|onForChecked|offForAll")?.unwrap_or_default(); + rs.motion_blur = m != SwitchOverride::OffForAll; + rs.motion_blur_mode = m; + } + None => {} + } + if let Some(m) = enum_p(p, "frameBlending", cmd, SwitchOverride::parse, "current|onForChecked|offForAll")? { + rs.frame_blending = m; + } + if let Some(f) = enum_p(p, "fieldRender", cmd, FieldRender::parse, "off|upper|lower")? { + rs.field_render = f; + } + if let Some(f) = enum_p(p, "pulldown", cmd, Pulldown::parse, "off|WSSWW|SSWWW|SWWWS|WWWSS|WWSSW")? { + rs.pulldown = f; + } + if let Some(e) = enum_p(p, "effects", cmd, EffectsMode::parse, "current|allOn|allOff")? { + rs.effects = e; + } + if let Some(e) = enum_p(p, "solo", cmd, CurrentOrOff::parse, "current|allOff")? { + rs.solo = e; + } + if let Some(e) = enum_p(p, "guideLayers", cmd, CurrentOrOff::parse, "current|allOff")? { + rs.guide_layers = e; + } + if let Some(e) = enum_p(p, "colorDepth", cmd, ColorDepth::parse, "current|8|16|32")? { + rs.color_depth = e; } if let Some(b) = b_p(p, "skipExisting") { rs.skip_existing = b; + } + if let Some(b) = b_p(p, "storageOverflow") { + rs.storage_overflow = b; + } + if *rs != before { + rs.name = "Custom".into(); any = true; } Ok(any) } -/// Apply Output Module parameters. -fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { +/// Apply a template to a module, keeping its output path (with the extension fixed). +fn apply_om_template(om: &mut OutputModule, t: &OutputModule) { + let keep = om.output.clone(); + let old = om.format; + *om = t.clone(); + om.output = keep; + let f = om.format; + om.format = old; + om.set_format(f); + om.channels = t.channels; + om.name = t.name.clone(); +} + +/// Apply Output Module parameters (`template` first). `roi`: the viewer's region of interest +/// (captured by `crop.useRoi`). +fn apply_output(om: &mut OutputModule, templates: &RenderTemplates, roi: Option<[f64; 4]>, p: &Value, cmd: &str) -> Result { let mut any = false; + if let Some(t) = str_p(p, "template") { + let t = templates.output_module(t).ok_or_else(|| bad(cmd, format!("no Output Module template “{t}”")))?; + apply_om_template(om, &t); + any = true; + } + let before = om.clone(); if let Some(f) = str_p(p, "format") { let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); - any = true; } if let Some(c) = str_p(p, "channels") { om.channels = match c.to_ascii_lowercase().replace([' ', '+'], "").as_str() { "rgb" => Channels::Rgb, "rgba" | "rgbalpha" => Channels::Rgba, - _ => return Err(bad(cmd, "channels: rgb|rgba")), + "alpha" | "alphaonly" => Channels::Alpha, + _ => return Err(bad(cmd, "channels: rgb|rgba|alpha")), }; if om.channels == Channels::Rgba && !om.format.supports_alpha() { return Err(bad(cmd, format!("{} has no alpha channel", om.format.label()))); } - any = true; + } + if let Some(c) = str_p(p, "color").or(str_p(p, "alphaMode")) { + om.alpha_mode = match c.to_ascii_lowercase().replace([' ', '(', ')'], "").as_str() { + "straight" | "straightunmatted" => AlphaMode::Straight, + "premultiplied" | "premultipliedmatted" | "premul" => AlphaMode::Premultiplied, + _ => return Err(bad(cmd, "color: straight|premultiplied")), + }; } if let Some(q) = f_p(p, "quality") { om.quality = q.clamp(1.0, 100.0) as u8; - any = true; } if let Some(b) = f_p(p, "bitrate") { - om.bitrate_kbps = b.clamp(100.0, 500_000.0) as u32; - any = true; + om.bitrate_kbps = b.clamp(50.0, 500_000.0) as u32; + } + if let Some(b) = b_p(p, "webmBitrate") { + om.webm_bitrate = b; + } + if let Some(k) = p.get("keyframeInterval").and_then(Value::as_u64) { + om.keyframe_interval = k.min(100_000) as u32; } if let Some(pp) = str_p(p, "proresProfile") { om.prores_profile = ProResProfile::from_name(pp).ok_or_else(|| bad(cmd, "proresProfile: proxy|lt|standard|hq|4444|4444xq"))?; - any = true; + } + match p.get("crop") { + Some(Value::Bool(b)) => om.crop.enabled = *b, + Some(Value::Object(c)) => { + om.crop.enabled = c.get("enabled").and_then(Value::as_bool).unwrap_or(true); + for (k, v) in [("top", &mut om.crop.top), ("left", &mut om.crop.left), ("bottom", &mut om.crop.bottom), ("right", &mut om.crop.right)] { + if let Some(n) = c.get(k).and_then(Value::as_i64) { + *v = n.clamp(-20_000, 20_000) as i32; + } + } + if let Some(u) = c.get("useRoi").and_then(Value::as_bool) { + om.crop.use_roi = u; + if u { + om.crop.roi = c.get("roi").and_then(|r| serde_json::from_value::<[f64; 4]>(r.clone()).ok()).or(roi); + if om.crop.roi.is_none() { + return Err(bad( + cmd, + "Use Region of Interest: the composition has no region of interest (View ▸ Region of Interest) — pass crop.roi [x, y, w, h]", + )); + } + } + } + } + Some(Value::Null) | None => {} + Some(_) => return Err(bad(cmd, "crop: bool or {enabled?, useRoi?, roi?: [x,y,w,h], top?, left?, bottom?, right?}")), + } + // Flat keys (dialog forms): cropTop/cropLeft/cropBottom/cropRight, resizeWidth/resizeHeight. + for (k, v) in [("cropTop", &mut om.crop.top), ("cropLeft", &mut om.crop.left), ("cropBottom", &mut om.crop.bottom), ("cropRight", &mut om.crop.right)] { + if let Some(n) = p.get(k).and_then(Value::as_f64) { + *v = (n.round() as i64).clamp(-20_000, 20_000) as i32; + om.crop.enabled = true; + om.crop.use_roi = false; + } + } + for (k, v) in [("resizeWidth", &mut om.resize.width), ("resizeHeight", &mut om.resize.height)] { + if let Some(n) = p.get(k).and_then(Value::as_f64) { + *v = (n.round() as u32).clamp(1, 30_000); + om.resize.enabled = true; + } + } + match p.get("resize") { + Some(Value::Bool(b)) => om.resize.enabled = *b, + Some(Value::Object(r)) => { + om.resize.enabled = r.get("enabled").and_then(Value::as_bool).unwrap_or(true); + if let Some(name) = r.get("preset").and_then(Value::as_str) { + let (_, w, h) = + RESIZE_PRESETS.iter().find(|(n, _, _)| n.eq_ignore_ascii_case(name)).ok_or_else(|| bad(cmd, format!("no resize preset “{name}”")))?; + (om.resize.width, om.resize.height) = (*w, *h); + om.resize.lock_aspect = false; + } + if let Some(l) = r.get("lockAspect").and_then(Value::as_bool) { + om.resize.lock_aspect = l; + } + if let Some(w) = r.get("width").and_then(Value::as_u64) { + om.resize.width = (w as u32).clamp(1, 30_000); + } + if let Some(h) = r.get("height").and_then(Value::as_u64) { + om.resize.height = (h as u32).clamp(1, 30_000); + } + if let Some(q) = r.get("quality").and_then(Value::as_str) { + om.resize.quality = match q.to_ascii_lowercase().as_str() { + "low" | "bilinear" => ResizeQuality::Low, + "high" | "bicubic" => ResizeQuality::High, + _ => return Err(bad(cmd, "resize.quality: low|high")), + }; + } + } + Some(Value::Null) | None => {} + Some(_) => return Err(bad(cmd, "resize: bool or {enabled?, preset?, width?, height?, lockAspect?, quality?: low|high}")), } if let Some(a) = p.get("audio") { om.audio = match a { @@ -174,15 +330,29 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { }, _ => return Err(bad(cmd, "audio: auto|on|off")), }; - any = true; } if let Some(r) = p.get("sampleRate").and_then(Value::as_u64) { om.audio_sample_rate = (r as u32).clamp(8_000, 192_000); - any = true; + } + match p.get("audioChannels") { + Some(Value::Number(n)) => om.audio_channels = if n.as_u64() == Some(1) { 1 } else { 2 }, + Some(Value::String(c)) => { + om.audio_channels = match c.to_ascii_lowercase().as_str() { + "mono" | "1" => 1, + "stereo" | "2" => 2, + _ => return Err(bad(cmd, "audioChannels: mono|stereo")), + } + } + _ => {} + } + if let Some(f) = enum_p(p, "audioFormat", cmd, AudioFormat::parse, "16|24|32 (float)")? { + om.audio_format = f; } if let Some(l) = b_p(p, "loop") { om.gif_loop = l; - any = true; + } + if let Some(l) = b_p(p, "includeProjectLink") { + om.include_project_link = l; } if let Some(o) = str_p(p, "output").or(str_p(p, "path")) { om.output = o.to_string(); @@ -194,6 +364,14 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { om.set_format(f); om.output = keep; } + } + if *om != before { + // Output To alone doesn't make the module custom. + let mut o = om.clone(); + o.output = before.output.clone(); + if o != before { + om.name = "Custom".into(); + } any = true; } Ok(any) @@ -227,7 +405,7 @@ fn requeue(it: &mut RenderQueueItem) { fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { let comp = s.project.comp(it.comp); let name = s.project.item(it.comp).map(|i| i.name.clone()); - let (w, h) = comp.map(|c| it.settings.output_size(c)).map(|(w, h)| it.output.format.coded_size(w, h)).unwrap_or((0, 0)); + let (w, h) = comp.map(|c| it.output.output_size(c, &it.settings)).unwrap_or((0, 0)); let frames = comp.map(|c| it.settings.frame_count(c)).unwrap_or(0); let mut v = serde_json::to_value(it).unwrap_or(Value::Null); if let Some(o) = v.as_object_mut() { @@ -249,6 +427,10 @@ fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { o.insert("outputModules".into(), json!(it.extra_outputs.len() + 1)); o.insert("extraOutputs".into(), json!(extra)); o.insert("postRenderAction".into(), json!(it.post_render.label())); + o.insert("logLabel".into(), json!(it.log.label())); + if let Some(p) = s.resolve_output(it) { + o.insert("logPath".into(), json!(effectcraft_project::render_queue::log_path(&p))); + } o.insert("width".into(), json!(w)); o.insert("height".into(), json!(h)); o.insert("frames".into(), json!(frames)); @@ -259,16 +441,25 @@ fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); + // New items start from the Movie Default templates (Edit ▸ Templates). + let templates = s.project.render_templates.clone(); + it.settings = templates.default_render_settings(TemplateSlot::Movie); + it.output = templates.default_output_module(TemplateSlot::Movie); if let Some(f) = str_p(p, "format") { let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; - it.output = OutputModule::for_format(f); + if f != it.output.format { + it.output = OutputModule::for_format(f); + } } - apply_settings(&mut it.settings, p, "renderQueue.add")?; + apply_settings(&mut it.settings, &templates, p, "renderQueue.add")?; let mut op = p.clone(); if let Some(o) = op.as_object_mut() { o.remove("format"); } - apply_output(&mut it.output, &op, "renderQueue.add")?; + apply_output(&mut it.output, &templates, s.state.region_of_interest, &op, "renderQueue.add")?; + if let Some(l) = enum_p(p, "log", "renderQueue.add", RenderLog::parse, "errorsOnly|plusSettings|plusPerFrameInfo")? { + it.log = l; + } let id = s.edit("Add to Render Queue", None, |proj, _| { it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; let id = it.id; @@ -305,7 +496,7 @@ fn set_render(s: &mut Session, p: &Value) -> Result { fn set_render_settings(s: &mut Session, p: &Value) -> Result { let i = item_index(s, p, "renderQueue.setRenderSettings")?; let mut rs = s.project.render_queue[i].settings.clone(); - if !apply_settings(&mut rs, p, "renderQueue.setRenderSettings")? { + if !apply_settings(&mut rs, &s.project.render_templates, p, "renderQueue.setRenderSettings")? { return Err(bad("renderQueue.setRenderSettings", "nothing to change")); } s.edit("Render Settings", None, |proj, _| { @@ -321,12 +512,20 @@ fn set_output_module(s: &mut Session, p: &Value) -> Result { let i = item_index(s, p, "renderQueue.setOutputModule")?; let m = module_p(s, i, p, "renderQueue.setOutputModule")?; let mut om = module_ref(&s.project.render_queue[i], m).clone(); - if !apply_output(&mut om, p, "renderQueue.setOutputModule")? { + let changed = apply_output(&mut om, &s.project.render_templates, s.state.region_of_interest, p, "renderQueue.setOutputModule")?; + let post = match str_p(p, "postRenderAction") { + Some(a) => Some(post_render_parse(a).ok_or_else(|| bad("renderQueue.setOutputModule", "postRenderAction: none|import|importAndReplace|setProxy"))?), + None => None, + }; + if !changed && post.is_none() { return Err(bad("renderQueue.setOutputModule", "nothing to change")); } s.edit("Output Module Settings", None, |proj, _| { let it = &mut proj.render_queue[i]; *module_mut(it, m) = om; + if let Some(a) = post { + it.post_render = a; + } requeue(it); Ok(()) })?; @@ -338,7 +537,7 @@ fn set_output(s: &mut Session, p: &Value) -> Result { let path = str_p(p, "path").or(str_p(p, "output")).ok_or_else(|| bad("renderQueue.setOutput", "pass `path` (file path or template)"))?; let m = module_p(s, i, p, "renderQueue.setOutput")?; let mut om = module_ref(&s.project.render_queue[i], m).clone(); - apply_output(&mut om, &json!({"output": path}), "renderQueue.setOutput")?; + apply_output(&mut om, &s.project.render_templates, None, &json!({"output": path}), "renderQueue.setOutput")?; s.edit("Output To", None, |proj, _| { let it = &mut proj.render_queue[i]; *module_mut(it, m) = om; @@ -452,7 +651,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { if let Some(o) = op.as_object_mut() { o.remove("format"); } - apply_output(&mut om, &op, "render.addOutputModule")?; + apply_output(&mut om, &s.project.render_templates, s.state.region_of_interest, &op, "render.addOutputModule")?; s.edit("Add Output Module", None, |proj, _| { let it = &mut proj.render_queue[i]; it.extra_outputs.push(om); @@ -469,11 +668,11 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { fn pre_render(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); - it.output = OutputModule::for_format(OutputFormat::ProRes); - it.output.prores_profile = ProResProfile::P4444; - it.output.channels = Channels::Rgba; + let templates = s.project.render_templates.clone(); + it.settings = templates.default_render_settings(TemplateSlot::PreRender); + it.output = templates.default_output_module(TemplateSlot::PreRender); it.output.output = "[compName]_prerender.[fileExtension]".into(); - apply_output(&mut it.output, p, "render.preRender")?; + apply_output(&mut it.output, &templates, s.state.region_of_interest, p, "render.preRender")?; it.post_render = PostRenderAction::ImportAndReplace; let id = s.edit("Pre-render", None, |proj, _| { it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; @@ -506,6 +705,8 @@ fn save_current_preview(s: &mut Session, p: &Value) -> Result { accel: s.accel.as_deref(), item: &item, path: &path, + storage: s.storage_quota.as_deref(), + label: "Save Current Preview".into(), nested_switches: s.prefs.general.switches_affect_nested_comps, }; let r = exporter.export(&job, &mut |_, _| true).map_err(EngineError::Other)?; @@ -513,6 +714,187 @@ fn save_current_preview(s: &mut Session, p: &Value) -> Result { Ok(serde_json::to_value(&r).unwrap_or_default()) } +// ---------------------------------------------------------------- templates (Edit ▸ Templates) + +fn kind_p(p: &Value, cmd: &str) -> Result { + str_p(p, "kind").and_then(TemplateKind::parse).ok_or_else(|| bad(cmd, "kind: renderSettings|outputModule")) +} + +fn name_p<'a>(p: &'a Value, cmd: &str) -> Result<&'a str> { + str_p(p, "name").map(str::trim).filter(|n| !n.is_empty()).ok_or_else(|| bad(cmd, "pass `name`")) +} + +/// Templates as JSON (lists, built-in flags, defaults per slot). +fn templates_json(t: &RenderTemplates) -> Value { + let rs: Vec = t + .render_settings_list() + .iter() + .map(|r| json!({"name": r.name, "builtin": RenderTemplates::is_builtin(TemplateKind::RenderSettings, &r.name), "settings": r})) + .collect(); + let om: Vec = t + .output_module_list() + .iter() + .map(|m| json!({"name": m.name, "builtin": RenderTemplates::is_builtin(TemplateKind::OutputModule, &m.name), "summary": format!("{} · {}", m.format.label(), m.channels.label()), "module": m})) + .collect(); + let slots = |k: TemplateKind| -> Value { + TemplateSlot::ALL.iter().map(|sl| (format!("{sl:?}"), json!(t.default_name(k, *sl)))).collect::>().into() + }; + json!({ + "renderSettings": rs, + "outputModules": om, + "defaults": {"renderSettings": slots(TemplateKind::RenderSettings), "outputModules": slots(TemplateKind::OutputModule)}, + "slots": TemplateSlot::ALL.iter().map(|sl| json!({"id": format!("{sl:?}"), "label": sl.label()})).collect::>(), + "resizePresets": RESIZE_PRESETS.iter().map(|(n, w, h)| json!({"name": n, "width": w, "height": h})).collect::>(), + }) +} + +fn templates(s: &mut Session, _: &Value) -> Result { + Ok(templates_json(&s.project.render_templates)) +} + +/// Save a template: from an item's settings (`item`/`index`, `module`), or the slot default, +/// adjusted by `params` (the same keys as setRenderSettings / setOutputModule). +fn save_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.saveTemplate"; + let kind = kind_p(p, C)?; + let name = name_p(p, C)?.to_string(); + if name.eq_ignore_ascii_case("custom") { + return Err(bad(C, "“Custom” is reserved")); + } + let item = if p.get("item").is_some() || p.get("index").is_some() { Some(item_index(s, p, C)?) } else { None }; + let params = p.get("params").cloned().unwrap_or(Value::Null); + let from = str_p(p, "from"); + let mut t = s.project.render_templates.clone(); + match kind { + TemplateKind::RenderSettings => { + let mut rs = match (item, from) { + (Some(i), _) => s.project.render_queue[i].settings.clone(), + (None, Some(f)) => t.render_settings(f).ok_or_else(|| bad(C, format!("no Render Settings template “{f}”")))?, + (None, None) => t.default_render_settings(TemplateSlot::Movie), + }; + if params.is_object() { + apply_settings(&mut rs, &t, ¶ms, C)?; + } + t.save_render_settings(&name, rs); + } + TemplateKind::OutputModule => { + let mut om = match (item, from) { + (Some(i), _) => module_ref(&s.project.render_queue[i], module_p(s, i, p, C)?).clone(), + (None, Some(f)) => t.output_module(f).ok_or_else(|| bad(C, format!("no Output Module template “{f}”")))?, + (None, None) => t.default_output_module(TemplateSlot::Movie), + }; + if params.is_object() { + apply_output(&mut om, &t, s.state.region_of_interest, ¶ms, C)?; + } + // Templates name files after the composition, not a particular path. + om.output = if om.format.is_sequence() { DEFAULT_SEQUENCE_TEMPLATE } else { DEFAULT_TEMPLATE }.into(); + t.save_output_module(&name, om); + } + } + s.edit("Save Template", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"saved": name, "templates": templates_json(&s.project.render_templates)})) +} + +fn delete_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.deleteTemplate"; + let kind = kind_p(p, C)?; + let name = name_p(p, C)?.to_string(); + let mut t = s.project.render_templates.clone(); + t.delete(kind, &name).map_err(|e| bad(C, e))?; + s.edit("Delete Template", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"deleted": name})) +} + +fn set_template_default(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setTemplateDefault"; + let kind = kind_p(p, C)?; + let slot = str_p(p, "slot").and_then(TemplateSlot::parse).ok_or_else(|| bad(C, "slot: movie|still|preRender|movieProxy|stillProxy"))?; + let name = name_p(p, C)?.to_string(); + let mut t = s.project.render_templates.clone(); + t.set_default(kind, slot, &name).map_err(|e| bad(C, e))?; + s.edit("Template Default", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"slot": format!("{slot:?}"), "name": name})) +} + +/// Apply templates to an item: `renderSettings` and/or `outputModule` (to `module`). +fn apply_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.applyTemplate"; + let i = item_index(s, p, C)?; + let m = module_p(s, i, p, C)?; + let t = s.project.render_templates.clone(); + let rs = match str_p(p, "renderSettings") { + Some(n) => Some(t.render_settings(n).ok_or_else(|| bad(C, format!("no Render Settings template “{n}”")))?), + None => None, + }; + let om = match str_p(p, "outputModule") { + Some(n) => { + let tpl = t.output_module(n).ok_or_else(|| bad(C, format!("no Output Module template “{n}”")))?; + let mut om = module_ref(&s.project.render_queue[i], m).clone(); + apply_om_template(&mut om, &tpl); + Some(om) + } + None => None, + }; + if rs.is_none() && om.is_none() { + return Err(bad(C, "pass `renderSettings` and/or `outputModule` (template names)")); + } + s.edit("Apply Template", None, |proj, _| { + let it = &mut proj.render_queue[i]; + if let Some(rs) = rs { + it.settings = rs; + } + if let Some(om) = om { + *module_mut(it, m) = om; + } + requeue(it); + Ok(()) + })?; + Ok(item_json(s, &s.project.render_queue[i], i)) +} + +fn set_log(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setLog"; + let i = item_index(s, p, C)?; + let log = enum_p(p, "log", C, RenderLog::parse, "errorsOnly|plusSettings|plusPerFrameInfo")?.ok_or_else(|| bad(C, "pass `log`"))?; + s.edit("Render Queue: Log", None, |proj, _| { + proj.render_queue[i].log = log; + Ok(()) + })?; + Ok(item_json(s, &s.project.render_queue[i], i)) +} + +fn set_notify(s: &mut Session, p: &Value) -> Result { + let on = b_p(p, "notify").or(b_p(p, "value")).unwrap_or(!s.project.render_prefs.notify); + s.edit("Render Queue: Notify", None, |proj, _| { + proj.render_prefs.notify = on; + Ok(()) + })?; + Ok(json!({"notify": on})) +} + +fn set_overflow(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setOverflowFolders"; + let folders: Vec = match p.get("folders") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_str).map(str::trim).filter(|f| !f.is_empty()).map(String::from).collect(), + Some(Value::Null) | None => vec![], + _ => return Err(bad(C, "folders: [path, …]")), + }; + s.edit("Storage Overflow Folders", None, |proj, _| { + proj.render_prefs.overflow_folders = folders.clone(); + Ok(()) + })?; + Ok(json!({"folders": s.project.render_prefs.overflow_folders})) +} + pub fn specs() -> Vec { vec![ cmd!( @@ -520,7 +902,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, template?: Render Settings template, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, log?: errorsOnly|plusSettings|plusPerFrameInfo, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, proxyUse?, effects?, solo?, guideLayers?, colorDepth?, frameBlending?, fieldRender?, pulldown?, motionBlur?, timeSpan?: workArea|comp|custom, start?: s, end?: s, duration?: s, frameRate?: fps|null, skipExisting?: bool, storageOverflow?: bool, channels?: rgb|rgba|alpha, color?, bitrate?: kbps, webmBitrate?, keyframeInterval?, proresProfile?, crop?, resize?, includeProjectLink?, audio?: auto|on|off, sampleRate?, audioChannels?, audioFormat?, loop?: bool (values as in setRenderSettings / setOutputModule)}", can_add, add ), @@ -531,7 +913,7 @@ pub fn specs() -> Vec { "Render Settings...", [], None, - "{item?|index?, quality?: best|draft, resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none}", + "{item?|index?, template?: name, quality?: best|draft, resolution?: full|half|third|quarter|scale, proxyUse?: current|all|comp|none, effects?: current|allOn|allOff, solo?: current|allOff, guideLayers?: current|allOff, colorDepth?: current|8|16|32, frameBlending?: current|onForChecked|offForAll, fieldRender?: off|upper|lower, pulldown?: off|WSSWW|SSWWW|SWWWS|WWWSS|WWSSW, motionBlur?: bool|current|onForChecked|offForAll, timeSpan?: workArea|comp|custom, start?: s, end?: s, duration?: s, frameRate?: fps|null, skipExisting?: bool, storageOverflow?: bool}", has_items, set_render_settings ), @@ -540,7 +922,7 @@ pub fn specs() -> Vec { "Output Module Settings...", [], None, - "{item?|index?, module?: n (1 = first), format?, channels?: rgb|rgba, quality?: 1-100, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool, output?}", + "{item?|index?, module?: n (1 = first), template?: name, format?, channels?: rgb|rgba|alpha, color?: straight|premultiplied, quality?: 1-100, bitrate?: kbps, webmBitrate?: bool, keyframeInterval?: frames, proresProfile?: proxy|lt|standard|hq|4444|4444xq, crop?: bool|{useRoi?, roi?, top?, left?, bottom?, right?}, cropTop?, cropLeft?, cropBottom?, cropRight?, resize?: bool|{preset?, width?, height?, lockAspect?, quality?: low|high}, resizeWidth?, resizeHeight?, postRenderAction?: none|import|importAndReplace|setProxy, includeProjectLink?: bool, audio?: auto|on|off, sampleRate?, audioChannels?: mono|stereo, audioFormat?: 16|24|32, loop?: bool, output?}", has_items, set_output_module ), @@ -569,6 +951,46 @@ pub fn specs() -> Vec { cmd!("renderQueue.render", "Render", [], None, "{wait?: bool (default true; the UI renders in the background)}", has_queued, render), cmd!("renderQueue.stop", "Stop Rendering", [], None, "{}", rendering, stop), query!("renderQueue.list", "Render Queue Items", "{}", list), + query!("renderQueue.templates", "Render Templates", "{}", templates), + cmd!( + "renderQueue.saveTemplate", + "Save Template...", + [], + None, + "{kind: renderSettings|outputModule, name, item?|index? (save from this item), from?: template to copy, module?, params?: {setRenderSettings / setOutputModule keys}}", + always, + save_template + ), + cmd!("renderQueue.deleteTemplate", "Delete Template", [], None, "{kind: renderSettings|outputModule, name}", always, delete_template), + cmd!( + "renderQueue.setTemplateDefault", + "Set Template Default", + [], + None, + "{kind: renderSettings|outputModule, slot: movie|still|preRender|movieProxy|stillProxy, name}", + always, + set_template_default + ), + cmd!( + "renderQueue.applyTemplate", + "Apply Template", + [], + None, + "{item?|index?, renderSettings?: template name, outputModule?: template name, module?: n}", + has_items, + apply_template + ), + cmd!("renderQueue.setLog", "Render Queue: Log", [], None, "{item?|index?, log: errorsOnly|plusSettings|plusPerFrameInfo}", has_items, set_log), + cmd!("renderQueue.setNotify", "Notify When Done", [], None, "{notify?: bool (toggles)}", always, set_notify), + cmd!( + "renderQueue.setOverflowFolders", + "Storage Overflow Folders", + [], + None, + "{folders: [path, …] (used in order when the output volume is full)}", + always, + set_overflow + ), query!("renderQueue.formats", "Output Formats", "{}", formats), ] } diff --git a/crates/engine/src/commands/scripts.rs b/crates/engine/src/commands/scripts.rs new file mode 100644 index 00000000..628533af --- /dev/null +++ b/crates/engine/src/commands/scripts.rs @@ -0,0 +1,222 @@ +//! File ▸ Scripts and ScriptUI. +//! +//! * **Installed scripts** live in the settings store under `Scripts/` (File ▸ Scripts ▸ Install +//! Script File…) and `Scripts/ScriptUI Panels/` (Install ScriptUI Panel…), next to the +//! settings on the desktop, so dropping `.jsx` files in those folders works too. File ▸ Scripts +//! lists them with the bundled sample scripts ([`SAMPLES`]) and runs them by name; ScriptUI +//! panels appear at the bottom of the Window menu and open as dockable panels +//! (`window.scriptPanel`), running with `this` = the panel as in After Effects. +//! * **ScriptUI windows** (`scriptui.*`): list the open script windows, read their control +//! trees, click buttons and set values like a user, close them (see [`crate::scriptui`]). + +use serde::Serialize; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, str_p}; +use crate::{EngineError, Event, Result, Session, cmd, scriptui}; + +/// The settings-store folders of installed scripts. +pub const SCRIPTS_DIR: &str = "Scripts"; +pub const PANELS_DIR: &str = "Scripts/ScriptUI Panels"; + +/// Sample scripts that ship with EffectCraft (original work): (file name, ScriptUI panel, code). +pub const SAMPLES: &[(&str, bool, &str)] = &[ + ("Create Null at Selected Layers.jsx", false, include_str!("../../scripts/Create Null at Selected Layers.jsx")), + ("Rename Layers.jsx", false, include_str!("../../scripts/Rename Layers.jsx")), + ("Render Queue Batch.jsx", false, include_str!("../../scripts/Render Queue Batch.jsx")), + ("Sort Layers by In Point.jsx", false, include_str!("../../scripts/Sort Layers by In Point.jsx")), + ("Layer Tools.jsx", true, include_str!("../../scripts/ScriptUI Panels/Layer Tools.jsx")), +]; + +/// A script File ▸ Scripts (or the Window menu, for panels) offers. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ScriptEntry { + /// File name (`Rename Layers.jsx`): what the menus show and commands take. + pub name: String, + /// In the ScriptUI Panels folder: opens as a dockable panel. + pub panel: bool, + /// `installed` or `sample`. + pub source: &'static str, +} + +fn is_script(name: &str) -> bool { + let l = name.to_ascii_lowercase(); + l.ends_with(".jsx") || l.ends_with(".js") +} + +/// Installed scripts and panels, then the samples not shadowed by an installed one, by name. +pub fn scripts(s: &Session) -> Vec { + let mut v: Vec = vec![]; + if let Some(cfg) = &s.config { + for (dir, panel) in [(SCRIPTS_DIR, false), (PANELS_DIR, true)] { + for n in cfg.list(dir).into_iter().filter(|n| is_script(n)) { + v.push(ScriptEntry { name: n, panel, source: "installed" }); + } + } + } + for (name, panel, _) in SAMPLES { + if !v.iter().any(|e| e.name == *name && e.panel == *panel) { + v.push(ScriptEntry { name: name.to_string(), panel: *panel, source: "sample" }); + } + } + v.sort_by(|a, b| a.panel.cmp(&b.panel).then(a.name.to_lowercase().cmp(&b.name.to_lowercase()))); + v +} + +/// The code of an installed or sample script / panel. +pub fn script_code(s: &Session, name: &str, panel: Option) -> Option<(bool, String)> { + if let Some(cfg) = &s.config { + for (dir, is_panel) in [(SCRIPTS_DIR, false), (PANELS_DIR, true)] { + if panel.is_some_and(|p| p != is_panel) { + continue; + } + if let Some(code) = cfg.read(&format!("{dir}/{name}")) { + return Some((is_panel, code)); + } + } + } + SAMPLES.iter().find(|(n, p, _)| *n == name && panel.is_none_or(|w| w == *p)).map(|(_, p, c)| (*p, c.to_string())) +} + +fn install(s: &mut Session, p: &Value, panel: bool) -> Result { + let c = if panel { "file.installScriptUIPanel" } else { "file.installScript" }; + let path = str_p(p, "path").ok_or_else(|| bad(c, "missing `path` (a .jsx / .js script)"))?; + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + if !is_script(&name) { + return Err(bad(c, "scripts are .jsx or .js files (.jsxbin is not supported)")); + } + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let code = String::from_utf8(bytes).map_err(|_| bad(c, "the script is not UTF-8 text"))?; + let cfg = s.config.clone().ok_or_else(|| EngineError::Other("this session has no settings folder to install scripts into".into()))?; + let dir = if panel { PANELS_DIR } else { SCRIPTS_DIR }; + cfg.write(&format!("{dir}/{name}"), &code).map_err(|e| EngineError::Other(format!("cannot install {name}: {e}")))?; + // After Effects asks for a restart; our menus pick it up at once. + s.toast(if panel { + format!("Installed ScriptUI panel {name}: find it at the bottom of the Window menu") + } else { + format!("Installed script {name}: find it in File ▸ Scripts") + }); + Ok(json!({"name": name, "panel": panel})) +} + +fn list(s: &mut Session, _: &Value) -> Result { + Ok(json!(scripts(s))) +} + +fn uninstall(s: &mut Session, p: &Value) -> Result { + let c = "file.uninstallScript"; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name`"))?; + let cfg = s.config.clone().ok_or_else(|| bad(c, "no settings folder"))?; + let mut removed = false; + for dir in [SCRIPTS_DIR, PANELS_DIR] { + let key = format!("{dir}/{name}"); + if cfg.read(&key).is_some() { + cfg.remove(&key).map_err(|e| EngineError::Other(e.to_string()))?; + removed = true; + } + } + if removed { Ok(json!({"removed": name})) } else { Err(bad(c, format!("`{name}` is not installed"))) } +} + +/// Run an installed or sample script by name (File ▸ Scripts ▸ ). A ScriptUI panel run +/// this way gets `this` = a floating palette-like panel window. +pub(crate) fn run_named(s: &mut Session, name: &str) -> Result { + let (panel, code) = script_code(s, name, None).ok_or_else(|| bad("file.runScript", format!("no installed or sample script `{name}`")))?; + if panel { + return open_panel(s, &json!({"name": name})); + } + super::file_more::run_js_named(s, &code, name) +} + +/// The code a ScriptUI panel runs with: `this` is a dockable Panel titled after the script. +pub fn panel_wrapper(title: &str, code: &str) -> String { + // On one line, so line numbers in errors stay those of the file. + format!("(function () {{ {code}\n}}).call(__uiDockPanel({}));", Value::String(title.to_string())) +} + +/// Window ▸ : run the panel script (or bring its panel forward). +fn open_panel(s: &mut Session, p: &Value) -> Result { + let c = "window.scriptPanel"; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name` (a script in the ScriptUI Panels folder)"))?.to_string(); + let title = name.trim_end_matches(".jsx").trim_end_matches(".js").to_string(); + if let Some(w) = s.script_ui.windows.iter().find(|w| w.kind == scriptui::WindowKind::Panel && w.script == name) { + let id = w.id; + s.events.push(Event::Frontend { command: c.into(), params: json!({"window": id}) }); + return Ok(json!({"window": id, "open": true})); + } + let (_, code) = script_code(s, &name, Some(true)).or_else(|| script_code(s, &name, None)).ok_or_else(|| bad(c, format!("no ScriptUI panel `{name}`")))?; + let out = super::file_more::run_js_named(s, &panel_wrapper(&title, &code), &name)?; + let id = s.script_ui.windows.iter().rev().find(|w| w.kind == scriptui::WindowKind::Panel && w.script == name).map(|w| w.id); + if let Some(id) = id { + s.events.push(Event::Frontend { command: c.into(), params: json!({"window": id}) }); + } + Ok(json!({"window": id, "result": out})) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "file.installScript", + "Install Script File...", + ["File", "Scripts"], + None, + "{path (.jsx / .js)} → copies it to the Scripts folder; it appears in File ▸ Scripts", + always, + |s, p| install(s, p, false) + ), + cmd!( + "file.installScriptUIPanel", + "Install ScriptUI Panel...", + ["File", "Scripts"], + None, + "{path (.jsx / .js)} → copies it to the ScriptUI Panels folder; it appears in the Window menu", + always, + |s, p| install(s, p, true) + ), + cmd!("file.uninstallScript", "Uninstall Script", [], None, "{name}", always, uninstall), + crate::query!("file.scripts.list", "List Scripts", "{} → [{name, panel, source: installed|sample}]", list), + cmd!( + "window.scriptPanel", + "ScriptUI Panel", + [], + None, + "{name (a script in the ScriptUI Panels folder, e.g. `Layer Tools.jsx`)} → opens it as a dockable panel", + always, + open_panel + ), + crate::query!("scriptui.list", "List Script Windows", "{} → [{window, title, kind: dialog|palette|window|panel, script, modal, size}]", scriptui::list), + crate::query!( + "scriptui.get", + "Script Window Controls", + "{window?: id | title} → {id, title, kind, root: {id, type, name, text, value, checked, items, selection, bounds, enabled, children…}}", + scriptui::get + ), + cmd!( + "scriptui.click", + "Click Script Window Control", + [], + None, + "{window?: id | title, widget: id | \"#id\" | properties.name | text} (buttons, checkboxes, radio buttons, tabs)", + always, + scriptui::click + ), + cmd!( + "scriptui.set", + "Set Script Window Control", + [], + None, + "{window?, widget, value: text | number | bool | item index | item text} (edit text, sliders, checkboxes, lists) → fires onChanging / onChange", + always, + scriptui::set + ), + cmd!( + "scriptui.close", + "Close Script Window", + [], + None, + "{window?, result? (a dialog's show() returns it; default 2 = Cancel)}", + always, + scriptui::close + ), + ] +} diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rs index ae7cfe09..0608dc6c 100644 --- a/crates/engine/src/config.rs +++ b/crates/engine/src/config.rs @@ -16,6 +16,12 @@ pub trait ConfigStore: Send + Sync { fn read(&self, name: &str) -> Option; fn write(&self, name: &str, data: &str) -> std::io::Result<()>; fn remove(&self, name: &str) -> std::io::Result<()>; + /// Names of the entries directly in `dir` (a `/`-separated name prefix such as `Scripts`), + /// sorted. Stores that can't list return nothing. + fn list(&self, dir: &str) -> Vec { + let _ = dir; + vec![] + } /// The folder next to the configuration (default Auto-Save folder for untitled projects), /// in the namespace of [`ConfigStore::files`]; `None` when there is none. fn dir(&self) -> Option { @@ -86,6 +92,11 @@ impl ConfigStore for MemoryConfig { } Ok(()) } + fn list(&self, dir: &str) -> Vec { + let pre = format!("{}/", dir.trim_end_matches('/')); + let Ok(m) = self.files.lock() else { return vec![] }; + m.keys().filter_map(|k| k.strip_prefix(&pre)).filter(|n| !n.is_empty() && !n.contains('/')).map(str::to_string).collect() + } } /// Files in one directory, written atomically. @@ -104,8 +115,15 @@ impl ConfigStore for DirConfig { std::fs::read_to_string(self.dir.join(name)).ok() } fn write(&self, name: &str, data: &str) -> std::io::Result<()> { - std::fs::create_dir_all(&self.dir)?; - atomic_write(&self.dir.join(name), data.as_bytes()) + let path = self.dir.join(name); + std::fs::create_dir_all(path.parent().unwrap_or(&self.dir))?; + atomic_write(&path, data.as_bytes()) + } + fn list(&self, dir: &str) -> Vec { + let Ok(rd) = std::fs::read_dir(self.dir.join(dir)) else { return vec![] }; + let mut v: Vec = rd.flatten().filter(|e| e.path().is_file()).map(|e| e.file_name().to_string_lossy().to_string()).collect(); + v.sort(); + v } fn remove(&self, name: &str) -> std::io::Result<()> { match std::fs::remove_file(self.dir.join(name)) { diff --git a/crates/engine/src/history.rs b/crates/engine/src/history.rs new file mode 100644 index 00000000..36630969 --- /dev/null +++ b/crates/engine/src/history.rs @@ -0,0 +1,346 @@ +//! Branching undo history (the History panel; better than After Effects, whose undo is a +//! straight line). +//! +//! [`History`] keeps After Effects' linear model as its working line — `undo` (the states +//! before each step, oldest first) and `redo` (undone states, next first) — and adds +//! **branches**: when you undo and then do something new, the undone states are not thrown +//! away but kept as a [`Branch`] hanging off the state they grew from. [`History::tree`] lists +//! every state as a tree and [`Session::goto_history`] jumps to any of them, on any branch: the +//! path to the target becomes the working line, and the line you left becomes a branch. +//! +//! States are whole-project snapshots (`Arc`), identified by their address while the +//! history holds them ([`HistoryNode::id`]), so ids stay valid across undo, redo and jumps. + +use std::collections::HashMap; +use std::sync::Arc; + +use effectcraft_project::Project; +use serde::Serialize; + +use crate::Session; + +/// Label of the oldest state in the history (the project as opened, or the oldest state kept). +pub const ROOT_LABEL: &str = "Original"; + +/// Undo history of whole-project snapshots. +#[derive(Clone, Default)] +pub struct History { + /// The states before each step on the working line, oldest first: (step label, state). + pub undo: Vec<(String, Arc)>, + /// Undone steps, the next one last: (step label, the state it leads to). + pub redo: Vec<(String, Arc)>, + /// Key of the last merged step: a continuous gesture with the same key folds into one step. + pub merge_key: Option, + /// Alternative futures kept when a new step was made after undoing. + pub branches: Vec, +} + +/// A line of states that grows from another state of the history. +#[derive(Clone)] +pub struct Branch { + /// The state the branch grows from. + pub parent: Arc, + /// The branch's states in order: (label of the step that made it, state). + pub steps: Vec<(String, Arc)>, +} + +/// One state of the history tree (`edit.history.list`). +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryNode { + /// Position in the listing (depth-first, the working line's continuation first). + pub index: usize, + /// Stable id of the state while the history keeps it. + pub id: String, + /// The step that made this state ([`ROOT_LABEL`] for the oldest). + pub label: String, + /// Index of the state it was made from. + pub parent: Option, + /// Branch nesting (0 = the working line). + pub depth: usize, + /// The current state. + pub current: bool, + /// On the working line (Undo / Redo walk it). + pub line: bool, + /// On the working line after the current state (Redo reaches it). + pub future: bool, +} + +fn key(p: &Arc) -> usize { + Arc::as_ptr(p) as usize +} + +/// Id of a state (its address while the history holds it). +pub fn state_id(p: &Arc) -> String { + format!("s{:x}", key(p)) +} + +/// The whole tree as an arena (internal). +struct Arena { + label: Vec, + state: Vec>, + parent: Vec>, + children: Vec>, + /// Index of each working-line state, oldest first, and the current state's position in it. + line: Vec, + current: usize, +} + +impl Arena { + fn build(h: &History, current: &Arc) -> Arena { + let mut a = Arena { label: vec![], state: vec![], parent: vec![], children: vec![], line: vec![], current: 0 }; + let add = |a: &mut Arena, label: String, st: Arc, parent: Option| { + let i = a.state.len(); + a.label.push(label); + a.state.push(st); + a.parent.push(parent); + a.children.push(vec![]); + if let Some(p) = parent { + a.children[p].push(i); + } + i + }; + // The working line: undo states, the current state, then the redo states. + let mut prev: Option = None; + let mut next_label = ROOT_LABEL.to_string(); + for (label, st) in &h.undo { + let i = add(&mut a, std::mem::take(&mut next_label), st.clone(), prev); + a.line.push(i); + prev = Some(i); + next_label = label.clone(); + } + let cur = add(&mut a, next_label, current.clone(), prev); + a.line.push(cur); + a.current = cur; + prev = Some(cur); + for (label, st) in h.redo.iter().rev() { + let i = add(&mut a, label.clone(), st.clone(), prev); + a.line.push(i); + prev = Some(i); + } + // Branches, attached to whichever state they grow from (branches of branches need the + // parent branch first, hence the passes). Orphans (their state was dropped) are skipped. + let mut index: HashMap = a.state.iter().enumerate().map(|(i, s)| (key(s), i)).collect(); + let mut pending: Vec<&Branch> = h.branches.iter().collect(); + loop { + let before = pending.len(); + pending.retain(|b| { + let Some(&p) = index.get(&key(&b.parent)) else { return true }; + let mut prev = p; + for (label, st) in &b.steps { + if index.contains_key(&key(st)) { + // Already in the tree (shared state): continue from it. + prev = index[&key(st)]; + continue; + } + let i = add(&mut a, label.clone(), st.clone(), Some(prev)); + index.insert(key(st), i); + prev = i; + } + false + }); + if pending.is_empty() || pending.len() == before { + break; + } + } + a + } + + fn on_line(&self) -> Vec { + let mut v = vec![false; self.state.len()]; + for &i in &self.line { + v[i] = true; + } + v + } + + /// Depth-first listing: the working line's child first, then branches in creation order. + fn listing(&self) -> Vec { + let on_line = self.on_line(); + let pos_in_line: HashMap = self.line.iter().enumerate().map(|(k, &i)| (i, k)).collect(); + let cur_pos = pos_in_line[&self.current]; + let mut out: Vec = vec![]; + let mut order: HashMap = HashMap::new(); + // (node, depth) + let mut stack: Vec<(usize, usize)> = vec![(self.line[0], 0)]; + while let Some((n, depth)) = stack.pop() { + order.insert(n, out.len()); + let line = on_line[n]; + out.push(HistoryNode { + index: out.len(), + id: state_id(&self.state[n]), + label: self.label[n].clone(), + parent: self.parent[n].and_then(|p| order.get(&p).copied()), + depth, + current: n == self.current, + line, + future: line && pos_in_line[&n] > cur_pos, + }); + // Push in reverse so the first child pops first; the line child continues at the + // same depth, branch children one deeper; a branch's own continuation (its first + // child) stays at the branch's depth. + for (k, &c) in self.children[n].iter().enumerate().rev() { + let cont = if line { on_line[c] } else { k == 0 }; + stack.push((c, if cont { depth } else { depth + 1 })); + } + } + out + } +} + +impl History { + /// Record a new step made from `before`: the redo line (if any) is kept as a branch. + pub fn record(&mut self, label: &str, before: Arc, levels: usize) { + self.abandon_redo(&before); + self.undo.push((label.to_string(), before.clone())); + self.merge_key = None; + self.trim(levels, &before); + } + + /// Keep the undone states as a branch growing from `from` (the state a new step starts from). + pub fn abandon_redo(&mut self, from: &Arc) { + if self.redo.is_empty() { + return; + } + let steps: Vec<(String, Arc)> = self.redo.drain(..).rev().collect(); + self.branches.push(Branch { parent: from.clone(), steps }); + } + + /// Apply the undo-levels limit: drop the oldest working-line states, the branches that grew + /// from them, and the oldest branches when branches hold more states than `levels`. + pub fn trim(&mut self, levels: usize, current: &Arc) { + let levels = levels.max(1); + if self.undo.len() > levels { + let extra = self.undo.len() - levels; + self.undo.drain(..extra); + } + let mut total: usize = self.branches.iter().map(|b| b.steps.len()).sum(); + while total > levels && !self.branches.is_empty() { + total -= self.branches.remove(0).steps.len(); + } + self.prune(current); + } + + /// Drop branches whose parent state is no longer in the history (`current` = the current + /// state, which the history doesn't store). + pub fn prune(&mut self, current: &Arc) { + if self.branches.is_empty() { + return; + } + let mut live: std::collections::HashSet = self.undo.iter().chain(self.redo.iter()).map(|(_, p)| key(p)).collect(); + live.insert(key(current)); + let mut keep = vec![false; self.branches.len()]; + loop { + let mut changed = false; + for (i, b) in self.branches.iter().enumerate() { + if !keep[i] && live.contains(&key(&b.parent)) { + keep[i] = true; + live.extend(b.steps.iter().map(|(_, p)| key(p))); + changed = true; + } + } + if !changed { + break; + } + } + let mut k = keep.into_iter(); + self.branches.retain(|_| k.next().unwrap_or(false)); + } + + /// Drop everything (Edit ▸ Purge ▸ Undo, New / Open Project). + pub fn clear(&mut self) { + *self = History::default(); + } + + /// The history as a tree, given the current state. + pub fn tree(&self, current: &Arc) -> Vec { + Arena::build(self, current).listing() + } + + /// Number of states on branches (not on the working line). + pub fn branch_states(&self) -> usize { + self.branches.iter().map(|b| b.steps.len()).sum() + } +} + +impl Session { + /// The undo history as a tree (the History panel, `edit.history.list`). + pub fn history_tree(&self) -> Vec { + self.history.tree(&self.project) + } + + /// Jump to any state of the history (`id` from [`HistoryNode::id`]), including states on + /// other branches: the path to it becomes the working line (its undone continuation the + /// redo line), and the line left behind is kept as a branch. Returns false for an unknown + /// id. + pub fn goto_history(&mut self, id: &str) -> bool { + let a = Arena::build(&self.history, &self.project); + let Some(target) = a.state.iter().position(|s| state_id(s) == id) else { return false }; + if target == a.current { + return true; + } + let on_line = a.on_line(); + // Path root → target. + let mut path = vec![target]; + while let Some(p) = a.parent[*path.last().unwrap_or(&target)] { + path.push(p); + } + path.reverse(); + // Continue past the target: the working line where it runs on, else each branch's own + // continuation (its first child). + let mut new_line = path.clone(); + let mut n = target; + loop { + let kids = &a.children[n]; + let next = if on_line[n] { kids.iter().copied().find(|&c| on_line[c]) } else { kids.first().copied() }; + match next { + Some(c) => { + new_line.push(c); + n = c; + } + None => break, + } + } + let mut in_new = vec![false; a.state.len()]; + for &i in &new_line { + in_new[i] = true; + } + // Every state off the new line goes into branches: a chain per first child, the other + // children starting branches of their own. + fn chain(a: &Arena, start: usize, parent: usize, in_new: &[bool], out: &mut Vec) { + let mut steps = vec![]; + let mut n = start; + let mut extra: Vec<(usize, usize)> = vec![]; + loop { + steps.push((a.label[n].clone(), a.state[n].clone())); + let kids: Vec = a.children[n].iter().copied().filter(|&c| !in_new[c]).collect(); + let Some((&first, rest)) = kids.split_first() else { break }; + extra.extend(rest.iter().map(|&c| (c, n))); + n = first; + } + out.push(Branch { parent: a.state[parent].clone(), steps }); + for (c, p) in extra { + chain(a, c, p, in_new, out); + } + } + let mut branches = vec![]; + for &n in &new_line { + for &c in &a.children[n] { + if !in_new[c] { + chain(&a, c, n, &in_new, &mut branches); + } + } + } + let k = new_line.iter().position(|&i| i == target).unwrap_or(0); + let undo: Vec<(String, Arc)> = (0..k).map(|i| (a.label[new_line[i + 1]].clone(), a.state[new_line[i]].clone())).collect(); + let redo: Vec<(String, Arc)> = new_line[k + 1..].iter().rev().map(|&i| (a.label[i].clone(), a.state[i].clone())).collect(); + self.project = a.state[target].clone(); + self.history.undo = undo; + self.history.redo = redo; + self.history.branches = branches; + self.history.merge_key = None; + self.sanitize_state(); + self.bump(); + true + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 25c88881..71d20f98 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -13,6 +13,7 @@ pub mod camera_track; pub mod commands; pub mod config; pub mod demo; +pub mod history; pub mod jobs; pub mod learn; pub mod links; @@ -26,6 +27,7 @@ pub mod prefs; pub mod psd_import; pub mod render_queue; pub mod roto; +pub mod scriptui; mod session_settings; pub mod shortcuts; pub mod sysinfo; @@ -55,6 +57,7 @@ pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; pub use effectcraft_track as track; +pub use history::{Branch, History, HistoryNode}; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -98,15 +101,6 @@ pub trait Importer: Send + Sync { fn probe(&self, path: &str) -> std::result::Result; } -/// Undo history of whole-project snapshots. -#[derive(Clone, Default)] -pub struct History { - pub undo: Vec<(String, Arc)>, - pub redo: Vec<(String, Arc)>, - /// Key of the last merged step: a continuous gesture with the same key folds into one step. - pub merge_key: Option, -} - /// A keyframe reference: layer, property uid, key time (layer time). #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct KeyRef { @@ -292,6 +286,8 @@ pub struct Session { pub importer: Option>, /// Render Queue encoder (the export layer); `None` = export unavailable. pub exporter: Option>, + /// Free-space hook for Render Settings ▸ Use Storage Overflow (`None`: never full). + pub storage_quota: Option>, /// The running (or finished, not yet polled) render. pub render_job: Option, /// The running (or finished, not yet polled) track analysis. @@ -337,6 +333,11 @@ pub struct Session { /// The JavaScript scripting engine (set by the host that links `effectcraft-script`): /// `script.run`, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`) and the Script Console. pub script: Option, + /// Loads WebAssembly effect plug-ins (set by the host that links `effectcraft-plugin` with + /// its runtime): `effect.plugins.load`. + pub plugin_loader: Option, + /// ScriptUI windows and panels opened by scripts (see [`scriptui`]). + pub script_ui: scriptui::ScriptUi, /// The last physical-memory reading (Settings ▸ Memory & CPU budgets); `None` = unknown /// (headless sessions don't query it, see [`Session::memory_tick`]). pub sys_memory: Option, @@ -370,6 +371,10 @@ pub struct ScriptRequest<'a> { /// column, file} | null}`. pub type ScriptRunner = fn(&mut Session, &ScriptRequest) -> Value; +/// Loads one WebAssembly effect plug-in (`bytes`, `source` path) into the effect registry and +/// describes it (`{id, name, category, version, api, params}`). +pub type PluginLoader = fn(&[u8], &str) -> std::result::Result; + impl Default for Session { fn default() -> Self { Session { @@ -386,6 +391,7 @@ impl Default for Session { expr_check: None, importer: None, exporter: None, + storage_quota: None, render_job: None, track_job: None, mask_job: None, @@ -408,6 +414,8 @@ impl Default for Session { autosave: autosave::AutoSaveState::default(), snapshot: None, script: None, + plugin_loader: None, + script_ui: scriptui::ScriptUi::default(), sys_memory: None, applied_nested_switches: None, offload: None, @@ -489,15 +497,13 @@ impl Session { } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { - self.history.undo.push((label.to_string(), before)); + // Undone steps aren't lost: they stay in the History panel as a branch. let levels = self.prefs.general.undo_levels.max(1) as usize; - if self.history.undo.len() > levels { - let extra = self.history.undo.len() - levels; - self.history.undo.drain(..extra); - } + self.history.record(label, before, levels); + } else { + self.history.abandon_redo(&before); } self.history.merge_key = merge.map(str::to_string); - self.history.redo.clear(); self.project = Arc::new(p); self.state = st; self.bump(); @@ -779,6 +785,8 @@ mod tests_effects; #[cfg(test)] mod tests_essential; #[cfg(test)] +mod tests_face; +#[cfg(test)] mod tests_fidelity; #[cfg(test)] mod tests_lottie; @@ -896,6 +904,8 @@ pub fn font_menu(prefs: &prefs::Prefs) -> Vec { out } #[cfg(test)] +mod tests_history; +#[cfg(test)] mod tests_paint; #[cfg(test)] mod tests_panels; diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs index 00315198..9174258e 100644 --- a/crates/engine/src/mask_track.rs +++ b/crates/engine/src/mask_track.rs @@ -6,6 +6,12 @@ //! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the //! chosen motion model), and every frame's motion is applied to the mask's vertices and //! tangents, so the shape's Bezier structure is kept. +//! +//! The two **Face Tracking** methods use `effectcraft_track::face` instead: the mask (drawn +//! around a face) seeds the tracker, and every frame's face outline becomes the Mask Path +//! (*Outline Only*); *Detailed Features* also keys the facial landmarks into a **Face Track +//! Points** effect on the layer (created on first use), from which Extract & Copy Face +//! Measurements (`track.extractFaceMeasurements`) derives a keyed **Face Measurements** effect. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -31,11 +37,22 @@ pub enum MaskMethod { PositionScaleRotation, PositionScaleRotationSkew, Perspective, + /// Face Tracking (Outline Only). + FaceOutline, + /// Face Tracking (Detailed Features). + FaceDetailed, } impl MaskMethod { - pub const ALL: [MaskMethod; 5] = - [MaskMethod::Position, MaskMethod::PositionScale, MaskMethod::PositionScaleRotation, MaskMethod::PositionScaleRotationSkew, MaskMethod::Perspective]; + pub const ALL: [MaskMethod; 7] = [ + MaskMethod::Position, + MaskMethod::PositionScale, + MaskMethod::PositionScaleRotation, + MaskMethod::PositionScaleRotationSkew, + MaskMethod::Perspective, + MaskMethod::FaceOutline, + MaskMethod::FaceDetailed, + ]; pub fn label(self) -> &'static str { match self { MaskMethod::Position => "Position", @@ -43,8 +60,14 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "Position, Scale & Rotation", MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew", MaskMethod::Perspective => "Perspective", + MaskMethod::FaceOutline => "Face Tracking (Outline Only)", + MaskMethod::FaceDetailed => "Face Tracking (Detailed Features)", } } + /// One of the Face Tracking methods. + pub fn is_face(self) -> bool { + matches!(self, MaskMethod::FaceOutline | MaskMethod::FaceDetailed) + } pub fn id(self) -> &'static str { match self { MaskMethod::Position => "position", @@ -52,6 +75,8 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "positionScaleRotation", MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew", MaskMethod::Perspective => "perspective", + MaskMethod::FaceOutline => "faceOutline", + MaskMethod::FaceDetailed => "faceDetailed", } } pub fn from_name(s: &str) -> Option { @@ -67,6 +92,7 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => Model::Similarity, MaskMethod::PositionScaleRotationSkew => Model::Affine, MaskMethod::Perspective => Model::Homography, + MaskMethod::FaceOutline | MaskMethod::FaceDetailed => Model::Similarity, } } } @@ -109,8 +135,29 @@ pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath { } } -/// One tracked frame: (layer time, comp time, mask path). -type MaskFrame = (Tick, Tick, ShapePath); +/// A closed, smooth mask path through a face outline polygon: `n` vertices with Catmull-Rom +/// tangents. `like` lends its feather points. +pub fn outline_path(poly: &[[f64; 2]], n: usize, like: &ShapePath) -> ShapePath { + let m = poly.len(); + let n = n.clamp(3, m.max(3)); + let v: Vec<[f64; 2]> = (0..n).map(|i| poly[(i * m / n) % m.max(1)]).collect(); + let tan: Vec<[f64; 2]> = (0..n) + .map(|i| { + let (a, b) = (v[(i + n - 1) % n], v[(i + 1) % n]); + [(b[0] - a[0]) / 6.0, (b[1] - a[1]) / 6.0] + }) + .collect(); + ShapePath { + vertices: v, + in_tangents: tan.iter().map(|t| [-t[0], -t[1]]).collect(), + out_tangents: tan, + closed: true, + feather: if like.len() == n { like.feather.clone() } else { vec![] }, + } +} + +/// One tracked frame: (layer time, comp time, mask path, face landmarks for Detailed Features). +type MaskFrame = (Tick, Tick, ShapePath, Option<[[f64; 2]; trk::face::N]>); #[derive(Default)] pub struct MaskTrackShared { @@ -169,9 +216,12 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); lock(&shared.state).total = w.times.len().saturating_sub(1) as u64; let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + if w.method.is_face() { + return run_face(&w, shared, first, &frame_at, layer, t0); + } let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 }); let mut path = w.path.clone(); - lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None)); let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; for k in 1..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { @@ -190,7 +240,7 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { break; }; path = transform_path(&path, &step.motion); - lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone(), None)); let mut s = lock(&shared.state); s.done = k as u64; s.elapsed = t0.elapsed().as_secs_f64(); @@ -203,12 +253,99 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { s.finished = true; } +/// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks. +fn run_face( + w: &MaskWork, + shared: &MaskTrackShared, + first: (Arc, [f64; 2]), + frame_at: &(dyn Fn(Tick) -> Option<(Arc, [f64; 2])> + Sync), + layer: &effectcraft_project::Layer, + t0: web_time::Instant, +) { + let detailed = w.method == MaskMethod::FaceDetailed; + let b = trk::mask::bounds(&flatten(&w.path, 8)); + let Some((mut tracker, fit)) = trk::face::FaceTracker::new(&trk::Frame { img: &first.0, offset: first.1 }, b) else { + let mut s = lock(&shared.state); + s.error = Some("no face found inside the mask (draw the mask around a face)".into()); + s.finished = true; + return; + }; + let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks)); + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts)); + let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + for k in 1..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + lock(&shared.state).cancelled = true; + break; + } + let Some(cur) = next.take() else { + lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); + break; + }; + let (res, nf) = + rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + next = nf; + let Some(fit) = res else { + lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds())); + break; + }; + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path, pts)); + let mut s = lock(&shared.state); + s.done = k as u64; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 }; + s.time = w.times[k].seconds(); + crate::offload::report(s.done, s.total); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// The layer's Face Track Points effect (created when missing). Returns its uid. +pub(crate) fn face_points_group(p: &mut Project, comp: ItemId, layer: LayerId) -> Option { + face_effect(p, comp, layer, effectcraft_effects::face_track::POINTS_ID) +} + +/// The layer's effect `id`, applied when missing (data effects of face tracking). +pub(crate) fn face_effect(p: &mut Project, comp: ItemId, layer: LayerId, id: &str) -> Option { + let spec = effectcraft_effects::find(id)?; + let l = p.comp(comp)?.layer(layer)?; + if let Some(g) = l.effects().and_then(|fx| fx.groups().find(|g| g.match_id == spec.id)) { + return Some(g.uid); + } + let (w, h) = effectcraft_render::source_size(p, l); + let size = if w == 0 { p.comp(comp).map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) } else { [w as f64, h as f64] }; + let mut next = p.next_id; + let g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut next), spec.name, size); + let uid = g.uid; + let fx = p.comp_mut(comp)?.layer_mut(layer)?.props.sub_mut("effects")?; + fx.children.push(g.into()); + p.next_id = next; + Some(uid) +} + fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) { + let points = if frames.iter().any(|f| f.3.is_some()) { face_points_group(p, job.comp, job.layer) } else { None }; let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return }; - let Some(g) = layer.props.find_group_mut(job.mask) else { return }; - let Some(pr) = g.get_mut("path") else { return }; - for (lt, _, path) in frames { - effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + if let Some(g) = layer.props.find_group_mut(job.mask) + && let Some(pr) = g.get_mut("path") + { + for (lt, _, path, _) in frames { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + } + } + if let Some(g) = points.and_then(|u| layer.props.find_group_mut(u)) { + for (lt, _, _, pts) in frames { + let Some(pts) = pts else { continue }; + for (i, (id, _, _)) in trk::face::LANDMARKS.iter().enumerate() { + if let Some(pr) = g.get_mut(id) { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Vec2(pts[i]))); + } + } + } } } @@ -242,14 +379,13 @@ impl Session { return Err("a track analysis is already running".into()); } self.poll_mask_track(); - if !wait && self.offloads() { + if !wait && self.offloads() && !work.method.is_face() { let MaskWork { comp, layer, path, method, times, .. } = work; return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times }); } let wait = wait || cfg!(target_arch = "wasm32"); - self.history.undo.push(("Track Mask".into(), self.project.clone())); - self.history.redo.clear(); - self.history.merge_key = None; + let levels = self.prefs.general.undo_levels as usize; + self.history.record("Track Mask", self.project.clone(), levels); let shared = Arc::new(MaskTrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index fce44a90..c286f5b9 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -71,7 +71,7 @@ fn holder_of(s: &Session, keys: &str) -> Option { /// The entries of a `@dynamic:` node, and the disabled placeholder shown when there are /// none: `recentProjects`, `recentFootage`, `recentPresets`, `history` (undo steps, newest /// first), `view3dShortcuts`, `workspaceShortcuts` and `openViewers` (open compositions other -/// than the active one). +/// than the active one), `scripts` (File ▸ Scripts) and `scriptPanels` (ScriptUI panels). pub fn dynamic(s: &Session, name: &str, cx: &DynCtx) -> (Vec, Option<&'static str>) { use serde_json::json; let indexed = |list: &[String], cmd: &str, stem: bool| -> Vec { @@ -131,6 +131,19 @@ pub fn dynamic(s: &Session, name: &str, cx: &DynCtx) -> (Vec, Option< .collect(), None, ), + // File ▸ Scripts: installed and sample scripts; the Window menu's ScriptUI panels. + "scripts" | "scriptPanels" => { + let panels = name == "scriptPanels"; + let cmd = if panels { "window.scriptPanel" } else { "file.runScript" }; + ( + crate::commands::scripts::scripts(s) + .into_iter() + .filter(|e| e.panel == panels) + .map(|e| dyn_entry(e.name.clone(), cmd, json!({"name": e.name}))) + .collect(), + None, + ) + } _ => (vec![], None), } } @@ -419,7 +432,7 @@ pub fn parse(text: &str, mac: bool) -> Result, String> { /// Effect ▸ category submenus from the effect registry. fn effect_categories() -> Vec { let mut cats: Vec<(&str, Vec)> = vec![]; - for e in crate::effects::registry() { + for e in crate::effects::all() { let entry = MenuNode::Item(MenuEntry { label: e.name.into(), command: "effect.apply".into(), params: serde_json::json!({"effect": e.id}), shortcut: None }); match cats.iter_mut().find(|(c, _)| *c == e.category) { @@ -517,6 +530,10 @@ File Find Missing Footage | file.findMissing {"what":"footage"} --- Scripts + @dynamic:scripts + --- + Install Script File... | file.installScript + Install ScriptUI Panel... | file.installScriptUIPanel Run Script File... | file.runScript --- Create Proxy @@ -908,6 +925,7 @@ Effect Remove All | effect.removeAll --- Manage Effects... | effect.manage + Load Effect Plug-in... | effect.plugins.load --- @effects Animation @@ -1146,6 +1164,8 @@ Window Render Queue | window.panel {"panel":"renderQueue"} | Cmd+Alt+0 Timeline | window.panel {"panel":"timeline"} @dynamic:openViewers + --- + @dynamic:scriptPanels Help EffectCraft Help... | help.docs {"page":"help"} | F1 Scripting Help... | help.docs {"page":"scripting"} diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index c01d8513..5e77131c 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -27,6 +27,10 @@ pub struct ExportJob<'a> { pub accel: Option<&'a dyn effectcraft_render::Accelerator>, /// Resolved output path (templates expanded except the `#` frame-number run). pub path: &'a str, + /// Free-space hook (Use Storage Overflow). + pub storage: Option<&'a dyn effectcraft_project::render_queue::StorageQuota>, + /// "#n Comp Name", for the render log. + pub label: String, /// Settings ▸ General ▸ Switches Affect Nested Comps. pub nested_switches: bool, } @@ -41,6 +45,12 @@ pub struct ExportResult { pub bytes: u64, pub seconds: f64, pub audio: bool, + /// The render log written next to the output, if any. + #[serde(skip_serializing_if = "Option::is_none")] + pub log: Option, + /// Files that went to storage overflow folders. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub overflow: Vec, } /// Encodes render queue items (implemented by the export layer). @@ -129,6 +139,7 @@ struct Work { expr: Option>, accel: Option>, exporter: Arc, + storage: Option>, /// Per queue item: one (item with that output module, resolved path) per output module /// (and per segment, see [`Session::segments`]). items: Vec>, @@ -199,6 +210,7 @@ pub(crate) fn run_items( expr: s.expr.clone(), accel: s.accel.clone(), exporter, + storage: s.storage_quota.clone(), items, nested_switches: s.prefs.general.switches_affect_nested_comps, }; @@ -232,6 +244,8 @@ fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared)) { // first module's file is the item's output. let mut r: Result = Err("no output module".into()); for (mi, (mitem, path)) in modules.iter().enumerate() { + let name = w.project.item(mitem.comp).map(|i| i.name.clone()).unwrap_or_default(); + let index = w.project.render_queue.iter().position(|i| i.id == mitem.id).map_or(0, |i| i + 1); let job = ExportJob { project: &w.project, footage: w.footage.as_ref(), @@ -239,6 +253,8 @@ fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared)) { accel: w.accel.as_deref(), item: mitem, path, + storage: w.storage.as_deref(), + label: format!("#{index} {name} (output module {})", mi + 1), nested_switches: w.nested_switches, }; let rr = w.exporter.export(&job, &mut |done, total| { @@ -301,8 +317,7 @@ impl Session { if tpl.is_empty() { return None; } - let (w, h) = item.settings.output_size(comp); - let (w, h) = item.output.format.coded_size(w, h); + let (w, h) = item.output.output_size(comp, &item.settings); let rate = item.settings.rate(comp); let n = item.settings.frame_count(comp) as i64; let f0 = item.settings.first_frame(comp); @@ -453,6 +468,7 @@ impl Session { expr: self.expr.clone(), accel: self.accel.clone(), exporter, + storage: self.storage_quota.clone(), items, nested_switches: self.prefs.general.switches_affect_nested_comps, }; @@ -552,6 +568,10 @@ impl Session { } else { format!("Rendered {} item(s) in {:.1} s", st.items_total, st.elapsed) }; + if self.project.render_prefs.notify { + // Notify: the frontend plays a sound and shows a system notification. + self.events.push(Event::Frontend { command: "renderQueue.notify".into(), params: serde_json::json!({"message": msg, "error": failed > 0}) }); + } self.events.push(Event::Toast { message: msg, error: failed > 0 }); } for (id, path) in post { diff --git a/crates/engine/src/scriptui.rs b/crates/engine/src/scriptui.rs new file mode 100644 index 00000000..2a76ba04 --- /dev/null +++ b/crates/engine/src/scriptui.rs @@ -0,0 +1,572 @@ +//! ScriptUI: the windows, dialogs and dockable panels scripts build with After Effects' +//! ScriptUI object model (`new Window("dialog", "Title")`, `win.add("button", undefined, "OK")`, +//! `orientation`, `alignChildren`, `onClick`/`onChange`, `show()`, `close()`, +//! `layout.layout()`…). +//! +//! The scripting engine (`effectcraft-script`) owns the live JavaScript objects; after every +//! script step it publishes a serde description of each open window here ([`ScriptWindow`], +//! a [`Widget`] tree with computed [`Widget::bounds`]). Frontends draw that description (the egui +//! UI renders dialogs, palettes and dockable panels from it) and report what the user does as +//! [`ScriptUiEvent`]s through the `scriptui.*` commands, which hand them to the script's +//! handlers ([`ScriptUi::dispatch`]). Agents use the same commands: list the open script +//! windows, read their widget trees, click buttons and set values. +//! +//! [`layout`] is ScriptUI's automatic layout (orientation, alignChildren / alignment, spacing, +//! margins, preferredSize), so every frontend places widgets the same way. + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{EngineError, Result, Session}; + +/// What kind of window a script made. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WindowKind { + /// Modal: the script waits in `show()` until it closes. + #[default] + Dialog, + /// Floating, non-modal. + Palette, + /// A non-modal document window. + Window, + /// A dockable panel (a script from the ScriptUI Panels folder, opened from the Window menu). + Panel, +} + +/// ScriptUI control types (the `type` passed to `add()`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WidgetKind { + /// The window itself (the tree's root). + #[default] + Window, + Panel, + Group, + Button, + IconButton, + StaticText, + EditText, + Checkbox, + RadioButton, + Slider, + Scrollbar, + Progressbar, + DropDownList, + ListBox, + TabbedPanel, + Tab, + Image, + /// A type we don't draw (it still lays out as an empty box). + #[serde(other)] + Unknown, +} + +impl WidgetKind { + pub fn is_container(self) -> bool { + matches!(self, WidgetKind::Window | WidgetKind::Panel | WidgetKind::Group | WidgetKind::TabbedPanel | WidgetKind::Tab) + } +} + +/// One control (or container) of a script window. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Widget { + /// Unique within its window (the window itself is 0). + pub id: u32, + #[serde(rename = "type")] + pub kind: WidgetKind, + /// `properties.name` (agents can address controls by it). + pub name: String, + pub text: String, + /// Slider / scrollbar / progressbar value. + pub value: f64, + pub min: f64, + pub max: f64, + /// Checkbox / radio button state. + pub checked: bool, + /// Drop-down list / list box items. + pub items: Vec, + /// Selected item indices. + pub selection: Vec, + /// `column` (default for windows and panels), `row` (groups) or `stack`. + pub orientation: String, + /// `[horizontal, vertical]`: left / center / right / fill, top / center / bottom / fill. + pub align_children: Vec, + /// This control's own alignment in its parent (overrides the parent's alignChildren). + pub alignment: Vec, + pub spacing: Option, + /// `[left, top, right, bottom]`. + pub margins: Option<[f64; 4]>, + pub preferred_size: Option<[f64; 2]>, + /// Bounds set by the script (`[left, top, right, bottom]` relative to the parent). + pub fixed_bounds: Option<[f64; 4]>, + /// Laid-out bounds `[x, y, width, height]` relative to the window's content. + pub bounds: [f64; 4], + pub enabled: bool, + pub visible: bool, + pub help_tip: String, + pub multiline: bool, + pub read_only: bool, + /// Event handlers the script attached (`onClick`, `onChange`, `onChanging`…). + pub handlers: Vec, + /// The active tab of a tabbed panel (child index). + pub active_tab: usize, + pub children: Vec, +} + +impl Widget { + pub fn find(&self, id: u32) -> Option<&Widget> { + if self.id == id { + return Some(self); + } + self.children.iter().find_map(|c| c.find(id)) + } + pub fn find_mut(&mut self, id: u32) -> Option<&mut Widget> { + if self.id == id { + return Some(self); + } + self.children.iter_mut().find_map(|c| c.find_mut(id)) + } + /// Every control, depth first. + pub fn walk<'a>(&'a self, out: &mut Vec<&'a Widget>) { + out.push(self); + for c in &self.children { + c.walk(out); + } + } + /// A control by id (number or `"#12"`), `properties.name` or text. + pub fn lookup(&self, key: &Value) -> Option<&Widget> { + let mut all = vec![]; + self.walk(&mut all); + match key { + Value::Number(n) => n.as_u64().and_then(|n| self.find(n as u32)), + Value::String(s) => { + if let Some(n) = s.strip_prefix('#').and_then(|n| n.parse::().ok()) { + return self.find(n); + } + all.iter().find(|w| !w.name.is_empty() && w.name == *s).or_else(|| all.iter().find(|w| w.id != 0 && w.text == *s)).copied() + } + _ => None, + } + } +} + +/// An open script window. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ScriptWindow { + /// Unique in the process. + pub id: u32, + /// The script context that owns it (its handlers run there). + pub host: u32, + pub kind: WindowKind, + pub title: String, + /// The script that made it. + pub script: String, + /// Shown and not closed. + pub visible: bool, + /// The script is waiting in this dialog's `show()`. + pub modal: bool, + /// The window and its controls (`root.bounds` is the content size). + pub root: Widget, +} + +/// A user (or agent) action on a script window control. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptUiEvent { + pub window: u32, + pub widget: u32, + /// `click`, `change`, `changing` or `close`. + pub kind: String, + /// The new value: text, number, bool, or selected index(es); for `close`, the result. + pub value: Value, +} + +/// Hands an event to the script that owns the window; returns what the handler reported +/// (`{ok, output, error}`) and updates [`Session::script_ui`]. +pub type ScriptUiDispatch = fn(&mut Session, &ScriptUiEvent) -> std::result::Result; + +/// The open script windows. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct ScriptUi { + pub windows: Vec, + /// Bumped whenever a window changes. + pub revision: u64, + /// The scripting engine's event hand-off (set with [`Session::script`]). + #[serde(skip)] + pub dispatch: Option, +} + +impl ScriptUi { + pub fn window(&self, id: u32) -> Option<&ScriptWindow> { + self.windows.iter().find(|w| w.id == id) + } + + /// Replace the windows a script host published (laying them out). + pub fn publish(&mut self, host: u32, mut windows: Vec) { + for w in &mut windows { + w.host = host; + layout(w); + } + windows.retain(|w| w.visible); + let mut out: Vec = vec![]; + // Keep the order windows first appeared in. + for old in self.windows.drain(..) { + if old.host != host { + out.push(old); + } else if let Some(i) = windows.iter().position(|w| w.id == old.id) { + out.push(windows.remove(i)); + } + } + out.extend(windows); + self.windows = out; + self.revision += 1; + } +} + +// ---------------------------------------------------------------- layout + +/// Average character width and line height of the default dialog font (points). +const CHAR_W: f64 = 7.0; +const LINE_H: f64 = 16.0; + +fn text_w(s: &str) -> f64 { + s.lines().map(|l| l.chars().count()).max().unwrap_or(0) as f64 * CHAR_W +} + +fn lines(s: &str) -> f64 { + s.lines().count().max(1) as f64 +} + +fn orientation(w: &Widget) -> &str { + match w.orientation.as_str() { + "" => match w.kind { + WidgetKind::Group => "row", + WidgetKind::TabbedPanel => "stack", + _ => "column", + }, + o => o, + } +} + +fn margins(w: &Widget) -> [f64; 4] { + w.margins.unwrap_or(match w.kind { + WidgetKind::Window => [15.0; 4], + WidgetKind::Panel => [10.0, 15.0, 10.0, 10.0], + WidgetKind::TabbedPanel => [0.0, 24.0, 0.0, 0.0], + WidgetKind::Tab => [10.0; 4], + _ => [0.0; 4], + }) +} + +fn spacing(w: &Widget) -> f64 { + w.spacing.unwrap_or(10.0) +} + +fn align_children(w: &Widget) -> (String, String) { + let d = match w.kind { + WidgetKind::Group => ("center", "center"), + _ => ("center", "top"), + }; + let a = |i: usize, d: &str| w.align_children.get(i).filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| d.to_string()); + // A single value applies to the axis it names. + if w.align_children.len() == 1 { + let v = w.align_children[0].clone(); + return match v.as_str() { + "top" | "bottom" => (d.0.into(), v), + "fill" if orientation(w) == "row" => (d.0.into(), v), + _ => (v, d.1.into()), + }; + } + (a(0, d.0), a(1, d.1)) +} + +/// Preferred size of a control (its own content; containers include their children). +pub fn preferred(w: &Widget) -> [f64; 2] { + let ps = w.preferred_size.unwrap_or([0.0; 2]); + let content = if w.kind.is_container() { + let kids: Vec<[f64; 2]> = w.children.iter().filter(|c| c.visible).map(preferred).collect(); + let m = margins(w); + let sp = spacing(w); + let n = kids.len() as f64; + let (cw, ch) = match orientation(w) { + "row" => (kids.iter().map(|k| k[0]).sum::() + sp * (n - 1.0).max(0.0), kids.iter().map(|k| k[1]).fold(0.0, f64::max)), + "stack" => (kids.iter().map(|k| k[0]).fold(0.0, f64::max), kids.iter().map(|k| k[1]).fold(0.0, f64::max)), + _ => (kids.iter().map(|k| k[0]).fold(0.0, f64::max), kids.iter().map(|k| k[1]).sum::() + sp * (n - 1.0).max(0.0)), + }; + let title_w = if matches!(w.kind, WidgetKind::Panel) { text_w(&w.text) + 20.0 } else { 0.0 }; + [(cw + m[0] + m[2]).max(title_w), ch + m[1] + m[3]] + } else { + match w.kind { + WidgetKind::Button | WidgetKind::IconButton => [(text_w(&w.text) + 24.0).max(80.0), 25.0], + WidgetKind::StaticText => [text_w(&w.text) + 4.0, LINE_H * lines(&w.text)], + WidgetKind::EditText => { + if w.multiline { + [(text_w(&w.text) + 12.0).max(160.0), (LINE_H * lines(&w.text) + 8.0).max(60.0)] + } else { + [(text_w(&w.text) + 12.0).max(40.0), 22.0] + } + } + WidgetKind::Checkbox | WidgetKind::RadioButton => [text_w(&w.text) + 24.0, 18.0], + WidgetKind::Slider | WidgetKind::Scrollbar => [100.0, 22.0], + WidgetKind::Progressbar => [100.0, 10.0], + WidgetKind::DropDownList => [w.items.iter().map(|i| text_w(i)).fold(0.0, f64::max) + 36.0, 22.0], + WidgetKind::ListBox => [(w.items.iter().map(|i| text_w(i)).fold(0.0, f64::max) + 24.0).max(80.0), (w.items.len().clamp(3, 10) as f64 * 18.0 + 4.0)], + _ => [20.0, 20.0], + } + }; + [if ps[0] > 0.0 { ps[0] } else { content[0] }, if ps[1] > 0.0 { ps[1] } else { content[1] }] +} + +fn place(w: &mut Widget, x: f64, y: f64, width: f64, height: f64) { + w.bounds = [x, y, width, height]; + if !w.kind.is_container() { + return; + } + let m = margins(w); + let sp = spacing(w); + let (ah, av) = align_children(w); + let inner = [m[0], m[1], (width - m[0] - m[2]).max(0.0), (height - m[1] - m[3]).max(0.0)]; + let orient = orientation(w).to_string(); + let mut cursor = 0.0; + for c in w.children.iter_mut() { + if !c.visible { + c.bounds = [0.0; 4]; + continue; + } + if let Some(b) = c.fixed_bounds { + let (cx, cy, cw, ch) = (x + b[0], y + b[1], b[2] - b[0], b[3] - b[1]); + place(c, cx, cy, cw, ch); + continue; + } + let pref = preferred(c); + let h_align = c.alignment.first().filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| ah.clone()); + let v_align = c.alignment.get(1).filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| av.clone()); + let along = |a: &str, free: f64| match a { + "center" => free / 2.0, + "right" | "bottom" => free, + _ => 0.0, + }; + let (cx, cy, cw, ch) = match orient.as_str() { + "row" => { + let ch = if v_align == "fill" { inner[3] } else { pref[1] }; + let cy = inner[1] + along(&v_align, inner[3] - ch); + let r = (inner[0] + cursor, cy, pref[0], ch); + cursor += pref[0] + sp; + r + } + "stack" => { + let cw = if h_align == "fill" { inner[2] } else { pref[0] }; + let ch = if v_align == "fill" { inner[3] } else { pref[1] }; + (inner[0] + along(&h_align, inner[2] - cw), inner[1] + along(&v_align, inner[3] - ch), cw, ch) + } + _ => { + let cw = if h_align == "fill" { inner[2] } else { pref[0] }; + let cx = inner[0] + along(&h_align, inner[2] - cw); + let r = (cx, inner[1] + cursor, cw, pref[1]); + cursor += pref[1] + sp; + r + } + }; + place(c, x + cx, y + cy, cw, ch); + } + // Rows centre their content run as a whole when it is narrower than the row. + if orient == "row" && (ah == "center" || ah == "right") { + let used = (cursor - sp).max(0.0); + let shift = along_shift(&ah, inner[2] - used); + if shift > 0.0 { + for c in w.children.iter_mut().filter(|c| c.visible && c.fixed_bounds.is_none()) { + shift_all(c, shift, 0.0); + } + } + } + // Columns aligned to the bottom / centre move the whole run. + if orient == "column" && (av == "center" || av == "bottom") { + let used = (cursor - sp).max(0.0); + let shift = along_shift(&av, inner[3] - used); + if shift > 0.0 { + for c in w.children.iter_mut().filter(|c| c.visible && c.fixed_bounds.is_none()) { + shift_all(c, 0.0, shift); + } + } + } +} + +fn along_shift(a: &str, free: f64) -> f64 { + match a { + "center" => (free / 2.0).max(0.0), + "right" | "bottom" => free.max(0.0), + _ => 0.0, + } +} + +fn shift_all(w: &mut Widget, dx: f64, dy: f64) { + w.bounds[0] += dx; + w.bounds[1] += dy; + for c in &mut w.children { + shift_all(c, dx, dy); + } +} + +/// ScriptUI automatic layout: size the window to its preferred size (or the size the script +/// gave it) and place every control (`bounds` relative to the window's top-left). +pub fn layout(w: &mut ScriptWindow) { + let pref = preferred(&w.root); + let (width, height) = match w.root.fixed_bounds { + Some(b) if b[2] > b[0] && b[3] > b[1] => (b[2] - b[0], b[3] - b[1]), + _ => (pref[0], pref[1]), + }; + place(&mut w.root, 0.0, 0.0, width, height); +} + +// ---------------------------------------------------------------- commands + +fn window_p(s: &Session, p: &Value, cmd: &str) -> Result { + let key = p.get("window"); + let w = match key { + Some(Value::Number(n)) => n.as_u64().map(|n| n as u32).filter(|n| s.script_ui.window(*n).is_some()), + Some(Value::String(t)) => s.script_ui.windows.iter().find(|w| w.title == *t).map(|w| w.id), + None if s.script_ui.windows.len() == 1 => s.script_ui.windows.first().map(|w| w.id), + _ => None, + }; + w.ok_or_else(|| crate::commands::bad(cmd, "no such script window (give `window`: id or title from scriptui.list)")) +} + +fn widget_p(s: &Session, win: u32, p: &Value, cmd: &str) -> Result { + let w = s.script_ui.window(win).ok_or_else(|| crate::commands::bad(cmd, "window closed"))?; + let key = p.get("widget").ok_or_else(|| crate::commands::bad(cmd, "missing `widget` (id, `#id`, properties.name or text)"))?; + w.root.lookup(key).map(|w| w.id).ok_or_else(|| crate::commands::bad(cmd, format!("no control {key} in window {win}"))) +} + +fn dispatch(s: &mut Session, ev: ScriptUiEvent) -> Result { + let f = s.script_ui.dispatch.ok_or_else(|| EngineError::Other("scripting is not available in this build".into()))?; + f(s, &ev).map_err(EngineError::Other) +} + +/// `scriptui.list`: the open script windows. +pub(crate) fn list(s: &mut Session, _: &Value) -> Result { + Ok(json!( + s.script_ui + .windows + .iter() + .map(|w| json!({"window": w.id, "title": w.title, "kind": w.kind, "script": w.script, "modal": w.modal, "size": [w.root.bounds[2], w.root.bounds[3]]})) + .collect::>() + )) +} + +/// `scriptui.get`: one window's control tree. +pub(crate) fn get(s: &mut Session, p: &Value) -> Result { + let id = window_p(s, p, "scriptui.get")?; + Ok(serde_json::to_value(s.script_ui.window(id)).unwrap_or(Value::Null)) +} + +/// `scriptui.click`: press a button / toggle a checkbox / pick a radio button. +pub(crate) fn click(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.click"; + let window = window_p(s, p, c)?; + let widget = widget_p(s, window, p, c)?; + let w = s.script_ui.window(window).and_then(|w| w.root.find(widget)).cloned().unwrap_or_default(); + if !w.enabled { + return Err(crate::commands::bad(c, format!("control {widget} is disabled"))); + } + dispatch(s, ScriptUiEvent { window, widget, kind: "click".into(), value: Value::Null }) +} + +/// `scriptui.set`: set a control's value (edit text, slider, checkbox, list selection) as the +/// user would, firing onChanging / onChange. +pub(crate) fn set(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.set"; + let window = window_p(s, p, c)?; + let widget = widget_p(s, window, p, c)?; + let w = s.script_ui.window(window).and_then(|w| w.root.find(widget)).cloned().unwrap_or_default(); + if !w.enabled { + return Err(crate::commands::bad(c, format!("control {widget} is disabled"))); + } + let mut value = p.get("value").cloned().ok_or_else(|| crate::commands::bad(c, "missing `value`"))?; + // List selections by item text. + if matches!(w.kind, WidgetKind::DropDownList | WidgetKind::ListBox) + && let Value::String(t) = &value + { + let i = w.items.iter().position(|it| it == t).ok_or_else(|| crate::commands::bad(c, format!("no item `{t}` (items: {:?})", w.items)))?; + value = json!(i); + } + dispatch(s, ScriptUiEvent { window, widget, kind: "change".into(), value }) +} + +/// `scriptui.close`: close a window (a dialog's `show()` returns `result`, default 2 = Cancel). +pub(crate) fn close(s: &mut Session, p: &Value) -> Result { + let window = window_p(s, p, "scriptui.close")?; + let result = p.get("result").cloned().unwrap_or(json!(2)); + dispatch(s, ScriptUiEvent { window, widget: 0, kind: "close".into(), value: result }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn w(kind: WidgetKind, text: &str) -> Widget { + Widget { kind, text: text.into(), enabled: true, visible: true, ..Default::default() } + } + + #[test] + fn column_and_row_layout() { + let mut row = w(WidgetKind::Group, ""); + row.id = 1; + let mut ok = w(WidgetKind::Button, "OK"); + ok.id = 2; + let mut cancel = w(WidgetKind::Button, "Cancel"); + cancel.id = 3; + row.children = vec![ok, cancel]; + let mut label = w(WidgetKind::StaticText, "Name:"); + label.id = 4; + let mut root = w(WidgetKind::Window, "Dialog"); + root.children = vec![label, row]; + let mut win = ScriptWindow { id: 1, root, visible: true, ..Default::default() }; + layout(&mut win); + let r = &win.root; + // Window: 15 px margins, children stacked with 10 px spacing. + let row = r.find(1).unwrap(); + assert_eq!(row.bounds[2], 170.0, "two 80 px buttons + 10 spacing"); + assert_eq!(r.bounds[2], 200.0); + assert_eq!(r.find(4).unwrap().bounds[1], 15.0); + assert_eq!(row.bounds[1], 15.0 + 16.0 + 10.0); + // Centred (the default alignChildren of windows). + assert_eq!(row.bounds[0], 15.0); + let label = r.find(4).unwrap(); + assert!((label.bounds[0] + label.bounds[2] / 2.0 - 100.0).abs() < 1e-9); + assert_eq!(r.find(3).unwrap().bounds[0], 15.0 + 80.0 + 10.0); + // alignChildren left / fill. + win.root.align_children = vec!["fill".into(), "top".into()]; + layout(&mut win); + assert_eq!(win.root.find(4).unwrap().bounds[2], 170.0); + win.root.align_children = vec!["left".into()]; + layout(&mut win); + assert_eq!(win.root.find(4).unwrap().bounds[0], 15.0); + // preferredSize wins. + win.root.find_mut(2).unwrap().preferred_size = Some([120.0, 30.0]); + layout(&mut win); + assert_eq!(win.root.find(2).unwrap().bounds[2..], [120.0, 30.0]); + // Lookup by name / text / id. + win.root.find_mut(2).unwrap().name = "ok".into(); + assert_eq!(win.root.lookup(&json!("ok")).unwrap().id, 2); + assert_eq!(win.root.lookup(&json!("Cancel")).unwrap().id, 3); + assert_eq!(win.root.lookup(&json!("#4")).unwrap().id, 4); + assert_eq!(win.root.lookup(&json!(1)).unwrap().id, 1); + } + + #[test] + fn publish_replaces_a_hosts_windows() { + let mut ui = ScriptUi::default(); + let win = |id: u32, visible: bool| ScriptWindow { id, visible, root: w(WidgetKind::Window, ""), ..Default::default() }; + ui.publish(7, vec![win(1, true), win(2, true)]); + ui.publish(8, vec![win(3, true)]); + assert_eq!(ui.windows.iter().map(|w| (w.id, w.host)).collect::>(), [(1, 7), (2, 7), (3, 8)]); + ui.publish(7, vec![win(2, true), win(1, false)]); + assert_eq!(ui.windows.iter().map(|w| w.id).collect::>(), [2, 3]); + let j = serde_json::to_value(&ui).unwrap(); + assert_eq!(j["windows"][0]["root"]["type"], "window"); + } +} diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index d68496a2..96cd5a29 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -36,10 +36,8 @@ impl Session { } // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; - if self.history.undo.len() > levels { - let extra = self.history.undo.len() - levels; - self.history.undo.drain(..extra); - } + let current = self.project.clone(); + self.history.trim(levels, ¤t); self.prefs_revision += 1; } diff --git a/crates/engine/src/tests_face.rs b/crates/engine/src/tests_face.rs new file mode 100644 index 00000000..dad97935 --- /dev/null +++ b/crates/engine/src/tests_face.rs @@ -0,0 +1,106 @@ +//! Face tracking end to end through the commands, on synthetic face footage generated in-test +//! (a moving, rotating face that opens its mouth and blinks). + +use effectcraft_keyframe::{ShapePath, Value as KV}; +use effectcraft_track::face::{FaceParams, LANDMARKS, MEASUREMENTS, render_synthetic, synth_landmarks}; +use serde_json::json; + +use crate::mask_track::MaskMethod; +use crate::tests_track::{H, W, frame_time, setup}; + +fn face(f: u32) -> FaceParams { + let t = f as f64; + FaceParams { + center: [160.0 + 25.0 * (t * 0.2).sin(), 120.0 + 8.0 * (t * 0.15).cos()], + height: 70.0 * (1.0 + 0.05 * (t * 0.1).sin()), + roll: 12.0 * (t * 0.18).sin(), + yaw: 10.0 * (t * 0.12).sin(), + mouth_open: 0.5 + 0.45 * (t * 0.3).sin(), + eye_open: [0.85, 0.85], + ..Default::default() + } +} + +#[test] +fn face_tracking_keys_outline_points_and_measurements() { + let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5)); + assert_eq!(MaskMethod::from_name("faceDetailed"), Some(MaskMethod::FaceDetailed)); + assert_eq!(MaskMethod::from_name("Face Tracking (Outline Only)"), Some(MaskMethod::FaceOutline)); + // A rough mask around the face. + let c = face(0).center; + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let mask = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap(); + assert_eq!(r["frames"], 24, "{r}"); + assert_eq!(s.state.mask_track_method, MaskMethod::FaceDetailed); + let comp = s.active_comp().unwrap(); + let layer = comp.layer(clip).unwrap(); + // The mask follows the face outline. + let path = layer.props.find_group(mask).unwrap().get("path").unwrap(); + assert_eq!(path.keys.len(), 25); + for f in [0, 12, 24] { + let KV::Path(p) = path.value_at(frame_time(f)) else { panic!() }; + let fp = face(f); + let (sn, cs) = fp.roll.to_radians().sin_cos(); + for v in &p.vertices { + let d = [(v[0] - fp.center[0]) / fp.height, (v[1] - fp.center[1]) / fp.height]; + let (u, w) = (cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]); + let r = ((u / 0.75).powi(2) + w * w).sqrt(); + assert!((r - 1.0).abs() < 0.06, "frame {f}: vertex {v:?} at {r}"); + } + } + // Face Track Points: one keyed point per landmark, on the face. + let fx = layer.effects().unwrap(); + let pts = fx.groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points"); + let mut worst: f64 = 0.0; + for f in [0, 6, 12, 18, 24] { + let truth = synth_landmarks(&face(f)); + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + let pr = pts.get(id).unwrap(); + assert_eq!(pr.keys.len(), 25, "{id}"); + let KV::Vec2(v) = pr.value_at(frame_time(f)) else { panic!() }; + worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height); + } + } + assert!(worst < 0.1, "worst landmark error {worst} face heights"); + // Extract & Copy Face Measurements. + assert!(s.is_enabled("track.extractFaceMeasurements")); + let r = s.execute_checked("track.extractFaceMeasurements", json!({"layer": clip.0})).unwrap(); + assert_eq!(r["frames"], 25); + assert!(r["clipboard"].as_str().unwrap().contains("Mouth Openness")); + assert_eq!(s.state.key_clipboard.len(), MEASUREMENTS.len()); + assert!(s.state.clip_is_keys); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + let m = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID).expect("Face Measurements"); + let mo = m.get("mouthOpenness").unwrap(); + assert_eq!(mo.keys.len(), 25); + // Mouth openness follows the footage: widest open where the generator opens it most. + let open = |f: u32| mo.value_at(frame_time(f)).as_f64(); + let (fmax, fmin) = ( + (0..25).max_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + (0..25).min_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + ); + assert!(open(fmax) > open(fmin) + 20.0, "{} vs {}", open(fmax), open(fmin)); + let roll = m.get("headOrientationZ").unwrap(); + for f in [0, 10, 20] { + assert!((roll.value_at(frame_time(f)).as_f64() - face(f).roll).abs() < 3.0); + } + // One undo step for the extraction. + assert!(s.undo()); + assert!(!s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().any(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID)); + // Outline Only keys just the mask. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let m2 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + s.execute_checked("track.mask", json!({"layer": clip.0, "mask": m2, "method": "faceOutline", "direction": "frameForward", "wait": true})).unwrap(); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert_eq!(layer.props.find_group(m2).unwrap().get("path").unwrap().keys.len(), 2); + // A mask without a face reports it. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect([20.0, 20.0], 30.0, 30.0).vertices, "closed": true})).unwrap(); + let m3 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let _ = s.execute("track.mask", json!({"layer": clip.0, "mask": m3, "method": "faceOutline", "direction": "forward", "wait": true})); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert!(layer.props.find_group(m3).unwrap().get("path").unwrap().keys.is_empty()); +} diff --git a/crates/engine/src/tests_history.rs b/crates/engine/src/tests_history.rs new file mode 100644 index 00000000..967c29e3 --- /dev/null +++ b/crates/engine/src/tests_history.rs @@ -0,0 +1,135 @@ +//! Branching history: undo then a new step keeps the undone line as a branch; the tree lists +//! every state; jumping to any state (on any branch) restores it and keeps everything else. + +use serde_json::{Value, json}; + +use crate::Session; + +fn names(s: &Session) -> Vec { + s.project.comps().next().map(|(_, c)| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +fn solid(s: &mut Session, name: &str) { + s.execute("layer.newSolid", json!({"name": name, "color": "#ff0000", "width": 10, "height": 10})).unwrap(); +} + +fn list(s: &mut Session) -> Vec { + s.execute("edit.history.list", json!({})).unwrap()["states"].as_array().cloned().unwrap() +} + +fn index_of(states: &[Value], label: &str, nth: usize) -> u64 { + states.iter().filter(|n| n["label"] == label).nth(nth).unwrap_or_else(|| panic!("no state {label} in {states:?}"))["index"].as_u64().unwrap() +} + +#[test] +fn undo_then_new_step_keeps_a_branch() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + solid(&mut s, "A"); + solid(&mut s, "B"); + solid(&mut s, "C"); + // Linear so far: Original, New Composition, 3 solids. + let st = list(&mut s); + assert_eq!(st.len(), 5, "{st:?}"); + assert!(st.iter().all(|n| n["depth"] == 0)); + assert_eq!(st.last().unwrap()["current"], true); + // Undo twice (B and C undone), then a new step: AE would forget B and C. + assert!(s.undo() && s.undo()); + let st = list(&mut s); + assert_eq!(st.iter().filter(|n| n["future"] == true).count(), 2); + solid(&mut s, "D"); + assert!(s.history.redo.is_empty(), "the new step clears Redo, as in After Effects"); + assert_eq!(s.history.branches.len(), 1); + assert_eq!(names(&s), ["D", "A"]); + let st = list(&mut s); + assert_eq!(st.len(), 6, "the undone states are still listed: {st:?}"); + let branch: Vec<&Value> = st.iter().filter(|n| n["depth"] == 1).collect(); + assert_eq!(branch.len(), 2, "B and C sit on a branch: {st:?}"); + // Jump to C on the old branch. + let c = st.iter().rfind(|n| n["depth"] == 1).unwrap()["index"].as_u64().unwrap(); + let r = s.execute("edit.history.goto", json!({"index": c})).unwrap(); + assert_eq!(r["label"], "New Solid", "{r}"); + assert_eq!(names(&s), ["C", "B", "A"]); + // Now D is the branch, and Undo walks the B/C line. + assert_eq!(s.history.branches.len(), 1); + assert_eq!(s.history.branches[0].steps.len(), 1); + assert!(s.undo()); + assert_eq!(names(&s), ["B", "A"]); + assert!(s.redo()); + // ...and back to D by id. + let st = list(&mut s); + let d = st.iter().find(|n| n["depth"] == 1).unwrap()["id"].as_str().unwrap().to_string(); + s.execute("edit.history.goto", json!({"id": d})).unwrap(); + assert_eq!(names(&s), ["D", "A"]); + let st = list(&mut s); + assert_eq!(st.len(), 6); + assert_eq!(st.iter().filter(|n| n["current"] == true).count(), 1); + // The original state. + let root = index_of(&st, "Original", 0); + s.execute("edit.history.goto", json!({"index": root})).unwrap(); + assert!(s.project.comps().next().is_none()); + assert!(s.history.undo.is_empty()); + assert_eq!(list(&mut s).len(), 6, "nothing is lost by jumping"); + // Relative steps along the line. + s.execute("edit.history.goto", json!({"steps": 2})).unwrap(); + assert_eq!(names(&s), ["A"]); + assert!(s.execute("edit.history.goto", json!({"id": "nope"})).is_err()); + assert!(s.execute("edit.history.goto", json!({})).is_err()); +} + +#[test] +fn branches_of_branches_and_purge() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + solid(&mut s, "A"); + solid(&mut s, "B"); + s.undo(); + solid(&mut s, "C"); // branch 1: B + s.undo(); + solid(&mut s, "D"); // branch 2: C + let st = list(&mut s); + assert_eq!(st.len(), 6, "{st:?}"); + // Go to B, undo to A, make E: D's line becomes a branch, and a branch of the new line. + let b = st.iter().find(|n| n["depth"] == 1 && n["label"] == "New Solid").unwrap()["index"].as_u64().unwrap(); + s.execute("edit.history.goto", json!({"index": b})).unwrap(); + let which = names(&s); + assert!(which == ["B", "A"] || which == ["C", "A"], "{which:?}"); + s.undo(); + solid(&mut s, "E"); + let st = list(&mut s); + assert_eq!(st.len(), 7, "{st:?}"); + // Every state is reachable and restores its own layers. + let mut seen = std::collections::BTreeSet::new(); + for i in 0..st.len() { + s.execute("edit.history.goto", json!({"index": i})).unwrap(); + seen.insert(names(&s).join(",")); + assert_eq!(list(&mut s).len(), 7); + } + for want in ["", "A", "B,A", "C,A", "D,A", "E,A"] { + assert!(seen.contains(want), "{want:?} not reachable: {seen:?}"); + } + s.execute("edit.purgeUndo", json!({})).unwrap(); + assert_eq!(list(&mut s).len(), 1); +} + +#[test] +fn undo_levels_trim_branches() { + let mut s = Session::default(); + s.prefs.general.undo_levels = 3; + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + for n in ["A", "B", "C", "D", "E"] { + solid(&mut s, n); + } + assert_eq!(s.history.undo.len(), 3); + s.undo(); + s.undo(); + solid(&mut s, "F"); + // The branch (D, E) grew from a state still kept. + assert_eq!(s.history.branch_states(), 2); + for n in ["G", "H", "I"] { + solid(&mut s, n); + } + // Its parent fell off the end of the undo levels: the branch goes too. + assert_eq!(s.history.branch_states(), 0); + assert_eq!(list(&mut s).len(), 4); +} diff --git a/crates/engine/src/tests_paint.rs b/crates/engine/src/tests_paint.rs index 9926e04c..9407ddd1 100644 --- a/crates/engine/src/tests_paint.rs +++ b/crates/engine/src/tests_paint.rs @@ -250,3 +250,56 @@ fn liquify_stroke_adds_the_effect_and_warps() { assert!(layer(&s, id).effects().unwrap().groups().next().is_none()); assert!(px(&s, 0.0, 55, 50)[3] > 0.99); } + +fn pin_keys(s: &Session, id: u64, pin: u64) -> Vec<(f64, [f64; 2])> { + let l = layer(s, id); + let pos = l.props.find_group(pin).unwrap().get("position").unwrap().clone(); + pos.keys.iter().map(|k| (k.time.seconds(), k.value.as_vec2())).collect() +} + +#[test] +fn puppet_pin_recording_makes_keys_at_the_frame_rate() { + let (mut s, id) = setup(); + let a = s.execute("puppet.addPin", json!({"layer": id, "position": [10, 30]})).unwrap(); + s.execute("puppet.addPin", json!({"layer": id, "position": [90, 30]})).unwrap(); + let pin = a["pin"].as_u64().unwrap(); + s.set_time(Tick::from_seconds_f64(0.5)); + // A 0.5 s drag sampled at irregular wall-clock times: right 30 px, linear in time. + let samples: Vec = [0.0, 0.07, 0.2, 0.33, 0.41, 0.5].iter().map(|t| json!([t, 10.0 + 60.0 * t, 30.0])).collect(); + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples, "smoothing": 0})).unwrap(); + // 0.5 s at 30 fps from 0.5 s: frames 15..=30, one key per frame. + assert_eq!(r["frames"], 16, "{r}"); + let keys = pin_keys(&s, id, pin); + let rec: Vec<&(f64, [f64; 2])> = keys.iter().filter(|k| k.0 >= 0.5 - 1e-9).collect(); + assert_eq!(rec.len(), 16, "{keys:?}"); + for (i, (t, v)) in rec.iter().enumerate() { + assert!((t - (0.5 + i as f64 / 30.0)).abs() < 1e-6, "{t}"); + assert!((v[0] - (10.0 + 60.0 * i as f64 / 30.0)).abs() < 1e-6, "{v:?}"); + } + assert_eq!(keys.len(), 17, "the pin's first key (0 s) is kept"); + assert_eq!(s.time().seconds(), 0.5, "the current time returns to where recording began"); + // The recording deforms the layer: by 1 s the left edge has followed the pin 30 px right. + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); + assert!(px(&s, 1.0, 55, 50)[3] < 0.5); + // Speed 200 %: the same drag plays back over twice the frames; recording again replaces + // the keys in the span. + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples, "speed": 200, "smoothing": 0})).unwrap(); + assert_eq!(r["frames"], 31); + let keys = pin_keys(&s, id, pin); + assert_eq!(keys.len(), 32); + let k = keys.iter().find(|k| (k.0 - (0.5 + 10.0 / 30.0)).abs() < 1e-6).unwrap(); + assert!((k.1[0] - (10.0 + 60.0 * 5.0 / 30.0)).abs() < 1e-6, "{k:?}"); + // Smoothing removes the extraneous keys of a straight-line drag. + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples})).unwrap(); + assert!(r["keys"].as_u64().unwrap() < 6, "{r}"); + // One undo step per recording. + s.undo(); + assert_eq!(pin_keys(&s, id, pin).len(), 32); + // Record Options are tool state (serde for agents). + let o = s.execute("puppet.recordOptions", json!({"speed": 50, "smoothing": 0, "useDraftDeformation": true, "showMesh": true})).unwrap(); + assert_eq!(o["speed"], 50.0); + assert!(s.state.puppet.record_draft && s.state.puppet.record_show_mesh); + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples})).unwrap(); + assert_eq!(r["frames"], 9, "50 %: half the frames"); + assert!(s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": []})).is_err()); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 58a1b620..decd8a72 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -312,6 +312,123 @@ fn pre_render_imports_and_replaces() { assert!(matches!(s.project.comp(outer).unwrap().layers[0].source, LayerSource::Comp { .. })); } +#[test] +fn post_render_actions_import_and_set_proxy() { + let (mut s, _) = rq(); + let cid = s.active_comp_id().unwrap(); + let items_before = s.project.items.len(); + // Import: the rendered file becomes a footage item. + let a = s.execute("renderQueue.add", json!({"output": "/tmp/post/a.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": a["item"], "postRenderAction": "import"})).unwrap(); + assert_eq!(s.project.render_queue[0].post_render, PostRenderAction::Import); + s.execute("renderQueue.render", json!({})).unwrap(); + assert_eq!(s.project.items.len(), items_before + 1); + assert!(s.project.items.values().any(|i| matches!(&i.kind, effectcraft_project::ItemKind::Footage(f) if f.path == "/tmp/post/a.mov"))); + // Set Proxy: the comp gets the render as its proxy. + let b = s.execute("renderQueue.add", json!({"output": "/tmp/post/b.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": b["item"], "postRenderAction": "Set Proxy"})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + let px = s.project.item(cid).unwrap().proxy.clone().expect("proxy set"); + assert_eq!(px.footage.path, "/tmp/post/b.mov"); + // None: nothing happens. + let n = s.project.items.len(); + let c = s.execute("renderQueue.add", json!({"output": "/tmp/post/c.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": c["item"], "postRenderAction": "none"})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + assert_eq!(s.project.items.len(), n); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": c["item"], "postRenderAction": "explode"})).is_err()); +} + +#[test] +fn templates_save_apply_default_and_persist() { + let (mut s, _) = rq(); + let t = s.execute("renderQueue.templates", json!({})).unwrap(); + let names: Vec<&str> = t["renderSettings"].as_array().unwrap().iter().map(|r| r["name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Best Settings", "Draft Settings", "DV Settings", "Multi-Machine Settings", "Current Settings"]); + assert!(t["outputModules"].as_array().unwrap().iter().any(|m| m["name"] == "High Quality with Alpha")); + assert_eq!(t["defaults"]["renderSettings"]["Movie"], "Best Settings"); + // A new item starts from the Movie defaults. + let a = s.execute("renderQueue.add", json!({"output": "/tmp/t/a.mp4"})).unwrap(); + assert_eq!(a["renderSettingsSummary"], "Best Settings"); + assert_eq!(a["outputModuleSummary"], "H.264 - Match Render Settings - 15 Mbps"); + // Edit it: Custom; save templates from it; make them the defaults. + let r = s + .execute( + "renderQueue.setRenderSettings", + json!({"item": a["item"], "fieldRender": "upper", "pulldown": "WSSWW", "colorDepth": 16, "effects": "allOff", "solo": "allOff", "guideLayers": "current", "frameBlending": "offForAll", "motionBlur": "onForChecked", "duration": 0.5, "storageOverflow": false}), + ) + .unwrap(); + assert_eq!(r["renderSettingsSummary"], "Custom"); + assert_eq!(r["settings"]["field_render"], "UpperFirst"); + s.execute( + "renderQueue.setOutputModule", + json!({"item": a["item"], "format": "png", "channels": "rgba", "color": "premultiplied", "crop": {"top": 2, "bottom": 2}, "resize": {"width": 320, "height": 200, "lockAspect": false, "quality": "low"}}), + ) + .unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "renderSettings", "name": "Interlaced 16", "item": a["item"]})).unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "outputModule", "name": "Small PNG", "item": a["item"], "params": {"quality": 50}})).unwrap(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "renderSettings", "slot": "movie", "name": "Interlaced 16"})).unwrap(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "movie", "name": "Small PNG"})).unwrap(); + assert!(s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "movie", "name": "Nope"})).is_err()); + let b = s.execute("renderQueue.add", json!({"output": "/tmp/t/b_[####].png"})).unwrap(); + assert_eq!(b["renderSettingsSummary"], "Interlaced 16"); + assert_eq!(b["outputModuleSummary"], "Small PNG"); + let it = s.project.render_queue.last().unwrap().clone(); + assert_eq!(it.output.resize.width, 320); + assert_eq!(it.output.output, "/tmp/t/b_[####].png"); + assert_eq!((b["width"].as_u64(), b["height"].as_u64()), (Some(320), Some(200))); + // Apply built-ins to an item; the output path keeps its name with the new extension. + let r = s.execute("renderQueue.applyTemplate", json!({"item": a["item"], "renderSettings": "DV Settings", "outputModule": "High Quality"})).unwrap(); + assert_eq!(r["renderSettingsSummary"], "DV Settings"); + assert_eq!(r["outputModuleSummary"], "High Quality"); + assert!(r["outputPath"].as_str().unwrap().ends_with("a.mov"), "{r}"); + // Persisted with the project. + let p = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + assert_eq!(p.render_templates, s.project.render_templates); + assert_eq!(p.render_queue, s.project.render_queue); + // Delete: saved ones go; built-ins stay. Undo restores. + s.execute("renderQueue.deleteTemplate", json!({"kind": "outputModule", "name": "Small PNG"})).unwrap(); + assert!(s.project.render_templates.output_module("Small PNG").is_none()); + assert!(s.execute("renderQueue.deleteTemplate", json!({"kind": "renderSettings", "name": "Best Settings"})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.render_templates.output_module("Small PNG").is_some()); +} + +#[test] +fn save_frame_as_queues_with_the_frame_defaults() { + let (mut s, _) = rq(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "still", "name": "TIFF Sequence with Alpha"})).unwrap(); + let r = s.execute("comp.saveFrameAs", json!({"queue": true, "time": 1.0})).unwrap(); + assert_eq!(r["queued"], true); + let it = s.project.render_queue.last().unwrap(); + assert_eq!(it.output.format, OutputFormat::TiffSequence); + assert_eq!(it.settings.name, "Current Settings"); + let comp = s.project.comp(it.comp).unwrap(); + assert_eq!(it.settings.frame_count(comp), 1); + assert_eq!(it.settings.frame_time(comp, 0), Tick::from_seconds_f64(1.0)); +} + +#[test] +fn log_notify_and_overflow_settings() { + let (mut s, _) = rq(); + let a = s.execute("renderQueue.add", json!({"output": "/tmp/n/a.mp4", "log": "plusPerFrameInfo"})).unwrap(); + assert_eq!(a["logLabel"], "Plus Per Frame Info"); + assert_eq!(a["logPath"], "/tmp/n/a_RenderLog.txt"); + s.execute("renderQueue.setLog", json!({"item": a["item"], "log": "plusSettings"})).unwrap(); + assert_eq!(s.project.render_queue[0].log, effectcraft_project::render_queue::RenderLog::PlusSettings); + s.execute("renderQueue.setOverflowFolders", json!({"folders": ["/tmp/o1", " ", "/tmp/o2"]})).unwrap(); + assert_eq!(s.project.render_prefs.overflow_folders, ["/tmp/o1", "/tmp/o2"]); + assert_eq!(s.execute("renderQueue.setNotify", json!({})).unwrap()["notify"], true); + s.drain_events(); + s.execute("renderQueue.render", json!({})).unwrap(); + let ev = s.drain_events(); + assert!(ev.iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == "renderQueue.notify")), "{ev:?}"); + s.execute("renderQueue.setNotify", json!({"notify": false})).unwrap(); + s.execute("renderQueue.setRender", json!({"index": 1, "render": true})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + assert!(!s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == "renderQueue.notify"))); +} + #[test] fn save_current_preview_writes_a_movie() { let (mut s, ex) = rq(); diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs index b4863d54..a4386199 100644 --- a/crates/engine/src/tracking.rs +++ b/crates/engine/src/tracking.rs @@ -358,9 +358,8 @@ impl Session { return self.offload_analysis(crate::offload::WorkerJob::Track { comp, layer, tracker, direction, settings, points, times }); } let wait = wait || cfg!(target_arch = "wasm32"); - self.history.undo.push(("Analyze Track".into(), self.project.clone())); - self.history.redo.clear(); - self.history.merge_key = None; + let levels = self.prefs.general.undo_levels as usize; + self.history.record("Analyze Track", self.project.clone(), levels); let shared = Arc::new(TrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 15f9b17d..6900c204 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -32,3 +32,4 @@ effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } filmcraft-opus = { workspace = true } filmcraft-vp9 = { workspace = true } +effectcraft-keyframe = { workspace = true } diff --git a/crates/export/src/encode.rs b/crates/export/src/encode.rs index d6cf3150..b0566d9b 100644 --- a/crates/export/src/encode.rs +++ b/crates/export/src/encode.rs @@ -4,12 +4,12 @@ use std::io::Write; use effectcraft_project::Comp; -use effectcraft_project::render_queue::{Channels, OutputFormat, ProResProfile}; +use effectcraft_project::render_queue::{AudioFormat, Channels, OutputFormat, ProResProfile}; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; use filmcraft_isobmff::{Brand, FourCc, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; use rayon::prelude::*; -use crate::{ExportError, Job, Report, Result, State, batch_size, render_frame, rgba8, wants_audio}; +use crate::{Cx, ExportError, Report, Result, State, batch_size, wants_audio}; fn enc(e: impl std::fmt::Display) -> ExportError { ExportError::Encode(e.to_string()) @@ -208,29 +208,60 @@ enum Audio { Pcm, } -const AUDIO_CHANNELS: u32 = 2; +fn deinterleave(buf: &[f32], channels: usize) -> Vec> { + (0..channels).map(|c| buf.chunks_exact(channels).map(|p| p[c]).collect()).collect() +} -fn deinterleave(buf: &[f32]) -> [Vec; 2] { - let l = buf.chunks_exact(2).map(|p| p[0]).collect(); - let r = buf.chunks_exact(2).map(|p| p[1]).collect(); - [l, r] +/// The comp's audio over `n` samples from `start`, interleaved with the module's channel count +/// (mono = the average of left and right). +pub(crate) fn mix(cx: &Cx, start: effectcraft_time::Tick, n: usize, sr: u32) -> Vec { + let st = effectcraft_render::audio::mix_comp(&cx.project, cx.footage, cx.expr, cx.comp, start, n, sr); + if cx.output.audio_channels == 1 { st.chunks_exact(2).map(|p| (p[0] + p[1]) * 0.5).collect() } else { st } +} + +/// PCM bytes of samples in `fmt` (little or big endian). +pub(crate) fn pcm_bytes(buf: &[f32], fmt: AudioFormat, big_endian: bool) -> Vec { + let mut out = Vec::with_capacity(buf.len() * 4); + for &s in buf { + let s = s.clamp(-1.0, 1.0); + match fmt { + AudioFormat::S16 => { + let v = (s * 32767.0).round() as i16; + out.extend_from_slice(&if big_endian { v.to_be_bytes() } else { v.to_le_bytes() }); + } + AudioFormat::S24 => { + let v = (s * 8_388_607.0).round() as i32; + let b = v.to_le_bytes(); + if big_endian { + out.extend_from_slice(&[b[2], b[1], b[0]]); + } else { + out.extend_from_slice(&b[..3]); + } + } + AudioFormat::F32 => out.extend_from_slice(&if big_endian { s.to_be_bytes() } else { s.to_le_bytes() }), + } + } + out } // ---------------------------------------------------------------- the movie type Writer<'f, 's> = Mp4Writer<&'f mut crate::out::Out<'s>>; -pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) fn movie(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { let rate = job.settings.rate(comp); let fmt = job.output.format; - let channels = if fmt == OutputFormat::ProRes { job.output.channels } else { Channels::Rgb }; + let channels = match job.output.channels { + Channels::Rgba if fmt != OutputFormat::ProRes => Channels::Rgb, + c => c, + }; let mut venc: Box = match fmt { OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps)?), - _ => Box::new(ProRes::new(w, h, job.output.prores_profile, channels)), + _ => Box::new(ProRes::new(w, h, job.output.prores_profile, if channels == Channels::Alpha { Channels::Rgb } else { channels })), }; let brand = if fmt == OutputFormat::H264 { Brand::Mp4 } else { Brand::Mov }; let batch = batch_size(); - let render = |k: u64| -> Vec { rgba8(&render_frame(job, comp, k), comp, channels, w, h) }; + let render = |k: u64| -> Vec { job.pixels(&job.frame(comp, k), comp, channels, w, h) }; // Encode the first batch before creating tracks: encoders finalise their config on frame 1. let first_end = batch.min(st.total); @@ -242,7 +273,8 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R drop(first); st.advance(first_end)?; - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = crate::out::create(job.sink, &path)?; let opts = WriterOptions::new(brand); let movie_ts = opts.movie_timescale.max(1) as i128; let mut mux: Writer = Mp4Writer::new(&mut file, opts).map_err(mux_err)?; @@ -257,15 +289,22 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R let vt = mux.add_track(vcfg).map_err(mux_err)?; let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); let with_audio = wants_audio(job); + let chans = if job.output.audio_channels == 1 { 1u32 } else { 2 }; + let afmt = job.output.audio_format; let mut audio = None; if with_audio { let (a, entry, start) = if brand == Brand::Mp4 { - let e = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(sr, AUDIO_CHANNELS as usize, 320_000)).map_err(enc)?; - let entry = SampleEntry::aac(e.audio_specific_config(), AUDIO_CHANNELS, sr); + let e = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(sr, chans as usize, 160_000 * chans)).map_err(enc)?; + let entry = SampleEntry::aac(e.audio_specific_config(), chans, sr); let priming = e.priming_samples() as i64; (Audio::Aac(Box::new(e)), entry, Some(priming)) } else { - let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: AUDIO_CHANNELS, sample_rate: sr as f64 }; + let (bits, float) = match afmt { + AudioFormat::S16 => (16, false), + AudioFormat::S24 => (24, false), + AudioFormat::F32 => (32, true), + }; + let pcm = PcmConfig { bits, float, big_endian: false, signed: true, channels: chans, sample_rate: sr as f64 }; (Audio::Pcm, SampleEntry::pcm(pcm), None) }; let mut c = TrackConfig::new(entry, sr); @@ -290,21 +329,18 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R } let n = (end - *cursor) as usize; let start = Tick(((*cursor as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); - let buf = effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, sr); + let buf = mix(job, start, n, sr); *cursor = end; match a { Audio::Aac(e) => { - let planar = deinterleave(&buf); + let planar = deinterleave(&buf, chans as usize); let refs: Vec<&[f32]> = planar.iter().map(Vec::as_slice).collect(); for au in e.encode(&refs) { mux.write_sample(*at, WriteSample { data: &au, duration: 1024, composition_offset: 0, is_sync: true }).map_err(mux_err)?; } } Audio::Pcm => { - let mut pcm = Vec::with_capacity(n * 4); - for s in buf { - pcm.extend_from_slice(&((s.clamp(-1.0, 1.0) * 32767.0).round() as i16).to_le_bytes()); - } + let pcm = pcm_bytes(&buf, afmt, false); mux.write_sample(*at, WriteSample { data: &pcm, duration: n as u32, composition_offset: 0, is_sync: true }).map_err(mux_err)?; } } @@ -336,7 +372,7 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R let out = mux.finish().map_err(mux_err)?; out.flush().map_err(mux_err)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 1f0c4407..f1e3269f 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -2,13 +2,21 @@ //! //! | Format | Container / files | Video | Alpha | Audio | //! |---|---|---|---|---| -//! | H.264 | MP4 | FilmCraft H.264 High, 8-bit 4:2:0 BT.709 limited, VBR | no | AAC-LC (FilmCraft), stereo | -//! | ProRes | QuickTime MOV | FilmCraft ProRes 422 Proxy/LT/422/HQ, 10-bit; 4444/4444 XQ 4:4:4 | 4444 (16-bit alpha coding) | 16-bit PCM, stereo | +//! | H.264 | MP4 | FilmCraft H.264 High, 8-bit 4:2:0 BT.709 limited, VBR | no | AAC-LC (FilmCraft), mono/stereo | +//! | ProRes | QuickTime MOV | FilmCraft ProRes 422 Proxy/LT/422/HQ, 10-bit; 4444/4444 XQ 4:4:4 | 4444 (16-bit alpha coding) | PCM 16/24-bit or 32-bit float, mono/stereo | //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | -//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | -//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | +//! | WebM | WebM (Matroska) | VP9 profile 0 (`effectcraft-vp9enc`: key + inter frames, a key frame every 2 s, loop filter), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | PCM 16/24-bit (WAV also 32-bit float), mono/stereo (audio only) | +//! +//! Every frame goes through [`pipeline`]: the Render Settings overrides (Effects, Solo Switches, +//! Guide Layers, Frame Blending, Motion Blur, Color Depth), field rendering (two fields half a +//! frame apart woven line by line, with optional 3:2 pulldown from film frames at 4/5 of the +//! rate), then the Output Module's Crop (edges or the region of interest) and Resize. Channels +//! may be RGB, RGB + Alpha (straight or premultiplied) or the alpha matte alone. With Use Storage +//! Overflow and a [`StorageQuota`] hook, files that don't fit go to the overflow folders. A +//! render log (Errors Only / Plus Settings / Plus Per Frame Info) is written next to the output. //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -20,19 +28,21 @@ mod encode; mod out; +mod pipeline; mod webm; use web_time::Instant; -use effectcraft_project::render_queue::{AudioOutput, Channels, OutputFormat, OutputModule, RenderQuality, RenderSettings, sequence_path}; +use effectcraft_project::render_queue::{AudioOutput, Channels, OutputFormat, OutputModule, RenderLog, RenderQuality, RenderSettings, sequence_path}; use effectcraft_project::{Comp, ItemId, Project}; use effectcraft_raster::Image; -use effectcraft_render::{ExprHost, FootageSource, RenderOpts, Renderer}; -use effectcraft_time::Tick; +use effectcraft_render::{ExprHost, FootageSource}; use rayon::prelude::*; pub use effectcraft_project::render_queue; +pub use effectcraft_project::render_queue::StorageQuota; pub use out::Sink; +pub(crate) use pipeline::Cx; #[derive(Debug, thiserror::Error)] pub enum ExportError { @@ -71,6 +81,21 @@ pub struct Job<'a> { pub sink: Option<&'a Sink>, /// Settings ▸ General ▸ Switches Affect Nested Comps (see `RenderOpts::nested_switches`). pub nested_switches: bool, + /// Render log, storage overflow. + pub options: JobOptions<'a>, +} + +/// Job options beyond the Render Settings and Output Module. +#[derive(Default)] +pub struct JobOptions<'a> { + /// The render log written next to the output. + pub log: RenderLog, + /// What the log calls the job (e.g. "#2 Main Comp"). + pub label: String, + /// Free-space hook for Use Storage Overflow (`None`: never full). + pub storage: Option<&'a dyn StorageQuota>, + /// Overflow folders, in order. + pub overflow: Vec, } /// Progress after a batch of frames. @@ -102,6 +127,10 @@ pub struct Report { pub seconds: f64, pub bytes: u64, pub audio: bool, + /// The render log, when one was written. + pub log: Option, + /// Files that went to an overflow folder. + pub overflow: Vec, } /// Formats this build can write (all of them; kept as the hook for optional encoders). @@ -109,10 +138,9 @@ pub fn available_formats() -> Vec { OutputFormat::ALL.to_vec() } -/// The size of the written frames for a job (H.264 rounds to even). +/// The size of the written frames for a job (crop, resize; H.264 rounds to even). pub fn output_size(comp: &Comp, settings: &RenderSettings, output: &OutputModule) -> (u32, u32) { - let (w, h) = settings.output_size(comp); - output.format.coded_size(w, h) + output.output_size(comp, settings) } /// Whether the job writes an audio track. @@ -127,19 +155,30 @@ pub fn wants_audio(job: &Job) -> bool { /// Run the export (blocking). `progress` is called after every batch; returning `false` cancels. pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result { - let comp = job.project.comp(job.comp).ok_or(ExportError::NoComp)?; let t0 = Instant::now(); - let total = job.settings.frame_count(comp); + let cx = Cx::new(job); + let mut r = export_with(&cx, progress, t0); + let log = cx.write_log(&r, t0.elapsed().as_secs_f64()); + if let Ok(rep) = &mut r { + rep.log = log; + rep.overflow = cx.overflowed.lock().unwrap_or_else(|e| e.into_inner()).clone(); + } + r +} + +fn export_with(cx: &Cx, progress: &mut dyn FnMut(&Progress) -> bool, t0: Instant) -> Result { + let comp = cx.comp().ok_or(ExportError::NoComp)?; + let total = cx.settings.frame_count(comp); let mut st = State { t0, total, done: 0, progress }; st.advance(0)?; - let (w, h) = output_size(comp, job.settings, job.output); - let mut report = match job.output.format { - OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, - OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, - OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, - OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, - OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, - f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, + let (w, h) = output_size(comp, cx.settings, cx.output); + let mut report = match cx.output.format { + OutputFormat::H264 | OutputFormat::ProRes => encode::movie(cx, comp, w, h, &mut st)?, + OutputFormat::Gif => gif_export(cx, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(cx, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(cx, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(cx, comp, true, &mut st)?, + f if f.is_sequence() => sequence(cx, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; report.seconds = t0.elapsed().as_secs_f64(); @@ -172,39 +211,8 @@ pub(crate) fn batch_size() -> u64 { rayon::current_num_threads().clamp(2, 16) as u64 } -/// Render output frame `i` of the job at the settings' resolution. -pub(crate) fn render_frame(job: &Job, comp: &Comp, i: u64) -> Image { - let t: Tick = job.settings.frame_time(comp, i); - let opts = RenderOpts { - scale: job.settings.resolution.clamp(0.01, 4.0), - motion_blur: job.settings.motion_blur, - guides: false, - draft: job.settings.quality == RenderQuality::Draft, - // Output renders always look through the comp's active camera. - view: None, - backend: effectcraft_render::Backend::Auto, - roi: None, - nested_switches: job.nested_switches, - draft_shadows: true, - proxy: job.settings.proxy_use, - }; - let mut r = Renderer::new(job.project, job.footage, opts); - r.expr = job.expr; - r.accel = job.accel; - r.comp_frame(job.comp, t) -} - -/// 8-bit RGBA of the requested channels (RGB: over the comp background, opaque; RGBA: straight), -/// cropped/padded to `w`×`h`. -pub(crate) fn rgba8(img: &Image, comp: &Comp, channels: Channels, w: u32, h: u32) -> Vec { - let px = match channels { - Channels::Rgb => img.to_rgba8_over(comp.background), - Channels::Rgba => img.to_rgba8(), - }; - fit(px, img.width, img.height, w, h) -} - -fn fit(px: Vec, iw: u32, ih: u32, w: u32, h: u32) -> Vec { +/// Crop/pad 8-bit RGBA to `w`×`h` (codec rounding). +pub(crate) fn fit(px: Vec, iw: u32, ih: u32, w: u32, h: u32) -> Vec { if iw == w && ih == h { return px; } @@ -221,10 +229,13 @@ fn first_frame_number(job: &Job, comp: &Comp) -> i64 { job.settings.first_frame(comp) } -fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { - let f0 = first_frame_number(job, comp); - let fmt = job.output.format; - let channels = if fmt.supports_alpha() { job.output.channels } else { Channels::Rgb }; +fn sequence(cx: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let f0 = first_frame_number(cx, comp); + let fmt = cx.output.format; + let channels = match cx.output.channels { + Channels::Rgba if !fmt.supports_alpha() => Channels::Rgb, + c => c, + }; let batch = batch_size(); let mut bytes = 0u64; let mut i = 0; @@ -233,13 +244,18 @@ fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result> = (i..end) .into_par_iter() .map(|k| { - let path = sequence_path(job.path, f0 + k as i64); - if job.settings.skip_existing && out::exists(job.sink, &path) { + let path = sequence_path(cx.path, f0 + k as i64); + if cx.settings.skip_existing && out::exists(cx.sink, &path) { return Ok(0); } - let img = render_frame(job, comp, k); - let mut f = out::create(job.sink, &path)?; - write_still(&mut f, fmt, &img, comp, channels, w, h, job.output.quality)?; + let img = cx.frame(comp, k); + // Encode in memory first: the storage hook decides where it fits. + let mut buf = std::io::Cursor::new(Vec::new()); + write_still(cx, &mut buf, fmt, &img, comp, channels, w, h, cx.output.quality)?; + let data = buf.into_inner(); + let path = cx.place(&path, data.len() as u64); + let mut f = out::create(cx.sink, &path)?; + std::io::Write::write_all(&mut f, &data).map_err(io)?; f.finish() }) .collect(); @@ -249,11 +265,21 @@ fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result Result<()> { +fn write_still( + cx: &Cx, + file: &mut W, + fmt: OutputFormat, + img: &Image, + comp: &Comp, + channels: Channels, + w: u32, + h: u32, + quality: u8, +) -> Result<()> { use image::{ExtendedColorType, ImageEncoder, ImageFormat}; match fmt { OutputFormat::ExrSequence => { @@ -264,6 +290,10 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, for x in 0..w { let p = if x < img.width && y < img.height { img.data[img.idx(x, y)] } else { [0.0; 4] }; let (c, a) = match channels { + Channels::Alpha => { + let a = p[3].clamp(0.0, 1.0); + ([lin(a); 3], 1.0) + } Channels::Rgba => { let a = p[3].clamp(0.0, 1.0); let s = |c: f32| if a > 0.0 { lin((c / a).clamp(0.0, 1.0)) * a } else { 0.0 }; @@ -282,16 +312,17 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, image::DynamicImage::ImageRgba32F(buf).write_to(file, ImageFormat::OpenExr).map_err(|e| ExportError::Encode(e.to_string())) } OutputFormat::JpegSequence => { - let px = rgba8(img, comp, Channels::Rgb, w, h); + let px = cx.pixels(img, comp, if channels == Channels::Alpha { Channels::Alpha } else { Channels::Rgb }, w, h); let rgb: Vec = px.chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect(); image::codecs::jpeg::JpegEncoder::new_with_quality(file, quality.clamp(1, 100)) .write_image(&rgb, w, h, ExtendedColorType::Rgb8) .map_err(|e| ExportError::Encode(e.to_string())) } OutputFormat::PngSequence | OutputFormat::TiffSequence => { - let px = rgba8(img, comp, channels, w, h); + let px = cx.pixels(img, comp, channels, w, h); let (data, ct) = match channels { Channels::Rgba => (px, ExtendedColorType::Rgba8), + Channels::Alpha => (px.chunks_exact(4).map(|p| p[0]).collect(), ExtendedColorType::L8), Channels::Rgb => (px.chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect(), ExtendedColorType::Rgb8), }; if fmt == OutputFormat::PngSequence { @@ -305,11 +336,12 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, } } -fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +fn gif_export(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { if w > u16::MAX as u32 || h > u16::MAX as u32 { return Err(ExportError::Unsupported("GIF frames are limited to 65535 px".into())); } - let mut file = out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = out::create(job.sink, &path)?; let mut enc = gif::Encoder::new(&mut file, w as u16, h as u16, &[]).map_err(|e| ExportError::Encode(e.to_string()))?; enc.set_repeat(if job.output.gif_loop { gif::Repeat::Infinite } else { gif::Repeat::Finite(0) }).map_err(|e| ExportError::Encode(e.to_string()))?; let rate = job.settings.rate(comp); @@ -324,8 +356,8 @@ fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result< let frames: Vec> = (i..end) .into_par_iter() .map(|k| { - let img = render_frame(job, comp, k); - let mut px = rgba8(&img, comp, job.output.channels, w, h); + let img = job.frame(comp, k); + let mut px = job.pixels(&img, comp, job.output.channels, w, h); gif::Frame::from_rgba_speed(w as u16, h as u16, &mut px, 10) }) .collect(); @@ -344,7 +376,7 @@ fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result< } drop(enc); let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: false }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: false, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs new file mode 100644 index 00000000..d1ad01bf --- /dev/null +++ b/crates/export/src/pipeline.rs @@ -0,0 +1,329 @@ +//! The frame pipeline shared by every output format: Render Settings overrides (effects, solo, +//! guide layers, frame blending, motion blur), field rendering with 3:2 pulldown, colour depth, +//! the Output Module's crop and resize, alpha handling, storage overflow and the render log. + +use std::sync::Mutex; + +use effectcraft_project::render_queue::{ + AlphaMode, Channels, ColorDepth, CurrentOrOff, EffectsMode, FrameSample, RenderLog, ResizeQuality, SwitchOverride, log_path, +}; +use effectcraft_project::{Comp, Project}; +use effectcraft_raster::Image; +use effectcraft_render::{RenderOpts, Renderer}; +use effectcraft_time::Tick; +use web_time::Instant; + +use crate::{Job, RenderQuality, out}; + +/// A job being exported: the job, the project with the Render Settings overrides applied, and +/// per-frame timing for the render log. +pub(crate) struct Cx<'a> { + pub job: &'a Job<'a>, + pub project: Project, + /// (output frame, seconds) of every rendered frame (Plus Per Frame Info logs). + pub frame_times: Option>>, + /// Files written to an overflow folder. + pub overflowed: Mutex>, +} + +impl<'a> std::ops::Deref for Cx<'a> { + type Target = Job<'a>; + fn deref(&self) -> &Job<'a> { + self.job + } +} + +/// The project as the Render Settings render it: Effects All On / All Off, Solo Switches All +/// Off, Frame Blending and Motion Blur "On for Checked Layers" (the composition switches on). +pub(crate) fn apply_overrides(p: &mut Project, job: &Job) { + let s = job.settings; + let ids: Vec<_> = p.comps().map(|(i, _)| *i).collect(); + for id in ids { + let Some(c) = p.comp_mut(id) else { continue }; + match s.frame_blending { + SwitchOverride::Current => {} + SwitchOverride::OnForChecked => c.enable_frame_blending = true, + SwitchOverride::OffForAll => c.enable_frame_blending = false, + } + match s.motion_blur_override() { + SwitchOverride::Current => {} + SwitchOverride::OnForChecked => c.enable_motion_blur = true, + SwitchOverride::OffForAll => c.enable_motion_blur = false, + } + for l in c.layers.iter_mut() { + match s.effects { + EffectsMode::Current => {} + EffectsMode::AllOn => l.switches.effects = true, + EffectsMode::AllOff => l.switches.effects = false, + } + if s.solo == CurrentOrOff::AllOff { + l.switches.solo = false; + } + } + } +} + +impl<'a> Cx<'a> { + pub fn new(job: &'a Job<'a>) -> Cx<'a> { + let mut project = job.project.clone(); + apply_overrides(&mut project, job); + let frame_times = (job.options.log == RenderLog::PlusPerFrameInfo).then(|| Mutex::new(Vec::new())); + Cx { job, project, frame_times, overflowed: Mutex::new(vec![]) } + } + + pub fn comp(&self) -> Option<&Comp> { + self.project.comp(self.job.comp) + } + + fn render_at(&self, t: Tick) -> Image { + let s = self.job.settings; + let opts = RenderOpts { + scale: s.resolution.clamp(0.01, 4.0), + motion_blur: s.motion_blur_override() != SwitchOverride::OffForAll, + guides: s.guide_layers == CurrentOrOff::Current, + draft: s.quality == RenderQuality::Draft, + // Output renders always look through the comp's active camera. + view: None, + backend: effectcraft_render::Backend::Auto, + roi: None, + nested_switches: self.job.nested_switches, + draft_shadows: true, + proxy: s.proxy_use, + }; + let mut r = Renderer::new(&self.project, self.job.footage, opts); + r.expr = self.job.expr; + r.accel = self.job.accel; + r.comp_frame(self.job.comp, t) + } + + /// Output frame `i`: sampled (fields, pulldown), at the colour depth, cropped and resized. + pub fn frame(&self, comp: &Comp, i: u64) -> Image { + let t0 = Instant::now(); + let img = match self.job.settings.sample(comp, i) { + FrameSample::Progressive(t) => self.render_at(t), + FrameSample::Fields { first, second, upper_first } => { + let a = self.render_at(first); + if second == first { a } else { interleave(&a, &self.render_at(second), upper_first) } + } + }; + let mut img = img; + // Color Depth: renders run at the project's depth; another depth quantises the frame. + let depth = self.job.settings.color_depth; + if depth != ColorDepth::Current + && let Some(levels) = depth.resolve(self.project.settings.bit_depth).levels() + { + img.quantize(levels); + } + let img = self.crop_resize(img); + if let Some(ft) = &self.frame_times { + ft.lock().unwrap_or_else(|e| e.into_inner()).push((i, t0.elapsed().as_secs_f64())); + } + img + } + + /// Output Module ▸ Crop, then Resize. + fn crop_resize(&self, img: Image) -> Image { + let om = self.job.output; + let scale = self.job.settings.resolution.clamp(0.01, 4.0); + let (t, l, b, r) = om.crop.edges(img.width, img.height, scale); + let img = if (t, l, b, r) == (0, 0, 0, 0) { img } else { crop(&img, t, l, b, r) }; + if !om.resize.enabled { + return img; + } + let (w, h) = om.frame_size(img.width, img.height, 1.0); + if (w, h) == (img.width, img.height) { + return img; + } + match om.resize.quality { + ResizeQuality::Low => effectcraft_raster::warp::resample(&img, w, h), + ResizeQuality::High => bicubic(&img, w, h), + } + } + + /// 8-bit RGBA of the requested channels cropped/padded to `w`×`h` (codec rounding): RGB over + /// the comp background (opaque), RGB + Alpha straight or premultiplied, or the alpha matte. + pub fn pixels(&self, img: &Image, comp: &Comp, channels: Channels, w: u32, h: u32) -> Vec { + let px = match channels { + Channels::Rgb => img.to_rgba8_over(comp.background), + Channels::Rgba => match self.job.output.alpha_mode { + AlphaMode::Straight => img.to_rgba8(), + AlphaMode::Premultiplied => premultiplied8(img), + }, + Channels::Alpha => img + .data + .iter() + .flat_map(|p| { + let a = (p[3].clamp(0.0, 1.0) * 255.0 + 0.5) as u8; + [a, a, a, 255] + }) + .collect(), + }; + crate::fit(px, img.width, img.height, w, h) + } + + /// Where a file goes: `path`, or (Use Storage Overflow) the first overflow folder with room + /// for `bytes` when the quota hook says the output volume is full. + pub fn place(&self, path: &str, bytes: u64) -> String { + let Some(q) = self.job.options.storage else { return path.to_string() }; + if !self.job.settings.storage_overflow || q.has_room(path, bytes) { + return path.to_string(); + } + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); + for dir in &self.job.options.overflow { + let p = std::path::Path::new(dir).join(&name).to_string_lossy().to_string(); + if q.has_room(&p, bytes) { + self.overflowed.lock().unwrap_or_else(|e| e.into_inner()).push(p.clone()); + return p; + } + } + path.to_string() + } + + /// Write the render log next to the output, as the Log setting asks. Returns its path. + pub fn write_log(&self, result: &Result, seconds: f64) -> Option { + let level = self.job.options.log; + // Errors Only: a log for failures (a user stop is not an error). + if level == RenderLog::ErrorsOnly && matches!(result, Ok(_) | Err(crate::ExportError::Cancelled)) { + return None; + } + let comp = self.comp(); + let path = log_path(self.job.path); + let mut s = String::new(); + s.push_str("EffectCraft Render Log\n"); + s.push_str(&format!("Item: {}\n", if self.job.options.label.is_empty() { "(render)" } else { &self.job.options.label })); + s.push_str(&format!("Output: {}\n", self.job.path)); + match result { + Ok(r) => s.push_str(&format!("Result: Done ({} frames, {}×{}, {} bytes) in {seconds:.2} s\n", r.frames, r.width, r.height, r.bytes)), + Err(e) => s.push_str(&format!("Result: Failed\nError: {e}\n")), + } + let over = self.overflowed.lock().unwrap_or_else(|e| e.into_inner()); + if !over.is_empty() { + s.push_str(&format!("Storage overflow: {} file(s) written to overflow folders (first: {})\n", over.len(), over[0])); + } + if level != RenderLog::ErrorsOnly { + s.push_str("\nRender Settings:\n"); + for (k, v) in self.job.settings.describe(comp) { + s.push_str(&format!(" {k}: {v}\n")); + } + s.push_str("\nOutput Module:\n"); + for (k, v) in self.job.output.describe() { + s.push_str(&format!(" {k}: {v}\n")); + } + } + if let (RenderLog::PlusPerFrameInfo, Some(ft), Some(c)) = (level, &self.frame_times, comp) { + let mut v = ft.lock().unwrap_or_else(|e| e.into_inner()).clone(); + v.sort_by_key(|(i, _)| *i); + s.push_str("\nFrames:\n"); + let f0 = self.job.settings.first_frame(c); + for (i, secs) in v { + let t = self.job.settings.frame_time(c, i); + s.push_str(&format!(" Frame {} ({:.3} s): rendered in {:.1} ms\n", f0 + i as i64, t.seconds(), secs * 1000.0)); + } + } + let mut f = out::create(self.job.sink, &path).ok()?; + std::io::Write::write_all(&mut f, s.as_bytes()).ok()?; + f.finish().ok()?; + Some(path) + } +} + +/// Weave two fields: the dominant field's lines (even lines when `upper_first`) from `first`. +pub(crate) fn interleave(first: &Image, second: &Image, upper_first: bool) -> Image { + let mut out = first.clone(); + let w = out.width as usize; + for y in 0..out.height as usize { + let from_first = (y % 2 == 0) == upper_first; + if !from_first && y < second.height as usize && second.width as usize == w { + out.data[y * w..(y + 1) * w].copy_from_slice(&second.data[y * w..(y + 1) * w]); + } + } + out +} + +/// Remove `t`/`l`/`b`/`r` pixels from the edges (negative values add transparent pixels). +pub(crate) fn crop(img: &Image, t: i32, l: i32, b: i32, r: i32) -> Image { + let w = (img.width as i32 - l - r).max(1) as u32; + let h = (img.height as i32 - t - b).max(1) as u32; + let mut out = Image::new(w, h); + for y in 0..h as i32 { + let sy = y + t; + if sy < 0 || sy >= img.height as i32 { + continue; + } + for x in 0..w as i32 { + let sx = x + l; + if sx >= 0 && sx < img.width as i32 { + let o = out.idx(x as u32, y as u32); + out.data[o] = img.data[img.idx(sx as u32, sy as u32)]; + } + } + } + out +} + +/// Bicubic resize (Resize Quality: High). +fn bicubic(src: &Image, w: u32, h: u32) -> Image { + // Shrinking a lot: bilinear with its box pre-filter first, then bicubic for the last step. + if src.width > w * 2 && src.height > h * 2 { + let mid = effectcraft_raster::warp::resample(src, w * 2, h * 2); + return bicubic(&mid, w, h); + } + let mut out = Image::new(w, h); + let (sx, sy) = (src.width as f64 / w as f64, src.height as f64 / h as f64); + for y in 0..h { + for x in 0..w { + let o = out.idx(x, y); + // Clamp to the edge so borders stay opaque. + let fx = ((x as f64 + 0.5) * sx).clamp(0.5, src.width as f64 - 0.5); + let fy = ((y as f64 + 0.5) * sy).clamp(0.5, src.height as f64 - 0.5); + let mut p = src.sample_bicubic(fx, fy); + p[3] = p[3].min(1.0); + out.data[o] = p; + } + } + out +} + +/// Premultiplied 8-bit RGBA (colour matted with black). +fn premultiplied8(img: &Image) -> Vec { + img.data + .iter() + .flat_map(|p| { + let a = p[3].clamp(0.0, 1.0); + [0, 1, 2].map(|c| (p[c].clamp(0.0, a) * 255.0 + 0.5) as u8).into_iter().chain([(a * 255.0 + 0.5) as u8]) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interleave_takes_alternate_lines() { + let a = Image::filled(3, 4, [1.0, 0.0, 0.0, 1.0]); + let b = Image::filled(3, 4, [0.0, 0.0, 1.0, 1.0]); + let up = interleave(&a, &b, true); + let lo = interleave(&a, &b, false); + for y in 0..4u32 { + assert_eq!(up.data[up.idx(1, y)][0], if y % 2 == 0 { 1.0 } else { 0.0 }, "upper first: line {y}"); + assert_eq!(lo.data[lo.idx(1, y)][0], if y % 2 == 1 { 1.0 } else { 0.0 }, "lower first: line {y}"); + } + } + + #[test] + fn crop_and_pad() { + let mut img = Image::new(4, 3); + for (i, p) in img.data.iter_mut().enumerate() { + *p = [i as f32, 0.0, 0.0, 1.0]; + } + let c = crop(&img, 1, 1, 0, 1); + assert_eq!((c.width, c.height), (2, 2)); + assert_eq!(c.data[0][0], 5.0); + let p = crop(&img, -1, 0, 0, 0); + assert_eq!((p.width, p.height), (4, 4)); + assert_eq!(p.data[0], [0.0; 4]); + assert_eq!(p.data[4][0], 0.0); + assert_eq!(p.data[5][0], 1.0); + } +} diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index eb99cd22..af38eaeb 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -5,19 +5,22 @@ //! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are //! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose //! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, -//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every -//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (key +//! frames every two seconds, inter frames between them: SimpleBlocks carry the key flag, inter +//! BlockGroups a ReferenceBlock; the alpha stream's key frames line up with the colour stream's), +//! Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, //! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit //! extended sample rate) follow their published file layouts. use std::io::{Seek, SeekFrom, Write}; use effectcraft_project::Comp; -use effectcraft_project::render_queue::Channels; +use effectcraft_project::render_queue::{AudioFormat, Channels}; use effectcraft_time::{TICKS_PER_SECOND, Tick}; use rayon::prelude::*; -use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; +use crate::encode::{mix, pcm_bytes}; +use crate::{Cx, Report, Result, State, batch_size, io, wants_audio}; const OPUS_RATE: u32 = 48_000; const OPUS_BITRATE: u32 = 192_000; @@ -82,6 +85,10 @@ fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { struct Frame { ms: i64, track: u64, + /// A key frame (VP9) or any audio packet. + key: bool, + /// Timestamp of the previous video frame (ReferenceBlock of inter frames). + prev_ms: Option, data: Vec, alpha: Option>, } @@ -94,10 +101,15 @@ fn cluster(frames: &mut [Frame]) -> Vec { for f in frames.iter() { let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; match &f.alpha { - None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + None => el(&mut body, 0xA3, &block(f.track, rel, if f.key { 0x80 } else { 0x00 }, &f.data)), Some(a) => { let mut g = vec![]; el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + if let (false, Some(p)) = (f.key, f.prev_ms) { + // ReferenceBlock: a signed timestamp relative to this block (an inter frame). + let d = (p - f.ms).clamp(i16::MIN as i64, -1) as i16; + el(&mut g, 0xFB, &d.to_be_bytes()); + } let mut more = vec![]; el_uint(&mut more, 0xEE, 1); el(&mut more, 0xA5, a); @@ -113,18 +125,33 @@ fn cluster(frames: &mut [Frame]) -> Vec { out } -pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) fn webm(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { let rate = job.settings.rate(comp); let alpha = job.output.channels == Channels::Rgba; + let colour_channels = if job.output.channels == Channels::Alpha { Channels::Alpha } else { Channels::Rgb }; let quality = job.output.quality.clamp(1, 100); - let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let fps = rate.as_f64().max(1.0); + let cfg = effectcraft_vp9enc::EncoderConfig { + width: w, + height: h, + quality, + full_range: false, + keyframe_interval: if job.output.keyframe_interval > 0 { job.output.keyframe_interval } else { (fps * 2.0).round().max(1.0) as u32 }, + target_kbps: job.output.webm_bitrate.then_some(job.output.bitrate_kbps.max(50)), + frame_rate: fps, + loop_filter: true, + }; + let mut color_enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let mut alpha_enc = alpha.then(|| effectcraft_vp9enc::Vp9Encoder::new(cfg.clone())); let with_audio = wants_audio(job); - let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let opus_channels = if job.output.audio_channels == 1 { 1usize } else { 2 }; + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(opus_channels as _, OPUS_BITRATE)); let total = st.total; let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; let duration_ms = frame_ms(total) as f64; - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = crate::out::create(job.sink, &path)?; // EBML header. let mut head = vec![]; let mut eb = vec![]; @@ -178,7 +205,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re el_uint(&mut a, 0x56BB, 80_000_000); let mut au = vec![]; el_float(&mut au, 0xB5, OPUS_RATE as f64); - el_uint(&mut au, 0x9F, 2); + el_uint(&mut au, 0x9F, opus_channels as u64); el(&mut a, 0xE1, &au); el(&mut tracks, TRACK_ENTRY, &a); } @@ -197,10 +224,10 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re if end > cursor { let n = (end - cursor) as usize; let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); - pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + pcm_left.extend(mix(job, start, n, OPUS_RATE)); cursor = end; } - let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * opus_channels; if finish && !pcm_left.is_empty() { // Pad the last packet (pre-skip + padding cover the encoder delay). let pad = (fs - pcm_left.len() % fs) % fs + fs; @@ -209,7 +236,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re while pcm_left.len() >= fs { let pkt = enc.encode_float(&pcm_left[..fs]); pcm_left.drain(..fs); - frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + frames.push(Frame { ms: packets as i64 * 20, track: 2, key: true, prev_ms: None, data: pkt, alpha: None }); packets += 1; } }; @@ -218,23 +245,28 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re let mut i = 0; while i < total { let end = (i + batch).min(total); - let encoded: Vec<(Vec, Option>)> = (i..end) - .into_par_iter() - .map(|k| { - let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); - let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); - let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); - let color = enc.encode_yuv420(&y, &u, &vv); - let a = alpha.then(|| { - let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); - let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); - ea.encode_yuv420(&y, &u, &vv) - }); - (color, a) - }) - .collect(); - let mut frames: Vec = - encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + // Frames render in parallel; the VP9 streams (inter-coded) encode in order, colour and + // alpha side by side. + let pixels: Vec> = + (i..end).into_par_iter().map(|k| job.pixels(&job.frame(comp, k), comp, if alpha { Channels::Rgba } else { colour_channels }, w, h)).collect(); + let mut frames: Vec = Vec::with_capacity(pixels.len()); + for (j, px) in pixels.iter().enumerate() { + let (color, a) = rayon::join( + || { + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(px, w, h, false); + color_enc.encode_frame(&y, &u, &vv, false) + }, + || { + alpha_enc.as_mut().map(|ea| { + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(px, w, h); + ea.encode_frame(&y, &u, &vv, false).data + }) + }, + ); + let k = i + j as u64; + frames.push(Frame { ms: frame_ms(k), track: 1, key: color.key, prev_ms: (k > 0).then(|| frame_ms(k - 1)), data: color.data, alpha: a }); + } + drop(pixels); let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; audio_until(t_end, &mut frames, end >= total); let c = cluster(&mut frames); @@ -260,7 +292,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re file.write_all(&size).map_err(io)?; file.seek(SeekFrom::End(0)).map_err(io)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio, log: None, overflow: vec![] }) } // ---------------------------------------------------------------- audio-only outputs @@ -278,14 +310,25 @@ fn ieee_extended(v: f64) -> [u8; 10] { out } -/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. -pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { +/// WAV or AIFF: the comp's audio over the render span (Audio Output: sample rate, mono/stereo, +/// 16/24-bit integer or 32-bit float; AIFF has no float in plain AIFF, so it writes 24-bit). +pub(crate) fn audio_file(job: &Cx, comp: &Comp, aiff: bool, st: &mut State) -> Result { let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let channels = if job.output.audio_channels == 1 { 1u16 } else { 2 }; + let fmt = match job.output.audio_format { + AudioFormat::F32 if aiff => AudioFormat::S24, + f => f, + }; + let bytes_per_sample: u16 = match fmt { + AudioFormat::S16 => 2, + AudioFormat::S24 => 3, + AudioFormat::F32 => 4, + }; let (span_start, span_end) = job.settings.span(comp); let start = span_start.to_units_floor(sr as i64); let end = span_end.to_units_floor(sr as i64); let n = (end - start).max(0) as usize; - let mut pcm = Vec::with_capacity(n * 4); + let mut pcm = Vec::with_capacity(n * (channels * bytes_per_sample) as usize); // Mix in one-second blocks, reporting progress in frames. let block = sr as usize; let mut done = 0usize; @@ -294,10 +337,7 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> while done < n { let k = block.min(n - done); let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); - for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { - let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; - pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); - } + pcm.extend(pcm_bytes(&mix(job, t, k, sr), fmt, aiff)); done += k; let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); if now > reported && st.total > 0 { @@ -308,13 +348,14 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> if reported < st.total { st.advance(st.total - reported)?; } + let block_align = channels * bytes_per_sample; + let frames = (pcm.len() / block_align as usize) as u32; let mut out = vec![]; - let frames = (pcm.len() / 4) as u32; if aiff { let mut comm = vec![]; - comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&channels.to_be_bytes()); comm.extend_from_slice(&frames.to_be_bytes()); - comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&(bytes_per_sample * 8).to_be_bytes()); comm.extend_from_slice(&ieee_extended(sr as f64)); let ssnd_len = 8 + pcm.len(); out.extend_from_slice(b"FORM"); @@ -331,24 +372,30 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> out.push(0); } } else { + // WAVE_FORMAT_PCM (1) or WAVE_FORMAT_IEEE_FLOAT (3). + let tag: u16 = if fmt == AudioFormat::F32 { 3 } else { 1 }; out.extend_from_slice(b"RIFF"); - out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(&((36 + pcm.len() + pcm.len() % 2) as u32).to_le_bytes()); out.extend_from_slice(b"WAVEfmt "); out.extend_from_slice(&16u32.to_le_bytes()); - out.extend_from_slice(&1u16.to_le_bytes()); - out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&tag.to_le_bytes()); + out.extend_from_slice(&channels.to_le_bytes()); out.extend_from_slice(&sr.to_le_bytes()); - out.extend_from_slice(&(sr * 4).to_le_bytes()); - out.extend_from_slice(&4u16.to_le_bytes()); - out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(&(sr * block_align as u32).to_le_bytes()); + out.extend_from_slice(&block_align.to_le_bytes()); + out.extend_from_slice(&(bytes_per_sample * 8).to_le_bytes()); out.extend_from_slice(b"data"); out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); out.extend_from_slice(&pcm); + if pcm.len() % 2 == 1 { + out.push(0); + } } - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, out.len() as u64); + let mut file = crate::out::create(job.sink, &path)?; file.write_all(&out).map_err(io)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) + Ok(Report { path, frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/tests/render_settings.rs b/crates/export/tests/render_settings.rs new file mode 100644 index 00000000..c59411b3 --- /dev/null +++ b/crates/export/tests/render_settings.rs @@ -0,0 +1,322 @@ +//! Render Settings and Output Module features end to end (through a [`Sink`], so nothing touches +//! the disk): field rendering and 3:2 pulldown, crop / region of interest / resize, alpha-only +//! and premultiplied output, the solo and guide-layer overrides, the render log, storage +//! overflow and PCM audio formats. + +use std::sync::{Arc, Mutex}; + +use effectcraft_color::Label; +use effectcraft_export::render_queue::*; +use effectcraft_export::{Job, JobOptions, Report, Sink, StorageQuota, export}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, Project, Solid, build}; +use effectcraft_render::NoFootage; +use effectcraft_time::{FrameRate, Tick}; + +const W: u32 = 64; +const H: u32 = 48; + +/// A comp at `rate`, black, with a white bar 8 px wide sweeping left to right (64 px/s) and a +/// static red block at the bottom left; `extra` adds layers. +fn project(rate: FrameRate, extra: impl FnOnce(&mut Project, &mut Comp)) -> (Project, ItemId) { + let mut p = Project::default(); + let mut comp = Comp::new(W, H, rate, Tick::from_seconds_f64(2.0)); + comp.background = [0.0, 0.0, 0.0]; + let bar = p.add_item("Bar", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: 8, height: 32, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Bar", LayerSource::Solid { item: bar }, (8, 32), None); + l.props.prop_mut("transform/position").unwrap().keys = + vec![Keyframe::new(Tick::ZERO, Value::Vec3([0.0, 16.0, 0.0])), Keyframe::new(Tick::from_seconds_f64(1.0), Value::Vec3([64.0, 16.0, 0.0]))]; + comp.layers.push(l); + let red = p.add_item("Block", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 0.0, 0.0], width: 16, height: 8, pixel_aspect: 1.0 })); + let mut b = build::layer(&mut p, &comp, "Block", LayerSource::Solid { item: red }, (16, 8), None); + b.props.prop_mut("transform/position").unwrap().value = Value::Vec3([8.0, 44.0, 0.0]); + comp.layers.push(b); + extra(&mut p, &mut comp); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + +type Files = Arc)>>>; + +fn custom(start: f64, end: f64) -> RenderSettings { + RenderSettings { time_span: TimeSpan::Custom { start: Tick::from_seconds_f64(start), end: Tick::from_seconds_f64(end) }, ..Default::default() } +} + +fn run_with(p: &Project, cid: ItemId, s: &RenderSettings, om: &OutputModule, path: &str, options: JobOptions, cancel: bool) -> (Result, Files) { + let files: Files = Default::default(); + let f = files.clone(); + let sink: Box = Box::new(move |path: &str, data: Vec| f.lock().unwrap().push((path.to_string(), data))); + let job = Job { + project: p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: s, + output: om, + path, + sink: Some(&*sink), + options, + nested_switches: true, + }; + let r = export(&job, &mut |pr| !(cancel && pr.done > 0)).map_err(|e| e.to_string()); + (r, files) +} + +fn run(p: &Project, cid: ItemId, s: &RenderSettings, om: &OutputModule, path: &str) -> Files { + let (r, files) = run_with(p, cid, s, om, path, JobOptions::default(), false); + r.expect("export"); + files +} + +fn png(files: &Files, i: usize) -> image::RgbaImage { + let f = files.lock().unwrap(); + let pngs: Vec<&(String, Vec)> = f.iter().filter(|(p, _)| p.ends_with(".png")).collect(); + let mut pngs = pngs; + pngs.sort_by(|a, b| a.0.cmp(&b.0)); + image::load_from_memory(&pngs[i].1).expect("png").to_rgba8() +} + +fn row(img: &image::RgbaImage, y: u32) -> Vec { + (0..img.width()).flat_map(|x| img.get_pixel(x, y).0).collect() +} + +#[test] +fn field_render_interleaves_two_times() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + // Frame 10 at 25 fps with fields: 0.40 s and 0.42 s. + for (fr, upper) in [(FieldRender::UpperFirst, true), (FieldRender::LowerFirst, false)] { + let s = RenderSettings { field_render: fr, ..custom(0.4, 0.44) }; + let fields = png(&run(&p, cid, &s, &om, "/f/field_[#####].png"), 0); + // Progressive references at 50 fps: frames 20 (0.40 s) and 21 (0.42 s). + let r = RenderSettings { frame_rate: Some(FrameRate::new(50, 1)), ..custom(0.4, 0.44) }; + let refs = run(&p, cid, &r, &om, "/f/ref_[#####].png"); + let (a, b) = (png(&refs, 0), png(&refs, 1)); + assert_ne!(row(&a, 10), row(&b, 10), "the bar moved between the fields"); + for y in 0..H { + let first = (y % 2 == 0) == upper; + assert_eq!(row(&fields, y), row(if first { &a } else { &b }, y), "{fr:?}: line {y}"); + } + } +} + +#[test] +fn pulldown_follows_the_cadence() { + let ntsc = FrameRate::new(30000, 1001); + let (p, cid) = project(ntsc, |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let five = 5.0 * 1001.0 / 30000.0; + let s = RenderSettings { field_render: FieldRender::UpperFirst, pulldown: Pulldown::Wssww, ..custom(0.0, five) }; + let out = run(&p, cid, &s, &om, "/p/pd_[#####].png"); + assert_eq!(out.lock().unwrap().len(), 5); + // Film frames at 23.976 fps. + let r = RenderSettings { frame_rate: Some(FrameRate::new(24000, 1001)), ..custom(0.0, 0.2) }; + let film = run(&p, cid, &r, &om, "/p/film_[#####].png"); + let mut pattern = String::new(); + for k in 0..5 { + let (a, b) = Pulldown::Wssww.fields(k as u64).unwrap(); + pattern.push(if a == b { 'W' } else { 'S' }); + let img = png(&out, k); + let (fa, fb) = (png(&film, a as usize), png(&film, b as usize)); + for y in 0..H { + assert_eq!(row(&img, y), row(if y % 2 == 0 { &fa } else { &fb }, y), "frame {k} line {y}"); + } + } + assert_eq!(pattern, "WSSWW"); +} + +#[test] +fn crop_roi_and_resize() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let s = custom(0.4, 0.44); + let full = png(&run(&p, cid, &s, &OutputModule::for_format(OutputFormat::PngSequence), "/c/full_[#####].png"), 0); + let mut om = OutputModule::for_format(OutputFormat::PngSequence); + om.crop = Crop { enabled: true, top: 4, left: 10, bottom: 6, right: 2, ..Default::default() }; + let c = png(&run(&p, cid, &s, &om, "/c/crop_[#####].png"), 0); + assert_eq!((c.width(), c.height()), (52, 38)); + for (x, y) in [(0, 0), (20, 11), (51, 37), (3, 30)] { + assert_eq!(c.get_pixel(x, y), full.get_pixel(x + 10, y + 4), "({x}, {y})"); + } + // Region of interest at half resolution. + om.crop = Crop { enabled: true, use_roi: true, roi: Some([16.0, 8.0, 32.0, 32.0]), ..Default::default() }; + let half = RenderSettings { resolution: 0.5, ..s.clone() }; + let r = png(&run(&p, cid, &half, &om, "/c/roi_[#####].png"), 0); + assert_eq!((r.width(), r.height()), (16, 16)); + // Resize (fixed size, low and high quality) after the crop. + om.crop = Crop::default(); + for q in [ResizeQuality::Low, ResizeQuality::High] { + om.resize = Resize { enabled: true, width: 32, height: 20, lock_aspect: false, quality: q }; + let z = png(&run(&p, cid, &s, &om, "/c/resize_[#####].png"), 0); + assert_eq!((z.width(), z.height()), (32, 20)); + // The red block (bottom left) survives the resize. + let px = z.get_pixel(4, 18).0; + assert!(px[0] > 150 && px[1] < 80, "{q:?}: {px:?}"); + } + om.resize.lock_aspect = true; + let z = png(&run(&p, cid, &s, &om, "/c/aspect_[#####].png"), 0); + assert_eq!((z.width(), z.height()), (32, 24)); +} + +#[test] +fn alpha_only_and_premultiplied() { + let (p, cid) = project(FrameRate::new(25, 1), |p, comp| { + // A half-transparent green layer over the right half. + let g = p.add_item("Green", Label::Green, None, ItemKind::Solid(Solid { color: [0.0, 1.0, 0.0], width: 32, height: 48, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, "Green", LayerSource::Solid { item: g }, (32, 48), None); + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3([48.0, 24.0, 0.0]); + l.props.prop_mut("transform/opacity").unwrap().value = Value::Scalar(50.0); + comp.layers.insert(0, l); + }); + let s = custom(0.0, 0.04); + let mut om = OutputModule::for_format(OutputFormat::PngSequence); + om.channels = Channels::Alpha; + let a = png(&run(&p, cid, &s, &om, "/a/alpha_[#####].png"), 0); + assert_eq!(a.get_pixel(60, 2).0[0], 128, "half-opaque green"); + assert_eq!(a.get_pixel(20, 2).0[0], 0, "transparent"); + assert_eq!(a.get_pixel(8, 44).0[0], 255, "opaque block"); + om.channels = Channels::Rgba; + let straight = png(&run(&p, cid, &s, &om, "/a/s_[#####].png"), 0); + om.alpha_mode = AlphaMode::Premultiplied; + let premul = png(&run(&p, cid, &s, &om, "/a/p_[#####].png"), 0); + assert_eq!(straight.get_pixel(60, 2).0, [0, 255, 0, 128]); + assert_eq!(premul.get_pixel(60, 2).0, [0, 128, 0, 128]); +} + +#[test] +fn solo_and_guide_layer_overrides() { + let (p, cid) = project(FrameRate::new(25, 1), |p, comp| { + let g = p.add_item("Guide", Label::Green, None, ItemKind::Solid(Solid { color: [0.0, 0.0, 1.0], width: 8, height: 8, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, "Guide", LayerSource::Solid { item: g }, (8, 8), None); + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3([56.0, 44.0, 0.0]); + l.switches.guide = true; + comp.layers.insert(0, l); + // Solo the moving bar. + comp.layers[1].switches.solo = true; + }); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = custom(0.4, 0.44); + let cur = png(&run(&p, cid, &s, &om, "/s/cur_[#####].png"), 0); + assert_eq!(cur.get_pixel(8, 44).0[0], 0, "solo: the red block is hidden"); + assert_eq!(cur.get_pixel(56, 44).0[2], 0, "guide layers off by default"); + let off = RenderSettings { solo: CurrentOrOff::AllOff, guide_layers: CurrentOrOff::Current, ..s.clone() }; + let all = png(&run(&p, cid, &off, &om, "/s/off_[#####].png"), 0); + assert_eq!(all.get_pixel(8, 44).0[0], 255, "Solo Switches: All Off"); + // (with solo off the guide renders too: Guide Layers: Current Settings) + assert_eq!(all.get_pixel(56, 44).0[2], 255); +} + +#[test] +fn render_logs() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = RenderSettings { field_render: FieldRender::UpperFirst, name: "DV-ish".into(), ..custom(0.0, 0.2) }; + let log_of = |files: &Files| { + files.lock().unwrap().iter().find(|(p, _)| p.ends_with("_RenderLog.txt")).map(|(p, d)| (p.clone(), String::from_utf8(d.clone()).unwrap())) + }; + // Errors Only + success: no log. + let (r, files) = run_with(&p, cid, &s, &om, "/l/a_[#####].png", JobOptions::default(), false); + assert!(r.unwrap().log.is_none()); + assert!(log_of(&files).is_none()); + // Plus Per Frame Info: settings and one line per frame. + let opts = JobOptions { log: RenderLog::PlusPerFrameInfo, label: "#1 Comp".into(), ..Default::default() }; + let (r, files) = run_with(&p, cid, &s, &om, "/l/b_[#####].png", opts, false); + let r = r.unwrap(); + let (path, text) = log_of(&files).expect("log"); + assert_eq!(r.log.as_deref(), Some(path.as_str())); + assert_eq!(path, "/l/b_RenderLog.txt"); + assert!(text.contains("Item: #1 Comp") && text.contains("Result: Done (5 frames"), "{text}"); + assert!(text.contains("Field Render: Upper Field First") && text.contains("Template: DV-ish") && text.contains("Format: PNG Sequence"), "{text}"); + assert_eq!(text.lines().filter(|l| l.contains("rendered in")).count(), 5, "{text}"); + assert!(text.contains("Frame 0 (0.000 s): rendered in") && text.contains("Frame 4 (0.160 s)"), "{text}"); + // Plus Settings: no per-frame lines. + let opts = JobOptions { log: RenderLog::PlusSettings, ..Default::default() }; + let (_, files) = run_with(&p, cid, &s, &om, "/l/c.gif", opts, false); + let (_, text) = log_of(&files).expect("log"); + assert!(text.contains("Render Settings:") && !text.contains("rendered in"), "{text}"); + // Errors Only + failure (the composition is gone): the error is logged. + let gif = OutputModule::for_format(OutputFormat::Gif); + let (r, files) = run_with(&p, ItemId(424_242), &s, &gif, "/l/d.gif", JobOptions::default(), false); + assert!(r.is_err()); + let (path, text) = log_of(&files).expect("error log"); + assert_eq!(path, "/l/d_RenderLog.txt"); + assert!(text.contains("Result: Failed") && text.contains("Error: composition not found") && !text.contains("Render Settings:"), "{text}"); + // A user stop is not an error: no log. + let (r, files) = run_with(&p, cid, &s, &gif, "/l/e.gif", JobOptions::default(), true); + assert!(r.is_err()); + assert!(log_of(&files).is_none()); +} + +/// The primary folder holds `limit` bytes; overflow folders are unlimited. +struct Quota { + limit: u64, + used: Mutex, +} + +impl StorageQuota for Quota { + fn has_room(&self, path: &str, bytes: u64) -> bool { + if !path.starts_with("/primary/") { + return true; + } + let mut u = self.used.lock().unwrap(); + if *u + bytes > self.limit { + return false; + } + *u += bytes; + true + } +} + +#[test] +fn storage_overflow_moves_files_to_the_overflow_folder() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = RenderSettings { skip_existing: false, ..custom(0.0, 0.4) }; + let q = Quota { limit: 1, used: Mutex::new(0) }; + let opts = JobOptions { storage: Some(&q), overflow: vec!["/overflow".into()], log: RenderLog::PlusSettings, ..Default::default() }; + let (r, files) = run_with(&p, cid, &s, &om, "/primary/seq_[#####].png", opts, false); + let r = r.unwrap(); + assert_eq!(r.overflow.len(), 10, "{:?}", r.overflow); + let f = files.lock().unwrap(); + assert_eq!(f.iter().filter(|(p, _)| p.starts_with("/overflow/seq_") && p.ends_with(".png")).count(), 10); + let log = f.iter().find(|(p, _)| p.ends_with("_RenderLog.txt")).map(|(_, d)| String::from_utf8_lossy(d).to_string()).unwrap(); + assert!(log.contains("Storage overflow: 10 file(s)"), "{log}"); + drop(f); + // Use Storage Overflow off: everything stays in place. + let off = RenderSettings { storage_overflow: false, ..s }; + let q = Quota { limit: 1, used: Mutex::new(0) }; + let opts = JobOptions { storage: Some(&q), overflow: vec!["/overflow".into()], ..Default::default() }; + let (r, files) = run_with(&p, cid, &off, &om, "/primary/seq_[#####].png", opts, false); + assert!(r.unwrap().overflow.is_empty()); + assert!(files.lock().unwrap().iter().all(|(p, _)| p.starts_with("/primary/"))); +} + +#[test] +fn wav_and_aiff_channels_and_formats() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let s = custom(0.0, 0.5); + let mut om = OutputModule::for_format(OutputFormat::Wav); + om.audio_channels = 1; + om.audio_format = AudioFormat::S24; + om.audio_sample_rate = 44_100; + let files = run(&p, cid, &s, &om, "/w/a.wav"); + let d = files.lock().unwrap()[0].1.clone(); + assert_eq!(&d[..4], b"RIFF"); + let u16_at = |o: usize| u16::from_le_bytes([d[o], d[o + 1]]); + let u32_at = |o: usize| u32::from_le_bytes([d[o], d[o + 1], d[o + 2], d[o + 3]]); + assert_eq!((u16_at(20), u16_at(22), u32_at(24), u16_at(32), u16_at(34)), (1, 1, 44_100, 3, 24)); + assert_eq!(u32_at(40), 22_050 * 3, "half a second of mono 24-bit"); + om.audio_channels = 2; + om.audio_format = AudioFormat::F32; + let files = run(&p, cid, &s, &om, "/w/b.wav"); + let d = files.lock().unwrap()[0].1.clone(); + let u16_at = |o: usize| u16::from_le_bytes([d[o], d[o + 1]]); + assert_eq!((u16_at(20), u16_at(22), u16_at(32), u16_at(34)), (3, 2, 8, 32), "IEEE float stereo"); + // AIFF has no float: 24-bit. + let mut aiff = OutputModule::for_format(OutputFormat::Aiff); + aiff.audio_format = AudioFormat::F32; + let files = run(&p, cid, &s, &aiff, "/w/c.aif"); + let d = files.lock().unwrap()[0].1.clone(); + assert_eq!(&d[8..12], b"AIFF"); + assert_eq!(u16::from_be_bytes([d[26], d[27]]), 24, "sample size"); +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index f8bef9f2..4c2f6bb4 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -55,7 +55,19 @@ fn settings() -> RenderSettings { fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report { let s = settings(); let path = path.to_string_lossy().to_string(); - let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None, nested_switches: true }; + let job = Job { + project: p, + footage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: om, + path: &path, + sink: None, + options: Default::default(), + nested_switches: true, + }; let mut last = Progress::default(); let mut calls = 0; let r = export(&job, &mut |pr| { @@ -167,8 +179,19 @@ fn h264_odd_size_is_cropped_even() { let comp = p.comp(cid).expect("comp"); assert_eq!(s.output_size(comp).0 % 2, 1); let path = d.join("odd.mp4").to_string_lossy().to_string(); - let job = - Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None, nested_switches: true }; + let job = Job { + project: &p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + nested_switches: true, + }; let r = export(&job, &mut |_| true).expect("export"); assert_eq!((r.width % 2, r.height % 2), (0, 0)); } @@ -288,8 +311,19 @@ fn cancel_stops_early() { let om = OutputModule::for_format(OutputFormat::PngSequence); let s = settings(); let path = d.join("c_[#####].png").to_string_lossy().to_string(); - let job = - Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None, nested_switches: true }; + let job = Job { + project: &p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + nested_switches: true, + }; let r = export(&job, &mut |_| false); assert!(matches!(r, Err(effectcraft_export::ExportError::Cancelled))); let written = std::fs::read_dir(&d).map(|r| r.count()).unwrap_or(0); diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs index 8156d850..5bf46e0a 100644 --- a/crates/export/tests/vp9_roundtrip.rs +++ b/crates/export/tests/vp9_roundtrip.rs @@ -61,7 +61,7 @@ fn lossless_is_exact_for_odd_sizes() { for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { for kind in 0..3 { let (y, u, v) = source(w, h, kind); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); assert_eq!((p.width, p.height), (w as u32, h as u32)); @@ -80,7 +80,7 @@ fn lossy_quality_and_size() { let (y, u, v) = source(w, h, kind); let mut last = 0usize; for q in [30u8, 60, 80, 95] { - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); let got = crop(&plane_u8(&p.y), p.y_stride, w, h); @@ -103,7 +103,7 @@ fn wide_frames_use_tile_columns() { // 4160 px > 64 superblocks: two tile columns are required. let (w, h) = (4160, 16); let (y, u, v) = source(w, h, 0); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); @@ -113,7 +113,7 @@ fn wide_frames_use_tile_columns() { fn ffmpeg_decodes_our_stream() { let (w, h) = (96, 64); let (y, u, v) = source(w, h, 0); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false, ..Default::default() }); let frame = e.encode_yuv420(&y, &u, &v); // IVF container (one frame). let mut ivf = Vec::new(); @@ -147,3 +147,176 @@ fn ffmpeg_decodes_our_stream() { assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); } + +// ---------------------------------------------------------------- inter frames + +/// Frame `t` of a moving sequence: a diagonal gradient with a sinusoidal texture and bright +/// squares, all translating by a fractional amount per frame (`kind` 1 adds noise that changes +/// every frame). +fn moving(w: usize, h: usize, t: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let (dx, dy) = (1.75 * t as f64, 0.6 * t as f64); + let mut s = 1234u32 + t as u32 * 77; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 28) as f64 + }; + let lum = |x: f64, y: f64| -> f64 { + let g = 40.0 + 120.0 * (x / w as f64) + 50.0 * (y / h as f64); + let tex = 18.0 * ((x * 0.31).sin() * (y * 0.23).cos()); + let sq = if (x - 20.0).rem_euclid(64.0) < 18.0 && (y - 10.0).rem_euclid(48.0) < 14.0 { 45.0 } else { 0.0 }; + g + tex + sq + }; + let mut y = vec![0u8; w * h]; + for yy in 0..h { + for x in 0..w { + let n = if kind == 1 { rnd() - 7.5 } else { 0.0 }; + y[yy * w + x] = (lum(x as f64 - dx, yy as f64 - dy) + n).round().clamp(0.0, 255.0) as u8; + } + } + let u: Vec = (0..cw * ch).map(|i| (100.0 + 40.0 * (((i % cw) as f64 * 2.0 - dx) / w as f64)).round() as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (150.0 - 30.0 * (((i / cw) as f64 * 2.0 - dy) / h as f64)).round() as u8).collect(); + (y, u, v) +} + +fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec; 3] { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + [crop(&plane_u8(&p.y), p.y_stride, w, h), crop(&plane_u8(&p.u), p.uv_stride, cw, ch), crop(&plane_u8(&p.v), p.uv_stride, cw, ch)] +} + +/// Encode `n` frames and decode them with one decoder: every decoded frame must equal the +/// encoder's reconstruction. Returns (total bytes, mean luma PSNR vs the source). +fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) { + let (w, h) = (cfg.width as usize, cfg.height as usize); + let mut e = Vp9Encoder::new(cfg); + let mut d = Decoder::new(); + let (mut bytes, mut db) = (0usize, 0.0); + for t in 0..n { + let (y, u, v) = moving(w, h, t, kind); + let f = e.encode_frame(&y, &u, &v, false); + bytes += f.data.len(); + let kind_s = if f.key { "key" } else { "inter" }; + let mut pics = d.decode(&f.data, t as i64).unwrap_or_else(|err| panic!("{w}x{h} frame {t} ({kind_s}): {err:?}")); + assert_eq!(pics.len(), 1); + let got = planes_of(&pics.remove(0), w, h); + let rec = e.reconstruction().expect("reconstruction"); + for k in 0..3 { + if got[k] != rec[k] { + let first = got[k].iter().zip(&rec[k]).position(|(a, b)| a != b).unwrap_or(0); + let pw = if k == 0 { w } else { w.div_ceil(2) }; + panic!("{w}x{h} frame {t} ({kind_s}, q{} lf{}): plane {k} differs first at ({}, {})", f.q_idx, f.filter_level, first % pw, first / pw); + } + } + db += psnr(&got[0], &y); + } + (bytes, db / n as f64) +} + +#[test] +fn inter_frames_decode_exactly() { + for (w, h) in [(64, 64), (96, 72), (33, 17), (130, 66)] { + for (q, kind) in [(30u8, 0u32), (70, 1), (92, 0), (100, 0)] { + let cfg = EncoderConfig { width: w as u32, height: h as u32, quality: q, keyframe_interval: 6, ..Default::default() }; + run_sequence(cfg, 8, kind); + } + } +} + +#[test] +fn inter_frames_with_tile_columns_decode_exactly() { + let cfg = EncoderConfig { width: 4160, height: 24, quality: 80, keyframe_interval: 0, ..Default::default() }; + run_sequence(cfg, 3, 0); +} + +#[test] +fn inter_psnr_rises_with_bitrate() { + let (w, h) = (160, 96); + let mut last = (0usize, 0.0f64); + let mut rows = vec![]; + for kbps in [60u32, 150, 400, 1200] { + let cfg = EncoderConfig { width: w, height: h, quality: 60, target_kbps: Some(kbps), frame_rate: 30.0, keyframe_interval: 30, ..Default::default() }; + let (bytes, db) = run_sequence(cfg, 20, 0); + rows.push(format!("{kbps} kbps target: {:.0} kbps actual, {db:.2} dB", bytes as f64 * 8.0 * 30.0 / 20.0 / 1000.0)); + assert!(bytes > last.0 && db > last.1, "{rows:#?}"); + last = (bytes, db); + } + eprintln!("{rows:#?}"); +} + +#[test] +fn inter_frames_are_much_smaller_than_intra() { + let (w, h) = (192u32, 108u32); + let mut report = vec![]; + for q in [50u8, 75, 90] { + let intra = EncoderConfig { width: w, height: h, quality: q, keyframe_interval: 1, ..Default::default() }; + let inter = EncoderConfig { keyframe_interval: 0, ..intra.clone() }; + let (bi, pi) = run_sequence(intra, 24, 0); + let (bp, pp) = run_sequence(inter, 24, 0); + report.push(format!("q{q}: intra {bi} B {pi:.2} dB, inter {bp} B {pp:.2} dB, ratio {:.2}", bi as f64 / bp as f64)); + assert!(pp > pi - 1.0, "{report:#?}"); + assert!(bi as f64 >= 3.0 * bp as f64, "{report:#?}"); + } + eprintln!("{report:#?}"); +} + +#[test] +fn ffmpeg_decodes_inter_frames_like_the_encoder() { + let (w, h) = (96usize, 64usize); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 75, keyframe_interval: 5, ..Default::default() }); + let n = 7; + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&(n as u32).to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + let mut recon = vec![]; + for t in 0..n { + let (y, u, v) = moving(w, h, t, 0); + let f = e.encode_frame(&y, &u, &v, false); + ivf.extend_from_slice(&(f.data.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&(t as u64).to_le_bytes()); + ivf.extend_from_slice(&f.data); + recon.push(e.reconstruction().expect("recon")); + } + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-inter-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let fsz = w * h + 2 * w.div_ceil(2) * h.div_ceil(2); + assert_eq!(out.stdout.len(), fsz * n); + for (t, r) in recon.iter().enumerate() { + let frame = &out.stdout[t * fsz..(t + 1) * fsz]; + assert_eq!(&frame[..w * h], &r[0][..], "ffmpeg luma of frame {t}"); + assert_eq!(&frame[w * h..w * h + r[1].len()], &r[1][..], "ffmpeg U of frame {t}"); + } +} + +/// Encoder speed on 720p (run with `--ignored --nocapture`). +#[test] +#[ignore] +fn bench_720p() { + let (w, h) = (1280usize, 720usize); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 80, keyframe_interval: 0, ..Default::default() }); + for t in 0..6 { + let (y, u, v) = moving(w, h, t, 0); + let t0 = std::time::Instant::now(); + let f = e.encode_frame(&y, &u, &v, false); + eprintln!("frame {t} ({}): {} B in {:.0} ms", if f.key { "key" } else { "inter" }, f.data.len(), t0.elapsed().as_secs_f64() * 1000.0); + } +} diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index 564efb3b..437a24c2 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -16,6 +16,11 @@ effectcraft-media = { workspace = true } effectcraft-expr = { workspace = true } effectcraft-script = { workspace = true } effectcraft-export = { workspace = true } +effectcraft-plugin = { workspace = true } + +# WebAssembly effect plug-ins run in wasmi on native builds. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +effectcraft-plugin = { workspace = true, features = ["wasm"] } [dev-dependencies] serde_json = { workspace = true } @@ -24,3 +29,4 @@ effectcraft-svg = { workspace = true } effectcraft-pdf = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-time = { workspace = true } +rayon = { workspace = true } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index cfa5f825..f0115f6f 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -40,10 +40,26 @@ impl Exporter for FileExporter { output: &job.item.output, path: job.path, sink: self.sink.as_deref(), + options: effectcraft_export::JobOptions { + log: job.item.log, + label: job.label.clone(), + storage: job.storage, + overflow: job.project.render_prefs.overflow_folders.clone(), + }, nested_switches: job.nested_switches, }; match effectcraft_export::export(&j, &mut |p| progress(p.done, p.total)) { - Ok(r) => Ok(ExportResult { path: r.path, frames: r.frames, width: r.width, height: r.height, bytes: r.bytes, seconds: r.seconds, audio: r.audio }), + Ok(r) => Ok(ExportResult { + path: r.path, + frames: r.frames, + width: r.width, + height: r.height, + bytes: r.bytes, + seconds: r.seconds, + audio: r.audio, + log: r.log, + overflow: r.overflow, + }), Err(effectcraft_export::ExportError::Cancelled) => Err(effectcraft_engine::render_queue::CANCELLED.into()), Err(e) => Err(e.to_string()), } @@ -59,6 +75,8 @@ pub fn session() -> Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), script: Some(effectcraft_script::runner), + script_ui: effectcraft_engine::scriptui::ScriptUi { dispatch: Some(effectcraft_script::dispatch_ui), ..Default::default() }, + plugin_loader: effectcraft_plugin::wasm_available().then_some(effectcraft_plugin::loader as effectcraft_engine::PluginLoader), ..Default::default() } } diff --git a/crates/host/tests/determinism.rs b/crates/host/tests/determinism.rs new file mode 100644 index 00000000..58a1b933 --- /dev/null +++ b/crates/host/tests/determinism.rs @@ -0,0 +1,74 @@ +//! Deterministic rendering (better than After Effects): the same project renders bit-identical +//! frames every time and whatever the number of render threads. Renders the demo comps (and a +//! comp of seeded-random effects: particles, grain, noise, wiggle expressions, Shatter) twice +//! on one thread and on several, and compares every float of every pixel. + +use effectcraft_engine::Session; +use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::{RenderOpts, Renderer}; +use effectcraft_engine::time::Tick; +use serde_json::json; + +fn render(s: &Session, p: &Project, comp: ItemId, t: f64) -> Vec { + let mut r = Renderer::new(p, s.footage.as_ref(), RenderOpts { scale: 0.25, ..Default::default() }); + r.expr = s.expr.as_deref(); + // No layer cache: every render computes everything. + let img = r.comp_frame(comp, Tick::from_seconds_f64(t)); + img.data.iter().flat_map(|px| px.iter().map(|c| c.to_bits())).collect() +} + +fn pool(n: usize) -> rayon::ThreadPool { + rayon::ThreadPoolBuilder::new().num_threads(n).build().expect("thread pool") +} + +/// A comp of effects that draw random numbers (all seeded per instance). +fn random_comp(s: &mut Session) -> ItemId { + s.execute("comp.new", json!({"name": "Random", "width": 480, "height": 270, "frameRate": 24, "duration": 4})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let solid = |s: &mut Session, name: &str, color: &str| { + s.execute("layer.newSolid", json!({"name": name, "color": color, "width": 480, "height": 270})).unwrap()["layer"].as_u64().unwrap() + }; + let a = solid(s, "Particles", "#000000"); + for fx in ["ec.sim.ccparticleworld", "ec.noise.addgrain"] { + s.execute("effect.apply", json!({"layer": a, "effect": fx})).unwrap(); + } + let b = solid(s, "Noise", "#336699"); + for fx in ["ec.noise.fractal", "ec.sim.ccsnowfall", "ec.noise.noisehls"] { + s.execute("effect.apply", json!({"layer": b, "effect": fx})).unwrap(); + } + s.execute("prop.setExpression", json!({"layer": b, "path": "transform/position", "expression": "wiggle(3, 40)"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": b, "path": "transform/opacity", "expression": "random(40, 90)"})).unwrap(); + let c = solid(s, "Shatter", "#cc8844"); + s.execute("layer.setTransform", json!({"layer": c, "prop": "scale", "value": [50, 50]})).unwrap(); + s.execute("effect.apply", json!({"layer": c, "effect": "ec.sim.shatter"})).unwrap(); + cid +} + +#[test] +fn renders_are_bit_identical_across_runs_and_thread_counts() { + let mut s = effectcraft_host::session(); + s.replace_project(effectcraft_engine::demo::demo_project(), None); + let random = random_comp(&mut s); + let p = (*s.project).clone(); + let mut comps: Vec<(ItemId, f64)> = p.comps().map(|(id, c)| (*id, c.duration.seconds())).collect(); + comps.sort_by_key(|c| c.0); + assert!(comps.len() >= 3, "demo comps + the random comp"); + assert!(comps.iter().any(|c| c.0 == random)); + let (one, many) = (pool(1), pool(6)); + let mut frames = 0; + for (comp, dur) in comps { + for t in [0.0, dur * 0.37, dur * 0.81] { + let a = one.install(|| render(&s, &p, comp, t)); + let b = many.install(|| render(&s, &p, comp, t)); + let c = many.install(|| render(&s, &p, comp, t)); + let d = one.install(|| render(&s, &p, comp, t)); + let name = &p.item(comp).unwrap().name; + assert!(!a.is_empty()); + assert!(a == b, "{name} @ {t:.2}s: 1 thread vs 6 threads differ"); + assert!(b == c, "{name} @ {t:.2}s: two 6-thread renders differ"); + assert!(a == d, "{name} @ {t:.2}s: two 1-thread renders differ"); + frames += 1; + } + } + assert!(frames >= 9); +} diff --git a/crates/host/tests/plugins.rs b/crates/host/tests/plugins.rs new file mode 100644 index 00000000..c8a9b368 --- /dev/null +++ b/crates/host/tests/plugins.rs @@ -0,0 +1,106 @@ +//! Effect plug-ins in a full session: a Rust plug-in registered through the trait and the +//! example WebAssembly plug-in (examples/plugins/posterize-bands) loaded with +//! `effect.plugins.load`, both applied by id like built-ins and rendered. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::plugin::{EffectPlugin, PluginFrame, PluginManifest, PluginParam, PluginParamKind, PluginParams, register_plugin}; +use effectcraft_engine::time::Tick; +use serde_json::json; + +struct Swap { + m: PluginManifest, +} + +impl EffectPlugin for Swap { + fn manifest(&self) -> &PluginManifest { + &self.m + } + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, _time: f64) -> Result<(), String> { + if params.b("on") { + for p in frame.pixels.iter_mut() { + p.swap(0, 2); + } + } + Ok(()) + } +} + +fn setup() -> (Session, u64) { + let mut s = effectcraft_host::session(); + s.execute("comp.new", json!({"name": "P", "width": 40, "height": 20, "duration": 1})).unwrap(); + let l = s.execute("layer.newSolid", json!({"color": [1.0, 0.5, 0.0], "width": 40, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + (s, l) +} + +fn px(s: &Session) -> [f32; 4] { + let cid = s.active_comp_id().unwrap(); + s.render(cid, Tick::ZERO, Default::default()).get(20, 10) +} + +#[test] +fn rust_plugins_apply_by_id() { + let m = PluginManifest { + api: 1, + id: "org.test.swap".into(), + name: "Swap Red Blue".into(), + category: "Channel".into(), + version: "1".into(), + author: String::new(), + description: String::new(), + params: vec![PluginParam { id: "on".into(), name: "On".into(), kind: PluginParamKind::Checkbox { default: true } }], + }; + register_plugin(Arc::new(Swap { m })).unwrap(); + let (mut s, l) = setup(); + s.execute("effect.apply", json!({"layer": l, "effect": "org.test.swap"})).unwrap(); + let p = px(&s); + assert!(p[0] < 0.01 && (p[1] - 0.5).abs() < 0.01 && p[2] > 0.99, "{p:?}"); + // Its parameters are ordinary properties: set (and animate) them like any effect's. + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/on", "value": false})).unwrap(); + let p = px(&s); + assert!(p[0] > 0.99 && p[2] < 0.01, "{p:?}"); + let list = s.execute("effect.plugins.list", json!({})).unwrap(); + assert!(list["plugins"].as_array().unwrap().iter().any(|p| p["id"] == "org.test.swap" && p["category"] == "Channel")); + assert_eq!(list["api"], 1); + assert!(s.execute("effect.list", json!({"filter": "swap red"})).unwrap().as_array().unwrap().len() == 1); +} + +/// Build the example plug-in (skipped when the wasm32 target isn't installed). +fn build_example() -> Option { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/plugins/posterize-bands"); + let target = std::env::var_os("CARGO_TARGET_DIR").map(std::path::PathBuf::from).unwrap_or_else(|| root.join("../../../target")).join("plugin-example"); + let target = if target.is_absolute() { target } else { root.join("../../..").join(target) }; + let st = std::process::Command::new(env!("CARGO")) + .current_dir(&root) + .args(["build", "--release", "--target", "wasm32-unknown-unknown", "--target-dir"]) + .arg(&target) + .status() + .ok()?; + if !st.success() { + eprintln!("skipping: the example plug-in didn't build (rustup target add wasm32-unknown-unknown)"); + return None; + } + Some(target.join("wasm32-unknown-unknown/release/posterize_bands.wasm")) +} + +#[test] +fn example_wasm_plugin_loads_and_renders() { + let Some(wasm) = build_example() else { return }; + let (mut s, l) = setup(); + let r = s.execute("effect.plugins.load", json!({"path": wasm.to_string_lossy()})).unwrap(); + assert_eq!(r["loaded"][0]["id"], "org.effectcraft.example.posterize-bands", "{r}"); + s.execute("effect.apply", json!({"layer": l, "effect": "Posterize Bands"})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/mix", "value": 0})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/levels", "value": 2})).unwrap(); + // Luminance of (1, .5, 0) is ~0.57: with 2 levels it bands to 1, scaling the colour by 1/y + // (8 bpc: red clips at 1). + let p = px(&s); + let y = 0.2126 + 0.7152 * 0.5; + assert!((p[1] - 0.5 / y as f32).abs() < 0.01 && p[0] > 0.99 && p[2] < 0.01, "{p:?}"); + // Invert: band 0. + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/invert", "value": true})).unwrap(); + assert_eq!(px(&s)[..3], [0.0, 0.0, 0.0]); + let list = s.execute("effect.plugins.list", json!({})).unwrap(); + assert!(list["plugins"].as_array().unwrap().iter().any(|p| p["source"].as_str().unwrap_or("").ends_with(".wasm"))); +} diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml new file mode 100644 index 00000000..4081b61b --- /dev/null +++ b/crates/plugin/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "effectcraft-plugin" +description = "EffectCraft effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1) into the effect registry" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +# The WebAssembly runtime (wasmi, a pure-Rust interpreter). Hosts enable it on native; without +# it `load_wasm` reports that WebAssembly plug-ins are unavailable. +wasm = ["dep:wasmi"] + +[dependencies] +effectcraft-effects = { workspace = true } +serde_json = { workspace = true } +wasmi = { workspace = true, optional = true } + +[dev-dependencies] +effectcraft-keyframe = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-raster = { workspace = true } diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs new file mode 100644 index 00000000..1231d171 --- /dev/null +++ b/crates/plugin/src/lib.rs @@ -0,0 +1,66 @@ +//! # effectcraft-plugin (L3) +//! +//! Loads **WebAssembly effect plug-ins** (plug-in API v1, see `docs/plugins.md`) into the effect +//! registry ([`effectcraft_effects::plugin`]). A plug-in is a `.wasm` (or `.wat`) module with +//! no imports that exports: +//! +//! | export | signature | | +//! |---|---|---| +//! | `memory` | memory | linear memory shared with the host | +//! | `ec_api_version` | `() -> i32` | must return 1 | +//! | `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | a UTF-8 JSON [`PluginManifest`] in memory | +//! | `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | +//! | `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; 0 = ok | +//! +//! For each render the host asks `ec_alloc` for one buffer holding the parameters (`nparams` +//! little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s, +//! premultiplied RGBA, row-major), calls `ec_render`, and reads the pixels back. +//! +//! Modules run in the [wasmi](https://github.com/wasmi-labs/wasmi) interpreter (pure Rust, +//! MIT/Apache-2.0) with deterministic floats, no imports (no files, network or clock) and a +//! fuel limit per frame, so a plug-in can't escape its sandbox, hang the app, or render +//! differently on another machine. The runtime is the `wasm` feature (on for the desktop app and +//! the CLI); without it [`load_wasm`] says plug-ins are unavailable. + +use effectcraft_effects::EffectSpec; +pub use effectcraft_effects::plugin::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams, register_plugin}; + +#[cfg(feature = "wasm")] +mod wasm; + +/// Whether this build can load WebAssembly plug-ins. +pub fn wasm_available() -> bool { + cfg!(feature = "wasm") +} + +/// Load and register a WebAssembly plug-in (`bytes` = a `.wasm` binary or `.wat` text; +/// `source` = where it came from, for listings). Returns the registered effect. +pub fn load_wasm(bytes: &[u8], source: &str) -> Result<&'static EffectSpec, String> { + #[cfg(feature = "wasm")] + { + let p = wasm::WasmPlugin::new(bytes, source)?; + register_plugin(std::sync::Arc::new(p)) + } + #[cfg(not(feature = "wasm"))] + { + let _ = (bytes, source); + Err("WebAssembly plug-ins are not available in this build".into()) + } +} + +/// [`load_wasm`] for the engine's plug-in loader hook: `{id, name, category, version, api}`. +pub fn loader(bytes: &[u8], source: &str) -> Result { + let spec = load_wasm(bytes, source)?; + let m = effectcraft_effects::plugin::plugin(spec.id).map(|p| p.manifest().clone()); + Ok(serde_json::json!({ + "id": spec.id, + "name": spec.name, + "category": spec.category, + "version": m.as_ref().map(|m| m.version.clone()).unwrap_or_default(), + "api": PLUGIN_API_VERSION, + "params": spec.params.len(), + })) +} + +#[cfg(all(test, feature = "wasm"))] +mod tests; diff --git a/crates/plugin/src/tests.rs b/crates/plugin/src/tests.rs new file mode 100644 index 00000000..9363ab3e --- /dev/null +++ b/crates/plugin/src/tests.rs @@ -0,0 +1,164 @@ +//! WebAssembly plug-ins: load, register, render through the effect registry, sandbox limits. + +use effectcraft_effects::{Buf, EffectCtx, EffectEnv, Image}; +use effectcraft_project::build::Ids; + +use crate::load_wasm; + +fn load_err(r: Result<&'static effectcraft_effects::EffectSpec, String>) -> String { + match r { + Err(e) => e, + Ok(s) => panic!("unexpectedly loaded {}", s.id), + } +} + +/// A plug-in in WebAssembly text: multiplies RGB by `gain`, adds `lift` × `tint` when `on`. +fn gain_module(id: &str, render_body: &str) -> String { + let manifest = format!( + r#"{{"api":1,"id":"{id}","name":"Gain {id}","category":"Test Plug-ins","version":"1.0","params":[{{"id":"gain","name":"Gain","type":"slider","default":0.5,"min":0,"max":4}},{{"id":"on","name":"Lift","type":"checkbox","default":false}},{{"id":"tint","name":"Tint","type":"color","default":[0,0,1,1]}}]}}"# + ); + let len = manifest.len(); + let escaped = manifest.replace('\\', "\\\\").replace('"', "\\\""); + format!( + r#"(module + (memory (export "memory") 2) + (data (i32.const 16) "{escaped}") + (func (export "ec_api_version") (result i32) i32.const 1) + (func (export "ec_manifest_ptr") (result i32) i32.const 16) + (func (export "ec_manifest_len") (result i32) i32.const {len}) + (func (export "ec_alloc") (param $n i32) (result i32) + (local $need i32) + local.get $n + i32.const 131071 + i32.add + i32.const 16 + i32.shr_u + memory.size + i32.sub + local.tee $need + i32.const 0 + i32.gt_s + if + local.get $need + memory.grow + drop + end + i32.const 65536) + (func (export "ec_render") (param $px i32) (param $w i32) (param $h i32) (param $params i32) (param $np i32) (param $t f64) (param $scale f64) (result i32) + (local $i i32) (local $end i32) (local $g f32) (local $on f64) + {render_body} + local.get $params + f64.load + f32.demote_f64 + local.set $g + local.get $params + f64.load offset=8 + local.set $on + local.get $px + local.get $w + local.get $h + i32.mul + i32.const 16 + i32.mul + i32.add + local.set $end + local.get $px + local.set $i + block $done + loop $l + local.get $i + local.get $end + i32.ge_u + br_if $done + local.get $i + local.get $i + f32.load + local.get $g + f32.mul + f32.store + local.get $i + local.get $i + f32.load offset=4 + local.get $g + f32.mul + f32.store offset=4 + local.get $i + local.get $i + f32.load offset=8 + local.get $g + f32.mul + local.get $on + local.get $params + f64.load offset=32 + f64.mul + f32.demote_f64 + f32.add + f32.store offset=8 + local.get $i + i32.const 16 + i32.add + local.set $i + br $l + end + end + i32.const 0))"# + ) +} + +fn render(spec: &effectcraft_effects::EffectSpec, gain: f64, on: bool) -> Image { + let mut next = 1; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [4.0, 2.0]); + let mut params = effectcraft_effects::flatten_params(&g, &mut |p| p.value.clone()); + params.values.insert("gain".into(), effectcraft_keyframe::Value::Scalar(gain)); + params.values.insert("on".into(), effectcraft_keyframe::Value::Bool(on)); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [4.0, 2.0], seed: 1, adjustment: false, env: EffectEnv::default() }; + let buf = Buf { img: Image::filled(4, 2, [0.8, 0.6, 0.2, 1.0]), offset: [0.0; 2], scale: 1.0 }; + (spec.render)(&ctx, buf).img +} + +#[test] +fn wasm_plugin_registers_and_renders() { + let spec = load_wasm(gain_module("org.test.gain", "").as_bytes(), "gain.wat").unwrap(); + assert_eq!(spec.id, "org.test.gain"); + assert_eq!(spec.category, "Test Plug-ins"); + assert_eq!(spec.params.len(), 3); + // Found like a built-in, listed under its own category. + assert!(effectcraft_effects::find("org.test.gain").is_some()); + assert!(effectcraft_effects::lookup("Gain org.test.gain").is_some()); + assert!(effectcraft_effects::all().iter().any(|e| e.id == "org.test.gain")); + assert!(effectcraft_effects::categories().contains(&"Test Plug-ins")); + let img = render(spec, 0.5, false); + assert_eq!(img.get(1, 1), [0.4, 0.3, 0.1, 1.0]); + // The checkbox and colour parameters arrive flattened (tint blue = 1 → +1 on blue). + let img = render(spec, 1.0, true); + assert_eq!(img.get(3, 0), [0.8, 0.6, 1.2, 1.0]); + // Deterministic, and safe to run on many threads at once (each takes an instance). + let a: Vec = (0..8).map(|_| render(spec, 0.25, false)).collect(); + std::thread::scope(|sc| { + let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| render(spec, 0.25, false))).collect(); + for h in hs { + assert_eq!(h.join().unwrap(), a[0]); + } + }); + assert!(a.iter().all(|i| *i == a[0])); + // The same id can't be registered twice. + assert!(load_err(load_wasm(gain_module("org.test.gain", "").as_bytes(), "again.wat")).contains("already registered")); +} + +#[test] +fn wasm_plugins_are_sandboxed() { + // A runaway loop runs out of fuel: the frame comes back unchanged instead of hanging. + let spec = load_wasm(gain_module("org.test.spin", "(loop $spin br $spin)").as_bytes(), "spin.wat").unwrap(); + let img = render(spec, 0.5, false); + assert_eq!(img.get(0, 0), [0.8, 0.6, 0.2, 1.0]); + // No imports (files, clock, network…). + let imports = r#"(module (import "env" "now" (func $now (result f64))) (memory (export "memory") 1))"#; + assert!(load_err(load_wasm(imports.as_bytes(), "imports.wat")).contains("import")); + // Garbage, wrong API version, bad manifests. + assert!(load_wasm(b"\0asm nope", "bad.wasm").is_err()); + let v2 = gain_module("org.test.v2", "") + .replace("(func (export \"ec_api_version\") (result i32) i32.const 1)", "(func (export \"ec_api_version\") (result i32) i32.const 2)"); + assert!(load_err(load_wasm(v2.as_bytes(), "v2.wat")).contains("API 2")); + let reserved = gain_module("ec.blur.mine", ""); + assert!(load_err(load_wasm(reserved.as_bytes(), "reserved.wat")).contains("reserved")); +} diff --git a/crates/plugin/src/wasm.rs b/crates/plugin/src/wasm.rs new file mode 100644 index 00000000..8015f7ee --- /dev/null +++ b/crates/plugin/src/wasm.rs @@ -0,0 +1,151 @@ +//! The wasmi-backed [`EffectPlugin`] (plug-in API v1 ABI, see the crate docs). + +use std::sync::Mutex; + +use wasmi::{Config, Engine, Instance, Linker, Memory, Module, Store, TypedFunc, WasmParams, WasmResults}; + +use crate::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams}; + +/// Instruction budget per frame: a fixed allowance plus some per pixel. Generous for per-pixel +/// work, but a runaway loop stops instead of hanging the render. +const FUEL_BASE: u64 = 50_000_000; +const FUEL_PER_PIXEL: u64 = 20_000; + +type RenderFn = TypedFunc<(i32, i32, i32, i32, i32, f64, f64), i32>; + +/// One instantiated module (renders running on several threads each take their own). +struct Inst { + store: Store<()>, + instance: Instance, + memory: Memory, + alloc: TypedFunc, + render: RenderFn, +} + +impl Inst { + fn func(&self, name: &str) -> Result, String> { + self.instance.get_typed_func::(&self.store, name).map_err(|_| missing(name)) + } +} + +fn missing(name: &str) -> String { + format!("the module exports no `{name}` (or with the wrong signature)") +} + +fn err(e: impl std::fmt::Display) -> String { + e.to_string() +} + +fn instantiate(engine: &Engine, module: &Module) -> Result { + let mut store = Store::new(engine, ()); + store.set_fuel(FUEL_BASE).map_err(err)?; + let linker = Linker::<()>::new(engine); + let instance = linker.instantiate_and_start(&mut store, module).map_err(|e| format!("instantiation failed: {e}"))?; + let memory = instance.get_memory(&store, "memory").ok_or("the module exports no `memory`")?; + let alloc = instance.get_typed_func::(&store, "ec_alloc").map_err(|_| missing("ec_alloc"))?; + let render = instance.get_typed_func::<(i32, i32, i32, i32, i32, f64, f64), i32>(&store, "ec_render").map_err(|_| missing("ec_render"))?; + Ok(Inst { store, instance, memory, alloc, render }) +} + +/// A loaded WebAssembly effect plug-in. +pub struct WasmPlugin { + manifest: PluginManifest, + source: String, + engine: Engine, + module: Module, + /// Idle instances. + pool: Mutex>, +} + +impl WasmPlugin { + pub fn new(bytes: &[u8], source: &str) -> Result { + let mut config = Config::default(); + config.consume_fuel(true); + let engine = Engine::new(&config); + let module = Module::new(&engine, bytes).map_err(|e| format!("not a valid WebAssembly module: {e}"))?; + if let Some(i) = module.imports().next() { + return Err(format!("plug-ins can't import anything (the module imports `{}.{}`)", i.module(), i.name())); + } + let mut inst = instantiate(&engine, &module)?; + let version = inst.func::<(), i32>("ec_api_version")?; + let v = version.call(&mut inst.store, ()).map_err(err)?; + if v != PLUGIN_API_VERSION as i32 { + return Err(format!("the module implements plug-in API {v}, this build implements API {PLUGIN_API_VERSION}")); + } + let ptr = inst.func::<(), i32>("ec_manifest_ptr")?.call(&mut inst.store, ()).map_err(err)?; + let len = inst.func::<(), i32>("ec_manifest_len")?.call(&mut inst.store, ()).map_err(err)?; + let mut text = vec![0u8; len.max(0) as usize]; + inst.memory.read(&inst.store, ptr as u32 as usize, &mut text).map_err(|e| format!("manifest out of bounds: {e}"))?; + let manifest: PluginManifest = serde_json::from_slice(&text).map_err(|e| format!("bad manifest JSON: {e}"))?; + manifest.validate()?; + Ok(WasmPlugin { manifest, source: source.to_string(), engine, module, pool: Mutex::new(vec![inst]) }) + } + + fn take(&self) -> Result { + if let Some(i) = self.pool.lock().ok().and_then(|mut p| p.pop()) { + return Ok(i); + } + instantiate(&self.engine, &self.module) + } + + fn give_back(&self, inst: Inst) { + if let Ok(mut p) = self.pool.lock() { + p.push(inst); + } + } +} + +impl EffectPlugin for WasmPlugin { + fn manifest(&self) -> &PluginManifest { + &self.manifest + } + + fn source(&self) -> String { + self.source.clone() + } + + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String> { + let n = frame.pixels.len(); + let params_bytes = params.flat.len() * 8; + let total = params_bytes + n * 16; + if total > i32::MAX as usize { + return Err("frame too large for a 32-bit plug-in".into()); + } + let mut inst = self.take()?; + inst.store.set_fuel(FUEL_BASE + FUEL_PER_PIXEL * n as u64).map_err(err)?; + let base = inst.alloc.call(&mut inst.store, total as i32).map_err(err)? as u32 as usize; + // Parameters, then pixels (little-endian, as WebAssembly memory is). + let mut buf: Vec = Vec::with_capacity(total); + for v in params.flat { + buf.extend_from_slice(&v.to_le_bytes()); + } + for px in frame.pixels.iter() { + for c in px { + buf.extend_from_slice(&c.to_le_bytes()); + } + } + inst.memory.write(&mut inst.store, base, &buf).map_err(|e| format!("ec_alloc returned a buffer out of bounds: {e}"))?; + let px_ptr = (base + params_bytes) as i32; + let r = inst + .render + .call(&mut inst.store, (px_ptr, frame.width as i32, frame.height as i32, base as i32, params.flat.len() as i32, time, frame.scale)) + .map_err(|e| format!("ec_render trapped: {e}")); + let r = match r { + Ok(0) => { + inst.memory.read(&inst.store, base + params_bytes, &mut buf[params_bytes..]).map_err(err)?; + for (px, b) in frame.pixels.iter_mut().zip(buf[params_bytes..].chunks_exact(16)) { + for (c, v) in px.iter_mut().zip(b.chunks_exact(4)) { + *c = f32::from_le_bytes([v[0], v[1], v[2], v[3]]); + } + } + Ok(()) + } + Ok(code) => Err(format!("ec_render returned error {code}")), + Err(e) => Err(e), + }; + if r.is_ok() { + self.give_back(inst); + } + r + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index dc1dd8ab..887bcc70 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -8,6 +8,7 @@ pub mod build; pub mod essential; pub mod props; pub mod render_queue; +pub mod render_templates; pub mod styles; pub mod tracking; @@ -940,11 +941,25 @@ pub struct Project { /// The Render Queue (Composition ▸ Add to Render Queue). #[serde(default, skip_serializing_if = "Vec::is_empty")] pub render_queue: Vec, + /// Render Settings / Output Module templates and their defaults. + #[serde(default, skip_serializing_if = "render_templates::RenderTemplates::is_default")] + pub render_templates: render_templates::RenderTemplates, + /// Render Queue preferences (Notify, storage overflow folders). + #[serde(default, skip_serializing_if = "render_queue::RenderQueuePrefs::is_default")] + pub render_prefs: render_queue::RenderQueuePrefs, } impl Default for Project { fn default() -> Self { - Project { schema: SCHEMA_VERSION, settings: ProjectSettings::default(), items: BTreeMap::new(), next_id: 1, render_queue: Vec::new() } + Project { + schema: SCHEMA_VERSION, + settings: ProjectSettings::default(), + items: BTreeMap::new(), + next_id: 1, + render_queue: Vec::new(), + render_templates: Default::default(), + render_prefs: Default::default(), + } } } diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index d0f0c91f..35aed0be 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -84,38 +84,329 @@ impl ProxyUse { } } +/// Render Settings ▸ Field Render: interlaced output, each frame woven from two fields rendered +/// half a frame apart. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum FieldRender { + #[default] + Off, + /// The upper field (lines 0, 2, 4…) holds the earlier time. + UpperFirst, + LowerFirst, +} + +impl FieldRender { + pub const ALL: [FieldRender; 3] = [FieldRender::Off, FieldRender::UpperFirst, FieldRender::LowerFirst]; + pub fn label(self) -> &'static str { + match self { + FieldRender::Off => "Off", + FieldRender::UpperFirst => "Upper Field First", + FieldRender::LowerFirst => "Lower Field First", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "off" | "none" | "progressive" => Some(FieldRender::Off), + "upper" | "upperfirst" | "upperfieldfirst" => Some(FieldRender::UpperFirst), + "lower" | "lowerfirst" | "lowerfieldfirst" => Some(FieldRender::LowerFirst), + _ => None, + } + } +} + +fn norm(s: &str) -> String { + s.to_ascii_lowercase().replace([' ', '_', '-', '.', ':', '&'], "") +} + +/// Render Settings ▸ 3:2 Pulldown: with field rendering, four film frames (at 4/5 of the output +/// rate) spread over five interlaced frames in 2:3 cadence. The phase names which of the five +/// frames are Whole (both fields from one film frame) or Split. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum Pulldown { + #[default] + Off, + Wssww, + Sswww, + Swwws, + Wwwss, + Wwssw, +} + +/// Film frame (within a group of four) of each of the ten fields of the 2:3 cadence. +const CADENCE: [i64; 10] = [0, 0, 1, 1, 1, 2, 2, 3, 3, 3]; + +impl Pulldown { + pub const ALL: [Pulldown; 6] = [Pulldown::Off, Pulldown::Wssww, Pulldown::Sswww, Pulldown::Swwws, Pulldown::Wwwss, Pulldown::Wwssw]; + pub fn label(self) -> &'static str { + match self { + Pulldown::Off => "Off", + Pulldown::Wssww => "WSSWW", + Pulldown::Sswww => "SSWWW", + Pulldown::Swwws => "SWWWS", + Pulldown::Wwwss => "WWWSS", + Pulldown::Wwssw => "WWSSW", + } + } + pub fn parse(s: &str) -> Option { + let n = norm(s); + if matches!(n.as_str(), "off" | "none") { + return Some(Pulldown::Off); + } + Pulldown::ALL.into_iter().find(|p| p.label().eq_ignore_ascii_case(&n)) + } + /// Offset (in frames) into the WWSSW base cadence. + fn rotation(self) -> Option { + Some(match self { + Pulldown::Off => return None, + Pulldown::Wwssw => 0, + Pulldown::Wssww => 1, + Pulldown::Sswww => 2, + Pulldown::Swwws => 3, + Pulldown::Wwwss => 4, + }) + } + /// The film frames (counted from the first film frame of the render) of the first and + /// second field of output frame `k`. + pub fn fields(self, k: u64) -> Option<(i64, i64)> { + let r = self.rotation()?; + let film = |f: i64| f.div_euclid(10) * 4 + CADENCE[f.rem_euclid(10) as usize]; + let f0 = 2 * (k as i64 + r); + let base = film(2 * r); + Some((film(f0) - base, film(f0 + 1) - base)) + } + /// The W/S pattern of five consecutive output frames. + pub fn pattern(self) -> String { + (0..5).filter_map(|k| self.fields(k).map(|(a, b)| if a == b { 'W' } else { 'S' })).collect() + } +} + +/// A layer switch override (Render Settings ▸ Frame Blending / Motion Blur). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum SwitchOverride { + /// The layer switches and the composition's enable switch. + #[default] + Current, + /// Every layer whose switch is on, whatever the composition's enable switch. + OnForChecked, + OffForAll, +} + +impl SwitchOverride { + pub const ALL: [SwitchOverride; 3] = [SwitchOverride::Current, SwitchOverride::OnForChecked, SwitchOverride::OffForAll]; + pub fn label(self) -> &'static str { + match self { + SwitchOverride::Current => "Current Settings", + SwitchOverride::OnForChecked => "On for Checked Layers", + SwitchOverride::OffForAll => "Off for All Layers", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(SwitchOverride::Current), + "on" | "onforchecked" | "onforcheckedlayers" | "true" => Some(SwitchOverride::OnForChecked), + "off" | "offforall" | "offforalllayers" | "false" => Some(SwitchOverride::OffForAll), + _ => None, + } + } +} + +/// Render Settings ▸ Effects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum EffectsMode { + #[default] + Current, + AllOn, + AllOff, +} + +impl EffectsMode { + pub const ALL: [EffectsMode; 3] = [EffectsMode::Current, EffectsMode::AllOn, EffectsMode::AllOff]; + pub fn label(self) -> &'static str { + match self { + EffectsMode::Current => "Current Settings", + EffectsMode::AllOn => "All On", + EffectsMode::AllOff => "All Off", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(EffectsMode::Current), + "allon" | "on" => Some(EffectsMode::AllOn), + "alloff" | "off" => Some(EffectsMode::AllOff), + _ => None, + } + } +} + +/// Render Settings ▸ Solo Switches / Guide Layers: the current switches, or all off. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum CurrentOrOff { + #[default] + Current, + AllOff, +} + +impl CurrentOrOff { + pub fn label(self) -> &'static str { + match self { + CurrentOrOff::Current => "Current Settings", + CurrentOrOff::AllOff => "All Off", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" | "on" => Some(CurrentOrOff::Current), + "alloff" | "off" => Some(CurrentOrOff::AllOff), + _ => None, + } + } +} + +/// Render Settings ▸ Color Depth. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ColorDepth { + /// The project's bit depth. + #[default] + Current, + Bpc8, + Bpc16, + Bpc32, +} + +impl ColorDepth { + pub const ALL: [ColorDepth; 4] = [ColorDepth::Current, ColorDepth::Bpc8, ColorDepth::Bpc16, ColorDepth::Bpc32]; + pub fn label(self) -> &'static str { + match self { + ColorDepth::Current => "Current Settings", + ColorDepth::Bpc8 => "8 bits per channel", + ColorDepth::Bpc16 => "16 bits per channel", + ColorDepth::Bpc32 => "32 bits per channel", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(ColorDepth::Current), + "8" | "8bpc" | "8bitsperchannel" => Some(ColorDepth::Bpc8), + "16" | "16bpc" | "16bitsperchannel" => Some(ColorDepth::Bpc16), + "32" | "32bpc" | "32bitsperchannel" | "float" => Some(ColorDepth::Bpc32), + _ => None, + } + } + /// The depth renders use in a project of depth `project`. + pub fn resolve(self, project: crate::BitDepth) -> crate::BitDepth { + match self { + ColorDepth::Current => project, + ColorDepth::Bpc8 => crate::BitDepth::Bpc8, + ColorDepth::Bpc16 => crate::BitDepth::Bpc16, + ColorDepth::Bpc32 => crate::BitDepth::Bpc32, + } + } +} + +/// How one output frame is sampled in time. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameSample { + Progressive(Tick), + /// Two fields woven together: `first` goes to the dominant field. + Fields { + first: Tick, + second: Tick, + upper_first: bool, + }, +} + +fn default_true() -> bool { + true +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct RenderSettings { + /// The template these settings came from ("" or "Custom" once edited). + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, pub quality: RenderQuality, /// Output scale (1 = Full, 0.5 = Half, 1/3 = Third, 0.25 = Quarter). pub resolution: f64, pub time_span: TimeSpan, - /// `None` = use the comp's frame rate. + /// `None` = use the comp's frame rate (Time Sampling ▸ Use comp's frame rate). pub frame_rate: Option, - /// Motion blur for layers with the switch on (and the comp's Enable Motion Blur). + /// Motion blur allowed at all (false = Off for All Layers; see `motion_blur_mode`). pub motion_blur: bool, + /// Motion Blur: Current Settings / On for Checked Layers / Off for All Layers. + pub motion_blur_mode: SwitchOverride, + /// Frame Blending: Current Settings / On for Checked Layers / Off for All Layers. + pub frame_blending: SwitchOverride, + /// Field Render. + pub field_render: FieldRender, + /// 3:2 Pulldown phase (with field rendering). + pub pulldown: Pulldown, + /// Effects: Current Settings / All On / All Off. + pub effects: EffectsMode, + /// Solo Switches: Current Settings / All Off. + pub solo: CurrentOrOff, + /// Guide Layers: Current Settings / All Off. + pub guide_layers: CurrentOrOff, + /// Color Depth. + pub color_depth: ColorDepth, /// Image sequences: skip frames whose file already exists. pub skip_existing: bool, /// Proxy Use (Best Settings: Use No Proxies). pub proxy_use: ProxyUse, + /// Use Storage Overflow: when the output volume is full, continue in the project's overflow + /// folders. + #[serde(default = "default_true")] + pub storage_overflow: bool, } impl Default for RenderSettings { fn default() -> Self { RenderSettings { + name: String::new(), quality: RenderQuality::Best, resolution: 1.0, time_span: TimeSpan::WorkArea, frame_rate: None, motion_blur: true, + motion_blur_mode: SwitchOverride::Current, + frame_blending: SwitchOverride::Current, + field_render: FieldRender::Off, + pulldown: Pulldown::Off, + effects: EffectsMode::Current, + solo: CurrentOrOff::Current, + guide_layers: CurrentOrOff::AllOff, + color_depth: ColorDepth::Current, skip_existing: false, proxy_use: ProxyUse::UseNone, + storage_overflow: true, } } } impl RenderSettings { + /// Effective Motion Blur override. + pub fn motion_blur_override(&self) -> SwitchOverride { + if self.motion_blur { self.motion_blur_mode } else { SwitchOverride::OffForAll } + } + /// How output frame `i` is sampled (field rendering and 3:2 pulldown). + pub fn sample(&self, comp: &Comp, i: u64) -> FrameSample { + let upper_first = match self.field_render { + FieldRender::Off => return FrameSample::Progressive(self.frame_time(comp, i)), + FieldRender::UpperFirst => true, + FieldRender::LowerFirst => false, + }; + let r = self.rate(comp); + if let Some((a, b)) = self.pulldown.fields(i) { + // Film frames at 4/5 of the output rate, from the first one starting in the span. + let film = FrameRate::new(r.num * 4, r.den * 5); + let f0 = Self::ceil_frame(film, self.span(comp).0); + return FrameSample::Fields { first: film.tick_of(f0 + a), second: film.tick_of(f0 + b), upper_first }; + } + let f = self.first_frame(comp) + i as i64; + let t = r.tick_of(f); + let half = Tick((r.tick_of(f + 1).0 - t.0) / 2); + FrameSample::Fields { first: t, second: Tick(t.0 + half.0), upper_first } + } /// The comp-time span `[start, end)` to render. pub fn span(&self, comp: &Comp) -> (Tick, Tick) { let (a, b) = match self.time_span { @@ -162,11 +453,45 @@ impl RenderSettings { _ => "Custom", } } - /// One-line summary shown next to "Render Settings:" (AE shows the template name). + /// One-line summary shown next to "Render Settings:" (the template name, like AE). pub fn summary(&self) -> String { + if !self.name.is_empty() { + return self.name.clone(); + } let q = if self.quality == RenderQuality::Best { "Best Settings" } else { "Draft Settings" }; format!("{q} · {}", self.resolution_label()) } + /// Every setting as `(label, value)` lines (render logs, the settings dialog). + pub fn describe(&self, comp: Option<&Comp>) -> Vec<(&'static str, String)> { + let span = match self.time_span { + TimeSpan::WorkArea => "Work Area Only".to_string(), + TimeSpan::LengthOfComp => "Length of Comp".to_string(), + TimeSpan::Custom { start, end } => format!("Custom ({:.3}s – {:.3}s)", start.seconds(), end.seconds()), + }; + let rate = match (self.frame_rate, comp) { + (Some(r), _) => format!("{:.3} fps", r.as_f64()), + (None, Some(c)) => format!("Use comp's frame rate ({:.3})", c.frame_rate.as_f64()), + (None, None) => "Use comp's frame rate".into(), + }; + vec![ + ("Template", if self.name.is_empty() { "Custom".into() } else { self.name.clone() }), + ("Quality", format!("{:?}", self.quality)), + ("Resolution", format!("{} ({:.3})", self.resolution_label(), self.resolution)), + ("Proxy Use", self.proxy_use.label().into()), + ("Effects", self.effects.label().into()), + ("Solo Switches", self.solo.label().into()), + ("Guide Layers", self.guide_layers.label().into()), + ("Color Depth", self.color_depth.label().into()), + ("Frame Blending", self.frame_blending.label().into()), + ("Field Render", self.field_render.label().into()), + ("3:2 Pulldown", self.pulldown.label().into()), + ("Motion Blur", self.motion_blur_override().label().into()), + ("Time Span", span), + ("Frame Rate", rate), + ("Skip Existing Files", if self.skip_existing { "On" } else { "Off" }.into()), + ("Use Storage Overflow", if self.storage_overflow { "On" } else { "Off" }.into()), + ] + } } /// Output Module ▸ Format. @@ -287,8 +612,30 @@ pub enum Channels { /// Opaque, composited over the comp background colour. #[default] Rgb, - /// With straight (unmatted) alpha. + /// With alpha (straight or premultiplied: [`AlphaMode`]). Rgba, + /// The alpha channel alone, as an opaque greyscale matte. + Alpha, +} + +impl Channels { + pub fn label(self) -> &'static str { + match self { + Channels::Rgb => "RGB", + Channels::Rgba => "RGB + Alpha", + Channels::Alpha => "Alpha", + } + } +} + +/// Output Module ▸ Color: how RGB is stored with alpha. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum AlphaMode { + /// Unmatted colour. + #[default] + Straight, + /// Colour multiplied by alpha (matted with black). + Premultiplied, } /// Output Module ▸ Audio Output. @@ -301,6 +648,121 @@ pub enum AudioOutput { Off, } +/// Output Module ▸ Audio Output ▸ sample format (PCM outputs: WAV, AIFF, QuickTime). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum AudioFormat { + #[default] + S16, + S24, + /// 32-bit float (WAV, QuickTime; AIFF writes 24-bit). + F32, +} + +impl AudioFormat { + pub fn label(self) -> &'static str { + match self { + AudioFormat::S16 => "16 Bit", + AudioFormat::S24 => "24 Bit", + AudioFormat::F32 => "32 Bit Float", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "16" | "16bit" | "s16" => Some(AudioFormat::S16), + "24" | "24bit" | "s24" => Some(AudioFormat::S24), + "32" | "32bit" | "32bitfloat" | "f32" | "float" => Some(AudioFormat::F32), + _ => None, + } + } +} + +/// Output Module ▸ Crop (pixels of the rendered frame; negative values add transparent pixels). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Crop { + pub enabled: bool, + /// Use Region of Interest: crop to `roi` (comp pixels, captured from the viewer's region of + /// interest when the option is turned on) instead of the edge values. + pub use_roi: bool, + pub roi: Option<[f64; 4]>, + pub top: i32, + pub left: i32, + pub bottom: i32, + pub right: i32, +} + +impl Crop { + /// Edges `(top, left, bottom, right)` in pixels of a `w`×`h` frame rendered at `scale`. + pub fn edges(&self, w: u32, h: u32, scale: f64) -> (i32, i32, i32, i32) { + if !self.enabled { + return (0, 0, 0, 0); + } + if self.use_roi + && let Some([x, y, rw, rh]) = self.roi + { + let l = (x * scale).round() as i32; + let t = (y * scale).round() as i32; + let r = w as i32 - ((x + rw) * scale).round() as i32; + let b = h as i32 - ((y + rh) * scale).round() as i32; + return (t, l, b, r); + } + (self.top, self.left, self.bottom, self.right) + } +} + +/// Output Module ▸ Resize quality. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ResizeQuality { + /// Bilinear. + Low, + /// Bicubic. + #[default] + High, +} + +/// Output Module ▸ Resize. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Resize { + pub enabled: bool, + pub width: u32, + pub height: u32, + /// Lock Aspect Ratio: setting one dimension derives the other from the rendered frame. + pub lock_aspect: bool, + pub quality: ResizeQuality, +} + +impl Default for Resize { + fn default() -> Self { + Resize { enabled: false, width: 1920, height: 1080, lock_aspect: true, quality: ResizeQuality::High } + } +} + +/// Output Module ▸ Resize ▸ Resize to: presets. +pub const RESIZE_PRESETS: [(&str, u32, u32); 10] = [ + ("HDTV 1080", 1920, 1080), + ("HDTV 720", 1280, 720), + ("UHD 4K", 3840, 2160), + ("DCI 2K", 2048, 1080), + ("DCI 4K", 4096, 2160), + ("NTSC DV", 720, 480), + ("PAL D1/DV", 720, 576), + ("Square 1080", 1080, 1080), + ("Vertical 1080×1920", 1080, 1920), + ("Web 640×360", 640, 360), +]; + +/// Output Module ▸ Post-Render Action. +pub fn post_render_parse(s: &str) -> Option { + match norm(s).as_str() { + "none" => Some(PostRenderAction::None), + "import" => Some(PostRenderAction::Import), + "importreplace" | "importandreplace" | "importandreplaceusage" | "importreplaceusage" | "replace" => Some(PostRenderAction::ImportAndReplace), + "setproxy" | "proxy" => Some(PostRenderAction::SetProxy), + _ => None, + } +} + /// ProRes flavour. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum ProResProfile { @@ -341,38 +803,133 @@ impl ProResProfile { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct OutputModule { + /// The template this module came from ("" or "Custom" once edited). + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, pub format: OutputFormat, pub channels: Channels, + /// Color: straight or premultiplied RGB with alpha. + pub alpha_mode: AlphaMode, /// Output path or template (see the module docs). pub output: String, - /// JPEG quality 1–100. + /// JPEG / WebM quality 1–100. pub quality: u8, - /// H.264 target bitrate. + /// H.264 target bitrate (and WebM's when `webm_bitrate`). pub bitrate_kbps: u32, + /// WebM: rate control by `bitrate_kbps` instead of `quality`. + pub webm_bitrate: bool, + /// Movies: frames between key frames (0 = automatic, two seconds). + pub keyframe_interval: u32, pub prores_profile: ProResProfile, + pub crop: Crop, + pub resize: Resize, pub audio: AudioOutput, pub audio_sample_rate: u32, + /// 1 (mono) or 2 (stereo). + pub audio_channels: u8, + pub audio_format: AudioFormat, /// GIF: loop forever. pub gif_loop: bool, + /// Include Project Link: kept for parity; EffectCraft's writers have no place to store a + /// link back to the project, so it changes nothing in the file. + pub include_project_link: bool, } impl Default for OutputModule { fn default() -> Self { OutputModule { + name: String::new(), format: OutputFormat::H264, channels: Channels::Rgb, + alpha_mode: AlphaMode::Straight, output: DEFAULT_TEMPLATE.into(), quality: 90, bitrate_kbps: 10_000, + webm_bitrate: false, + keyframe_interval: 0, prores_profile: ProResProfile::Hq, + crop: Crop::default(), + resize: Resize::default(), audio: AudioOutput::Auto, audio_sample_rate: 48_000, + audio_channels: 2, + audio_format: AudioFormat::S16, gif_loop: true, + include_project_link: true, } } } impl OutputModule { + /// The size of the frames this module writes for frames rendered at `w`×`h` with render + /// scale `scale`: after Crop and Resize (before codec rounding). + pub fn frame_size(&self, w: u32, h: u32, scale: f64) -> (u32, u32) { + let (t, l, b, r) = self.crop.edges(w, h, scale); + let cw = (w as i32 - l - r).max(1) as u32; + let ch = (h as i32 - t - b).max(1) as u32; + if !self.resize.enabled { + return (cw, ch); + } + let (mut rw, mut rh) = (self.resize.width.max(1), self.resize.height.max(1)); + if self.resize.lock_aspect { + // Width rules; height follows the cropped frame's aspect. + rh = ((rw as f64 * ch as f64 / cw as f64).round() as u32).max(1); + } + rw = rw.min(30_000); + rh = rh.min(30_000); + (rw, rh) + } + /// The written frame size of an item (Render Settings resolution, Crop, Resize, codec + /// rounding). + pub fn output_size(&self, comp: &Comp, settings: &RenderSettings) -> (u32, u32) { + let (w, h) = settings.output_size(comp); + let (w, h) = self.frame_size(w, h, settings.resolution.clamp(0.01, 4.0)); + self.format.coded_size(w, h) + } + /// Every setting as `(label, value)` lines (render logs). + pub fn describe(&self) -> Vec<(&'static str, String)> { + let mut v = vec![ + ("Template", if self.name.is_empty() { "Custom".into() } else { self.name.clone() }), + ("Format", self.format.label().into()), + ("Channels", self.channels.label().into()), + ("Output To", self.output.clone()), + ]; + if self.channels == Channels::Rgba { + v.insert(3, ("Color", format!("{:?}", self.alpha_mode))); + } + match self.format { + OutputFormat::H264 => v.push(("Bitrate", format!("{} kbps", self.bitrate_kbps))), + OutputFormat::ProRes => v.push(("Codec", self.prores_profile.label().into())), + OutputFormat::WebM if self.webm_bitrate => v.push(("Bitrate", format!("{} kbps", self.bitrate_kbps))), + OutputFormat::WebM | OutputFormat::JpegSequence => v.push(("Quality", self.quality.to_string())), + _ => {} + } + if self.crop.enabled { + let c = &self.crop; + v.push(( + "Crop", + if c.use_roi { format!("Region of Interest {:?}", c.roi) } else { format!("T {} L {} B {} R {}", c.top, c.left, c.bottom, c.right) }, + )); + } + if self.resize.enabled { + let r = &self.resize; + v.push(("Resize", format!("{}×{} ({:?} quality{})", r.width, r.height, r.quality, if r.lock_aspect { ", aspect locked" } else { "" }))); + } + if self.format.supports_audio() { + v.push(( + "Audio Output", + format!( + "{:?} · {} Hz · {} · {}", + self.audio, + self.audio_sample_rate, + if self.audio_channels == 1 { "Mono" } else { "Stereo" }, + self.audio_format.label() + ), + )); + } + v.push(("Include Project Link", if self.include_project_link { "On" } else { "Off" }.into())); + v + } pub fn for_format(format: OutputFormat) -> OutputModule { let mut m = OutputModule { format, ..Default::default() }; if format.is_sequence() { @@ -384,7 +941,7 @@ impl OutputModule { pub fn set_format(&mut self, format: OutputFormat) { let old = self.format; self.format = format; - if !format.supports_alpha() { + if !format.supports_alpha() && self.channels == Channels::Rgba { self.channels = Channels::Rgb; } if self.output.contains("[fileExtension]") { @@ -406,10 +963,10 @@ impl OutputModule { } /// One-line summary shown next to "Output Module:". pub fn summary(&self) -> String { - let ch = match self.channels { - Channels::Rgb => "RGB", - Channels::Rgba => "RGB + Alpha", - }; + if !self.name.is_empty() { + return self.name.clone(); + } + let ch = self.channels.label(); match self.format { OutputFormat::H264 => format!("H.264 · {} kbps", self.bitrate_kbps), OutputFormat::ProRes => format!( @@ -478,6 +1035,85 @@ pub struct RenderQueueItem { /// Post-Render Action of the first output module. #[serde(default, skip_serializing_if = "PostRenderAction::is_none")] pub post_render: PostRenderAction, + /// The Log menu: what the render log next to the output records. + #[serde(default, skip_serializing_if = "RenderLog::is_default")] + pub log: RenderLog, +} + +/// Render Queue item ▸ Log. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum RenderLog { + /// A log only when the render fails. + #[default] + ErrorsOnly, + /// Always: the result plus every Render Settings and Output Module setting. + PlusSettings, + /// Also one line per rendered frame. + PlusPerFrameInfo, +} + +impl RenderLog { + pub const ALL: [RenderLog; 3] = [RenderLog::ErrorsOnly, RenderLog::PlusSettings, RenderLog::PlusPerFrameInfo]; + pub fn is_default(&self) -> bool { + *self == RenderLog::ErrorsOnly + } + pub fn label(self) -> &'static str { + match self { + RenderLog::ErrorsOnly => "Errors Only", + RenderLog::PlusSettings => "Plus Settings", + RenderLog::PlusPerFrameInfo => "Plus Per Frame Info", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "errors" | "errorsonly" => Some(RenderLog::ErrorsOnly), + "settings" | "plussettings" => Some(RenderLog::PlusSettings), + "perframe" | "plusperframe" | "plusperframeinfo" | "frames" => Some(RenderLog::PlusPerFrameInfo), + _ => None, + } + } +} + +/// The render log's path for an output: next to it, `_RenderLog.txt` (a sequence's +/// frame-number run is dropped). +pub fn log_path(output: &str) -> String { + let p = std::path::Path::new(output); + let stem = p.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); + let mut stem: String = stem.replace("_[#####]", "").replace("[#####]", ""); + if let Some(i) = stem.find('#') { + let n = stem[i..].chars().take_while(|c| *c == '#').count(); + stem.replace_range(i..i + n, ""); + } + let stem = stem.trim_end_matches(['_', '.', ' ']); + let name = format!("{}_RenderLog.txt", if stem.is_empty() { "Render" } else { stem }); + match p.parent().filter(|d| !d.as_os_str().is_empty()) { + Some(d) => d.join(name).to_string_lossy().to_string(), + None => name, + } +} + +/// Storage hook (Render Settings ▸ Use Storage Overflow): whether `bytes` more fit on the volume +/// holding `path`. The desktop app has no quota by default; tests and hosts with quotas supply +/// one. +pub trait StorageQuota: Send + Sync { + fn has_room(&self, path: &str, bytes: u64) -> bool; +} + +/// Project-wide Render Queue preferences. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RenderQueuePrefs { + /// Notify when the render finishes (a sound and a notification in the desktop app). + pub notify: bool, + /// Storage overflow folders, used in order when the output volume is full. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub overflow_folders: Vec, +} + +impl RenderQueuePrefs { + pub fn is_default(&self) -> bool { + *self == RenderQueuePrefs::default() + } } /// What happens after an item renders (Output Module ▸ Post-Render Action). @@ -521,6 +1157,7 @@ impl RenderQueueItem { last_output: None, extra_outputs: vec![], post_render: PostRenderAction::None, + log: RenderLog::ErrorsOnly, } } /// Every output module, the first one first. @@ -612,6 +1249,90 @@ mod tests { assert_eq!(ProResProfile::from_name("hq"), Some(ProResProfile::Hq)); } + #[test] + fn pulldown_cadence() { + assert_eq!(Pulldown::Wwssw.pattern(), "WWSSW"); + for p in &Pulldown::ALL[1..] { + assert_eq!(p.pattern(), p.label(), "{p:?}"); + } + // 2:3 cadence: ten fields carry four film frames; every output frame starts on film 0. + for p in &Pulldown::ALL[1..] { + let mut fields = vec![]; + for k in 0..10 { + let (a, b) = p.fields(k).unwrap(); + fields.extend([a, b]); + } + assert_eq!(fields[0], 0, "{p:?}"); + assert!(fields.windows(2).all(|w| w[1] == w[0] || w[1] == w[0] + 1), "{p:?}: {fields:?}"); + // 20 fields = 8 film frames' worth of cadence (a phase may start or end mid-frame). + assert!((7..=8).contains(&(*fields.last().unwrap() - fields[0])), "{p:?} {fields:?}"); + // Each film frame appears in 2 or 3 consecutive fields, alternating. + let mut runs = vec![]; + let mut n = 1; + for w in fields.windows(2) { + if w[1] == w[0] { + n += 1; + } else { + runs.push(n); + n = 1; + } + } + assert!(runs[1..].iter().all(|r| *r == 2 || *r == 3), "{p:?}: {runs:?}"); + assert!(runs[1..].windows(2).all(|w| w[0] != w[1]), "2 and 3 alternate: {p:?}: {runs:?}"); + } + assert_eq!(Pulldown::parse("sswww"), Some(Pulldown::Sswww)); + // With field rendering at 29.97, film frames are 23.976 fps. + let c = Comp::new(64, 48, FrameRate::new(30000, 1001), Tick::from_seconds_f64(2.0)); + let s = RenderSettings { field_render: FieldRender::UpperFirst, pulldown: Pulldown::Wssww, time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let film = FrameRate::new(24000, 1001); + match s.sample(&c, 1) { + FrameSample::Fields { first, second, upper_first } => { + assert!(upper_first); + assert_eq!((first, second), (film.tick_of(0), film.tick_of(1)), "WSSWW: frame 1 is split between film frames 0 and 1"); + } + f => panic!("{f:?}"), + } + } + + #[test] + fn field_samples_are_half_a_frame_apart() { + let c = Comp::new(64, 48, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); + let s = RenderSettings { field_render: FieldRender::LowerFirst, time_span: TimeSpan::LengthOfComp, ..Default::default() }; + assert_eq!(s.sample(&c, 3), FrameSample::Fields { first: Tick::from_seconds_f64(0.12), second: Tick::from_seconds_f64(0.14), upper_first: false }); + let p = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + assert_eq!(p.sample(&c, 3), FrameSample::Progressive(Tick::from_seconds_f64(0.12))); + } + + #[test] + fn crop_and_resize_sizes() { + let c = Comp::new(1920, 1080, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0)); + let rs = RenderSettings::default(); + let mut m = OutputModule::default(); + assert_eq!(m.output_size(&c, &rs), (1920, 1080)); + m.crop = Crop { enabled: true, top: 10, left: 20, bottom: 30, right: 41, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1858, 1040), "H.264 rounds the odd width down"); + m.format = OutputFormat::PngSequence; + assert_eq!(m.output_size(&c, &rs), (1859, 1040)); + // Region of interest, at half resolution. + m.crop = Crop { enabled: true, use_roi: true, roi: Some([100.0, 50.0, 800.0, 400.0]), ..Default::default() }; + let half = RenderSettings { resolution: 0.5, ..Default::default() }; + assert_eq!(m.output_size(&c, &half), (400, 200)); + // Negative crop pads. + m.crop = Crop { enabled: true, top: -10, bottom: -10, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1920, 1100)); + // Resize: fixed, and with the aspect locked to the (cropped) frame. + m.crop = Crop::default(); + m.resize = Resize { enabled: true, width: 1280, height: 999, lock_aspect: false, quality: ResizeQuality::High }; + assert_eq!(m.output_size(&c, &rs), (1280, 999)); + m.resize.lock_aspect = true; + assert_eq!(m.output_size(&c, &rs), (1280, 720)); + m.crop = Crop { enabled: true, left: 480, right: 480, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1280, 1440), "960×1080 cropped, then 1280 wide at 8:9"); + assert_eq!(log_path("/out/Main_[#####].png"), "/out/Main_RenderLog.txt"); + assert_eq!(log_path("/out/a.mov"), "/out/a_RenderLog.txt"); + assert_eq!(log_path("seq###.tif"), "seq_RenderLog.txt"); + } + #[test] fn spans() { let mut c = Comp::new(100, 50, FrameRate::new(25, 1), Tick::from_seconds_f64(4.0)); diff --git a/crates/project/src/render_templates.rs b/crates/project/src/render_templates.rs new file mode 100644 index 00000000..cf52b000 --- /dev/null +++ b/crates/project/src/render_templates.rs @@ -0,0 +1,323 @@ +//! Render Settings and Output Module templates (Edit ▸ Templates ▸ Render Settings… / Output +//! Module…): the built-in presets, templates saved with the project, and the defaults new +//! Render Queue items, pre-renders and proxies start from. +//! +//! Built-ins always exist; a saved template with a built-in's name overrides it. Templates +//! carry their name in [`RenderSettings::name`] / [`OutputModule::name`], which the Render Queue +//! shows like After Effects does. + +use serde::{Deserialize, Serialize}; + +use crate::render_queue::*; +use effectcraft_time::FrameRate; + +/// A default slot (the Defaults section of the Templates dialogs). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum TemplateSlot { + /// New Render Queue items rendering a movie. + Movie, + /// Single-frame renders (Composition ▸ Save Frame As ▸ File…). + Still, + /// Composition ▸ Pre-render…. + PreRender, + /// File ▸ Create Proxy ▸ Movie…. + MovieProxy, + /// File ▸ Create Proxy ▸ Still…. + StillProxy, +} + +impl TemplateSlot { + pub const ALL: [TemplateSlot; 5] = [TemplateSlot::Movie, TemplateSlot::Still, TemplateSlot::PreRender, TemplateSlot::MovieProxy, TemplateSlot::StillProxy]; + pub fn label(self) -> &'static str { + match self { + TemplateSlot::Movie => "Movie Default", + TemplateSlot::Still => "Frame Default", + TemplateSlot::PreRender => "Pre-Render Default", + TemplateSlot::MovieProxy => "Movie Proxy Default", + TemplateSlot::StillProxy => "Still Proxy Default", + } + } + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "movie" | "moviedefault" => Some(TemplateSlot::Movie), + "still" | "frame" | "framedefault" | "stilldefault" => Some(TemplateSlot::Still), + "prerender" | "prerenderdefault" => Some(TemplateSlot::PreRender), + "movieproxy" | "movieproxydefault" => Some(TemplateSlot::MovieProxy), + "stillproxy" | "stillproxydefault" => Some(TemplateSlot::StillProxy), + _ => None, + } + } +} + +/// Which template list. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum TemplateKind { + RenderSettings, + OutputModule, +} + +impl TemplateKind { + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "rendersettings" | "settings" | "render" => Some(TemplateKind::RenderSettings), + "outputmodule" | "output" | "module" => Some(TemplateKind::OutputModule), + _ => None, + } + } +} + +/// Defaults per slot (template names). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct TemplateDefaults { + pub render_settings: Vec<(TemplateSlot, String)>, + pub output_modules: Vec<(TemplateSlot, String)>, +} + +impl Default for TemplateDefaults { + fn default() -> Self { + use TemplateSlot::*; + let s = |v: &[(TemplateSlot, &str)]| v.iter().map(|(k, n)| (*k, n.to_string())).collect(); + TemplateDefaults { + render_settings: s(&[ + (Movie, "Best Settings"), + (Still, "Current Settings"), + (PreRender, "Best Settings"), + (MovieProxy, "Draft Settings"), + (StillProxy, "Best Settings"), + ]), + output_modules: s(&[ + (Movie, "H.264 - Match Render Settings - 15 Mbps"), + (Still, "PNG Sequence"), + (PreRender, "Lossless with Alpha"), + (MovieProxy, "Lossless with Alpha"), + (StillProxy, "PNG Sequence with Alpha"), + ]), + } + } +} + +/// Templates saved with the project, plus the defaults. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RenderTemplates { + #[serde(skip_serializing_if = "Vec::is_empty")] + pub render_settings: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub output_modules: Vec, + pub defaults: TemplateDefaults, +} + +/// The built-in Render Settings templates. +pub fn builtin_render_settings() -> Vec { + let best = RenderSettings { + name: "Best Settings".into(), + frame_blending: SwitchOverride::OnForChecked, + motion_blur_mode: SwitchOverride::OnForChecked, + ..RenderSettings::default() + }; + let draft = RenderSettings { + name: "Draft Settings".into(), + quality: RenderQuality::Draft, + resolution: 0.5, + proxy_use: ProxyUse::CurrentSettings, + frame_blending: SwitchOverride::Current, + motion_blur_mode: SwitchOverride::Current, + ..RenderSettings::default() + }; + let dv = + RenderSettings { name: "DV Settings".into(), field_render: FieldRender::LowerFirst, frame_rate: Some(FrameRate::new(30000, 1001)), ..best.clone() }; + let multi = RenderSettings { name: "Multi-Machine Settings".into(), skip_existing: true, ..best.clone() }; + let current = RenderSettings { + name: "Current Settings".into(), + proxy_use: ProxyUse::CurrentSettings, + guide_layers: CurrentOrOff::Current, + ..RenderSettings::default() + }; + vec![best, draft, dv, multi, current] +} + +fn om(name: &str, format: OutputFormat) -> OutputModule { + let mut m = OutputModule::for_format(format); + m.name = name.into(); + m +} + +/// The built-in Output Module templates. +pub fn builtin_output_modules() -> Vec { + let mut v = vec![]; + let mut hq = om("High Quality", OutputFormat::ProRes); + hq.prores_profile = ProResProfile::Hq; + v.push(hq); + let mut hqa = om("High Quality with Alpha", OutputFormat::ProRes); + hqa.prores_profile = ProResProfile::P4444; + hqa.channels = Channels::Rgba; + hqa.alpha_mode = AlphaMode::Premultiplied; + v.push(hqa); + // Lossless: the highest-fidelity movie codec this build writes (ProRes 4444 XQ). + let mut ll = om("Lossless", OutputFormat::ProRes); + ll.prores_profile = ProResProfile::P4444Xq; + v.push(ll); + let mut lla = om("Lossless with Alpha", OutputFormat::ProRes); + lla.prores_profile = ProResProfile::P4444Xq; + lla.channels = Channels::Rgba; + v.push(lla); + for mbps in [5u32, 15, 40] { + let mut h = om(&format!("H.264 - Match Render Settings - {mbps} Mbps"), OutputFormat::H264); + h.bitrate_kbps = mbps * 1000; + v.push(h); + } + let mut alpha = om("Alpha Only", OutputFormat::ProRes); + alpha.channels = Channels::Alpha; + v.push(alpha); + v.push(om("AIFF 48kHz", OutputFormat::Aiff)); + v.push(om("WAV 48kHz", OutputFormat::Wav)); + v.push(om("PNG Sequence", OutputFormat::PngSequence)); + let mut pa = om("PNG Sequence with Alpha", OutputFormat::PngSequence); + pa.channels = Channels::Rgba; + v.push(pa); + let mut ta = om("TIFF Sequence with Alpha", OutputFormat::TiffSequence); + ta.channels = Channels::Rgba; + v.push(ta); + v.push(om("JPEG Sequence", OutputFormat::JpegSequence)); + let mut exr = om("OpenEXR Sequence with Alpha", OutputFormat::ExrSequence); + exr.channels = Channels::Rgba; + v.push(exr); + v.push(om("Animated GIF", OutputFormat::Gif)); + v.push(om("WebM (VP9 + Opus)", OutputFormat::WebM)); + let mut wa = om("WebM with Alpha", OutputFormat::WebM); + wa.channels = Channels::Rgba; + v.push(wa); + v +} + +fn same(a: &str, b: &str) -> bool { + a.trim().eq_ignore_ascii_case(b.trim()) +} + +impl RenderTemplates { + pub fn is_default(&self) -> bool { + *self == RenderTemplates::default() + } + /// Every Render Settings template: built-ins (unless overridden), then saved ones. + pub fn render_settings_list(&self) -> Vec { + let mut v: Vec = + builtin_render_settings().into_iter().filter(|b| !self.render_settings.iter().any(|u| same(&u.name, &b.name))).collect(); + v.extend(self.render_settings.iter().cloned()); + v + } + pub fn output_module_list(&self) -> Vec { + let mut v: Vec = builtin_output_modules().into_iter().filter(|b| !self.output_modules.iter().any(|u| same(&u.name, &b.name))).collect(); + v.extend(self.output_modules.iter().cloned()); + v + } + pub fn render_settings(&self, name: &str) -> Option { + self.render_settings_list().into_iter().find(|t| same(&t.name, name)) + } + pub fn output_module(&self, name: &str) -> Option { + self.output_module_list().into_iter().find(|t| same(&t.name, name)) + } + pub fn is_builtin(kind: TemplateKind, name: &str) -> bool { + match kind { + TemplateKind::RenderSettings => builtin_render_settings().iter().any(|t| same(&t.name, name)), + TemplateKind::OutputModule => builtin_output_modules().iter().any(|t| same(&t.name, name)), + } + } + /// Save (or replace) a template under `name`. + pub fn save_render_settings(&mut self, name: &str, mut s: RenderSettings) { + s.name = name.trim().to_string(); + self.render_settings.retain(|t| !same(&t.name, name)); + self.render_settings.push(s); + } + pub fn save_output_module(&mut self, name: &str, mut m: OutputModule) { + m.name = name.trim().to_string(); + self.output_modules.retain(|t| !same(&t.name, name)); + self.output_modules.push(m); + } + /// Delete a saved template (a built-in it overrode comes back). Built-ins themselves can't + /// be deleted. + pub fn delete(&mut self, kind: TemplateKind, name: &str) -> Result<(), String> { + let before = (self.render_settings.len(), self.output_modules.len()); + match kind { + TemplateKind::RenderSettings => self.render_settings.retain(|t| !same(&t.name, name)), + TemplateKind::OutputModule => self.output_modules.retain(|t| !same(&t.name, name)), + } + if before != (self.render_settings.len(), self.output_modules.len()) { + return Ok(()); + } + if Self::is_builtin(kind, name) { Err(format!("“{name}” is a built-in template")) } else { Err(format!("no template “{name}”")) } + } + /// The default template name of a slot. + pub fn default_name(&self, kind: TemplateKind, slot: TemplateSlot) -> String { + let list = match kind { + TemplateKind::RenderSettings => &self.defaults.render_settings, + TemplateKind::OutputModule => &self.defaults.output_modules, + }; + list.iter().find(|(s, _)| *s == slot).map(|(_, n)| n.clone()).unwrap_or_else(|| { + let d = TemplateDefaults::default(); + let l = if kind == TemplateKind::RenderSettings { d.render_settings } else { d.output_modules }; + l.into_iter().find(|(s, _)| *s == slot).map(|(_, n)| n).unwrap_or_default() + }) + } + pub fn set_default(&mut self, kind: TemplateKind, slot: TemplateSlot, name: &str) -> Result<(), String> { + let exists = match kind { + TemplateKind::RenderSettings => self.render_settings(name).is_some(), + TemplateKind::OutputModule => self.output_module(name).is_some(), + }; + if !exists { + return Err(format!("no template “{name}”")); + } + let list = match kind { + TemplateKind::RenderSettings => &mut self.defaults.render_settings, + TemplateKind::OutputModule => &mut self.defaults.output_modules, + }; + list.retain(|(s, _)| *s != slot); + list.push((slot, name.trim().to_string())); + Ok(()) + } + /// The Render Settings a slot starts from (a missing template falls back to the built-in + /// default). + pub fn default_render_settings(&self, slot: TemplateSlot) -> RenderSettings { + self.render_settings(&self.default_name(TemplateKind::RenderSettings, slot)).unwrap_or_else(|| builtin_render_settings().remove(0)) + } + pub fn default_output_module(&self, slot: TemplateSlot) -> OutputModule { + self.output_module(&self.default_name(TemplateKind::OutputModule, slot)).unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builtins_defaults_and_round_trip() { + let mut t = RenderTemplates::default(); + let names: Vec = t.render_settings_list().into_iter().map(|s| s.name).collect(); + assert_eq!(names, ["Best Settings", "Draft Settings", "DV Settings", "Multi-Machine Settings", "Current Settings"]); + assert_eq!(t.default_render_settings(TemplateSlot::Movie).name, "Best Settings"); + assert_eq!(t.default_output_module(TemplateSlot::PreRender).channels, Channels::Rgba); + let dv = t.render_settings("dv settings").unwrap(); + assert_eq!(dv.field_render, FieldRender::LowerFirst); + // Save a custom template, make it the movie default, round-trip through JSON. + let mut s = RenderSettings { resolution: 0.25, ..Default::default() }; + s.color_depth = ColorDepth::Bpc16; + t.save_render_settings("Quarter 16", s); + t.set_default(TemplateKind::RenderSettings, TemplateSlot::Movie, "Quarter 16").unwrap(); + let mut m = OutputModule::for_format(OutputFormat::WebM); + m.crop = Crop { enabled: true, top: 4, left: 2, ..Default::default() }; + t.save_output_module("Web Crop", m); + let json = serde_json::to_string(&t).unwrap(); + let back: RenderTemplates = serde_json::from_str(&json).unwrap(); + assert_eq!(back, t); + assert_eq!(back.default_render_settings(TemplateSlot::Movie).color_depth, ColorDepth::Bpc16); + assert_eq!(back.output_module("web crop").unwrap().crop.top, 4); + assert!(t.delete(TemplateKind::RenderSettings, "Best Settings").is_err()); + t.delete(TemplateKind::OutputModule, "Web Crop").unwrap(); + assert!(t.output_module("Web Crop").is_none()); + // Overriding a built-in, then deleting the override, restores it. + t.save_output_module("High Quality", OutputModule::for_format(OutputFormat::Gif)); + assert_eq!(t.output_module("High Quality").unwrap().format, OutputFormat::Gif); + t.delete(TemplateKind::OutputModule, "High Quality").unwrap(); + assert_eq!(t.output_module("High Quality").unwrap().format, OutputFormat::ProRes); + } +} diff --git a/crates/script/src/lib.rs b/crates/script/src/lib.rs index a1e9e5d4..a7a85213 100644 --- a/crates/script/src/lib.rs +++ b/crates/script/src/lib.rs @@ -30,6 +30,14 @@ //! * **Security**: scripts can import/open/save projects and render, but `File` reads are limited //! to the project's folder and writes (and the network) are off unless Preferences ▸ Scripting //! & Expressions ▸ Allow Scripts to Write Files and Access Network is on. +//! * **ScriptUI** (`scriptui.js`, `ui.rs`): `Window` (dialog / palette / window), `Panel`, +//! `Group`, `Button`, `StaticText`, `EditText`, `Checkbox`, `RadioButton`, `Slider`, +//! `Progressbar`, `DropDownList`, `ListBox`, `TabbedPanel`/`Tab`, with `add()`, `orientation`, +//! `alignChildren`, `alignment`, `margins`, `spacing`, `preferredSize`, `onClick` / +//! `onChange` / `onChanging` / `onClose`, `show()` / `close()` and `layout.layout()`. Windows +//! are published to [`Session::script_ui`](effectcraft_engine::Session::script_ui) for +//! frontends and agents (`scriptui.*` commands); modal dialogs block `show()` until they close. +//! Scripts in the ScriptUI Panels folder run with `this` = a dockable `Panel`. //! //! Entry points: [`install`] sets [`Session::script`](effectcraft_engine::Session::script), which //! the `script.run` command, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`), the Script Console @@ -40,19 +48,22 @@ pub mod matchnames; mod model; mod runtime; +mod ui; use effectcraft_engine::{ScriptRequest, Session}; pub use runtime::{Outcome, ScriptError, run}; +pub use ui::dispatch_ui; /// The [`effectcraft_engine::ScriptRunner`] this crate provides. pub fn runner(s: &mut Session, req: &ScriptRequest) -> serde_json::Value { run(s, req).to_json() } -/// Enable scripting on a session. +/// Enable scripting (and ScriptUI) on a session. pub fn install(s: &mut Session) { s.script = Some(runner); + s.script_ui.dispatch = Some(dispatch_ui); } /// Run `code` as a one-off script named `name`. @@ -62,3 +73,5 @@ pub fn run_code(s: &mut Session, code: &str, name: &str) -> Outcome { #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_ui; diff --git a/crates/script/src/model.rs b/crates/script/src/model.rs index 03e80bb5..18270590 100644 --- a/crates/script/src/model.rs +++ b/crates/script/src/model.rs @@ -59,8 +59,8 @@ pub fn query(s: &mut Session, kind: &str, a: &J) -> R { } "labels" => Ok(json!(Label::ALL.iter().map(|l| l.name()).collect::>())), "effects" => Ok(json!( - effectcraft_engine::effects::registry() - .iter() + effectcraft_engine::effects::all() + .into_iter() .map(|e| json!({"id": e.id, "name": e.name, "matchName": matchnames::effect(e.id), "category": e.category})) .collect::>() )), diff --git a/crates/script/src/runtime.rs b/crates/script/src/runtime.rs index c494a6bf..82e456af 100644 --- a/crates/script/src/runtime.rs +++ b/crates/script/src/runtime.rs @@ -26,6 +26,9 @@ pub struct Outcome { /// `writeLn` / `$.writeln` / `alert` output, one entry per line. pub output: Vec, pub error: Option, + /// The script is waiting in a modal ScriptUI dialog's `show()` (it goes on when the dialog + /// closes; see [`crate::dispatch_ui`]). + pub waiting: bool, } /// An uncaught script error. @@ -42,36 +45,49 @@ impl Outcome { self.error.is_none() } pub fn to_json(&self) -> J { - json!({ + let mut j = json!({ "ok": self.ok(), "result": self.result, "output": self.output.join("\n"), "error": self.error.as_ref().map(|e| json!({"message": e.message, "line": e.line, "column": e.column, "file": e.file})), - }) + }); + if self.waiting { + j["waiting"] = json!(true); + } + j } } /// The session and per-run state while a script runs (the natives reach it here; nothing /// borrowed ever enters the JS heap). -struct Active { - session: Session, - output: Vec, +pub(crate) struct Active { + pub(crate) session: Session, + pub(crate) output: Vec, /// Open undo groups: (name, project snapshot when the outermost group began). - groups: Vec<(String, Arc)>, + pub(crate) groups: Vec<(String, Arc)>, /// `app.scheduleTask` queue: (id, code, repeat). - tasks: Vec<(u32, String, bool)>, - next_task: u32, - name: String, + pub(crate) tasks: Vec<(u32, String, bool)>, + pub(crate) next_task: u32, + pub(crate) name: String, + /// Running on a script-host thread: modal dialogs hand the session back to the caller + /// through this link while they wait. + pub(crate) link: Option, +} + +impl Active { + pub(crate) fn new(session: Session, name: &str) -> Active { + Active { session, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: name.to_string(), link: None } + } } thread_local! { - static ACTIVE: RefCell> = const { RefCell::new(None) }; + pub(crate) static ACTIVE: RefCell> = const { RefCell::new(None) }; /// The Script Console's persistent context (never dropped, like the expression runtime: boa's /// own thread-local heap may be torn down first at thread exit). - static CONSOLE: RefCell>> = const { RefCell::new(None) }; + pub(crate) static CONSOLE: RefCell>> = const { RefCell::new(None) }; } -fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { +pub(crate) fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { ACTIVE.with(|a| match a.try_borrow_mut() { Ok(mut g) => match g.as_mut() { Some(act) => f(act), @@ -81,11 +97,11 @@ fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result) -> boa_engine::JsError { +pub(crate) fn throw(msg: impl Into) -> boa_engine::JsError { JsNativeError::error().with_message(msg.into()).into() } -fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { +pub(crate) fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { Ok(args.get(i).cloned().unwrap_or_default().to_string(ctx)?.to_std_string_escaped()) } @@ -97,7 +113,7 @@ fn arg_json(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { serde_json::from_str(&s).map_err(|e| throw(format!("bad parameters: {e}"))) } -fn js_str(s: &str) -> JsValue { +pub(crate) fn js_str(s: &str) -> JsValue { JsValue::from(JsString::from(s)) } @@ -205,7 +221,7 @@ fn native_undo(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult) { +pub(crate) fn collapse(s: &mut Session, name: &str, snap: &Arc) { let h = &mut s.history; let Some(start) = h.undo.iter().rposition(|(_, p)| Arc::ptr_eq(p, snap)) else { // Nothing was recorded (or history was reset by New / Open Project). @@ -354,28 +370,32 @@ fn native_file(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult Result { +pub(crate) fn new_context() -> Result { let mut ctx = Context::default(); let limits = ctx.runtime_limits_mut(); limits.set_loop_iteration_limit(200_000_000); limits.set_recursion_limit(2000); - let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 6] = [ + let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 9] = [ (js_string!("__exec"), 2, native_exec), (js_string!("__query"), 2, native_query), (js_string!("__print"), 1, native_print), (js_string!("__undo"), 2, native_undo), (js_string!("__task"), 3, native_task), (js_string!("__file"), 3, native_file), + (js_string!("__uiNewId"), 0, crate::ui::native_new_id), + (js_string!("__uiModal"), 1, crate::ui::native_modal), + (js_string!("__uiLayoutNative"), 1, crate::ui::native_layout), ]; for (name, len, f) in natives { ctx.register_global_callable(name, len, NativeFunction::from_fn_ptr(f)).map_err(|e| e.to_string())?; } ctx.eval(Source::from_bytes(PRELUDE)).map_err(|e| format!("script prelude: {e}"))?; + ctx.eval(Source::from_bytes(crate::ui::SCRIPTUI)).map_err(|e| format!("ScriptUI prelude: {e}"))?; Ok(ctx) } /// Turn a boa error into a message with line and column (`code` is the user's script). -fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { +pub(crate) fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { // Thrown values: `throw new Error("x")`, `throw "x"`, native errors. let display = e.to_string(); let message = match e.as_opaque() { @@ -447,7 +467,7 @@ fn position(msg: &str, file: &str, code: &str) -> (Option, Option) { } /// Evaluate `code` in `ctx` and convert the result. -fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { +pub(crate) fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { let src = Source::from_bytes(code.as_bytes()).with_path(std::path::Path::new(file)); let script = boa_engine::Script::parse(src, None, ctx).map_err(|e| script_error(&e, ctx, file, code))?; let v = script.evaluate(ctx).map_err(|e| script_error(&e, ctx, file, code))?; @@ -471,8 +491,8 @@ fn result_json(v: &JsValue, ctx: &mut Context) -> J { } /// Restores the caller's session when a run ends (on every exit path). -struct Restore<'a> { - target: &'a mut Session, +pub(crate) struct Restore<'a> { + pub(crate) target: &'a mut Session, } impl Drop for Restore<'_> { @@ -483,8 +503,54 @@ impl Drop for Restore<'_> { } } +/// End of a script step: close undo groups left open, drop selections that no longer exist, +/// and take the output so far. +pub(crate) fn settle() -> Vec { + with_active(|a| { + if let Some((name, snap)) = a.groups.first().cloned() { + collapse(&mut a.session, &name, &snap); + a.groups.clear(); + } + a.session.sanitize_state(); + Ok(std::mem::take(&mut a.output)) + }) + .unwrap_or_default() +} + +/// Evaluate a script file's code, then the one-shot tasks it scheduled (`app.scheduleTask`). +pub(crate) fn eval_with_tasks(ctx: &mut Context, code: &str, file: &str) -> Result { + let mut r = eval_in(ctx, code, file); + let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); + for (_, code, repeat) in tasks { + if repeat { + continue; + } + if let Err(e) = eval_in(ctx, &code, "scheduled task") + && r.is_ok() + { + r = Err(e); + } + } + r +} + +pub(crate) fn outcome(output: Vec, result: Result) -> Outcome { + let mut out = Outcome { output, ..Default::default() }; + match result { + Ok(v) => out.result = v, + Err(e) => out.error = Some(e), + } + out +} + /// Run a script against `session`: the object model sees and edits that session through engine /// commands. Script errors come back in [`Outcome::error`]. +/// +/// On native builds a script file runs on its own script-host thread, so a ScriptUI dialog's +/// `show()` can wait for the user (the session comes back here meanwhile, and the dialog's +/// events reach the host through [`crate::dispatch_ui`]); palettes and panels keep their host +/// alive for their handlers. The Script Console (and the web build) run inline, where dialogs +/// don't block. pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { let running = ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)); if running { @@ -493,59 +559,49 @@ pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { ..Default::default() }; } + #[cfg(not(target_arch = "wasm32"))] + if !req.console { + return crate::ui::run_threaded(session, req); + } + run_inline(session, req) +} + +/// Run on this thread (the Script Console's persistent context, or a fresh one). +pub(crate) fn run_inline(session: &mut Session, req: &ScriptRequest) -> Outcome { + let host = if req.console { crate::ui::CONSOLE_HOST } else { crate::ui::new_host_id() }; let s = std::mem::take(session); - ACTIVE.with(|a| *a.borrow_mut() = Some(Active { session: s, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: req.name.to_string() })); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active::new(s, req.name))); let guard = Restore { target: session }; let file = if req.name.is_empty() { "script" } else { req.name }; - let mut result = if req.console { + let (result, windows) = if req.console { CONSOLE.with(|c| { let mut c = c.borrow_mut(); if c.is_none() { match new_context() { Ok(ctx) => *c = Some(ManuallyDrop::new(ctx)), - Err(e) => return Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + Err(e) => return (Err(ScriptError { message: e, file: file.into(), ..Default::default() }), vec![]), } } let ctx = c.as_mut().map(|m| &mut **m).expect("console context"); - eval_in(ctx, req.code, file) + let r = eval_in(ctx, req.code, file); + (r, crate::ui::snapshot(ctx)) }) } else { match new_context() { Ok(mut ctx) => { - let r = eval_in(&mut ctx, req.code, file); - // app.scheduleTask: run the queued one-shot tasks after the script, in order. - let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); - let mut r = r; - for (_, code, repeat) in tasks { - if repeat { - continue; - } - if let Err(e) = eval_in(&mut ctx, &code, "scheduled task") - && r.is_ok() - { - r = Err(e); - } + let r = eval_with_tasks(&mut ctx, req.code, file); + let windows = crate::ui::snapshot(&mut ctx); + if windows.iter().any(|w| w.visible) { + // Palettes and panels outlive the run: keep their context for the handlers. + crate::ui::keep_inline(host, ctx); } - r + (r, windows) } - Err(e) => Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + Err(e) => (Err(ScriptError { message: e, file: file.into(), ..Default::default() }), vec![]), } }; - // Close undo groups left open; drop selections that no longer exist. - let output = with_active(|a| { - if let Some((name, snap)) = a.groups.first().cloned() { - collapse(&mut a.session, &name, &snap); - a.groups.clear(); - } - a.session.sanitize_state(); - Ok(std::mem::take(&mut a.output)) - }) - .unwrap_or_default(); + let output = settle(); drop(guard); - let mut out = Outcome { output, ..Default::default() }; - match result.as_mut() { - Ok(v) => out.result = std::mem::take(v), - Err(e) => out.error = Some(e.clone()), - } - out + crate::ui::publish(session, host, req.name, windows); + outcome(output, result) } diff --git a/crates/script/src/scriptui.js b/crates/script/src/scriptui.js new file mode 100644 index 00000000..e214be30 --- /dev/null +++ b/crates/script/src/scriptui.js @@ -0,0 +1,544 @@ +// EffectCraft scripting: ScriptUI, the dialog / panel object model After Effects scripts use +// (`new Window("dialog", "Title")`, `win.add("button", undefined, "OK")`, `onClick`, `show()`…). +// +// Written from the behaviour Adobe's public JavaScript Tools Guide documents. Controls are plain +// JavaScript objects; after each script step the host reads `__uiSnapshot()` (a JSON +// description of the open windows) and lays them out, frontends draw them, and user actions come +// back through `__uiDispatch(window, control, kind, valueJson)`. Modal dialogs block in +// `show()` (`__uiModal`) until they close, as in After Effects. + +var __uiWins = []; +// show() results of closed dialogs, by window id. +var __uiResults = {}; + +function __uiModalResult(id) { + var r = __uiResults[id]; + if (r !== undefined) { + delete __uiResults[id]; + return r; + } + return __uiWin(id) ? null : 2; +} + +function __dim(w, h) { + var d = [w, h]; + Object.defineProperty(d, "width", { get: function () { return this[0]; }, set: function (v) { this[0] = v; } }); + Object.defineProperty(d, "height", { get: function () { return this[1]; }, set: function (v) { this[1] = v; } }); + return d; +} +function __bounds4(l, t, r, b) { + var a = [l, t, r, b]; + Object.defineProperty(a, "left", { get: function () { return this[0]; } }); + Object.defineProperty(a, "top", { get: function () { return this[1]; } }); + Object.defineProperty(a, "right", { get: function () { return this[2]; } }); + Object.defineProperty(a, "bottom", { get: function () { return this[3]; } }); + Object.defineProperty(a, "x", { get: function () { return this[0]; } }); + Object.defineProperty(a, "y", { get: function () { return this[1]; } }); + Object.defineProperty(a, "width", { get: function () { return this[2] - this[0]; } }); + Object.defineProperty(a, "height", { get: function () { return this[3] - this[1]; } }); + return a; +} +function __toBounds(b) { + if (b === undefined || b === null) return null; + if (b.length === 4) return [__num(b[0]), __num(b[1]), __num(b[2]), __num(b[3])]; + if (b.width !== undefined) { + var x = b.x || b.left || 0, y = b.y || b.top || 0; + return [x, y, x + __num(b.width), y + __num(b.height)]; + } + return null; +} + +var __uiTypes = { + window: "Window", panel: "Panel", group: "Group", button: "Button", iconbutton: "IconButton", + statictext: "StaticText", edittext: "EditText", checkbox: "Checkbox", radiobutton: "RadioButton", + slider: "Slider", scrollbar: "Scrollbar", progressbar: "Progressbar", dropdownlist: "DropDownList", + listbox: "ListBox", tabbedpanel: "TabbedPanel", tab: "Tab", image: "ScriptUIImage" +}; +var __uiContainers = { window: 1, panel: 1, group: 1, tabbedpanel: 1, tab: 1 }; + +function Window(type, title, bounds, props) { + if (!(this instanceof Window)) return new Window(type, title, bounds, props); + type = String(type === undefined ? "window" : type).toLowerCase(); + if (type.indexOf("{") >= 0) throw __err("ScriptUI resource strings are not supported: build the window with add()"); + if (type !== "dialog" && type !== "palette" && type !== "window") throw __err("Bad window type " + type); + __uiInit(this, "window", null, null, bounds, title, props); + this.__kind = type; + this.__root = true; + __uiWins.push(this); +} +Window.alert = function (s) { alert(s); }; +Window.confirm = function (s) { return confirm(s); }; +Window.prompt = function (m, d) { return prompt(m, d); }; +Window.find = function (type, title) { + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].text === title) return __uiWins[i]; + return null; +}; + +function Panel() {} +function Group() {} +function Button() {} +function IconButton() {} +function StaticText() {} +function EditText() {} +function Checkbox() {} +function RadioButton() {} +function Slider() {} +function Scrollbar() {} +function Progressbar() {} +function DropDownList() {} +function ListBox() {} +function TabbedPanel() {} +function Tab() {} +function ScriptUIImage() {} +function ListItem() {} + +var ScriptUI = { + version: "6.2.2", + frameworkName: "EffectCraft", + environment: { keyboardState: { shiftKey: false, ctrlKey: false, altKey: false, metaKey: false, keyName: "" } }, + FontStyle: { REGULAR: 0, BOLD: 1, ITALIC: 2, BOLDITALIC: 3 }, + BrushType: { SOLID_COLOR: 0, THEME_COLOR: 1 }, + PenType: { SOLID_COLOR: 0, THEME_COLOR: 1 }, + Alignment: { LEFT: "left", RIGHT: "right", TOP: "top", BOTTOM: "bottom", CENTER: "center", FILL: "fill" }, + newFont: function (name, style, size) { return { name: name, style: style, size: size }; }, + newImage: function () { return {}; }, + getResourceText: function (t) { return t; }, + events: { createEvent: function (t) { return { type: t }; } } +}; + +function __uiGraphics() { + return { + newBrush: function (t, c) { return { type: t, color: c }; }, + newPen: function (t, c, w) { return { type: t, color: c, lineWidth: w }; }, + newPath: function () {}, moveTo: function () {}, lineTo: function () {}, rectPath: function () {}, + ellipsePath: function () {}, fillPath: function () {}, strokePath: function () {}, closePath: function () {}, + drawString: function () {}, drawOSControl: function () {}, drawImage: function () {}, drawFocusRing: function () {}, + measureString: function (s) { return __dim(String(s).length * 7, 16); }, + font: null, foregroundColor: null, backgroundColor: null, disabledForegroundColor: null, disabledBackgroundColor: null + }; +} + +function __uiInit(o, type, win, parent, bounds, text, props) { + o.type = type; + o.window = win || o; + o.parent = parent || null; + o.__id = win ? win.__next++ : 0; + if (!win) { + o.__next = 1; + o.__wid = __uiNewId(); + } + o.properties = props || {}; + o.text = text === undefined || text === null ? "" : String(text); + o.enabled = true; + o.visible = true; + o.helpTip = ""; + o.children = []; + o.__ub = __toBounds(bounds); + o.__lb = [0, 0, 0, 0]; + o.preferredSize = __dim(-1, -1); + o.minimumSize = __dim(0, 0); + o.maximumSize = __dim(10000, 10000); + o.alignment = null; + o.graphics = __uiGraphics(); + o.__listeners = {}; + if (o.properties.name) o.name = o.properties.name; + if (__uiContainers[type]) { + o.orientation = type === "group" ? "row" : type === "tabbedpanel" ? "stack" : "column"; + o.alignChildren = type === "group" ? ["center", "center"] : ["center", "top"]; + o.spacing = 10; + o.margins = type === "window" ? [15, 15, 15, 15] : type === "panel" ? [10, 15, 10, 10] : type === "tab" ? [10, 10, 10, 10] : [0, 0, 0, 0]; + var self = o; + o.layout = { + layout: function () { __uiLayout(self.window); }, + resize: function () { __uiLayout(self.window); }, + margins: o.margins, + spacing: o.spacing + }; + } + if (type === "checkbox" || type === "radiobutton") o.value = !!o.properties.value; + if (type === "slider" || type === "scrollbar" || type === "progressbar") { + o.value = 0; + o.minvalue = 0; + o.maxvalue = 100; + } + if (type === "edittext") { + o.characters = 0; + o.textselection = ""; + o.active = false; + } + if (type === "dropdownlist" || type === "listbox") { + o.items = []; + o.__sel = []; + var it = o.properties.items; + if (it) for (var i = 0; i < it.length; i++) o.add("item", it[i]); + } + if (type === "tabbedpanel") o.__sel = []; + return o; +} + +function __uiProto() { + return { + add: function (type, bounds, text, props) { + var t = String(type).toLowerCase(); + if (t === "item" || t === "separator") return this.__addItem(t === "separator" ? "-" : bounds); + if (!__uiContainers[this.type]) throw __err("add(): a " + this.type + " can't contain controls"); + var ctor = __uiTypes[t]; + if (!ctor || t === "window") throw __err("add(): unknown control type " + type); + var c = Object.create(__ecGlobal[ctor].prototype); + // Value arguments after `text`: slider (value, min, max), progressbar (value, max), lists (items). + __uiInit(c, t, this.window, this, bounds, typeof text === "object" && text !== null && !(text instanceof Array) ? "" : text, props); + if ((t === "dropdownlist" || t === "listbox") && text instanceof Array) { + for (var i = 0; i < text.length; i++) c.add("item", text[i]); + c.text = ""; + } + if (t === "slider" || t === "scrollbar") { + c.text = ""; + c.value = text === undefined ? 0 : __num(text); + if (arguments.length > 3 && typeof props === "number") { + c.minvalue = props; + c.maxvalue = arguments[4] === undefined ? 100 : __num(arguments[4]); + c.properties = arguments[5] || {}; + } + } + if (t === "progressbar") { + c.text = ""; + c.value = text === undefined ? 0 : __num(text); + if (typeof props === "number") { + c.maxvalue = props; + c.properties = arguments[4] || {}; + } + } + if (t === "checkbox" || t === "radiobutton") c.value = false; + if (t === "edittext" && c.properties.multiline) c.__multiline = true; + if (t === "edittext" && c.properties.readonly) c.__readonly = true; + if (c.properties && c.properties.name) c.name = c.properties.name; + this.children.push(c); + if (t === "tab" && this.type === "tabbedpanel" && this.__sel.length === 0) this.__sel = [this.children.length - 1]; + return c; + }, + remove: function (what) { + var list = this.items && this.type !== "tabbedpanel" ? this.items : this.children; + var i = typeof what === "number" ? what : list.indexOf(what); + if (i < 0 || i >= list.length) return; + list.splice(i, 1); + if (this.items) { + for (var k = 0; k < this.items.length; k++) this.items[k].index = k; + this.__sel = this.__sel.filter(function (s) { return s !== i; }).map(function (s) { return s > i ? s - 1 : s; }); + } + }, + removeAll: function () { + if (this.items) { + this.items = []; + this.__sel = []; + } else this.children = []; + }, + find: function (text) { + var list = this.items || this.children; + for (var i = 0; i < list.length; i++) if (list[i].text === text) return list[i]; + return null; + }, + __addItem: function (text) { + if (!this.items) throw __err("add(\"item\"): only lists have items"); + var it = Object.create(ListItem.prototype); + it.text = String(text === undefined ? "" : text); + it.type = text === "-" ? "separator" : "item"; + it.index = this.items.length; + it.parent = this; + it.image = null; + it.checked = false; + this.items.push(it); + return it; + }, + show: function () { + if (!this.__root) { + this.visible = true; + return; + } + this.__shown = true; + this.__closed = false; + this.__fire("onShow"); + if (this.__kind === "dialog") { + this.__modal = true; + var r = __uiModal(this.__wid); + this.__modal = false; + return r === null ? undefined : r; + } + return undefined; + }, + hide: function () { + if (this.__root) { + this.close(2); + } else this.visible = false; + }, + close: function (result) { + if (!this.__root) return; + if (this.__fire("onClose") === false) return; + this.__closed = true; + this.__shown = false; + this.__result = result === undefined ? 0 : result; + if (this.__modal) __uiResults[this.__wid] = this.__result; + }, + center: function () {}, + notify: function (ev) { + var e = ev || (this.type === "edittext" || this.type === "slider" || this.type === "dropdownlist" || this.type === "listbox" ? "onChange" : "onClick"); + if (e.indexOf("on") !== 0) e = "on" + e.charAt(0).toUpperCase() + e.slice(1); + this.__fire(e); + }, + addEventListener: function (name, fn) { + var k = String(name).toLowerCase(); + (this.__listeners[k] = this.__listeners[k] || []).push(fn); + }, + removeEventListener: function (name, fn) { + var k = String(name).toLowerCase(); + if (this.__listeners[k]) this.__listeners[k] = this.__listeners[k].filter(function (f) { return f !== fn; }); + }, + dispatchEvent: function (e) { this.__fire("on" + String(e.type || e).charAt(0).toUpperCase() + String(e.type || e).slice(1)); }, + __fire: function (name) { + var r; + if (typeof this[name] === "function") r = this[name].call(this); + var l = this.__listeners[name.slice(2).toLowerCase()]; + if (l) for (var i = 0; i < l.length; i++) l[i].call(this, { type: name.slice(2).toLowerCase(), target: this }); + return r; + }, + __handlers: function () { + var out = []; + var names = ["onClick", "onChange", "onChanging", "onDoubleClick", "onClose", "onShow", "onActivate", "onDeactivate", "onResize"]; + for (var i = 0; i < names.length; i++) { + var n = names[i]; + if (typeof this[n] === "function" || (this.__listeners[n.slice(2).toLowerCase()] || []).length) out.push(n); + } + return out; + }, + toString: function () { return "[object " + (__uiTypes[this.type] || "Window") + "]"; } + }; +} + +(function () { + var all = ["Window", "Panel", "Group", "Button", "IconButton", "StaticText", "EditText", "Checkbox", "RadioButton", "Slider", + "Scrollbar", "Progressbar", "DropDownList", "ListBox", "TabbedPanel", "Tab", "ScriptUIImage"]; + for (var i = 0; i < all.length; i++) { + var C = __ecGlobal[all[i]]; + var p = __uiProto(); + for (var k in p) C.prototype[k] = p[k]; + Object.defineProperty(C.prototype, "bounds", { + get: function () { var b = this.__ub || this.__lb; return __bounds4(b[0], b[1], b[2], b[3]); }, + set: function (v) { this.__ub = __toBounds(v); }, + configurable: true + }); + Object.defineProperty(C.prototype, "size", { + get: function () { var b = this.bounds; return __dim(b[2] - b[0], b[3] - b[1]); }, + set: function (v) { + var b = this.bounds; + var w = v.width !== undefined ? v.width : v[0], h = v.height !== undefined ? v.height : v[1]; + this.__ub = [b[0], b[1], b[0] + __num(w), b[1] + __num(h)]; + }, + configurable: true + }); + Object.defineProperty(C.prototype, "location", { + get: function () { var b = this.bounds; return __dim(b[0], b[1]); }, + set: function (v) { + var b = this.bounds; + var x = v.x !== undefined ? v.x : v[0], y = v.y !== undefined ? v.y : v[1]; + this.__ub = [__num(x), __num(y), __num(x) + b[2] - b[0], __num(y) + b[3] - b[1]]; + }, + configurable: true + }); + } + var sel = { + get: function () { + var self = this; + if (this.type === "tabbedpanel") return this.__sel.length ? this.children[this.__sel[0]] : null; + var items = this.__sel.map(function (i) { return self.items[i]; }); + if (this.properties.multiselect) return items.length ? items : null; + return items.length ? items[0] : null; + }, + set: function (v) { + var self = this; + var list = this.type === "tabbedpanel" ? this.children : this.items; + var idx = function (x) { return typeof x === "number" ? x : list.indexOf(x); }; + if (v === null || v === undefined) this.__sel = []; + else if (v instanceof Array) this.__sel = v.map(idx).filter(function (i) { return i >= 0 && i < list.length; }); + else { + var i = idx(v); + this.__sel = i >= 0 && i < list.length ? [i] : []; + } + if (this.items) for (var k = 0; k < this.items.length; k++) this.items[k].selected = this.__sel.indexOf(k) >= 0; + }, + configurable: true + }; + Object.defineProperty(DropDownList.prototype, "selection", sel); + Object.defineProperty(ListBox.prototype, "selection", sel); + Object.defineProperty(TabbedPanel.prototype, "selection", sel); + Object.defineProperty(ListItem.prototype, "toString", { value: function () { return this.text; } }); +})(); + +// A dockable ScriptUI panel (scripts in the ScriptUI Panels folder run with `this` = it). +function __uiDockPanel(title) { + var p = Object.create(Panel.prototype); + __uiInit(p, "panel", null, null, undefined, title, {}); + p.type = "panel"; + p.__kind = "panel"; + p.__root = true; + p.__shown = true; + p.margins = [10, 10, 10, 10]; + __uiWins.push(p); + return p; +} + +function __uiAlign(v) { + if (v === null || v === undefined) return []; + if (v instanceof Array) return v.map(function (x) { return String(x).toLowerCase(); }); + return [String(v).toLowerCase()]; +} +function __uiMargins(m) { + if (m === null || m === undefined) return null; + if (typeof m === "number") return [m, m, m, m]; + if (m.length === 4) return [m[0], m[1], m[2], m[3]]; + return null; +} +function __uiSize(d) { + if (!d) return null; + var w = d.width !== undefined ? d.width : d[0], h = d.height !== undefined ? d.height : d[1]; + w = Number(w); + h = Number(h); + if (!(w > 0) && !(h > 0)) return null; + return [w > 0 ? w : 0, h > 0 ? h : 0]; +} + +function __uiJson(c) { + var ps = __uiSize(c.preferredSize); + if (c.type === "edittext" && c.characters > 0) ps = [c.characters * 7 + 12, ps ? ps[1] : 0]; + var o = { + id: c.__id, + type: c.type, + name: String(c.name || c.properties.name || ""), + text: String(c.text === undefined || c.text === null ? "" : c.text), + enabled: c.enabled !== false, + visible: c.visible !== false, + helpTip: String(c.helpTip || ""), + handlers: c.__handlers(), + children: [] + }; + if (ps) o.preferredSize = ps; + if (c.__ub && !c.__root) o.fixedBounds = c.__ub; + if (c.__root && c.__ub) o.fixedBounds = [0, 0, c.__ub[2] - c.__ub[0], c.__ub[3] - c.__ub[1]]; + if (__uiContainers[c.type]) { + o.orientation = String(c.orientation || "").toLowerCase(); + o.alignChildren = __uiAlign(c.alignChildren); + var m = __uiMargins(c.margins); + if (m) o.margins = m; + if (typeof c.spacing === "number") o.spacing = c.spacing; + var kids = c.children; + for (var i = 0; i < kids.length; i++) o.children.push(__uiJson(kids[i])); + } + if (c.alignment !== null && c.alignment !== undefined) o.alignment = __uiAlign(c.alignment); + if (c.type === "checkbox" || c.type === "radiobutton") o.checked = !!c.value; + if (c.type === "slider" || c.type === "scrollbar" || c.type === "progressbar") { + o.value = Number(c.value) || 0; + o.min = Number(c.minvalue) || 0; + o.max = Number(c.maxvalue); + if (!(o.max === o.max)) o.max = 100; + } + if (c.items) { + o.items = c.items.map(function (it) { return String(it.text); }); + o.selection = c.__sel.slice(); + } + if (c.type === "tabbedpanel") o.activeTab = c.__sel.length ? c.__sel[0] : 0; + if (c.type === "edittext") { + o.multiline = !!c.__multiline; + o.readOnly = !!c.__readonly; + } + return o; +} + +// The open windows of this script, for the host. +function __uiSnapshot() { + var out = []; + for (var i = 0; i < __uiWins.length; i++) { + var w = __uiWins[i]; + out.push({ id: w.__wid, kind: w.__kind, title: String(w.text || ""), visible: !!w.__shown && !w.__closed, modal: !!w.__modal, root: __uiJson(w) }); + } + // Closed windows are forgotten. + __uiWins = __uiWins.filter(function (w) { return !w.__closed; }); + return JSON.stringify(out); +} + +function __uiOpenCount() { + var n = 0; + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].__shown && !__uiWins[i].__closed) n++; + return n; +} + +function __uiWin(id) { + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].__wid === id) return __uiWins[i]; + return null; +} +function __uiFind(c, id) { + if (c.__id === id) return c; + for (var i = 0; i < (c.children || []).length; i++) { + var r = __uiFind(c.children[i], id); + if (r) return r; + } + return null; +} + +// layout.layout(): lay the window out now (the host does it before drawing anyway) so scripts +// can read bounds and sizes. +function __uiLayout(win) { + var b = JSON.parse(__uiLayoutNative(JSON.stringify({ id: win.__wid, kind: win.__kind, title: String(win.text || ""), visible: true, root: __uiJson(win) }))); + // Bounds are relative to the parent container, as in ScriptUI. + (function apply(c, ox, oy) { + var r = b[c.__id]; + if (!r) return; + c.__lb = [r[0] - ox, r[1] - oy, r[0] - ox + r[2], r[1] - oy + r[3]]; + for (var i = 0; i < (c.children || []).length; i++) apply(c.children[i], r[0], r[1]); + })(win, 0, 0); +} + +// A user action from the frontend. +function __uiDispatch(winId, id, kind, valueJson) { + var w = __uiWin(winId); + if (!w) return; + var v = valueJson === undefined || valueJson === "" ? null : JSON.parse(valueJson); + if (kind === "close") { + w.close(v === null ? 2 : v); + return; + } + var c = __uiFind(w, id); + if (!c || c.enabled === false) return; + var radio = function () { + var sib = c.parent ? c.parent.children : []; + for (var i = 0; i < sib.length; i++) if (sib[i] !== c && sib[i].type === "radiobutton") sib[i].value = false; + c.value = true; + }; + if (kind === "click") { + if (c.type === "checkbox") c.value = !c.value; + else if (c.type === "radiobutton") radio(); + else if (c.type === "tab" && c.parent) { + c.parent.selection = c; + c.parent.__fire("onChange"); + return; + } + var handled = c.__handlers().indexOf("onClick") >= 0; + c.__fire("onClick"); + // Dialog buttons without handlers: OK closes with 1, Cancel with 2. + if (!handled && c.type === "button" && w.__kind === "dialog") { + var n = String(c.name || c.properties.name || c.text).toLowerCase(); + if (c === w.defaultElement || n === "ok") w.close(1); + else if (c === w.cancelElement || n === "cancel") w.close(2); + } + return; + } + // change / changing + if (c.type === "edittext") c.text = v === null ? "" : String(v); + else if (c.type === "statictext") c.text = v === null ? "" : String(v); + else if (c.type === "slider" || c.type === "scrollbar" || c.type === "progressbar") c.value = Number(v); + else if (c.type === "checkbox") c.value = !!v; + else if (c.type === "radiobutton") { if (v) radio(); else c.value = false; } + else if (c.type === "dropdownlist" || c.type === "listbox" || c.type === "tabbedpanel") c.selection = v; + if (c.type === "edittext" || c.type === "slider" || c.type === "scrollbar") c.__fire("onChanging"); + if (kind === "change") { + if (c.type === "checkbox" || c.type === "radiobutton") c.__fire("onClick"); + else c.__fire("onChange"); + } +} diff --git a/crates/script/src/tests_ui.rs b/crates/script/src/tests_ui.rs new file mode 100644 index 00000000..667a3242 --- /dev/null +++ b/crates/script/src/tests_ui.rs @@ -0,0 +1,299 @@ +//! ScriptUI: windows built by scripts are published to the session, drawn by frontends and +//! driven through the `scriptui.*` commands; dialogs block `show()` until they close; dockable +//! panels; File ▸ Scripts (installed and sample scripts). + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::scriptui::{WidgetKind, WindowKind}; +use serde_json::{Value, json}; + +use crate::run_code; + +fn session() -> Session { + let mut s = Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), ..Default::default() }; + crate::install(&mut s); + s.config = Some(Arc::new(effectcraft_engine::config::MemoryConfig::default())); + s +} + +fn comp_names(s: &Session) -> Vec { + s.project.items.values().filter(|i| i.as_comp().is_some()).map(|i| i.name.clone()).collect() +} + +fn layer_names(s: &Session) -> Vec { + s.active_comp().map(|c| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +const PALETTE: &str = r#" +var made = 0; +var w = new Window("palette", "Maker"); +w.orientation = "column"; +w.alignChildren = ["fill", "top"]; +var name = w.add("edittext", undefined, "Box"); +name.characters = 12; +var big = w.add("checkbox", undefined, "Big"); +var row = w.add("group"); +var make = row.add("button", undefined, "Make", { name: "make" }); +var label = row.add("statictext", undefined, "none yet"); +var size = w.add("slider", undefined, 50, 10, 200); +var kind = w.add("dropdownlist", undefined, ["Solid", "Null"]); +kind.selection = 0; +var echo = ""; +name.onChange = function () { echo = name.text; }; +make.onClick = function () { + var c = app.project.activeItem; + var s = big.value ? 200 : Math.round(size.value); + if (kind.selection.index === 0) c.layers.addSolid([1, 0, 0], name.text, s, s, 1); + else c.layers.addNull().name = name.text; + made++; + label.text = "made " + made; + writeLn("made " + name.text + " " + s); +}; +w.show(); +"palette up" +"#; + +#[test] +fn palette_controls_run_their_handlers() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 2})).unwrap(); + let o = run_code(&mut s, PALETTE, "maker.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(o.result, json!("palette up")); + assert_eq!(s.script_ui.windows.len(), 1); + let w = s.script_ui.windows[0].clone(); + assert_eq!((w.kind, w.title.as_str(), w.script.as_str()), (WindowKind::Palette, "Maker", "maker.jsx")); + let kinds: Vec = w.root.children.iter().map(|c| c.kind).collect(); + assert_eq!(kinds, [WidgetKind::EditText, WidgetKind::Checkbox, WidgetKind::Group, WidgetKind::Slider, WidgetKind::DropDownList]); + // Laid out: the window has a size, the fill-aligned controls span its width. + assert!(w.root.bounds[2] > 100.0 && w.root.bounds[3] > 100.0, "{:?}", w.root.bounds); + let edit = &w.root.children[0]; + assert_eq!(edit.bounds[2], w.root.bounds[2] - 30.0); + assert_eq!(w.root.children[3].value, 50.0); + assert_eq!((w.root.children[3].min, w.root.children[3].max), (10.0, 200.0)); + assert_eq!(w.root.children[4].items, ["Solid", "Null"]); + assert_eq!(w.root.children[4].selection, [0]); + assert!(w.root.children[2].children[0].handlers.contains(&"onClick".to_string())); + // Agents: list, read, click. + let list = s.execute("scriptui.list", json!({})).unwrap(); + assert_eq!(list[0]["title"], "Maker"); + let tree = s.execute("scriptui.get", json!({"window": w.id})).unwrap(); + assert_eq!(tree["root"]["children"][2]["children"][0]["name"], "make"); + let r = s.execute("scriptui.click", json!({"widget": "make"})).unwrap(); + assert_eq!(r["output"], "made Box 50", "{r}"); + assert_eq!(layer_names(&s), ["Box"]); + // Handlers' edits are undoable engine commands. + assert!(s.undo()); + assert!(layer_names(&s).is_empty()); + // Set values like a user: text (onChange), checkbox, slider, list by item text. + s.execute("scriptui.set", json!({"widget": "#1", "value": "Crate"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "Big"})).unwrap(); + s.execute("scriptui.set", json!({"window": "Maker", "widget": 7, "value": "Null"})).unwrap(); + let r = s.execute("scriptui.click", json!({"widget": "make"})).unwrap(); + assert_eq!(r["output"], "made Crate 200"); + assert_eq!(layer_names(&s), ["Crate"]); + assert!(s.active_comp().unwrap().layers[0].source == effectcraft_engine::project::LayerSource::Null); + // The handler updated a label: the published tree follows. + let w = s.script_ui.windows[0].clone(); + assert_eq!(w.root.children[2].children[1].text, "made 2"); + assert!(w.root.children[1].checked); + assert_eq!(w.root.children[4].selection, [1]); + // Errors. + assert!(s.execute("scriptui.click", json!({"widget": "nope"})).is_err()); + assert!(s.execute("scriptui.get", json!({"window": 999_999})).is_err()); + // Closing the window ends the script host. + s.execute("scriptui.close", json!({})).unwrap(); + assert!(s.script_ui.windows.is_empty()); + assert!(s.execute("scriptui.list", json!({})).unwrap().as_array().unwrap().is_empty()); +} + +const DIALOG: &str = r#" +var d = new Window("dialog", "New Comp"); +d.alignChildren = "left"; +var e = d.add("edittext", undefined, "Untitled"); +var fps = d.add("radiobutton", undefined, "24 fps"); +var fps30 = d.add("radiobutton", undefined, "30 fps"); +fps.value = true; +var g = d.add("group"); +g.add("button", undefined, "Cancel", { name: "cancel" }); +var ok = g.add("button", undefined, "OK", { name: "ok" }); +writeLn("before"); +var r = d.show(); +writeLn("result " + r + " " + e.text + " " + (fps30.value ? 30 : 24)); +if (r === 1) app.project.items.addComp(e.text, 100, 100, 1, 1, fps30.value ? 30 : 24); +r +"#; + +#[test] +fn dialogs_block_show_until_they_close() { + let mut s = session(); + let o = run_code(&mut s, DIALOG, "dialog.jsx"); + // The script waits in show(); the session is back here meanwhile. + assert!(o.waiting, "{o:?}"); + assert_eq!(o.output, ["before"]); + assert_eq!(s.script_ui.windows.len(), 1); + let w = s.script_ui.windows[0].clone(); + assert!(w.modal && w.kind == WindowKind::Dialog); + // The app keeps working while the dialog is up. + s.execute("comp.new", json!({"name": "Other", "width": 64, "height": 64, "duration": 1})).unwrap(); + s.execute("scriptui.set", json!({"widget": "#1", "value": "Shot 010"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "30 fps"})).unwrap(); + let w = s.script_ui.windows[0].clone(); + assert!(!w.root.children[1].checked && w.root.children[2].checked, "radio buttons are exclusive"); + // OK closes the dialog with 1: the script runs on. + let r = s.execute("scriptui.click", json!({"widget": "ok"})).unwrap(); + assert_eq!(r["output"], "result 1 Shot 010 30", "{r}"); + assert_eq!(r["result"], 1); + assert!(r.get("waiting").is_none()); + assert!(s.script_ui.windows.is_empty()); + let names = comp_names(&s); + assert!(names.contains(&"Shot 010".to_string()) && names.contains(&"Other".to_string()), "{names:?}"); + // Cancel (or closing the window) returns 2. + let o = run_code(&mut s, DIALOG, "dialog.jsx"); + assert!(o.waiting); + let r = s.execute("scriptui.close", json!({})).unwrap(); + assert_eq!(r["output"], "result 2 Untitled 24"); + assert_eq!(comp_names(&s).len(), 2); +} + +#[test] +fn layout_and_handler_errors() { + let mut s = session(); + let code = r#" + var w = new Window("palette", "L"); + var g = w.add("group"); + g.orientation = "row"; + var a = g.add("button", undefined, "A"); + var b = g.add("button", undefined, "B"); + b.preferredSize = [120, 30]; + var p = w.add("panel", undefined, "Options"); + p.add("checkbox", undefined, "One"); + var bad = w.add("button", undefined, "Bad"); + bad.onClick = function () { nope(); }; + w.layout.layout(true); + w.show(); + [a.bounds.width, b.size.width, b.size.height, b.bounds[0] - a.bounds[0], w.bounds.width > 200, p.bounds.top > a.bounds.bottom] + "#; + let o = run_code(&mut s, code, "layout.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(o.result, json!([80, 120, 30, 90, true, true])); + // An error in a handler is reported, the window stays. + let r = s.execute("scriptui.click", json!({"widget": "Bad"})).unwrap(); + assert_eq!(r["ok"], false); + assert!(r["error"]["message"].as_str().unwrap().contains("nope"), "{r}"); + assert_eq!(s.script_ui.windows.len(), 1); + // Resource strings aren't supported (a clear error). + let o = run_code(&mut s, "new Window(\"dialog { text: 'x' }\")", "res.jsx"); + assert!(o.error.unwrap().message.contains("resource")); + s.execute("scriptui.close", json!({})).unwrap(); +} + +#[test] +fn console_windows_run_inline() { + let mut s = session(); + let req = effectcraft_engine::ScriptRequest { + code: "var cw = new Window('palette', 'Console UI'); var cb = cw.add('button', undefined, 'Hi'); var hits = 0; cb.onClick = function () { hits++; writeLn('hit ' + hits); }; cw.show(); 1", + name: "Script Console", + console: true, + }; + let o = crate::run(&mut s, &req); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(s.script_ui.windows.len(), 1); + let r = s.execute("scriptui.click", json!({"widget": "Hi"})).unwrap(); + assert_eq!(r["output"], "hit 1"); + // The console context sees the handler's state. + let o = crate::run(&mut s, &effectcraft_engine::ScriptRequest { code: "hits", name: "Script Console", console: true }); + assert_eq!(o.result, json!(1)); + // Dialogs don't block in the console. + let o = crate::run( + &mut s, + &effectcraft_engine::ScriptRequest { code: "var dd = new Window('dialog', 'D'); dd.show(); 'went on'", name: "Script Console", console: true }, + ); + assert_eq!(o.result, json!("went on")); + assert_eq!(s.script_ui.windows.len(), 2); + s.execute("scriptui.close", json!({"window": "D"})).unwrap(); + s.execute("scriptui.close", json!({"window": "Console UI"})).unwrap(); + assert!(s.script_ui.windows.is_empty()); +} + +fn solid(s: &mut Session, name: &str, at: f64) -> u64 { + let id = s.execute("layer.newSolid", json!({"name": name, "color": "#808080", "width": 20, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.timing", json!({"layers": [id], "in": at})).unwrap(); + id +} + +#[test] +fn sample_scripts_and_installing() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Main", "width": 400, "height": 200, "duration": 10})).unwrap(); + for (n, at) in [("C", 2.0), ("A", 0.0), ("B", 1.0)] { + solid(&mut s, n, at); + } + // File ▸ Scripts lists the samples. + let list = s.execute("file.scripts.list", json!({})).unwrap(); + let names: Vec<&str> = list.as_array().unwrap().iter().map(|e| e["name"].as_str().unwrap()).collect(); + for want in ["Rename Layers.jsx", "Sort Layers by In Point.jsx", "Create Null at Selected Layers.jsx", "Render Queue Batch.jsx", "Layer Tools.jsx"] { + assert!(names.contains(&want), "{names:?}"); + } + // Sort Layers by In Point (all layers when none are selected). + s.execute("edit.deselectAll", json!({})).unwrap(); + s.execute("file.runScript", json!({"name": "Sort Layers by In Point.jsx"})).unwrap(); + assert_eq!(layer_names(&s), ["A", "B", "C"]); + assert!(s.undo()); + assert_eq!(layer_names(&s), ["B", "A", "C"]); + // Create Null at Selected Layers. + let ids: Vec = s.active_comp().unwrap().layers.iter().take(2).map(|l| l.id.0).collect(); + s.execute("layer.select", json!({"layers": ids})).unwrap(); + s.execute("file.runScript", json!({"name": "Create Null at Selected Layers.jsx"})).unwrap(); + let c = s.active_comp().unwrap(); + let null = c.layers.iter().find(|l| l.name == "Centroid").unwrap(); + assert!(c.layers.iter().filter(|l| ids.contains(&l.id.0)).all(|l| l.parent == Some(null.id))); + // Rename Layers: a dialog; fill it in and press OK. + s.execute("layer.select", json!({"layers": ids})).unwrap(); + let r = s.execute("file.runScript", json!({"name": "Rename Layers.jsx"})).unwrap(); + assert_eq!(r["waiting"], true); + s.execute("scriptui.set", json!({"widget": "#11", "value": "Shot_"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "Number the layers"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "ok"})).unwrap(); + let names = layer_names(&s); + assert!(names.contains(&"Shot_B 1".to_string()) && names.contains(&"Shot_A 2".to_string()), "{names:?}"); + // Install a script and a ScriptUI panel (they live in the settings store). + let dir = std::env::temp_dir().join(format!("ec-scripts-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("Hello.jsx"); + std::fs::write(&file, "writeLn('hello from ' + $.fileName); 42").unwrap(); + s.execute("file.installScript", json!({"path": file.to_string_lossy()})).unwrap(); + let panel = dir.join("Mini Panel.jsx"); + std::fs::write(&panel, "var p = this; p.add('button', undefined, 'Go').onClick = function () { writeLn('go'); }; p.layout.layout(true);").unwrap(); + s.execute("file.installScriptUIPanel", json!({"path": panel.to_string_lossy()})).unwrap(); + let list = s.execute("file.scripts.list", json!({})).unwrap(); + assert!(list.as_array().unwrap().iter().any(|e| e["name"] == "Hello.jsx" && e["source"] == "installed" && e["panel"] == false)); + assert!(list.as_array().unwrap().iter().any(|e| e["name"] == "Mini Panel.jsx" && e["panel"] == true)); + let r = s.execute("file.runScript", json!({"name": "Hello.jsx"})).unwrap(); + assert_eq!(r["result"], 42); + // Window ▸ Mini Panel.jsx: a dockable panel; `this` is the panel. + let r = s.execute("window.scriptPanel", json!({"name": "Mini Panel.jsx"})).unwrap(); + let wid = r["window"].as_u64().unwrap(); + let w = s.script_ui.window(wid as u32).unwrap().clone(); + assert_eq!((w.kind, w.title.as_str()), (WindowKind::Panel, "Mini Panel")); + let r = s.execute("scriptui.click", json!({"window": wid, "widget": "Go"})).unwrap(); + assert_eq!(r["output"], "go"); + // Opening it again brings the same panel forward. + let r = s.execute("window.scriptPanel", json!({"name": "Mini Panel.jsx"})).unwrap(); + assert_eq!(r["window"], wid); + // The sample panel docks too, and works. + let r = s.execute("window.scriptPanel", json!({"name": "Layer Tools.jsx"})).unwrap(); + let tools = r["window"].as_u64().unwrap(); + let before = s.active_comp().unwrap().layers.len(); + s.execute("layer.select", json!({"layers": ids})).unwrap(); + s.execute("scriptui.click", json!({"window": tools, "widget": "Null at Centroid"})).unwrap(); + assert_eq!(s.active_comp().unwrap().layers.len(), before + 1); + s.execute("scriptui.close", json!({"window": tools})).unwrap(); + s.execute("scriptui.close", json!({"window": wid})).unwrap(); + assert!(s.execute("file.installScript", json!({"path": "/nope/x.txt"})).is_err()); + s.execute("file.uninstallScript", json!({"name": "Hello.jsx"})).unwrap(); + assert!(s.execute("file.runScript", json!({"name": "Hello.jsx"})).is_err()); + let _ = std::fs::remove_dir_all(&dir); + let _: Value = json!(null); +} diff --git a/crates/script/src/ui.rs b/crates/script/src/ui.rs new file mode 100644 index 00000000..66574ce0 --- /dev/null +++ b/crates/script/src/ui.rs @@ -0,0 +1,339 @@ +//! ScriptUI hosts: who owns a script window's JavaScript objects, and how user actions reach +//! its handlers. +//! +//! * **Script files** (native builds) run on a *script-host thread*. The caller's session moves +//! to that thread while JavaScript runs and comes back whenever the script stops: at the end, +//! or while a modal dialog waits in `show()`. Each [`ScriptUiEvent`] moves the session there +//! again, runs the handler, and brings it back with the windows' new state. The host ends +//! when its last window closes. Only one side holds the session at a time, so nothing runs +//! concurrently and every edit is still an ordinary undoable engine command. +//! * **The Script Console** (and every run on the web) runs inline: its windows' handlers run in +//! the console's (or a kept) context on the caller's thread, and dialogs don't block. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::mem::ManuallyDrop; +use std::sync::atomic::{AtomicU32, Ordering}; + +use boa_engine::{Context, JsResult, JsValue, js_string}; +use effectcraft_engine::Session; +use effectcraft_engine::scriptui::{ScriptUiEvent, ScriptWindow, layout}; +use serde_json::{Value as J, json}; + +use crate::runtime::{ACTIVE, Active, CONSOLE, Restore, ScriptError, arg_string, js_str, outcome, script_error, settle, throw, with_active}; + +pub(crate) const SCRIPTUI: &str = include_str!("scriptui.js"); + +/// The Script Console's host id. +pub(crate) const CONSOLE_HOST: u32 = u32::MAX; + +static NEXT_WINDOW: AtomicU32 = AtomicU32::new(1); +static NEXT_HOST: AtomicU32 = AtomicU32::new(1); + +pub(crate) fn new_host_id() -> u32 { + NEXT_HOST.fetch_add(1, Ordering::Relaxed) +} + +thread_local! { + /// Contexts of inline runs whose palettes / panels are still open (dropped when they close). + static INLINE: RefCell>> = const { RefCell::new(BTreeMap::new()) }; +} + +pub(crate) fn keep_inline(host: u32, ctx: Context) { + INLINE.with(|m| m.borrow_mut().insert(host, ManuallyDrop::new(ctx))); +} + +fn drop_inline(host: u32) { + if let Some(ctx) = INLINE.with(|m| m.borrow_mut().remove(&host)) { + drop(ManuallyDrop::into_inner(ctx)); + } +} + +// ---------------------------------------------------------------- JavaScript side + +fn call_global(ctx: &mut Context, name: &str, args: &[JsValue]) -> JsResult { + let f = ctx.global_object().get(js_string!(name), ctx)?; + let f = f.as_object().ok_or_else(|| throw(format!("{name} is not defined")))?; + f.call(&JsValue::undefined(), args, ctx) +} + +/// The open windows of a context (closed ones are reported once, invisible, then forgotten). +pub(crate) fn snapshot(ctx: &mut Context) -> Vec { + let Ok(v) = call_global(ctx, "__uiSnapshot", &[]) else { return vec![] }; + let s = v.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + serde_json::from_str(&s).unwrap_or_default() +} + +/// Run a window event's handlers in `ctx`. +fn dispatch_js(ctx: &mut Context, ev: &ScriptUiEvent) -> Result { + let args = [JsValue::from(ev.window), JsValue::from(ev.widget), js_str(&ev.kind), js_str(&ev.value.to_string())]; + match call_global(ctx, "__uiDispatch", &args) { + Ok(_) => Ok(J::Null), + Err(e) => { + let mut err = script_error(&e, ctx, "ScriptUI handler", ""); + err.message = format!("{} handler: {}", ev.kind, err.message); + Err(err) + } + } +} + +/// `__uiNewId()`: a window id unique in the process. +pub(crate) fn native_new_id(_: &JsValue, _: &[JsValue], _: &mut Context) -> JsResult { + Ok(JsValue::from(NEXT_WINDOW.fetch_add(1, Ordering::Relaxed))) +} + +/// `__uiLayoutNative(windowJson)` → `{controlId: [x, y, width, height]}`: ScriptUI's automatic +/// layout ([`effectcraft_engine::scriptui::layout`]) for `layout.layout()`. +pub(crate) fn native_layout(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let s = arg_string(args, 0, ctx)?; + let mut w: ScriptWindow = serde_json::from_str(&s).map_err(|e| throw(format!("layout: {e}")))?; + layout(&mut w); + let mut all = vec![]; + w.root.walk(&mut all); + let map: serde_json::Map = all.iter().map(|c| (c.id.to_string(), json!(c.bounds))).collect(); + Ok(js_str(&J::Object(map).to_string())) +} + +/// `__uiModal(windowId)`: a dialog's `show()`. On a script-host thread it hands the session back +/// to the caller and waits for the dialog's events until it closes, returning its result. Inline +/// (Script Console, web) it returns `null` at once: the dialog stays open like a palette. +pub(crate) fn native_modal(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let id = args.first().cloned().unwrap_or_default().to_number(ctx)? as u32; + let linked = with_active(|a| Ok(a.link.is_some())).unwrap_or(false); + if !linked { + return Ok(JsValue::null()); + } + loop { + let r = call_global(ctx, "__uiModalResult", &[JsValue::from(id)])?; + if !r.is_null() && !r.is_undefined() { + return Ok(r); + } + let windows = snapshot(ctx); + let out = crate::Outcome { output: settle(), waiting: true, ..Default::default() }; + let Some(ev) = yield_wait(out, windows, false) else { + // The caller went away: cancel the dialog. + return Ok(JsValue::from(2)); + }; + if let Err(e) = dispatch_js(ctx, &ev) { + let line = e.line.map(|l| format!(" (line {l})")).unwrap_or_default(); + let _ = with_active(|a| { + a.output.push(format!("Error{line}: {}", e.message)); + Ok(()) + }); + } + } +} + +// ---------------------------------------------------------------- script-host threads + +pub(crate) enum ToHost { + Start(Session), + Event(Session, ScriptUiEvent), +} + +pub(crate) struct FromHost { + session: Session, + outcome: crate::Outcome, + windows: Vec, + name: String, + /// The host thread has ended (no windows left). + done: bool, +} + +/// The host thread's end of the hand-off. +pub(crate) struct Link { + rx: std::sync::mpsc::Receiver, + tx: std::sync::mpsc::Sender, +} + +/// The caller's end. +struct HostEnd { + tx: std::sync::mpsc::Sender, + rx: std::sync::mpsc::Receiver, +} + +fn hosts() -> &'static std::sync::Mutex> { + static H: std::sync::OnceLock>> = std::sync::OnceLock::new(); + H.get_or_init(Default::default) +} + +// The session crosses to script-host threads and back. +const _: fn() = || { + fn send() {} + send::(); +}; + +/// On the host thread: hand the session (with the windows' state and what the script printed) +/// back to the caller, and wait for the next event. `None` when the caller has gone. +fn yield_wait(outcome: crate::Outcome, windows: Vec, done: bool) -> Option { + let (session, link, name) = ACTIVE.with(|a| { + let mut a = a.borrow_mut(); + let a = a.as_mut()?; + Some((std::mem::take(&mut a.session), a.link.take()?, a.name.clone())) + })?; + if link.tx.send(FromHost { session, outcome, windows, name, done }).is_err() { + return None; + } + if done { + return None; + } + match link.rx.recv() { + Ok(ToHost::Event(s, ev)) => { + ACTIVE.with(|a| { + if let Some(a) = a.borrow_mut().as_mut() { + a.session = s; + a.link = Some(link); + } + }); + Some(ev) + } + _ => None, + } +} + +fn host_main(name: String, code: String, link: Link) { + let Ok(ToHost::Start(session)) = link.rx.recv() else { return }; + let mut a = Active::new(session, &name); + a.link = Some(link); + ACTIVE.with(|c| *c.borrow_mut() = Some(a)); + let file = if name.is_empty() { "script".to_string() } else { name.clone() }; + let body = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut ctx = match crate::runtime::new_context() { + Ok(c) => c, + Err(e) => { + yield_wait(outcome(settle(), Err(ScriptError { message: e, file: file.clone(), ..Default::default() })), vec![], true); + return; + } + }; + let r = crate::runtime::eval_with_tasks(&mut ctx, &code, &file); + let mut out = outcome(settle(), r); + loop { + let windows = snapshot(&mut ctx); + let open = windows.iter().any(|w| w.visible); + match yield_wait(out, windows, !open) { + Some(ev) => { + let r = dispatch_js(&mut ctx, &ev); + out = outcome(settle(), r); + } + None => return, + } + } + })); + // A panic: give the session back all the same. + let left = ACTIVE.with(|c| c.borrow_mut().take()); + if let Some(mut a) = left + && let Some(link) = a.link.take() + { + let message = if body.is_err() { "the script engine stopped unexpectedly" } else { "the script ended" }; + let out = crate::Outcome { error: Some(ScriptError { message: message.into(), file, ..Default::default() }), ..Default::default() }; + let _ = link.tx.send(FromHost { session: std::mem::take(&mut a.session), outcome: out, windows: vec![], name, done: true }); + } +} + +/// Wait for the host to hand the session back; keep its link while it has windows open. +fn receive(session: &mut Session, host: u32, end: HostEnd) -> crate::Outcome { + match end.rx.recv() { + Ok(FromHost { session: s, outcome, windows, name, done }) => { + *session = s; + publish(session, host, &name, windows); + if done { + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + } else if let Ok(mut h) = hosts().lock() { + h.insert(host, end); + } + outcome + } + Err(_) => crate::Outcome { error: Some(ScriptError { message: "the script host stopped".into(), ..Default::default() }), ..Default::default() }, + } +} + +/// Run a script file on a new script-host thread (see the module docs). +pub(crate) fn run_threaded(session: &mut Session, req: &effectcraft_engine::ScriptRequest) -> crate::Outcome { + let host = new_host_id(); + let (to_tx, to_rx) = std::sync::mpsc::channel(); + let (from_tx, from_rx) = std::sync::mpsc::channel(); + let (name, code) = (req.name.to_string(), req.code.to_string()); + let spawned = + std::thread::Builder::new().name(format!("script-{host}")).stack_size(64 << 20).spawn(move || host_main(name, code, Link { rx: to_rx, tx: from_tx })); + if spawned.is_err() { + return crate::runtime::run_inline(session, req); + } + if let Err(std::sync::mpsc::SendError(ToHost::Start(s) | ToHost::Event(s, _))) = to_tx.send(ToHost::Start(std::mem::take(session))) { + *session = s; + return crate::Outcome { error: Some(ScriptError { message: "the script host did not start".into(), ..Default::default() }), ..Default::default() }; + } + receive(session, host, HostEnd { tx: to_tx, rx: from_rx }) +} + +/// Record the windows a host published (named after the script that made them). +pub(crate) fn publish(session: &mut Session, host: u32, name: &str, mut windows: Vec) { + for w in &mut windows { + if w.script.is_empty() { + w.script = name.to_string(); + } + } + let had = session.script_ui.windows.iter().any(|w| w.host == host); + if windows.is_empty() && !had { + return; + } + session.script_ui.publish(host, windows); + if host != CONSOLE_HOST && !session.script_ui.windows.iter().any(|w| w.host == host) { + drop_inline(host); + } +} + +/// The [`effectcraft_engine::scriptui::ScriptUiDispatch`] this crate provides: run a window +/// event's handlers in the script that made the window, then publish its windows' new state. +/// Returns the handler run's `{ok, output, error, waiting?}`. +pub fn dispatch_ui(session: &mut Session, ev: &ScriptUiEvent) -> Result { + if ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)) { + return Err("scripts can't drive script windows".into()); + } + let w = session.script_ui.window(ev.window).ok_or_else(|| format!("no script window {}", ev.window))?; + let (host, name) = (w.host, w.script.clone()); + let inline = host == CONSOLE_HOST || INLINE.with(|m| m.borrow().contains_key(&host)); + if inline { + return Ok(dispatch_inline(session, host, &name, ev).to_json()); + } + let end = hosts().lock().ok().and_then(|mut h| h.remove(&host)); + let Some(end) = end else { + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + return Err("the script that made this window has ended".into()); + }; + if let Err(std::sync::mpsc::SendError(ToHost::Event(s, _) | ToHost::Start(s))) = end.tx.send(ToHost::Event(std::mem::take(session), ev.clone())) { + *session = s; + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + return Err("the script that made this window has ended".into()); + } + Ok(receive(session, host, end).to_json()) +} + +fn dispatch_inline(session: &mut Session, host: u32, name: &str, ev: &ScriptUiEvent) -> crate::Outcome { + let s = std::mem::take(session); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active::new(s, name))); + let guard = Restore { target: session }; + let (r, windows) = if host == CONSOLE_HOST { + CONSOLE.with(|c| match c.borrow_mut().as_mut() { + Some(ctx) => { + let r = dispatch_js(ctx, ev); + (r, snapshot(ctx)) + } + None => (Ok(J::Null), vec![]), + }) + } else { + INLINE.with(|m| match m.borrow_mut().get_mut(&host) { + Some(ctx) => { + let r = dispatch_js(ctx, ev); + (r, snapshot(ctx)) + } + None => (Ok(J::Null), vec![]), + }) + }; + let output = settle(); + drop(guard); + publish(session, host, name, windows); + outcome(output, r) +} diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 72cb2f17..077f04b7 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -10,6 +10,7 @@ use rayon::prelude::*; +use super::Distortion; use super::geometry::Pose; use super::linalg::*; @@ -19,6 +20,8 @@ pub struct BaCam { pub pose: Pose, /// Focal length (pixels). pub f: f64, + /// Lens distortion (shared by all cameras; adjusted when [`BaOpts::distortion`]). + pub dist: Distortion, } /// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal @@ -50,11 +53,13 @@ pub struct BaOpts { /// Huber threshold (pixels). pub huber: f64, pub max_iter: usize, + /// Adjust the shared radial distortion `(k1, k2)` too. + pub distortion: bool, } impl Default for BaOpts { fn default() -> Self { - BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30, distortion: false } } } @@ -77,7 +82,8 @@ pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { if p[2] <= 1e-9 { return None; } - Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) + let q = c.dist.distort([c.f * p[0] / p[2], c.f * p[1] / p[2]]); + Some([q[0] - uv[0], q[1] - uv[1]]) } /// Total robust cost. @@ -108,6 +114,8 @@ struct Layout { fidx: Vec, /// Index of the shared focal parameter. shared: Option, + /// Index of `k1` (then `k2`) when the distortion is adjusted. + dist: Option, n: usize, rot_only: bool, } @@ -132,7 +140,11 @@ fn layout(ncam: usize, o: &BaOpts) -> Layout { n += 1; n - 1 }); - Layout { start, fidx, shared, n, rot_only: o.fix_centers } + let dist = o.distortion.then(|| { + n += 2; + n - 2 + }); + Layout { start, fidx, shared, dist, n, rot_only: o.fix_centers } } /// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). @@ -204,12 +216,21 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> continue; } let iz = 1.0 / xc[2]; - let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; - let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let p0 = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let pd = c.dist.distort(p0); + let r = [pd[0] - ob.uv[0], pd[1] - ob.uv[1]]; let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); let w = if e <= o.huber { 1.0 } else { o.huber / e }; - let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + let a0 = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + // Through the distortion: d r = D d p. + let (dj, dk) = c.dist.jacobian(p0); + let a = [0, 1].map(|i| [0, 1, 2].map(|k| dj[i][0] * a0[0][k] + dj[i][1] * a0[1][k])); + let p = [dj[0][0] * p0[0] + dj[0][1] * p0[1], dj[1][0] * p0[0] + dj[1][1] * p0[1]]; cam_jac(&l, ci, c, xc, &a, p, &mut jc); + if let Some(g) = l.dist { + jc.push((g, [dk[0][0], dk[1][0]])); + jc.push((g + 1, [dk[0][1], dk[1][1]])); + } // Point Jacobian: A R. let jx = if o.fix_points { [[0.0; 3]; 2] @@ -236,7 +257,8 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; // Each observation of a point is on another camera: only the shared focal // length repeats. - let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + let repeats = Some(*ia) == l.shared || l.dist.is_some_and(|g| *ia == g || *ia == g + 1); + let found = if repeats { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), @@ -385,6 +407,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> if let Some(g) = l.shared { c.f *= dc[g].clamp(-0.5, 0.5).exp(); } + if let Some(g) = l.dist { + c.dist.k1 += dc[g].clamp(-0.2, 0.2); + c.dist.k2 += dc[g + 1].clamp(-0.2, 0.2); + } } let mut np: Vec = pts.to_vec(); if !o.fix_points { @@ -445,7 +471,11 @@ mod tests { let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); let f = 900.0; let cams_true: Vec = (0..6) - .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .map(|i| BaCam { + pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, + f, + dist: Distortion::NONE, + }) .collect(); let mut obs = vec![]; for (ci, c) in cams_true.iter().enumerate() { diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 30b019e5..1775a304 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -28,11 +28,19 @@ pub struct TrackAnalyzer { frame: u32, size: [f64; 2], factor: f64, + /// (analysis long side, wanted features) overriding the options' defaults. + budget: Option<(u32, usize)>, } impl TrackAnalyzer { pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { - TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0, budget: None } + } + + /// An analyzer working at most at `max_side` pixels with about `want` live features (the + /// Warp Stabilizer's trajectories for Subspace Warp). + pub fn with_budget(size: [f64; 2], max_side: u32, want: usize) -> TrackAnalyzer { + TrackAnalyzer { budget: Some((max_side, want.max(8))), ..TrackAnalyzer::new(size, AnalyzeOpts::default()) } } pub fn frames(&self) -> u32 { @@ -45,6 +53,9 @@ impl TrackAnalyzer { } fn params(&self) -> (u32, usize) { + if let Some(b) = self.budget { + return b; + } if self.opts.detailed { (1280, 900) } else { (720, 500) } } diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 76c0637f..e8cd28da 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -28,8 +28,15 @@ //! tripod solution when it explains the tracks as well as the general one (a quick rotation fit //! at the general solution's focal length skips the tripod search when it clearly fails). //! -//! The camera model has square pixels, the principal point at the layer centre and no lens -//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! 5. **Lens distortion** (Solve Lens Distortion, or Detailed Analysis): radial distortion +//! `p_d = p (1 + k1 ρ² + k2 ρ⁴)` (ρ = distance from the principal point over half the frame +//! diagonal; Brown's model without the tangential terms) is solved jointly with the cameras +//! and points in a final bundle adjustment on the raw tracks; the whole solve is then repeated +//! on tracks undistorted with that estimate (the linear initialisations assume a pinhole +//! camera) and adjusted jointly again. The solve stores `k1, k2`; the effect can undistort +//! its output so composited 3D layers line up. +//! +//! The camera model has square pixels and the principal point at the layer centre. Camera space is x right, y down, z forward (After Effects' convention); a camera //! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. //! //! The solve is stored in a canonical frame: the first solved frame's camera is at the origin @@ -105,6 +112,69 @@ impl CameraTracks { } } +/// Radial lens distortion around the principal point (see the module docs). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Distortion { + pub k1: f64, + pub k2: f64, + /// Normalisation radius ρ = 1 (pixels): half the frame diagonal; 0 = no distortion. + pub radius: f64, +} + +impl Distortion { + pub const NONE: Distortion = Distortion { k1: 0.0, k2: 0.0, radius: 0.0 }; + + /// No distortion for frames of `size` (the radius set, coefficients zero). + pub fn for_size(size: [f64; 2]) -> Distortion { + Distortion { k1: 0.0, k2: 0.0, radius: 0.5 * size[0].hypot(size[1]) } + } + pub fn is_none(&self) -> bool { + self.radius <= 0.0 || (self.k1 == 0.0 && self.k2 == 0.0) + } + fn rho2(&self, p: [f64; 2]) -> f64 { + (p[0] * p[0] + p[1] * p[1]) / (self.radius * self.radius) + } + /// Ideal → distorted (pixels relative to the principal point). + pub fn distort(&self, p: [f64; 2]) -> [f64; 2] { + if self.radius <= 0.0 { + return p; + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + [p[0] * d, p[1] * d] + } + /// Distorted → ideal (fixed-point iteration). + pub fn undistort(&self, q: [f64; 2]) -> [f64; 2] { + if self.is_none() { + return q; + } + let mut p = q; + for _ in 0..30 { + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + if d.abs() < 1e-6 { + break; + } + p = [q[0] / d, q[1] / d]; + } + p + } + /// `d distort / d p` (2 × 2, row-major) and `d distort / d (k1, k2)` (rows x, y). + pub fn jacobian(&self, p: [f64; 2]) -> ([[f64; 2]; 2], [[f64; 2]; 2]) { + if self.radius <= 0.0 { + return ([[1.0, 0.0], [0.0, 1.0]], [[0.0; 2]; 2]); + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + // d(r2)/dp = 2 p / R². + let dd = (self.k1 + 2.0 * self.k2 * r2) * 2.0 / (self.radius * self.radius); + let j = [[d + p[0] * dd * p[0], p[0] * dd * p[1]], [p[1] * dd * p[0], d + p[1] * dd * p[1]]]; + let k = [[p[0] * r2, p[0] * r2 * r2], [p[1] * r2, p[1] * r2 * r2]]; + (j, k) + } +} + /// 3D Camera Tracker ▸ Shot Type. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -178,6 +248,8 @@ pub struct SolveSettings { pub method: SolveMethod, /// Track ids the user deleted. pub deleted: Vec, + /// Solve radial lens distortion jointly with the cameras. + pub lens_distortion: bool, } /// The solved camera of one frame. @@ -261,6 +333,9 @@ pub struct CameraSolve { pub points: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub ground: Option, + /// The solved lens distortion (Solve Lens Distortion). + #[serde(skip_serializing_if = "Option::is_none")] + pub distortion: Option, } impl CameraSolve { @@ -355,6 +430,36 @@ impl CameraSolve { } } + /// An ideal (pinhole) image point → where it appears in the footage (layer pixels), through + /// the solved lens distortion. + pub fn to_image(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.distort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// A footage point → its ideal (undistorted) position. + pub fn to_ideal(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.undistort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// Project world point `x` on frame `k`: through the lens distortion into the footage, or + /// (`undistorted`) into the undistorted image. + pub fn project(&self, k: usize, x: V3, undistorted: bool) -> Option<[f64; 2]> { + let q = self.frames.get(k)?.project(self.size, x)?; + Some(if undistorted { q } else { self.to_image(q) }) + } + /// The first solved frame. pub fn first_solved(&self) -> Option { self.frames.iter().position(|c| c.solved) diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index 85edd2bd..5ed7b81b 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -8,7 +8,7 @@ use rayon::prelude::*; use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; use super::geometry::{self, Pose}; use super::linalg::*; -use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use super::{CameraSolve, CameraTracks, Distortion, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; use crate::fit::{Model, ransac}; fn debug() -> bool { @@ -142,6 +142,8 @@ struct Cfg { focal: FocalMode, /// Registration estimates a focal length per frame. per_frame_f: bool, + /// The global adjustments also adjust the lens distortion. + distortion: bool, } const F_PRIOR: f64 = 30.0; @@ -183,7 +185,7 @@ impl Recon { &mut cams, &mut pts, &obs, - &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters, distortion: cfg.distortion }, ); for (k, c) in frames.iter().zip(&cams) { self.cams.insert(*k, *c); @@ -289,6 +291,7 @@ impl Recon { fixed: vec![false], huber, max_iter: 40, + distortion: false, }; let mut best: Option<(usize, BaCam)> = None; // Robust start (large Huber threshold), then tighten; from the prediction and, failing @@ -371,7 +374,11 @@ fn predict(r: &Recon, k: usize) -> Option { match (before, after) { (Some((ka, a)), Some((kb, b))) => { let t = (k - ka) as f64 / (kb - ka) as f64; - Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + Some(BaCam { + pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, + f: a.f * (b.f / a.f).powf(t), + dist: a.dist, + }) } (Some((ka, a)), None) => { // Constant velocity from the two previous cameras. @@ -381,7 +388,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: a.f, dist: a.dist }) } None => Some(a), } @@ -392,7 +399,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: b.f, dist: b.dist }) } None => Some(b), }, @@ -459,8 +466,8 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let (a, b) = (kfs[0], kfs[j]); let mut r = Recon::default(); - r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); - r.cams.insert(b, BaCam { pose: init.0, f }); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); + r.cams.insert(b, BaCam { pose: init.0, f, dist: Distortion::NONE }); r.pts = init.1.clone(); let inner = |r: &mut Recon, frames: &[usize]| { let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; @@ -495,7 +502,7 @@ fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f /// Tripod reconstruction over keyframes `kfs` at focal length `f`. fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let mut r = Recon::default(); - r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); for w in kfs.windows(2) { if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { @@ -519,7 +526,7 @@ fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&Atomi rel = geometry::procrustes(&ia, &ib)?; } let pa = r.cams[&a].pose.r; - r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f, dist: Distortion::NONE }); } r.triangulate(d, Kind::Tripod, 0.0, 20.0); let frames: Vec = r.cams.keys().copied().collect(); @@ -640,8 +647,101 @@ fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { if best.1 <= errs[i] { best.0.exp() } else { fs[i] } } +/// A solve before its canonical output. +struct Core { + d: Data, + r: Recon, + method: SolveMethod, + kind: Kind, + cfg: Cfg, + focal: FocalMode, + kfs: Vec, +} + +/// `tracks` with every position undistorted by `dist`. +fn undistorted(tracks: &CameraTracks, dist: &Distortion) -> CameraTracks { + let c = [tracks.size[0] * 0.5, tracks.size[1] * 0.5]; + let mut t = tracks.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = dist.undistort([p[0] as f64 - c[0], p[1] as f64 - c[1]]); + *p = [(q[0] + c[0]) as f32, (q[1] + c[1]) as f32]; + } + } + t +} + /// Solve the camera of a tracked clip. pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + if !s.lens_distortion { + let c = solve_core(tracks, s, cancel)?; + return Ok(finish(&c.d, tracks, s, c.r, c.method, c.kind)); + } + // Lens distortion: solve, adjust (k1, k2) jointly on the raw tracks, then solve again on + // tracks undistorted with the estimate and adjust jointly once more. + let mut dist = Distortion::for_size(tracks.size); + let mut out = None; + for pass in 0..2 { + let und = (pass > 0).then(|| undistorted(tracks, &dist)); + let c = solve_core(und.as_ref().unwrap_or(tracks), s, cancel)?; + let d = Data::new(tracks, &s.deleted); + d.set_noise(c.d.noise()); + let mut r = c.r; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + let cfg = Cfg { distortion: true, ..c.cfg }; + let all: Vec = r.cams.keys().copied().collect(); + let frames: Vec = if all.len() <= 160 { all.clone() } else { c.kfs.iter().copied().filter(|k| r.cams.contains_key(k)).collect() }; + if frames.len() < 2 { + return err("the camera could not be solved (not enough parallax or features)"); + } + // The pinhole solve's outliers include the distorted frame edges: start from all + // observations (the Huber loss keeps real outliers in check), then reject again. + r.outliers.clear(); + r.adjust(&d, &frames, &cfg, c.focal, 30); + r.outliers.clear(); + r.reject(&d); + r.adjust(&d, &frames, &cfg, c.focal, 30); + dist = r.cams[&frames[0]].dist; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + if frames.len() < all.len() { + // Refine the other frames against the adjusted points and distortion. + let base = r.clone(); + let rcfg = Cfg { distortion: false, ..cfg }; + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &rcfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, cam) in refined { + if let Some(cam) = cam { + r.cams.insert(k, cam); + } + } + r.reject(&d); + } + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return err("cancelled"); + } + if debug() { + eprintln!("lens distortion pass {pass}: k1 {:.5} k2 {:.5}", dist.k1, dist.k2); + } + let mut sol = finish(&d, tracks, s, r, c.method, c.kind); + sol.distortion = Some(dist); + out = Some(sol); + } + out.ok_or_else(|| SolveError("the camera could not be solved".into())) +} + +fn solve_core(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { let d = Data::new(tracks, &s.deleted); if d.n < 2 { return err("the clip must be at least two frames long"); @@ -653,7 +753,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); let variable = s.shot == ShotType::VariableZoom; - let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false, distortion: false }; let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; // The focal-length search runs on the first keyframes only. let mini: Vec = kfs.iter().copied().take(10).collect(); @@ -738,7 +838,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } let (att, _) = results.swap_remove(pick); let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; - let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable, distortion: false }; let mut r = att.recon; let focal = match (fixed_f, variable) { (Some(_), _) => FocalMode::Fixed, @@ -845,7 +945,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (m, _) => m, }; lap("final adjustment"); - Ok(finish(&d, tracks, s, r, method, kind)) + Ok(Core { d, r, method, kind, cfg, focal, kfs: kf_frames }) } /// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). @@ -949,5 +1049,6 @@ fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: frames, points, ground: None, + distortion: None, } } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 4b5a8346..ff92cfe4 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -105,7 +105,7 @@ fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { } fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { - SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } + SolveSettings { shot, hfov: 0.0, method, deleted: vec![], lens_distortion: false } } #[test] @@ -149,7 +149,8 @@ fn specified_angle_of_view_is_kept() { let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); let tr = tracks_of(&pts, &t, 6, true); let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![], lens_distortion: false }, None).unwrap(); assert!((s.focal() - f).abs() < 1e-6); assert!(s.average_error < 0.5); let (pe, _) = path_error(&s, &t); @@ -226,7 +227,9 @@ fn mostly_flat_scene_and_ground_plane() { } // Ground plane from the solved (coplanar) points: every point lies on it. let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let mut s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![], lens_distortion: false }, None) + .expect("solve"); let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); let g = s.ground_from(&ids, 0).expect("ground"); let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); @@ -251,7 +254,9 @@ fn deleted_tracks_are_ignored_and_cancel_stops() { let tr = tracks_of(&pts, &t, 14, true); let deleted: Vec = (0..40).collect(); let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone(), lens_distortion: false }, None) + .unwrap(); assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); let stop = std::sync::atomic::AtomicBool::new(true); let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); @@ -436,3 +441,55 @@ fn solve_saved_tracks() { t0.elapsed() ); } + +/// Pass every track position through a lens distortion (around the frame centre). +fn distort_tracks(t: &CameraTracks, d: &Distortion) -> CameraTracks { + let mut t = t.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = d.distort([p[0] as f64 - W / 2.0, p[1] as f64 - H / 2.0]); + *p = [(q[0] + W / 2.0) as f32, (q[1] + H / 2.0) as f32]; + } + } + t +} + +#[test] +fn lens_distortion_is_recovered_in_the_bundle_adjustment() { + let f = 1000.0; + let t = moving_camera(36, |_| f); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let truth = Distortion { k1: -0.12, k2: 0.03, radius: 0.5 * W.hypot(H) }; + let tr = distort_tracks(&tracks_of(&pts, &t, 8, true), &truth); + let plain = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("pinhole solve"); + let lens = solve(&tr, &SolveSettings { lens_distortion: true, ..settings(ShotType::FixedAngle, SolveMethod::Typical) }, None).expect("lens solve"); + let d = lens.distortion.expect("distortion solved"); + eprintln!( + "pinhole error {:.3} px (focal {:.1}); with distortion {:.3} px (focal {:.1}), k1 {:.4} k2 {:.4}", + plain.average_error, + plain.focal(), + lens.average_error, + lens.focal(), + d.k1, + d.k2 + ); + assert!(plain.distortion.is_none()); + assert!((d.k1 - truth.k1).abs() < 0.005, "k1 {}", d.k1); + assert!((d.k2 - truth.k2).abs() < 0.01, "k2 {}", d.k2); + // The distortion model explains the tracks: about the noise level, far below the pinhole fit. + assert!(lens.average_error < 0.3, "{}", lens.average_error); + assert!(lens.average_error < 0.5 * plain.average_error, "{} vs {}", lens.average_error, plain.average_error); + assert!((lens.focal() - f).abs() < 0.02 * f, "focal {}", lens.focal()); + let (path, rot) = path_error(&lens, &t); + assert!(path < 0.01 && rot < 0.3, "path {path} rot {rot}"); + // Projection through the lens lands on the (distorted) tracks; undistorting inverts it. + let p = [100.0, 80.0]; + let q = lens.to_image(p); + let back = lens.to_ideal(q); + assert!((back[0] - p[0]).abs() < 1e-6 && (back[1] - p[1]).abs() < 1e-6); + let sp = &lens.points[0]; + let k = sp.first as usize; + let obs = tr.track(sp.id).and_then(|t| t.at(k as u32)).expect("observed"); + let proj = lens.project(k, sp.pos, false).expect("in front"); + assert!((proj[0] - obs[0]).hypot(proj[1] - obs[1]) < 2.0, "{proj:?} vs {obs:?}"); +} diff --git a/crates/track/src/face.rs b/crates/track/src/face.rs new file mode 100644 index 00000000..62073d0a --- /dev/null +++ b/crates/track/src/face.rs @@ -0,0 +1,1147 @@ +//! Face tracking (Tracker panel ▸ Method ▸ Face Tracking (Outline Only) / (Detailed Features)), +//! a classical pipeline with no learned weights: +//! +//! 1. **Skin model**: on the first frame, the colour of the middle of the mask (luma and +//! normalised r, g chromaticity: median and median absolute deviation) defines skin; every +//! later frame is classified against it inside a search window around the previous face. +//! 2. **Face region and outline**: the skin component containing the face centre is flood-filled; +//! its outer boundary is found along 48 rays from its centroid (so the non-skin holes of eyes, +//! brows, nostrils and mouth don't matter), median-smoothed, and an ellipse is fitted to the +//! outline polygon from its area moments (centre, half-height, roll). The outline is what +//! *Outline Only* keys into the mask. +//! 3. **Facial features**: the non-skin connected components inside the face are matched to the +//! eyes, brows, nostrils and mouth by their position in the face frame (predicted from the +//! previous frame); landmark points are measured on each component (extremes along the face +//! axes for corners and lids, a darkness-weighted centroid for the pupil). Chin and jaw points +//! come from the outline. +//! 4. **Shape model**: an active-shape point distribution model (Cootes, Taylor, Cooper & Graham, +//! "Active Shape Models — Their Training and Application", CVIU 61(1), 1995) regularises the +//! landmarks: a weighted similarity alignment and shape-parameter fit (parameters clamped to +//! ±3 standard deviations) fills landmarks whose feature was not found (a closed eye, a +//! merged component) and replaces detections that disagree with the model. The model is +//! *trained on synthetic shapes* produced by our own parametric face generator +//! ([`synth_shape`]: eye and mouth openness, mouth width, brow raise, head yaw and pitch) by +//! principal component analysis — no external data or weights. +//! +//! Coordinates are layer pixels; the face frame is centred on the face ellipse, with the +//! half-height as unit, `u` right and `v` down (image-left is "Left"). + +use std::sync::OnceLock; + +use effectcraft_raster::Image; + +use crate::Frame; +use crate::camtrack::linalg::{cholesky_solve, sym_eigen}; + +/// Landmarks: (id, display name, canonical position in the face frame). +pub const LANDMARKS: [(&str, &str, [f64; 2]); 26] = [ + ("leftEyebrowInner", "Left Eyebrow Inner", [-0.16, -0.44]), + ("leftEyebrowMiddle", "Left Eyebrow Middle", [-0.32, -0.50]), + ("leftEyebrowOuter", "Left Eyebrow Outer", [-0.48, -0.44]), + ("rightEyebrowInner", "Right Eyebrow Inner", [0.16, -0.44]), + ("rightEyebrowMiddle", "Right Eyebrow Middle", [0.32, -0.50]), + ("rightEyebrowOuter", "Right Eyebrow Outer", [0.48, -0.44]), + ("leftEyeInner", "Left Eye Inner", [-0.17, -0.22]), + ("leftEyeOuter", "Left Eye Outer", [-0.45, -0.22]), + ("leftEyeTop", "Left Eye Top", [-0.31, -0.292]), + ("leftEyeBottom", "Left Eye Bottom", [-0.31, -0.148]), + ("leftPupil", "Left Pupil", [-0.31, -0.22]), + ("rightEyeInner", "Right Eye Inner", [0.17, -0.22]), + ("rightEyeOuter", "Right Eye Outer", [0.45, -0.22]), + ("rightEyeTop", "Right Eye Top", [0.31, -0.292]), + ("rightEyeBottom", "Right Eye Bottom", [0.31, -0.148]), + ("rightPupil", "Right Pupil", [0.31, -0.22]), + ("noseTip", "Nose Tip", [0.0, 0.14]), + ("leftNostril", "Left Nostril", [-0.08, 0.20]), + ("rightNostril", "Right Nostril", [0.08, 0.20]), + ("mouthLeft", "Mouth Left", [-0.24, 0.50]), + ("mouthRight", "Mouth Right", [0.24, 0.50]), + ("mouthTop", "Mouth Top", [0.0, 0.435]), + ("mouthBottom", "Mouth Bottom", [0.0, 0.565]), + ("chin", "Chin", [0.0, 1.0]), + ("leftJaw", "Left Jaw", [-0.614, 0.574]), + ("rightJaw", "Right Jaw", [0.614, 0.574]), +]; + +/// Number of landmarks. +pub const N: usize = LANDMARKS.len(); + +/// Landmark index by id. +pub fn landmark(id: &str) -> Option { + LANDMARKS.iter().position(|l| l.0 == id) +} + +const BROW: [[usize; 3]; 2] = [[0, 1, 2], [3, 4, 5]]; +/// Eye landmarks: inner, outer, top, bottom, pupil. +const EYE: [[usize; 5]; 2] = [[6, 7, 8, 9, 10], [11, 12, 13, 14, 15]]; +const NOSE_TIP: usize = 16; +const NOSTRIL: [usize; 2] = [17, 18]; +/// Mouth: left, right, top, bottom. +const MOUTH: [usize; 4] = [19, 20, 21, 22]; +const CHIN: usize = 23; +const JAW: [usize; 2] = [24, 25]; +/// Half-width of the face relative to its half-height. +const ASPECT: f64 = 0.75; + +// ---------------------------------------------------------------- synthetic faces + +/// Parameters of a synthetic face (the shape model's training data and the tests' footage). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FaceParams { + /// Face centre (layer pixels) and half-height (pixels). + pub center: [f64; 2], + pub height: f64, + /// Head rotation in degrees: roll (in the image plane), yaw (turn), pitch (nod). + pub roll: f64, + pub yaw: f64, + pub pitch: f64, + /// Eye openness (0 closed … 1 open), left and right. + pub eye_open: [f64; 2], + /// Mouth openness (0 … 1) and width (1 = neutral). + pub mouth_open: f64, + pub mouth_width: f64, + /// Eyebrow raise (−1 … 1), left and right. + pub brow_raise: [f64; 2], +} + +impl Default for FaceParams { + fn default() -> Self { + FaceParams { + center: [0.0; 2], + height: 60.0, + roll: 0.0, + yaw: 0.0, + pitch: 0.0, + eye_open: [1.0; 2], + mouth_open: 0.2, + mouth_width: 1.0, + brow_raise: [0.0; 2], + } + } +} + +fn eye_half_height(open: f64) -> f64 { + 0.012 + 0.06 * open.clamp(0.0, 1.0) +} +fn mouth_half_height(open: f64) -> f64 { + 0.015 + 0.11 * open.clamp(0.0, 1.0) +} + +/// The landmarks of a synthetic face in the face frame (before roll, translation and scale). +pub fn synth_shape(p: &FaceParams) -> [[f64; 2]; N] { + let mut s = [[0.0; 2]; N]; + let (sy, sp) = (p.yaw.to_radians().sin(), p.pitch.to_radians().sin()); + // Inner features slide with the head's turn; the nose (closer to the camera) slides more. + let shift = |depth: f64| [0.35 * sy * depth, 0.3 * sp * depth]; + for (i, l) in LANDMARKS.iter().enumerate() { + s[i] = l.2; + } + for side in 0..2 { + let sg = if side == 0 { -1.0 } else { 1.0 }; + let raise = -0.06 * p.brow_raise[side].clamp(-1.0, 1.0); + for &i in &BROW[side] { + s[i][1] += raise; + } + let h = eye_half_height(p.eye_open[side]); + let e = EYE[side]; + s[e[2]] = [sg * 0.31, -0.22 - h]; + s[e[3]] = [sg * 0.31, -0.22 + h]; + } + let mw = 0.24 * p.mouth_width.clamp(0.5, 1.6); + let mh = mouth_half_height(p.mouth_open); + s[MOUTH[0]] = [-mw, 0.5]; + s[MOUTH[1]] = [mw, 0.5]; + s[MOUTH[2]] = [0.0, 0.5 - mh]; + s[MOUTH[3]] = [0.0, 0.5 + mh]; + for (i, q) in s.iter_mut().enumerate() { + if i == CHIN || JAW.contains(&i) { + continue; + } + let d = if i == NOSE_TIP { + 1.3 + } else if NOSTRIL.contains(&i) { + 1.15 + } else { + 1.0 + }; + let o = shift(d); + q[0] += o[0]; + q[1] += o[1]; + } + s +} + +/// Face frame → layer pixels. +fn to_layer(c: [f64; 2], b: f64, roll_deg: f64, q: [f64; 2]) -> [f64; 2] { + let (sn, cs) = roll_deg.to_radians().sin_cos(); + [c[0] + b * (cs * q[0] - sn * q[1]), c[1] + b * (sn * q[0] + cs * q[1])] +} + +/// Layer pixels → face frame. +fn to_face(c: [f64; 2], b: f64, roll_deg: f64, p: [f64; 2]) -> [f64; 2] { + let (sn, cs) = roll_deg.to_radians().sin_cos(); + let d = [(p[0] - c[0]) / b, (p[1] - c[1]) / b]; + [cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]] +} + +/// A synthetic face's landmarks in layer pixels. +pub fn synth_landmarks(p: &FaceParams) -> [[f64; 2]; N] { + synth_shape(p).map(|q| to_layer(p.center, p.height, p.roll, q)) +} + +fn hash(i: i64, j: i64) -> f64 { + let mut v = (i.wrapping_mul(73_856_093) ^ j.wrapping_mul(19_349_663)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1_e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 +} + +fn seg_dist(p: [f64; 2], a: [f64; 2], b: [f64; 2]) -> f64 { + let d = [b[0] - a[0], b[1] - a[1]]; + let l2 = d[0] * d[0] + d[1] * d[1]; + let t = if l2 > 0.0 { (((p[0] - a[0]) * d[0] + (p[1] - a[1]) * d[1]) / l2).clamp(0.0, 1.0) } else { 0.0 }; + (p[0] - a[0] - t * d[0]).hypot(p[1] - a[1] - t * d[1]) +} + +/// Colour of a synthetic face at face-frame point `q` (`None` outside the face). +fn face_color(p: &FaceParams, s: &[[f64; 2]; N], q: [f64; 2]) -> Option<[f32; 3]> { + if (q[0] / ASPECT).powi(2) + q[1] * q[1] > 1.0 { + return None; + } + let shade = (0.95 - 0.08 * q[1]) as f32; + let mut col = [0.86 * shade, 0.64 * shade, 0.52 * shade]; + for side in 0..2 { + let b = BROW[side]; + let d = seg_dist(q, s[b[0]], s[b[1]]).min(seg_dist(q, s[b[1]], s[b[2]])); + if d < 0.035 { + col = [0.28, 0.18, 0.12]; + } + let e = EYE[side]; + let ec = [(s[e[0]][0] + s[e[1]][0]) / 2.0, (s[e[2]][1] + s[e[3]][1]) / 2.0]; + let (hw, hh) = ((s[e[1]][0] - s[e[0]][0]).abs() / 2.0, (s[e[3]][1] - s[e[2]][1]).abs() / 2.0); + if ((q[0] - ec[0]) / hw).powi(2) + ((q[1] - ec[1]) / hh).powi(2) <= 1.0 { + let pupil = s[e[4]]; + col = if (q[0] - pupil[0]).hypot(q[1] - pupil[1]) < 0.055 { [0.12, 0.08, 0.06] } else { [0.95, 0.95, 0.93] }; + } + } + let _ = p; + for &n in &NOSTRIL { + if ((q[0] - s[n][0]) / 0.035).powi(2) + ((q[1] - s[n][1]) / 0.022).powi(2) <= 1.0 { + col = [0.3, 0.13, 0.1]; + } + } + let m = MOUTH; + let mc = [(s[m[0]][0] + s[m[1]][0]) / 2.0, (s[m[2]][1] + s[m[3]][1]) / 2.0]; + let (mw, mh) = ((s[m[1]][0] - s[m[0]][0]) / 2.0, (s[m[3]][1] - s[m[2]][1]) / 2.0); + if ((q[0] - mc[0]) / mw).powi(2) + ((q[1] - mc[1]) / mh).powi(2) <= 1.0 { + col = [0.42, 0.08, 0.1]; + } + Some(col) +} + +/// Render a synthetic face over a textured blue-green background (2 × 2 supersampled): the +/// test footage of the face tracker. +pub fn render_synthetic(w: u32, h: u32, p: &FaceParams, background_seed: i64) -> Image { + use rayon::prelude::*; + let s = synth_shape(p); + let mut img = Image::new(w, h); + img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let mut acc = [0.0f32; 3]; + for sy in 0..2 { + for sx in 0..2 { + let l = [x as f64 + 0.25 + 0.5 * sx as f64, y as f64 + 0.25 + 0.5 * sy as f64]; + let q = to_face(p.center, p.height, p.roll, l); + let c = face_color(p, &s, q).unwrap_or_else(|| { + let n = hash((l[0] / 9.0) as i64 + background_seed, (l[1] / 9.0) as i64) as f32; + [0.15 + 0.1 * n, 0.35 + 0.2 * n, 0.45 + 0.25 * n] + }); + for k in 0..3 { + acc[k] += c[k] * 0.25; + } + } + } + *px = [acc[0], acc[1], acc[2], 1.0]; + } + }); + img +} + +// ---------------------------------------------------------------- shape model + +/// Point distribution model: mean shape and principal modes (face frame). +pub struct Pdm { + pub mean: Vec, + /// Modes as rows of length `2N`. + pub modes: Vec>, + /// Variance of each mode. + pub var: Vec, +} + +/// The shape model, trained once on synthetic shapes. +pub fn pdm() -> &'static Pdm { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| { + let samples = 600; + let dim = 2 * N; + let mut rng = 0x2545_f491_4f6c_dd1du64; + let mut u = || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + (rng % 1_000_000) as f64 / 1_000_000.0 + }; + let mut data = Vec::with_capacity(samples); + for _ in 0..samples { + let p = FaceParams { + yaw: (u() - 0.5) * 60.0, + pitch: (u() - 0.5) * 40.0, + eye_open: [u(), u()], + mouth_open: u(), + mouth_width: 0.8 + 0.45 * u(), + brow_raise: [u() * 2.0 - 1.0, u() * 2.0 - 1.0], + ..Default::default() + }; + data.push(synth_shape(&p).iter().flat_map(|q| [q[0], q[1]]).collect::>()); + } + let mut mean = vec![0.0; dim]; + for d in &data { + for (m, v) in mean.iter_mut().zip(d) { + *m += v / samples as f64; + } + } + let mut cov = vec![0.0; dim * dim]; + for d in &data { + for a in 0..dim { + let da = d[a] - mean[a]; + for b in 0..dim { + cov[a * dim + b] += da * (d[b] - mean[b]) / samples as f64; + } + } + } + let (vals, vecs) = sym_eigen(&cov, dim); + let total: f64 = vals.iter().map(|v| v.max(0.0)).sum(); + let mut modes = vec![]; + let mut var = vec![]; + let mut acc = 0.0; + for k in (0..dim).rev() { + if acc >= 0.995 * total || vals[k] <= 1e-12 { + break; + } + acc += vals[k]; + modes.push((0..dim).map(|r| vecs[r * dim + k]).collect()); + var.push(vals[k]); + } + Pdm { mean, modes, var } + }) +} + +/// A similarity `p ↦ s R q + t` as `[a, b, tx, ty]` (`x = a u − b v + tx`, `y = b u + a v + ty`). +fn fit_similarity(src: &[[f64; 2]], dst: &[[f64; 2]], w: &[f64]) -> Option<[f64; 4]> { + let ws: f64 = w.iter().sum(); + if ws < 2.0 - 1e-9 { + return None; + } + let mut cs = [0.0; 2]; + let mut cd = [0.0; 2]; + for i in 0..src.len() { + for k in 0..2 { + cs[k] += w[i] * src[i][k] / ws; + cd[k] += w[i] * dst[i][k] / ws; + } + } + let (mut ss, mut dt, mut cr) = (0.0, 0.0, 0.0); + for i in 0..src.len() { + let a = [src[i][0] - cs[0], src[i][1] - cs[1]]; + let b = [dst[i][0] - cd[0], dst[i][1] - cd[1]]; + ss += w[i] * (a[0] * a[0] + a[1] * a[1]); + dt += w[i] * (a[0] * b[0] + a[1] * b[1]); + cr += w[i] * (a[0] * b[1] - a[1] * b[0]); + } + if ss < 1e-12 { + return None; + } + let (a, b) = (dt / ss, cr / ss); + Some([a, b, cd[0] - (a * cs[0] - b * cs[1]), cd[1] - (b * cs[0] + a * cs[1])]) +} + +fn sim_apply(t: &[f64; 4], q: [f64; 2]) -> [f64; 2] { + [t[0] * q[0] - t[1] * q[1] + t[2], t[1] * q[0] + t[0] * q[1] + t[3]] +} + +fn sim_inverse(t: &[f64; 4], p: [f64; 2]) -> [f64; 2] { + let d = [p[0] - t[2], p[1] - t[3]]; + let s2 = t[0] * t[0] + t[1] * t[1]; + [(t[0] * d[0] + t[1] * d[1]) / s2, (-t[1] * d[0] + t[0] * d[1]) / s2] +} + +/// Fit the shape model to detected landmarks (`w` = 0 for missing ones), starting from the +/// face-frame similarity `init`. Returns (model landmarks in layer pixels, similarity). +pub fn fit_shape(detected: &[[f64; 2]; N], w: &[f64; N], init: [f64; 4]) -> ([[f64; 2]; N], [f64; 4]) { + let m = pdm(); + let k = m.modes.len(); + let mut b = vec![0.0; k]; + let mut t = init; + let shape = |b: &[f64]| -> [[f64; 2]; N] { + let mut s = [[0.0; 2]; N]; + for i in 0..N { + for d in 0..2 { + let j = 2 * i + d; + s[i][d] = m.mean[j] + m.modes.iter().zip(b).map(|(row, bk)| row[j] * bk).sum::(); + } + } + s + }; + for _ in 0..10 { + let x = shape(&b); + if let Some(nt) = fit_similarity(&x, detected, w) { + t = nt; + } + // Detections in the model frame. + let y: Vec<[f64; 2]> = detected.iter().map(|p| sim_inverse(&t, *p)).collect(); + let mut ata = vec![0.0; k * k]; + let mut atb = vec![0.0; k]; + for i in 0..N { + if w[i] <= 0.0 { + continue; + } + for d in 0..2 { + let j = 2 * i + d; + let r = y[i][d] - m.mean[j]; + for a in 0..k { + atb[a] += w[i] * m.modes[a][j] * r; + for c in 0..k { + ata[a * k + c] += w[i] * m.modes[a][j] * m.modes[c][j]; + } + } + } + } + for a in 0..k { + // A weak prior towards the mean (Tikhonov, scaled by the mode's variance). + ata[a * k + a] += 1e-3 / m.var[a].max(1e-9) * 1e-3 + 1e-9; + } + if cholesky_solve(&mut ata, k, &mut atb) { + for a in 0..k { + let lim = 3.0 * m.var[a].sqrt(); + b[a] = atb[a].clamp(-lim, lim); + } + } + } + let x = shape(&b); + (x.map(|q| sim_apply(&t, q)), t) +} + +// ---------------------------------------------------------------- tracking + +/// The skin colour model (luma, r and g chromaticity: median and spread). +#[derive(Clone, Copy, Debug, PartialEq)] +struct Skin { + med: [f32; 3], + tol: [f32; 3], +} + +fn features(px: [f32; 4]) -> [f32; 3] { + let s = (px[0] + px[1] + px[2]).max(1e-4); + [0.2126 * px[0] + 0.7152 * px[1] + 0.0722 * px[2], px[0] / s, px[1] / s] +} + +impl Skin { + fn is_skin(&self, px: [f32; 4]) -> bool { + let f = features(px); + (0..3).all(|k| (f[k] - self.med[k]).abs() <= self.tol[k]) + } +} + +/// One frame's face fit. +#[derive(Clone, Debug, PartialEq)] +pub struct FaceFit { + /// Landmarks ([`LANDMARKS`] order), layer pixels. + pub landmarks: [[f64; 2]; N], + /// Which landmarks were measured on this frame (the others come from the shape model). + pub detected: [bool; N], + /// The face outline: a closed polygon (layer pixels). + pub outline: Vec<[f64; 2]>, + /// Face ellipse centre, half-height (pixels) and roll (degrees). + pub center: [f64; 2], + pub height: f64, + pub roll: f64, +} + +/// Follows one face through a clip. +#[derive(Clone, Debug)] +pub struct FaceTracker { + skin: Skin, + prev: FaceFit, + /// Previous landmarks in the face frame (the expected feature positions). + expect: [[f64; 2]; N], +} + +/// A pixel window of a frame in layer coordinates. +struct Window { + x0: i64, + y0: i64, + w: usize, + h: usize, +} + +impl Window { + fn around(frame: &Frame, c: [f64; 2], rx: f64, ry: f64) -> Option { + let (ox, oy) = (frame.offset[0], frame.offset[1]); + let x0 = ((c[0] - rx + ox).floor() as i64).max(0); + let y0 = ((c[1] - ry + oy).floor() as i64).max(0); + let x1 = ((c[0] + rx + ox).ceil() as i64).min(frame.img.width as i64); + let y1 = ((c[1] + ry + oy).ceil() as i64).min(frame.img.height as i64); + (x1 > x0 + 4 && y1 > y0 + 4).then(|| Window { x0, y0, w: (x1 - x0) as usize, h: (y1 - y0) as usize }) + } + fn px(&self, frame: &Frame, i: usize, j: usize) -> [f32; 4] { + frame.img.get(self.x0 + i as i64, self.y0 + j as i64) + } + /// Layer position of the centre of window pixel `(i, j)`. + fn layer(&self, frame: &Frame, i: usize, j: usize) -> [f64; 2] { + [(self.x0 + i as i64) as f64 + 0.5 - frame.offset[0], (self.y0 + j as i64) as f64 + 0.5 - frame.offset[1]] + } + fn cell(&self, frame: &Frame, p: [f64; 2]) -> Option<(usize, usize)> { + let x = (p[0] + frame.offset[0]).floor() as i64 - self.x0; + let y = (p[1] + frame.offset[1]).floor() as i64 - self.y0; + (x >= 0 && y >= 0 && (x as usize) < self.w && (y as usize) < self.h).then_some((x as usize, y as usize)) + } +} + +fn median(v: &mut [f32]) -> f32 { + v.sort_by(f32::total_cmp); + v[v.len() / 2] +} + +/// Area-moment ellipse of a closed polygon: (centroid, half-height along the major axis, +/// half-width, major axis angle from vertical in degrees). +fn polygon_ellipse(poly: &[[f64; 2]]) -> Option<([f64; 2], f64, f64, f64)> { + let n = poly.len(); + let (mut a, mut cx, mut cy) = (0.0, 0.0, 0.0); + let (mut ixx, mut iyy, mut ixy) = (0.0, 0.0, 0.0); + for i in 0..n { + let (p, q) = (poly[i], poly[(i + 1) % n]); + let cr = p[0] * q[1] - q[0] * p[1]; + a += cr; + cx += (p[0] + q[0]) * cr; + cy += (p[1] + q[1]) * cr; + ixx += (p[1] * p[1] + p[1] * q[1] + q[1] * q[1]) * cr; + iyy += (p[0] * p[0] + p[0] * q[0] + q[0] * q[0]) * cr; + ixy += (p[0] * q[1] + 2.0 * p[0] * p[1] + 2.0 * q[0] * q[1] + q[0] * p[1]) * cr; + } + a *= 0.5; + if a.abs() < 1e-9 { + return None; + } + cx /= 6.0 * a; + cy /= 6.0 * a; + // Central second moments (per unit area): var x, var y, cov xy. + let vxx = iyy / (12.0 * a) - cx * cx; + let vyy = ixx / (12.0 * a) - cy * cy; + let vxy = ixy / (24.0 * a) - cx * cy; + let tr = vxx + vyy; + let det = vxx * vyy - vxy * vxy; + let disc = (tr * tr / 4.0 - det).max(0.0).sqrt(); + let (l1, l2) = (tr / 2.0 + disc, (tr / 2.0 - disc).max(0.0)); + // Major axis direction. + let ang = 0.5 * (2.0 * vxy).atan2(vxx - vyy); // from +x + let mut from_vertical = ang.to_degrees() - 90.0; + while from_vertical > 90.0 { + from_vertical -= 180.0; + } + while from_vertical < -90.0 { + from_vertical += 180.0; + } + Some(([cx, cy], 2.0 * l1.sqrt(), 2.0 * l2.sqrt(), from_vertical)) +} + +/// A non-skin component inside the face. +struct Blob { + /// Face-frame positions of its pixels and their darkness (skin luma − luma). + pts: Vec<([f64; 2], f64)>, + centroid: [f64; 2], +} + +impl Blob { + fn extreme(&self, dir: [f64; 2]) -> [f64; 2] { + self.pts.iter().map(|p| p.0).max_by(|a, b| (a[0] * dir[0] + a[1] * dir[1]).total_cmp(&(b[0] * dir[0] + b[1] * dir[1]))).unwrap_or(self.centroid) + } + /// Extreme along `v` (`sign` −1 = top) among the pixels within `band` of the column `u`. + fn lid(&self, u: f64, band: f64, sign: f64) -> [f64; 2] { + let c = self.pts.iter().filter(|p| (p.0[0] - u).abs() <= band).map(|p| p.0); + let best = c.max_by(|a, b| (sign * a[1]).total_cmp(&(sign * b[1]))); + best.map(|p| [u, p[1]]).unwrap_or(self.centroid) + } + fn dark_centroid(&self) -> [f64; 2] { + let (mut s, mut w) = ([0.0; 2], 0.0); + for (q, d) in &self.pts { + let k = d.max(0.0).powi(2); + s[0] += k * q[0]; + s[1] += k * q[1]; + w += k; + } + if w > 1e-12 { [s[0] / w, s[1] / w] } else { self.centroid } + } +} + +impl FaceTracker { + /// Start on `frame` with the face inside `region` (`[x0, y0, x1, y1]`, layer pixels: the + /// mask's bounds). `None` when no face-like skin region is found. + pub fn new(frame: &Frame, region: [f64; 4]) -> Option<(FaceTracker, FaceFit)> { + let c = [(region[0] + region[2]) / 2.0, (region[1] + region[3]) / 2.0]; + let (hw, hh) = ((region[2] - region[0]) / 2.0, (region[3] - region[1]) / 2.0); + if !(hw > 4.0 && hh > 4.0) { + return None; + } + // Skin colour: the middle of the region (cheeks and nose). + let win = Window::around(frame, c, hw * 0.35, hh * 0.3)?; + let mut f: [Vec; 3] = [vec![], vec![], vec![]]; + for j in 0..win.h { + for i in 0..win.w { + let v = features(win.px(frame, i, j)); + for k in 0..3 { + f[k].push(v[k]); + } + } + } + let mut med = [0.0f32; 3]; + let mut tol = [0.0f32; 3]; + let floors = [0.12f32, 0.025, 0.025]; + for k in 0..3 { + med[k] = median(&mut f[k]); + let mut dev: Vec = f[k].iter().map(|v| (v - med[k]).abs()).collect(); + tol[k] = (4.0 * median(&mut dev)).max(floors[k]); + } + let skin = Skin { med, tol }; + let canon: [[f64; 2]; N] = LANDMARKS.map(|l| l.2); + let guess = FaceFit { + landmarks: canon.map(|q| to_layer(c, hh, 0.0, q)), + detected: [false; N], + outline: vec![], + center: c, + height: hh.max(hw / ASPECT), + roll: 0.0, + }; + let mut t = FaceTracker { skin, prev: guess, expect: canon }; + let fit = t.fit(frame)?; + // A face shows at least two of: left eye, right eye, mouth. + let found = [EYE[0][4], EYE[1][4], MOUTH[0]].iter().filter(|i| fit.detected[**i]).count(); + if found < 2 { + return None; + } + t.accept(&fit); + Some((t, fit)) + } + + /// The fit of the last tracked frame. + pub fn last(&self) -> &FaceFit { + &self.prev + } + + fn accept(&mut self, fit: &FaceFit) { + self.expect = fit.landmarks.map(|p| to_face(fit.center, fit.height, fit.roll, p)); + self.prev = fit.clone(); + } + + /// Track into the next frame (either direction). `None` when the face is lost. + pub fn step(&mut self, frame: &Frame) -> Option { + let fit = self.fit(frame)?; + self.accept(&fit); + Some(fit) + } + + fn fit(&self, frame: &Frame) -> Option { + let pv = &self.prev; + let b0 = pv.height; + let reach = 1.7 * b0; + let win = Window::around(frame, pv.center, reach, reach)?; + let (w, h) = (win.w, win.h); + let mut skin = vec![false; w * h]; + let mut luma = vec![0.0f32; w * h]; + for j in 0..h { + for i in 0..w { + let px = win.px(frame, i, j); + skin[j * w + i] = self.skin.is_skin(px); + luma[j * w + i] = features(px)[0]; + } + } + // Seed: the skin pixel nearest the predicted centre. + let (ci, cj) = win.cell(frame, pv.center)?; + let mut seed = None; + 'search: for r in 0..(0.6 * b0).ceil() as i64 { + for dj in -r..=r { + for di in -r..=r { + if di.abs() != r && dj.abs() != r { + continue; + } + let (i, j) = (ci as i64 + di, cj as i64 + dj); + if i >= 0 && j >= 0 && (i as usize) < w && (j as usize) < h && skin[j as usize * w + i as usize] { + seed = Some((i as usize, j as usize)); + break 'search; + } + } + } + } + let (si, sj) = seed?; + // Flood fill (4-connected). + let mut comp = vec![false; w * h]; + let mut stack = vec![(si, sj)]; + comp[sj * w + si] = true; + let mut count = 0usize; + let (mut mx, mut my) = (0.0, 0.0); + while let Some((i, j)) = stack.pop() { + count += 1; + let p = win.layer(frame, i, j); + mx += p[0]; + my += p[1]; + let nb = [(i.wrapping_sub(1), j), (i + 1, j), (i, j.wrapping_sub(1)), (i, j + 1)]; + for (a, b) in nb { + if a < w && b < h && !comp[b * w + a] && skin[b * w + a] { + comp[b * w + a] = true; + stack.push((a, b)); + } + } + } + if (count as f64) < 0.15 * std::f64::consts::PI * ASPECT * b0 * b0 { + return None; + } + let c0 = [mx / count as f64, my / count as f64]; + // Outline along rays: the farthest component pixel. + let rays = 48; + let mut radii = vec![0.0f64; rays]; + for (k, r) in radii.iter_mut().enumerate() { + let a = k as f64 / rays as f64 * std::f64::consts::TAU; + let d = [a.cos(), a.sin()]; + let mut t = 0.0; + while t < reach * 1.4 { + let p = [c0[0] + d[0] * t, c0[1] + d[1] * t]; + match win.cell(frame, p) { + Some((i, j)) => { + if comp[j * w + i] { + *r = t + 0.5; + } + } + None => break, + } + t += 0.5; + } + } + // Median of 5, circularly. + let sm: Vec = (0..rays) + .map(|k| { + let mut v: Vec = (0..5).map(|d| radii[(k + rays + d - 2) % rays]).collect(); + v.sort_by(f64::total_cmp); + v[2] + }) + .collect(); + let outline: Vec<[f64; 2]> = (0..rays) + .map(|k| { + let a = k as f64 / rays as f64 * std::f64::consts::TAU; + [c0[0] + a.cos() * sm[k], c0[1] + a.sin() * sm[k]] + }) + .collect(); + let (center, major, _minor, ang) = polygon_ellipse(&outline)?; + if !(major > 0.3 * b0 && major < 2.5 * b0) { + return None; + } + // Roll: the ellipse axis nearest the previous roll (it is ambiguous by 180°). + let mut roll = ang; + while roll - pv.roll > 90.0 { + roll -= 180.0; + } + while roll - pv.roll < -90.0 { + roll += 180.0; + } + let b = major; + // Non-skin components inside the face outline (shrunk a little: anti-aliased edges). + let inner: Vec<[f64; 2]> = outline.iter().map(|p| [center[0] + 0.9 * (p[0] - center[0]), center[1] + 0.9 * (p[1] - center[1])]).collect(); + let mut feat = vec![false; w * h]; + for j in 0..h { + for i in 0..w { + if !skin[j * w + i] && crate::mask::inside_polygon(&inner, win.layer(frame, i, j)) { + feat[j * w + i] = true; + } + } + } + let min_area = (0.0006 * b * b).max(2.0) as usize; + let mut seen = vec![false; w * h]; + let mut blobs: Vec = vec![]; + let skin_luma = self.skin.med[0] as f64; + for j0 in 0..h { + for i0 in 0..w { + if !feat[j0 * w + i0] || seen[j0 * w + i0] { + continue; + } + let mut st = vec![(i0, j0)]; + seen[j0 * w + i0] = true; + let mut pts = vec![]; + while let Some((i, j)) = st.pop() { + let q = to_face(center, b, roll, win.layer(frame, i, j)); + pts.push((q, skin_luma - luma[j * w + i] as f64)); + for dj in -1i64..=1 { + for di in -1i64..=1 { + let (a, c) = (i as i64 + di, j as i64 + dj); + if a < 0 || c < 0 || a as usize >= w || c as usize >= h { + continue; + } + let (a, c) = (a as usize, c as usize); + if feat[c * w + a] && !seen[c * w + a] { + seen[c * w + a] = true; + st.push((a, c)); + } + } + } + } + if pts.len() >= min_area { + let n = pts.len() as f64; + let centroid = [pts.iter().map(|p| p.0[0]).sum::() / n, pts.iter().map(|p| p.0[1]).sum::() / n]; + blobs.push(Blob { pts, centroid }); + } + } + } + // Match components to features near their expected positions. + let ex = &self.expect; + let mean_of = |ids: &[usize]| { + let n = ids.len() as f64; + [ids.iter().map(|i| ex[*i][0]).sum::() / n, ids.iter().map(|i| ex[*i][1]).sum::() / n] + }; + let mut used = vec![false; blobs.len()]; + let take = |target: [f64; 2], radius: f64, prefer_large: bool, used: &mut Vec| -> Option { + let mut best: Option<(usize, f64)> = None; + for (k, bl) in blobs.iter().enumerate() { + if used[k] { + continue; + } + let d = (bl.centroid[0] - target[0]).hypot(bl.centroid[1] - target[1]); + if d > radius { + continue; + } + let score = if prefer_large { -(bl.pts.len() as f64) } else { d }; + if best.is_none_or(|b| score < b.1) { + best = Some((k, score)); + } + } + if let Some((k, _)) = best { + used[k] = true; + } + best.map(|b| b.0) + }; + let mut det = [[0.0; 2]; N]; + let mut wgt = [0.0; N]; + let set = |i: usize, q: [f64; 2], det: &mut [[f64; 2]; N], wgt: &mut [f64; N]| { + det[i] = to_layer(center, b, roll, q); + wgt[i] = 1.0; + }; + let eye_ids = [EYE[0].to_vec(), EYE[1].to_vec()]; + let mouth = take(mean_of(&MOUTH), 0.25, true, &mut used); + let eyes = [take(mean_of(&eye_ids[0]), 0.15, false, &mut used), take(mean_of(&eye_ids[1]), 0.15, false, &mut used)]; + let brows = [take(mean_of(&BROW[0]), 0.15, false, &mut used), take(mean_of(&BROW[1]), 0.15, false, &mut used)]; + let nostrils = [take(ex[NOSTRIL[0]], 0.07, false, &mut used), take(ex[NOSTRIL[1]], 0.07, false, &mut used)]; + for side in 0..2 { + let sg = if side == 0 { -1.0 } else { 1.0 }; + if let Some(k) = eyes[side] { + let bl = &blobs[k]; + let e = EYE[side]; + let inner = bl.extreme([-sg, 0.0]); + let outer = bl.extreme([sg, 0.0]); + let mid = 0.5 * (inner[0] + outer[0]); + let band = 0.15 * (outer[0] - inner[0]).abs(); + let px = 0.5 / b; + set(e[0], [inner[0] - sg * px, inner[1]], &mut det, &mut wgt); + set(e[1], [outer[0] + sg * px, outer[1]], &mut det, &mut wgt); + let top = bl.lid(mid, band, -1.0); + let bot = bl.lid(mid, band, 1.0); + set(e[2], [top[0], top[1] - px], &mut det, &mut wgt); + set(e[3], [bot[0], bot[1] + px], &mut det, &mut wgt); + set(e[4], bl.dark_centroid(), &mut det, &mut wgt); + } + if let Some(k) = brows[side] { + let bl = &blobs[k]; + let br = BROW[side]; + let inner = bl.extreme([-sg, 0.0]); + let outer = bl.extreme([sg, 0.0]); + set(br[0], inner, &mut det, &mut wgt); + set(br[2], outer, &mut det, &mut wgt); + // The middle: the brow's centre line at its mid column. + let mid = 0.5 * (inner[0] + outer[0]); + let band = 0.1 * (outer[0] - inner[0]).abs().max(0.05); + let col: Vec = bl.pts.iter().filter(|p| (p.0[0] - mid).abs() <= band).map(|p| p.0[1]).collect(); + if !col.is_empty() { + set(br[1], [mid, col.iter().sum::() / col.len() as f64], &mut det, &mut wgt); + } + } + if let Some(k) = nostrils[side] { + set(NOSTRIL[side], blobs[k].centroid, &mut det, &mut wgt); + } + } + if let (Some(a), Some(c)) = (nostrils[0], nostrils[1]) { + let (p, q) = (blobs[a].centroid, blobs[c].centroid); + set(NOSE_TIP, [0.5 * (p[0] + q[0]), 0.5 * (p[1] + q[1]) - 0.06], &mut det, &mut wgt); + } + if let Some(k) = mouth { + let bl = &blobs[k]; + let l = bl.extreme([-1.0, 0.0]); + let r = bl.extreme([1.0, 0.0]); + let px = 0.5 / b; + set(MOUTH[0], [l[0] - px, l[1]], &mut det, &mut wgt); + set(MOUTH[1], [r[0] + px, r[1]], &mut det, &mut wgt); + let mid = 0.5 * (l[0] + r[0]); + let band = 0.12 * (r[0] - l[0]).abs(); + let top = bl.lid(mid, band, -1.0); + let bot = bl.lid(mid, band, 1.0); + set(MOUTH[2], [top[0], top[1] - px], &mut det, &mut wgt); + set(MOUTH[3], [bot[0], bot[1] + px], &mut det, &mut wgt); + } + // Chin and jaw from the outline, along their canonical directions. + for i in [CHIN, JAW[0], JAW[1]] { + let q = LANDMARKS[i].2; + let dir = to_layer([0.0; 2], 1.0, roll, q); + let ang = dir[1].atan2(dir[0]); + let rad = ray_radius(&outline, center, ang); + let len = dir[0].hypot(dir[1]); + let p = [center[0] + dir[0] / len * rad, center[1] + dir[1] / len * rad]; + det[i] = p; + wgt[i] = 1.0; + } + // Shape model: fill the missing landmarks and replace inconsistent detections. + let (sn, cs) = roll.to_radians().sin_cos(); + let init = [b * cs, b * sn, center[0], center[1]]; + let (model, _) = fit_shape(&det, &wgt, init); + let mut w2 = wgt; + for i in 0..N { + if wgt[i] > 0.0 && (det[i][0] - model[i][0]).hypot(det[i][1] - model[i][1]) > 0.12 * b { + w2[i] = 0.0; + } + } + let (model, sim) = if w2 != wgt { fit_shape(&det, &w2, init) } else { (model, init) }; + let _ = sim; + let mut landmarks = model; + let mut detected = [false; N]; + for i in 0..N { + if w2[i] > 0.0 { + landmarks[i] = det[i]; + detected[i] = true; + } + } + // Roll from the jaw line (it does not move with yaw); fall back to the ellipse. + let jl = landmarks[JAW[0]]; + let jr = landmarks[JAW[1]]; + let roll_jaw = (jr[1] - jl[1]).atan2(jr[0] - jl[0]).to_degrees(); + let roll = if (roll_jaw - roll).abs() < 20.0 { roll_jaw } else { roll }; + Some(FaceFit { landmarks, detected, outline, center, height: b, roll }) + } +} + +/// Distance from `c` to the polygon `poly` (star-shaped around `c`) along angle `ang`. +fn ray_radius(poly: &[[f64; 2]], c: [f64; 2], ang: f64) -> f64 { + let d = [ang.cos(), ang.sin()]; + let mut best = 0.0f64; + let n = poly.len(); + for i in 0..n { + let (p, q) = (poly[i], poly[(i + 1) % n]); + let e = [q[0] - p[0], q[1] - p[1]]; + let den = d[0] * e[1] - d[1] * e[0]; + if den.abs() < 1e-12 { + continue; + } + let w = [p[0] - c[0], p[1] - c[1]]; + let t = (w[0] * e[1] - w[1] * e[0]) / den; + let s = (w[0] * d[1] - w[1] * d[0]) / den; + if t > 0.0 && (-1e-9..=1.0 + 1e-9).contains(&s) { + best = best.max(t); + } + } + best +} + +// ---------------------------------------------------------------- measurements + +/// Face measurements (Extract & Copy Face Measurements): (id, display name). +pub const MEASUREMENTS: [(&str, &str); 14] = [ + ("headPositionX", "Head Position X"), + ("headPositionY", "Head Position Y"), + ("headScale", "Head Scale"), + ("headOrientationX", "Head Orientation X"), + ("headOrientationY", "Head Orientation Y"), + ("headOrientationZ", "Head Orientation Z"), + ("leftEyeOpenness", "Left Eye Openness"), + ("rightEyeOpenness", "Right Eye Openness"), + ("leftEyebrowRaise", "Left Eyebrow Distance from Eye"), + ("rightEyebrowRaise", "Right Eyebrow Distance from Eye"), + ("mouthOpenness", "Mouth Openness"), + ("mouthWidth", "Mouth Width"), + ("mouthOffsetX", "Mouth Offset X"), + ("mouthOffsetY", "Mouth Offset Y"), +]; + +/// Face centre, half-height and roll from the chin and jaw landmarks. +pub fn face_frame(l: &[[f64; 2]; N]) -> ([f64; 2], f64, f64) { + let (ch, jl, jr) = (l[CHIN], l[JAW[0]], l[JAW[1]]); + let jm = [0.5 * (jl[0] + jr[0]), 0.5 * (jl[1] + jr[1])]; + let k = 1.0 - LANDMARKS[JAW[0]].2[1]; + let c = [ch[0] + (jm[0] - ch[0]) / k, ch[1] + (jm[1] - ch[1]) / k]; + // Half-height from the chin and from the jaw width (averaged: less noise). + let jw = (jr[0] - jl[0]).hypot(jr[1] - jl[1]) / (2.0 * LANDMARKS[JAW[1]].2[0]); + let b = (0.5 * ((ch[0] - c[0]).hypot(ch[1] - c[1]) + jw)).max(1e-9); + let roll = (jr[1] - jl[1]).atan2(jr[0] - jl[0]).to_degrees(); + (c, b, roll) +} + +/// The measurements of a frame's landmarks ([`MEASUREMENTS`] order); `reference_height`: the +/// face half-height that is Head Scale 100 %. +pub fn measure(l: &[[f64; 2]; N], reference_height: f64) -> [f64; 14] { + let (c, b, roll) = face_frame(l); + let q = l.map(|p| to_face(c, b, roll, p)); + let canon = LANDMARKS.map(|x| x.2); + // Inner features slide with yaw and pitch (see `synth_shape`). + let inner = [EYE[0][4], EYE[1][4], MOUTH[0], MOUTH[1], EYE[0][0], EYE[1][0], EYE[0][1], EYE[1][1]]; + let n = inner.len() as f64; + let du = inner.iter().map(|i| q[*i][0] - canon[*i][0]).sum::() / n; + let dv = inner.iter().map(|i| q[*i][1] - canon[*i][1]).sum::() / n; + let yaw = (du / 0.35).clamp(-1.0, 1.0).asin().to_degrees(); + let pitch = (dv / 0.3).clamp(-1.0, 1.0).asin().to_degrees(); + let open = |e: [usize; 5]| { + let wdt = (q[e[1]][0] - q[e[0]][0]).abs().max(1e-9); + (q[e[3]][1] - q[e[2]][1]).max(0.0) / wdt * 100.0 + }; + let brow = |side: usize| { + let e = EYE[side]; + let eye_v = 0.5 * (q[e[2]][1] + q[e[3]][1]); + (eye_v - q[BROW[side][1]][1]) * 100.0 + }; + let mw = (q[MOUTH[1]][0] - q[MOUTH[0]][0]).abs(); + let mc = [0.5 * (q[MOUTH[0]][0] + q[MOUTH[1]][0]), 0.5 * (q[MOUTH[2]][1] + q[MOUTH[3]][1])]; + [ + c[0], + c[1], + b / reference_height.max(1e-9) * 100.0, + pitch, + yaw, + roll, + open(EYE[0]), + open(EYE[1]), + brow(0), + brow(1), + (q[MOUTH[3]][1] - q[MOUTH[2]][1]).max(0.0) / mw.max(1e-9) * 100.0, + mw / (2.0 * ASPECT) * 100.0, + (mc[0] - q[NOSE_TIP][0]) * 100.0, + (mc[1] - q[NOSE_TIP][1]) * 100.0, + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame_params(t: usize) -> FaceParams { + let tf = t as f64; + FaceParams { + center: [160.0 + 30.0 * (tf * 0.15).sin(), 120.0 + 12.0 * (tf * 0.1).cos()], + height: 62.0 * (1.0 + 0.08 * (tf * 0.07).sin()), + roll: 14.0 * (tf * 0.12).sin(), + yaw: 12.0 * (tf * 0.09).sin(), + pitch: 6.0 * (tf * 0.11).cos(), + eye_open: [0.8 + 0.2 * (tf * 0.3).sin(), 0.8 + 0.2 * (tf * 0.3).sin()], + mouth_open: 0.5 + 0.45 * (tf * 0.2).sin(), + mouth_width: 1.0 + 0.1 * (tf * 0.13).cos(), + brow_raise: [0.4 * (tf * 0.17).sin(), 0.4 * (tf * 0.17).sin()], + } + } + + #[test] + fn shape_model_spans_the_synthetic_shapes() { + let m = pdm(); + assert!(m.modes.len() >= 5 && m.modes.len() <= 14, "{} modes", m.modes.len()); + // A synthetic shape is reproduced from half its landmarks. + let p = FaceParams { yaw: 15.0, mouth_open: 0.8, eye_open: [0.3, 0.9], center: [100.0, 80.0], height: 50.0, roll: 10.0, ..Default::default() }; + let truth = synth_landmarks(&p); + let mut w = [0.0; N]; + for (i, wi) in w.iter_mut().enumerate() { + *wi = if i % 2 == 0 || i >= 19 { 1.0 } else { 0.0 }; + } + let (sn, cs) = 10f64.to_radians().sin_cos(); + let (fit, _) = fit_shape(&truth, &w, [50.0 * cs, 50.0 * sn, 100.0, 80.0]); + let err = (0..N).map(|i| (fit[i][0] - truth[i][0]).hypot(fit[i][1] - truth[i][1])).fold(0.0, f64::max); + assert!(err < 0.06 * 50.0, "max error {err}"); + } + + #[test] + fn tracks_a_moving_rotating_synthetic_face() { + let (w, h) = (320, 240); + let p0 = frame_params(0); + let img0 = render_synthetic(w, h, &p0, 3); + // The user's mask: a loose box around the face. + let region = [p0.center[0] - 60.0, p0.center[1] - 75.0, p0.center[0] + 60.0, p0.center[1] + 75.0]; + let (mut tr, fit0) = FaceTracker::new(&Frame { img: &img0, offset: [0.0; 2] }, region).expect("face found"); + let mut fits = vec![fit0]; + let n = 40; + for t in 1..n { + let img = render_synthetic(w, h, &frame_params(t), 3); + fits.push(tr.step(&Frame { img: &img, offset: [0.0; 2] }).unwrap_or_else(|| panic!("lost at {t}"))); + } + let mut per = [0.0f64; N]; + let mut worst = 0.0f64; + let mut outline_err = 0.0; + let mut meas = vec![]; + for (t, f) in fits.iter().enumerate() { + let p = frame_params(t); + let truth = synth_landmarks(&p); + for i in 0..N { + let e = (f.landmarks[i][0] - truth[i][0]).hypot(f.landmarks[i][1] - truth[i][1]) / p.height; + per[i] += e / n as f64; + worst = worst.max(e); + } + // Outline: on the face ellipse. + let mut oe = 0.0; + for q in &f.outline { + let u = to_face(p.center, p.height, p.roll, *q); + oe += ((u[0] / ASPECT).powi(2) + u[1] * u[1]).sqrt() - 1.0; + } + outline_err += (oe / f.outline.len() as f64).abs() / n as f64; + meas.push((p, measure(&f.landmarks, fits[0].height))); + assert!((f.roll - p.roll).abs() < 3.0, "frame {t}: roll {} vs {}", f.roll, p.roll); + } + let mean = per.iter().sum::() / N as f64; + eprintln!("face landmarks: mean error {mean:.4}, worst {worst:.4} face heights; outline {outline_err:.4}"); + for (i, e) in per.iter().enumerate() { + assert!(*e < 0.04, "{}: mean error {e:.3} (face heights)", LANDMARKS[i].0); + } + assert!(mean < 0.02, "mean landmark error {mean:.4}"); + assert!(worst < 0.08, "worst landmark error {worst:.3}"); + assert!(outline_err < 0.04, "outline error {outline_err:.3}"); + // Measurements follow the generator: mouth openness and yaw correlate strongly. + let corr = |a: &[f64], b: &[f64]| { + let n = a.len() as f64; + let (ma, mb) = (a.iter().sum::() / n, b.iter().sum::() / n); + let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0); + for (x, y) in a.iter().zip(b) { + sab += (x - ma) * (y - mb); + saa += (x - ma).powi(2); + sbb += (y - mb).powi(2); + } + sab / (saa * sbb).sqrt() + }; + let mo: Vec = meas.iter().map(|m| m.1[10]).collect(); + let mt: Vec = meas.iter().map(|m| m.0.mouth_open).collect(); + assert!(corr(&mo, &mt) > 0.95, "mouth openness r = {}", corr(&mo, &mt)); + let yw: Vec = meas.iter().map(|m| m.1[4]).collect(); + let yt: Vec = meas.iter().map(|m| m.0.yaw).collect(); + assert!(corr(&yw, &yt) > 0.9, "yaw r = {}", corr(&yw, &yt)); + let rl: Vec = meas.iter().map(|m| m.1[5]).collect(); + let rt: Vec = meas.iter().map(|m| m.0.roll).collect(); + assert!(corr(&rl, &rt) > 0.98, "roll r = {}", corr(&rl, &rt)); + let sc: Vec = meas.iter().map(|m| m.1[2]).collect(); + let st: Vec = meas.iter().map(|m| m.0.height).collect(); + assert!(corr(&sc, &st) > 0.95, "scale r = {}", corr(&sc, &st)); + } + + #[test] + fn measurements_of_the_synthetic_model_are_exact() { + let p = FaceParams { center: [50.0, 60.0], height: 40.0, roll: 20.0, yaw: 10.0, pitch: -5.0, ..Default::default() }; + let m = measure(&synth_landmarks(&p), 40.0); + assert!((m[0] - 50.0).abs() < 1e-6 && (m[1] - 60.0).abs() < 1e-6, "{m:?}"); + assert!((m[2] - 100.0).abs() < 1e-6); + assert!((m[5] - 20.0).abs() < 1e-6); + assert!((m[4] - 10.0).abs() < 0.5 && (m[3] + 5.0).abs() < 0.5, "{m:?}"); + } +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index 1c6e6a15..f4a7b88f 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -19,6 +19,7 @@ //! after subtracting the frame's `offset` (see [`Frame`]). pub mod camtrack; +pub mod face; pub mod fit; pub mod klt; pub mod mask; @@ -26,6 +27,7 @@ pub mod plane; pub mod roto; pub mod solve; pub mod stabilize; +pub mod subspace; use effectcraft_raster::Image; use rayon::prelude::*; diff --git a/crates/track/src/solve.rs b/crates/track/src/solve.rs index 6c267721..693c52f7 100644 --- a/crates/track/src/solve.rs +++ b/crates/track/src/solve.rs @@ -37,6 +37,12 @@ pub fn solve_linear(mut a: [[f64; N]; N], mut b: [f64; N]) -> Op #[derive(Clone, Copy, Debug, PartialEq)] pub struct Homography(pub [[f64; 3]; 3]); +impl Default for Homography { + fn default() -> Self { + Homography::IDENTITY + } +} + impl Homography { pub const IDENTITY: Homography = Homography([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]); diff --git a/crates/track/src/stabilize.rs b/crates/track/src/stabilize.rs index 1abe79f9..9769fcbd 100644 --- a/crates/track/src/stabilize.rs +++ b/crates/track/src/stabilize.rs @@ -18,14 +18,23 @@ //! Video Stabilization with Robust L1 Optimal Camera Paths" (CVPR 2011), applied here //! per frame rather than through a linear program. //! +//! +//! *Subspace Warp* additionally follows long feature trajectories through the clip +//! ([`crate::camtrack::TrackAnalyzer`]) and stabilises each frame with a content-preserving mesh +//! warp fitted to the subspace-smoothed trajectories ([`crate::subspace`]): scenes with depth +//! (parallax) that one homography per frame cannot hold still are stabilised locally. The +//! framing (crop, auto-scale, Maximum Scale relaxation) works the same on meshes. +//! //! Coordinates are layer pixels; transforms map *source* frame pixels to *stabilized* pixels. use serde::{Deserialize, Serialize}; use crate::Frame; -use crate::fit::{Model, ransac}; +use crate::camtrack::{Track2D, TrackAnalyzer}; +use crate::fit::{Model, least_squares, ransac}; use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; use crate::solve::Homography; +use crate::subspace::{Mesh, MeshOpts, smooth_trajectories, solve_mesh}; /// Inter-frame motion of one frame (from the previous frame to this one) for each model. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -82,6 +91,10 @@ pub struct WarpAnalysis { pub detailed: bool, /// Per frame; frame 0's motion is the identity. pub frames: Vec, + /// Long feature trajectories (layer pixels, 0.1 px) for Subspace Warp; empty in analyses + /// made before it (which then stabilise like Perspective). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tracks: Vec, } impl WarpAnalysis { @@ -120,6 +133,7 @@ pub struct Analyzer { prev: Option, frames: Vec, size: [f64; 2], + trajectories: TrackAnalyzer, } fn round_pts(p: &[[f64; 2]], n: usize) -> Vec<[f32; 2]> { @@ -129,7 +143,8 @@ fn round_pts(p: &[[f64; 2]], n: usize) -> Vec<[f32; 2]> { impl Analyzer { pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> Analyzer { - Analyzer { opts, prev: None, frames: vec![], size } + let (side, want) = if opts.detailed { (640, 260) } else { (480, 160) }; + Analyzer { opts, prev: None, frames: vec![], size, trajectories: TrackAnalyzer::with_budget(size, side, want) } } pub fn len(&self) -> usize { @@ -144,6 +159,7 @@ impl Analyzer { let (max_side, max_feat) = if self.opts.detailed { (960, 700) } else { (480, 350) }; let factor = analysis_factor(frame.img.width, frame.img.height, max_side); let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 3); + self.trajectories.push(frame); let mut fm = FrameMotion::default(); if let Some(prev) = &self.prev && prev.width() == pyr.width() @@ -184,7 +200,17 @@ impl Analyzer { } pub fn finish(self, start: f64, frame_duration: f64) -> WarpAnalysis { - WarpAnalysis { version: 1, start, frame_duration, size: self.size, detailed: self.opts.detailed, frames: self.frames } + let mut tracks = self.trajectories.finish(start, frame_duration).tracks; + // Short trajectories hardly constrain the subspace: keep those of at least 6 frames, at + // 0.1 px (smaller JSON). + tracks.retain(|t| t.pts.len() >= 6); + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 10.0).round() / 10.0, (p[1] * 10.0).round() / 10.0]; + } + } + WarpAnalysis { version: 2, start, frame_duration, size: self.size, detailed: self.opts.detailed, frames: self.frames, tracks } } } @@ -205,11 +231,31 @@ pub enum Method { /// Position, Scale, Rotation. Similarity, Perspective, - /// Approximated by Perspective (see the module docs of the effect). + /// Content-preserving mesh warps fitted to subspace-smoothed trajectories. #[default] SubspaceWarp, } +/// Advanced ▸ Rolling Shutter Ripple: how freely Subspace Warp's mesh may bend row by row (a +/// rolling-shutter camera exposes rows at different times, so shake wobbles the frame). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Ripple { + /// A 12 × 8 mesh. + #[default] + Automatic, + /// Twice the rows and a softer shape term: follows row-wise wobble more closely. + Enhanced, +} + +impl Ripple { + pub fn mesh(self) -> MeshOpts { + match self { + Ripple::Automatic => MeshOpts { cols: 12, rows: 8, alpha: 1.0, prior: 0.02 }, + Ripple::Enhanced => MeshOpts { cols: 12, rows: 16, alpha: 0.5, prior: 0.02 }, + } + } +} + /// Borders ▸ Framing. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub enum Framing { @@ -239,6 +285,7 @@ pub struct StabSettings { pub crop_less_smooth_more: f64, /// Frames per second (for Smoothness). pub fps: f64, + pub ripple: Ripple, } impl Default for StabSettings { @@ -254,6 +301,7 @@ impl Default for StabSettings { additional_scale: 100.0, crop_less_smooth_more: 0.0, fps: 30.0, + ripple: Ripple::Automatic, } } } @@ -269,8 +317,15 @@ impl StabSettings { /// The per-frame stabilization of a clip. #[derive(Clone, Debug, Default)] pub struct Plan { - /// Final source → output transform per frame (layer pixels), framing scale included. + /// Final source → output transform per frame (layer pixels), framing scale included. For + /// mesh warps, the closest homography (for overlays and edge synthesis). pub warps: Vec, + /// Subspace Warp: per frame, the mesh (stabilised pixels before framing → source pixels); + /// empty for the other methods. + pub meshes: Vec, + /// The framing transform (auto-scale and Additional Scale about the centre) after the + /// stabilisation. + pub framing: Homography, /// The visible region in output pixels `[x0, y0, x1, y1]` (Stabilize, Crop / Auto-scale); /// `None` = everything the warp produces. pub crop: Option<[f64; 4]>, @@ -280,6 +335,24 @@ pub struct Plan { pub valid_fraction: f64, } +impl Plan { + /// The source position (frame `k`'s layer pixels) shown at output point `p`. + pub fn source_of(&self, k: usize, p: [f64; 2]) -> Option<[f64; 2]> { + if let Some(m) = self.meshes.get(k) { + let inv = self.framing.inverse()?; + return Some(m.sample(inv.apply(p))); + } + Some(self.warps.get(k)?.inverse()?.apply(p)) + } + /// Where source point `q` of frame `k` lands in the output. + pub fn output_of(&self, k: usize, q: [f64; 2]) -> Option<[f64; 2]> { + if let Some(m) = self.meshes.get(k) { + return m.forward(q).map(|p| self.framing.apply(p)); + } + Some(self.warps.get(k)?.apply(q)) + } +} + fn convex_contains(q: &[[f64; 2]; 4], p: [f64; 2]) -> bool { let mut sign = 0.0; for i in 0..4 { @@ -418,14 +491,100 @@ pub fn corrections(a: &WarpAnalysis, s: &StabSettings) -> Vec { out } +/// One frame's stabilisation before framing. +#[derive(Clone, Debug)] +enum Warp { + H(Homography), + M(Mesh), +} + +impl Warp { + fn valid(&self, size: [f64; 2]) -> f64 { + match self { + Warp::H(h) => valid_fraction(h, size), + Warp::M(m) => m.valid_fraction(), + } + } + /// Blend towards the identity (`l` = 1 keeps the warp). + fn relax(&self, l: f64) -> Warp { + match self { + Warp::H(h) => Warp::H(Homography::IDENTITY.lerp(h, l)), + Warp::M(m) => Warp::M(m.lerp_identity(l)), + } + } + fn homography(&self) -> Homography { + match self { + Warp::H(h) => *h, + Warp::M(m) => m.homography(), + } + } +} + +/// Remove the uniform scale of the correspondences' best similarity (Preserve Scale): the +/// wanted positions are scaled back about their centroid. +fn unscale(pairs: &mut [([f64; 2], [f64; 2])]) { + let (src, dst): (Vec<[f64; 2]>, Vec<[f64; 2]>) = pairs.iter().copied().unzip(); + let Some(h) = least_squares(Model::Similarity, &src, &dst) else { return }; + let m = h.0; + let k = (m[0][0] * m[1][1] - m[0][1] * m[1][0]).abs().sqrt(); + if k < 1e-6 { + return; + } + let n = dst.len() as f64; + let c = [dst.iter().map(|q| q[0]).sum::() / n, dst.iter().map(|q| q[1]).sum::() / n]; + for (_, d) in pairs.iter_mut() { + *d = [c[0] + (d[0] - c[0]) / k, c[1] + (d[1] - c[1]) / k]; + } +} + +/// Subspace Warp's meshes (stabilised → source, before framing) for every frame; `None` for the +/// other methods and for analyses without trajectories. `base`: the perspective corrections +/// ([`corrections`]), the meshes' prior. +pub fn meshes(a: &WarpAnalysis, s: &StabSettings, base: &[Homography]) -> Option> { + if s.method != Method::SubspaceWarp || a.tracks.is_empty() || base.len() != a.frames.len() { + return None; + } + let n = a.frames.len(); + let o = s.ripple.mesh(); + let size = a.size; + let pairs: Vec> = match s.result { + StabResult::NoMotion => (0..n).map(|t| a.tracks.iter().filter_map(|tr| tr.at(t as u32)).map(|p| (p, base[t].apply(p))).collect()).collect(), + StabResult::SmoothMotion => { + let sigma = s.sigma(); + if sigma < 0.3 { + return Some((0..n).map(|_| Mesh::identity(o.cols, o.rows, size)).collect()); + } + smooth_trajectories(&a.tracks, n, sigma, 9) + } + }; + use rayon::prelude::*; + Some( + pairs + .into_par_iter() + .enumerate() + .map(|(t, mut pr)| { + if s.preserve_scale && pr.len() >= 2 { + unscale(&mut pr); + } + if pr.len() < 6 { Mesh::from_homography(o.cols, o.rows, size, &base[t]) } else { solve_mesh(size, &o, &pr, &base[t]) } + }) + .collect(), + ) +} + /// Stabilization plan: corrections plus framing. pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { - let mut b = corrections(a, s); + use rayon::prelude::*; + let base = corrections(a, s); + let mut b: Vec = match meshes(a, s, &base) { + Some(m) => m.into_iter().map(Warp::M).collect(), + None => base.into_iter().map(Warp::H).collect(), + }; let size = a.size; let c = [size[0] / 2.0, size[1] / 2.0]; let add = (s.additional_scale / 100.0).max(0.01); let z_add = Homography::scale_about(add, c); - let fracs = |b: &[Homography]| b.iter().map(|h| valid_fraction(h, size)).collect::>(); + let fracs = |b: &[Warp]| b.par_iter().map(|w| w.valid(size)).collect::>(); let mut fr = fracs(&b); let mut vmin = fr.iter().copied().fold(1.0, f64::min); let margin = (s.action_safe / 100.0).clamp(0.0, 0.45); @@ -435,22 +594,24 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { // Relax the correction on frames that need more than Maximum Scale. let target = need_cover / max_scale; let n = b.len(); - let mut lam = vec![1.0f64; n]; - for t in 0..n { - if fr[t] >= target { - continue; - } - let (mut lo, mut hi) = (0.0, 1.0); - for _ in 0..24 { - let mid = 0.5 * (lo + hi); - if valid_fraction(&Homography::IDENTITY.lerp(&b[t], mid), size) >= target { - lo = mid; - } else { - hi = mid; + let lam: Vec = (0..n) + .into_par_iter() + .map(|t| { + if fr[t] >= target { + return 1.0; } - } - lam[t] = lo; - } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..24 { + let mid = 0.5 * (lo + hi); + if b[t].relax(mid).valid(size) >= target { + lo = mid; + } else { + hi = mid; + } + } + lo + }) + .collect(); // Spread the relaxation smoothly (min filter, then a Gaussian of the same radius) so // the camera doesn't jump. let r = ((s.sigma() * 0.5).ceil() as usize).max(1); @@ -468,7 +629,7 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { (acc / ws).min(lam[t]) }) .collect(); - b = b.iter().zip(&smooth).map(|(h, l)| if *l >= 1.0 { *h } else { Homography::IDENTITY.lerp(h, *l) }).collect(); + b = b.iter().zip(&smooth).map(|(w, l)| if *l >= 1.0 { w.clone() } else { w.relax(*l) }).collect(); fr = fracs(&b); vmin = fr.iter().copied().fold(1.0, f64::min); } @@ -483,13 +644,23 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { } }; let z = Homography::scale_about(auto, c); - let warps = b.iter().map(|h| z_add.then_after(&z.then_after(h))).collect(); - Plan { warps, crop, auto_scale: auto, valid_fraction: vmin } + let framing = z_add.then_after(&z); + let warps = b.par_iter().map(|w| framing.then_after(&w.homography())).collect(); + let meshes = b + .into_iter() + .filter_map(|w| match w { + Warp::M(m) => Some(m), + Warp::H(_) => None, + }) + .collect(); + Plan { warps, meshes, framing, crop, auto_scale: auto, valid_fraction: vmin } } #[cfg(test)] mod tests { use super::*; + use effectcraft_raster::Image; + use rayon::prelude::*; /// A synthetic analysis: a slow pan plus per-frame jitter. fn shaky(n: usize) -> (WarpAnalysis, Vec<[f64; 2]>) { @@ -497,7 +668,7 @@ mod tests { let j = [((k * 7919) % 13) as f64 - 6.0, ((k * 104729) % 11) as f64 - 5.0]; [2.0 * k as f64 + j[0], 0.5 * k as f64 + j[1]] }; - let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 30.0, size: [640.0, 360.0], detailed: false, frames: vec![] }; + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 30.0, size: [640.0, 360.0], detailed: false, frames: vec![], tracks: vec![] }; let path: Vec<[f64; 2]> = (0..n).map(pos).collect(); for k in 0..n { let mut f = FrameMotion::default(); @@ -573,4 +744,139 @@ mod tests { let j = WarpAnalysis::from_json(&a.to_json()).unwrap(); assert_eq!(j, a); } + + // ------------------------------------------------------------ synthetic shaky footage + + fn hash(i: i64, j: i64, k: i64) -> f64 { + let mut v = (i.wrapping_mul(73_856_093) ^ j.wrapping_mul(19_349_663) ^ k.wrapping_mul(83_492_791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1_e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + + fn value_noise(x: f64, y: f64, cell: f64, seed: i64) -> f64 { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (u, v) = (fx - i as f64, fy - j as f64); + let (u, v) = (u * u * (3.0 - 2.0 * u), v * v * (3.0 - 2.0 * v)); + let a = hash(i, j, seed) * (1.0 - u) + hash(i + 1, j, seed) * u; + let b = hash(i, j + 1, seed) * (1.0 - u) + hash(i + 1, j + 1, seed) * u; + a * (1.0 - v) + b * v + } + + fn texture(x: f64, y: f64) -> f32 { + (0.55 * value_noise(x, y, 5.0, 1) + 0.45 * value_noise(x, y, 13.0, 2)) as f32 + } + + /// A scene with depth: a near bump in the middle moves `1 + depth` times as much as the + /// background when the camera shakes (parallax a homography cannot model). + struct Scene { + size: [f64; 2], + cams: Vec<[f64; 2]>, + } + + impl Scene { + fn new(n: usize) -> Scene { + let cams = (0..n) + .map(|t| { + let j = [5.0 * (hash(t as i64, 7, 3) - 0.5), 5.0 * (hash(t as i64, 11, 5) - 0.5)]; + [0.4 * t as f64 + j[0], 0.15 * t as f64 + j[1]] + }) + .collect(); + Scene { size: [320.0, 180.0], cams } + } + fn depth(&self, s: [f64; 2]) -> f64 { + 1.0 + 1.2 * (-((s[0] - 160.0).powi(2) + (s[1] - 90.0).powi(2)) / (2.0 * 38.0f64.powi(2))).exp() + } + /// Where scene point `s` appears on frame `t`. + fn image_of(&self, t: usize, s: [f64; 2]) -> [f64; 2] { + let (c, d) = (self.cams[t], self.depth(s)); + [s[0] + c[0] * d, s[1] + c[1] * d] + } + fn frame(&self, t: usize) -> Image { + let (w, h) = (self.size[0] as u32, self.size[1] as u32); + let mut im = Image::new(w, h); + let c = self.cams[t]; + im.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let mut s = p; + for _ in 0..10 { + let d = self.depth(s); + s = [p[0] - c[0] * d, p[1] - c[1] * d]; + } + let v = texture(s[0], s[1]); + *px = [v, v, v, 1.0]; + } + }); + im + } + } + + fn residual_jitter(scene: &Scene, out: impl Fn(usize, [f64; 2]) -> Option<[f64; 2]>) -> f64 { + let n = scene.cams.len(); + let mut tot = 0.0; + let mut cnt = 0; + for gy in 0..5 { + for gx in 0..7 { + let s = [70.0 + gx as f64 * 30.0, 45.0 + gy as f64 * 22.0]; + let ys: Vec> = (0..n).map(|t| out(t, scene.image_of(t, s))).collect(); + for t in 6..n - 6 { + if let (Some(a), Some(b), Some(c)) = (ys[t - 1], ys[t], ys[t + 1]) { + tot += (c[0] - 2.0 * b[0] + a[0]).hypot(c[1] - 2.0 * b[1] + a[1]); + cnt += 1; + } + } + } + } + tot / cnt.max(1) as f64 + } + + #[test] + fn subspace_warp_beats_perspective_on_parallax() { + let n = 48; + let scene = Scene::new(n); + let mut an = Analyzer::new(scene.size, AnalyzeOpts::default()); + for t in 0..n { + let img = scene.frame(t); + an.push(&Frame { img: &img, offset: [0.0; 2] }); + } + let a = an.finish(0.0, 1.0 / 24.0); + assert!(a.tracks.len() >= 40, "trajectories: {}", a.tracks.len()); + // The analysis round-trips with its trajectories. + assert_eq!(WarpAnalysis::from_json(&a.to_json()).unwrap(), a); + let base = StabSettings { framing: Framing::StabilizeOnly, fps: 24.0, ..Default::default() }; + let before = residual_jitter(&scene, |_, x| Some(x)); + let ppl = plan(&a, &StabSettings { method: Method::Perspective, ..base }); + assert!(ppl.meshes.is_empty()); + let persp = residual_jitter(&scene, |t, x| ppl.output_of(t, x)); + let spl = plan(&a, &StabSettings { method: Method::SubspaceWarp, ..base }); + assert_eq!(spl.meshes.len(), n); + let sub = residual_jitter(&scene, |t, x| spl.output_of(t, x)); + let epl = plan(&a, &StabSettings { method: Method::SubspaceWarp, ripple: Ripple::Enhanced, ..base }); + let enh = residual_jitter(&scene, |t, x| epl.output_of(t, x)); + eprintln!("jitter: before {before:.3}, perspective {persp:.3}, subspace {sub:.3}, enhanced {enh:.3}"); + assert!(persp < 0.8 * before, "perspective {persp} vs {before}"); + assert!(sub < 0.6 * persp, "subspace {sub} vs perspective {persp}"); + assert!(enh < 0.6 * persp, "enhanced {enh} vs perspective {persp}"); + // source_of and output_of agree. + let q = [150.0, 80.0]; + let o = spl.output_of(10, q).unwrap(); + let back = spl.source_of(10, o).unwrap(); + assert!((back[0] - q[0]).abs() < 1e-3 && (back[1] - q[1]).abs() < 1e-3); + // Auto-scale framing covers the frame with mesh warps too. + let fpl = plan(&a, &StabSettings { method: Method::SubspaceWarp, fps: 24.0, ..Default::default() }); + assert!(fpl.auto_scale >= 1.0); + for k in [0, n / 2, n - 1] { + for p in [[1.0, 1.0], [319.0, 1.0], [319.0, 179.0], [1.0, 179.0]] { + let q = fpl.source_of(k, p).unwrap(); + let inside = q[0] >= -0.5 && q[1] >= -0.5 && q[0] <= 320.5 && q[1] <= 180.5; + assert!(inside || fpl.crop.is_some(), "frame {k}: {p:?} → {q:?}"); + } + } + // Preserve Scale keeps working with meshes. + let psp = plan(&a, &StabSettings { method: Method::SubspaceWarp, preserve_scale: true, ..base }); + assert_eq!(psp.meshes.len(), n); + } } diff --git a/crates/track/src/subspace.rs b/crates/track/src/subspace.rs new file mode 100644 index 00000000..339f9708 --- /dev/null +++ b/crates/track/src/subspace.rs @@ -0,0 +1,447 @@ +//! Warp Stabilizer ▸ Method ▸ **Subspace Warp**: content-preserving warps fitted to smoothed +//! feature trajectories, clean-room from the published method: +//! +//! - Liu, Gleicher, Wang, Jin & Agarwala, "Subspace Video Stabilization", ACM TOG 30(1), 2011: +//! the feature trajectories of a short window form a matrix `M` (two rows per trajectory, one +//! column per frame) that is close to low rank (rank 9 for a moving camera over a static +//! scene); its factorisation `M ≈ W C` gives *eigen-trajectories* `C`, which are smoothed (here +//! with a Gaussian) instead of the trajectories themselves. Every trajectory's smoothed +//! position is `W_i Ĉ_t`; trajectories that do not span the whole window get their +//! coefficients `W_i` by least squares over the frames they cover (the paper's handling of +//! incomplete trajectories in a moving factorisation). Smoothing in the subspace keeps the +//! smoothed trajectories consistent with one another, so the warp does not tear the scene. +//! - Liu, Gleicher, Jin & Agarwala, "Content-Preserving Warps for 3D Video Stabilization", ACM +//! TOG 28(3), 2009: each frame is warped by a grid mesh fitted to the feature correspondences +//! (original → smoothed position) in least squares, with a **similarity term** asking every +//! grid triangle to keep its shape (`V1 = V2 + u (V3 − V2) + v R90 (V3 − V2)` with `u, v` from +//! the undeformed grid) so the warp bends where the scene's depth demands it but stays locally +//! rigid elsewhere. +//! +//! The mesh is stored *inversely*: a regular grid over the stabilised frame whose vertices hold +//! the source position they show ([`Mesh`]), so rendering is a bilinear lookup per output pixel. +//! The data term is the same linear constraint read the other way (the smoothed position's grid +//! cell, bilinearly, must land on the original position). A weak prior ties every vertex to the +//! frame's perspective correction, which fills feature-less regions and keeps the system well +//! posed. + +use rayon::prelude::*; + +use crate::camtrack::Track2D; +use crate::camtrack::linalg::{cholesky_solve, sym_eigen}; +use crate::fit::{Model, least_squares}; +use crate::solve::Homography; + +/// A warp mesh: a regular `cols × rows` grid over the stabilised frame (`size`), each vertex +/// holding the source-frame position it shows. +#[derive(Clone, Debug, PartialEq)] +pub struct Mesh { + pub cols: usize, + pub rows: usize, + pub size: [f64; 2], + /// `(rows + 1) × (cols + 1)` source positions, row-major. + pub verts: Vec<[f64; 2]>, +} + +impl Mesh { + /// The identity mesh. + pub fn identity(cols: usize, rows: usize, size: [f64; 2]) -> Mesh { + let mut m = Mesh { cols: cols.max(1), rows: rows.max(1), size, verts: vec![] }; + m.verts = (0..=m.rows).flat_map(|j| (0..=m.cols).map(move |i| (i, j))).map(|(i, j)| m.grid(i, j)).collect(); + m + } + + /// The mesh of a homography (`h`: source → stabilised). + pub fn from_homography(cols: usize, rows: usize, size: [f64; 2], h: &Homography) -> Mesh { + let inv = h.inverse().unwrap_or(Homography::IDENTITY); + let mut m = Mesh::identity(cols, rows, size); + for v in m.verts.iter_mut() { + *v = inv.apply(*v); + } + m + } + + fn cell(&self) -> [f64; 2] { + [self.size[0] / self.cols as f64, self.size[1] / self.rows as f64] + } + + /// The undeformed position of vertex `(i, j)` (column, row). + pub fn grid(&self, i: usize, j: usize) -> [f64; 2] { + let c = self.cell(); + [i as f64 * c[0], j as f64 * c[1]] + } + + fn idx(&self, i: usize, j: usize) -> usize { + j * (self.cols + 1) + i + } + + /// Cell and bilinear coordinates of a stabilised point (the outer cells extend beyond the + /// frame, so points outside it extrapolate). + fn locate(&self, p: [f64; 2]) -> (usize, usize, f64, f64) { + let c = self.cell(); + let fx = p[0] / c[0]; + let fy = p[1] / c[1]; + let i = (fx.floor().max(0.0) as usize).min(self.cols - 1); + let j = (fy.floor().max(0.0) as usize).min(self.rows - 1); + (i, j, fx - i as f64, fy - j as f64) + } + + /// The source position shown at stabilised point `p`. + pub fn sample(&self, p: [f64; 2]) -> [f64; 2] { + let (i, j, u, v) = self.locate(p); + let a = self.verts[self.idx(i, j)]; + let b = self.verts[self.idx(i + 1, j)]; + let c = self.verts[self.idx(i, j + 1)]; + let d = self.verts[self.idx(i + 1, j + 1)]; + let w = [(1.0 - u) * (1.0 - v), u * (1.0 - v), (1.0 - u) * v, u * v]; + [0, 1].map(|k| w[0] * a[k] + w[1] * b[k] + w[2] * c[k] + w[3] * d[k]) + } + + /// Where source point `q` lands in the stabilised frame (Newton iterations on + /// [`Mesh::sample`]); `None` when it does not converge (a folded mesh). + pub fn forward(&self, q: [f64; 2]) -> Option<[f64; 2]> { + // Start from the mesh's own offset at q. + let s = self.sample(q); + let mut p = [2.0 * q[0] - s[0], 2.0 * q[1] - s[1]]; + for _ in 0..30 { + let f = self.sample(p); + let e = [f[0] - q[0], f[1] - q[1]]; + if e[0].hypot(e[1]) < 1e-6 { + return Some(p); + } + let h = 0.5; + let fx = self.sample([p[0] + h, p[1]]); + let fy = self.sample([p[0], p[1] + h]); + let j = [[(fx[0] - f[0]) / h, (fy[0] - f[0]) / h], [(fx[1] - f[1]) / h, (fy[1] - f[1]) / h]]; + let det = j[0][0] * j[1][1] - j[0][1] * j[1][0]; + if det.abs() < 1e-12 { + return None; + } + p[0] -= (j[1][1] * e[0] - j[0][1] * e[1]) / det; + p[1] -= (-j[1][0] * e[0] + j[0][0] * e[1]) / det; + } + let f = self.sample(p); + ((f[0] - q[0]).hypot(f[1] - q[1]) < 1e-3).then_some(p) + } + + /// `(1 − t) · identity + t · self` per vertex. + pub fn lerp_identity(&self, t: f64) -> Mesh { + let mut m = self.clone(); + for j in 0..=self.rows { + for i in 0..=self.cols { + let g = self.grid(i, j); + let k = self.idx(i, j); + m.verts[k] = [g[0] + (self.verts[k][0] - g[0]) * t, g[1] + (self.verts[k][1] - g[1]) * t]; + } + } + m + } + + /// Largest `s ∈ [0, 1]` such that the centred `s·W × s·H` rectangle of the stabilised frame + /// shows only source pixels (its outline, sampled densely, maps inside the source frame). + pub fn valid_fraction(&self) -> f64 { + let [w, h] = self.size; + let c = [w / 2.0, h / 2.0]; + let tol = 1e-6; + let inside = |q: [f64; 2]| q[0] >= -tol && q[1] >= -tol && q[0] <= w + tol && q[1] <= h + tol; + let fits = |s: f64| { + let (hw, hh) = (s * w / 2.0, s * h / 2.0); + let n = 48; + (0..=n).all(|k| { + let t = k as f64 / n as f64; + let x = c[0] - hw + 2.0 * hw * t; + let y = c[1] - hh + 2.0 * hh * t; + inside(self.sample([x, c[1] - hh])) + && inside(self.sample([x, c[1] + hh])) + && inside(self.sample([c[0] - hw, y])) + && inside(self.sample([c[0] + hw, y])) + }) + }; + if fits(1.0) { + return 1.0; + } + if !fits(0.0) { + return 0.0; + } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..24 { + let mid = 0.5 * (lo + hi); + if fits(mid) { + lo = mid; + } else { + hi = mid; + } + } + lo + } + + /// The homography (source → stabilised) closest to the mesh in least squares. + pub fn homography(&self) -> Homography { + let mut src = vec![]; + let mut dst = vec![]; + for j in 0..=self.rows { + for i in 0..=self.cols { + src.push(self.verts[self.idx(i, j)]); + dst.push(self.grid(i, j)); + } + } + least_squares(Model::Homography, &src, &dst).unwrap_or(Homography::IDENTITY) + } +} + +/// Mesh resolution and rigidity. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct MeshOpts { + pub cols: usize, + pub rows: usize, + /// Weight of the similarity (shape-preserving) term relative to one feature. + pub alpha: f64, + /// Weight of the prior towards the frame's perspective correction, per vertex. + pub prior: f64, +} + +impl Default for MeshOpts { + fn default() -> Self { + MeshOpts { cols: 12, rows: 8, alpha: 1.0, prior: 0.02 } + } +} + +/// Normal equations `AᵀA x = Aᵀb`, accumulated row by row. +struct Normal { + n: usize, + a: Vec, + b: Vec, +} + +impl Normal { + fn row(&mut self, coefs: &[(usize, f64)], rhs: f64, w: f64) { + for (ia, ca) in coefs { + self.b[*ia] += w * ca * rhs; + for (ib, cb) in coefs { + self.a[ia * self.n + ib] += w * ca * cb; + } + } + } +} + +/// Fit the content-preserving warp of one frame: `pairs` are (source position, wanted +/// stabilised position) of its features; `base` is the frame's perspective correction (source → +/// stabilised), the prior. +pub fn solve_mesh(size: [f64; 2], o: &MeshOpts, pairs: &[([f64; 2], [f64; 2])], base: &Homography) -> Mesh { + let mut m = Mesh::from_homography(o.cols, o.rows, size, base); + let nv = (m.cols + 1) * (m.rows + 1); + let n = 2 * nv; + let mut ne = Normal { n, a: vec![0.0; n * n], b: vec![0.0; n] }; + // Prior. + for k in 0..nv { + let v = m.verts[k]; + ne.row(&[(2 * k, 1.0)], v[0], o.prior); + ne.row(&[(2 * k + 1, 1.0)], v[1], o.prior); + } + // Data: the bilinear combination of the wanted position's cell lands on the source position. + let mut used = 0; + for (src, dst) in pairs { + if !(dst[0] >= 0.0 && dst[1] >= 0.0 && dst[0] <= size[0] && dst[1] <= size[1] && src[0].is_finite() && src[1].is_finite()) { + continue; + } + let (i, j, u, v) = m.locate(*dst); + let ks = [m.idx(i, j), m.idx(i + 1, j), m.idx(i, j + 1), m.idx(i + 1, j + 1)]; + let w = [(1.0 - u) * (1.0 - v), u * (1.0 - v), (1.0 - u) * v, u * v]; + for d in 0..2 { + let coefs: Vec<(usize, f64)> = ks.iter().zip(w).map(|(k, w)| (2 * k + d, w)).collect(); + ne.row(&coefs, src[d], 1.0); + } + used += 1; + } + if used == 0 { + return m; + } + // Similarity: four triangles per cell, each corner against its two cell neighbours. + let a2 = o.alpha * o.alpha; + for j in 0..m.rows { + for i in 0..m.cols { + let (a, b, c, d) = ((i, j), (i + 1, j), (i + 1, j + 1), (i, j + 1)); + for (v1, v2, v3) in [(a, b, d), (b, c, a), (c, d, b), (d, a, c)] { + let (g1, g2, g3) = (m.grid(v1.0, v1.1), m.grid(v2.0, v2.1), m.grid(v3.0, v3.1)); + let dv = [g3[0] - g2[0], g3[1] - g2[1]]; + let e = [g1[0] - g2[0], g1[1] - g2[1]]; + let l2 = dv[0] * dv[0] + dv[1] * dv[1]; + // R90(d) = (−d.y, d.x). + let u = (e[0] * dv[0] + e[1] * dv[1]) / l2; + let v = (e[0] * -dv[1] + e[1] * dv[0]) / l2; + let (k1, k2, k3) = (m.idx(v1.0, v1.1), m.idx(v2.0, v2.1), m.idx(v3.0, v3.1)); + // x: V1x − V2x − u (V3x − V2x) + v (V3y − V2y) = 0 + ne.row(&[(2 * k1, 1.0), (2 * k2, u - 1.0), (2 * k3, -u), (2 * k3 + 1, v), (2 * k2 + 1, -v)], 0.0, a2); + // y: V1y − V2y − u (V3y − V2y) − v (V3x − V2x) = 0 + ne.row(&[(2 * k1 + 1, 1.0), (2 * k2 + 1, u - 1.0), (2 * k3 + 1, -u), (2 * k3, -v), (2 * k2, v)], 0.0, a2); + } + } + } + let mut x = ne.b; + if cholesky_solve(&mut ne.a, n, &mut x) { + for k in 0..nv { + m.verts[k] = [x[2 * k], x[2 * k + 1]]; + } + } + m +} + +/// Gaussian weights of the frames `lo..=hi` around `t`. +fn gauss(lo: usize, hi: usize, t: usize, sigma: f64) -> Vec { + let g: Vec = (lo..=hi).map(|j| (-((j as f64 - t as f64).powi(2)) / (2.0 * sigma * sigma)).exp()).collect(); + let s: f64 = g.iter().sum(); + g.into_iter().map(|v| v / s.max(1e-300)).collect() +} + +/// Subspace-smoothed feature positions: for every frame `t < frames`, the (original, smoothed) +/// positions of the trajectories seen on it. `sigma`: Gaussian smoothing in frames; `rank`: the +/// subspace dimension (9 in the paper). Frames without enough trajectories get no pairs. +pub fn smooth_trajectories(tracks: &[Track2D], frames: usize, sigma: f64, rank: usize) -> Vec> { + let max_r = ((3.0 * sigma).ceil() as usize).clamp(1, 30); + (0..frames) + .into_par_iter() + .map(|t| { + let t32 = t as u32; + let mut r = max_r; + // The window around t: shrink it until enough trajectories span it. + let (lo, hi, complete) = loop { + let lo = t.saturating_sub(r); + let hi = (t + r).min(frames.saturating_sub(1)); + let complete: Vec<&Track2D> = tracks.iter().filter(|tr| tr.start as usize <= lo && tr.end() as usize > hi).collect(); + if complete.len() >= 8 || r <= 2 { + break (lo, hi, complete); + } + r = (r * 2 / 3).max(2); + }; + if complete.len() < 4 { + return vec![]; + } + let m = hi - lo + 1; + let rows = 2 * complete.len(); + let k = rank.min(m).min(rows).max(1); + // M: rows × m (row-major). + let mut mat = vec![0.0; rows * m]; + for (i, tr) in complete.iter().enumerate() { + for c in 0..m { + let p = tr.at((lo + c) as u32).unwrap_or([0.0; 2]); + mat[(2 * i) * m + c] = p[0]; + mat[(2 * i + 1) * m + c] = p[1]; + } + } + // Gram MᵀM and its top eigenvectors: the eigen-trajectories (orthonormal rows of C). + let mut gram = vec![0.0; m * m]; + for a in 0..m { + for b in a..m { + let s: f64 = (0..rows).map(|r| mat[r * m + a] * mat[r * m + b]).sum(); + gram[a * m + b] = s; + gram[b * m + a] = s; + } + } + let (_, vecs) = sym_eigen(&gram, m); + // C[q][c] = vecs[c][m − 1 − q] (largest first). + let cmat: Vec> = (0..k).map(|q| (0..m).map(|c| vecs[c * m + (m - 1 - q)]).collect()).collect(); + let g = gauss(lo, hi, t, sigma.max(1e-6)); + let chat: Vec = cmat.iter().map(|row| row.iter().zip(&g).map(|(a, b)| a * b).sum()).collect(); + let tc = t - lo; + let mut out = Vec::new(); + // Complete trajectories: W = M Cᵀ. + for (i, _) in complete.iter().enumerate() { + let src = [mat[(2 * i) * m + tc], mat[(2 * i + 1) * m + tc]]; + let mut dst = [0.0; 2]; + for (d, slot) in dst.iter_mut().enumerate() { + let row = &mat[(2 * i + d) * m..(2 * i + d + 1) * m]; + for q in 0..k { + let w: f64 = row.iter().zip(&cmat[q]).map(|(a, b)| a * b).sum(); + *slot += w * chat[q]; + } + } + out.push((src, dst)); + } + // Incomplete trajectories seen on t: coefficients by least squares over the frames + // they cover in the window. + let need = (k + 3).max(m / 2); + for tr in tracks.iter().filter(|tr| tr.at(t32).is_some() && !(tr.start as usize <= lo && tr.end() as usize > hi)) { + let cols: Vec = (0..m).filter(|c| tr.at((lo + c) as u32).is_some()).collect(); + if cols.len() < need { + continue; + } + let mut ata = vec![0.0; k * k]; + let mut atb = [vec![0.0; k], vec![0.0; k]]; + for &c in &cols { + let p = tr.at((lo + c) as u32).expect("covered"); + for a in 0..k { + for d in 0..2 { + atb[d][a] += cmat[a][c] * p[d]; + } + for b in 0..k { + ata[a * k + b] += cmat[a][c] * cmat[b][c]; + } + } + } + for a in 0..k { + ata[a * k + a] += 1e-9; + } + let mut dst = [0.0; 2]; + let mut ok = true; + for d in 0..2 { + let mut x = atb[d].clone(); + let mut aa = ata.clone(); + if !cholesky_solve(&mut aa, k, &mut x) { + ok = false; + break; + } + dst[d] = x.iter().zip(&chat).map(|(a, b)| a * b).sum(); + } + if ok { + out.push((tr.at(t32).expect("seen"), dst)); + } + } + out + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mesh_identity_sample_and_forward() { + let m = Mesh::identity(4, 3, [100.0, 60.0]); + assert_eq!(m.sample([37.0, 21.0]), [37.0, 21.0]); + assert_eq!(m.valid_fraction(), 1.0); + let h = Homography([[1.0, 0.02, 5.0], [-0.01, 1.0, -3.0], [0.0, 0.0, 1.0]]); + let mh = Mesh::from_homography(4, 3, [100.0, 60.0], &h); + // Affine maps are exact on a bilinear mesh. + let q = [40.0, 30.0]; + let p = mh.forward(q).unwrap(); + let e = h.apply(q); + assert!((p[0] - e[0]).abs() < 1e-6 && (p[1] - e[1]).abs() < 1e-6, "{p:?} {e:?}"); + assert!(mh.homography().max_diff(&h) < 1e-6); + assert!(mh.valid_fraction() < 1.0); + } + + #[test] + fn mesh_fits_a_translation_and_bends_for_local_motion() { + let size = [200.0, 120.0]; + let mut pairs = vec![]; + for y in 0..12 { + for x in 0..20 { + let p = [5.0 + x as f64 * 10.0, 5.0 + y as f64 * 10.0]; + // Everything moves by (3, −2); a blob near the centre by (8, −2). + let bump = (-((p[0] - 100.0).powi(2) + (p[1] - 60.0).powi(2)) / (2.0 * 20.0f64.powi(2))).exp(); + pairs.push((p, [p[0] + 3.0 + 5.0 * bump, p[1] - 2.0])); + } + } + let base = Homography::translation([3.0, -2.0]); + let m = solve_mesh(size, &MeshOpts { cols: 10, rows: 6, alpha: 0.5, prior: 0.01 }, &pairs, &base); + let mut err_mesh = 0.0; + let mut err_h = 0.0; + for (s, d) in &pairs { + let q = m.sample(*d); + err_mesh += (q[0] - s[0]).hypot(q[1] - s[1]); + let qh = base.inverse().unwrap().apply(*d); + err_h += (qh[0] - s[0]).hypot(qh[1] - s[1]); + } + assert!(err_mesh < 0.35 * err_h, "{err_mesh} vs {err_h}"); + } +} diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 316b4ea3..28c04837 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -36,6 +36,8 @@ pub enum PanelKind { MaskInterpolation, ScriptConsole, EssentialGraphics, + /// A dockable ScriptUI panel (a script from the ScriptUI Panels folder), by script window id. + ScriptPanel(u32), LumetriScopes, Footage, MediaBrowser, @@ -107,6 +109,7 @@ impl PanelKind { PanelKind::MaskInterpolation => "Mask Interpolation", PanelKind::ScriptConsole => "Script Console", PanelKind::EssentialGraphics => "Essential Graphics", + PanelKind::ScriptPanel(_) => "ScriptUI Panel", PanelKind::LumetriScopes => "Lumetri Scopes", PanelKind::Footage => "Footage", PanelKind::MediaBrowser => "Media Browser", diff --git a/crates/ui-egui/src/dock_ui.rs b/crates/ui-egui/src/dock_ui.rs index 3b222570..61fa5ad6 100644 --- a/crates/ui-egui/src/dock_ui.rs +++ b/crates/ui-egui/src/dock_ui.rs @@ -18,6 +18,12 @@ impl EffectcraftApp { /// "Effect Controls Title", "Properties: Title", and the Timeline tab named after its comp. fn tab_titles(&self) -> Vec<(PanelKind, String)> { let mut out = Vec::new(); + // ScriptUI panels are named after their script. + for w in self.session.script_ui.windows.iter().filter(|w| w.kind == effectcraft_engine::scriptui::WindowKind::Panel) { + if let Some(t) = panels::scriptui_view::panel_title(self, w.id) { + out.push((PanelKind::ScriptPanel(w.id), t)); + } + } let Some(comp) = self.session.active_comp() else { return out }; let cname = self.session.active_comp_id().and_then(|id| self.session.project.item(id)).map(|i| i.name.clone()).unwrap_or_default(); out.push((PanelKind::Composition, format!("Composition {cname}"))); @@ -69,6 +75,13 @@ impl EffectcraftApp { /// Close a panel wherever it is (docked or floating). pub fn close_panel(&mut self, p: PanelKind) { + // Closing a ScriptUI panel closes its script window (its script's onClose runs). + if let PanelKind::ScriptPanel(id) = p + && self.session.script_ui.window(id).is_some() + && let Err(e) = self.session.execute("scriptui.close", serde_json::json!({"window": id})) + { + self.ui.status = e.to_string(); + } self.ui.dock.close(p); self.edit_layout(|l| l.unfloat(p)); if self.ui.maximized == Some(p) { diff --git a/crates/ui-egui/src/header.rs b/crates/ui-egui/src/header.rs index d185b52c..2b7f3759 100644 --- a/crates/ui-egui/src/header.rs +++ b/crates/ui-egui/src/header.rs @@ -223,7 +223,21 @@ fn puppet_options(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter } x = r.max.x + 12.0; } - x + 4.0 + // Record Options… (⌘/Ctrl-drag a pin records its motion in real time). + let label = "Record Options..."; + let w = 104.0; + let r = Rect::from_min_size(pos2(x, cy - 10.0), vec2(w, 20.0)); + let resp = ui.interact(r, egui::Id::new("puppet-record-options"), egui::Sense::click()).on_hover_text("⌘/Ctrl-drag a pin to record its motion"); + p.rect_stroke(r, 3.0, egui::Stroke::new(1.0, if resp.hovered() { t.accent } else { t.field_border }), egui::StrokeKind::Inside); + p.text(r.center(), Align2::CENTER_CENTER, label, Tokens::ui(11.5), t.text); + app.auto.add("header.puppet.recordOptions", r, label); + if resp.clicked() { + let ctx = ui.ctx().clone(); + if let Err(e) = crate::menus::invoke(app, &ctx, "puppet.recordOptions", json!({})) { + app.ui.status = e; + } + } + r.max.x + 16.0 } fn ws_menu(app: &mut EffectcraftApp, ui: &mut egui::Ui) { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index d873a333..48587f9e 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -74,6 +74,8 @@ pub enum Dialog { Precompose, /// Layer ▸ Layer Styles ▸ Layer Style Options… LayerStyles, + /// Edit ▸ Templates ▸ Render Settings… / Output Module…. + RenderTemplates, } /// Host hooks provided by the native app (file pickers etc.). @@ -855,6 +857,8 @@ impl EffectcraftApp { panels::precomp::mini_flowchart(self, &ctx); panels::home::capture_thumbnail(self); panels::dialogs::show(self, &ctx); + panels::scriptui_view::sync_panels(self); + panels::scriptui_view::show_windows(self, &ctx); panels::learn::coach(self, &ctx); self.draw_toast(ui, full); } diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 28b42605..f3800ef6 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -456,6 +456,12 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu app.dialog = Some(crate::Dialog::About); Value::Null } + // Window ▸ : dock (or bring forward) the panel the script built. + "window.scriptPanel" => { + let id = p.get("window").and_then(Value::as_u64).ok_or("no ScriptUI panel window")? as u32; + app.show_panel(PanelKind::ScriptPanel(id)); + json!({"window": id}) + } "layer.style.options" => { crate::panels::layer_styles_dialog::open(app, &p)?; Value::Null @@ -506,8 +512,18 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu } "app.templates" => { let kind = p.get("kind").and_then(Value::as_str).unwrap_or("renderSettings"); - let title = if kind == "outputModule" { "Output Module Templates" } else { "Render Settings Templates" }; - crate::panels::dialogs::info(app, title, "Render and output templates are managed from the Render Queue panel (Window ▸ Render Queue)."); + crate::panels::rq_templates::open(app, ctx, kind); + Value::Null + } + "renderQueue.notify" => { + // Render finished with Notify on: a toast, the dock/taskbar attention request, and the + // host's system sound (the desktop app handles `renderQueue.notify`). + let msg = p.get("message").and_then(Value::as_str).unwrap_or("Render finished").to_string(); + app.toast = Some((msg, now)); + ctx.send_viewport_cmd(egui::ViewportCommand::RequestUserAttention(egui::UserAttentionType::Informational)); + if let Some(f) = app.hooks.app_action.as_ref() { + f("renderQueue.notify"); + } Value::Null } "app.find" => { @@ -822,7 +838,9 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Save("Animation.json")), "file.importLottie" => ("path", Ask::Open(&["json", "lottie"])), "render.saveCurrentPreview" => ("path", Ask::Save("Preview.mp4")), - "file.runScript" => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), + "file.runScript" if params.get("name").is_none() => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), + "file.installScript" | "file.installScriptUIPanel" => ("path", Ask::Open(&["jsx", "js"])), + "effect.plugins.load" if params.get("folder").is_none() => ("path", Ask::Open(&["wasm", "wat"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), "file.saveCopyAsXml" => ("path", Ask::Save("Untitled Project.ecprojx")), diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs index 41fb7204..c9c40f05 100644 --- a/crates/ui-egui/src/panels/camera_tracker_ui.rs +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -38,6 +38,8 @@ struct Active { show_3d: bool, point_size: f64, target_size: f64, + /// Undistort Footage is on: points project without the lens distortion. + undistort: bool, /// Layer pixels → comp pixels. l2c: Mat3, frame: usize, @@ -66,6 +68,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye let t = tracks.as_ref()?; (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) }); + let undistort = solve.as_ref().is_some_and(|sv| ct::undistorts(¶ms, sv)); return Some(Active { layer: *lid, uid: *uid, @@ -74,6 +77,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + undistort, l2c: l2c(ectx, l), frame: frame.unwrap_or(0), }); @@ -98,18 +102,20 @@ fn l2c_scale(m: &Mat3) -> f32 { /// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { - let Some(cam) = sv.frames.get(a.frame) else { return }; + if sv.frames.get(a.frame).is_none() { + return; + } let r = t.size * 0.5 * a.target_size / 100.0; for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { let ring = ct::target_circle(t.center, t.normal, r * frac, 48); - let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + let pts: Vec = ring.iter().filter_map(|x| sv.project(a.frame, *x, a.undistort)).map(|q| to_screen(a, map, q)).collect(); if pts.len() < 3 { return; } let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); } - if let Some(c) = cam.project(sv.size, t.center) { + if let Some(c) = sv.project(a.frame, t.center, a.undistort) { let c = to_screen(a, map, c); p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); } @@ -168,7 +174,7 @@ pub fn viewer_hook( let mut hits: Vec = vec![]; match (&a.solve, a.show_3d) { (Some(sv), true) => { - let pts = ct::projected(sv, a.frame); + let pts = ct::projected_with(sv, a.frame, a.undistort); let med = ct::median_depth(&pts); for (id, q, z) in pts { let sp = to_screen(&a, map, q); diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index cabc8703..8992b69a 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -129,6 +129,7 @@ pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context) { Dialog::LayerStyles => super::layer_styles_dialog::show(app, ctx, &t), Dialog::TrackTarget => super::tracker::target_dialog(app, ctx, &t), Dialog::TrackApply => super::tracker::apply_dialog(app, ctx, &t), + Dialog::RenderTemplates => super::rq_templates::show(app, ctx, &t), } } diff --git a/crates/ui-egui/src/panels/effects_presets.rs b/crates/ui-egui/src/panels/effects_presets.rs index 888f9d86..21d63dec 100644 --- a/crates/ui-egui/src/panels/effects_presets.rs +++ b/crates/ui-egui/src/panels/effects_presets.rs @@ -322,7 +322,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let searching = !query.is_empty(); let mut y = list.min.y - scroll; let mut apply: Option = None; - let reg = effectcraft_engine::effects::registry(); + let reg = effectcraft_engine::effects::all(); let keep = |e: &effectcraft_engine::effects::EffectSpec| depth_keeps(&view.depth, e) && (query.is_empty() || e.name.to_lowercase().contains(&query)); let lookup = |id: &str| reg.iter().find(|e| e.id == id); @@ -381,7 +381,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { effect_row(app, ui, &lp, list, &mut y, e, -16.0, &query, "effects.item", &mut apply); } } else { - let mut cats: Vec<&str> = effectcraft_engine::effects::CATEGORIES.to_vec(); + let mut cats: Vec<&str> = effectcraft_engine::effects::categories(); cats.retain(|c| reg.iter().any(|e| e.category == *c)); for cat in cats { let items: Vec<_> = reg.iter().filter(|e| e.category == cat && keep(e)).collect(); diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 20382a3e..8d2df347 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -136,6 +136,19 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { 1, )], ), + // Puppet tool ▸ Record Options. + "puppet.recordOptions" if !has(p, &["speed", "smoothing", "useDraftDeformation", "showMesh"]) => { + let o = &s.state.puppet; + ( + "Puppet Record Options".into(), + vec![ + Field::num("speed", "Speed (%)", o.record_speed), + Field::num("smoothing", "Smoothing", o.record_smoothing), + Field::bool("useDraftDeformation", "Use Draft Deformation", o.record_draft), + Field::bool("showMesh", "Show Mesh", o.record_show_mesh), + ], + ) + } "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 69d95f66..ea12591c 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -172,34 +172,43 @@ pub fn audio(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { pub fn history(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let t = app.tokens; let p = ui.painter().with_clip_rect(rect); + // The branching history (EffectCraft's History panel): every state, branches indented under + // the state they grew from; click any state to go there (`edit.history.goto`). + let nodes = app.session.history_tree(); + let row_h = 22.0; + let cur = nodes.iter().position(|n| n.current).unwrap_or(0); + // Keep the current state in view. + let visible = ((rect.height() - 12.0) / row_h).max(1.0) as usize; + let first = (cur + 2).saturating_sub(visible).min(nodes.len().saturating_sub(visible)); let mut y = rect.min.y + 6.0; - let undo: Vec = app.session.history.undo.iter().map(|u| u.0.clone()).collect(); - let redo: Vec = app.session.history.redo.iter().rev().map(|u| u.0.clone()).collect(); - let n = undo.len(); - let mut jump: Option = None; - for (i, label) in undo.iter().chain(redo.iter()).enumerate() { - let r = Rect::from_min_size(pos2(rect.min.x, y), vec2(rect.width(), 22.0)); - y += 22.0; - let current = i + 1 == n; - let future = i >= n; - let resp = ui.interact(r, egui::Id::new(("hist", i)), Sense::click()); - if current { + let mut jump: Option = None; + for n in nodes.iter().skip(first).take(visible + 1) { + let r = Rect::from_min_size(pos2(rect.min.x, y), vec2(rect.width(), row_h)); + y += row_h; + let resp = ui.interact(r, egui::Id::new(("hist", n.id.as_str())), Sense::click()); + if n.current { p.rect_filled(r, 0.0, t.row_selected); } else if resp.hovered() { p.rect_filled(r, 0.0, t.hover); } - p.text(pos2(r.min.x + 12.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::ui(12.0), if future { t.text_faint } else { t.text }); + let x = r.min.x + 12.0 + n.depth as f32 * 14.0; + if n.depth > 0 { + // Branch marker. + let c = pos2(x - 7.0, r.center().y); + p.line_segment([pos2(c.x, r.min.y), c], egui::Stroke::new(1.0, t.text_faint)); + p.line_segment([c, pos2(c.x + 5.0, c.y)], egui::Stroke::new(1.0, t.text_faint)); + } + let col = if !n.current && (n.future || !n.line) { t.text_faint } else { t.text }; + p.text(pos2(x, r.center().y), Align2::LEFT_CENTER, &n.label, Tokens::ui(12.0), col); + app.auto.add(&format!("history.state.{}", n.index), r, &n.label); if resp.clicked() { - jump = Some(i as i64 + 1 - n as i64); + jump = Some(n.index); } } - if let Some(d) = jump { - for _ in 0..d.unsigned_abs() { - if d < 0 { - app.session.undo(); - } else { - app.session.redo(); - } + if let Some(i) = jump { + let ctx = ui.ctx().clone(); + if let Err(e) = crate::menus::invoke(app, &ctx, "edit.history.goto", serde_json::json!({"index": i})) { + app.ui.status = e; } } } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index ed766f0d..d82c6d97 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -34,8 +34,10 @@ pub mod properties; pub mod puppet_tool; pub mod render_queue; pub mod roto_tool; +pub mod rq_templates; pub mod scopes_panel; pub mod script_console; +pub mod scriptui_view; pub mod settings; pub mod shortcut_editor; pub mod text_panels; @@ -107,6 +109,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), PanelKind::EssentialGraphics => essential::show(app, ui, rect), + PanelKind::ScriptPanel(id) => scriptui_view::panel(app, ui, rect, id), PanelKind::Flowchart => flowchart::show(app, ui, rect), PanelKind::LumetriScopes => scopes_panel::show(app, ui, rect), PanelKind::Footage => footage_panel::show(app, ui, rect), diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index c02090bc..ec0853a1 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -4,10 +4,18 @@ //! Output Module and Output To rows, and a status footer. //! //! Every control dispatches an engine `renderQueue.*` command (so agents can do the same) and -//! registers an automation id: `renderQueue.render`, `renderQueue.stop`, -//! `renderQueue.item..{row,twirl,render,renderSettings,outputModule,outputTo,outputPath}`. +//! registers an automation id: `renderQueue.render`, `renderQueue.stop`, `renderQueue.notify`, +//! `renderQueue.item..{row,twirl,render,renderSettings,outputModule,outputTo,outputPath,log}`. +//! +//! The Render Settings and Output Module menus list the templates first (Edit ▸ Templates), then +//! every setting; entries needing numbers (custom time span and frame rate, crop, resize, bitrate, +//! key frame interval, Make Template) open a form that runs the same command. -use effectcraft_engine::project::render_queue::{AudioOutput, Channels, OutputFormat, ProResProfile, RenderQuality, RenderQueueItem, RenderStatus, TimeSpan}; +use effectcraft_engine::project::render_queue::{ + AlphaMode, AudioFormat, AudioOutput, Channels, ColorDepth, CurrentOrOff, EffectsMode, FieldRender, OutputFormat, PostRenderAction, ProResProfile, Pulldown, + RESIZE_PRESETS, RenderLog, RenderQuality, RenderQueueItem, RenderStatus, ResizeQuality, SwitchOverride, TimeSpan, +}; +use effectcraft_engine::project::render_templates::RenderTemplates; use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use serde_json::{Value, json}; @@ -53,42 +61,82 @@ fn status_color(t: &Tokens, s: &RenderStatus) -> Color32 { } } -/// Render Settings menu: (label, params). -fn settings_menu(it: &RenderQueueItem) -> Vec<(String, Value)> { +fn mark(on: bool, l: &str) -> String { + if on { format!("✓ {l}") } else { format!(" {l}") } +} + +/// Render Settings menu: (label, params). `{"form": …}` entries open a form. +fn settings_menu(it: &RenderQueueItem, templates: &RenderTemplates) -> Vec<(String, Value)> { let s = &it.settings; - let mark = |on: bool, l: &str| if on { format!("✓ {l}") } else { format!(" {l}") }; - let mut v = vec![ - (mark(s.quality == RenderQuality::Best, "Best Settings"), json!({"quality": "best"})), - (mark(s.quality == RenderQuality::Draft, "Draft Settings"), json!({"quality": "draft"})), - ("-".into(), Value::Null), - ]; + let mut v = vec![]; + for t in templates.render_settings_list() { + v.push((mark(s.name == t.name, &t.name), json!({"template": t.name}))); + } + v.push((" Make Template…".into(), json!({"form": "makeRenderTemplate"}))); + v.push(("-".into(), Value::Null)); + v.push((mark(s.quality == RenderQuality::Best, "Quality: Best"), json!({"quality": "best"}))); + v.push((mark(s.quality == RenderQuality::Draft, "Quality: Draft"), json!({"quality": "draft"}))); for (l, k) in [("Full", "full"), ("Half", "half"), ("Third", "third"), ("Quarter", "quarter")] { v.push((mark(s.resolution_label() == l, &format!("Resolution: {l}")), json!({"resolution": k}))); } v.push(("-".into(), Value::Null)); + for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { + v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); + } + for (e, k) in EffectsMode::ALL.iter().zip(["current", "allOn", "allOff"]) { + v.push((mark(s.effects == *e, &format!("Effects: {}", e.label())), json!({"effects": k}))); + } + for (e, k) in [(CurrentOrOff::Current, "current"), (CurrentOrOff::AllOff, "allOff")] { + v.push((mark(s.solo == e, &format!("Solo Switches: {}", e.label())), json!({"solo": k}))); + } + for (e, k) in [(CurrentOrOff::Current, "current"), (CurrentOrOff::AllOff, "allOff")] { + v.push((mark(s.guide_layers == e, &format!("Guide Layers: {}", e.label())), json!({"guideLayers": k}))); + } + for (d, k) in ColorDepth::ALL.iter().zip(["current", "8", "16", "32"]) { + v.push((mark(s.color_depth == *d, &format!("Color Depth: {}", d.label())), json!({"colorDepth": k}))); + } + v.push(("-".into(), Value::Null)); + for (m, k) in SwitchOverride::ALL.iter().zip(["current", "onForChecked", "offForAll"]) { + v.push((mark(s.frame_blending == *m, &format!("Frame Blending: {}", m.label())), json!({"frameBlending": k}))); + } + for (m, k) in SwitchOverride::ALL.iter().zip(["current", "onForChecked", "offForAll"]) { + v.push((mark(s.motion_blur_override() == *m, &format!("Motion Blur: {}", m.label())), json!({"motionBlur": k}))); + } + for (f, k) in FieldRender::ALL.iter().zip(["off", "upper", "lower"]) { + v.push((mark(s.field_render == *f, &format!("Field Render: {}", f.label())), json!({"fieldRender": k}))); + } + if s.field_render != FieldRender::Off { + for pd in Pulldown::ALL { + v.push((mark(s.pulldown == pd, &format!("3:2 Pulldown: {}", pd.label())), json!({"pulldown": pd.label()}))); + } + } + v.push(("-".into(), Value::Null)); v.push((mark(s.time_span == TimeSpan::WorkArea, "Time Span: Work Area Only"), json!({"timeSpan": "workArea"}))); v.push((mark(s.time_span == TimeSpan::LengthOfComp, "Time Span: Length of Comp"), json!({"timeSpan": "comp"}))); - v.push(("-".into(), Value::Null)); + v.push((mark(matches!(s.time_span, TimeSpan::Custom { .. }), "Time Span: Custom…"), json!({"form": "timeSpan"}))); v.push((mark(s.frame_rate.is_none(), "Frame Rate: Use comp's frame rate"), json!({"frameRate": null}))); for fps in [12.0, 15.0, 23.976, 24.0, 25.0, 29.97, 30.0, 50.0, 60.0] { let on = s.frame_rate.is_some_and(|r| (r.as_f64() - fps).abs() < 0.01); v.push((mark(on, &format!("Frame Rate: {fps}")), json!({"frameRate": fps}))); } + v.push((" Frame Rate: Custom…".into(), json!({"form": "frameRate"}))); v.push(("-".into(), Value::Null)); - v.push((mark(s.motion_blur, "Motion Blur: On for Checked Layers"), json!({"motionBlur": !s.motion_blur}))); v.push((mark(s.skip_existing, "Skip Existing Files"), json!({"skipExisting": !s.skip_existing}))); - v.push(("-".into(), Value::Null)); - for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { - v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); - } + v.push((mark(s.storage_overflow, "Use Storage Overflow"), json!({"storageOverflow": !s.storage_overflow}))); v } -/// Output Module menu: (label, params). -fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, Value)> { +/// Output Module menu: (label, params). `{"form": …}` entries open a form. +fn output_menu(it: &RenderQueueItem, available: &[OutputFormat], templates: &RenderTemplates) -> Vec<(String, Value)> { let o = &it.output; - let mark = |on: bool, l: &str| if on { format!("✓ {l}") } else { format!(" {l}") }; let mut v = Vec::new(); + for t in templates.output_module_list() { + if available.contains(&t.format) { + v.push((mark(o.name == t.name, &t.name), json!({"template": t.name}))); + } + } + v.push((" Make Template…".into(), json!({"form": "makeOutputTemplate"}))); + v.push(("-".into(), Value::Null)); for f in OutputFormat::ALL { if available.contains(&f) { v.push((mark(o.format == f, &format!("Format: {}", f.label())), json!({"format": format!("{f:?}")}))); @@ -99,6 +147,13 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, if o.format.supports_alpha() { v.push((mark(o.channels == Channels::Rgba, "Channels: RGB + Alpha"), json!({"channels": "rgba"}))); } + if !o.format.is_audio_only() { + v.push((mark(o.channels == Channels::Alpha, "Channels: Alpha"), json!({"channels": "alpha"}))); + } + if o.channels == Channels::Rgba { + v.push((mark(o.alpha_mode == AlphaMode::Straight, "Color: Straight (Unmatted)"), json!({"color": "straight"}))); + v.push((mark(o.alpha_mode == AlphaMode::Premultiplied, "Color: Premultiplied (Matted)"), json!({"color": "premultiplied"}))); + } match o.format { OutputFormat::ProRes => { v.push(("-".into(), Value::Null)); @@ -108,14 +163,18 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, } OutputFormat::H264 => { v.push(("-".into(), Value::Null)); - for kbps in [2_000, 5_000, 10_000, 20_000, 40_000] { + for kbps in [2_000, 5_000, 10_000, 15_000, 20_000, 40_000] { v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { - v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); + v.push((mark(o.quality == q && !o.webm_bitrate, &format!("Quality: {q}")), json!({"quality": q, "webmBitrate": false}))); + } + if o.format == OutputFormat::WebM { + v.push((mark(o.webm_bitrate, &format!("Target Bitrate ({} kbps)…", o.bitrate_kbps)), json!({"form": "bitrate"}))); + v.push((mark(o.keyframe_interval > 0, "Key Frame Interval…"), json!({"form": "keyframes"}))); } } OutputFormat::Gif => { @@ -124,20 +183,131 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, } _ => {} } + if !o.format.is_audio_only() { + v.push(("-".into(), Value::Null)); + v.push((mark(!o.crop.enabled, "Crop: Off"), json!({"crop": false}))); + v.push((mark(o.crop.enabled && o.crop.use_roi, "Crop: Use Region of Interest"), json!({"crop": {"useRoi": true}}))); + v.push((mark(o.crop.enabled && !o.crop.use_roi, "Crop: Edges…"), json!({"form": "crop"}))); + v.push((mark(!o.resize.enabled, "Resize: Off"), json!({"resize": false}))); + for (n, w, h) in RESIZE_PRESETS { + let on = o.resize.enabled && o.resize.width == w && o.resize.height == h && !o.resize.lock_aspect; + v.push((mark(on, &format!("Resize to: {n} ({w}×{h})")), json!({"resize": {"preset": n}}))); + } + v.push((mark(o.resize.enabled, "Resize to: Custom…"), json!({"form": "resize"}))); + v.push(( + mark(o.resize.lock_aspect, "Resize: Lock Aspect Ratio"), + json!({"resize": {"enabled": o.resize.enabled, "lockAspect": !o.resize.lock_aspect}}), + )); + for (q, k) in [(ResizeQuality::Low, "low"), (ResizeQuality::High, "high")] { + v.push((mark(o.resize.quality == q, &format!("Resize Quality: {q:?}")), json!({"resize": {"enabled": o.resize.enabled, "quality": k}}))); + } + } + v.push(("-".into(), Value::Null)); + for (a, k) in [ + (PostRenderAction::None, "none"), + (PostRenderAction::Import, "import"), + (PostRenderAction::ImportAndReplace, "importAndReplace"), + (PostRenderAction::SetProxy, "setProxy"), + ] { + v.push((mark(it.post_render == a, &format!("Post-Render Action: {}", a.label())), json!({"postRenderAction": k}))); + } + v.push((mark(o.include_project_link, "Include Project Link"), json!({"includeProjectLink": !o.include_project_link}))); if o.format.supports_audio() { v.push(("-".into(), Value::Null)); for (a, l) in [(AudioOutput::Auto, "auto"), (AudioOutput::On, "on"), (AudioOutput::Off, "off")] { let label = match a { AudioOutput::Auto => "Audio Output: Auto", - AudioOutput::On => "Audio Output: On (48 kHz stereo)", + AudioOutput::On => "Audio Output: On", AudioOutput::Off => "Audio Output: Off", }; v.push((mark(o.audio == a, label), json!({"audio": l}))); } + for sr in [44_100u32, 48_000, 96_000] { + v.push((mark(o.audio_sample_rate == sr, &format!("Sample Rate: {:.1} kHz", sr as f64 / 1000.0)), json!({"sampleRate": sr}))); + } + v.push((mark(o.audio_channels == 1, "Audio Channels: Mono"), json!({"audioChannels": "mono"}))); + v.push((mark(o.audio_channels != 1, "Audio Channels: Stereo"), json!({"audioChannels": "stereo"}))); + if matches!(o.format, OutputFormat::ProRes | OutputFormat::Wav | OutputFormat::Aiff) { + for f in [AudioFormat::S16, AudioFormat::S24, AudioFormat::F32] { + v.push((mark(o.audio_format == f, &format!("Sample Format: {}", f.label())), json!({"audioFormat": f.label()}))); + } + } } v } +/// Open the form behind a `{"form": …}` menu entry. +fn open_menu_form(app: &mut EffectcraftApp, it: &RenderQueueItem, form: &str) { + use crate::panels::forms::{Field, form as open}; + let id = it.id; + let o = &it.output; + match form { + "makeRenderTemplate" => open( + app, + "Make Render Settings Template", + "renderQueue.saveTemplate", + json!({"kind": "renderSettings", "item": id}), + vec![Field::text("name", "Template name", "")], + ), + "makeOutputTemplate" => open( + app, + "Make Output Module Template", + "renderQueue.saveTemplate", + json!({"kind": "outputModule", "item": id}), + vec![Field::text("name", "Template name", "")], + ), + "timeSpan" => { + let comp = app.session.project.comp(it.comp); + let (a, b) = comp.map(|c| it.settings.span(c)).unwrap_or_default(); + open( + app, + "Custom Time Span", + "renderQueue.setRenderSettings", + json!({"item": id, "timeSpan": "custom"}), + vec![Field::num("start", "Start (s)", a.seconds()), Field::num("duration", "Duration (s)", (b.seconds() - a.seconds()).max(0.04))], + ) + } + "frameRate" => { + let r = app.session.project.comp(it.comp).map(|c| it.settings.rate(c).as_f64()).unwrap_or(30.0); + open(app, "Use This Frame Rate", "renderQueue.setRenderSettings", json!({"item": id}), vec![Field::num("frameRate", "Frames per second", r)]) + } + "crop" => open( + app, + "Crop", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![ + Field::num("cropTop", "Top", o.crop.top as f64), + Field::num("cropLeft", "Left", o.crop.left as f64), + Field::num("cropBottom", "Bottom", o.crop.bottom as f64), + Field::num("cropRight", "Right", o.crop.right as f64), + ], + ), + "resize" => open( + app, + "Resize to", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![Field::num("resizeWidth", "Width", o.resize.width as f64), Field::num("resizeHeight", "Height", o.resize.height as f64)], + ), + "bitrate" => open( + app, + "WebM Target Bitrate", + "renderQueue.setOutputModule", + json!({"item": id, "webmBitrate": true}), + vec![Field::num("bitrate", "Bitrate (kbps)", o.bitrate_kbps as f64)], + ), + "keyframes" => open( + app, + "Key Frame Interval", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![Field::num("keyframeInterval", "Frames between key frames (0 = every 2 s)", o.keyframe_interval as f64)], + ), + _ => {} + } +} + const OUTPUT_TEMPLATES: [&str; 4] = [ "[compName].[fileExtension]", "[compName]_[width]x[height].[fileExtension]", @@ -166,6 +336,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let rendering = app.session.is_rendering(); let available = app.session.exporter.as_ref().map(|e| e.formats()).unwrap_or_default(); let mut actions: Vec<(&'static str, Value)> = Vec::new(); + let mut forms: Vec<(RenderQueueItem, String)> = Vec::new(); let state_id = egui::Id::new("rq-ui"); let (mut open, mut selected): (Vec, Option) = ui.data(|d| d.get_temp(state_id).unwrap_or_default()); @@ -194,6 +365,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("renderQueue.render", json!({"wait": false}))); } app.auto.add("renderQueue.render", render_b, "Render"); + // Notify (sound + notification when the queue finishes). + let notify = app.session.project.render_prefs.notify; + let nb = Rect::from_min_size(pos2(render_b.min.x - 8.0 - 2.0 * bw - 8.0, render_b.min.y + 4.0), vec2(18.0, 18.0)); + if widgets::checkbox(ui, nb, notify, &t, egui::Id::new("rq-notify")).clicked() { + actions.push(("renderQueue.setNotify", json!({"notify": !notify}))); + } + p.text(pos2(nb.max.x + 5.0, nb.center().y), Align2::LEFT_CENTER, "Notify", Tokens::ui(11.5), t.text_dim); + app.auto.add("renderQueue.notify", nb, if notify { "Notify: on" } else { "Notify: off" }); if rendering { if widgets::text_button(ui, stop_b, "Stop", false, &t, egui::Id::new("rq-stop")).clicked() { actions.push(("renderQueue.stop", json!({}))); @@ -201,8 +380,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.auto.add("renderQueue.stop", stop_b, "Stop"); } // Progress bar. - let bar = - Rect::from_min_max(pos2(x0, render_b.min.y + 4.0), pos2(if rendering { stop_b.min.x - 16.0 } else { render_b.min.x - 16.0 }, render_b.max.y - 4.0)); + let bar = Rect::from_min_max(pos2(x0, render_b.min.y + 4.0), pos2(nb.min.x - 16.0, render_b.max.y - 4.0)); p.rect_filled(bar, 3.0, t.field_bg); p.rect_stroke(bar, 3.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); let frac = progress.as_ref().filter(|p| p.total > 0).map(|p| (p.done as f32 / p.total as f32).clamp(0.0, 1.0)).unwrap_or(0.0); @@ -355,22 +533,28 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = Rect::from_min_size(pos2(list.min.x, y), vec2(list.width(), ROW_H)); y += ROW_H; lp.rect_filled(r, 0.0, t.panel_bg); + let templates = &app.session.project.render_templates; let (label, summary, key, menu): (&str, String, &str, Vec<(String, Value)>) = if sub == 0 { - ("Render Settings:", it.settings.summary(), "renderSettings", settings_menu(it)) + ("Render Settings:", it.settings.summary(), "renderSettings", settings_menu(it, templates)) } else { - ("Output Module:", it.output.summary(), "outputModule", output_menu(it, &available)) + ("Output Module:", it.output.summary(), "outputModule", output_menu(it, &available, templates)) }; lp.text(pos2(cx_num - 6.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::ui(11.5), t.text_dim); let dd = Rect::from_min_size(pos2(ix + 70.0, r.min.y + 3.0), vec2(220.0, ROW_H - 6.0)); let did = egui::Id::new(("rq-dd", it.id, sub)); - if widgets::dropdown(ui, dd, &summary, &t, did).clicked() && !rendering { + let shown: String = if summary.chars().count() > 31 { summary.chars().take(30).chain(std::iter::once('…')).collect() } else { summary.clone() }; + if widgets::dropdown(ui, dd, &shown, &t, did).clicked() && !rendering { widgets::open_popup(ui, did); } let labels: Vec = menu.iter().map(|(l, _)| l.clone()).collect(); if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &labels, None) { let mut params = menu[i].1.clone(); - params["item"] = json!(it.id); - actions.push((if sub == 0 { "renderQueue.setRenderSettings" } else { "renderQueue.setOutputModule" }, params)); + if let Some(f) = params.get("form").and_then(Value::as_str) { + forms.push((it.clone(), f.to_string())); + } else { + params["item"] = json!(it.id); + actions.push((if sub == 0 { "renderQueue.setRenderSettings" } else { "renderQueue.setOutputModule" }, params)); + } } app.auto.add(&aid(key), dd, &summary); if sub == 0 { @@ -387,6 +571,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ); lp.text(pos2(dd.max.x + 14.0, r.center().y), Align2::LEFT_CENTER, info, Tokens::ui(11.0), t.text_faint); } + // Log: Errors Only / Plus Settings / Plus Per Frame Info. + let ld = Rect::from_min_size(pos2(r.max.x - 190.0, r.min.y + 3.0), vec2(150.0, ROW_H - 6.0)); + lp.text(pos2(ld.min.x - 6.0, r.center().y), Align2::RIGHT_CENTER, "Log:", Tokens::ui(11.5), t.text_dim); + let lid = egui::Id::new(("rq-log", it.id)); + if widgets::dropdown(ui, ld, it.log.label(), &t, lid).clicked() && !rendering { + widgets::open_popup(ui, lid); + } + let llabels: Vec = RenderLog::ALL.iter().map(|l| mark(it.log == *l, l.label())).collect(); + if let Some(i) = widgets::popup_menu(ui, lid, ld.left_bottom(), &llabels, None) { + actions.push(("renderQueue.setLog", json!({"item": it.id, "log": RenderLog::ALL[i].label()}))); + } + app.auto.add(&aid("log"), ld, it.log.label()); continue; } // Output To. @@ -490,6 +686,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { open.retain(|id| queue.iter().any(|i| i.id == *id)); ui.data_mut(|d| d.insert_temp(state_id, (open, selected))); let ctx = ui.ctx().clone(); + for (it, f) in forms { + open_menu_form(app, &it, &f); + } for (id, params) in actions { if let Err(e) = crate::menus::invoke(app, &ctx, id, params) { app.ui.status = e; diff --git a/crates/ui-egui/src/panels/rq_templates.rs b/crates/ui-egui/src/panels/rq_templates.rs new file mode 100644 index 00000000..c60c3009 --- /dev/null +++ b/crates/ui-egui/src/panels/rq_templates.rs @@ -0,0 +1,166 @@ +//! Edit ▸ Templates ▸ Render Settings… / Output Module…: the Defaults (which template new +//! Render Queue items, frames, pre-renders and proxies start from) and the template list +//! (settings of the selected template; Duplicate, Delete, New from a Render Queue item). +//! +//! Every action runs a `renderQueue.*Template*` engine command. Automation ids: +//! `templates.default.`, `templates.select`, `templates.duplicate`, `templates.delete`, +//! `templates.newName`, `templates.saveFromItem`, `templates.ok`. + +use effectcraft_engine::project::render_templates::{RenderTemplates, TemplateKind, TemplateSlot}; +use egui::{Color32, vec2}; +use serde_json::{Value, json}; + +use crate::theme::Tokens; +use crate::{Dialog, EffectcraftApp}; + +#[derive(Clone, Default)] +struct State { + output: bool, + selected: String, + new_name: String, +} + +fn state_id() -> egui::Id { + egui::Id::new("rq-templates-dialog") +} + +/// Open the dialog for `kind` (`renderSettings` | `outputModule`). +pub fn open(app: &mut EffectcraftApp, ctx: &egui::Context, kind: &str) { + let output = TemplateKind::parse(kind) == Some(TemplateKind::OutputModule); + let t = &app.session.project.render_templates; + let k = if output { TemplateKind::OutputModule } else { TemplateKind::RenderSettings }; + let selected = t.default_name(k, TemplateSlot::Movie); + ctx.data_mut(|d| d.insert_temp(state_id(), State { output, selected, new_name: String::new() })); + app.dialog = Some(Dialog::RenderTemplates); +} + +fn names(t: &RenderTemplates, output: bool) -> Vec { + if output { t.output_module_list().into_iter().map(|m| m.name).collect() } else { t.render_settings_list().into_iter().map(|r| r.name).collect() } +} + +pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) { + let mut st: State = ctx.data(|d| d.get_temp(state_id())).unwrap_or_default(); + let kind = if st.output { "outputModule" } else { "renderSettings" }; + let k = if st.output { TemplateKind::OutputModule } else { TemplateKind::RenderSettings }; + let templates = app.session.project.render_templates.clone(); + let list = names(&templates, st.output); + if !list.contains(&st.selected) { + st.selected = list.first().cloned().unwrap_or_default(); + } + let mut actions: Vec<(&'static str, Value)> = vec![]; + let mut close = false; + let title = if st.output { "Output Module Templates" } else { "Render Settings Templates" }; + super::dialogs::modal(ctx, title, vec2(600.0, 560.0), t, |ui| { + ui.label(egui::RichText::new("Defaults").font(Tokens::semibold(12.5)).color(t.text)); + egui::Grid::new("tpl-defaults").num_columns(2).spacing(vec2(14.0, 6.0)).show(ui, |ui| { + for slot in TemplateSlot::ALL { + ui.label(slot.label()); + let cur = templates.default_name(k, slot); + let mut pick = cur.clone(); + let r = egui::ComboBox::from_id_salt(("tpl-def", slot.label())).selected_text(&cur).width(300.0).show_ui(ui, |ui| { + for n in &list { + ui.selectable_value(&mut pick, n.clone(), n); + } + }); + app.auto.add(&format!("templates.default.{slot:?}"), r.response.rect, &cur); + if pick != cur { + actions.push(("renderQueue.setTemplateDefault", json!({"kind": kind, "slot": format!("{slot:?}"), "name": pick}))); + } + ui.end_row(); + } + }); + ui.add_space(12.0); + ui.separator(); + ui.label(egui::RichText::new("Settings").font(Tokens::semibold(12.5)).color(t.text)); + ui.horizontal(|ui| { + ui.label("Settings Name:"); + let r = egui::ComboBox::from_id_salt("tpl-select").selected_text(&st.selected).width(300.0).show_ui(ui, |ui| { + for n in &list { + ui.selectable_value(&mut st.selected, n.clone(), n); + } + }); + app.auto.add("templates.select", r.response.rect, &st.selected); + }); + let builtin = RenderTemplates::is_builtin(k, &st.selected) + && !match k { + TemplateKind::RenderSettings => templates.render_settings.iter().any(|r| r.name.eq_ignore_ascii_case(&st.selected)), + TemplateKind::OutputModule => templates.output_modules.iter().any(|m| m.name.eq_ignore_ascii_case(&st.selected)), + }; + // The selected template's settings. + let lines: Vec<(&'static str, String)> = if st.output { + templates.output_module(&st.selected).map(|m| m.describe()).unwrap_or_default() + } else { + templates.render_settings(&st.selected).map(|r| r.describe(None)).unwrap_or_default() + }; + egui::Frame::new().fill(t.field_bg).inner_margin(egui::Margin::same(8)).show(ui, |ui| { + ui.set_min_height(180.0); + ui.set_width(ui.available_width()); + egui::ScrollArea::vertical().max_height(200.0).show(ui, |ui| { + egui::Grid::new("tpl-lines").num_columns(2).spacing(vec2(12.0, 2.0)).show(ui, |ui| { + for (k, v) in &lines { + ui.label(egui::RichText::new(*k).color(t.text_dim).size(11.5)); + ui.label(egui::RichText::new(v).size(11.5)); + ui.end_row(); + } + }); + }); + }); + ui.horizontal(|ui| { + let r = ui.button("Duplicate"); + app.auto.add("templates.duplicate", r.rect, "Duplicate"); + if r.clicked() { + let mut n = format!("{} copy", st.selected); + let mut i = 2; + while list.iter().any(|x| x.eq_ignore_ascii_case(&n)) { + n = format!("{} copy {i}", st.selected); + i += 1; + } + actions.push(("renderQueue.saveTemplate", json!({"kind": kind, "name": n, "from": st.selected}))); + st.selected = n; + } + let r = ui.add_enabled(!builtin, egui::Button::new("Delete")); + app.auto.add("templates.delete", r.rect, "Delete"); + if r.clicked() { + actions.push(("renderQueue.deleteTemplate", json!({"kind": kind, "name": st.selected}))); + } + if builtin { + ui.label(egui::RichText::new("built-in").color(t.text_faint).size(11.0)); + } + }); + ui.add_space(8.0); + ui.separator(); + // New template from the selected (or only) Render Queue item. + let queue = &app.session.project.render_queue; + ui.horizontal(|ui| { + ui.label("New from Render Queue item:"); + let r = ui.add(egui::TextEdit::singleline(&mut st.new_name).hint_text("template name").desired_width(180.0)); + app.auto.add("templates.newName", r.rect, &st.new_name); + let item = queue.last().map(|i| i.id); + let ok = item.is_some() && !st.new_name.trim().is_empty(); + let r = ui.add_enabled(ok, egui::Button::new(format!("Save from #{}", queue.len().max(1)))); + app.auto.add("templates.saveFromItem", r.rect, "Save from item"); + if r.clicked() + && let Some(id) = item + { + actions.push(("renderQueue.saveTemplate", json!({"kind": kind, "name": st.new_name.trim(), "item": id}))); + st.selected = st.new_name.trim().to_string(); + st.new_name.clear(); + } + }); + ui.add_space(12.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let r = ui.add(egui::Button::new(egui::RichText::new(" OK ").color(Color32::WHITE)).fill(t.accent)); + app.auto.add("templates.ok", r.rect, "OK"); + close = r.clicked(); + }); + }); + for (id, p) in actions { + if let Err(e) = app.session.execute(id, p) { + app.ui.status = e.to_string(); + } + } + ctx.data_mut(|d| d.insert_temp(state_id(), st)); + if close || ctx.input(|i| i.key_pressed(egui::Key::Escape)) { + app.dialog = None; + } +} diff --git a/crates/ui-egui/src/panels/scriptui_view.rs b/crates/ui-egui/src/panels/scriptui_view.rs new file mode 100644 index 00000000..580e287b --- /dev/null +++ b/crates/ui-egui/src/panels/scriptui_view.rs @@ -0,0 +1,321 @@ +//! ScriptUI windows: the dialogs, palettes and dockable panels scripts build +//! (`Session::script_ui`). Each control is drawn at the bounds the engine's ScriptUI layout +//! computed; what the user does goes back through the `scriptui.click` / `scriptui.set` / +//! `scriptui.close` commands, which run the script's handlers. Every control registers an +//! automation id `scriptui..` (and `scriptui..` when the script +//! named it). + +use effectcraft_engine::scriptui::{ScriptWindow, Widget, WidgetKind, WindowKind, layout}; +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, UiBuilder, pos2, vec2}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::dock::PanelKind; +use crate::theme::Tokens; + +enum Act { + Click(u32, u32), + Set(u32, u32, Value), + Close(u32), +} + +fn rect_of(origin: egui::Pos2, b: [f64; 4]) -> Rect { + Rect::from_min_size(origin + vec2(b[0] as f32, b[1] as f32), vec2(b[2] as f32, b[3] as f32)) +} + +fn register(app: &mut EffectcraftApp, win: u32, w: &Widget, rect: Rect) { + let label = if w.text.is_empty() { format!("{:?}", w.kind) } else { w.text.clone() }; + app.auto.add(&format!("scriptui.{win}.{}", w.id), rect, &label); + if !w.name.is_empty() { + app.auto.add(&format!("scriptui.{win}.{}", w.name), rect, &label); + } +} + +/// Draw `w`'s children (and, for containers, their children) at `origin` + bounds. +fn draw_children(app: &mut EffectcraftApp, ui: &mut egui::Ui, origin: egui::Pos2, win: u32, w: &Widget, acts: &mut Vec) { + let t = app.tokens; + if w.kind == WidgetKind::TabbedPanel { + // Tab headers, then the active tab only. + let r = rect_of(origin, w.bounds); + let mut x = r.min.x; + for (i, tab) in w.children.iter().enumerate() { + let tw = (tab.text.chars().count() as f32 * 7.0 + 20.0).max(48.0); + let hr = Rect::from_min_size(pos2(x, r.min.y), vec2(tw, 22.0)); + let active = i == w.active_tab; + let resp = ui.interact(hr, egui::Id::new(("sui-tab", win, tab.id)), Sense::click()); + ui.painter().rect_filled( + hr, + 3.0, + if active { + t.row_selected + } else if resp.hovered() { + t.hover + } else { + t.field_bg + }, + ); + ui.painter().text(hr.center(), Align2::CENTER_CENTER, &tab.text, Tokens::ui(12.0), t.text); + register(app, win, tab, hr); + if resp.clicked() && !active && tab.enabled { + acts.push(Act::Click(win, tab.id)); + } + x += tw + 2.0; + } + ui.painter().rect_stroke(Rect::from_min_max(pos2(r.min.x, r.min.y + 22.0), r.max), 3.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + if let Some(tab) = w.children.get(w.active_tab) { + draw_children(app, ui, origin, win, tab, acts); + } + return; + } + for c in w.children.iter().filter(|c| c.visible) { + draw_widget(app, ui, origin, win, c, acts); + } +} + +fn draw_widget(app: &mut EffectcraftApp, ui: &mut egui::Ui, origin: egui::Pos2, win: u32, w: &Widget, acts: &mut Vec) { + let t = app.tokens; + let rect = rect_of(origin, w.bounds); + register(app, win, w, rect); + let id = egui::Id::new(("sui", win, w.id)); + let tip = |r: egui::Response| if w.help_tip.is_empty() { r } else { r.on_hover_text(&w.help_tip) }; + match w.kind { + WidgetKind::Group | WidgetKind::Tab | WidgetKind::Window | WidgetKind::TabbedPanel => draw_children(app, ui, origin, win, w, acts), + WidgetKind::Panel => { + let p = ui.painter(); + p.rect_stroke(rect.shrink(1.0), 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + if !w.text.is_empty() { + let g = p.layout_no_wrap(w.text.clone(), Tokens::ui(11.5), t.text_dim); + let at = pos2(rect.min.x + 10.0, rect.min.y - 1.0); + p.rect_filled(Rect::from_min_size(at - vec2(3.0, 0.0), g.size() + vec2(6.0, 0.0)), 0.0, t.panel_bg); + p.galley(at, g, t.text_dim); + } + draw_children(app, ui, origin, win, w, acts); + } + WidgetKind::Button | WidgetKind::IconButton => { + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(rect, egui::Button::new(&w.text))).inner; + if tip(r).clicked() { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::StaticText => { + let col = if w.enabled { t.text } else { t.text_faint }; + ui.painter().with_clip_rect(rect.expand(2.0)).text(pos2(rect.min.x, rect.min.y + 1.0), Align2::LEFT_TOP, &w.text, Tokens::ui(12.0), col); + } + WidgetKind::EditText => { + // The text being typed lives here until the field loses focus (then onChange). + let mut buf: String = ui.data(|d| d.get_temp::(id)).unwrap_or_else(|| w.text.clone()); + let focused = ui.memory(|m| m.has_focus(id)); + if !focused { + buf = w.text.clone(); + } + let te = if w.multiline { egui::TextEdit::multiline(&mut buf) } else { egui::TextEdit::singleline(&mut buf) }; + let te = te.id(id).interactive(!w.read_only).font(Tokens::ui(12.0)); + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(rect, te)).inner; + let r = tip(r); + if r.lost_focus() && buf != w.text { + acts.push(Act::Set(win, w.id, json!(buf))); + } + ui.data_mut(|d| d.insert_temp(id, buf)); + } + WidgetKind::Checkbox => { + let mut v = w.checked; + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(snug(rect, &w.text), egui::Checkbox::new(&mut v, &w.text))).inner; + if tip(r).clicked() { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::RadioButton => { + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(snug(rect, &w.text), egui::RadioButton::new(w.checked, &w.text))).inner; + if tip(r).clicked() && !w.checked { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::Slider | WidgetKind::Scrollbar => { + // While dragging, the value lives here; the release sends it (onChanging / onChange). + let mut v: f64 = ui.data(|d| d.get_temp::(id)).unwrap_or(w.value); + let (lo, hi) = if w.max > w.min { (w.min, w.max) } else { (0.0, 100.0) }; + let r = ui + .add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect), |ui| { + ui.spacing_mut().slider_width = rect.width() - 4.0; + ui.add(egui::Slider::new(&mut v, lo..=hi).show_value(false)) + }) + .inner + }) + .inner; + let r = tip(r); + if r.dragged() { + ui.data_mut(|d| d.insert_temp(id, v)); + } else { + ui.data_mut(|d| d.remove::(id)); + if r.drag_stopped() || r.changed() { + acts.push(Act::Set(win, w.id, json!(v))); + } + } + } + WidgetKind::Progressbar => { + let span = if w.max > w.min { w.max - w.min } else { 100.0 }; + let f = ((w.value - w.min) / span).clamp(0.0, 1.0) as f32; + ui.put(rect, egui::ProgressBar::new(f).desired_height(rect.height())); + } + WidgetKind::DropDownList => { + let cur = w.selection.first().and_then(|i| w.items.get(*i)).cloned().unwrap_or_default(); + let mut pick: Option = None; + ui.add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect), |ui| { + egui::ComboBox::from_id_salt(id).selected_text(cur).width(rect.width() - 8.0).show_ui(ui, |ui| { + for (i, it) in w.items.iter().enumerate() { + if it == "-" { + ui.separator(); + } else if ui.selectable_label(w.selection.contains(&i), it).clicked() { + pick = Some(i); + } + } + }); + }); + }); + if let Some(i) = pick + && !w.selection.contains(&i) + { + acts.push(Act::Set(win, w.id, json!(i))); + } + } + WidgetKind::ListBox => { + let mut pick: Option = None; + ui.painter().rect_filled(rect, 2.0, t.field_bg); + ui.add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect.shrink(2.0)), |ui| { + egui::ScrollArea::vertical().id_salt(id).max_height(rect.height() - 4.0).show(ui, |ui| { + ui.set_width(rect.width() - 8.0); + for (i, it) in w.items.iter().enumerate() { + if ui.selectable_label(w.selection.contains(&i), it).clicked() { + pick = Some(i); + } + } + }); + }); + }); + if let Some(i) = pick { + acts.push(Act::Set(win, w.id, json!(i))); + } + } + WidgetKind::Image | WidgetKind::Unknown => { + ui.painter().rect_stroke(rect, 2.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + } + } +} + +/// Checkboxes and radio buttons sit at the left of their (possibly filled) bounds. +fn snug(r: Rect, text: &str) -> Rect { + let w = (text.chars().count() as f32 * 7.0 + 26.0).min(r.width()); + Rect::from_min_size(r.min, vec2(w, r.height())) +} + +fn perform(app: &mut EffectcraftApp, ctx: &egui::Context, acts: Vec) { + for a in acts { + let (cmd, p) = match a { + Act::Click(w, id) => ("scriptui.click", json!({"window": w, "widget": id})), + Act::Set(w, id, v) => ("scriptui.set", json!({"window": w, "widget": id, "value": v})), + Act::Close(w) => ("scriptui.close", json!({"window": w})), + }; + match crate::menus::invoke(app, ctx, cmd, p) { + Ok(r) => { + if let Some(e) = r.get("error").filter(|e| !e.is_null()) { + let msg = e["message"].as_str().unwrap_or("script error").to_string(); + app.toast = Some((msg, ctx.input(|i| i.time))); + } else if let Some(o) = r["output"].as_str().filter(|o| !o.is_empty()) { + app.ui.status = o.lines().last().unwrap_or_default().to_string(); + } + } + Err(e) => app.ui.status = e, + } + } +} + +/// The floating script windows (dialogs, palettes, windows), drawn over the app. +pub fn show_windows(app: &mut EffectcraftApp, ctx: &egui::Context) { + let windows: Vec = app.session.script_ui.windows.iter().filter(|w| w.kind != WindowKind::Panel).cloned().collect(); + if windows.is_empty() { + return; + } + let t = app.tokens; + let mut acts = vec![]; + let screen = ctx.content_rect(); + if windows.iter().any(|w| w.modal) { + // A modal dialog blocks the app behind it, as in After Effects. + let r = egui::Area::new(egui::Id::new("scriptui-modal-block")).order(egui::Order::Middle).fixed_pos(screen.min).show(ctx, |ui| { + ui.allocate_exact_size(screen.size(), Sense::click_and_drag()); + ui.painter().rect_filled(screen, 0.0, Color32::from_black_alpha(90)); + }); + let _ = r; + } + for w in &windows { + let size = vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32); + let mut open = true; + let order = if w.modal { egui::Order::Foreground } else { egui::Order::Middle }; + let resp = egui::Window::new(if w.title.is_empty() { &w.script } else { &w.title }) + .id(egui::Id::new(("scriptui-window", w.id))) + .order(order) + .collapsible(false) + .resizable(false) + .open(&mut open) + .default_pos(screen.center() - size / 2.0) + .frame(egui::Frame::window(&ctx.global_style()).fill(t.panel_bg)) + .show(ctx, |ui| { + let (rect, _) = ui.allocate_exact_size(size, Sense::hover()); + draw_children(app, ui, rect.min, w.id, &w.root, &mut acts); + rect + }); + if let Some(r) = resp { + app.auto.add(&format!("scriptui.{}", w.id), r.response.rect, &w.title); + } + if !open { + acts.push(Act::Close(w.id)); + } + } + perform(app, ctx, acts); +} + +/// A dockable ScriptUI panel's content (Window ▸ .jsx). +pub fn panel(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, id: u32) { + let t = app.tokens; + let Some(mut w) = app.session.script_ui.window(id).cloned() else { + ui.painter().text(rect.center(), Align2::CENTER_CENTER, "This ScriptUI panel has closed", Tokens::ui(12.0), t.text_faint); + return; + }; + // Docked panels fill their frame (ScriptUI lays them out at the panel's size). + let (pw, ph) = (rect.width() as f64, rect.height() as f64); + w.root.fixed_bounds = Some([0.0, 0.0, pw.max(w.root.bounds[2]), ph.max(w.root.bounds[3])]); + layout(&mut w); + let mut acts = vec![]; + let content = rect; + egui::ScrollArea::both().id_salt(("scriptui-panel", id)).max_width(content.width()).max_height(content.height()).show(ui, |ui| { + let (r, _) = ui.allocate_exact_size(vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32), Sense::hover()); + draw_children(app, ui, r.min, id, &w.root, &mut acts); + }); + app.auto.add(&format!("scriptui.{id}"), rect, &w.title); + let ctx = ui.ctx().clone(); + perform(app, &ctx, acts); +} + +/// Tab title of a script panel. +pub fn panel_title(app: &EffectcraftApp, id: u32) -> Option { + app.session.script_ui.window(id).map(|w| if w.title.is_empty() { w.script.clone() } else { w.title.clone() }) +} + +/// Close the dock panels whose script window has gone (the script closed it, or a saved +/// workspace from an earlier run). +pub fn sync_panels(app: &mut EffectcraftApp) { + let mut all = vec![]; + app.ui.dock.panels(&mut all); + for f in &app.ui.floating { + all.extend(f.panels.iter().copied()); + } + for p in all { + if let PanelKind::ScriptPanel(id) = p + && app.session.script_ui.window(id).is_none() + { + app.close_panel(p); + } + } +} diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index 364e114e..02c59636 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -404,6 +404,27 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p bx += 34.0; } y += 34.0; + // Face Tracking (Detailed Features): Extract & Copy Face Measurements. + let has_points = app + .session + .active_comp() + .and_then(|c| c.layer(layer)) + .and_then(|l| l.effects()) + .is_some_and(|fx| fx.groups().any(|g| g.match_id == effectcraft_engine::effects::face_track::POINTS_ID)); + if method == MaskMethod::FaceDetailed || has_points { + let r = Rect::from_min_size(pos2(x0, y), vec2(w.min(260.0), 24.0)); + if button(app, ui, r, "Extract & Copy Face Measurements", has_points && !running, "tracker.mask.extractFace") { + match crate::menus::invoke(app, ctx, "track.extractFaceMeasurements", json!({"layer": layer.0})) { + Ok(v) => { + if let Some(text) = v.get("clipboard").and_then(|t| t.as_str()) { + ctx.copy_text(text.to_string()); + } + } + Err(e) => app.ui.status = e, + } + } + y += 32.0; + } if let Some(pr) = app.session.mask_track_progress() { let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0)); p.rect_filled(bar, 3.0, t.field_bg); @@ -419,7 +440,12 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p ); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } else { - p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Tracks the pixels inside the mask and keys its Mask Path.", Tokens::ui(11.0), t.text_faint); + let hint = if method.is_face() { + "Finds the face inside the mask and keys its outline (and, with Detailed Features, Face Track Points)." + } else { + "Tracks the pixels inside the mask and keys its Mask Path." + }; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, hint, Tokens::ui(11.0), t.text_faint); } } diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index b8ec8d41..66c3c94a 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -121,6 +121,17 @@ enum Gesture { pin: u64, inv: Mat3, }, + /// Puppet tool Record mode (⌘/Ctrl-drag a pin): the drag is sampled in real time (seconds + /// since it began, layer space) and becomes keyframes on release (`puppet.recordPin`); the + /// current time runs along while recording. + PuppetRecord { + layer: LayerId, + pin: u64, + inv: Mat3, + began: f64, + cti: Tick, + samples: Vec<(f64, [f64; 2])>, + }, /// Pen tool: pulling the tangents of the vertex just placed. Pen { layer: LayerId, @@ -934,7 +945,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { t if t.is_shape() => Some(Gesture::Create { tool: t, start: cpt }), t if t.puppet_kind().is_some() => pin_hits.iter().rev().find(|h| h.pos.distance(press) < 8.0).and_then(|h| { let l = comp.layer(h.layer)?; - Some(Gesture::PuppetPin { layer: h.layer, pin: h.pin, inv: l2c(&ectx, l).0.inverse()? }) + let inv = l2c(&ectx, l).0.inverse()?; + if mods.command { + let lp = inv.apply(gv2(cpt[0], cpt[1])); + let began = ui.input(|i| i.time); + return Some(Gesture::PuppetRecord { layer: h.layer, pin: h.pin, inv, began, cti: time, samples: vec![(0.0, [lp.x, lp.y])] }); + } + Some(Gesture::PuppetPin { layer: h.layer, pin: h.pin, inv }) }), Tool::Orbit | Tool::PanCamera | Tool::Dolly => Some(Gesture::Camera { tool }), Tool::Selection if super::tracker::hit_at(&track_hits, press).is_some() => { @@ -1170,6 +1187,20 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let _ = app.session.execute("mask.setVertex", params); } + Gesture::PuppetRecord { layer, pin, inv, began, cti, mut samples } => { + let lp = inv.apply(gv2(cpt[0], cpt[1])); + let now = ui.input(|i| i.time) - began; + if samples.last().is_none_or(|s| now > s.0) { + samples.push((now, [lp.x, lp.y])); + } + // The current time runs with the recording (Record Options ▸ Speed). + let speed = app.session.state.puppet.record_speed.max(1.0) / 100.0; + app.session.set_time(cti + Tick::from_seconds_f64(now / speed)); + painter.circle_filled(pos, 6.0, Color32::from_rgb(0xe0, 0x30, 0x30)); + painter.text(pos + vec2(10.0, -10.0), Align2::LEFT_BOTTOM, "Recording", Tokens::ui(11.0), Color32::from_rgb(0xff, 0x60, 0x60)); + ui.data_mut(|dd| dd.insert_temp(gid, Gesture::PuppetRecord { layer, pin, inv, began, cti, samples })); + ui.ctx().request_repaint(); + } Gesture::PuppetPin { layer, pin, inv } => { let lp = inv.apply(gv2(cpt[0], cpt[1])); if let Err(e) = app.session.execute("puppet.movePin", json!({"layer": layer.0, "pin": pin, "position": [lp.x, lp.y], "merge": merge})) { @@ -1202,6 +1233,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if resp.drag_stopped() { if let (Some(g), Some(pos)) = (gesture, resp.interact_pointer_pos()) { match g { + Gesture::PuppetRecord { layer, pin, cti, samples, .. } => { + let pts: Vec = samples.iter().map(|(t, p)| json!([t, p[0], p[1]])).collect(); + if let Err(e) = app.session.execute("puppet.recordPin", json!({"layer": layer.0, "pin": pin, "samples": pts, "start": cti.seconds()})) { + app.ui.status = e.to_string(); + } + } Gesture::Create { tool, start } => { let end = map.to_comp(pos); create_shape(app, tool, start, end, mods.shift); diff --git a/crates/ui-egui/tests/render_queue_snapshot.rs b/crates/ui-egui/tests/render_queue_snapshot.rs index 5029a5f2..41ce13d2 100644 --- a/crates/ui-egui/tests/render_queue_snapshot.rs +++ b/crates/ui-egui/tests/render_queue_snapshot.rs @@ -41,3 +41,26 @@ fn render_queue_panel_snapshot() { } img.save(&out).unwrap(); } + +/// Edit ▸ Templates ▸ Output Module… (`RQ_TEMPLATES_SNAPSHOT=/abs/out.png`). +#[test] +#[ignore] +fn templates_dialog_snapshot() { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("renderQueue.add", json!({"output": "/renders/[compName].[fileExtension]"})).unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "outputModule", "name": "Web Delivery", "item": s.project.render_queue[0].id, "params": {"format": "webm", "resize": {"preset": "HDTV 720"}}})) + .unwrap(); + let kind = std::env::var("RQ_TEMPLATES_KIND").unwrap_or_else(|_| "outputModule".into()); + s.execute("app.templates", json!({"kind": kind})).unwrap(); + let mut app = Some(EffectcraftApp::new(s)); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app.take().expect("app")); + h.run_steps(4); + assert!(h.state().dialog.is_some(), "the Templates dialog is open"); + let img = h.render().expect("render"); + let out = std::env::var("RQ_TEMPLATES_SNAPSHOT").unwrap_or_else(|_| concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/test-out/rq_templates.png").into()); + if let Some(d) = std::path::Path::new(&out).parent() { + std::fs::create_dir_all(d).unwrap(); + } + img.save(&out).unwrap(); +} diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs index 45374fba..963ce3b6 100644 --- a/crates/ui-egui/tests/ui_camera_tracker.rs +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -41,6 +41,7 @@ fn app() -> (EffectcraftApp, u64, u64) { }) .collect(), ground: None, + distortion: None, }; s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); s.camera_pending.clear(); diff --git a/crates/ui-egui/tests/ui_scriptui.rs b/crates/ui-egui/tests/ui_scriptui.rs new file mode 100644 index 00000000..bbfd9e56 --- /dev/null +++ b/crates/ui-egui/tests/ui_scriptui.rs @@ -0,0 +1,113 @@ +//! M13.1 UI: ScriptUI dialogs, palettes and dockable panels drawn from the session's script +//! windows (clicks reach the script's handlers), the branching History panel, and the Window / +//! File ▸ Scripts menus listing scripts (egui_kittest, UI logic only). + +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui::{Event, Rect, pos2, vec2}; +use egui_kittest::Harness; +use serde_json::json; + +fn harness() -> Harness<'static, EffectcraftApp> { + let mut s = effectcraft_host::session(); + s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 4})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.run_steps(3); + h +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, id: &str) { + let c = rect(h, id).center(); + h.input_mut().events.push(Event::PointerMoved(c)); + for pressed in [true, false] { + h.input_mut().events.push(Event::PointerButton { pos: c, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(3); +} + +fn layer_names(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().session.active_comp().map(|c| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +#[test] +fn script_windows_are_drawn_and_clickable() { + let mut h = harness(); + let code = r#" + var w = new Window("palette", "Quick Solid"); + var n = w.add("edittext", undefined, "Card"); + n.characters = 10; + var go = w.add("button", undefined, "Add", { name: "add" }); + go.onClick = function () { app.project.activeItem.layers.addSolid([0, 1, 0], n.text, 50, 50, 1); }; + w.show(); + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "quick.jsx"})).unwrap(); + assert_eq!(r["ok"], true, "{r}"); + h.run_steps(3); + let win = h.state().session.script_ui.windows[0].id; + // Every control has an automation id (by id and by name). + for id in [format!("scriptui.{win}"), format!("scriptui.{win}.1"), format!("scriptui.{win}.2"), format!("scriptui.{win}.add")] { + assert!(h.state().auto.find(&id).is_some(), "missing {id}"); + } + click(&mut h, &format!("scriptui.{win}.add")); + assert_eq!(layer_names(&h), ["Card"]); + // A modal dialog: the script waits in show() until OK is clicked. + let code = r#" + var d = new Window("dialog", "Confirm"); + d.add("statictext", undefined, "Make a null?"); + var g = d.add("group"); + g.add("button", undefined, "Cancel", { name: "cancel" }); + g.add("button", undefined, "OK", { name: "ok" }); + if (d.show() === 1) app.project.activeItem.layers.addNull().name = "Yes"; + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "confirm.jsx"})).unwrap(); + assert_eq!(r["waiting"], true, "{r}"); + h.run_steps(3); + let d = h.state().session.script_ui.windows.iter().find(|w| w.modal).unwrap().id; + click(&mut h, &format!("scriptui.{d}.ok")); + assert_eq!(layer_names(&h), ["Yes", "Card"]); + assert!(h.state().session.script_ui.windows.iter().all(|w| !w.modal)); + // Window ▸ : the sample ScriptUI panel docks as its own tab. + h.state_mut().session.execute("window.scriptPanel", json!({"name": "Layer Tools.jsx"})).unwrap(); + h.run_steps(3); + let panel = h.state().session.script_ui.windows.iter().find(|w| w.script == "Layer Tools.jsx").unwrap().id; + assert!(h.state().ui.dock.contains(PanelKind::ScriptPanel(panel)), "the panel docks"); + assert!(h.state().auto.find(&format!("scriptui.{panel}")).is_some()); + // The menus list scripts and panels. + let cx = effectcraft_engine::menus::DynCtx::default(); + let (panels, _) = effectcraft_engine::menus::dynamic(&h.state().session, "scriptPanels", &cx); + assert!(panels.iter().any(|e| e.label == "Layer Tools.jsx" && e.command == "window.scriptPanel")); + let (scripts, _) = effectcraft_engine::menus::dynamic(&h.state().session, "scripts", &cx); + assert!(scripts.iter().any(|e| e.label == "Rename Layers.jsx" && e.command == "file.runScript")); + // Closing the panel's tab closes its script window. + h.state_mut().close_panel(PanelKind::ScriptPanel(panel)); + h.run_steps(2); + assert!(h.state().session.script_ui.window(panel).is_none()); + h.state_mut().session.execute("scriptui.close", json!({})).unwrap(); +} + +#[test] +fn history_panel_jumps_between_branches() { + let mut h = harness(); + for n in ["A", "B"] { + h.state_mut().session.execute("layer.newSolid", json!({"name": n, "color": "#ffffff", "width": 10, "height": 10})).unwrap(); + } + h.state_mut().session.undo(); + h.state_mut().session.execute("layer.newSolid", json!({"name": "C", "color": "#ffffff", "width": 10, "height": 10})).unwrap(); + assert_eq!(layer_names(&h), ["C", "A"]); + h.state_mut().show_panel(PanelKind::History); + h.run_steps(3); + // Original, New Composition, A, C, and B on its branch. + let states = h.state().session.history_tree(); + assert_eq!(states.len(), 5); + let b = states.iter().find(|n| n.depth == 1).unwrap().index; + click(&mut h, &format!("history.state.{b}")); + assert_eq!(layer_names(&h), ["B", "A"]); + let c = h.state().session.history_tree().iter().find(|n| n.depth == 1).unwrap().index; + click(&mut h, &format!("history.state.{c}")); + assert_eq!(layer_names(&h), ["C", "A"]); +} diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml index 62144b25..fe2e767c 100644 --- a/crates/vp9enc/Cargo.toml +++ b/crates/vp9enc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-vp9enc" -description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +description = "VP9 encoder (profile 0, 8-bit 4:2:0; intra + inter frames, loop filter, rate control) for WebM export, from the VP9 Bitstream & Decoding Process Specification" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md index c8f781b8..b9f94fae 100644 --- a/crates/vp9enc/README.md +++ b/crates/vp9enc/README.md @@ -1,15 +1,47 @@ # effectcraft-vp9enc -A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, -BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 -DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 -(mathematically lossless). +A small VP9 encoder for WebM export (profile 0, 8-bit 4:2:0, BT.709). + +* **Key frames**: 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 DCT/ADST + transforms, or the Walsh–Hadamard transform at quality 100 (mathematically lossless). +* **Inter frames**: one reference (the previous frame, LAST), 16×16 or 8×8 blocks chosen by a + cost comparison; motion search from the predicted vectors (NearestMv / NearMv / last frame's + co-located vector) with a whole-sample cross/diamond search refined to half and quarter + samples with the regular 8-tap interpolation filters; NEARESTMV / NEARMV / ZEROMV / NEWMV + chosen by bit cost; blocks with no coefficients worth coding are skipped; 4×4 DCT residual + with a rate–distortion check per transform block. Inter frames are coded error resilient + (`error_resilient_mode` = 1: no dependence on earlier probability contexts or motion vectors), + so `UsePrevFrameMvs` is never needed. No high-precision (1/8) vectors, no compound prediction, + no intra blocks inside inter frames. +* **Loop filter**: the in-loop deblocking filter (§8.8) on every lossy frame; the level is the + best of three trials against the source (sharpness 0, no mode/reference deltas). +* **Probabilities**: per-frame forward updates in the compressed header (`diff_update_prob` / + `update_mv_prob`) of the coefficient, skip, inter-mode, is-inter, reference, partition and + motion-vector probabilities. Tiles are recorded as symbols first, the frame's own counts pick + the updates that pay for their signalling, then the symbols are entropy coded with the updated + probabilities. There is no backward adaptation (`refresh_frame_context` = 0). +* **Rate control**: constant quality (quality → `base_q_idx`) or a target bitrate: a per-frame + q index from a bits ≈ complexity / step model per frame type, key frames getting a larger share + of each key-frame interval, with the running over/undershoot paid back over about a second. +* **Key frame interval** is configurable (1 = every frame a key frame, the old intra-only mode). + +The encoder reconstructs exactly what a decoder reconstructs (inter prediction with the +decoder's vector clamping and edge-clamped reference fetches, integer inverse transforms, the +loop filter) and exposes it (`Vp9Encoder::reconstruction`); the tests require FilmCraft's VP9 +decoder (and ffmpeg, when installed) to output exactly that reconstruction, frame by frame, for +moving synthetic sequences at several sizes and qualities, with tile columns, and lossless. + +On a moving-gradient sequence (192×108, 24 frames) inter coding is 3.9–7× smaller than all-intra +at the same quality setting with equal or better PSNR. Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March 2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode -info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) -and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The -boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec -tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR -Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an -external test oracle. No libvpx or other third-party encoder/decoder source was consulted. +info, motion vector and token syntax (§6.4), derives motion vector candidates with the §6.4 +`find_mv_refs` / `find_best_ref_mvs` processes, and reconstructs with the specification's intra +and inter prediction (§8.5), integer inverse transforms (§8.7) and loop filter (§8.8) so it stays +in step with any conformant decoder. The boolean encoder is the inverse of the §9.2 decoder in +the form given in RFC 6386 §7.3. Spec tables (§10) are the specification's; the key-frame tables +were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0), which +is also the decoder the tests round-trip through and whose behaviour (loop filter edge rules, +vector candidate order) the encoder was checked against; ffmpeg is used only as an external test +oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs index 45db92dd..0cd91b0e 100644 --- a/crates/vp9enc/src/bool.rs +++ b/crates/vp9enc/src/bool.rs @@ -64,31 +64,6 @@ impl BoolEncoder { } } - /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). - pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { - // Find the path to the leaf. - fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { - for b in 0..2 { - let t = tree[node + b]; - out.push((node >> 1, b == 1)); - if t <= 0 { - if (-t) as u8 == value { - return true; - } - } else if path(tree, t as usize, value, out) { - return true; - } - out.pop(); - } - false - } - let mut p = Vec::with_capacity(8); - path(tree, 0, value, &mut p); - for (i, bit) in p { - self.write(bit, probs[i]); - } - } - /// Flush (RFC 6386 §7.3) and return the bytes. pub fn finish(mut self) -> Vec { let mut c = self.bit_count; diff --git a/crates/vp9enc/src/lf.rs b/crates/vp9enc/src/lf.rs new file mode 100644 index 00000000..d781144f --- /dev/null +++ b/crates/vp9enc/src/lf.rs @@ -0,0 +1,124 @@ +//! The in-loop deblocking filter (spec 8.8), as the decoder applies it to a frame coded with 4×4 +//! transforms, 8×8 / 16×16 blocks, no segmentation and no reference/mode deltas: superblocks in +//! raster order, for each plane the vertical edges (pass 0) then the horizontal edges (pass 1). +//! Transform edges inside a skipped inter block are left alone; every 4×4 edge uses the 4-tap +//! filter except edges on a 32-sample boundary, which get the 8-tap (flat) filter. + +use crate::{BLOCK_16X16, Mi, Plane}; + +#[derive(Clone, Copy)] +struct Limits { + limit: i32, + blimit: i32, + thresh: i32, +} + +/// Filter the reconstructed planes (Y, U, V, or a prefix of them) in place at `level` (0 = off; +/// sharpness 0). +pub fn filter_frame(planes: &mut [Plane], mi: &[Mi], mi_rows: usize, mi_cols: usize, level: u8) { + if level == 0 { + return; + } + let l = level as i32; + let limit = l.max(1); + let lim = Limits { limit, blimit: 2 * (l + 2) + limit, thresh: l >> 4 }; + for sbr in 0..mi_rows.div_ceil(8) { + for sbc in 0..mi_cols.div_ceil(8) { + for (plane, p) in planes.iter_mut().enumerate() { + for pass in 0..2 { + sb_plane(p, mi, mi_rows, mi_cols, plane, pass, sbr * 8, sbc * 8, lim); + } + } + } + } +} + +#[allow(clippy::too_many_arguments)] +fn sb_plane(p: &mut Plane, mi: &[Mi], mi_rows: usize, mi_cols: usize, plane: usize, pass: usize, row: usize, col: usize, lim: Limits) { + let (sx, sy) = if plane > 0 { (1, 1) } else { (0, 0) }; + let (sub, edge_len) = if pass == 0 { (sx, 64 >> sy) } else { (sy, 64 >> sx) }; + let stride = p.w; + for edge in 0..(16usize >> sub) { + let mut i = 0; + while i < edge_len { + let (x, y) = if pass == 0 { (col * 8 + edge * (4 << sx), row * 8 + (i << sy)) } else { (col * 8 + (i << sx), row * 8 + edge * (4 << sy)) }; + if x >= 8 * mi_cols || y >= 8 * mi_rows || (pass == 0 && x == 0) || (pass == 1 && y == 0) { + i += 8; + continue; + } + let lc = ((x >> 3) >> sx) << sx; + let lr = ((y >> 3) >> sy) << sy; + let m = mi[lr * mi_cols + lc]; + let size = if sub == 0 { m.size } else { m.size.max(BLOCK_16X16) }; + let n8 = if size >= BLOCK_16X16 { 2 } else { 1 }; + let is_block_edge = if pass == 0 { x & (8 * n8 - 1) == 0 } else { y & (8 * n8 - 1) == 0 }; + let tx_ok = !m.inter || !m.skip; + if !is_block_edge && !tx_ok { + i += 8; + continue; + } + let wide = edge % 8 == 0; + let n = if pass == 0 { (8 * mi_rows - y).div_ceil(1 << sy) } else { (8 * mi_cols - x).div_ceil(1 << sx) }.min(8).min(edge_len - i); + // A transform edge is not filtered in the last odd chroma column. + let odd_last = pass == 1 && sx == 1 && mi_cols & 1 == 1 && edge & 1 == 1; + let (px, py) = (x >> sx, y >> sy); + for l in 0..n { + let is_tx_edge = tx_ok && !(odd_last && x + (l << sx) + 8 >= mi_cols * 8); + if !(is_block_edge || is_tx_edge) { + continue; + } + let (pos, step) = if pass == 0 { ((py + l) * stride + px, 1) } else { (py * stride + px + l, stride) }; + filter_lane(&mut p.rec, pos, step, wide, lim); + } + i += 8; + } + } +} + +/// Filter mask, narrow (4-tap) and 8-tap flat filters of spec 8.8.5 across one edge position; +/// `pos` is q0, samples across the edge are `step` apart. +fn filter_lane(d: &mut [u8], pos: usize, step: usize, wide: bool, lim: Limits) { + let at = |k: isize| d[(pos as isize + k * step as isize) as usize] as i32; + let (p3, p2, p1, p0) = (at(-4), at(-3), at(-2), at(-1)); + let (q0, q1, q2, q3) = (at(0), at(1), at(2), at(3)); + let Limits { limit, blimit, thresh } = lim; + let mask = (p3 - p2).abs() <= limit + && (p2 - p1).abs() <= limit + && (p1 - p0).abs() <= limit + && (q1 - q0).abs() <= limit + && (q2 - q1).abs() <= limit + && (q3 - q2).abs() <= limit + && (p0 - q0).abs() * 2 + (p1 - q1).abs() / 2 <= blimit; + if !mask { + return; + } + let flat = + wide && (p1 - p0).abs() <= 1 && (q1 - q0).abs() <= 1 && (p2 - p0).abs() <= 1 && (q2 - q0).abs() <= 1 && (p3 - p0).abs() <= 1 && (q3 - q0).abs() <= 1; + let mut set = |k: isize, v: i32| d[(pos as isize + k * step as isize) as usize] = v as u8; + if flat { + // 8-tap: positions p2..q2, each the rounded mean of a 7-wide clamped window plus itself. + let s = [p3, p2, p1, p0, q0, q1, q2, q3]; + let px = |k: isize| s[(k.clamp(-4, 3) + 4) as usize]; + for k in -3isize..3 { + let mut sum = px(k); + for j in -3..=3 { + sum += px(k + j); + } + set(k, (sum + 4) >> 3); + } + return; + } + let c = |v: i32| v.clamp(-128, 127); + let hev = (p1 - p0).abs() > thresh || (q1 - q0).abs() > thresh; + let (ps1, ps0, qs0, qs1) = (p1 - 128, p0 - 128, q0 - 128, q1 - 128); + let filter = c(if hev { c(ps1 - qs1) } else { 0 } + 3 * (qs0 - ps0)); + let f1 = c(filter + 4) >> 3; + let f2 = c(filter + 3) >> 3; + set(0, c(qs0 - f1) + 128); + set(-1, c(ps0 + f2) + 128); + if !hev { + let f = (f1 + 1) >> 1; + set(1, c(qs1 - f) + 128); + set(-2, c(ps1 + f) + 128); + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs index 02cbb0e2..8f72cb7b 100644 --- a/crates/vp9enc/src/lib.rs +++ b/crates/vp9enc/src/lib.rs @@ -1,290 +1,89 @@ -//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & -//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see -//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width -//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard -//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by -//! prediction error, default probabilities (no probability updates). The encoder reconstructs -//! exactly what a decoder reconstructs (the specification's integer inverse transforms and -//! intra edge rules), so prediction never drifts. +//! A VP9 encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & Decoding Process +//! Specification* v0.6 by mirroring its decoding process in reverse (see README.md). +//! +//! * **Key frames**: 64×64 superblocks split to 8×8 intra blocks (DC / V / H / TM chosen per +//! block), 4×4 DCT/ADST transforms (the Walsh–Hadamard transform at quality 100: lossless). +//! * **Inter frames** (error resilient, single reference: the previous frame as LAST): 16×16 or +//! 8×8 blocks with motion vectors found by a candidate + diamond search refined to quarter +//! samples with the regular 8-tap filters; NEARESTMV / NEARMV / ZEROMV / NEWMV chosen by cost, +//! motion vector prediction exactly as the decoder derives it; skipped blocks; 4×4 DCT residual. +//! * **In-loop deblocking** (8.8) on every lossy frame; the level is picked per frame by trial. +//! * **Probabilities**: per-frame forward updates (compressed header) of the coefficient, skip, +//! inter mode, reference, partition and motion vector probabilities, chosen from the frame's +//! own symbol counts (no backward adaptation: `refresh_frame_context` = 0). +//! * **Rate control**: constant quality (quality → base_q_idx), or a target bitrate with a +//! per-frame q_idx model. +//! +//! The encoder reconstructs exactly what a decoder reconstructs (prediction, integer inverse +//! transforms, loop filter), so prediction never drifts; [`Vp9Encoder::reconstruction`] returns +//! it. mod bool; +mod lf; +mod mc; +mod probs; mod tables; +mod tile; +mod transform; -use bool::BoolEncoder; -use tables::*; +pub use mc::Mv; +use probs::Probs; +use tables::{AC_QLOOKUP, DC_QLOOKUP}; +use transform::Quant; -const DC_PRED: u8 = 0; -const V_PRED: u8 = 1; -const H_PRED: u8 = 2; -const TM_PRED: u8 = 9; -const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; - -const DCT_DCT: u8 = 0; -const ADST_DCT: u8 = 1; -const DCT_ADST: u8 = 2; -const ADST_ADST: u8 = 3; - -/// mode2txfm_map for the modes used here. -fn mode_tx_type(mode: u8) -> u8 { - match mode { - V_PRED => ADST_DCT, - H_PRED => DCT_ADST, - TM_PRED => ADST_ADST, - _ => DCT_DCT, - } -} - -const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; -const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; - -/// extra_bits[token] = (cat, numExtra, base). -const EXTRA_BITS: [(u8, u8, u16); 11] = - [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; -const CAT_PROBS: [&[u8]; 7] = [ - &[], - &[159], - &[165, 145], - &[173, 148, 140], - &[176, 155, 140, 135], - &[180, 157, 141, 134, 130], - &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], -]; +pub(crate) const BLOCK_8X8: u8 = 3; +pub(crate) const BLOCK_16X16: u8 = 6; /// Encoder settings. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub struct EncoderConfig { pub width: u32, pub height: u32, - /// 1..=100; 100 is lossless. + /// 1..=100; 100 is lossless. With a target bitrate: the quality of the first frame. pub quality: u8, /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. pub full_range: bool, + /// A key frame every this many frames (1 = every frame is a key frame; 0 = only the first). + pub keyframe_interval: u32, + /// Target bitrate in kbit/s (`None`: constant quality). + pub target_kbps: Option, + /// Frames per second (rate control). + pub frame_rate: f64, + /// The in-loop deblocking filter. + pub loop_filter: bool, } -/// Probabilities of the full token tree (9.3.2) for model probability `p`. -fn pareto(p: u8) -> [u8; 8] { - let p = p.max(1) as usize; - let x = (p - 1) / 2; - let mut t = [0u8; 8]; - for n in 0..8 { - t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; - } - t -} - -fn scan(tx_type: u8) -> &'static [u16; 16] { - match tx_type { - ADST_DCT => &ROW_SCAN_4X4, - DCT_ADST => &COL_SCAN_4X4, - _ => &DEFAULT_SCAN_4X4, +impl Default for EncoderConfig { + fn default() -> Self { + EncoderConfig { width: 0, height: 0, quality: 80, full_range: false, keyframe_interval: 60, target_kbps: None, frame_rate: 30.0, loop_filter: true } } } -/// Coefficient context neighbours (9.3.2) of each scan position. -fn neighbors(tx_type: u8) -> [(usize, usize); 16] { - let sc = scan(tx_type); - let mut nb = [(0, 0); 16]; - for (c, &pos) in sc.iter().enumerate().skip(1) { - let pos = pos as usize; - let (i, j) = (pos / 4, pos % 4); - nb[c] = if i > 0 && j > 0 { - let a = (i - 1) * 4 + j; - let b = i * 4 + j - 1; - match tx_type { - DCT_ADST => (a, a), - ADST_DCT => (b, b), - _ => (a, b), - } - } else if i > 0 { - ((i - 1) * 4 + j, (i - 1) * 4 + j) - } else { - (j - 1, j - 1) - }; - } - nb +/// One coded frame. +#[derive(Clone, Debug)] +pub struct EncodedFrame { + pub data: Vec, + pub key: bool, + pub q_idx: u8, + pub filter_level: u8, } -// ---------------------------------------------------------------- transforms (8.7) - -fn r14(v: i32) -> i32 { - (v + (1 << 13)) >> 14 -} - -fn idct4(x: [i32; 4]) -> [i32; 4] { - let v0 = r14(x[0] * 11585 - x[2] * 11585); - let v1 = r14(x[0] * 11585 + x[2] * 11585); - let v2 = r14(x[1] * 6270 - x[3] * 15137); - let v3 = r14(x[1] * 15137 + x[3] * 6270); - [v1 + v3, v0 + v2, v0 - v2, v1 - v3] -} - -fn iadst4(x: [i32; 4]) -> [i32; 4] { - let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); - let x: [i64; 4] = x.map(|v| v as i64); - let s0 = s19 * x[0]; - let s1 = s29 * x[0]; - let s2 = s39 * x[1]; - let s3 = s49 * x[2]; - let s4 = s19 * x[2]; - let s5 = s29 * x[3]; - let s6 = s49 * x[3]; - let s7 = s39 * (x[0] - x[2] + x[3]); - let x0 = s0 + s3 + s5; - let x1 = s1 - s4 - s6; - let x2 = s7; - let x3 = s2; - let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; - [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] -} - -/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. -fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { - let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); - let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); - let mut t = [[0i32; 4]; 4]; - for i in 0..4 { - let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; - t[i] = if row_adst { iadst4(r) } else { idct4(r) }; - } - let mut out = [0i32; 16]; - for j in 0..4 { - let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; - let c = if col_adst { iadst4(c) } else { idct4(c) }; - for i in 0..4 { - out[i * 4 + j] = (c[i] + 8) >> 4; - } - } - out -} - -/// Inverse Walsh–Hadamard (8.7.1.10) on one line. -fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { - let mut a = x[0] >> shift; - let mut c = x[1] >> shift; - let mut d = x[2] >> shift; - let mut b = x[3] >> shift; - a += c; - d -= b; - let e = (a - d) >> 1; - b = e - b; - c = e - c; - a -= b; - d += c; - [a, b, c, d] -} - -/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. -fn fwht(y: [i32; 4]) -> [i32; 4] { - let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); - let d1 = dd - cc; - let a1 = aa + bb; - let e = (a1 - d1) >> 1; - let b0 = e - bb; - let c0 = e - cc; - let a0 = a1 - c0; - let d0 = d1 + b0; - [a0, c0, d0, b0] -} - -fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { - let mut t = [[0i32; 4]; 4]; - for i in 0..4 { - t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); - } - let mut out = [0i32; 16]; - for j in 0..4 { - let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); - for i in 0..4 { - out[i * 4 + j] = r[i]; - } - } - out -} - -/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. -fn forward_wht(res: &[i32; 16]) -> [i32; 16] { - let mut t = [[0i32; 4]; 4]; - for j in 0..4 { - let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); - for i in 0..4 { - t[i][j] = col[i]; - } - } - let mut lv = [0i32; 16]; - for i in 0..4 { - let r = fwht(t[i]); - lv[i * 4..i * 4 + 4].copy_from_slice(&r); - } - lv -} - -/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse -/// transforms, so `inverse4(forward(res)) ≈ res`. -fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { - use std::sync::OnceLock; - static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); - M.get_or_init(|| { - let mut out = [[[0.0; 16]; 16]; 4]; - for (ty, o) in out.iter_mut().enumerate() { - // Columns of the inverse: response to a scaled unit coefficient. - let k = 1 << 16; - let mut inv = [[0.0f64; 16]; 16]; - for c in 0..16 { - let mut e = [0i32; 16]; - e[c] = k; - let r = inverse4(&e, ty as u8); - for p in 0..16 { - inv[p][c] = r[p] as f64 / k as f64; - } - } - *o = invert16(inv); - } - out - }) -} - -fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { - let mut a = m; - let mut inv = [[0.0; 16]; 16]; - for (i, r) in inv.iter_mut().enumerate() { - r[i] = 1.0; - } - for col in 0..16 { - let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); - a.swap(col, piv); - inv.swap(col, piv); - let d = a[col][col]; - for j in 0..16 { - a[col][j] /= d; - inv[col][j] /= d; - } - for r in 0..16 { - if r != col { - let f = a[r][col]; - if f != 0.0 { - for j in 0..16 { - a[r][j] -= f * a[col][j]; - inv[r][j] -= f * inv[col][j]; - } - } - } - } - } - inv -} - -// ---------------------------------------------------------------- planes and prediction - -struct Plane { - w: usize, - /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). - max_x: usize, - max_y: usize, - src: Vec, - rec: Vec, +/// A plane being coded: source and reconstruction at superblock-aligned size. +pub(crate) struct Plane { + /// Stride (superblock-aligned width). + pub w: usize, + /// Last valid column / row for intra prediction edges (the 8-aligned mode-info grid). + pub max_x: usize, + pub max_y: usize, + /// Visible size. + pub vis_w: usize, + pub vis_h: usize, + pub src: Vec, + pub rec: Vec, } impl Plane { + #[allow(clippy::too_many_arguments)] fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { let mut src = vec![0u8; w * h]; for y in 0..h { @@ -294,103 +93,141 @@ impl Plane { src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); } } - Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + Plane { w, max_x, max_y, vis_w: vw, vis_h: vh, src, rec: vec![0; w * h] } } } -/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. -fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { - let size = 1usize << log2; - let base = 128i32; - let buf = &p.rec; - let w = p.w; - let mut above = [0i32; 8]; - let mut left = [0i32; 8]; - if have_above { - for i in 0..size { - above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; - } - } else { - above[..size].fill(base - 1); - } - let top_left = if have_above && have_left { - buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 - } else if have_above { - base + 1 - } else { - base - 1 - }; - if have_left { - for i in 0..size { - left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; - } - } else { - left[..size].fill(base + 1); - } - let mut out = [0i32; 64]; - for i in 0..size { - for j in 0..size { - out[i * size + j] = match mode { - V_PRED => above[j], - H_PRED => left[i], - TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), - _ => 0, - }; - } - } - if mode == DC_PRED { - let v = match (have_left, have_above) { - (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), - (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, - (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, - (false, false) => base, - }; - out[..size * size].fill(v); - } - out +/// Mode info of one 8×8 position. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Mi { + pub inter: bool, + /// Intra mode (0..=9) or inter mode (NEARESTMV..=NEWMV). + pub mode: u8, + pub mv: Mv, + /// Skip as the loop filter sees it (inter blocks without coefficients). + pub skip: bool, + pub size: u8, } -// ---------------------------------------------------------------- the encoder +/// Rate control state (target bitrate mode). +#[derive(Clone, Debug)] +struct RateControl { + /// bits ≈ complexity / step: per frame type, from the last frame of that type. + key_complexity: Option, + inter_complexity: Option, + /// Bits spent minus bits budgeted so far. + debt: f64, + last_inter_q: Option, +} pub struct Vp9Encoder { cfg: EncoderConfig, - qidx: u8, + base_qidx: u8, lossless: bool, + frames: u64, + since_key: u32, + /// The reconstruction of the last frame (the LAST reference): planes at their strides. + reference: Option<[Plane; 3]>, + mvs: Vec, + rc: RateControl, } -/// Per-tile coding state. -struct Tile<'a> { - enc: &'a Vp9Encoder, - bc: BoolEncoder, - planes: &'a mut [Plane; 3], - mi_cols: usize, - mi_rows: usize, - col_start: usize, - above_nz: [Vec; 3], - left_nz: [Vec; 3], - above_part: Vec, - left_part: Vec, - /// y mode of every 8×8 block (for mode contexts). - modes: &'a mut [u8], +fn quality_to_qidx(q: u8) -> u8 { + // 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + ((100 - q.clamp(1, 99) as i32) * 255 / 85).clamp(1, 255) as u8 } impl Vp9Encoder { pub fn new(cfg: EncoderConfig) -> Self { let q = cfg.quality.clamp(1, 100); - let lossless = q >= 100; - // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. - let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; - Vp9Encoder { cfg, qidx, lossless } + let lossless = q >= 100 && cfg.target_kbps.is_none(); + let base_qidx = if lossless { 0 } else { quality_to_qidx(q) }; + Vp9Encoder { + cfg, + base_qidx, + lossless, + frames: 0, + since_key: 0, + reference: None, + mvs: vec![], + rc: RateControl { key_complexity: None, inter_complexity: None, debt: 0.0, last_inter_q: None }, + } } pub fn config(&self) -> &EncoderConfig { &self.cfg } - /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// Encode one frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: /// ((w+1)/2)×((h+1)/2), tightly packed). pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + self.encode_frame(y, u, v, false).data + } + + /// The decoder's reconstruction of the last coded frame (visible area, tightly packed Y, U, + /// V). + pub fn reconstruction(&self) -> Option<[Vec; 3]> { + let r = self.reference.as_ref()?; + Some(std::array::from_fn(|i| { + let p = &r[i]; + (0..p.vis_h).flat_map(|y| p.rec[y * p.w..y * p.w + p.vis_w].iter().copied()).collect() + })) + } + + fn is_key_due(&self, force: bool) -> bool { + force || self.reference.is_none() || self.cfg.keyframe_interval == 1 || (self.cfg.keyframe_interval > 1 && self.since_key >= self.cfg.keyframe_interval) + } + + /// The q index of the next frame. + fn pick_qidx(&mut self, key: bool) -> u8 { + if self.lossless { + return 0; + } + let Some(kbps) = self.cfg.target_kbps else { return self.base_qidx }; + let per_frame = kbps as f64 * 1000.0 / self.cfg.frame_rate.max(1.0); + let interval = if self.cfg.keyframe_interval == 0 { 300.0 } else { self.cfg.keyframe_interval as f64 }; + // Key frames get a larger share of the group's budget. + let boost = (interval / 6.0).clamp(1.0, 5.0); + let group = per_frame * interval; + let key_share = group * boost / (boost + interval - 1.0); + let inter_share = (group - key_share) / (interval - 1.0).max(1.0); + let mut target = if key { key_share } else { inter_share }; + // Pay back (or spend) the running difference over about a second. + let horizon = self.cfg.frame_rate.max(1.0); + target = (target - self.rc.debt / horizon).max(target * 0.25); + let complexity = if key { self.rc.key_complexity } else { self.rc.inter_complexity.or(self.rc.key_complexity.map(|c| c / (boost * 2.0))) }; + let Some(cx) = complexity else { return self.base_qidx }; + let step = (cx / target).max(4.0); + let mut q = (0..=255u8).min_by_key(|&q| ((AC_QLOOKUP[q as usize] as f64 - step).abs() * 16.0) as i64).unwrap_or(self.base_qidx); + if !key && let Some(last) = self.rc.last_inter_q { + q = q.clamp(last.saturating_sub(24), last.saturating_add(24)); + } + q.max(1) + } + + fn rc_update(&mut self, key: bool, qidx: u8, bytes: usize) { + let Some(kbps) = self.cfg.target_kbps else { return }; + let bits = bytes as f64 * 8.0; + let cx = bits * AC_QLOOKUP[qidx as usize] as f64; + if key { + self.rc.key_complexity = Some(cx); + } else { + self.rc.inter_complexity = Some(match self.rc.inter_complexity { + Some(c) => c * 0.5 + cx * 0.5, + None => cx, + }); + self.rc.last_inter_q = Some(qidx); + } + let per_frame = kbps as f64 * 1000.0 / self.cfg.frame_rate.max(1.0); + self.rc.debt += bits - per_frame; + } + + /// Encode one frame; `force_key` starts a new key frame. + pub fn encode_frame(&mut self, y: &[u8], u: &[u8], v: &[u8], force_key: bool) -> EncodedFrame { let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let key = self.is_key_due(force_key); + let qidx = self.pick_qidx(key); + let lossless = qidx == 0; let mi_cols = w.div_ceil(8); let mi_rows = h.div_ceil(8); let sb_cols = mi_cols.div_ceil(8); @@ -413,71 +250,77 @@ impl Vp9Encoder { } max_log2 -= 1; let tile_cols = 1usize << min_log2; - let mut modes = vec![0u8; mi_cols * mi_rows]; - let mut tiles = vec![]; + let quant = Quant { dcq: DC_QLOOKUP[qidx as usize], acq: AC_QLOOKUP[qidx as usize], lossless, round_dc: 0.5, round_ac: 0.38 }; + let step = AC_QLOOKUP[qidx as usize] as f64 / 8.0; + let lambda_sse = 0.35 * step * step; + let refs = if key { + None + } else { + self.reference.as_ref().map(|r| std::array::from_fn::<_, 3, _>(|i| mc::RefPlane { data: &r[i].rec, stride: r[i].w, w: r[i].vis_w, h: r[i].vis_h })) + }; + let prev = (!key && self.mvs.len() == mi_cols * mi_rows).then_some(&self.mvs[..]); + let phases = refs.as_ref().map(|r| mc::PhasePlanes::new(&r[0], 48)); + let fp = tile::FrameParams { key, quant, mi_cols, mi_rows, refs, phases, prev_mvs: prev, lambda_sad: lambda_sse.sqrt(), lambda_sse }; + let mut mi = vec![Mi::default(); mi_cols * mi_rows]; + let mut writers = vec![]; for t in 0..tile_cols { let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); - let mut tile = Tile { - enc: self, - bc: BoolEncoder::new(), - planes: &mut planes, - mi_cols, - mi_rows, - col_start: start, - above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], - left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], - above_part: vec![0; mi_cols + 8], - left_part: vec![0; 8], - modes: &mut modes, - }; - tile.bc.write(false, 128); - for r in (0..mi_rows).step_by(8) { - for p in 0..3 { - tile.left_nz[p].fill(0); - } - tile.left_part.fill(0); - for c in (start..end).step_by(8) { - tile.partition(r, c, 12); - } - } - tiles.push(tile.bc.finish()); + let mut tile = tile::Tile::new(&fp, &mut planes, &mut mi, start, end); + tile.code(); + writers.push(tile.w); } - // Compressed header. - let mut ch_enc = BoolEncoder::new(); - ch_enc.write(false, 128); - if !self.lossless { - ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + // Forward probability updates from this frame's counts. + let defaults = Probs::defaults(); + let mut counts = vec![[0u32; 2]; probs::NPROBS]; + for wr in &writers { + wr.count(&mut counts); } - ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) - for _ in 0..3 { - ch_enc.write(false, 252); // skip probabilities unchanged - } - let compressed = ch_enc.finish(); + let wanted = probs::plan_updates(&defaults, &counts, !key); + let (compressed, used) = probs::compressed_header(&defaults, &wanted, lossless, !key); + let tiles: Vec> = writers.iter().map(|wr| wr.emit(&used)).collect(); + drop(fp); + // Loop filter: the level with the least error against the source. + let level = if lossless || !self.cfg.loop_filter { 0 } else { choose_level(&mut planes, &mi, mi_rows, mi_cols, qidx, key) }; + lf::filter_frame(&mut planes, &mi, mi_rows, mi_cols, level); // Uncompressed header. let mut bw = BitWriter::default(); bw.put(2, 2); // frame_marker bw.put(0, 1); // profile low bit bw.put(0, 1); // profile high bit bw.put(0, 1); // show_existing_frame - bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(!key as u32, 1); // frame_type bw.put(1, 1); // show_frame - bw.put(0, 1); // error_resilient_mode - bw.put(0x49, 8); - bw.put(0x83, 8); - bw.put(0x42, 8); - bw.put(2, 3); // color_space CS_BT_709 - bw.put(self.cfg.full_range as u32, 1); - bw.put(w as u32 - 1, 16); - bw.put(h as u32 - 1, 16); - bw.put(0, 1); // render_and_frame_size_different - bw.put(0, 1); // refresh_frame_context - bw.put(1, 1); // frame_parallel_decoding_mode + if key { + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + } else { + bw.put(1, 1); // error_resilient_mode (no reliance on earlier contexts or vectors) + bw.put(0x01, 8); // refresh_frame_flags: slot 0 becomes the new LAST + for _ in 0..3 { + bw.put(0, 3); // ref_frame_idx: slot 0 + bw.put(0, 1); // sign bias + } + bw.put(1, 1); // found_ref: the size of LAST + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // allow_high_precision_mv + bw.put(0, 1); // is_filter_switchable + bw.put(1, 2); // raw_interpolation_filter → EIGHTTAP (regular) + } bw.put(0, 2); // frame_context_idx - bw.put(0, 6); // loop_filter_level + bw.put(level as u32, 6); // loop_filter_level bw.put(0, 3); // sharpness bw.put(0, 1); // mode_ref_delta_enabled - bw.put(self.qidx as u32, 8); + bw.put(qidx as u32, 8); bw.put(0, 1); // delta_q_y_dc bw.put(0, 1); // delta_q_uv_dc bw.put(0, 1); // delta_q_uv_ac @@ -496,10 +339,46 @@ impl Vp9Encoder { } out.extend_from_slice(&t); } - out + // State for the next frame. + self.mvs = mi.iter().map(|m| if m.inter { m.mv } else { Mv::ZERO }).collect(); + self.reference = Some(planes); + self.frames += 1; + self.since_key = if key { 1 } else { self.since_key + 1 }; + self.rc_update(key, qidx, out.len()); + EncodedFrame { data: out, key, q_idx: qidx, filter_level: level } } } +/// Try a few loop filter levels on the luma reconstruction; the one closest to the source +/// (squared error over the visible area) wins (0 when filtering doesn't help). +fn choose_level(planes: &mut [Plane; 3], mi: &[Mi], mi_rows: usize, mi_cols: usize, qidx: u8, key: bool) -> u8 { + let guess = ((qidx as f64 * if key { 0.25 } else { 0.2 }).round() as u8).clamp(1, 63); + let sse = |p: &Plane| -> u64 { + let mut s = 0u64; + for y in 0..p.vis_h { + for x in 0..p.vis_w { + let d = p.src[y * p.w + x] as i64 - p.rec[y * p.w + x] as i64; + s += (d * d) as u64; + } + } + s + }; + let mut best = (sse(&planes[0]), 0u8); + let saved = planes[0].rec.clone(); + for level in [guess / 2, guess, (guess as u16 * 3 / 2).min(63) as u8] { + if level == 0 { + continue; + } + lf::filter_frame(&mut planes[..1], mi, mi_rows, mi_cols, level); + let s = sse(&planes[0]); + planes[0].rec.copy_from_slice(&saved); + if s < best.0 { + best = (s, level); + } + } + best.1 +} + #[derive(Default)] struct BitWriter { out: Vec, @@ -527,254 +406,6 @@ impl BitWriter { } } -const BLOCK_8X8: u8 = 3; -const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; -const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; - -impl Tile<'_> { - fn partition(&mut self, r: usize, c: usize, bsize: u8) { - if r >= self.mi_rows || c >= self.mi_cols { - return; - } - let num8 = NUM_8X8[bsize as usize]; - let half = num8 >> 1; - let has_rows = (r + half) < self.mi_rows; - let has_cols = (c + half) < self.mi_cols; - let bsl = MI_WIDTH_LOG2[bsize as usize]; - let boffset = 3 - bsl; - let mut above = 0u8; - let mut left = 0u8; - for i in 0..num8 { - above |= self.above_part[c + i]; - left |= self.left_part[(r & 7) + i]; - } - let above = (above >> boffset) & 1; - let left = (left >> boffset) & 1; - let ctx = bsl * 4 + left as usize * 2 + above as usize; - let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; - if bsize == BLOCK_8X8 { - // PARTITION_NONE. - self.bc.write(false, probs[0]); - self.block(r, c); - self.above_part[c] = 15 >> 1; - self.left_part[r & 7] = 15 >> 1; - return; - } - // PARTITION_SPLIT. - if has_rows && has_cols { - self.bc.tree(&PARTITION_TREE, probs, 3); - } else if has_cols { - self.bc.write(true, probs[1]); - } else if has_rows { - self.bc.write(true, probs[2]); - } - let sub = bsize - 3; - self.partition(r, c, sub); - self.partition(r, c + half, sub); - self.partition(r + half, c, sub); - self.partition(r + half, c + half, sub); - } - - fn block(&mut self, r: usize, c: usize) { - let avail_u = r > 0; - let avail_l = c > self.col_start; - // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. - let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); - let uv_mode = { - let mut best = (i64::MAX, DC_PRED); - for m in MODES { - let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); - if cost < best.0 { - best = (cost, m); - } - } - best.1 - }; - // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). - self.bc.write(false, DEFAULT_SKIP_PROB[0]); - let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; - let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; - let o = (above_mode as usize * 10 + left_mode as usize) * 9; - self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); - let o = y_mode as usize * 9; - self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); - self.modes[r * self.mi_cols + c] = y_mode; - // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. - for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { - let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; - self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); - } - for p in 1..3 { - self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); - } - } - - fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { - let p = &self.planes[plane]; - let pred = predict(p, x, y, log2, mode, have_left, have_above); - let size = 1 << log2; - let mut s = 0i64; - for i in 0..size { - for j in 0..size { - s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; - } - } - s - } - - fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { - let mut best = (i64::MAX, DC_PRED); - for m in MODES { - let cost = self.cost(plane, x, y, log2, m, have_left, have_above); - if cost < best.0 { - best = (cost, m); - } - } - best.1 - } - - #[allow(clippy::too_many_arguments)] - fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { - let ss = (plane > 0) as usize; - let max_x = (self.mi_cols * 8) >> ss; - let max_y = (self.mi_rows * 8) >> ss; - let (x4, y4) = (x >> 2, y >> 2); - let ly = y4 & (15 >> ss); - if x >= max_x || y >= max_y { - self.above_nz[plane][x4] = 0; - self.left_nz[plane][ly] = 0; - return; - } - let p = &self.planes[plane]; - let pred = predict(p, x, y, 2, mode, have_left, have_above); - let mut res = [0i32; 16]; - for i in 0..4 { - for j in 0..4 { - res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; - } - } - let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); - // Quantised levels (raster order). - let levels: [i32; 16] = if self.enc.lossless { - forward_wht(&res) - } else { - let f = &forward_matrices()[tx_type as usize]; - let mut lv = [0i32; 16]; - for k in 0..16 { - let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); - let qs = if k == 0 { dcq } else { acq } as f64; - let a = coef.abs() / qs; - // Rounding with a small dead zone for AC coefficients. - let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; - lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; - } - lv - }; - let sc = scan(tx_type); - let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); - // Tokens. - let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; - self.tokens(plane, tx_type, &levels, eob, ctx0); - // Reconstruct exactly as the decoder does. - let mut deq = [0i32; 16]; - for k in 0..16 { - deq[k] = levels[k] * if k == 0 { dcq } else { acq }; - } - let resid = if eob == 0 { - [0i32; 16] - } else if self.enc.lossless { - inverse_wht(&deq) - } else { - inverse4(&deq, tx_type) - }; - let p = &mut self.planes[plane]; - for i in 0..4 { - for j in 0..4 { - p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; - } - } - let nz = (eob > 0) as u8; - self.above_nz[plane][x4] = nz; - self.left_nz[plane][ly] = nz; - } - - fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { - let sc = scan(tx_type); - let nb = neighbors(tx_type); - let ptype = (plane > 0) as usize; - // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. - let base = (ptype * 2) * 6 * 6 * 3; - let mut tc = [0u8; 16]; - let mut check_eob = true; - let mut c = 0; - while c < 16 { - let pos = sc[c] as usize; - let band = COEFBAND_4X4[c] as usize; - let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; - let o = base + (band * 6 + ctx) * 3; - let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; - if check_eob { - let more = c < eob; - self.bc.write(more, p[0]); - if !more { - break; - } - } - let v = levels[pos].unsigned_abs(); - if v == 0 { - self.bc.write(false, p[1]); - tc[pos] = 0; - check_eob = false; - c += 1; - continue; - } - self.bc.write(true, p[1]); - check_eob = true; - let token = match v { - 1 => 1, - 2 => 2, - 3 => 3, - 4 => 4, - 5..=6 => 5, - 7..=10 => 6, - 11..=18 => 7, - 19..=34 => 8, - 35..=66 => 9, - _ => 10, - }; - if token == 1 { - self.bc.write(false, p[2]); - } else { - self.bc.write(true, p[2]); - let pp = pareto(p[2]); - let path: &[(usize, bool)] = match token { - 2 => &[(0, false), (1, false)], - 3 => &[(0, false), (1, true), (2, false)], - 4 => &[(0, false), (1, true), (2, true)], - 5 => &[(0, true), (3, false), (4, false)], - 6 => &[(0, true), (3, false), (4, true)], - 7 => &[(0, true), (3, true), (5, false), (6, false)], - 8 => &[(0, true), (3, true), (5, false), (6, true)], - 9 => &[(0, true), (3, true), (5, true), (7, false)], - _ => &[(0, true), (3, true), (5, true), (7, true)], - }; - for &(n, b) in path { - self.bc.write(b, pp[n]); - } - let (cat, num_extra, base_v) = EXTRA_BITS[token]; - let extra = (v - base_v as u32).min((1 << num_extra) - 1); - let cp = CAT_PROBS[cat as usize]; - for e in 0..num_extra as usize { - self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); - } - } - tc[pos] = ENERGY_CLASS[token]; - self.bc.write(levels[pos] < 0, 128); - c += 1; - } - } -} - // ---------------------------------------------------------------- colour conversion /// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). @@ -822,36 +453,3 @@ pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); (y, vec![128; cw * ch], vec![128; cw * ch]) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn wht_is_exactly_invertible() { - let mut s = 7u32; - for _ in 0..2000 { - let mut r = [0i32; 16]; - for v in &mut r { - s = s.wrapping_mul(1664525).wrapping_add(1013904223); - *v = (s >> 16) as i32 % 511 - 255; - } - let lv = forward_wht(&r); - let deq = lv.map(|l| l * 4); - assert_eq!(inverse_wht(&deq), r); - } - } - - #[test] - fn forward_transforms_invert_the_decoder() { - for ty in 0..4u8 { - let f = &forward_matrices()[ty as usize]; - let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); - let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); - let back = inverse4(&c, ty); - for i in 0..16 { - assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); - } - } - } -} diff --git a/crates/vp9enc/src/mc.rs b/crates/vp9enc/src/mc.rs new file mode 100644 index 00000000..779a1ea5 --- /dev/null +++ b/crates/vp9enc/src/mc.rs @@ -0,0 +1,207 @@ +//! Inter prediction exactly as the decoder forms it: motion vector clamping (spec 8.5.2.2) and +//! the unscaled block inter prediction process (8.5.2.4) with the regular 8-tap filters and +//! reference samples clamped to the reference frame's visible area. + +use crate::tables::SUBPEL_REGULAR; + +/// A motion vector in 1/8 luma samples. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Mv { + pub row: i32, + pub col: i32, +} + +impl Mv { + pub const ZERO: Mv = Mv { row: 0, col: 0 }; + pub fn new(row: i32, col: i32) -> Mv { + Mv { row, col } + } + pub fn minus(self, o: Mv) -> Mv { + Mv::new(self.row - o.row, self.col - o.col) + } +} + +/// A reference plane: `w`×`h` visible samples at `stride`. +pub struct RefPlane<'a> { + pub data: &'a [u8], + pub stride: usize, + pub w: usize, + pub h: usize, +} + +/// The block's motion vector for `plane` in 1/16 samples of that plane, clamped as the +/// decoder clamps it. `n8`: block size in 8×8 units. +pub fn clamp_for_plane(mv: Mv, r: usize, c: usize, n8: usize, mi_rows: usize, mi_cols: usize, chroma: bool) -> (i32, i32) { + let s = chroma as i32; + let (r, c, b) = (r as i32, c as i32, n8 as i32); + let to_top = (-(r * 8 * 16)) >> s; + let to_bottom = (((mi_rows as i32 - b - r) * 8) * 16) >> s; + let to_left = (-(c * 8 * 16)) >> s; + let to_right = (((mi_cols as i32 - b - c) * 8) * 16) >> s; + let spel = (4 + ((b * 8) >> s)) << 4; + let row = ((2 * mv.row) >> s).clamp(to_top - spel, (to_bottom + spel - 16).max(to_top - spel)); + let col = ((2 * mv.col) >> s).clamp(to_left - spel, (to_right + spel - 16).max(to_left - spel)); + (row, col) +} + +/// Predict a `w`×`h` block whose top-left sample is at (`x16`, `y16`) (1/16 samples) into `out` +/// (stride `w`). +pub fn predict(r: &RefPlane, x16: i32, y16: i32, w: usize, h: usize, out: &mut [u8]) { + let (x0, y0) = (x16 >> 4, y16 >> 4); + let (fx, fy) = ((x16 & 15) as usize, (y16 & 15) as usize); + let (lx, ly) = (r.w as i32 - 1, r.h as i32 - 1); + let inside = x0 - 3 >= 0 && y0 - 3 >= 0 && x0 + w as i32 + 4 <= lx && y0 + h as i32 + 4 <= ly; + let at = |x: i32, y: i32| -> i32 { + if inside { r.data[y as usize * r.stride + x as usize] as i32 } else { r.data[y.clamp(0, ly) as usize * r.stride + x.clamp(0, lx) as usize] as i32 } + }; + let fh = &SUBPEL_REGULAR[fx]; + let fv = &SUBPEL_REGULAR[fy]; + if fx == 0 && fy == 0 { + for i in 0..h { + for j in 0..w { + out[i * w + j] = at(x0 + j as i32, y0 + i as i32) as u8; + } + } + return; + } + // Horizontal pass over h + 7 rows (from y0 - 3), then vertical. + let mut tmp = [0u8; 23 * 16]; + for i in 0..h + 7 { + let y = y0 - 3 + i as i32; + for j in 0..w { + let x = x0 + j as i32 - 3; + let mut acc = 64; + for t in 0..8 { + acc += fh[t] * at(x + t as i32, y); + } + tmp[i * w + j] = (acc >> 7).clamp(0, 255) as u8; + } + } + for i in 0..h { + for j in 0..w { + let mut acc = 64; + for t in 0..8 { + acc += fv[t] * tmp[(i + t) * w + j] as i32; + } + out[i * w + j] = (acc >> 7).clamp(0, 255) as u8; + } + } +} + +/// The reference luma pre-interpolated at the 16 quarter-sample phases the encoder uses (motion +/// vectors are even in 1/8 units: fractions 0, 4, 8, 12 in 1/16), over the visible area plus a +/// border. Each phase plane holds exactly what [`predict`] returns at that phase (the decoder's +/// separable filter with edge clamping), so motion search reads predictions directly. +pub struct PhasePlanes { + border: usize, + stride: usize, + rows: usize, + planes: Vec>, +} + +impl PhasePlanes { + pub fn new(r: &RefPlane, border: usize) -> PhasePlanes { + const M: usize = 4; + let (pw, ph) = (r.w + 2 * border, r.h + 2 * border); + // Edge-replicated source with M more samples on every side for the filter taps. + let rs = pw + 2 * M; + let rh = ph + 2 * M; + let off = (border + M) as i32; + let mut raw = vec![0u8; rs * rh]; + for y in 0..rh { + let sy = (y as i32 - off).clamp(0, r.h as i32 - 1) as usize; + let src = &r.data[sy * r.stride..sy * r.stride + r.w]; + for x in 0..rs { + raw[y * rs + x] = src[(x as i32 - off).clamp(0, r.w as i32 - 1) as usize]; + } + } + // Horizontal phases over every raw row (inner columns). + let hpass = |fx: usize| -> Vec { + let f = &SUBPEL_REGULAR[fx]; + let mut out = vec![0u8; pw * rh]; + for y in 0..rh { + let row = &raw[y * rs..y * rs + rs]; + for x in 0..pw { + let s = &row[x + M - 3..x + M + 5]; + let acc: i32 = 64 + (0..8).map(|t| f[t] * s[t] as i32).sum::(); + out[y * pw + x] = (acc >> 7).clamp(0, 255) as u8; + } + } + out + }; + let unfiltered: Vec = (0..rh).flat_map(|y| raw[y * rs + M..y * rs + M + pw].iter().copied()).collect(); + let h: Vec> = (0..4).map(|k| if k == 0 { unfiltered.clone() } else { hpass(k * 4) }).collect(); + let mut planes = Vec::with_capacity(16); + for fy in 0..4 { + for hp in &h { + if fy == 0 { + planes.push(hp[M * pw..(M + ph) * pw].to_vec()); + continue; + } + let f = &SUBPEL_REGULAR[fy * 4]; + let mut out = vec![0u8; pw * ph]; + for y in 0..ph { + for x in 0..pw { + let mut acc = 64; + for t in 0..8 { + acc += f[t] * hp[(y + M + t - 3) * pw + x] as i32; + } + out[y * pw + x] = (acc >> 7).clamp(0, 255) as u8; + } + } + planes.push(out); + } + } + PhasePlanes { border, stride: pw, rows: ph, planes } + } + + /// The prediction of the `n`×`n` block at (`x16`, `y16`) (fractions multiples of 4) as rows of + /// a phase plane: (plane, offset of the first sample, stride). `None` beyond the border. + pub fn block(&self, x16: i32, y16: i32, n: usize) -> Option<(&[u8], usize, usize)> { + let (x0, y0) = ((x16 >> 4) + self.border as i32, (y16 >> 4) + self.border as i32); + let (fx, fy) = ((x16 & 15) as usize, (y16 & 15) as usize); + if fx % 4 != 0 || fy % 4 != 0 || x0 < 0 || y0 < 0 || x0 as usize + n > self.stride || y0 as usize + n > self.rows { + return None; + } + Some((&self.planes[(fy / 4) * 4 + fx / 4], y0 as usize * self.stride + x0 as usize, self.stride)) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn phase_planes_match_predict() { + let (w, h) = (37usize, 29usize); + let data: Vec = (0..w * h).map(|i| ((i * 7919) % 251) as u8).collect(); + let r = RefPlane { data: &data, stride: w, w, h }; + let pp = PhasePlanes::new(&r, 24); + let mut out = [0u8; 64]; + for y16 in (-20 * 16..(h as i32 + 10) * 16).step_by(4 * 7) { + for x16 in (-20 * 16..(w as i32 + 10) * 16).step_by(4 * 5) { + let Some((p, o, s)) = pp.block(x16, y16, 8) else { continue }; + predict(&r, x16, y16, 8, 8, &mut out); + for i in 0..8 { + assert_eq!(&p[o + i * s..o + i * s + 8], &out[i * 8..i * 8 + 8], "({x16}, {y16}) row {i}"); + } + } + } + } + + use super::*; + + #[test] + fn whole_sample_and_linear_ramps() { + let (w, h) = (40usize, 30usize); + let data: Vec = (0..w * h).map(|i| ((i % w) * 4 + (i / w)) as u8).collect(); + let r = RefPlane { data: &data, stride: w, w, h }; + let mut out = [0u8; 64]; + predict(&r, 10 << 4, 8 << 4, 8, 8, &mut out); + assert_eq!(out[0], data[8 * w + 10]); + // Half a sample right on a linear ramp: exact midpoint. + predict(&r, (10 << 4) + 8, 8 << 4, 8, 8, &mut out); + assert_eq!(out[0], data[8 * w + 10] + 2); + // Off the left edge: clamped to column 0. + predict(&r, -(20 << 4), 0, 4, 4, &mut out); + assert_eq!(out[0], data[0]); + } +} diff --git a/crates/vp9enc/src/probs.rs b/crates/vp9enc/src/probs.rs new file mode 100644 index 00000000..8f55bbba --- /dev/null +++ b/crates/vp9enc/src/probs.rs @@ -0,0 +1,426 @@ +//! Probabilities: the frame's adaptable probability table (one flat array laid out in the order +//! the compressed header updates them), a symbol recorder that codes every boolean decision +//! against either a fixed probability or an entry of that table, bit-cost estimates, and the +//! per-frame forward updates (spec 6.3: `diff_update_prob` with `decode_term_subexp` / +//! `inv_remap_prob`, and `update_mv_prob`). +//! +//! Tiles are first recorded, then the encoder counts how often each table entry coded a 0 or a +//! 1, picks updates that pay for themselves, writes them in the compressed header and finally +//! entropy codes the recorded symbols with the updated probabilities. + +use crate::bool::BoolEncoder; +use crate::tables::*; + +/// Coefficient probabilities `[ptype][ref][band][ctx][3]` for 4×4 transforms (band 0 uses +/// contexts 0..3 only). +pub const COEF: usize = 0; +pub const SKIP: usize = 432; +pub const INTER_MODE: usize = SKIP + 3; +pub const IS_INTER: usize = INTER_MODE + 21; +pub const SINGLE_REF: usize = IS_INTER + 4; +pub const PARTITION: usize = SINGLE_REF + 10; +pub const MV_JOINT: usize = PARTITION + 48; +pub const MV_SIGN: usize = MV_JOINT + 3; +pub const MV_CLASS: usize = MV_SIGN + 2; +pub const MV_CLASS0_BIT: usize = MV_CLASS + 20; +pub const MV_BITS: usize = MV_CLASS0_BIT + 2; +pub const MV_CLASS0_FR: usize = MV_BITS + 20; +pub const MV_FR: usize = MV_CLASS0_FR + 12; +pub const NPROBS: usize = MV_FR + 6; + +/// Index of the first of the three coefficient probabilities of a context. +pub fn coef_index(ptype: usize, inter: bool, band: usize, ctx: usize) -> usize { + COEF + (((ptype * 2 + inter as usize) * 6 + band) * 6 + ctx) * 3 +} + +#[derive(Clone)] +pub struct Probs(pub [u8; NPROBS]); + +impl Probs { + /// The default probabilities (setup_past_independence / key frames). + pub fn defaults() -> Probs { + let mut p = [128u8; NPROBS]; + p[COEF..COEF + 432].copy_from_slice(&DEFAULT_COEF_PROBS); + p[SKIP..SKIP + 3].copy_from_slice(&DEFAULT_SKIP_PROB); + p[INTER_MODE..INTER_MODE + 21].copy_from_slice(&DEFAULT_INTER_MODE_PROBS); + p[IS_INTER..IS_INTER + 4].copy_from_slice(&DEFAULT_IS_INTER_PROB); + p[SINGLE_REF..SINGLE_REF + 10].copy_from_slice(&DEFAULT_SINGLE_REF_PROB); + p[PARTITION..PARTITION + 48].copy_from_slice(&DEFAULT_PARTITION_PROBS); + p[MV_JOINT..MV_JOINT + 3].copy_from_slice(&DEFAULT_MV_JOINT_PROBS); + p[MV_SIGN..MV_SIGN + 2].copy_from_slice(&DEFAULT_MV_SIGN_PROB); + p[MV_CLASS..MV_CLASS + 20].copy_from_slice(&DEFAULT_MV_CLASS_PROBS); + p[MV_CLASS0_BIT..MV_CLASS0_BIT + 2].copy_from_slice(&DEFAULT_MV_CLASS0_BIT_PROB); + p[MV_BITS..MV_BITS + 20].copy_from_slice(&DEFAULT_MV_BITS_PROB); + p[MV_CLASS0_FR..MV_CLASS0_FR + 12].copy_from_slice(&DEFAULT_MV_CLASS0_FR_PROBS); + p[MV_FR..MV_FR + 6].copy_from_slice(&DEFAULT_MV_FR_PROBS); + Probs(p) + } +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +pub fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +/// The probability a recorded symbol is coded with. +#[derive(Clone, Copy, Debug)] +pub enum P { + Fixed(u8), + /// An entry of the frame's [`Probs`]. + Ctx(u16), + /// Node `n` of the Pareto token tree derived from the coefficient probability at the index. + Pareto(u16, u8), +} + +#[derive(Clone, Copy)] +struct Sym { + bit: bool, + p: P, +} + +/// Records the boolean decisions of a tile. +#[derive(Default)] +pub struct Writer { + syms: Vec, +} + +impl Writer { + #[inline] + pub fn put(&mut self, bit: bool, p: P) { + self.syms.push(Sym { bit, p }); + } + #[inline] + pub fn fixed(&mut self, bit: bool, prob: u8) { + self.put(bit, P::Fixed(prob)); + } + #[inline] + pub fn ctx(&mut self, bit: bool, idx: usize) { + self.put(bit, P::Ctx(idx as u16)); + } + /// Write `value` with `tree` (leaves stored negated); node `i` uses `prob(i)`. + pub fn tree(&mut self, tree: &[i8], value: u8, prob: impl Fn(usize) -> P) { + for (node, bit) in tree_path(tree, value) { + self.put(bit, prob(node)); + } + } + + /// Add the 0/1 counts of every table entry this tile used. + pub fn count(&self, counts: &mut [[u32; 2]]) { + for s in &self.syms { + if let P::Ctx(i) = s.p { + counts[i as usize][s.bit as usize] += 1; + } + } + } + + /// Entropy code the recorded symbols. + pub fn emit(&self, probs: &Probs) -> Vec { + let mut bc = BoolEncoder::new(); + bc.write(false, 128); // marker bit + for s in &self.syms { + let p = match s.p { + P::Fixed(p) => p, + P::Ctx(i) => probs.0[i as usize], + P::Pareto(i, n) => pareto(probs.0[i as usize])[n as usize], + }; + bc.write(s.bit, p); + } + bc.finish() + } +} + +/// The (node, bit) path to `value` in a tree (node = tree index / 2). +pub fn tree_path(tree: &[i8], value: u8) -> impl Iterator { + struct Path { + steps: [(u8, bool); 16], + len: usize, + } + fn walk(tree: &[i8], node: usize, value: u8, out: &mut Path) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.steps[out.len] = ((node >> 1) as u8, b == 1); + out.len += 1; + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if walk(tree, t as usize, value, out) { + return true; + } + out.len -= 1; + } + false + } + let mut p = Path { steps: [(0, false); 16], len: 0 }; + walk(tree, 0, value, &mut p); + let Path { steps, len } = p; + steps.into_iter().take(len).map(|(n, b)| (n as usize, b)) +} + +// ---------------------------------------------------------------- costs + +/// Cost in 1/256 bit of coding a 0 (`[p][0]`) or a 1 (`[p][1]`) with probability `p`. +pub fn cost_table() -> &'static [[u32; 2]; 256] { + use std::sync::OnceLock; + static T: OnceLock<[[u32; 2]; 256]> = OnceLock::new(); + T.get_or_init(|| { + let mut t = [[0u32; 2]; 256]; + for (p, e) in t.iter_mut().enumerate() { + let q = (p.max(1) as f64) / 256.0; + e[0] = (-q.log2() * 256.0).round() as u32; + e[1] = (-(1.0 - q).log2() * 256.0).round() as u32; + } + t + }) +} + +#[inline] +pub fn cost(bit: bool, p: u8) -> u32 { + cost_table()[p as usize][bit as usize] +} + +/// Cost of `value` in a tree with probabilities `probs` (node i → probs[i]). +pub fn tree_cost(tree: &[i8], probs: &[u8], value: u8) -> u32 { + tree_path(tree, value).map(|(n, b)| cost(b, probs[n])).sum() +} + +// ---------------------------------------------------------------- forward updates + +fn inv_recenter_nonneg(v: i32, m: i32) -> i32 { + if v > 2 * m { + v + } else if v & 1 != 0 { + m - ((v + 1) >> 1) + } else { + m + (v >> 1) + } +} + +/// `inv_remap_prob` (6.3.5). +fn inv_remap_prob(delta: usize, prob: u8) -> u8 { + let v = inv_map_table()[delta.min(254)] as i32; + let m = prob as i32 - 1; + let r = if (m << 1) <= 255 { 1 + inv_recenter_nonneg(v, m) } else { 255 - inv_recenter_nonneg(v, 255 - 1 - m) }; + r.clamp(1, 255) as u8 +} + +/// The `decode_term_subexp` value that moves `old` to `new`. +fn delta_for(old: u8, new: u8) -> Option { + (0..255).find(|&d| inv_remap_prob(d, old) == new) +} + +/// Bits of `decode_term_subexp` for `d`. +fn subexp_bits(d: usize) -> u32 { + match d { + 0..16 => 5, + 16..32 => 6, + 32..64 => 8, + 64..129 => 10, + _ => 11, + } +} + +fn write_subexp(bc: &mut BoolEncoder, d: usize) { + let d = d as u32; + if d < 16 { + bc.literal(0, 1); + bc.literal(d, 4); + } else if d < 32 { + bc.literal(2, 2); + bc.literal(d - 16, 4); + } else if d < 64 { + bc.literal(6, 3); + bc.literal(d - 32, 5); + } else { + bc.literal(7, 3); + if d < 129 { + bc.literal(d - 64, 7); + } else { + let v = (d + 1) >> 1; + bc.literal(v, 7); + bc.literal((d + 1) & 1, 1); + } + } +} + +fn counts_cost(c: [u32; 2], p: u8) -> u64 { + c[0] as u64 * cost(false, p) as u64 + c[1] as u64 * cost(true, p) as u64 +} + +/// Whether table entry `i` is an MV probability (7-bit `update_mv_prob` rather than +/// `diff_update_prob`). +fn is_mv(i: usize) -> bool { + i >= MV_JOINT +} + +/// Choose updated probabilities: an entry changes when the bits saved on this frame's symbols +/// exceed the cost of signalling the change. `inter`: the inter-frame tables are updatable. +pub fn plan_updates(old: &Probs, counts: &[[u32; 2]], inter: bool) -> Probs { + let mut new = old.clone(); + let flag = cost(true, 252) as i64 - cost(false, 252) as i64; + let limit = if inter { NPROBS } else { INTER_MODE }; + for i in 0..limit { + let c = counts[i]; + let n = c[0] + c[1]; + if n == 0 { + continue; + } + let o = old.0[i]; + let ideal = ((c[0] as f64 * 256.0 / n as f64).round() as i32).clamp(1, 255) as u8; + let cands: Vec = if is_mv(i) { vec![ideal | 1, ideal.saturating_sub(1).max(1) | 1] } else { vec![ideal] }; + let base = counts_cost(c, o) as i64; + let mut best = (0i64, o); + for p in cands { + if p == o { + continue; + } + let signal = if is_mv(i) { + 7 * 256 + } else { + match delta_for(o, p) { + Some(d) => subexp_bits(d) as i64 * 256, + None => continue, + } + }; + let gain = base - counts_cost(c, p) as i64 - signal - flag; + if gain > best.0 { + best = (gain, p); + } + } + new.0[i] = best.1; + } + new +} + +/// `diff_update_prob` for one entry. +fn diff_update(bc: &mut BoolEncoder, old: &Probs, new: &Probs, i: usize) { + let (o, n) = (old.0[i], new.0[i]); + match (o != n).then(|| delta_for(o, n)).flatten() { + Some(d) => { + bc.write(true, 252); + write_subexp(bc, d); + } + None => bc.write(false, 252), + } +} + +/// `update_mv_prob` for one entry. +fn mv_update(bc: &mut BoolEncoder, old: &Probs, new: &Probs, i: usize) { + let n = new.0[i]; + if n != old.0[i] && n & 1 == 1 { + bc.write(true, 252); + bc.literal(n as u32 >> 1, 7); + } else { + bc.write(false, 252); + } +} + +/// The compressed header (6.3) of a frame coded with ONLY_4X4 transforms, single LAST +/// references and a fixed interpolation filter, moving the probabilities from `old` to `new`. +/// Entries `new` cannot signal are left at `old` (and must be coded with `old`): the caller +/// codes the tiles with the probabilities this returns. +pub fn compressed_header(old: &Probs, new: &Probs, lossless: bool, inter: bool) -> (Vec, Probs) { + let mut bc = BoolEncoder::new(); + bc.write(false, 128); + if !lossless { + bc.literal(0, 2); // tx_mode = ONLY_4X4 + } + // read_coef_probs: TX_4X4 only. + let coef_changed = (COEF..COEF + 432).any(|i| new.0[i] != old.0[i]); + bc.literal(coef_changed as u32, 1); + if coef_changed { + for i in 0..2 { + for j in 0..2 { + for k in 0..6 { + for l in 0..if k == 0 { 3 } else { 6 } { + for m in 0..3 { + diff_update(&mut bc, old, new, coef_index(i, j == 1, k, l) + m); + } + } + } + } + } + } + for i in 0..3 { + diff_update(&mut bc, old, new, SKIP + i); + } + if inter { + for i in 0..21 { + diff_update(&mut bc, old, new, INTER_MODE + i); + } + for i in 0..4 { + diff_update(&mut bc, old, new, IS_INTER + i); + } + // reference_mode is SINGLE_REFERENCE (no compound: equal sign biases). + for i in 0..10 { + diff_update(&mut bc, old, new, SINGLE_REF + i); + } + for _ in 0..36 { + bc.write(false, 252); // y mode probabilities (no intra blocks in inter frames) + } + for i in 0..48 { + diff_update(&mut bc, old, new, PARTITION + i); + } + for j in 0..3 { + mv_update(&mut bc, old, new, MV_JOINT + j); + } + for i in 0..2 { + mv_update(&mut bc, old, new, MV_SIGN + i); + for j in 0..10 { + mv_update(&mut bc, old, new, MV_CLASS + i * 10 + j); + } + mv_update(&mut bc, old, new, MV_CLASS0_BIT + i); + for j in 0..10 { + mv_update(&mut bc, old, new, MV_BITS + i * 10 + j); + } + } + for i in 0..2 { + for j in 0..6 { + mv_update(&mut bc, old, new, MV_CLASS0_FR + i * 6 + j); + } + for j in 0..3 { + mv_update(&mut bc, old, new, MV_FR + i * 3 + j); + } + } + } + // What the decoder ends up with. + let mut used = old.clone(); + for i in 0..NPROBS { + let ok = if is_mv(i) { new.0[i] & 1 == 1 } else { delta_for(old.0[i], new.0[i]).is_some() }; + if ok && (inter || i < INTER_MODE) { + used.0[i] = new.0[i]; + } + } + (bc.finish(), used) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remap_reaches_every_probability() { + for old in 1..=255u8 { + for new in 1..=255u8 { + if new != old { + assert!(delta_for(old, new).is_some(), "{old} -> {new}"); + } + } + } + } + + #[test] + fn inv_map_table_shape() { + let t = inv_map_table(); + assert_eq!(&t[..4], &[7, 20, 33, 46]); + assert_eq!(t[19], 254); + assert_eq!(&t[20..27], &[1, 2, 3, 4, 5, 6, 8]); + assert_eq!(&t[252..], &[252, 253, 253]); + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs index dc8027a6..ffb26641 100644 --- a/crates/vp9enc/src/tables.rs +++ b/crates/vp9enc/src/tables.rs @@ -128,3 +128,76 @@ pub const DEFAULT_COEF_PROBS: [u8; 432] = [ 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, ]; + +// ---------------------------------------------------------------- inter frames (spec section 10.5) + +/// `inv_map_table` (used by `inv_remap_prob`): the 20 values 7 + 13k first, then every other value +/// 1..=253 in increasing order, then 253 again. +pub fn inv_map_table() -> &'static [u8; 255] { + use std::sync::OnceLock; + static T: OnceLock<[u8; 255]> = OnceLock::new(); + T.get_or_init(|| { + let mut t = [253u8; 255]; + let mut n = 0; + for k in 0..20 { + t[n] = 7 + 13 * k; + n += 1; + } + for v in 1..=253u8 { + if v < 7 || (v - 7) % 13 != 0 { + t[n] = v; + n += 1; + } + } + t + }) +} + +/// The regular 8-tap sub-sample interpolation filters (`subpel_filters[EIGHTTAP]`), 16 positions. +pub const SUBPEL_REGULAR: [[i32; 8]; 16] = [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 1, -5, 126, 8, -3, 1, 0], + [-1, 3, -10, 122, 18, -6, 2, 0], + [-1, 4, -13, 118, 27, -9, 3, -1], + [-1, 4, -16, 112, 37, -11, 4, -1], + [-1, 5, -18, 105, 48, -14, 4, -1], + [-1, 5, -19, 97, 58, -16, 5, -1], + [-1, 6, -19, 88, 68, -18, 5, -1], + [-1, 6, -19, 78, 78, -19, 6, -1], + [-1, 5, -18, 68, 88, -19, 6, -1], + [-1, 5, -16, 58, 97, -19, 5, -1], + [-1, 4, -14, 48, 105, -18, 5, -1], + [-1, 4, -11, 37, 112, -16, 4, -1], + [-1, 3, -9, 27, 118, -13, 4, -1], + [0, 2, -6, 18, 122, -10, 3, -1], + [0, 1, -3, 8, 126, -5, 1, 0], +]; + +/// `mv_ref_blocks` (row, column offsets) for BLOCK_8X8 and BLOCK_16X16. +pub const MV_REF_8X8: [(i32, i32); 8] = [(-1, 0), (0, -1), (-1, -1), (-2, 0), (0, -2), (-2, -1), (-1, -2), (-2, -2)]; +pub const MV_REF_16X16: [(i32, i32); 8] = [(-1, 0), (0, -1), (-1, 1), (1, -1), (-1, -1), (-3, 0), (0, -3), (-3, -3)]; + +/// `mode_2_counter` indexed by y mode (intra modes 0..=9, then NEARESTMV, NEARMV, ZEROMV, NEWMV). +pub const MODE_2_COUNTER: [u8; 14] = [9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 3, 1]; + +/// `counter_to_context`; INVALID_CASE entries are 9. +pub const COUNTER_TO_CONTEXT: [u8; 19] = [2, 3, 4, 1, 3, 9, 0, 9, 9, 5, 5, 9, 5, 9, 9, 9, 9, 9, 6]; + +pub const DEFAULT_PARTITION_PROBS: [u8; 48] = [ + 199, 122, 141, 147, 63, 159, 148, 133, 118, 121, 104, 114, 174, 73, 87, 92, 41, 83, 82, 99, 50, 53, 39, 39, 177, 58, 59, 68, 26, 63, 52, 79, 25, 17, 14, + 12, 222, 34, 30, 72, 16, 44, 58, 32, 12, 10, 7, 6, +]; + +pub const DEFAULT_IS_INTER_PROB: [u8; 4] = [9, 102, 187, 225]; + +pub const DEFAULT_SINGLE_REF_PROB: [u8; 10] = [33, 16, 77, 74, 142, 142, 172, 170, 238, 247]; + +pub const DEFAULT_INTER_MODE_PROBS: [u8; 21] = [2, 173, 34, 7, 145, 85, 7, 166, 63, 7, 94, 66, 8, 64, 46, 17, 81, 31, 25, 29, 30]; + +pub const DEFAULT_MV_JOINT_PROBS: [u8; 3] = [32, 64, 96]; +pub const DEFAULT_MV_SIGN_PROB: [u8; 2] = [128, 128]; +pub const DEFAULT_MV_CLASS_PROBS: [u8; 20] = [224, 144, 192, 168, 192, 176, 192, 198, 198, 245, 216, 128, 176, 160, 176, 176, 192, 198, 198, 208]; +pub const DEFAULT_MV_CLASS0_BIT_PROB: [u8; 2] = [216, 208]; +pub const DEFAULT_MV_BITS_PROB: [u8; 20] = [136, 140, 148, 160, 176, 192, 224, 234, 234, 240, 136, 140, 148, 160, 176, 192, 224, 234, 234, 240]; +pub const DEFAULT_MV_CLASS0_FR_PROBS: [u8; 12] = [128, 128, 64, 96, 112, 64, 128, 128, 64, 96, 112, 64]; +pub const DEFAULT_MV_FR_PROBS: [u8; 6] = [64, 96, 64, 64, 96, 64]; diff --git a/crates/vp9enc/src/tile.rs b/crates/vp9enc/src/tile.rs new file mode 100644 index 00000000..fe3a1a49 --- /dev/null +++ b/crates/vp9enc/src/tile.rs @@ -0,0 +1,894 @@ +//! Tile coding: partitions, intra blocks (key frames) and inter blocks (motion search, mode +//! decision, motion vector prediction, residual), recorded as symbols (see [`crate::probs`]). +//! +//! Mode info, motion vector candidate lists and contexts follow the decoding process of the +//! VP9 specification (6.4, 6.4.x `find_mv_refs` / `find_best_ref_mvs`) for the subset of the +//! syntax this encoder emits: 8×8 and 16×16 blocks, single LAST references, no +//! `UsePrevFrameMvs` (inter frames are error resilient), no high-precision vectors. + +use crate::mc::{Mv, PhasePlanes, RefPlane, clamp_for_plane, predict}; +use crate::probs::{self, P, Writer, cost, tree_cost}; +use crate::tables::*; +use crate::transform::{ADST_ADST, ADST_DCT, DCT_ADST, DCT_DCT, Quant}; +use crate::{BLOCK_8X8, BLOCK_16X16, Mi, Plane}; + +pub const DC_PRED: u8 = 0; +pub const V_PRED: u8 = 1; +pub const H_PRED: u8 = 2; +pub const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; +pub const NEARESTMV: u8 = 10; +pub const NEARMV: u8 = 11; +pub const ZEROMV: u8 = 12; +pub const NEWMV: u8 = 13; + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; +/// Leaves are `mode - NEARESTMV`. +const INTER_MODE_TREE: [i8; 6] = [-2, 2, 0, 4, -1, -3]; +const MV_JOINT_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const MV_CLASS_TREE: [i8; 20] = [0, 2, -1, 4, 6, 8, -2, -3, 10, 12, -4, -5, -6, 14, 16, 18, -7, -8, -9, -10]; +const MV_FR_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// mode2txfm_map for the intra modes used here (inter blocks use DCT_DCT). +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +fn eob_of(levels: &[i32; 16], tx_type: u8) -> usize { + let sc = scan(tx_type); + (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1) +} + +/// Rough coefficient bit cost (1/256 bit) for rate-distortion decisions. +fn levels_cost(levels: &[i32; 16], eob: usize) -> u32 { + if eob == 0 { + return 0; + } + let mut bits = 256; + for &l in levels { + let a = l.unsigned_abs(); + bits += if a == 0 { 160 } else { 640 + 384 * (32 - a.leading_zeros()) }; + } + bits +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict_intra(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- motion vector coding + +/// `mv_class` of an offset (|component| - 1). +fn mv_class(offset: u32) -> u32 { + if offset < 16 { 0 } else { (31 - offset.leading_zeros()) - 3 } +} + +fn mv_component_cost(i: usize, v: i32) -> u32 { + let d = &probs_defaults().0; + let offset = v.unsigned_abs() - 1; + let c = mv_class(offset); + let mut bits = cost(v < 0, d[probs::MV_SIGN + i]); + bits += tree_cost(&MV_CLASS_TREE, &d[probs::MV_CLASS + i * 10..], c as u8); + let fr = ((offset >> 1) & 3) as u8; + if c == 0 { + let b = offset >> 3; + bits += cost(b == 1, d[probs::MV_CLASS0_BIT + i]); + bits += tree_cost(&MV_FR_TREE, &d[probs::MV_CLASS0_FR + i * 6 + b as usize * 3..], fr); + } else { + let dd = (offset - (1 << (c + 3))) >> 3; + for b in 0..c { + bits += cost((dd >> b) & 1 == 1, d[probs::MV_BITS + i * 10 + b as usize]); + } + bits += tree_cost(&MV_FR_TREE, &d[probs::MV_FR + i * 3..], fr); + } + bits +} + +const MV_TABLE: i32 = 8192; + +/// Component costs for |v| ≤ MV_TABLE (index v + MV_TABLE) and the four joint costs. +fn mv_tables() -> &'static ([Vec; 2], [u32; 4]) { + use std::sync::OnceLock; + static T: OnceLock<([Vec; 2], [u32; 4])> = OnceLock::new(); + T.get_or_init(|| { + let comp = std::array::from_fn(|i| (-MV_TABLE..=MV_TABLE).map(|v| if v == 0 { 0 } else { mv_component_cost(i, v) }).collect()); + let d = &probs_defaults().0; + let joints = std::array::from_fn(|j| tree_cost(&MV_JOINT_TREE, &d[probs::MV_JOINT..], j as u8)); + (comp, joints) + }) +} + +/// Bits (1/256) of a NEWMV difference with the default probabilities. +pub fn mv_cost(diff: Mv) -> u32 { + let (comp, joints) = mv_tables(); + let joint = (diff.row != 0) as usize * 2 + (diff.col != 0) as usize; + let c = |i: usize, v: i32| if v.abs() <= MV_TABLE { comp[i][(v + MV_TABLE) as usize] } else { mv_component_cost(i, v) }; + joints[joint] + if diff.row != 0 { c(0, diff.row) } else { 0 } + if diff.col != 0 { c(1, diff.col) } else { 0 } +} + +fn probs_defaults() -> &'static probs::Probs { + use std::sync::OnceLock; + static D: OnceLock = OnceLock::new(); + D.get_or_init(probs::Probs::defaults) +} + +// ---------------------------------------------------------------- the tile + +/// Per-frame coding parameters shared by the tiles. +pub struct FrameParams<'a> { + pub key: bool, + pub quant: Quant, + pub mi_cols: usize, + pub mi_rows: usize, + /// Reference planes (inter frames). + pub refs: Option<[RefPlane<'a>; 3]>, + /// The reference luma at every quarter-sample phase (motion search). + pub phases: Option, + /// Encoder-side motion hints from the previous frame (by MI position). + pub prev_mvs: Option<&'a [Mv]>, + /// Lagrangian multipliers: per bit, in SAD and in squared-error units. + pub lambda_sad: f64, + pub lambda_sse: f64, +} + +pub struct Tile<'a> { + pub f: &'a FrameParams<'a>, + pub w: Writer, + pub planes: &'a mut [Plane; 3], + pub mi: &'a mut [Mi], + pub col_start: usize, + pub col_end: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// Motion vectors of the four 8×8 blocks evaluated for the current 16×16 area. + pending: [(usize, usize, Mv); 4], +} + +impl<'a> Tile<'a> { + pub fn new(f: &'a FrameParams<'a>, planes: &'a mut [Plane; 3], mi: &'a mut [Mi], col_start: usize, col_end: usize) -> Tile<'a> { + let n = f.mi_cols * 2 + 16; + Tile { + f, + w: Writer::default(), + planes, + mi, + col_start, + col_end, + above_nz: [vec![0; n], vec![0; n], vec![0; n]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; f.mi_cols + 8], + left_part: vec![0; 8], + pending: [(usize::MAX, 0, Mv::ZERO); 4], + } + } + + /// Code the tile's superblocks. + pub fn code(&mut self) { + for r in (0..self.f.mi_rows).step_by(8) { + for p in 0..3 { + self.left_nz[p].fill(0); + } + self.left_part.fill(0); + for c in (self.col_start..self.col_end).step_by(8) { + self.partition(r, c, 12); + } + } + } + + fn mi_at(&self, r: usize, c: usize) -> Mi { + self.mi[r * self.f.mi_cols + c] + } + + fn set_mi(&mut self, r: usize, c: usize, n8: usize, m: Mi) { + for dr in 0..n8 { + for dc in 0..n8 { + if r + dr < self.f.mi_rows && c + dc < self.f.mi_cols { + self.mi[(r + dr) * self.f.mi_cols + c + dc] = m; + } + } + } + } + + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + let (mi_rows, mi_cols) = (self.f.mi_rows, self.f.mi_cols); + if r >= mi_rows || c >= mi_cols { + return; + } + let num8 = [1usize, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8][bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < mi_rows; + let has_cols = (c + half) < mi_cols; + let bsl = [0usize, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3][bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let ctx = bsl * 4 + (((left >> boffset) & 1) as usize) * 2 + ((above >> boffset) & 1) as usize; + let key = self.f.key; + let prob = |i: usize| if key { P::Fixed(KF_PARTITION_PROBS[ctx * 3 + i]) } else { P::Ctx((probs::PARTITION + ctx * 3 + i) as u16) }; + if bsize == BLOCK_8X8 { + self.w.put(false, prob(0)); // PARTITION_NONE + if key { + self.intra_block(r, c); + } else { + let mv = self.pending.iter().find(|p| p.0 == r && p.1 == c).map(|p| p.2); + let mv = mv.unwrap_or_else(|| self.search(r, c, BLOCK_8X8).0); + self.inter_block(r, c, BLOCK_8X8, mv); + } + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + if !key && bsize == BLOCK_16X16 && has_rows && has_cols { + if let Some(mv) = self.decide16(r, c) { + self.w.put(false, prob(0)); // PARTITION_NONE + self.inter_block(r, c, BLOCK_16X16, mv); + for i in 0..2 { + self.above_part[c + i] = 15 >> 2; + self.left_part[(r & 7) + i] = 15 >> 2; + } + return; + } + } else if !key && bsize == BLOCK_16X16 { + self.pending = [(usize::MAX, 0, Mv::ZERO); 4]; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.w.tree(&PARTITION_TREE, 3, prob); + } else if has_cols { + self.w.put(true, prob(1)); + } else if has_rows { + self.w.put(true, prob(2)); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + // ------------------------------------------------------------ intra (key frames) + + fn intra_block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + let y_mode = self.choose_intra(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.intra_cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (neighbours never skip in key frames: context 0); tx_size implied (4×4). + self.w.ctx(false, probs::SKIP); + let above_mode = if avail_u { self.mi_at(r - 1, c).mode } else { DC_PRED }; + let left_mode = if avail_l { self.mi_at(r, c - 1).mode } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.w.tree(&INTRA_MODE_TREE, y_mode, |i| P::Fixed(KF_Y_MODE_PROBS[o + i])); + let o = y_mode as usize * 9; + self.w.tree(&INTRA_MODE_TREE, uv_mode, |i| P::Fixed(KF_UV_MODE_PROBS[o + i])); + self.set_mi(r, c, 1, Mi { inter: false, mode: y_mode, mv: Mv::ZERO, skip: false, size: BLOCK_8X8 }); + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.f.quant.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.intra_tx(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.intra_tx(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + #[allow(clippy::too_many_arguments)] + fn intra_cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict_intra(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose_intra(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.intra_cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn intra_tx(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.f.mi_cols * 8) >> ss; + let max_y = (self.f.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict_intra(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let q = Quant { round_dc: 0.5, round_ac: 0.38, ..self.f.quant }; + let levels = q.levels(&res, tx_type); + let eob = eob_of(&levels, tx_type); + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, false, tx_type, &levels, eob, ctx0); + let resid = q.residual(&levels, tx_type, eob); + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, inter: bool, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = probs::coef_index(ptype, inter, band, ctx); + if check_eob { + let more = c < eob; + self.w.ctx(more, o); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.w.ctx(false, o + 1); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.w.ctx(true, o + 1); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.w.ctx(false, o + 2); + } else { + self.w.ctx(true, o + 2); + let path: &[(u8, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.w.put(b, P::Pareto((o + 2) as u16, n)); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.w.fixed((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.w.fixed(levels[pos] < 0, 128); + c += 1; + } + } + + // ------------------------------------------------------------ inter + + fn is_inside(&self, r: i32, c: i32) -> bool { + r >= 0 && (r as usize) < self.f.mi_rows && c >= self.col_start as i32 && (c as usize) < self.col_end + } + + /// `find_mv_refs` + `find_best_ref_mvs` for LAST_FRAME: (NearestMv, NearMv, mode context). + fn mv_refs(&self, r: usize, c: usize, bsize: u8) -> (Mv, Mv, usize) { + let search = if bsize == BLOCK_16X16 { &MV_REF_16X16 } else { &MV_REF_8X8 }; + let mut list = [Mv::ZERO; 2]; + let mut count = 0; + let add = |mv: Mv, list: &mut [Mv; 2], count: &mut usize| { + if *count >= 2 || (*count > 0 && mv == list[0]) { + return; + } + list[*count] = mv; + *count += 1; + }; + let mut counter = 0usize; + for (i, &(dr, dc)) in search.iter().enumerate() { + let (cr, cc) = (r as i32 + dr, c as i32 + dc); + if !self.is_inside(cr, cc) { + continue; + } + let m = self.mi_at(cr as usize, cc as usize); + if i < 2 { + counter += MODE_2_COUNTER[m.mode as usize] as usize; + } + if m.inter { + add(m.mv, &mut list, &mut count); + } + } + let ctx = COUNTER_TO_CONTEXT[counter.min(18)] as usize; + let n8 = if bsize == BLOCK_16X16 { 2 } else { 1 }; + let clamp = |mv: Mv, border: i32| -> Mv { + let (r, c, b) = (r as i32, c as i32, n8); + let to_top = -(r * 64); + let to_bottom = (self.f.mi_rows as i32 - b - r) * 64; + let to_left = -(c * 64); + let to_right = (self.f.mi_cols as i32 - b - c) * 64; + Mv::new( + mv.row.clamp(to_top - border, (to_bottom + border).max(to_top - border)), + mv.col.clamp(to_left - border, (to_right + border).max(to_left - border)), + ) + }; + for mv in list.iter_mut() { + *mv = clamp(*mv, 128); + // No high precision: odd components move toward zero. + let even = |v: i32| if v & 1 != 0 { v + if v > 0 { -1 } else { 1 } } else { v }; + *mv = clamp(Mv::new(even(mv.row), even(mv.col)), (160 - 4) << 3); + } + (list[0], list[1], ctx) + } + + /// The allowed motion vector range of a block (no decoder clamping inside it). + fn mv_range(&self, r: usize, c: usize, n: i32) -> (i32, i32, i32, i32) { + let n8 = n / 8; + let (r, c) = (r as i32, c as i32); + let lo_c = -(c * 64) - (4 + n) * 8; + let hi_c = (self.f.mi_cols as i32 - n8 - c) * 64 + (4 + n) * 8 - 8; + let lo_r = -(r * 64) - (4 + n) * 8; + let hi_r = (self.f.mi_rows as i32 - n8 - r) * 64 + (4 + n) * 8 - 8; + let lim = 8 * 1000; + (lo_r.max(-lim), hi_r.min(lim), lo_c.max(-lim), hi_c.min(lim)) + } + + /// SAD of the luma prediction at `mv` against the source. + fn sad(&self, r: usize, c: usize, n: usize, mv: Mv) -> u32 { + let p = &self.planes[0]; + let (x16, y16) = (((c * 8) << 4) as i32 + 2 * mv.col, ((r * 8) << 4) as i32 + 2 * mv.row); + let mut s = 0u32; + if let Some((pl, o, st)) = self.f.phases.as_ref().and_then(|ph| ph.block(x16, y16, n)) { + for i in 0..n { + let a = &p.src[(r * 8 + i) * p.w + c * 8..][..n]; + let b = &pl[o + i * st..o + i * st + n]; + s += a.iter().zip(b).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::(); + } + return s; + } + let refs = self.f.refs.as_ref().expect("reference"); + let mut pred = [0u8; 256]; + predict(&refs[0], x16, y16, n, n, &mut pred); + for i in 0..n { + let a = &p.src[(r * 8 + i) * p.w + c * 8..][..n]; + s += a.iter().zip(&pred[i * n..i * n + n]).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::(); + } + s + } + + /// Motion search for the block at (`r`, `c`): the best vector and its cost + /// (SAD + λ·bits). + fn search(&self, r: usize, c: usize, bsize: u8) -> (Mv, f64) { + let n = if bsize == BLOCK_16X16 { 16 } else { 8 }; + let (nearest, near, mctx) = self.mv_refs(r, c, bsize); + let (lo_r, hi_r, lo_c, hi_c) = self.mv_range(r, c, n as i32); + let inside = |mv: Mv| mv.row >= lo_r && mv.row <= hi_r && mv.col >= lo_c && mv.col <= hi_c; + let lambda = self.f.lambda_sad / 256.0; + let d = &probs_defaults().0; + let mb: [u32; 4] = std::array::from_fn(|m| tree_cost(&INTER_MODE_TREE, &d[probs::INTER_MODE + mctx * 3..], m as u8)); + let mode_bits = |m: u8| mb[(m - NEARESTMV) as usize]; + let cost = |mv: Mv| -> f64 { + let bits = if mv == nearest { + mode_bits(NEARESTMV) + } else if mv == Mv::ZERO { + mode_bits(ZEROMV) + } else if mv == near { + mode_bits(NEARMV) + } else { + mode_bits(NEWMV) + mv_cost(mv.minus(nearest)) + }; + self.sad(r, c, n, mv) as f64 + lambda * bits as f64 + }; + let mut best = (Mv::ZERO, cost(Mv::ZERO)); + let prev = self.f.prev_mvs.map(|p| p[r * self.f.mi_cols + c]); + for cand in [Some(nearest), Some(near), prev].into_iter().flatten() { + if cand != Mv::ZERO && inside(cand) { + let j = cost(cand); + if j < best.1 { + best = (cand, j); + } + } + } + // A perfect match needs no search. + let good_enough = lambda * 256.0 * 0.5; + if best.1 > good_enough { + // Whole-sample search from the best candidate: a coarse cross, then a small diamond. + let round8 = |v: i32| (v + 4).div_euclid(8) * 8; + let mut cur = Mv::new(round8(best.0.row), round8(best.0.col)); + let mut cur_j = if inside(cur) { cost(cur) } else { f64::MAX }; + if cur_j < best.1 { + best = (cur, cur_j); + } else { + cur = best.0; + cur_j = best.1; + } + for (step, max_iters) in [(16, 1), (8, 2), (4, 3), (2, 4), (1, 8)] { + let s = step * 8; + for _ in 0..max_iters { + let mut moved = false; + let center = cur; + for (dr, dc) in [(-s, 0), (s, 0), (0, -s), (0, s)] { + let cand = Mv::new(center.row + dr, center.col + dc); + if !inside(cand) { + continue; + } + let j = cost(cand); + if j < cur_j { + cur = cand; + cur_j = j; + moved = true; + } + } + if !moved { + break; + } + } + } + if cur_j < best.1 { + best = (cur, cur_j); + } + } + // Sub-sample refinement: half then quarter samples (1/8 units, even: no high precision). + for s in [4, 2] { + let center = best.0; + for (dr, dc) in [(-s, 0), (s, 0), (0, -s), (0, s), (-s, -s), (-s, s), (s, -s), (s, s)] { + let cand = Mv::new(center.row + dr, center.col + dc); + if !inside(cand) { + continue; + } + let j = cost(cand); + if j < best.1 { + best = (cand, j); + } + } + } + best + } + + /// Choose between one 16×16 block and four 8×8 blocks at (`r`, `c`): `Some(mv)` for 16×16. + fn decide16(&mut self, r: usize, c: usize) -> Option { + let (mv16, j16) = self.search(r, c, BLOCK_16X16); + let lambda = self.f.lambda_sad; + // A near-perfect 16×16 prediction needs no split. + if j16 < 2.0 * 256.0 / 256.0 * 16.0 { + return Some(mv16); + } + let mut j8 = lambda * 4.0; // partition split + three more blocks of mode info (≈ 4 bits) + for (k, (dr, dc)) in [(0, 0), (0, 1), (1, 0), (1, 1)].into_iter().enumerate() { + let (rr, cc) = (r + dr, c + dc); + let (mv, j) = self.search(rr, cc, BLOCK_8X8); + j8 += j; + self.pending[k] = (rr, cc, mv); + // Tentative mode info so the next 8×8 blocks see this one as a candidate. + self.set_mi(rr, cc, 1, Mi { inter: true, mode: NEWMV, mv, skip: false, size: BLOCK_8X8 }); + } + if j16 <= j8 { + self.pending = [(usize::MAX, 0, Mv::ZERO); 4]; + Some(mv16) + } else { + None + } + } + + /// Code an inter block with motion vector `mv`. + fn inter_block(&mut self, r: usize, c: usize, bsize: u8, mv: Mv) { + let (mi_rows, mi_cols) = (self.f.mi_rows, self.f.mi_cols); + let n8 = if bsize == BLOCK_16X16 { 2 } else { 1 }; + let n = 8 * n8; + let (nearest, near, mctx) = self.mv_refs(r, c, bsize); + let d = &probs_defaults().0; + let mode_bits = |m: u8| tree_cost(&INTER_MODE_TREE, &d[probs::INTER_MODE + mctx * 3..], m - NEARESTMV); + let mut mode = (NEWMV, mode_bits(NEWMV) + mv_cost(mv.minus(nearest))); + for (m, ok) in [(NEARESTMV, mv == nearest), (NEARMV, mv == near), (ZEROMV, mv == Mv::ZERO)] { + if ok && mode_bits(m) < mode.1 { + mode = (m, mode_bits(m)); + } + } + let mode = mode.0; + // Prediction of every plane (written straight into the reconstruction). + let refs = self.f.refs.as_ref().expect("reference"); + for plane in 0..3 { + let ch = plane > 0; + let (row16, col16) = clamp_for_plane(mv, r, c, n8, mi_rows, mi_cols, ch); + let bn = if ch { n / 2 } else { n }; + let (bx, by) = if ch { (c * 4, r * 4) } else { (c * 8, r * 8) }; + let mut pred = [0u8; 256]; + predict(&refs[plane], ((bx as i32) << 4) + col16, ((by as i32) << 4) + row16, bn, bn, &mut pred); + let p = &mut self.planes[plane]; + for i in 0..bn { + p.rec[(by + i) * p.w + bx..][..bn].copy_from_slice(&pred[i * bn..i * bn + bn]); + } + } + // Residual: quantise every 4×4, dropping blocks whose coefficients don't pay for themselves. + let q = Quant { round_dc: 0.5, round_ac: 0.42, ..self.f.quant }; + // Weight of the rate term when dropping a transform block's coefficients (low: keep quality on par + // with key frames at the same q index). + let zl = 0.07; + let tx = DCT_DCT; + let mut blocks: Vec<(usize, usize, usize, [i32; 16], usize)> = Vec::with_capacity(24); + let mut any = false; + for plane in 0..3 { + let ch = plane > 0; + let bn = if ch { n / 2 } else { n }; + let (bx, by) = if ch { (c * 4, r * 4) } else { (c * 8, r * 8) }; + let p = &self.planes[plane]; + for ty in (0..bn).step_by(4) { + for tx4 in (0..bn).step_by(4) { + let (x, y) = (bx + tx4, by + ty); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + let k = (y + i) * p.w + x + j; + res[i * 4 + j] = p.src[k] as i32 - p.rec[k] as i32; + } + } + let mut levels = q.levels(&res, tx); + let mut eob = eob_of(&levels, tx); + if eob > 0 && !q.lossless { + let back = q.residual(&levels, tx, eob); + let d_skip: i64 = res.iter().map(|v| (*v as i64) * (*v as i64)).sum(); + let d_code: i64 = res.iter().zip(&back).map(|(a, b)| ((a - b) as i64).pow(2)).sum(); + if d_skip as f64 <= d_code as f64 + zl * self.f.lambda_sse * levels_cost(&levels, eob) as f64 / 256.0 { + levels = [0; 16]; + eob = 0; + } + } + any |= eob > 0; + blocks.push((plane, x, y, levels, eob)); + } + } + } + let skip = !any; + // Mode info. + let avail_u = r > 0; + let avail_l = c > self.col_start; + let above = avail_u.then(|| self.mi_at(r - 1, c)); + let left = avail_l.then(|| self.mi_at(r, c - 1)); + let skip_ctx = above.is_some_and(|m| m.skip) as usize + left.is_some_and(|m| m.skip) as usize; + self.w.ctx(skip, probs::SKIP + skip_ctx); + // is_inter context: neighbours are inter (no intra blocks in inter frames). + let intra = |m: Option| m.is_some_and(|m| !m.inter); + let inter_ctx = match (above, left) { + (Some(_), Some(_)) => { + if intra(above) && intra(left) { + 3 + } else { + (intra(above) || intra(left)) as usize + } + } + (Some(_), None) => 2 * intra(above) as usize, + (None, Some(_)) => 2 * intra(left) as usize, + (None, None) => 0, + }; + self.w.ctx(true, probs::IS_INTER + inter_ctx); + // single_ref_p1 = 0 (LAST_FRAME); context for single-reference LAST neighbours. + let ref_ctx = match (above, left) { + (Some(a), Some(l)) => match (a.inter, l.inter) { + (false, false) => 2, + (false, true) => 4, + (true, false) => 4, + (true, true) => 4, + }, + (Some(m), None) | (None, Some(m)) => { + if m.inter { + 4 + } else { + 2 + } + } + (None, None) => 2, + }; + self.w.ctx(false, probs::SINGLE_REF + ref_ctx * 2); + self.w.tree(&INTER_MODE_TREE, mode - NEARESTMV, |i| P::Ctx((probs::INTER_MODE + mctx * 3 + i) as u16)); + if mode == NEWMV { + self.write_mv(mv.minus(nearest)); + } + self.set_mi(r, c, n8, Mi { inter: true, mode, mv, skip, size: bsize }); + // Tokens and reconstruction. + for (plane, x, y, levels, eob) in blocks { + let ss = (plane > 0) as usize; + let (x4, ly) = (x >> 2, (y >> 2) & (15 >> ss)); + if !skip { + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, true, tx, &levels, eob, ctx0); + if eob > 0 { + let resid = q.residual(&levels, tx, eob); + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + let k = (y + i) * p.w + x + j; + p.rec[k] = (p.rec[k] as i32 + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + } + } + let nz = (!skip && eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + } + + fn write_mv(&mut self, diff: Mv) { + let joint = (diff.row != 0) as u8 * 2 + (diff.col != 0) as u8; + self.w.tree(&MV_JOINT_TREE, joint, |i| P::Ctx((probs::MV_JOINT + i) as u16)); + if diff.row != 0 { + self.write_mv_component(0, diff.row); + } + if diff.col != 0 { + self.write_mv_component(1, diff.col); + } + } + + fn write_mv_component(&mut self, i: usize, v: i32) { + debug_assert!(v != 0 && v & 1 == 0, "vectors are even without high precision"); + let offset = v.unsigned_abs() - 1; + let c = mv_class(offset); + self.w.ctx(v < 0, probs::MV_SIGN + i); + self.w.tree(&MV_CLASS_TREE, c as u8, |k| P::Ctx((probs::MV_CLASS + i * 10 + k) as u16)); + let fr = ((offset >> 1) & 3) as u8; + if c == 0 { + let b = (offset >> 3) as usize; + self.w.ctx(b == 1, probs::MV_CLASS0_BIT + i); + self.w.tree(&MV_FR_TREE, fr, |k| P::Ctx((probs::MV_CLASS0_FR + i * 6 + b * 3 + k) as u16)); + } else { + let dd = (offset - (1 << (c + 3))) >> 3; + for b in 0..c { + self.w.ctx((dd >> b) & 1 == 1, probs::MV_BITS + i * 10 + b as usize); + } + self.w.tree(&MV_FR_TREE, fr, |k| P::Ctx((probs::MV_FR + i * 3 + k) as u16)); + } + } +} diff --git a/crates/vp9enc/src/transform.rs b/crates/vp9enc/src/transform.rs new file mode 100644 index 00000000..e6a685d5 --- /dev/null +++ b/crates/vp9enc/src/transform.rs @@ -0,0 +1,249 @@ +//! 4×4 transforms: the decoder's integer inverse DCT / ADST (spec 8.7.1) and Walsh–Hadamard +//! transform (8.7.1.10), and their forward counterparts (the exact inverse matrices of the +//! decoder's linear transforms, so `inverse4(forward(res)) ≈ res`; the WHT inverse is exact). + +pub const DCT_DCT: u8 = 0; +pub const ADST_DCT: u8 = 1; +pub const DCT_ADST: u8 = 2; +pub const ADST_ADST: u8 = 3; + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +pub fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +pub fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +pub fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms. +pub fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +/// Quantisation settings of one block. +#[derive(Clone, Copy)] +pub struct Quant { + pub dcq: i32, + pub acq: i32, + pub lossless: bool, + /// Rounding offsets (fractions of a step) for DC and AC; smaller = wider dead zone. + pub round_dc: f64, + pub round_ac: f64, +} + +impl Quant { + /// Quantised levels (raster order) of a residual. + pub fn levels(&self, res: &[i32; 16], tx_type: u8) -> [i32; 16] { + if self.lossless { + return forward_wht(res); + } + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let (qs, rnd) = if k == 0 { (self.dcq, self.round_dc) } else { (self.acq, self.round_ac) }; + let l = (coef.abs() / qs as f64 + rnd).floor() as i32; + lv[k] = if coef < 0.0 { -l } else { l }; + } + lv + } + + /// The decoder's residual for `levels` (all zero when `eob` is 0). + pub fn residual(&self, levels: &[i32; 16], tx_type: u8, eob: usize) -> [i32; 16] { + if eob == 0 { + return [0; 16]; + } + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { self.dcq } else { self.acq }; + } + if self.lossless { inverse_wht(&deq) } else { inverse4(&deq, tx_type) } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/docs/agents.md b/docs/agents.md index 8689b9da..330f24a9 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -105,11 +105,18 @@ effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \ `track.mask` follows the pixels inside a mask and keys its Mask Path on every frame (vertices and tangents move with the fitted motion). `method` is `position`, `positionScale`, -`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew` -or `perspective`; `direction` is `forward|backward|frameForward|frameBackward` from the current +`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew`, +`perspective`, `faceOutline` (Face Tracking (Outline Only): the mask, drawn around a face, becomes +the face outline on every frame) or `faceDetailed` (Face Tracking (Detailed Features): also keys a +Face Track Points effect with one point per landmark — eyebrows, eyes, pupils, nose tip, +nostrils, mouth corners and lips, chin, jaw); `direction` is `forward|backward|frameForward|frameBackward` from the current time (or `start` / `end` in seconds). Like `track.analyze` it runs in the background in the app (progress in `track.status` under `mask`, `track.stop` cancels and keeps what was tracked) and is -one undo step; `"wait": true` blocks. +one undo step; `"wait": true` blocks. `track.extractFaceMeasurements {layer?}` (Extract & Copy Face +Measurements) keys a Face Measurements effect from the Face Track Points (head position, scale and +orientation X/Y/Z, eye openness, eyebrow distance from eye, mouth openness, width and offset), puts +those keys on the keyframe clipboard and returns them per frame plus a tab-separated `clipboard` +text. `mask.interpolate` (Window ▸ Mask Interpolation ▸ Apply) adds in-between Mask Path keys between each pair of selected Mask Path keys (or the existing keys at `times`, in seconds), giving both @@ -138,7 +145,11 @@ the app re-analyses automatically (headless: run `warp.analyze` again). Settings properties under the instance's groups, e.g. `effects/#1/stabilization/result` (0 Smooth Motion, 1 No Motion), `effects/#1/stabilization/smoothness`, `effects/#1/stabilization/method`, `effects/#1/borders/framing` (0 Stabilize Only … 3 Stabilize, Synthesize Edges), -`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. +`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. Method 3 +(Subspace Warp, the default) bends a mesh per frame to the smoothed feature trajectories, so +scenes with depth stabilise where one perspective transform cannot; +`effects/#1/advanced/rollingShutterRipple` (0 Automatic Reduction, 1 Enhanced Reduction: a finer, +softer mesh) tunes it. ```sh effectcraft-cli run shaky.ecproj track.warpStabilizer '{"layer":"#1","wait":true}' \ @@ -150,13 +161,17 @@ effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ### 3D Camera Tracker Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: -fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?, +undistort?, wait?}`) apply Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: "Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. `camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, `averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` -(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +(position, orientation, zoom), `groundPlane` and the solved `lensDistortion` (`k1`, `k2`, normalised by +half the frame diagonal) when Solve Lens Distortion (`effects/#1/advanced/lensDistortion`, also on +with Detailed Analysis) is on; `effects/#1/advanced/undistort` renders the footage undistorted so +the solved camera's 3D layers line up. `camera.points {time?}` lists the solved points visible now with their `id`, `comp` position, `depth`, `world` position and `error`. Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / @@ -220,6 +235,35 @@ In expressions, `text.sourceText.style` / `getStyleAt(i, t)` read styles and the (`setFontSize(v, start?, count?)`, `setFillColor`, `setText`, `setJustification`…) return a styled document. +### Scripts and ScriptUI windows + +Scripts that build ScriptUI windows publish them to the session; agents drive them like a user: + +1. `scriptui.list` → `[{window, title, kind: dialog|palette|window|panel, script, modal, size}]`. +2. `scriptui.get {"window": id}` → the control tree (`type`, `name`, `text`, `value`, `checked`, + `items`, `selection`, laid-out `bounds`, `handlers`…). +3. `scriptui.click {"widget": "ok"}` presses a button / toggles a checkbox / picks a radio button or + tab; `scriptui.set {"widget": "#4", "value": "Shot_"}` types into edit text, moves a slider or + picks a list item (index or text); `scriptui.close {"result": 2}` closes. Controls are addressed + by id, `#id`, `properties.name` or text; `window` can be omitted when one window is open. Each + returns the handler run's `{ok, output, error}`. + +A dialog's `show()` waits for the user: the `script.run` / `file.runScript` reply carries +`"waiting": true`, and the script continues (its final output arrives in the reply of the click +that closes the dialog). File ▸ Scripts: `file.scripts.list`, `file.runScript {"name": …}`, +`file.installScript` / `file.installScriptUIPanel {"path": …}`, `window.scriptPanel {"name": …}`. + +### History, puppet recording, plug-ins + +* `edit.history.list` lists every undo state as a tree (undoing then editing keeps the undone + states as a branch); `edit.history.goto {"index": n}` (or `id`, or `steps`) jumps to any of + them. MCP: the `history` tool. +* `puppet.recordPin {"layer": "#1", "pin": "Puppet Pin 1", "samples": [[t, x, y]…]}` records a + drag (t = seconds since it began, layer space) into Position keys at the comp frame rate from the + current time; `puppet.recordOptions {speed, smoothing, useDraftDeformation, showMesh}`. +* `effect.plugins.load {"path": "x.wasm"}` / `effect.plugins.list`: WebAssembly effect plug-ins + ([plugins.md](plugins.md)), then `effect.apply` by id like a built-in. + ## CLI Each invocation runs a headless engine with no window. It opens the demo project unless you pass diff --git a/docs/architecture.md b/docs/architecture.md index 7c341818..ed86d9b5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,9 +35,10 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | -| L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | -| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) whose edits run engine commands; AE match names | -| L4 | `host` | A fully wired `Session` (media, expressions, scripting, exporter) for the frontends | +| L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) | +| L4 | `engine` | `Session`: project, branching undo history, editor state, the command registry and menus, the ScriptUI window model (`scriptui`) | +| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) and ScriptUI (dialogs, palettes, dockable panels; script-host threads so dialogs block `show()`), whose edits run engine commands; AE match names | +| L4 | `host` | A fully wired `Session` (media, expressions, scripting, WebAssembly plug-ins, exporter) for the frontends | | L5 | `ui-egui` | The desktop interface: docking, panels, viewer, timeline, graph editor, dialogs, control channel | | L5 | `automation` | The MCP server, headless or bridged to the running app | | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | @@ -229,6 +230,13 @@ tracks them with pyramidal Lucas–Kanade and fits the chosen model (position RANSAC; the motion moves the Mask Path's vertices and tangents and is keyed per frame. **Mask Interpolation** (`mask.interpolate`) uses `effectcraft-path`'s smart interpolation (arc-length vertex insertion, shape-context matching, rigid in-betweens) to key in-between shapes. +The two **Face Tracking** methods (`effectcraft_track::face`) fit a face inside the mask instead: +a skin colour model from the first frame, the face outline along rays from the skin component's +centre (an area-moment ellipse gives centre, size and roll), facial features as the non-skin +components inside it, and an active-shape point distribution model trained on synthetic face +shapes (our own generator, no external weights) that fills and checks the landmarks. The outline +keys the Mask Path; Detailed Features keys a Face Track Points effect, and +`track.extractFaceMeasurements` derives a keyed Face Measurements effect (and copies its keys). **Warp Stabilizer** (`effects::warp_stab`, `engine::warp`): `warp.analyze` renders the layer's input to the effect (`Renderer::layer_input`: source, masks and the effects above it) for every @@ -237,7 +245,10 @@ translation / similarity / homography fits) as JSON in the effect's hidden Analy with a key of the layer's source, In/Out, start, stretch and Time Remap. `Session::edit` clears analyses whose key no longer matches and queues them; the desktop app re-analyses them in the background. Rendering derives the stabilization plan (smoothed camera path or No Motion, framing, -auto-scale; cached per analysis and settings) and warps each frame; Synthesize Edges fills the +auto-scale; cached per analysis and settings) and warps each frame. Subspace Warp also stores long +feature trajectories in the analysis, smooths them in a low-rank subspace (Liu et al. 2011) and +warps each frame with a content-preserving mesh (Liu et al. 2009) fitted to the smoothed +positions, rendered as a per-pixel mesh lookup; Synthesize Edges fills the borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. @@ -268,7 +279,10 @@ the essential matrix (normalised 8-point in RANSAC) or a planar homography decom resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod -pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +pans, chosen by Auto Detect when it explains the tracks as well; with Solve Lens Distortion (or +Detailed Analysis) radial distortion `k1, k2` is adjusted jointly in a final bundle adjustment on +the raw tracks and the solve repeated on tracks undistorted with that estimate; Undistort Footage +renders the input through the inverse model. Tracks and solve are stored as JSON in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is diff --git a/docs/effects.md b/docs/effects.md index 84735286..fd29541b 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 304 effects with 2446 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 306 effects with 2488 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 59 effects also run on the GPU compositor with identical results. -- **32**: 304 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 262 implemented in full, 42 partial (what is missing is listed). +- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 264 implemented in full, 42 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -302,7 +302,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Effect | Id | Params | GPU | 32 | Status | |---|---|---:|:-:|:-:|---| -| 3D Camera Tracker | `ec.perspective.cameratracker` | 10 | | 32 | Implemented | +| 3D Camera Tracker | `ec.perspective.cameratracker` | 12 | | 32 | Implemented | | 3D Glasses | `ec.perspective.3dglasses` | 8 | | 32 | Implemented | | Bevel Alpha | `ec.perspective.bevelalpha` | 4 | | 32 | Implemented | | Bevel Edges | `ec.perspective.beveledges` | 4 | | 32 | Implemented | @@ -416,6 +416,8 @@ Effects are addressed by id (`ec..`) or display name in commands | CC Overbrights | `ec.utility.ccoverbrights` | 2 | | 32 | Implemented | | Cineon Converter | `ec.utility.cineon` | 7 | | 32 | Implemented | | Color Profile Converter | `ec.utility.colorprofileconverter` | 7 | | 32 | Implemented | +| Face Measurements | `ec.utility.facemeasurements` | 14 | | 32 | Implemented | +| Face Track Points | `ec.utility.facetrackpoints` | 26 | | 32 | Implemented | | Grow Bounds | `ec.utility.growbounds` | 1 | | 32 | Implemented | | HDR Compander | `ec.utility.hdrcompander` | 3 | | 32 | Implemented | | HDR Highlight Compression | `ec.utility.hdrcompression` | 1 | | 32 | Implemented | diff --git a/docs/parity.md b/docs/parity.md index 1cb71322..dc49af2e 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -2,27 +2,42 @@ How close EffectCraft is to After Effects 2026, feature by feature, and how much work is left. -## Current status (audit at commit `fa26ad9`, 2 October 2026, late) +## Current status (audit at commit `978e8d7`, 3 October 2026) | Measure | Value | |---|---| -| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 93%** | -| Unweighted | ≈ 91% | -| P0 / P1 / P2 | ≈ 96% / 90% / 65% | -| Features done / partial / missing | 41 / 50 / 1 of 92 (missing: a plug-in API) | -| **Effects** | **298 of 298** After Effects 2026 effects exist (some still simplified) | -| Disabled menu entries left | 26 commands (31 menu items) | -| Remaining work | ≈ 50–55 agent-hours at the pace measured so far (≈ 150 by the original conservative audit scale) | -| **Wall-clock estimate** | **≈ 11–14 hours** with five agents in parallel; ≈ 6–8 hours for 100% of P0 + P1 | +| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 94%** counting every partial feature as half done; ≈ 98% using per-feature fractions | +| Unweighted | ≈ 94% (half-credit) / 98% (fractions) | +| P0 / P1 / P2 | ≈ 96% / 92% / 85% (half-credit); 99% / 97% / 96% (fractions) | +| Features done / partial / missing | 80 / 12 / 0 of 92 | +| **Effects** | **298 of 298** After Effects 2026 effects exist; 42 still simplified ([effects.md](effects.md)); 58 run on the GPU | +| Disabled menu entries left | 0 (the stub list is empty) | +| Remaining work | ≈ 19–22 agent-hours at the pace measured so far (≈ 58 on the conservative audit scale) | +| **Wall-clock estimate** | **≈ 4–5 hours** with five agents in parallel; ≈ 3 hours for 100% of P0 + P1 | + +Partial features: effect implementations (42 simplified effects), the Preview panel's settings +(range, play from, skip, frame rate, full screen), GPU coverage (Advanced 3D, more GPU effects), +the web app (no threads for viewer frames and Roto Brush), the Learn panel and Align to Selection / +Distribute, Advanced 3D (iris shapes, collapsed precomps, extruded strokes), ScriptUI resource +strings and `onDraw`, Essential Graphics font and scale controls, and the learned-model quality of +Roto Brush and face tracking (both are classical). Menu items still absent: Save Frame As ▸ +Photoshop Layers / ProEXR, Open in Essential Graphics, Vanishing Point import, Watch Folder. + +### Previous audit (commit `fa26ad9`, 2 October 2026, late): ≈ 93% By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, -Tracking 88%, Effects 85%, Web 65%. +Tracking 96%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking -renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; -approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; text in imported -PDF/AI files and Illustrator procset EPS; more codecs; 59 preferences not yet wired; ScriptUI; face tracking; and the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). +renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; +approximated effects (Key Cleaner) and Liquify's viewer brush; text in imported +PDF/AI files and Illustrator procset EPS; and more codecs. Face tracking and a true Subspace Warp +landed in M13.3. ScriptUI (dialogs that block `show()`, palettes, dockable ScriptUI panels, File ▸ Scripts install +and sample scripts), puppet pin recording and the remaining "better than After Effects" items — a +versioned effect plug-in API with sandboxed WebAssembly plug-ins ([plugins.md](plugins.md)), +branching undo history in the History panel, and a bit-identical rendering test across runs and +thread counts — landed in M13.1 (GPU particles landed in M12.7). Render Queue field render, crop/resize, templates and logs, and VP9 inter frames landed in M10.2. The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for history. @@ -107,13 +122,13 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | +| Output | 97% | 0.8 | Render Settings complete (field render + 3:2 pulldown, effects/solo/guide/depth/blending/blur overrides, time sampling, storage overflow), Output Module crop/ROI/resize, alpha modes, post-render actions, PCM formats, templates with defaults, render logs, Notify (M10.2); WebM VP9 key + inter frames with motion search, loop filter and rate control. Left: Opus is CELT-only, Photoshop sequence output, overflow for movies only checks at file creation | | Audio | 85% | 0.5 | audio to keyframes | | Import | 94% | 1.0 | text and images inside PDF/AI files, Illustrator EPS relying on Adobe procsets, smart-object perspective/warps, PSD 3D layers (PDF / PDF-compatible AI / EPS vector footage with Continuously Rasterize, layered composition import and Create Shapes from Vector Layer, and PSD smart objects with embedded files landed in M13.2; PSD as footage/composition/retain layer sizes, SVG footage earlier) | -| Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | +| Automation | ≈ 97% | 0.3 | `.jsxbin`, sockets, ScriptUI resource strings and custom `onDraw` graphics (the core object model landed in M14.4; ScriptUI windows/dialogs/dockable panels with `scriptui.*` agent commands, File ▸ Scripts install + sample scripts, and the effect plug-in API (EFF-6, WebAssembly) landed in M13.1) | | Shapes | ≈ 80% | 1.5 | Lottie can't carry stroke taper/wave (stroke Taper and Wave, Dash 2/Gap 2/Dash 3/Gap 3 and radial-gradient Highlight Length/Angle landed in M13.5; pen tool for shape paths and vertex editing in M6.5) | | Compositions | ≈ 80% | 3.5 | Mocha-style planar tracks for templates, Essential Graphics' rare controls (font menus, mirrored properties) (the marker dialog, Composition Flowchart, Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-6, CMP-7) | -| Animation | 67% | 10.0 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | +| Animation | 70% | 9.0 | puppet depth beyond pins and recording (puppet pin recording with Record Options landed in M13.1), Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | | Text | ≈ 94% | 0.5 | no extruded strokes, variable-axis animation changes outlines but not advances (OpenType features — stylistic sets, discretionary ligatures, contextual / stylistic alternates, swash, titling, ordinals, fractions, figure styles, true small caps / all small caps and superior / inferior glyphs with faux fallback — per character with the Character panel's OpenType popup and `text.fontFeatures` landed in M13.2; vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | | Web | 85% | 1.0 | viewer frames and Roto Brush propagation still on the page's thread; GPU effects in the browser (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2) | | 3D | 85% | 6.5 | multi-view layouts, iris shapes in Advanced 3D's depth of field, collapsed 3D precomps in Advanced 3D, the Extended Viewer for Advanced 3D comps (Classic 3D Extended Viewer landed in M13.5 UI completion); collapsed precomps of another size seen through the parent's camera render (fixed in M13.2); stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | @@ -122,7 +137,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 66% | 4.0 | GPU bokeh depth of field, wireframes and Advanced 3D compositing (Classic 3D runs and adjustment layers composite on the GPU since M12.7; persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | +| Tracking | ≈ 96% | 0.3 | face tracking is a classical (skin model + feature components + shape model) fitter, not a learned detector: profile views and occluded faces are weak; Rolling Shutter Ripple is approximated by Subspace Warp's mesh density (face tracking (Outline Only / Detailed Features with Face Track Points and Extract & Copy Face Measurements), Subspace Warp's content-preserving mesh warp on subspace-smoothed trajectories, and radial lens distortion (k1, k2) in the camera bundle adjustment with Undistort Footage landed in M13.3; Rolling Shutter Repair in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 00000000..5d5b39c0 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,126 @@ +# Effect plug-ins + +EffectCraft loads third-party effects through a small, versioned plug-in API (**API version 1**). +A plug-in effect behaves exactly like a built-in one: it is listed in Effects & Presets (and the +Effect menu) under its category, applied by id (`effect.apply {"effect": "org.example.posterize"}`) +or display name, its parameters are ordinary properties (keyframes, expressions, Effect Controls, +`prop.set`), and it is saved in projects by id. + +There are two ways to write one: + +| | WebAssembly plug-in | Rust plug-in | +|---|---|---| +| What | a `.wasm` (or `.wat`) module | a type implementing `effectcraft_effects::plugin::EffectPlugin` | +| Loaded by | Effect ▸ Load Effect Plug-in…, `effect.plugins.load {path \| folder}`, or the `Plug-ins` folder next to the settings at start-up | `register_plugin(Arc::new(MyEffect))` in an app that links EffectCraft's crates | +| Sandbox | yes: no imports (no files, clock or network), fuel-limited, deterministic floats | none (it is your code) | +| Platforms | desktop and CLI (the wasmi interpreter, crate feature `effectcraft-plugin/wasm`); not the web build | everywhere | + +`effect.plugins.list` lists what is loaded: `{api, wasm, plugins: [{id, name, category, version, +author, source, params}]}`. + +## The contract + +* **Pixels** are premultiplied RGBA `f32` (0–1; values may exceed 1 in 32 bpc projects), row + major, processed **in place**. The buffer is the layer's (after masks and the effects above), + possibly at preview resolution (`scale` = buffer pixels per layer pixel) and padded. +* **Parameters** arrive flattened to `f64`s in the order they are declared: slider, angle, + checkbox (0 / 1) and popup (0-based index) take one value, point two (already in **buffer + pixels**), colour four (RGBA 0–1). +* **Time** is layer time in seconds. +* Rendering must be **deterministic** and may run on several threads at once (a WebAssembly + plug-in gets one instance per thread). +* A failed render (an error code, a trap, running out of fuel) leaves the frame unchanged. + +## The manifest + +Every plug-in describes itself with a JSON manifest: + +```json +{ + "api": 1, + "id": "org.effectcraft.example.posterize-bands", + "name": "Posterize Bands", + "category": "Stylize", + "version": "1.0.0", + "author": "EffectCraft contributors", + "description": "Quantizes luminance into tinted bands.", + "params": [ + {"id": "levels", "name": "Levels", "type": "slider", "default": 4, "min": 2, "max": 64, "sliderMax": 16, "decimals": 0}, + {"id": "mix", "name": "Tint Amount", "type": "slider", "default": 50, "min": 0, "max": 100}, + {"id": "tint", "name": "Tint", "type": "color", "default": [1.0, 0.55, 0.2, 1.0]}, + {"id": "invert", "name": "Invert Bands", "type": "checkbox", "default": false} + ] +} +``` + +* `id`: stable, reverse-domain; letters, digits, `.`, `_`, `-`. `ec.` is reserved for built-ins. + Ids must be unique (a second plug-in with a taken id is refused). +* `category`: one of the Effects & Presets categories (`Blur & Sharpen`, `Stylize`, …) or a new + one, which appears after the built-in categories. +* Parameter types: `slider` (`default`, `min`, `max`, optional `sliderMin`, `sliderMax`, + `decimals`), `angle` (`default` degrees), `checkbox` (`default` bool), `popup` (`options`, + `default` index), `point` (`default` as fractions of the layer size, `[0.5, 0.5]` = centre), + `color` (`default` RGBA 0–1). Parameter ids are letters, digits and `_`. + +## WebAssembly ABI (v1) + +A module with **no imports** that exports: + +| export | signature | | +|---|---|---| +| `memory` | memory | linear memory shared with the host | +| `ec_api_version` | `() -> i32` | must return `1` | +| `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | the UTF-8 JSON manifest in memory | +| `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | +| `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; return 0 on success | + +For each frame the host calls `ec_alloc` once for a buffer holding the parameters (`nparams` +little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s), fills it, +calls `ec_render`, and reads the pixels back. Each call gets a fuel budget proportional to the +frame size; a module that loops forever stops with an error instead of hanging the app. + +### Example + +[`examples/plugins/posterize-bands`](../examples/plugins/posterize-bands) is a complete plug-in in +plain Rust (no dependencies): + +```sh +cd examples/plugins/posterize-bands +cargo build --release --target wasm32-unknown-unknown +# then, in EffectCraft: Effect ▸ Load Effect Plug-in… → target/wasm32-unknown-unknown/release/posterize_bands.wasm +# or headless: +effectcraft-cli exec effect.plugins.load '{"path": "target/wasm32-unknown-unknown/release/posterize_bands.wasm"}' +``` + +Any language that compiles to WebAssembly works (C, Zig, AssemblyScript…) as long as the module +has no imports. `crates/plugin/src/tests.rs` has a plug-in written directly in WebAssembly text. + +## Rust plug-ins + +```rust +use std::sync::Arc; +use effectcraft_effects::plugin::*; + +struct Swap(PluginManifest); + +impl EffectPlugin for Swap { + fn manifest(&self) -> &PluginManifest { &self.0 } + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, _time: f64) -> Result<(), String> { + if params.b("on") { + for p in frame.pixels.iter_mut() { p.swap(0, 2); } + } + Ok(()) + } +} + +register_plugin(Arc::new(Swap(manifest)))?; // before the UI builds its menus +``` + +`PluginParams` reads values by parameter id (`f`, `b`, `point`, `color`). At most 64 plug-in +effects can be registered per process. + +## Compatibility + +The API version is checked when a plug-in loads: a manifest or module for another version is +refused with a message. Within a version the contract above does not change; additions (new +parameter types, new optional exports) bump the version. diff --git a/docs/web.md b/docs/web.md index 041ef76c..72eb75ff 100644 --- a/docs/web.md +++ b/docs/web.md @@ -207,5 +207,3 @@ every request and reply, a render through an in-process offload). - Cancelling an analysis drops its partial result (the worker is terminated). - Stored media count against the origin's storage quota; `effectcraft.listStored()` shows usage, `removeStored` frees it (there is no storage manager in the UI yet). -- `engine.execute` rejects the spread Render Settings / Output Module keys of `renderQueue.add` - (`resolution`, `timeSpan`…) through its strict parameter check; use `ui.menu.invoke` as above. diff --git a/examples/plugins/posterize-bands/Cargo.lock b/examples/plugins/posterize-bands/Cargo.lock new file mode 100644 index 00000000..c1732e16 --- /dev/null +++ b/examples/plugins/posterize-bands/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "posterize-bands" +version = "1.0.0" diff --git a/examples/plugins/posterize-bands/Cargo.toml b/examples/plugins/posterize-bands/Cargo.toml new file mode 100644 index 00000000..825d704a --- /dev/null +++ b/examples/plugins/posterize-bands/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "posterize-bands" +description = "Example EffectCraft effect plug-in (WebAssembly, plug-in API v1): posterizes luminance into tinted bands" +version = "1.0.0" +edition = "2024" +license = "MIT OR Apache-2.0" +publish = false + +# Standalone: build with `cargo build --release --target wasm32-unknown-unknown` in this folder. +[workspace] + +[lib] +crate-type = ["cdylib"] + +[profile.release] +opt-level = "s" +lto = true +panic = "abort" diff --git a/examples/plugins/posterize-bands/src/lib.rs b/examples/plugins/posterize-bands/src/lib.rs new file mode 100644 index 00000000..9af70943 --- /dev/null +++ b/examples/plugins/posterize-bands/src/lib.rs @@ -0,0 +1,93 @@ +//! Posterize Bands: an example EffectCraft effect plug-in (plug-in API v1, see +//! `docs/plugins.md`). It quantizes each pixel's luminance into `levels` bands and mixes a tint +//! colour in by `mix` percent, optionally only above a `threshold`. +//! +//! Build: `cargo build --release --target wasm32-unknown-unknown`, then load +//! `target/wasm32-unknown-unknown/release/posterize_bands.wasm` with Effect ▸ Load Effect Plug-in +//! (or `effect.plugins.load {"path": …}`), or drop it in the settings folder's `Plug-ins`. + +use std::cell::RefCell; + +/// The manifest: id, name, Effects & Presets category, parameters (flattened in this order: +/// levels, mix, tint r g b a, invert). +const MANIFEST: &str = r#"{ + "api": 1, + "id": "org.effectcraft.example.posterize-bands", + "name": "Posterize Bands", + "category": "Stylize", + "version": "1.0.0", + "author": "EffectCraft contributors", + "description": "Quantizes luminance into tinted bands.", + "params": [ + {"id": "levels", "name": "Levels", "type": "slider", "default": 4, "min": 2, "max": 64, "sliderMax": 16, "decimals": 0}, + {"id": "mix", "name": "Tint Amount", "type": "slider", "default": 50, "min": 0, "max": 100, "decimals": 1}, + {"id": "tint", "name": "Tint", "type": "color", "default": [1.0, 0.55, 0.2, 1.0]}, + {"id": "invert", "name": "Invert Bands", "type": "checkbox", "default": false} + ] +}"#; + +thread_local! { + static BUF: RefCell> = const { RefCell::new(Vec::new()) }; +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_api_version() -> i32 { + 1 +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_manifest_ptr() -> i32 { + MANIFEST.as_ptr() as i32 +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_manifest_len() -> i32 { + MANIFEST.len() as i32 +} + +/// One reusable, 8-byte-aligned buffer for the parameters and pixels. +#[unsafe(no_mangle)] +pub extern "C" fn ec_alloc(bytes: i32) -> i32 { + BUF.with(|b| { + let mut b = b.borrow_mut(); + let words = (bytes.max(0) as usize).div_ceil(8); + if b.len() < words { + b.resize(words, 0); + } + b.as_mut_ptr() as i32 + }) +} + +/// # Safety +/// The host passes pointers into the buffer `ec_alloc` returned. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ec_render(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, _time: f64, _scale: f64) -> i32 { + if nparams < 7 { + return 1; + } + let p = unsafe { std::slice::from_raw_parts(params as *const f64, nparams as usize) }; + let px = unsafe { std::slice::from_raw_parts_mut(pixels as *mut [f32; 4], (width.max(0) * height.max(0)) as usize) }; + let levels = p[0].round().max(2.0) as f32; + let mix = (p[1] / 100.0).clamp(0.0, 1.0) as f32; + let tint = [p[2] as f32, p[3] as f32, p[4] as f32]; + let invert = p[6] != 0.0; + for c in px.iter_mut() { + let a = c[3]; + if a <= 0.0 { + continue; + } + // Un-premultiply, band the luminance, re-premultiply. + let (r, g, b) = (c[0] / a, c[1] / a, c[2] / a); + let y = 0.2126 * r + 0.7152 * g + 0.0722 * b; + let mut band = (y * levels).floor().min(levels - 1.0) / (levels - 1.0); + if invert { + band = 1.0 - band; + } + for (i, ch) in [r, g, b].into_iter().enumerate() { + let banded = if y > 0.0 { ch * band / y.max(1e-6) } else { band }; + let v = banded * (1.0 - mix) + band * tint[i] * mix; + c[i] = v.max(0.0) * a; + } + } + 0 +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ad904eba..11899dc4 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -44,6 +44,7 @@ const LAYERS: &[(&str, u8)] = &[ ("gpu", 3), ("lottie", 3), ("format", 3), + ("plugin", 3), ("engine", 4), ("script", 4), ("host", 4),