From 83c7e14b944a44329db30bc269b091ff0cf83151 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:04:47 -0400 Subject: [PATCH 1/4] M7.4: model import New crate effectcraft-model (L2): glTF 2.0 (.gltf/.glb, data URIs, external buffers and images, PBR metallic-roughness with base/MR/normal/occlusion/emissive textures, node hierarchy, skins, step/linear/cubic animations) and OBJ/MTL import, parametric primitives, extruded/bevelled outline meshes and polygon triangulation with holes. Project: FootageKind::Model, LayerSource::Model/Primitive, Layer::environment, Geometry and Material Options builders, Environment light Source. Media probes and loads models (FootageSource::model). Engine: layer.newModel, layer.new3dPrimitive/newPrimitive, comp.renderer, layer.environment, material.set/reset/duplicateAssign/revealSource (stubs replaced); File > Import and layer.addItem pick up models. Hand-written fixtures with attribution sidecars. Co-Authored-By: Claude Opus 5.5 --- ATTRIBUTION.md | 4 + Cargo.lock | 14 + Cargo.toml | 1 + crates/engine/Cargo.toml | 1 + crates/engine/src/commands/comp.rs | 1 + crates/engine/src/commands/file.rs | 2 +- crates/engine/src/commands/layer.rs | 9 +- crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/model3d.rs | 397 ++++++++++ crates/engine/src/commands/stubs.rs | 5 - crates/engine/src/lib.rs | 2 + crates/engine/src/tests_model3d.rs | 194 +++++ crates/lottie/src/export.rs | 4 + crates/media/Cargo.toml | 1 + crates/media/src/lib.rs | 10 +- crates/media/src/pool.rs | 35 +- crates/media/src/probe.rs | 39 + crates/model/Cargo.toml | 17 + crates/model/src/extrude.rs | 301 ++++++++ crates/model/src/gltf.rs | 519 +++++++++++++ crates/model/src/lib.rs | 691 ++++++++++++++++++ crates/model/src/obj.rs | 262 +++++++ crates/model/src/prim.rs | 235 ++++++ crates/model/src/tests.rs | 318 ++++++++ crates/model/src/triangulate.rs | 187 +++++ crates/model/tests/fixtures/cube.mtl | 11 + .../model/tests/fixtures/cube.mtl.attribution | 7 + crates/model/tests/fixtures/cube.obj | 29 + .../model/tests/fixtures/cube.obj.attribution | 7 + crates/model/tests/fixtures/quad.glb | Bin 0 -> 2028 bytes .../model/tests/fixtures/quad.glb.attribution | 7 + crates/model/tests/fixtures/quad.gltf | 210 ++++++ .../tests/fixtures/quad.gltf.attribution | 7 + crates/project/src/build.rs | 100 ++- crates/project/src/lib.rs | 53 +- crates/render/Cargo.toml | 1 + crates/render/src/lib.rs | 4 + crates/ui-egui/src/menus.rs | 2 +- crates/ui-egui/src/panels/project.rs | 1 + crates/ui-egui/src/panels/timeline.rs | 1 + docs/architecture.md | 1 + xtask/src/main.rs | 1 + 42 files changed, 3676 insertions(+), 17 deletions(-) create mode 100644 crates/engine/src/commands/model3d.rs create mode 100644 crates/engine/src/tests_model3d.rs create mode 100644 crates/model/Cargo.toml create mode 100644 crates/model/src/extrude.rs create mode 100644 crates/model/src/gltf.rs create mode 100644 crates/model/src/lib.rs create mode 100644 crates/model/src/obj.rs create mode 100644 crates/model/src/prim.rs create mode 100644 crates/model/src/tests.rs create mode 100644 crates/model/src/triangulate.rs create mode 100644 crates/model/tests/fixtures/cube.mtl create mode 100644 crates/model/tests/fixtures/cube.mtl.attribution create mode 100644 crates/model/tests/fixtures/cube.obj create mode 100644 crates/model/tests/fixtures/cube.obj.attribution create mode 100644 crates/model/tests/fixtures/quad.glb create mode 100644 crates/model/tests/fixtures/quad.glb.attribution create mode 100644 crates/model/tests/fixtures/quad.gltf create mode 100644 crates/model/tests/fixtures/quad.gltf.attribution diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 9caf1c27..f89c0e87 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -36,6 +36,10 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/quad.gltf` | EffectCraft contributors | Original work: glTF 2.0 test fixture (textured quad, node hierarchy, translation animation; data URIs) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/quad.glb` | EffectCraft contributors | Original work: binary glTF test fixture (the same quad with its texture in the BIN chunk) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/cube.obj` | EffectCraft contributors | Original work: Wavefront OBJ test fixture (unit cube, two materials) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/cube.mtl` | EffectCraft contributors | Original work: MTL test fixture (materials of cube.obj) | MIT OR Apache-2.0 | ## Screenshots diff --git a/Cargo.lock b/Cargo.lock index 635fb0f5..fc3f4a02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,6 +1195,7 @@ dependencies = [ "effectcraft-geom", "effectcraft-keyframe", "effectcraft-lottie", + "effectcraft-model", "effectcraft-path", "effectcraft-project", "effectcraft-raster", @@ -1322,6 +1323,7 @@ dependencies = [ name = "effectcraft-media" version = "0.1.0" dependencies = [ + "effectcraft-model", "effectcraft-project", "effectcraft-raster", "effectcraft-render", @@ -1337,6 +1339,17 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-model" +version = "0.1.0" +dependencies = [ + "effectcraft-geom", + "image", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "effectcraft-path" version = "0.1.0" @@ -1379,6 +1392,7 @@ dependencies = [ "effectcraft-effects", "effectcraft-geom", "effectcraft-keyframe", + "effectcraft-model", "effectcraft-path", "effectcraft-project", "effectcraft-raster", diff --git a/Cargo.toml b/Cargo.toml index a1941cf4..a5294834 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ effectcraft-project = { path = "crates/project" } effectcraft-text = { path = "crates/text" } effectcraft-effects = { path = "crates/effects" } effectcraft-track = { path = "crates/track" } +effectcraft-model = { path = "crates/model" } effectcraft-render = { path = "crates/render" } effectcraft-gpu = { path = "crates/gpu" } effectcraft-media = { path = "crates/media" } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 132bdcac..a60242bb 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,6 +21,7 @@ effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } effectcraft-render = { workspace = true } effectcraft-track = { workspace = true } +effectcraft-model = { workspace = true } effectcraft-lottie = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/engine/src/commands/comp.rs b/crates/engine/src/commands/comp.rs index 5ebd491e..a33fd574 100644 --- a/crates/engine/src/commands/comp.rs +++ b/crates/engine/src/commands/comp.rs @@ -125,6 +125,7 @@ fn settings(s: &mut Session, p: &Value) -> Result { } let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; apply_settings(c, p); + super::model3d::sync_geometry_options(proj, cid); Ok(()) })?; let c = s.project.comp(cid).ok_or(EngineError::NoComp)?; diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index a85a391f..11e8a387 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -110,7 +110,7 @@ fn import(s: &mut Session, p: &Value) -> Result { let label = match f.kind { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, - FootageKind::Video => Label::Aqua, + FootageKind::Video | FootageKind::Model => Label::Aqua, }; let id = proj.add_item(&name, label, None, ItemKind::Footage(f)); ids.push(id.0); diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index 3dea6880..ac728200 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -24,7 +24,11 @@ pub(crate) fn color_p(p: &Value, k: &str) -> Option<[f32; 3]> { } /// Insert a new layer above the selection (or at the top) and select it. -pub(crate) fn insert_layer(proj: &mut Project, st: &mut crate::EditorState, cid: ItemId, layer: Layer) -> Result { +pub(crate) fn insert_layer(proj: &mut Project, st: &mut crate::EditorState, cid: ItemId, mut layer: Layer) -> Result { + // Text and shape layers in Advanced 3D comps get Geometry Options (extrusion, bevels). + if proj.comp(cid).is_some_and(|c| c.renderer == effectcraft_project::Renderer::Advanced3D) { + super::model3d::add_geometry_options(&mut proj.next_id, &mut layer); + } let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; let at = st.selected_layers.first().and_then(|id| comp.layers.iter().position(|l| l.id == *id)).unwrap_or(0); let id = layer.id; @@ -235,6 +239,9 @@ fn add_item(s: &mut Session, p: &Value) -> Result { let d = (s.prefs.import.still_footage == "seconds").then(|| Tick::from_seconds_f64(s.prefs.import.still_seconds)); (LayerSource::Footage { item }, (f.width, f.height), d) } + ItemKind::Footage(f) if f.kind == effectcraft_project::FootageKind::Model => { + return super::model3d::new_model(s, &serde_json::json!({"comp": cid.0, "item": item.0, "time": f_p(p, "time")})); + } ItemKind::Footage(f) => (LayerSource::Footage { item }, (f.width, f.height), Some(f.duration)), ItemKind::Solid(so) => (LayerSource::Solid { item }, (so.width, so.height), None), ItemKind::Folder => return Err(bad("layer.addItem", "folders can't be layers")), diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 26db85c9..519bce63 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -20,6 +20,7 @@ mod link; mod lottie; mod markers; mod mask; +pub(crate) mod model3d; pub mod paint; mod paths; mod project_items; @@ -99,6 +100,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(layer::specs()); v.extend(styles::specs()); v.extend(three_d::specs()); + v.extend(model3d::specs()); v.extend(text_anim::specs()); v.extend(text_edit::specs()); v.extend(layer_time::specs()); diff --git a/crates/engine/src/commands/model3d.rs b/crates/engine/src/commands/model3d.rs new file mode 100644 index 00000000..1ad9cc32 --- /dev/null +++ b/crates/engine/src/commands/model3d.rs @@ -0,0 +1,397 @@ +//! Advanced 3D: model layers (File ▸ Import of glTF/OBJ, Layer ▸ New ▸ 3D primitives), the +//! comp renderer switch, Layer ▸ Environment Layer, Geometry Options for extruded text and +//! shapes, and Layer ▸ Material commands. + +use effectcraft_keyframe::Value as KV; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{FootageKind, ItemId, ItemKind, Layer, LayerSource, Node, PrimitiveKind, PropGroup, Renderer}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::layer::{color_p, insert_layer}; +use super::{CommandSpec, b_p, bad, comp_id, f_p, has_comp, has_layers, layer_mut, layer_p, layers_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +// ---------------------------------------------------------------- geometry options + +/// Add Geometry Options (Bevel Style, Bevel Depth, Hole Bevel Depth, Extrusion Depth) to a text +/// or shape layer that has none. +pub(crate) fn add_geometry_options(next_id: &mut u64, l: &mut Layer) { + if !matches!(l.source, LayerSource::Text | LayerSource::Shape) || l.props.sub("geometryOptions").is_some() { + return; + } + let g = build::extrusion_geometry_options(&mut Ids(next_id)); + // After Transform, like After Effects. + let at = l.props.children.iter().position(|c| c.match_id() == "transform").map_or(l.props.children.len(), |i| i + 1); + l.props.children.insert(at, g.into()); +} + +/// Advanced 3D comps: give every text and shape layer Geometry Options. +pub(crate) fn sync_geometry_options(proj: &mut effectcraft_project::Project, cid: ItemId) { + if proj.comp(cid).is_none_or(|c| c.renderer != Renderer::Advanced3D) { + return; + } + let mut next = proj.next_id; + if let Some(c) = proj.comp_mut(cid) { + for l in &mut c.layers { + add_geometry_options(&mut next, l); + } + } + proj.next_id = next; +} + +fn renderer_p(p: &Value, cmd: &str) -> Result { + let v = str_p(p, "renderer").or_else(|| str_p(p, "value")).ok_or_else(|| bad(cmd, "missing `renderer` (advanced3d|classic3d)"))?; + match v.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "advanced3d" | "advanced" => Ok(Renderer::Advanced3D), + "classic3d" | "classic" => Ok(Renderer::Classic3D), + _ => Err(bad(cmd, format!("renderer: advanced3d|classic3d, not `{v}`"))), + } +} + +fn comp_renderer(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + if p.get("renderer").is_none() && p.get("value").is_none() { + let r = s.project.comp(cid).ok_or(EngineError::NoComp)?.renderer; + return Ok(json!({"renderer": renderer_id(r)})); + } + let r = renderer_p(p, "comp.renderer")?; + s.edit("Change Renderer", None, |proj, _| { + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.renderer = r; + sync_geometry_options(proj, cid); + Ok(()) + })?; + Ok(json!({"renderer": renderer_id(r)})) +} + +fn renderer_id(r: Renderer) -> &'static str { + match r { + Renderer::Advanced3D => "advanced3d", + Renderer::Classic3D => "classic3d", + } +} + +// ---------------------------------------------------------------- model layers + +/// Set a property's value at comp time `t` (adds a key when it is animated). +fn set_at(l: &mut Layer, path: &str, t: Tick, v: KV) -> bool { + let lt = l.layer_time(t); + match l.props.prop_mut(path) { + Some(pr) => { + pr.set_value_at(lt, v); + true + } + None => false, + } +} + +/// New 3D model layer from a model footage item (`item`) or a file (`path`, imported first). +pub(crate) fn new_model(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let item = match (p.get("item"), str_p(p, "path")) { + (Some(Value::Number(n)), _) => ItemId(n.as_u64().unwrap_or(0)), + (Some(Value::String(name)), _) => s.project.find_by_name(name).map(|i| i.id).ok_or_else(|| bad("layer.newModel", format!("no item `{name}`")))?, + (_, Some(path)) => { + let r = s.execute("file.import", json!({"paths": [path]}))?; + let id = r.get("items").and_then(|a| a.get(0)).and_then(Value::as_u64); + let err = r.get("errors").and_then(|e| e.get(0)).and_then(Value::as_str).unwrap_or("import failed").to_string(); + ItemId(id.ok_or_else(|| bad("layer.newModel", err))?) + } + _ => *s.state.project_selection.first().ok_or_else(|| bad("layer.newModel", "missing `item` or `path`"))?, + }; + let it = s.project.item(item).ok_or_else(|| bad("layer.newModel", "no such item"))?.clone(); + let ItemKind::Footage(f) = &it.kind else { return Err(bad("layer.newModel", format!("`{}` is not a 3D model", it.name))) }; + if f.kind != FootageKind::Model { + return Err(bad("layer.newModel", format!("`{}` is not a 3D model", it.name))); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + // Fit the model to half the comp height, and list its animation clips. + let model = s.footage.model(item, f); + let (scale, clips) = match &model { + Some(m) => { + let ext = m.bounds().map_or(1.0, |(lo, hi)| (0..3).map(|k| hi[k] - lo[k]).fold(0.0f64, f64::max)); + let scale = if ext > 1e-9 { comp.height as f64 * 0.5 / ext } else { 100.0 }; + let clips = m.animations.iter().enumerate().map(|(i, a)| if a.name.is_empty() { format!("Animation {}", i + 1) } else { a.name.clone() }).collect(); + (scale, clips) + } + None => (100.0, vec![]), + }; + let name = str_p(p, "name").unwrap_or(&it.name).to_string(); + let fr = comp.frame_rate; + let start = fr.snap_nearest(f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or(Tick::ZERO)); + let id = s.edit("New 3D Model Layer", None, |proj, st| { + let mut l = build::layer(proj, &comp, &name, LayerSource::Model { item }, (comp.width, comp.height), None); + let g = build::model_geometry_options(&mut Ids(&mut proj.next_id), scale, &clips); + if let Some(slot) = l.props.children.iter_mut().find(|c| c.match_id() == "geometryOptions") { + *slot = g.into(); + } + l.start_time = start; + l.in_point = start; + l.out_point = comp.duration.max(start + fr.frame_duration()); + insert_layer(proj, st, cid, l) + })?; + Ok(json!({"layer": id.0, "item": item.0})) +} + +fn kind_p(p: &Value, cmd: &str) -> Result { + let k = str_p(p, "kind").or_else(|| str_p(p, "type")).ok_or_else(|| bad(cmd, "missing `kind` (cube|sphere|plane|torus|cone|cylinder)"))?; + PrimitiveKind::parse(k).ok_or_else(|| bad(cmd, format!("kind: cube|sphere|plane|torus|cone|cylinder, not `{k}`"))) +} + +/// Parse Material Options parameters onto a layer at comp time `t`. +fn apply_material(l: &mut Layer, p: &Value, t: Tick, cmd: &str) -> Result { + let mut n = 0; + let tri = |v: &Value| -> Option { + match v { + Value::Bool(b) => Some(*b as u32), + Value::Number(x) => x.as_u64().map(|x| x.min(2) as u32), + Value::String(s) => match s.to_ascii_lowercase().as_str() { + "off" | "false" => Some(0), + "on" | "true" => Some(1), + "only" => Some(2), + _ => None, + }, + _ => None, + } + }; + for k in ["castsShadows", "acceptsShadows"] { + if let Some(v) = p.get(k) { + let e = tri(v).ok_or_else(|| bad(cmd, format!("{k}: off|on|only")))?; + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Enum(e)) as usize; + } + } + for k in ["acceptsLights", "appearsInReflections"] { + if let Some(v) = b_p(p, k) { + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Bool(v)) as usize; + } + } + for k in ["lightTransmission", "ambient", "diffuse", "specularIntensity", "specularShininess", "metal", "metallic", "roughness"] { + if let Some(v) = f_p(p, k) { + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Scalar(v)) as usize; + } + } + for k in ["baseColor", "emissive"] { + if let Some(c) = color_p(p, k) { + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Color([c[0] as f64, c[1] as f64, c[2] as f64, 1.0])) as usize; + } + } + Ok(n) +} + +fn new_primitive(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let kind = kind_p(p, "layer.newPrimitive")?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| kind.label().to_string()); + let t = super::time_p(s, p, Some(&comp)); + let id = s.edit(&format!("New {}", kind.label()), None, |proj, st| { + let mut l = build::layer(proj, &comp, &name, LayerSource::Primitive { kind }, (comp.width, comp.height), None); + for k in ["width", "height", "depth", "radius", "tubeRadius", "segments", "rings"] { + if let Some(v) = f_p(p, k) { + set_at(&mut l, &format!("geometryOptions/{k}"), t, KV::Scalar(v.max(0.0))); + } + } + if let Some(pos) = p.get("position").and_then(Value::as_array) { + let g = |i: usize| pos.get(i).and_then(Value::as_f64).unwrap_or(0.0); + set_at(&mut l, "transform/position", t, KV::Vec3([g(0), g(1), g(2)])); + } + apply_material(&mut l, p, t, "layer.newPrimitive")?; + insert_layer(proj, st, cid, l) + })?; + // Primitives are drawn by the Advanced 3D renderer. + let advanced = s.project.comp(cid).is_some_and(|c| c.renderer == Renderer::Advanced3D); + Ok(json!({"layer": id.0, "kind": kind.label(), "advanced3d": advanced})) +} + +// ---------------------------------------------------------------- environment layer + +fn environment(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.is_empty() { + return Err(bad("layer.environment", "select a footage or composition layer")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + for id in &ids { + let l = comp.layer(*id).ok_or(EngineError::NoComp)?; + if !l.source.is_av() || l.source.is_model() { + return Err(bad("layer.environment", format!("`{}` can't be an environment layer (use a footage, comp or solid layer)", l.name))); + } + } + let on = b_p(p, "on").unwrap_or_else(|| !comp.layer(ids[0]).is_some_and(|l| l.environment)); + s.edit("Environment Layer", None, |proj, _| { + for id in &ids { + let l = layer_mut(proj, cid, *id)?; + l.environment = on; + if on { + l.switches.three_d = true; + } + } + Ok(()) + })?; + Ok(json!({"environment": on, "layers": ids.iter().map(|l| l.0).collect::>()})) +} + +// ---------------------------------------------------------------- materials + +fn material_set(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.is_empty() { + return Err(bad("material.set", "no `layers` given and no layer selected")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let t = super::time_p(s, p, Some(&comp)); + let n = s.edit("Material Options", None, |proj, _| { + let mut n = 0; + for id in &ids { + n += apply_material(layer_mut(proj, cid, *id)?, p, t, "material.set")?; + } + Ok(n) + })?; + if n == 0 { + return Err(bad( + "material.set", + "no Material Options property matched (castsShadows, acceptsShadows, acceptsLights, baseColor, metallic, roughness, …)", + )); + } + Ok(json!({"set": n})) +} + +/// Fresh Material Options for a layer. +fn default_material(next: &mut u64, l: &Layer) -> Option { + let ids = &mut Ids(next); + match l.source { + LayerSource::Primitive { .. } => Some(build::model_material_options(ids, true)), + LayerSource::Model { .. } => Some(build::model_material_options(ids, false)), + _ if l.source.is_av() => Some(build::material_options(ids)), + _ => None, + } +} + +fn replace_material(l: &mut Layer, g: PropGroup) { + match l.props.children.iter_mut().find(|c| c.match_id() == "materialOptions") { + Some(slot) => *slot = Node::Group(g), + None => l.props.children.push(g.into()), + } +} + +fn material_reset(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.is_empty() { + return Err(bad("material.reset", "no `layers` given and no layer selected")); + } + s.edit("Reset Material", None, |proj, _| { + let mut next = proj.next_id; + for id in &ids { + let l = layer_mut(proj, cid, *id)?; + if let Some(g) = default_material(&mut next, l) { + replace_material(l, g); + } + } + proj.next_id = next; + Ok(()) + })?; + Ok(json!({"layers": ids.iter().map(|l| l.0).collect::>()})) +} + +/// Duplicate and Assign Material: the first layer's Material Options are copied onto the other +/// selected layers of the same kind. +fn material_duplicate(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.len() < 2 { + return Err(bad("material.duplicateAssign", "select the source layer first, then the layers to assign its material to")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let src = comp + .layer(ids[0]) + .and_then(|l| l.props.sub("materialOptions")) + .cloned() + .ok_or_else(|| bad("material.duplicateAssign", "the first layer has no material"))?; + let n = s.edit("Duplicate and Assign Material", None, |proj, _| { + let mut n = 0; + for id in &ids[1..] { + let l = layer_mut(proj, cid, *id)?; + let Some(dst) = l.props.sub("materialOptions") else { continue }; + // Property sets differ (models, primitives, cards): copy the values they share. + let mut g = dst.clone(); + for c in g.children.iter_mut() { + if let Node::Prop(d) = c + && let Some(sv) = src.get(&d.match_id) + { + d.value = sv.value.clone(); + d.keys = sv.keys.clone(); + d.expr = sv.expr.clone(); + } + } + replace_material(l, g); + n += 1; + } + Ok(n) + })?; + Ok(json!({"assigned": n})) +} + +fn material_reveal(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "material.revealSource")?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let LayerSource::Model { item } = l.source else { return Err(bad("material.revealSource", "the layer's material is its own (not from a model file)")) }; + s.state.project_selection = vec![item]; + Ok(json!({"item": item.0})) +} + +fn has_model_or_layers(s: &Session) -> std::result::Result<(), String> { + has_layers(s) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "layer.newModel", + "3D Model Layer", + [], + None, + "{item?: id|name (a 3D model footage item) | path?: .gltf|.glb|.obj file, name?, time? (s)}", + has_comp, + new_model + ), + cmd!( + "layer.new3dPrimitive", + "3D Primitive", + [], + None, + "{kind: cube|sphere|plane|torus|cone|cylinder, name?, width?, height?, depth?, radius?, tubeRadius?, segments?, rings?, position? [x,y,z], baseColor?, metallic?, roughness?, emissive?, castsShadows?, acceptsShadows?, acceptsLights?}", + has_comp, + new_primitive + ), + cmd!( + "layer.newPrimitive", + "New 3D Primitive", + [], + None, + "{kind: cube|sphere|plane|torus|cone|cylinder, name?, width?, height?, depth?, radius?, tubeRadius?, segments?, rings?, position? [x,y,z], baseColor?, metallic?, roughness?, emissive?, castsShadows?, acceptsShadows?, acceptsLights?}", + has_comp, + new_primitive + ), + cmd!("comp.renderer", "Renderer", [], None, "{renderer?: advanced3d|classic3d (omit to read), value?}", has_comp, comp_renderer), + cmd!("layer.environment", "Environment Layer", ["Layer"], None, "{layers?, on?}", has_layers, environment), + cmd!( + "material.set", + "Material Options", + [], + None, + "{layers?, castsShadows?: off|on|only, acceptsShadows?: off|on|only, acceptsLights?, appearsInReflections?, lightTransmission?, ambient?, diffuse?, specularIntensity?, specularShininess?, metal?, baseColor?, metallic?, roughness?, emissive?, time?}", + has_layers, + material_set + ), + cmd!("material.revealSource", "Reveal Material Source in Project", ["Layer", "Material"], None, "{layer?}", has_layers, material_reveal), + cmd!("material.reset", "Reset Material", ["Layer", "Material"], None, "{layers?}", has_model_or_layers, material_reset), + cmd!( + "material.duplicateAssign", + "Duplicate and Assign Material", + ["Layer", "Material"], + None, + "{layers? (first = source)}", + has_layers, + material_duplicate + ), + ] +} diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f7868d9c..01880a53 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -29,11 +29,9 @@ pub fn specs() -> Vec { stub!("keys.selectLabelGroup", "Select Keyframe Label Group", [], "{scope}"), // Layer ▸ New (3D / content-aware fill). stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), - stub!("layer.new3dPrimitive", "3D Primitive", [], "{kind: cube|sphere|plane|torus|cone|cylinder}"), // Mask options not in the mask model yet; pen / vertex editing (timeline milestone). // Time (timeline milestone). stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), - stub!("layer.environment", "Environment Layer", ["Layer"], "{}"), stub!("layer.create", "Create", [], "{op}"), stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), @@ -47,9 +45,6 @@ pub fn specs() -> Vec { stub!("light.fromModel", "Create Lights from 3D Model", ["Layer", "Light"], "{}"), stub!("light.controlWithCamera", "Control Light with Camera", ["Layer", "Light"], "{}"), stub!("light.environmentBackground", "Create Environment Light Background Layer", ["Layer", "Light"], "{}"), - stub!("material.revealSource", "Reveal Material Source in Project", ["Layer", "Material"], "{}"), - stub!("material.reset", "Reset Material", ["Layer", "Material"], "{}"), - stub!("material.duplicateAssign", "Duplicate and Assign Material", ["Layer", "Material"], "{}"), stub!("view.3d.default", "Default", ["View", "Switch 3D View"], "{}"), stub!("view.splitLockedViewer", "Split with New Locked Viewer", ["View"], "{}"), // Color management. diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f64ce817..eb234ea8 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -556,6 +556,8 @@ mod tests_markers; #[cfg(test)] mod tests_menu_cmds; #[cfg(test)] +mod tests_model3d; +#[cfg(test)] mod tests_project_items; #[cfg(test)] mod tests_settings; diff --git a/crates/engine/src/tests_model3d.rs b/crates/engine/src/tests_model3d.rs new file mode 100644 index 00000000..51f9173d --- /dev/null +++ b/crates/engine/src/tests_model3d.rs @@ -0,0 +1,194 @@ +//! Advanced 3D commands: model import → model layers, primitives, the renderer switch with +//! Geometry Options, Environment Layer and Material commands (undo/redo, serde). + +use std::sync::Arc; + +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{Footage, FootageKind, ItemId, LayerId, LayerSource, PrimitiveKind, Project, Renderer}; +use effectcraft_render::FootageSource; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::json; + +use crate::{Importer, Session}; + +fn fixture(name: &str) -> String { + format!("{}/../model/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) +} + +/// Loads models from disk (what `effectcraft-media` does in the app). +struct DiskModels; + +impl FootageSource for DiskModels { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn model(&self, _: ItemId, f: &Footage) -> Option> { + let dir = std::path::Path::new(&f.path).parent()?.to_path_buf(); + let bytes = std::fs::read(&f.path).ok()?; + effectcraft_model::load(&f.path, &bytes, &|u| std::fs::read(dir.join(u)).ok()).ok().map(Arc::new) + } +} + +struct ModelImporter; + +impl Importer for ModelImporter { + fn probe(&self, path: &str) -> Result { + effectcraft_model::format_of(path).ok_or("not a model")?; + Ok(Footage { + path: path.into(), + kind: FootageKind::Model, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + native_rate: None, + duration: Tick::ZERO, + has_video: false, + has_audio: false, + alpha: Default::default(), + premul_color: [0.0; 3], + loop_count: 1, + codec: "glTF".into(), + missing: false, + sequence: vec![], + color_profile: None, + }) + } +} + +fn session() -> Session { + let mut s = Session { footage: Arc::new(DiskModels), importer: Some(Arc::new(ModelImporter)), ..Default::default() }; + s.execute("comp.new", json!({"name": "C", "width": 640, "height": 360, "duration": 2})).unwrap(); + s +} + +fn layer(s: &Session, id: u64) -> effectcraft_project::Layer { + s.active_comp().unwrap().layer(LayerId(id)).unwrap().clone() +} + +#[test] +fn import_gltf_and_add_model_layer() { + let mut s = session(); + let r = s.execute("file.import", json!({"paths": [fixture("quad.gltf")]})).unwrap(); + let item = r["items"][0].as_u64().unwrap(); + assert_eq!(s.project.item(ItemId(item)).unwrap().type_name(), "3D Model"); + // Adding the item to the comp makes a model layer (3D, fitted, with the clip popup). + let id = s.execute("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + let l = layer(&s, id); + assert_eq!(l.source, LayerSource::Model { item: ItemId(item) }); + assert!(l.switches.three_d); + assert!(l.props.sub("masks").is_none() && l.props.sub("materialOptions").is_some()); + // Bounds 2 units tall → half the comp height. + let scale = l.props.prop("geometryOptions/unitScale").unwrap().value.as_f64(); + assert!((scale - 90.0).abs() < 1e-6, "{scale}"); + let anim = l.props.prop("geometryOptions/animation").unwrap(); + assert_eq!(anim.value, KV::Enum(1)); + match &anim.ui { + effectcraft_project::ParamUi::Popup { options } => assert_eq!(options, &vec!["None".to_string(), "Move".to_string()]), + u => panic!("{u:?}"), + } + s.undo(); + assert!(s.active_comp().unwrap().layers.is_empty()); + s.redo(); + assert_eq!(s.active_comp().unwrap().layers.len(), 1); + // layer.newModel imports and places in one step. + let r = s.execute("layer.newModel", json!({"path": fixture("cube.obj")})).unwrap(); + assert!(r["layer"].as_u64().is_some()); + assert!(s.execute("layer.newModel", json!({"path": "/nonexistent/thing.png"})).is_err()); +} + +#[test] +fn primitives_commands() { + let mut s = session(); + for k in PrimitiveKind::ALL { + let r = s.execute("layer.new3dPrimitive", json!({"kind": k.label().to_lowercase()})).unwrap(); + let l = layer(&s, r["layer"].as_u64().unwrap()); + assert_eq!(l.source, LayerSource::Primitive { kind: k }); + assert_eq!(l.name, k.label()); + assert!(l.switches.three_d); + assert!(l.props.sub("geometryOptions").is_some()); + } + let r = s + .execute("layer.newPrimitive", json!({"kind": "sphere", "radius": 80, "segments": 12, "baseColor": [1, 0, 0], "metallic": 100, "castsShadows": "on"})) + .unwrap(); + let l = layer(&s, r["layer"].as_u64().unwrap()); + assert_eq!(l.props.prop("geometryOptions/radius").unwrap().value.as_f64(), 80.0); + assert_eq!(l.props.prop("materialOptions/metallic").unwrap().value.as_f64(), 100.0); + assert_eq!(l.props.prop("materialOptions/castsShadows").unwrap().value, KV::Enum(1)); + assert!(s.execute("layer.newPrimitive", json!({"kind": "teapot"})).is_err()); + let n = s.active_comp().unwrap().layers.len(); + s.undo(); + assert_eq!(s.active_comp().unwrap().layers.len(), n - 1); + // Menu entries point at the real command now. + assert!(s.is_enabled("layer.new3dPrimitive")); +} + +#[test] +fn renderer_switch_adds_geometry_options() { + let mut s = session(); + let t = s.execute("layer.newText", json!({"text": "Hi"})).unwrap()["layer"].as_u64().unwrap(); + assert!(layer(&s, t).props.sub("geometryOptions").is_none()); + s.execute("comp.renderer", json!({"renderer": "advanced3d"})).unwrap(); + assert_eq!(s.active_comp().unwrap().renderer, Renderer::Advanced3D); + let g = layer(&s, t); + let g = g.props.sub("geometryOptions").unwrap(); + let ids: Vec<&str> = g.children.iter().map(|c| c.match_id()).collect(); + assert_eq!(ids, vec!["bevelStyle", "bevelDepth", "holeBevelDepth", "extrusionDepth"]); + // New shape layers in the Advanced 3D comp get them too. + let sh = s.execute("layer.newShape", json!({})).unwrap()["layer"].as_u64().unwrap(); + assert!(layer(&s, sh).props.sub("geometryOptions").is_some()); + assert_eq!(s.execute("comp.renderer", json!({})).unwrap()["renderer"], "advanced3d"); + s.undo(); + s.undo(); + assert_eq!(s.active_comp().unwrap().renderer, Renderer::Classic3D); + assert!(layer(&s, t).props.sub("geometryOptions").is_none()); + s.redo(); + assert_eq!(s.active_comp().unwrap().renderer, Renderer::Advanced3D); + assert!(s.execute("comp.renderer", json!({"renderer": "raytraced"})).is_err()); + // Extrusion values are ordinary properties. + s.execute("prop.set", json!({"layer": t, "path": "geometryOptions/extrusionDepth", "value": 40})).unwrap(); + assert_eq!(layer(&s, t).props.prop("geometryOptions/extrusionDepth").unwrap().value.as_f64(), 40.0); +} + +#[test] +fn environment_layer_toggle() { + let mut s = session(); + let id = s.execute("layer.newSolid", json!({"width": 64, "height": 32})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.environment", json!({"layer": id})).unwrap(); + assert!(layer(&s, id).environment && layer(&s, id).switches.three_d); + s.execute("layer.environment", json!({"layer": id})).unwrap(); + assert!(!layer(&s, id).environment); + s.undo(); + assert!(layer(&s, id).environment); + let cam = s.execute("layer.newCamera", json!({})).unwrap()["layer"].as_u64().unwrap(); + assert!(s.execute("layer.environment", json!({"layer": cam})).is_err()); + // An Environment light sources a layer. + let li = s.execute("layer.newLight", json!({"kind": "Environment"})).unwrap()["layer"].as_u64().unwrap(); + assert!(layer(&s, li).props.prop("lightOptions/source").is_some()); +} + +#[test] +fn material_commands_and_serde() { + let mut s = session(); + let a = s.execute("layer.newPrimitive", json!({"kind": "cube"})).unwrap()["layer"].as_u64().unwrap(); + let b = s.execute("layer.newPrimitive", json!({"kind": "torus"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("material.set", json!({"layers": [a], "baseColor": "#3366ff", "roughness": 20, "acceptsLights": false})).unwrap(); + assert_eq!(layer(&s, a).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 20.0); + assert!(s.execute("material.set", json!({"layers": [a], "castsShadows": "sometimes"})).is_err()); + s.execute("material.duplicateAssign", json!({"layers": [a, b]})).unwrap(); + assert_eq!(layer(&s, b).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 20.0); + assert_eq!(layer(&s, b).props.prop("materialOptions/acceptsLights").unwrap().value, KV::Bool(false)); + s.execute("material.reset", json!({"layers": [a]})).unwrap(); + assert_eq!(layer(&s, a).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 50.0); + s.undo(); + assert_eq!(layer(&s, a).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 20.0); + assert!(s.execute("material.revealSource", json!({"layer": a})).is_err()); + // The project (model and primitive sources, environment flag) round-trips through JSON. + let r = s.execute("file.import", json!({"paths": [fixture("quad.glb")]})).unwrap(); + let item = r["items"][0].as_u64().unwrap(); + let m = s.execute("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + assert_eq!(s.execute("material.revealSource", json!({"layer": m})).unwrap()["item"], item); + let json = s.project.to_json(); + let back = Project::from_json(&json).unwrap(); + assert_eq!(&back, &*s.project); +} diff --git a/crates/lottie/src/export.rs b/crates/lottie/src/export.rs index fa5447d8..eeb6e273 100644 --- a/crates/lottie/src/export.rs +++ b/crates/lottie/src/export.rs @@ -400,6 +400,10 @@ impl Doc<'_> { self.ex.warn(format!("{name}: cameras and lights are not exported (3D; exported as a hidden null to keep parenting)")); hidden = true; } + LayerSource::Model { .. } | LayerSource::Primitive { .. } => { + self.ex.warn(format!("{name}: 3D model layers are not part of Lottie (layer exported as a hidden null)")); + hidden = true; + } } o.insert("ty".into(), json!(ty)); if let Some(p) = l.parent.and_then(|p| comp.index_of(p)) { diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 89992569..010a6414 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -14,6 +14,7 @@ effectcraft-time = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-model = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } filmcraft-frame = { workspace = true } diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 29546f7e..fcc872d1 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -17,7 +17,7 @@ mod pool; mod probe; pub use pool::{DEFAULT_BUDGET, MediaPool, PoolStats}; -pub use probe::{DEFAULT_SEQUENCE_RATE, probe, probe_bytes, probe_single, sequence_files}; +pub use probe::{DEFAULT_SEQUENCE_RATE, probe, probe_bytes, probe_model, probe_single, sequence_files}; /// Errors from probing or decoding footage. #[derive(Debug, thiserror::Error)] @@ -50,10 +50,16 @@ pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; /// File extensions recognised as movies. pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "m4v", "mov", "mkv", "webm"]; +/// File extensions recognised as 3D models (Advanced 3D model layers). +pub const MODEL_EXTENSIONS: &[&str] = &["gltf", "glb", "obj"]; + /// Whether `path` has an extension we can import. pub fn is_importable(path: &std::path::Path) -> bool { let ext = ext_of(path); - STILL_EXTENSIONS.contains(&ext.as_str()) || AUDIO_EXTENSIONS.contains(&ext.as_str()) || VIDEO_EXTENSIONS.contains(&ext.as_str()) + MODEL_EXTENSIONS.contains(&ext.as_str()) + || STILL_EXTENSIONS.contains(&ext.as_str()) + || AUDIO_EXTENSIONS.contains(&ext.as_str()) + || VIDEO_EXTENSIONS.contains(&ext.as_str()) } pub(crate) fn ext_of(path: &std::path::Path) -> String { diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b4cc51ef..a7898fc6 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -122,6 +122,8 @@ struct Inner { sources: Mutex, Option>>, /// In-memory files registered with `add_bytes` (web builds, tests). files: Mutex>>, + /// Parsed 3D models by path (`None`: failed to load; not retried until `forget`). + models: Mutex>>>, cache: Mutex, done: Condvar, budget: AtomicU64, @@ -183,6 +185,7 @@ impl MediaPool { inner: Arc::new(Inner { sources: Mutex::default(), files: Mutex::default(), + models: Mutex::default(), cache: Mutex::default(), done: Condvar::new(), budget: AtomicU64::new(bytes as u64), @@ -234,6 +237,7 @@ impl MediaPool { /// Forget a file (after it changed on disk, or to retry a file that failed to open). pub fn forget(&self, path: &str) { lock(&self.inner.sources).remove(path); + lock(&self.inner.models).remove(path); let mut c = lock(&self.inner.cache); let keys: Vec = c.lru.map.keys().filter(|k| &*k.path == path).cloned().collect(); for k in keys { @@ -305,6 +309,29 @@ impl Inner { self.budget.load(Ordering::Relaxed) as usize } + /// A 3D model file and its sibling resources (buffers, textures, MTL files), parsed once. + fn model(&self, path: &str) -> Option> { + if let Some(m) = lock(&self.models).get(path) { + return m.clone(); + } + let loaded = self.read(path).map_err(|e| e.to_string()).and_then(|bytes| { + let dir = std::path::Path::new(path).parent().map(|d| d.to_path_buf()).unwrap_or_default(); + let resolve = |uri: &str| -> Option> { + let p = dir.join(uri); + self.read(&p.to_string_lossy()).ok().map(|b| b.to_vec()) + }; + effectcraft_model::load(path, &bytes, &resolve).map_err(|e| e.to_string()) + }); + let m = match loaded { + Ok(m) => Some(Arc::new(m)), + Err(e) => { + log::warn!("media: cannot load model {path}: {e}"); + None + } + }; + lock(&self.models).entry(path.to_string()).or_insert(m).clone() + } + fn read(&self, path: &str) -> Result> { if let Some(b) = lock(&self.files).get(path) { return Ok(b.clone()); @@ -359,7 +386,7 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } - FootageKind::Still | FootageKind::Sequence => Loc { key: key(&footage.path, 0), media_t: None }, + FootageKind::Still | FootageKind::Sequence | FootageKind::Model => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); @@ -451,6 +478,12 @@ impl Inner { } impl FootageSource for MediaPool { + fn model(&self, _item: ItemId, footage: &Footage) -> Option> { + if footage.missing || footage.kind != FootageKind::Model { + return None; + } + self.inner.model(&footage.path) + } fn set_cache_budget(&self, bytes: usize) { self.set_budget(bytes); } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 4e1c5611..e97544db 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -28,11 +28,50 @@ pub fn probe(path: impl AsRef) -> Result { pub fn probe_single(path: impl AsRef) -> Result { let path = path.as_ref(); let bytes = std::fs::read(path).map_err(|e| MediaError::Io(format!("{}: {e}", path.display())))?; + if crate::MODEL_EXTENSIONS.contains(&ext_of(path).as_str()) { + let dir = path.parent().map(Path::to_path_buf).unwrap_or_default(); + return probe_model(&path.to_string_lossy(), &bytes, &|uri| std::fs::read(dir.join(uri)).ok()); + } probe_bytes(&path.to_string_lossy(), bytes.into()) } +/// Probe a 3D model (glTF 2.0 / OBJ): a [`FootageKind::Model`] item whose duration is its +/// longest animation. `resolve` reads sibling resources. +pub fn probe_model(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + let m = effectcraft_model::load(path, bytes, resolve).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let s = m.summary(); + let dur = s.animations.iter().map(|a| a.1).fold(0.0f64, f64::max); + let codec = match effectcraft_model::format_of(path) { + Some("obj") => "OBJ", + Some("glb") => "GLB", + _ => "glTF", + }; + Ok(Footage { + path: path.to_string(), + kind: FootageKind::Model, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: DEFAULT_SEQUENCE_RATE, + native_rate: None, + duration: Tick::from_seconds_f64(dur), + has_video: false, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: format!("{codec} ({} meshes, {} triangles)", s.meshes, s.triangles), + missing: false, + sequence: Vec::new(), + color_profile: None, + }) +} + /// Probe in-memory file contents; `path` names the footage (and is stored as `Footage::path`). pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { + if crate::MODEL_EXTENSIONS.contains(&ext_of(Path::new(path)).as_str()) { + return probe_model(path, &bytes, &|_| None); + } if let Ok(fmt) = image::guess_format(&bytes) { return probe_still_bytes(path, &bytes, fmt); } diff --git a/crates/model/Cargo.toml b/crates/model/Cargo.toml new file mode 100644 index 00000000..20a1fde0 --- /dev/null +++ b/crates/model/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "effectcraft-model" +description = "3D models: glTF 2.0 (.gltf/.glb) and Wavefront OBJ/MTL import, PBR metallic-roughness materials, node hierarchy, skinning and animation, parametric primitives, and extruded/bevelled outline meshes" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-geom = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +image = { workspace = true } diff --git a/crates/model/src/extrude.rs b/crates/model/src/extrude.rs new file mode 100644 index 00000000..344c67cb --- /dev/null +++ b/crates/model/src/extrude.rs @@ -0,0 +1,301 @@ +//! Extruded, bevelled meshes from 2D outlines (Advanced 3D text and shape layers' Geometry +//! Options: Bevel Style, Bevel Depth, Hole Bevel Depth, Extrusion Depth). +//! +//! Input outlines are closed polylines in layer space (x right, y down). The mesh is in layer +//! space too: the front cap lies on the layer plane (z = 0, facing the camera, −z), the back cap +//! at z = depth. Each contour gets a ring of vertices per profile step (inset from the outline +//! by the bevel, at a depth along the profile); consecutive rings are joined by quads, the caps +//! are triangulated from the innermost front/back rings with holes. Caps and walls share +//! positions exactly, so the mesh is watertight (welded by position). + +use serde::{Deserialize, Serialize}; + +use crate::Primitive; +use crate::triangulate::{signed_area, triangulate}; + +type P = [f64; 2]; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum BevelStyle { + #[default] + None, + Angular, + Concave, + Convex, +} + +impl BevelStyle { + pub const ALL: [BevelStyle; 4] = [BevelStyle::None, BevelStyle::Angular, BevelStyle::Concave, BevelStyle::Convex]; + pub fn label(self) -> &'static str { + match self { + BevelStyle::None => "None", + BevelStyle::Angular => "Angular", + BevelStyle::Concave => "Concave", + BevelStyle::Convex => "Convex", + } + } + pub fn from_index(i: u32) -> BevelStyle { + BevelStyle::ALL.get(i as usize).copied().unwrap_or_default() + } + /// (inset fraction 1 → 0, depth fraction 0 → 1) along the profile at `t` in 0..1. + fn profile(self, t: f64) -> (f64, f64) { + let th = t * std::f64::consts::FRAC_PI_2; + match self { + BevelStyle::None | BevelStyle::Angular => (1.0 - t, t), + // Quarter circle bulging out towards the outline corner. + BevelStyle::Convex => (1.0 - th.sin(), 1.0 - th.cos()), + // Quarter circle scooped in. + BevelStyle::Concave => (th.cos(), th.sin()), + } + } + fn steps(self) -> usize { + match self { + BevelStyle::None => 0, + BevelStyle::Angular => 1, + _ => 6, + } + } +} + +/// Geometry Options of an extruded layer. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ExtrudeParams { + pub bevel: BevelStyle, + /// Pixels. + pub bevel_depth: f64, + /// Hole bevels as a fraction of the bevel depth (0..1). + pub hole_bevel: f64, + /// Pixels (> 0). + pub depth: f64, +} + +impl Default for ExtrudeParams { + fn default() -> Self { + ExtrudeParams { bevel: BevelStyle::None, bevel_depth: 2.0, hole_bevel: 1.0, depth: 50.0 } + } +} + +/// An outer contour with its holes. +#[derive(Clone, Debug, PartialEq)] +pub struct Outline { + pub outer: Vec

, + pub holes: Vec>, +} + +fn point_in(p: P, c: &[P]) -> bool { + let n = c.len(); + let mut inside = false; + let mut j = n - 1; + for i in 0..n { + let (a, b) = (c[i], c[j]); + if (a[1] > p[1]) != (b[1] > p[1]) && p[0] < (b[0] - a[0]) * (p[1] - a[1]) / (b[1] - a[1]) + a[0] { + inside = !inside; + } + j = i; + } + inside +} + +/// Group closed contours into outlines by nesting depth (even depth = outer, odd = hole of the +/// innermost containing outer contour). Degenerate contours are dropped. +pub fn group_contours(contours: Vec>) -> Vec { + let cs: Vec> = contours + .into_iter() + .map(|mut c| { + c.dedup(); + while c.len() > 1 && c.first() == c.last() { + c.pop(); + } + c + }) + .filter(|c| c.len() >= 3 && signed_area(c).abs() > 1e-9) + .collect(); + let n = cs.len(); + let area: Vec = cs.iter().map(|c| signed_area(c).abs()).collect(); + // Containers of each contour (by a vertex inside; nested contours do not cross). + let parents: Vec> = (0..n) + .map(|i| { + let probe = cs[i][0]; + (0..n).filter(|&j| j != i && area[j] > area[i] && point_in(probe, &cs[j])).collect() + }) + .collect(); + let mut outlines: Vec = vec![]; + let mut slot = vec![usize::MAX; n]; + let mut order: Vec = (0..n).collect(); + order.sort_by_key(|&i| parents[i].len()); + for i in order { + let depth = parents[i].len(); + if depth.is_multiple_of(2) { + slot[i] = outlines.len(); + outlines.push(Outline { outer: cs[i].clone(), holes: vec![] }); + } else if let Some(&p) = parents[i].iter().filter(|&&j| parents[j].len() == depth - 1).min_by(|&&a, &&b| area[a].total_cmp(&area[b])) + && slot[p] != usize::MAX + { + outlines[slot[p]].holes.push(cs[i].clone()); + } + } + outlines +} + +struct Contour { + pts: Vec

, + /// Outward unit normal of edge i (pts[i] → pts[i+1]). + edge_n: Vec

, + /// Miter offset direction per vertex (outward, length ≥ 1). + miter: Vec

, + /// Smooth-shaded vertex (shallow corner). + smooth: Vec, + bevel: f64, +} + +fn contour(mut pts: Vec

, outer: bool, bevel: f64) -> Contour { + // Outer counter-clockwise (in x/y), holes clockwise: the outward normal is (dy, −dx). + let ccw = signed_area(&pts) > 0.0; + if ccw != outer { + pts.reverse(); + } + let n = pts.len(); + let edge_n: Vec

= (0..n) + .map(|i| { + let (a, b) = (pts[i], pts[(i + 1) % n]); + let (dx, dy) = (b[0] - a[0], b[1] - a[1]); + let l = dx.hypot(dy).max(1e-12); + [dy / l, -dx / l] + }) + .collect(); + let mut miter = vec![[0.0; 2]; n]; + let mut smooth = vec![false; n]; + for i in 0..n { + let (n0, n1) = (edge_n[(i + n - 1) % n], edge_n[i]); + let d = n0[0] * n1[0] + n0[1] * n1[1]; + let s = [n0[0] + n1[0], n0[1] + n1[1]]; + let k = 1.0 + d; + miter[i] = if k < 0.25 { + // Very sharp corner: limit the miter (offset along the bisector, length 2.8). + let l = s[0].hypot(s[1]).max(1e-12); + [s[0] / l * 2.8, s[1] / l * 2.8] + } else { + [s[0] / k, s[1] / k] + }; + smooth[i] = d > 0.82; // < ~35° turn + } + Contour { pts, edge_n, miter, smooth, bevel } +} + +/// Rings of a contour: (inset, z) from the front cap to the back cap. +fn rings(style: BevelStyle, bevel: f64, depth: f64) -> Vec<(f64, f64)> { + let b = bevel.clamp(0.0, depth * 0.5); + let steps = if b > 1e-9 { style.steps() } else { 0 }; + if steps == 0 { + return vec![(0.0, 0.0), (0.0, depth)]; + } + let front: Vec<(f64, f64)> = (0..=steps) + .map(|k| { + let (f, g) = style.profile(k as f64 / steps as f64); + (b * f, b * g) + }) + .collect(); + let mut out = front.clone(); + let back: Vec<(f64, f64)> = front.iter().rev().map(|&(i, z)| (i, depth - z)).collect(); + // Skip a duplicate ring where the walls have zero height (bevel = depth / 2). + let start = if (back[0].1 - out[out.len() - 1].1).abs() < 1e-9 { 1 } else { 0 }; + out.extend_from_slice(&back[start..]); + out +} + +/// Build the extruded mesh of outlines (one primitive, material 0). +pub fn extrude(outlines: &[Outline], p: &ExtrudeParams) -> Primitive { + let mut prim = Primitive { material: Some(0), ..Default::default() }; + let depth = p.depth.max(1e-3); + let push = |prim: &mut Primitive, pos: [f64; 3], n: [f64; 3]| -> u32 { + prim.positions.push([pos[0] as f32, pos[1] as f32, pos[2] as f32]); + let l = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt().max(1e-12); + prim.normals.push([(n[0] / l) as f32, (n[1] / l) as f32, (n[2] / l) as f32]); + prim.uvs.push([0.0, 0.0]); + (prim.positions.len() - 1) as u32 + }; + for o in outlines { + let mut cs = vec![contour(o.outer.clone(), true, p.bevel_depth)]; + for h in &o.holes { + cs.push(contour(h.clone(), false, p.bevel_depth * p.hole_bevel.clamp(0.0, 1.0))); + } + // Positions per contour per ring. + let mut ring_pos: Vec>> = vec![]; + let mut ring_par: Vec> = vec![]; + for c in &cs { + let rs = rings(p.bevel, c.bevel, depth); + let pos: Vec> = + rs.iter().map(|&(inset, z)| c.pts.iter().zip(&c.miter).map(|(q, m)| [q[0] - m[0] * inset, q[1] - m[1] * inset, z]).collect()).collect(); + ring_pos.push(pos); + ring_par.push(rs); + } + // Walls and bevels. + for (ci, c) in cs.iter().enumerate() { + let rs = &ring_par[ci]; + let n = c.pts.len(); + let curved = matches!(p.bevel, BevelStyle::Concave | BevelStyle::Convex); + // Profile normal (outward weight, z weight) of segment k. + let seg_n = |k: usize| -> (f64, f64) { + let (i0, z0) = rs[k]; + let (i1, z1) = rs[k + 1]; + let (di, dz) = (i1 - i0, z1 - z0); + let l = di.hypot(dz).max(1e-12); + (dz / l, di / l) + }; + for k in 0..rs.len() - 1 { + let is_wall = (rs[k].0 - rs[k + 1].0).abs() < 1e-12; + let (a0, a1) = (seg_n(k), seg_n(k)); + // Curved bevels: average with the neighbouring bevel segments at shared rings. + let blend = |k2: Option, base: (f64, f64)| -> (f64, f64) { + match k2 { + Some(k2) if curved && !is_wall && k2 < rs.len() - 1 && (rs[k2].0 - rs[k2 + 1].0).abs() >= 1e-12 => { + let o = seg_n(k2); + (base.0 + o.0, base.1 + o.1) + } + _ => base, + } + }; + let pa = blend(k.checked_sub(1), a0); + let pb = blend(Some(k + 1), a1); + for i in 0..n { + let j = (i + 1) % n; + let en = c.edge_n[i]; + let vn = |v: usize| -> P { + if c.smooth[v] { + let m = c.miter[v]; + let l = m[0].hypot(m[1]).max(1e-12); + [m[0] / l, m[1] / l] + } else { + en + } + }; + let (ni, nj) = (vn(i), vn(j)); + let nrm = |xy: P, pr: (f64, f64)| [xy[0] * pr.0, xy[1] * pr.0, pr.1]; + let ai = push(&mut prim, ring_pos[ci][k][i], nrm(ni, pa)); + let aj = push(&mut prim, ring_pos[ci][k][j], nrm(nj, pa)); + let bj = push(&mut prim, ring_pos[ci][k + 1][j], nrm(nj, pb)); + let bi = push(&mut prim, ring_pos[ci][k + 1][i], nrm(ni, pb)); + prim.indices.extend([ai, aj, bj, ai, bj, bi]); + } + } + } + // Caps: triangulate in x/y (counter-clockwise → +z normal: the back cap; the front cap + // is reversed to face the camera). + let outer2: Vec

= ring_pos[0][0].iter().map(|q| [q[0], q[1]]).collect(); + let holes2: Vec> = ring_pos[1..].iter().map(|r| r[0].iter().map(|q| [q[0], q[1]]).collect()).collect(); + let tris = triangulate(&outer2, &holes2); + let flat: Vec<(usize, usize)> = (0..cs.len()).flat_map(|ci| (0..cs[ci].pts.len()).map(move |i| (ci, i))).collect(); + for (front, nz) in [(true, -1.0), (false, 1.0)] { + let base = prim.positions.len() as u32; + for &(ci, i) in &flat { + let r = if front { 0 } else { ring_pos[ci].len() - 1 }; + push(&mut prim, ring_pos[ci][r][i], [0.0, 0.0, nz]); + } + for t in &tris { + let (a, b, c) = (base + t[0] as u32, base + t[1] as u32, base + t[2] as u32); + if front { prim.indices.extend([a, c, b]) } else { prim.indices.extend([a, b, c]) } + } + } + } + prim +} diff --git a/crates/model/src/gltf.rs b/crates/model/src/gltf.rs new file mode 100644 index 00000000..375884bc --- /dev/null +++ b/crates/model/src/gltf.rs @@ -0,0 +1,519 @@ +//! glTF 2.0 import, written from the Khronos glTF 2.0 specification. +//! +//! Supports `.gltf` (JSON with external `.bin`/image files or `data:` URIs) and `.glb` (binary +//! container: a JSON chunk and a BIN chunk). Imports the default scene's node hierarchy (TRS or +//! matrix), triangle primitives (`TRIANGLES`, `TRIANGLE_STRIP`, `TRIANGLE_FAN`) with POSITION, +//! NORMAL, TANGENT, TEXCOORD_0, JOINTS_0 and WEIGHTS_0, PBR metallic-roughness materials +//! (base colour, metallic-roughness, normal, occlusion and emissive textures; alpha modes; +//! double-sided; `KHR_materials_emissive_strength`, `KHR_materials_unlit`), embedded or +//! external PNG/JPEG images with sampler wrap modes, skins and animations (step, linear and +//! cubic-spline translation/rotation/scale). Sparse accessors, morph targets, cameras and +//! lights are ignored. + +use std::sync::Arc; + +use serde_json::Value; + +use crate::{ + AlphaMode, Animation, Channel, ChannelPath, Interp, Mat4, Material, Mesh, Model, ModelError, Node, Primitive, Result, Skin, TexRef, Texture, Wrap, perr, +}; + +const GLB_MAGIC: u32 = 0x4654_6C67; +const CHUNK_JSON: u32 = 0x4E4F_534A; +const CHUNK_BIN: u32 = 0x004E_4942; + +/// Split a GLB container into its JSON text and BIN chunk. +pub fn split_glb(b: &[u8]) -> Result<(&[u8], Option<&[u8]>)> { + let u32_at = |o: usize| -> Option { b.get(o..o + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]])) }; + if u32_at(0) != Some(GLB_MAGIC) { + return Err(perr("not a GLB file")); + } + if u32_at(4) != Some(2) { + return Err(ModelError::Unsupported(format!("GLB version {}", u32_at(4).unwrap_or(0)))); + } + let total = (u32_at(8).unwrap_or(0) as usize).min(b.len()); + let mut o = 12; + let mut json = None; + let mut bin = None; + while o + 8 <= total { + let len = u32_at(o).unwrap_or(0) as usize; + let ty = u32_at(o + 4).unwrap_or(0); + let data = b.get(o + 8..o + 8 + len).ok_or_else(|| perr("GLB chunk runs past the end of the file"))?; + match ty { + CHUNK_JSON if json.is_none() => json = Some(data), + CHUNK_BIN if bin.is_none() => bin = Some(data), + _ => {} + } + o += 8 + len; + } + Ok((json.ok_or_else(|| perr("GLB without a JSON chunk"))?, bin)) +} + +/// Decode standard base64 (with or without padding; whitespace ignored). +pub fn base64_decode(s: &str) -> Option> { + let mut out = Vec::with_capacity(s.len() * 3 / 4); + let mut acc = 0u32; + let mut bits = 0; + for c in s.bytes() { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' | b'-' => 62, + b'/' | b'_' => 63, + b'=' => break, + b' ' | b'\n' | b'\r' | b'\t' => continue, + _ => return None, + } as u32; + acc = (acc << 6) | v; + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + acc &= (1 << bits) - 1; + } + } + Some(out) +} + +/// Percent-decode a relative URI. +pub fn uri_decode(s: &str) -> String { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' + && i + 2 < b.len() + && let Ok(v) = u8::from_str_radix(std::str::from_utf8(&b[i + 1..i + 3]).unwrap_or("zz"), 16) + { + out.push(v); + i += 3; + continue; + } + out.push(b[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// Bytes of a `data:` URI or an external resource. +fn load_uri(uri: &str, resolve: &dyn Fn(&str) -> Option>) -> Result> { + if let Some(rest) = uri.strip_prefix("data:") { + let (meta, data) = rest.split_once(',').ok_or_else(|| perr("malformed data URI"))?; + return if meta.ends_with(";base64") { base64_decode(data).ok_or_else(|| perr("bad base64 in data URI")) } else { Ok(uri_decode(data).into_bytes()) }; + } + let name = uri_decode(uri); + resolve(&name).ok_or(ModelError::Missing(name)) +} + +struct Doc<'a> { + j: &'a Value, + buffers: Vec>, +} + +fn arr<'v>(v: &'v Value, k: &str) -> &'v [Value] { + v.get(k).and_then(Value::as_array).map(Vec::as_slice).unwrap_or(&[]) +} +fn idx(v: &Value, k: &str) -> Option { + v.get(k).and_then(Value::as_u64).map(|x| x as usize) +} +fn num(v: &Value, k: &str, d: f64) -> f64 { + v.get(k).and_then(Value::as_f64).unwrap_or(d) +} +fn nums(v: &Value, k: &str, d: [f64; N]) -> [f64; N] { + let mut out = d; + if let Some(a) = v.get(k).and_then(Value::as_array) { + for (i, x) in a.iter().take(N).enumerate() { + out[i] = x.as_f64().unwrap_or(d[i]); + } + } + out +} + +impl Doc<'_> { + /// An accessor as `f32` rows of `n` components (normalised integers mapped to 0..1 / -1..1). + fn floats(&self, a: usize) -> Result<(Vec, usize)> { + let acc = arr(self.j, "accessors").get(a).ok_or_else(|| perr(format!("no accessor {a}")))?; + let n = match acc.get("type").and_then(Value::as_str).unwrap_or("") { + "SCALAR" => 1, + "VEC2" => 2, + "VEC3" => 3, + "VEC4" => 4, + "MAT2" => 4, + "MAT3" => 9, + "MAT4" => 16, + t => return Err(perr(format!("accessor type `{t}`"))), + }; + let count = idx(acc, "count").unwrap_or(0); + let ct = idx(acc, "componentType").unwrap_or(5126); + let norm = acc.get("normalized").and_then(Value::as_bool).unwrap_or(false); + let csize = match ct { + 5120 | 5121 => 1, + 5122 | 5123 => 2, + 5125 | 5126 => 4, + _ => return Err(perr(format!("componentType {ct}"))), + }; + let mut out = vec![0.0f32; count * n]; + let Some(bv) = idx(acc, "bufferView") else { + // No buffer view: all zeros (sparse substitution is not supported). + return Ok((out, n)); + }; + let view = arr(self.j, "bufferViews").get(bv).ok_or_else(|| perr(format!("no bufferView {bv}")))?; + let buf = self.buffers.get(idx(view, "buffer").unwrap_or(0)).ok_or_else(|| perr("bufferView buffer out of range"))?; + let base = idx(view, "byteOffset").unwrap_or(0) + idx(acc, "byteOffset").unwrap_or(0); + let elem = csize * n; + // Matrices of 1/2-byte components pad columns to 4 bytes; ignored (rare). + let stride = idx(view, "byteStride").filter(|s| *s > 0).unwrap_or(elem); + for i in 0..count { + for c in 0..n { + let o = base + i * stride + c * csize; + let s = buf.get(o..o + csize).ok_or_else(|| perr("accessor reads past its buffer"))?; + out[i * n + c] = match ct { + 5126 => f32::from_le_bytes([s[0], s[1], s[2], s[3]]), + 5125 => u32::from_le_bytes([s[0], s[1], s[2], s[3]]) as f32, + 5123 => { + let v = u16::from_le_bytes([s[0], s[1]]) as f32; + if norm { v / 65535.0 } else { v } + } + 5122 => { + let v = i16::from_le_bytes([s[0], s[1]]) as f32; + if norm { (v / 32767.0).max(-1.0) } else { v } + } + 5121 => { + let v = s[0] as f32; + if norm { v / 255.0 } else { v } + } + _ => { + let v = s[0] as i8 as f32; + if norm { (v / 127.0).max(-1.0) } else { v } + } + }; + } + } + Ok((out, n)) + } + + /// Integer accessor (indices, joints) read exactly. + fn ints(&self, a: usize) -> Result> { + let acc = arr(self.j, "accessors").get(a).ok_or_else(|| perr(format!("no accessor {a}")))?; + let ct = idx(acc, "componentType").unwrap_or(5125); + if ct == 5126 { + return Ok(self.floats(a)?.0.into_iter().map(|v| v as u32).collect()); + } + let n = match acc.get("type").and_then(Value::as_str).unwrap_or("SCALAR") { + "VEC4" => 4, + "VEC3" => 3, + "VEC2" => 2, + _ => 1, + }; + let count = idx(acc, "count").unwrap_or(0); + let csize = match ct { + 5120 | 5121 => 1, + 5122 | 5123 => 2, + _ => 4, + }; + let Some(bv) = idx(acc, "bufferView") else { return Ok(vec![0; count * n]) }; + let view = arr(self.j, "bufferViews").get(bv).ok_or_else(|| perr(format!("no bufferView {bv}")))?; + let buf = self.buffers.get(idx(view, "buffer").unwrap_or(0)).ok_or_else(|| perr("bufferView buffer out of range"))?; + let base = idx(view, "byteOffset").unwrap_or(0) + idx(acc, "byteOffset").unwrap_or(0); + let stride = idx(view, "byteStride").filter(|s| *s > 0).unwrap_or(csize * n); + let mut out = Vec::with_capacity(count * n); + for i in 0..count { + for c in 0..n { + let o = base + i * stride + c * csize; + let s = buf.get(o..o + csize).ok_or_else(|| perr("accessor reads past its buffer"))?; + out.push(match csize { + 1 => s[0] as u32, + 2 => u16::from_le_bytes([s[0], s[1]]) as u32, + _ => u32::from_le_bytes([s[0], s[1], s[2], s[3]]), + }); + } + } + Ok(out) + } +} + +/// Parse a `.gltf` (JSON) or `.glb` file. +pub fn parse(bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + let (json, bin) = if bytes.starts_with(b"glTF") { split_glb(bytes)? } else { (bytes, None) }; + let j: Value = serde_json::from_slice(json).map_err(|e| perr(format!("glTF JSON: {e}")))?; + let ver = j.get("asset").and_then(|a| a.get("version")).and_then(Value::as_str).unwrap_or(""); + if !ver.starts_with('2') { + return Err(ModelError::Unsupported(format!("glTF version `{ver}` (2.0 required)"))); + } + for ext in arr(&j, "extensionsRequired") { + let e = ext.as_str().unwrap_or(""); + if !matches!(e, "KHR_materials_unlit" | "KHR_materials_emissive_strength") { + return Err(ModelError::Unsupported(format!("required extension {e}"))); + } + } + let mut buffers = vec![]; + for (i, b) in arr(&j, "buffers").iter().enumerate() { + let data = match b.get("uri").and_then(Value::as_str) { + Some(u) => load_uri(u, resolve)?, + None if i == 0 => bin.map(<[u8]>::to_vec).ok_or_else(|| perr("buffer 0 has no uri and the file has no BIN chunk"))?, + None => vec![], + }; + buffers.push(data); + } + let d = Doc { j: &j, buffers }; + + // Images → textures (decoded once per image; samplers set the wrap modes). + let mut images: Vec>> = vec![]; + for im in arr(&j, "images") { + let data = if let Some(u) = im.get("uri").and_then(Value::as_str) { + load_uri(u, resolve).ok() + } else if let Some(bv) = idx(im, "bufferView") { + arr(&j, "bufferViews").get(bv).and_then(|v| { + let b = d.buffers.get(idx(v, "buffer").unwrap_or(0))?; + let o = idx(v, "byteOffset").unwrap_or(0); + b.get(o..o + idx(v, "byteLength").unwrap_or(0)).map(<[u8]>::to_vec) + }) + } else { + None + }; + images.push(data.and_then(|b| Texture::decode(&b).ok()).map(Arc::new)); + } + let wrap = |v: u64| match v { + 33071 => Wrap::Clamp, + 33648 => Wrap::Mirror, + _ => Wrap::Repeat, + }; + let mut textures: Vec> = vec![]; + let mut tex_map: Vec> = vec![]; + for t in arr(&j, "textures") { + let Some(img) = idx(t, "source").and_then(|s| images.get(s).cloned().flatten()) else { + tex_map.push(None); + continue; + }; + let samp = idx(t, "sampler").and_then(|s| arr(&j, "samplers").get(s)); + let ws = samp.and_then(|s| s.get("wrapS")).and_then(Value::as_u64).unwrap_or(10497); + let wt = samp.and_then(|s| s.get("wrapT")).and_then(Value::as_u64).unwrap_or(10497); + let w = [wrap(ws), wrap(wt)]; + let tex = if img.wrap == w { img } else { Arc::new(Texture { wrap: w, ..(*img).clone() }) }; + tex_map.push(Some(textures.len())); + textures.push(tex); + } + let tref = |v: Option<&Value>| -> Option { + let v = v?; + let t = tex_map.get(idx(v, "index")?).copied().flatten()?; + Some(TexRef { texture: t, uv_set: idx(v, "texCoord").unwrap_or(0) as u32 }) + }; + + let mut materials = vec![]; + for m in arr(&j, "materials") { + let pbr = m.get("pbrMetallicRoughness").cloned().unwrap_or(Value::Null); + let bc = nums(&pbr, "baseColorFactor", [1.0; 4]); + let ext = m.get("extensions"); + let strength = ext.and_then(|e| e.get("KHR_materials_emissive_strength")).map_or(1.0, |e| num(e, "emissiveStrength", 1.0)); + let em = nums(m, "emissiveFactor", [0.0; 3]); + materials.push(Material { + name: m.get("name").and_then(Value::as_str).unwrap_or("").to_string(), + base_color: bc.map(|v| v as f32), + base_color_tex: tref(pbr.get("baseColorTexture")), + metallic: num(&pbr, "metallicFactor", 1.0) as f32, + roughness: num(&pbr, "roughnessFactor", 1.0) as f32, + metallic_roughness_tex: tref(pbr.get("metallicRoughnessTexture")), + normal_tex: tref(m.get("normalTexture")), + normal_scale: m.get("normalTexture").map_or(1.0, |n| num(n, "scale", 1.0)) as f32, + occlusion_tex: tref(m.get("occlusionTexture")), + occlusion_strength: m.get("occlusionTexture").map_or(1.0, |n| num(n, "strength", 1.0)) as f32, + emissive: [(em[0] * strength) as f32, (em[1] * strength) as f32, (em[2] * strength) as f32], + emissive_tex: tref(m.get("emissiveTexture")), + alpha_mode: match m.get("alphaMode").and_then(Value::as_str) { + Some("MASK") => AlphaMode::Mask, + Some("BLEND") => AlphaMode::Blend, + _ => AlphaMode::Opaque, + }, + alpha_cutoff: num(m, "alphaCutoff", 0.5) as f32, + double_sided: m.get("doubleSided").and_then(Value::as_bool).unwrap_or(false), + unlit: ext.and_then(|e| e.get("KHR_materials_unlit")).is_some(), + }); + } + + let mut meshes = vec![]; + for m in arr(&j, "meshes") { + let mut mesh = Mesh { name: m.get("name").and_then(Value::as_str).unwrap_or("").to_string(), primitives: vec![] }; + for p in arr(m, "primitives") { + if let Some(prim) = primitive(&d, p, &materials)? { + mesh.primitives.push(prim); + } + } + meshes.push(mesh); + } + + let mut nodes = vec![]; + for n in arr(&j, "nodes") { + let matrix = n.get("matrix").and_then(Value::as_array).filter(|a| a.len() == 16).map(|a| { + // Column-major in the file. + let v: Vec = a.iter().map(|x| x.as_f64().unwrap_or(0.0)).collect(); + let mut m = [[0.0; 4]; 4]; + for c in 0..4 { + for r in 0..4 { + m[r][c] = v[c * 4 + r]; + } + } + Mat4(m) + }); + nodes.push(Node { + name: n.get("name").and_then(Value::as_str).unwrap_or("").to_string(), + translation: nums(n, "translation", [0.0; 3]), + rotation: nums(n, "rotation", [0.0, 0.0, 0.0, 1.0]), + scale: nums(n, "scale", [1.0; 3]), + matrix, + mesh: idx(n, "mesh").filter(|&m| m < meshes.len()), + skin: idx(n, "skin"), + children: arr(n, "children").iter().filter_map(Value::as_u64).map(|c| c as usize).collect(), + }); + } + + let scene = idx(&j, "scene").unwrap_or(0); + let roots: Vec = match arr(&j, "scenes").get(scene) { + Some(s) => arr(s, "nodes").iter().filter_map(Value::as_u64).map(|c| c as usize).collect(), + None => { + // No scene: every node that is nobody's child. + let mut child = vec![false; nodes.len()]; + for n in &nodes { + for &c in &n.children { + if c < child.len() { + child[c] = true; + } + } + } + (0..nodes.len()).filter(|&i| !child[i]).collect() + } + }; + + let mut skins = vec![]; + for s in arr(&j, "skins") { + let joints: Vec = arr(s, "joints").iter().filter_map(Value::as_u64).map(|c| c as usize).collect(); + let inverse_bind = match idx(s, "inverseBindMatrices") { + Some(a) => { + let (f, _) = d.floats(a)?; + f.chunks_exact(16) + .map(|v| { + let mut m = [[0.0; 4]; 4]; + for c in 0..4 { + for r in 0..4 { + m[r][c] = v[c * 4 + r] as f64; + } + } + Mat4(m) + }) + .collect() + } + None => vec![Mat4::IDENTITY; joints.len()], + }; + skins.push(Skin { joints, inverse_bind }); + } + + let mut animations = vec![]; + for a in arr(&j, "animations") { + let samplers = arr(a, "samplers"); + let mut anim = Animation { name: a.get("name").and_then(Value::as_str).unwrap_or("").to_string(), channels: vec![] }; + for c in arr(a, "channels") { + let Some(t) = c.get("target") else { continue }; + let Some(node) = idx(t, "node") else { continue }; + let path = match t.get("path").and_then(Value::as_str) { + Some("translation") => ChannelPath::Translation, + Some("rotation") => ChannelPath::Rotation, + Some("scale") => ChannelPath::Scale, + _ => continue, + }; + let Some(s) = idx(c, "sampler").and_then(|s| samplers.get(s)) else { continue }; + let interp = match s.get("interpolation").and_then(Value::as_str) { + Some("STEP") => Interp::Step, + Some("CUBICSPLINE") => Interp::CubicSpline, + _ => Interp::Linear, + }; + let (Some(input), Some(output)) = (idx(s, "input"), idx(s, "output")) else { continue }; + let (times, _) = d.floats(input)?; + let (values, _) = d.floats(output)?; + anim.channels.push(Channel { node, path, interp, times, values }); + } + animations.push(anim); + } + + Ok(Model { meshes, nodes, roots, materials, textures, skins, animations }) +} + +fn primitive(d: &Doc, p: &Value, materials: &[Material]) -> Result> { + let mode = idx(p, "mode").unwrap_or(4); + if !matches!(mode, 4..=6) { + return Ok(None); + } + let attrs = p.get("attributes").ok_or_else(|| perr("primitive without attributes"))?; + let Some(pa) = idx(attrs, "POSITION") else { return Ok(None) }; + let (pos, _) = d.floats(pa)?; + let positions: Vec<[f32; 3]> = pos.chunks_exact(3).map(|c| [c[0], c[1], c[2]]).collect(); + let n = positions.len(); + let vec3s = |a: Option| -> Result> { + Ok(match a { + Some(a) => d.floats(a)?.0.chunks_exact(3).map(|c| [c[0], c[1], c[2]]).collect(), + None => vec![], + }) + }; + let mut normals = vec3s(idx(attrs, "NORMAL"))?; + let uvs: Vec<[f32; 2]> = match idx(attrs, "TEXCOORD_0") { + Some(a) => d.floats(a)?.0.chunks_exact(2).map(|c| [c[0], c[1]]).collect(), + None => vec![], + }; + let tangents: Vec<[f32; 4]> = match idx(attrs, "TANGENT") { + Some(a) => d.floats(a)?.0.chunks_exact(4).map(|c| [c[0], c[1], c[2], c[3]]).collect(), + None => vec![], + }; + let joints: Vec<[u16; 4]> = match idx(attrs, "JOINTS_0") { + Some(a) => d.ints(a)?.chunks_exact(4).map(|c| [c[0] as u16, c[1] as u16, c[2] as u16, c[3] as u16]).collect(), + None => vec![], + }; + let weights: Vec<[f32; 4]> = match idx(attrs, "WEIGHTS_0") { + Some(a) => d.floats(a)?.0.chunks_exact(4).map(|c| [c[0], c[1], c[2], c[3]]).collect(), + None => vec![], + }; + let raw: Vec = match idx(p, "indices") { + Some(a) => d.ints(a)?, + None => (0..n as u32).collect(), + }; + if raw.iter().any(|&i| i as usize >= n) { + return Err(perr("index out of range")); + } + let mut indices = Vec::with_capacity(raw.len()); + match mode { + 5 => { + for i in 2..raw.len() { + let (a, b, c) = (raw[i - 2], raw[i - 1], raw[i]); + if i % 2 == 0 { indices.extend([a, b, c]) } else { indices.extend([b, a, c]) } + } + } + 6 => { + for i in 2..raw.len() { + indices.extend([raw[0], raw[i - 1], raw[i]]); + } + } + _ => indices.extend(raw.chunks_exact(3).flatten()), + } + if normals.len() != n { + normals.clear(); + } + let material = idx(p, "material").filter(|&m| m < materials.len()); + let skinned = joints.len() == n && weights.len() == n; + let mut prim = Primitive { + positions, + normals, + uvs: if uvs.len() == n { uvs } else { vec![] }, + tangents: if tangents.len() == n { tangents } else { vec![] }, + joints: if skinned { joints } else { vec![] }, + weights: if skinned { weights } else { vec![] }, + indices, + material, + }; + if prim.normals.is_empty() { + prim.flat_normals(); + prim.tangents.clear(); + } + let needs_tangents = material.is_some_and(|m| materials[m].normal_tex.is_some()); + if needs_tangents && prim.tangents.is_empty() { + prim.compute_tangents(); + } + Ok(Some(prim)) +} diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs new file mode 100644 index 00000000..290e5500 --- /dev/null +++ b/crates/model/src/lib.rs @@ -0,0 +1,691 @@ +//! 3D models for the Advanced 3D renderer. +//! +//! - [`gltf`]: glTF 2.0 import (`.gltf` with external or embedded buffers, `.glb`), written from +//! the Khronos glTF 2.0 specification: meshes, PBR metallic-roughness materials (base colour, +//! metallic-roughness, normal, occlusion and emissive maps), the node hierarchy, skins and +//! animations. +//! - [`obj`]: Wavefront OBJ geometry with MTL materials (mapped onto the metallic-roughness model). +//! - [`prim`]: parametric primitives (cube, sphere, plane, torus, cone, cylinder). +//! - [`extrude`]: extruded, bevelled meshes from 2D outlines (text and shape layers), with +//! [`triangulate`] for the caps. +//! +//! Models are in their own units with +Y up (glTF's convention); the renderer maps them into +//! After Effects' Y-down pixel space. + +pub mod extrude; +pub mod gltf; +pub mod obj; +pub mod prim; +pub mod triangulate; + +use std::sync::Arc; + +pub use effectcraft_geom::{Mat4, Vec3, vec3}; + +#[derive(Debug, thiserror::Error)] +pub enum ModelError { + #[error("{0}")] + Parse(String), + #[error("missing resource `{0}`")] + Missing(String), + #[error("unsupported: {0}")] + Unsupported(String), +} + +pub type Result = std::result::Result; + +pub(crate) fn perr(s: impl Into) -> ModelError { + ModelError::Parse(s.into()) +} + +/// Texture coordinate wrapping (glTF sampler wrap modes). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Wrap { + #[default] + Repeat, + Clamp, + Mirror, +} + +/// A decoded texture: straight RGBA, 0..1, as stored in the file (colour textures are still +/// sRGB-encoded; the renderer linearises them). +#[derive(Clone, Debug, PartialEq)] +pub struct Texture { + pub width: u32, + pub height: u32, + pub data: Vec<[f32; 4]>, + pub wrap: [Wrap; 2], +} + +impl Texture { + pub fn solid(c: [f32; 4]) -> Texture { + Texture { width: 1, height: 1, data: vec![c], wrap: [Wrap::Repeat; 2] } + } + /// Decode a PNG or JPEG. + pub fn decode(bytes: &[u8]) -> Result { + let img = image::load_from_memory(bytes).map_err(|e| perr(format!("image: {e}")))?.to_rgba32f(); + let (w, h) = (img.width(), img.height()); + let data = img.pixels().map(|p| p.0).collect(); + Ok(Texture { width: w, height: h, data, wrap: [Wrap::Repeat; 2] }) + } +} + +/// How a material's alpha is used (glTF `alphaMode`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AlphaMode { + #[default] + Opaque, + Mask, + Blend, +} + +/// A texture reference of a material. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TexRef { + pub texture: usize, + /// TEXCOORD set (only set 0 is imported; others fall back to set 0). + pub uv_set: u32, +} + +/// A metallic-roughness PBR material (glTF 2.0 core material model). +#[derive(Clone, Debug, PartialEq)] +pub struct Material { + pub name: String, + /// Linear RGBA factor. + pub base_color: [f32; 4], + pub base_color_tex: Option, + pub metallic: f32, + pub roughness: f32, + /// Roughness in G, metalness in B. + pub metallic_roughness_tex: Option, + pub normal_tex: Option, + pub normal_scale: f32, + /// Occlusion in R. + pub occlusion_tex: Option, + pub occlusion_strength: f32, + /// Linear emissive factor (× KHR_materials_emissive_strength). + pub emissive: [f32; 3], + pub emissive_tex: Option, + pub alpha_mode: AlphaMode, + pub alpha_cutoff: f32, + pub double_sided: bool, + /// KHR_materials_unlit. + pub unlit: bool, +} + +impl Default for Material { + fn default() -> Self { + Material { + name: String::new(), + base_color: [1.0; 4], + base_color_tex: None, + metallic: 1.0, + roughness: 1.0, + metallic_roughness_tex: None, + normal_tex: None, + normal_scale: 1.0, + occlusion_tex: None, + occlusion_strength: 1.0, + emissive: [0.0; 3], + emissive_tex: None, + alpha_mode: AlphaMode::Opaque, + alpha_cutoff: 0.5, + double_sided: false, + unlit: false, + } + } +} + +impl Material { + /// A plain dielectric material of a colour (primitives, OBJ without MTL). + pub fn plain(base: [f32; 4], metallic: f32, roughness: f32) -> Material { + Material { base_color: base, metallic, roughness, ..Material::default() } + } +} + +/// Triangles with one material. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Primitive { + pub positions: Vec<[f32; 3]>, + pub normals: Vec<[f32; 3]>, + /// TEXCOORD_0 (empty when the primitive has none). + pub uvs: Vec<[f32; 2]>, + /// Tangents (xyz, w = bitangent sign); empty when not needed. + pub tangents: Vec<[f32; 4]>, + /// JOINTS_0 / WEIGHTS_0 (skinned meshes; empty otherwise). + pub joints: Vec<[u16; 4]>, + pub weights: Vec<[f32; 4]>, + /// Triangle list. + pub indices: Vec, + pub material: Option, +} + +impl Primitive { + pub fn triangle_count(&self) -> usize { + self.indices.len() / 3 + } + /// Smooth vertex normals from the triangles (area weighted), for meshes without normals. + pub fn compute_normals(&mut self) { + let mut n = vec![[0.0f32; 3]; self.positions.len()]; + for t in self.indices.chunks_exact(3) { + let (a, b, c) = (self.positions[t[0] as usize], self.positions[t[1] as usize], self.positions[t[2] as usize]); + let f = cross(sub(b, a), sub(c, a)); + for &i in t { + let v = &mut n[i as usize]; + v[0] += f[0]; + v[1] += f[1]; + v[2] += f[2]; + } + } + self.normals = n.into_iter().map(|v| normalize(v).unwrap_or([0.0, 0.0, 1.0])).collect(); + } + /// Split every triangle into its own vertices with a face normal (glTF: flat normals when a + /// primitive has none). + pub fn flat_normals(&mut self) { + let mut p = Primitive { material: self.material, ..Default::default() }; + for t in self.indices.chunks_exact(3) { + let (a, b, c) = (self.positions[t[0] as usize], self.positions[t[1] as usize], self.positions[t[2] as usize]); + let f = normalize(cross(sub(b, a), sub(c, a))).unwrap_or([0.0, 0.0, 1.0]); + for &i in t { + let i = i as usize; + p.indices.push(p.positions.len() as u32); + p.positions.push(self.positions[i]); + p.normals.push(f); + if !self.uvs.is_empty() { + p.uvs.push(self.uvs[i]); + } + if !self.joints.is_empty() { + p.joints.push(self.joints[i]); + p.weights.push(self.weights[i]); + } + } + } + *self = p; + } + /// Per-vertex tangents from the UV layout (accumulated per triangle, Gram-Schmidt + /// orthogonalised against the normal, handedness in w). + pub fn compute_tangents(&mut self) { + let n = self.positions.len(); + if self.uvs.len() != n || self.normals.len() != n { + return; + } + let mut tan = vec![[0.0f32; 3]; n]; + let mut bit = vec![[0.0f32; 3]; n]; + for t in self.indices.chunks_exact(3) { + let (i0, i1, i2) = (t[0] as usize, t[1] as usize, t[2] as usize); + let (p0, p1, p2) = (self.positions[i0], self.positions[i1], self.positions[i2]); + let (w0, w1, w2) = (self.uvs[i0], self.uvs[i1], self.uvs[i2]); + let e1 = sub(p1, p0); + let e2 = sub(p2, p0); + let (du1, dv1, du2, dv2) = (w1[0] - w0[0], w1[1] - w0[1], w2[0] - w0[0], w2[1] - w0[1]); + let det = du1 * dv2 - du2 * dv1; + if det.abs() < 1e-12 { + continue; + } + let r = 1.0 / det; + let sdir = [(e1[0] * dv2 - e2[0] * dv1) * r, (e1[1] * dv2 - e2[1] * dv1) * r, (e1[2] * dv2 - e2[2] * dv1) * r]; + let tdir = [(e2[0] * du1 - e1[0] * du2) * r, (e2[1] * du1 - e1[1] * du2) * r, (e2[2] * du1 - e1[2] * du2) * r]; + for i in [i0, i1, i2] { + for k in 0..3 { + tan[i][k] += sdir[k]; + bit[i][k] += tdir[k]; + } + } + } + self.tangents = (0..n) + .map(|i| { + let nn = self.normals[i]; + let t = tan[i]; + let d = dot(nn, t); + let o = [t[0] - nn[0] * d, t[1] - nn[1] * d, t[2] - nn[2] * d]; + let o = normalize(o).unwrap_or_else(|| any_perpendicular(nn)); + let w = if dot(cross(nn, o), bit[i]) < 0.0 { -1.0 } else { 1.0 }; + [o[0], o[1], o[2], w] + }) + .collect(); + } + /// Axis-aligned bounds of the positions. + pub fn bounds(&self) -> Option<([f32; 3], [f32; 3])> { + let mut it = self.positions.iter(); + let f = *it.next()?; + let (mut lo, mut hi) = (f, f); + for p in it { + for k in 0..3 { + lo[k] = lo[k].min(p[k]); + hi[k] = hi[k].max(p[k]); + } + } + Some((lo, hi)) + } +} + +pub(crate) fn sub(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub(crate) fn cross(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub(crate) fn dot(a: [f32; 3], b: [f32; 3]) -> f32 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub(crate) fn normalize(a: [f32; 3]) -> Option<[f32; 3]> { + let l = dot(a, a).sqrt(); + (l > 1e-20 && l.is_finite()).then(|| [a[0] / l, a[1] / l, a[2] / l]) +} +fn any_perpendicular(n: [f32; 3]) -> [f32; 3] { + let a = if n[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + normalize(cross(n, a)).unwrap_or([1.0, 0.0, 0.0]) +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Mesh { + pub name: String, + pub primitives: Vec, +} + +/// A node of the scene hierarchy: local transform (TRS or a matrix), an optional mesh and skin. +#[derive(Clone, Debug, PartialEq)] +pub struct Node { + pub name: String, + pub translation: [f64; 3], + /// Unit quaternion (x, y, z, w). + pub rotation: [f64; 4], + pub scale: [f64; 3], + /// A fixed local matrix (nodes given by `matrix` are not animated). + pub matrix: Option, + pub mesh: Option, + pub skin: Option, + pub children: Vec, +} + +impl Default for Node { + fn default() -> Self { + Node { + name: String::new(), + translation: [0.0; 3], + rotation: [0.0, 0.0, 0.0, 1.0], + scale: [1.0; 3], + matrix: None, + mesh: None, + skin: None, + children: vec![], + } + } +} + +impl Node { + pub fn local(&self) -> Mat4 { + self.matrix.unwrap_or_else(|| trs(self.translation, self.rotation, self.scale)) + } +} + +/// Translation × rotation (quaternion x, y, z, w) × scale. +pub fn trs(t: [f64; 3], q: [f64; 4], s: [f64; 3]) -> Mat4 { + let r = quat_matrix(q); + let mut m = r; + for i in 0..3 { + for j in 0..3 { + m.0[i][j] *= s[j]; + } + m.0[i][3] = t[i]; + } + m +} + +/// Rotation matrix of a quaternion (x, y, z, w); normalised first. +pub fn quat_matrix(q: [f64; 4]) -> Mat4 { + let l = (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]).sqrt(); + let [x, y, z, w] = if l > 1e-12 { [q[0] / l, q[1] / l, q[2] / l, q[3] / l] } else { [0.0, 0.0, 0.0, 1.0] }; + Mat4([ + [1.0 - 2.0 * (y * y + z * z), 2.0 * (x * y - z * w), 2.0 * (x * z + y * w), 0.0], + [2.0 * (x * y + z * w), 1.0 - 2.0 * (x * x + z * z), 2.0 * (y * z - x * w), 0.0], + [2.0 * (x * z - y * w), 2.0 * (y * z + x * w), 1.0 - 2.0 * (x * x + y * y), 0.0], + [0.0, 0.0, 0.0, 1.0], + ]) +} + +/// Spherical linear interpolation of unit quaternions (shortest path). +pub fn slerp(a: [f64; 4], b: [f64; 4], t: f64) -> [f64; 4] { + let mut d = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3]; + let mut b = b; + if d < 0.0 { + d = -d; + b = [-b[0], -b[1], -b[2], -b[3]]; + } + let (ka, kb) = if d > 0.9995 { + (1.0 - t, t) + } else { + let th = d.clamp(-1.0, 1.0).acos(); + let s = th.sin(); + (((1.0 - t) * th).sin() / s, (t * th).sin() / s) + }; + let q = [a[0] * ka + b[0] * kb, a[1] * ka + b[1] * kb, a[2] * ka + b[2] * kb, a[3] * ka + b[3] * kb]; + let l = (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]).sqrt().max(1e-12); + [q[0] / l, q[1] / l, q[2] / l, q[3] / l] +} + +/// A skin: joint nodes and their inverse bind matrices. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Skin { + pub joints: Vec, + pub inverse_bind: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChannelPath { + Translation, + Rotation, + Scale, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Interp { + Step, + Linear, + CubicSpline, +} + +/// One animated node property. +#[derive(Clone, Debug, PartialEq)] +pub struct Channel { + pub node: usize, + pub path: ChannelPath, + pub interp: Interp, + /// Key times (seconds). + pub times: Vec, + /// Values, `width` floats per key (cubic spline: in-tangent, value, out-tangent per key). + pub values: Vec, +} + +impl Channel { + fn width(&self) -> usize { + match self.path { + ChannelPath::Rotation => 4, + _ => 3, + } + } + /// The value at time `t` (seconds), clamped to the key range. + pub fn sample(&self, t: f64) -> Option<[f64; 4]> { + let n = self.times.len(); + let w = self.width(); + let stride = if self.interp == Interp::CubicSpline { 3 * w } else { w }; + let off = if self.interp == Interp::CubicSpline { w } else { 0 }; + if n == 0 || self.values.len() < n * stride { + return None; + } + let get = |k: usize, o: usize| -> [f64; 4] { + let mut v = [0.0; 4]; + for i in 0..w { + v[i] = self.values[k * stride + o + i] as f64; + } + v + }; + let t0 = self.times[0] as f64; + if t <= t0 || n == 1 { + return Some(get(0, off)); + } + if t >= self.times[n - 1] as f64 { + return Some(get(n - 1, off)); + } + let k = self.times.partition_point(|&x| (x as f64) <= t).saturating_sub(1).min(n - 2); + let (ta, tb) = (self.times[k] as f64, self.times[k + 1] as f64); + let dt = (tb - ta).max(1e-9); + let u = ((t - ta) / dt).clamp(0.0, 1.0); + let (a, b) = (get(k, off), get(k + 1, off)); + let v = match self.interp { + Interp::Step => a, + Interp::Linear => { + if self.path == ChannelPath::Rotation { + slerp(a, b, u) + } else { + let mut v = [0.0; 4]; + for i in 0..w { + v[i] = a[i] + (b[i] - a[i]) * u; + } + v + } + } + Interp::CubicSpline => { + // Hermite spline with tangents scaled by the key interval (glTF Appendix C). + let out_a = get(k, 2 * w); + let in_b = get(k + 1, 0); + let (u2, u3) = (u * u, u * u * u); + let h00 = 2.0 * u3 - 3.0 * u2 + 1.0; + let h10 = u3 - 2.0 * u2 + u; + let h01 = -2.0 * u3 + 3.0 * u2; + let h11 = u3 - u2; + let mut v = [0.0; 4]; + for i in 0..w { + v[i] = h00 * a[i] + h10 * dt * out_a[i] + h01 * b[i] + h11 * dt * in_b[i]; + } + if self.path == ChannelPath::Rotation { + let l = (v[0] * v[0] + v[1] * v[1] + v[2] * v[2] + v[3] * v[3]).sqrt().max(1e-12); + for c in v.iter_mut() { + *c /= l; + } + } + v + } + }; + Some(v) + } +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Animation { + pub name: String, + pub channels: Vec, +} + +impl Animation { + /// Length in seconds (the last key time of any channel). + pub fn duration(&self) -> f64 { + self.channels.iter().filter_map(|c| c.times.last()).fold(0.0f64, |a, &b| a.max(b as f64)) + } +} + +/// An imported model (one glTF scene, one OBJ file, or a primitive). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Model { + pub meshes: Vec, + pub nodes: Vec, + /// Root nodes of the displayed scene. + pub roots: Vec, + pub materials: Vec, + pub textures: Vec>, + pub skins: Vec, + pub animations: Vec, +} + +/// One mesh placed in model space by [`Model::instances`]. +#[derive(Clone, Debug)] +pub struct Instance<'m> { + pub mesh: &'m Mesh, + /// Node → model space. + pub world: Mat4, + /// Skinned meshes: joint matrices (model space, `joint_world × inverse_bind`). + pub joints: Option>, +} + +impl Model { + /// A model of one mesh at the root. + pub fn single(mesh: Mesh, materials: Vec) -> Model { + Model { meshes: vec![mesh], nodes: vec![Node { mesh: Some(0), ..Node::default() }], roots: vec![0], materials, ..Model::default() } + } + + /// Local node matrices at animation `clip` time `t` (seconds); `None` = rest pose. + pub fn local_pose(&self, clip: Option, t: f64) -> Vec { + let mut trs_v: Vec<([f64; 3], [f64; 4], [f64; 3])> = self.nodes.iter().map(|n| (n.translation, n.rotation, n.scale)).collect(); + if let Some(a) = clip.and_then(|c| self.animations.get(c)) { + for ch in &a.channels { + let (Some(v), Some(e)) = (ch.sample(t), trs_v.get_mut(ch.node)) else { continue }; + match ch.path { + ChannelPath::Translation => e.0 = [v[0], v[1], v[2]], + ChannelPath::Rotation => e.1 = v, + ChannelPath::Scale => e.2 = [v[0], v[1], v[2]], + } + } + } + self.nodes.iter().zip(trs_v).map(|(n, (t, r, s))| n.matrix.unwrap_or_else(|| trs(t, r, s))).collect() + } + + /// Model-space matrices of every node (hierarchy applied) for a local pose. + pub fn world_pose(&self, local: &[Mat4]) -> Vec { + let mut world = vec![Mat4::IDENTITY; self.nodes.len()]; + let mut seen = vec![false; self.nodes.len()]; + let mut stack: Vec<(usize, Mat4)> = self.roots.iter().map(|&r| (r, Mat4::IDENTITY)).collect(); + while let Some((i, parent)) = stack.pop() { + if i >= self.nodes.len() || seen[i] { + continue; + } + seen[i] = true; + let w = parent * local[i]; + world[i] = w; + for &c in &self.nodes[i].children { + stack.push((c, w)); + } + } + world + } + + /// The meshes to draw at animation `clip` time `t`. + pub fn instances(&self, clip: Option, t: f64) -> Vec> { + let local = self.local_pose(clip, t); + let world = self.world_pose(&local); + let mut seen = vec![false; self.nodes.len()]; + let mut stack: Vec = self.roots.clone(); + let mut out = vec![]; + while let Some(i) = stack.pop() { + if i >= self.nodes.len() || seen[i] { + continue; + } + seen[i] = true; + let n = &self.nodes[i]; + if let Some(m) = n.mesh.and_then(|m| self.meshes.get(m)) { + let joints = n.skin.and_then(|s| self.skins.get(s)).map(|s| { + s.joints + .iter() + .enumerate() + .map(|(k, &j)| world.get(j).copied().unwrap_or(Mat4::IDENTITY) * s.inverse_bind.get(k).copied().unwrap_or(Mat4::IDENTITY)) + .collect() + }); + // Skinned vertices are placed by their joints alone (glTF: the node transform of + // a skinned mesh is ignored). + let w = if joints.is_some() { Mat4::IDENTITY } else { world[i] }; + out.push(Instance { mesh: m, world: w, joints }); + } + stack.extend(n.children.iter().rev()); + } + out + } + + /// Model-space bounds of the rest pose (None for an empty model). + pub fn bounds(&self) -> Option<([f64; 3], [f64; 3])> { + let mut lo = [f64::INFINITY; 3]; + let mut hi = [f64::NEG_INFINITY; 3]; + for inst in self.instances(None, 0.0) { + for p in &inst.mesh.primitives { + for (k, v) in p.positions.iter().enumerate() { + let q = skin_point(&inst, p, k, *v); + for c in 0..3 { + lo[c] = lo[c].min(q[c]); + hi[c] = hi[c].max(q[c]); + } + } + } + } + lo[0].is_finite().then_some((lo, hi)) + } + + pub fn vertex_count(&self) -> usize { + self.meshes.iter().flat_map(|m| &m.primitives).map(|p| p.positions.len()).sum() + } + pub fn triangle_count(&self) -> usize { + self.meshes.iter().flat_map(|m| &m.primitives).map(|p| p.triangle_count()).sum() + } +} + +/// The skinning (or node) matrix of vertex `k` of an instance's primitive. +pub fn vertex_matrix(inst: &Instance, p: &Primitive, k: usize) -> Mat4 { + let Some(j) = &inst.joints else { return inst.world }; + let (Some(ji), Some(wi)) = (p.joints.get(k), p.weights.get(k)) else { return inst.world }; + let mut m = [[0.0; 4]; 4]; + let mut total = 0.0; + for c in 0..4 { + let w = wi[c] as f64; + if w == 0.0 { + continue; + } + let Some(jm) = j.get(ji[c] as usize) else { continue }; + total += w; + for r in 0..4 { + for q in 0..4 { + m[r][q] += jm.0[r][q] * w; + } + } + } + if total <= 1e-9 { inst.world } else { Mat4(m) } +} + +/// A vertex position of an instance in model space. +pub fn skin_point(inst: &Instance, p: &Primitive, k: usize, v: [f32; 3]) -> [f64; 3] { + let q = vertex_matrix(inst, p, k).apply(vec3(v[0] as f64, v[1] as f64, v[2] as f64)); + [q.x, q.y, q.z] +} + +/// Which loader handles a file name (`gltf`, `glb`, `obj`), by extension. +pub fn format_of(path: &str) -> Option<&'static str> { + let ext = path.rsplit('.').next()?.to_ascii_lowercase(); + match ext.as_str() { + "gltf" => Some("gltf"), + "glb" => Some("glb"), + "obj" => Some("obj"), + _ => None, + } +} + +/// Load a model from its main file's bytes; `resolve(uri)` reads sibling resources (buffers, +/// textures, MTL files) by their relative URI. +pub fn load(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + match format_of(path) { + Some("obj") => obj::parse(bytes, resolve), + Some(_) => gltf::parse(bytes, resolve), + None if bytes.starts_with(b"glTF") => gltf::parse(bytes, resolve), + None => Err(ModelError::Unsupported(format!("{path}: not a glTF or OBJ file"))), + } +} + +/// Summary for footage metadata (Project panel, Interpret Footage). +#[derive(Clone, Debug, Default, PartialEq, serde::Serialize)] +pub struct Summary { + pub meshes: usize, + pub vertices: usize, + pub triangles: usize, + pub materials: usize, + pub textures: usize, + pub animations: Vec<(String, f64)>, + pub skinned: bool, + /// Rest-pose bounds (model units). + pub min: [f64; 3], + pub max: [f64; 3], +} + +impl Model { + pub fn summary(&self) -> Summary { + let (min, max) = self.bounds().unwrap_or(([0.0; 3], [0.0; 3])); + Summary { + meshes: self.meshes.len(), + vertices: self.vertex_count(), + triangles: self.triangle_count(), + materials: self.materials.len(), + textures: self.textures.len(), + animations: self.animations.iter().map(|a| (a.name.clone(), a.duration())).collect(), + skinned: !self.skins.is_empty(), + min, + max, + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/model/src/obj.rs b/crates/model/src/obj.rs new file mode 100644 index 00000000..e6727e73 --- /dev/null +++ b/crates/model/src/obj.rs @@ -0,0 +1,262 @@ +//! Wavefront OBJ geometry and MTL materials. +//! +//! OBJ: `v` (optional vertex colours ignored), `vt`, `vn`, `f` with `v`, `v/vt`, `v//vn`, +//! `v/vt/vn` references (negative = relative), polygons fan-triangulated, `o`/`g` groups, `s` +//! smoothing (normals computed per smoothing state when a face has none), `usemtl`, `mtllib`. +//! +//! MTL materials map onto metallic-roughness: `Kd` → base colour, `d`/`Tr` → alpha, `Ke` → +//! emissive, `Pr`/`Pm` (the PBR extension) → roughness/metallic, otherwise roughness from the +//! Phong exponent `Ns` (`sqrt(2 / (Ns + 2))`) and metallic 0; `map_Kd`, `map_Ke`, +//! `map_Bump`/`bump`/`norm` load base colour, emissive and normal textures. + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::{AlphaMode, Material, Mesh, Model, Primitive, Result, TexRef, Texture, perr}; + +/// Parse MTL text into named materials (textures appended to `textures`). +pub fn parse_mtl(text: &str, resolve: &dyn Fn(&str) -> Option>, textures: &mut Vec>) -> Vec { + let mut out: Vec = vec![]; + let mut loaded: HashMap> = HashMap::new(); + let mut tex = |name: &str, textures: &mut Vec>| -> Option { + // Options (`-bm 1 file.png`): the file name is the last token. + let file = name.split_whitespace().last()?.to_string(); + let t = *loaded.entry(file.clone()).or_insert_with(|| { + let b = resolve(&file)?; + let t = Texture::decode(&b).ok()?; + textures.push(Arc::new(t)); + Some(textures.len() - 1) + }); + t.map(|texture| TexRef { texture, uv_set: 0 }) + }; + let mut ns_seen = false; + let mut pr_seen = false; + for line in text.lines() { + let line = line.trim(); + let (key, rest) = line.split_once(char::is_whitespace).map(|(k, r)| (k, r.trim())).unwrap_or((line, "")); + let f3 = || -> [f32; 3] { + let v: Vec = rest.split_whitespace().filter_map(|x| x.parse().ok()).collect(); + match v.len() { + 0 => [0.0; 3], + 1 | 2 => [v[0]; 3], + _ => [v[0], v[1], v[2]], + } + }; + let f1 = || rest.split_whitespace().next().and_then(|x| x.parse::().ok()); + if key == "newmtl" { + if let Some(m) = out.last_mut() { + finish(m, ns_seen, pr_seen); + } + ns_seen = false; + pr_seen = false; + out.push(Material { name: rest.to_string(), metallic: 0.0, roughness: 0.5, ..Material::default() }); + continue; + } + let Some(m) = out.last_mut() else { continue }; + match key { + "Kd" => { + let c = f3(); + m.base_color = [srgb_to_linear(c[0]), srgb_to_linear(c[1]), srgb_to_linear(c[2]), m.base_color[3]]; + } + "Ke" => m.emissive = f3().map(srgb_to_linear), + "d" => m.base_color[3] = f1().unwrap_or(1.0), + "Tr" => m.base_color[3] = 1.0 - f1().unwrap_or(0.0), + "Ns" => { + let ns = f1().unwrap_or(0.0).max(0.0); + if !pr_seen { + m.roughness = (2.0 / (ns + 2.0)).sqrt(); + } + ns_seen = true; + } + "Pr" => { + m.roughness = f1().unwrap_or(0.5).clamp(0.0, 1.0); + pr_seen = true; + } + "Pm" => m.metallic = f1().unwrap_or(0.0).clamp(0.0, 1.0), + "map_Kd" => m.base_color_tex = tex(rest, textures), + "map_Ke" => { + m.emissive_tex = tex(rest, textures); + if m.emissive == [0.0; 3] { + m.emissive = [1.0; 3]; + } + } + "map_Bump" | "map_bump" | "bump" | "norm" => m.normal_tex = tex(rest, textures), + _ => {} + } + } + if let Some(m) = out.last_mut() { + finish(m, ns_seen, pr_seen); + } + out +} + +fn finish(m: &mut Material, _ns: bool, _pr: bool) { + if m.base_color[3] < 0.999 { + m.alpha_mode = AlphaMode::Blend; + } +} + +/// sRGB-encoded component → linear (MTL colours are display colours). +pub fn srgb_to_linear(c: f32) -> f32 { + if c <= 0.04045 { c / 12.92 } else { ((c + 0.055) / 1.055).powf(2.4) } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct Key { + v: u32, + t: u32, + n: u32, + /// Smoothing state (faces without normals get computed normals per state). + s: u32, +} + +/// Parse OBJ text (and its MTL libraries through `resolve`). +pub fn parse(bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + let text = String::from_utf8_lossy(bytes); + let mut v: Vec<[f32; 3]> = vec![]; + let mut vt: Vec<[f32; 2]> = vec![]; + let mut vn: Vec<[f32; 3]> = vec![]; + let mut materials: Vec = vec![]; + let mut textures: Vec> = vec![]; + // One primitive per material, in order of first use. + let mut prims: Vec<(Option, Primitive, HashMap, Vec)> = vec![]; + let mut current: Option = None; + let mut smooth = 0u32; + let mut flat_id = 1u32 << 31; + let mut faces = 0usize; + // Join `\` line continuations. + let joined = text.replace("\\\r\n", " ").replace("\\\n", " "); + for (ln, line) in joined.lines().enumerate() { + let line = line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + let mut it = line.split_whitespace(); + let key = it.next().unwrap_or(""); + let rest: Vec<&str> = it.collect(); + let fl = |i: usize| rest.get(i).and_then(|x| x.parse::().ok()); + match key { + "v" => v.push([fl(0).unwrap_or(0.0), fl(1).unwrap_or(0.0), fl(2).unwrap_or(0.0)]), + "vt" => vt.push([fl(0).unwrap_or(0.0), fl(1).unwrap_or(0.0)]), + "vn" => vn.push([fl(0).unwrap_or(0.0), fl(1).unwrap_or(0.0), fl(2).unwrap_or(1.0)]), + "s" => { + smooth = match rest.first().copied() { + None | Some("off") | Some("0") => 0, + Some(x) => x.parse().unwrap_or(1).max(1), + } + } + "mtllib" => { + let name = line["mtllib".len()..].trim(); + for file in std::iter::once(name).chain(name.split_whitespace()) { + if let Some(b) = resolve(file) { + let mut ms = parse_mtl(&String::from_utf8_lossy(&b), resolve, &mut textures); + materials.append(&mut ms); + break; + } + } + } + "usemtl" => { + let name = line["usemtl".len()..].trim(); + current = materials.iter().position(|m| m.name == name); + if current.is_none() { + // Unknown material: a named grey default. + materials.push(Material { name: name.to_string(), metallic: 0.0, roughness: 0.5, base_color: [0.8, 0.8, 0.8, 1.0], ..Material::default() }); + current = Some(materials.len() - 1); + } + } + "f" => { + if rest.len() < 3 { + continue; + } + faces += 1; + let resolve_i = |s: &str, len: usize| -> Option { + if s.is_empty() { + return None; + } + let i: i64 = s.parse().ok()?; + let k = if i < 0 { len as i64 + i } else { i - 1 }; + (k >= 0 && (k as usize) < len).then_some(k as u32) + }; + let mut refs = vec![]; + for r in &rest { + let mut parts = r.split('/'); + let vi = resolve_i(parts.next().unwrap_or(""), v.len()).ok_or_else(|| perr(format!("line {}: bad vertex index `{r}`", ln + 1)))?; + let ti = parts.next().and_then(|s| resolve_i(s, vt.len())); + let ni = parts.next().and_then(|s| resolve_i(s, vn.len())); + refs.push((vi, ti, ni)); + } + let has_n = refs.iter().all(|r| r.2.is_some()); + // Faces without normals: shared within a smoothing group, unique otherwise. + let s = if has_n { + 0 + } else if smooth == 0 { + flat_id += 1; + flat_id + } else { + smooth + }; + let slot = match prims.iter().position(|p| p.0 == current) { + Some(i) => i, + None => { + prims.push((current, Primitive { material: current, ..Default::default() }, HashMap::new(), vec![])); + prims.len() - 1 + } + }; + let (_, prim, map, need) = &mut prims[slot]; + let mut ids = vec![]; + for (vi, ti, ni) in refs { + let k = Key { v: vi, t: ti.map_or(u32::MAX, |x| x), n: ni.map_or(u32::MAX, |x| x), s }; + let id = *map.entry(k).or_insert_with(|| { + prim.positions.push(v[vi as usize]); + prim.uvs.push(ti.map_or([0.0, 0.0], |t| vt[t as usize])); + prim.normals.push(ni.map_or([0.0; 3], |n| vn[n as usize])); + need.push(ni.is_none()); + (prim.positions.len() - 1) as u32 + }); + ids.push(id); + } + for i in 2..ids.len() { + prim.indices.extend([ids[0], ids[i - 1], ids[i]]); + } + } + _ => {} + } + } + if faces == 0 { + return Err(perr("OBJ file without faces")); + } + let any_uv = !vt.is_empty(); + let mut mesh = Mesh { name: String::new(), primitives: vec![] }; + for (_, mut p, _, need) in prims { + if need.iter().any(|&b| b) { + // Fill missing normals from the triangles (area weighted), keep given ones. + let given = p.normals.clone(); + p.compute_normals(); + for (i, n) in p.normals.iter_mut().enumerate() { + if !need[i] { + *n = given[i]; + } + } + } + if !any_uv { + p.uvs.clear(); + } else { + // OBJ's v axis points up the image; textures are stored top row first. + for uv in p.uvs.iter_mut() { + uv[1] = 1.0 - uv[1]; + } + } + if p.material.is_some_and(|m| materials[m].normal_tex.is_some()) { + p.compute_tangents(); + } + mesh.primitives.push(p); + } + Ok(Model::single(mesh, materials).with_textures(textures)) +} + +impl Model { + fn with_textures(mut self, t: Vec>) -> Model { + self.textures = t; + self + } +} diff --git a/crates/model/src/prim.rs b/crates/model/src/prim.rs new file mode 100644 index 00000000..f04873df --- /dev/null +++ b/crates/model/src/prim.rs @@ -0,0 +1,235 @@ +//! Parametric primitives (Layer ▸ New ▸ Cube/Sphere/Plane/Torus/Cone/Cylinder). +//! +//! Meshes are built in model space (+Y up, counter-clockwise front faces, like glTF), centred +//! on the origin, in pixels, with normals, UVs (0..1 across each face/around each surface) and +//! tangents. + +use std::f32::consts::{PI, TAU}; + +use serde::{Deserialize, Serialize}; + +use crate::{Material, Mesh, Model, Primitive}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum PrimitiveKind { + #[default] + Cube, + Sphere, + Plane, + Torus, + Cone, + Cylinder, +} + +impl PrimitiveKind { + pub const ALL: [PrimitiveKind; 6] = + [PrimitiveKind::Cube, PrimitiveKind::Sphere, PrimitiveKind::Plane, PrimitiveKind::Torus, PrimitiveKind::Cone, PrimitiveKind::Cylinder]; + pub fn label(self) -> &'static str { + match self { + PrimitiveKind::Cube => "Cube", + PrimitiveKind::Sphere => "Sphere", + PrimitiveKind::Plane => "Plane", + PrimitiveKind::Torus => "Torus", + PrimitiveKind::Cone => "Cone", + PrimitiveKind::Cylinder => "Cylinder", + } + } + pub fn parse(s: &str) -> Option { + PrimitiveKind::ALL.into_iter().find(|k| k.label().eq_ignore_ascii_case(s.trim())) + } +} + +/// Primitive dimensions (pixels) and tessellation. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PrimParams { + /// Width, height, depth (cube, plane: width × height). + pub size: [f32; 3], + /// Sphere/cone/cylinder radius; torus ring radius. + pub radius: f32, + /// Torus tube radius. + pub tube: f32, + /// Cone/cylinder height. + pub height: f32, + /// Segments around. + pub segments: u32, + /// Sphere rings / torus tube sides. + pub rings: u32, +} + +impl Default for PrimParams { + fn default() -> Self { + PrimParams { size: [300.0; 3], radius: 150.0, tube: 50.0, height: 300.0, segments: 48, rings: 24 } + } +} + +struct B { + p: Primitive, +} + +impl B { + fn new() -> B { + B { p: Primitive { material: Some(0), ..Default::default() } } + } + fn v(&mut self, pos: [f32; 3], n: [f32; 3], uv: [f32; 2]) -> u32 { + self.p.positions.push(pos); + self.p.normals.push(n); + self.p.uvs.push(uv); + (self.p.positions.len() - 1) as u32 + } + fn tri(&mut self, a: u32, b: u32, c: u32) { + self.p.indices.extend([a, b, c]); + } + fn quad(&mut self, a: u32, b: u32, c: u32, d: u32) { + // a-b-c-d counter-clockwise. + self.tri(a, b, c); + self.tri(a, c, d); + } +} + +/// The mesh of a primitive. +pub fn mesh(kind: PrimitiveKind, pr: &PrimParams) -> Primitive { + let seg = pr.segments.clamp(3, 512); + let rings = pr.rings.clamp(2, 512); + let mut b = B::new(); + match kind { + PrimitiveKind::Cube => { + let [w, h, d] = pr.size.map(|v| v.abs() * 0.5); + // (normal, u axis, v axis) per face; v runs down the texture. + let faces: [([f32; 3], [f32; 3], [f32; 3]); 6] = [ + ([0.0, 0.0, 1.0], [1.0, 0.0, 0.0], [0.0, -1.0, 0.0]), + ([0.0, 0.0, -1.0], [-1.0, 0.0, 0.0], [0.0, -1.0, 0.0]), + ([1.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, -1.0, 0.0]), + ([-1.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, -1.0, 0.0]), + ([0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, -1.0]), + ]; + let half = [w, h, d]; + for (n, u, v) in faces { + let c = |su: f32, sv: f32| -> [f32; 3] { [0, 1, 2].map(|k| (n[k] + u[k] * su + v[k] * sv) * half[k]) }; + let i0 = b.v(c(-1.0, -1.0), n, [0.0, 0.0]); + let i1 = b.v(c(1.0, -1.0), n, [1.0, 0.0]); + let i2 = b.v(c(1.0, 1.0), n, [1.0, 1.0]); + let i3 = b.v(c(-1.0, 1.0), n, [0.0, 1.0]); + // u × v points along -n for these axes (v down): wind so the face is CCW from outside. + b.quad(i0, i3, i2, i1); + } + } + PrimitiveKind::Plane => { + let (w, h) = (pr.size[0].abs() * 0.5, pr.size[1].abs() * 0.5); + let n = [0.0, 0.0, 1.0]; + let i0 = b.v([-w, h, 0.0], n, [0.0, 0.0]); + let i1 = b.v([w, h, 0.0], n, [1.0, 0.0]); + let i2 = b.v([w, -h, 0.0], n, [1.0, 1.0]); + let i3 = b.v([-w, -h, 0.0], n, [0.0, 1.0]); + b.quad(i0, i3, i2, i1); + } + PrimitiveKind::Sphere => { + let r = pr.radius.abs(); + for j in 0..=rings { + let th = PI * j as f32 / rings as f32; + // Exact poles and seam (welded positions). + let (st, ct) = if j == 0 { + (0.0, 1.0) + } else if j == rings { + (0.0, -1.0) + } else { + th.sin_cos() + }; + for i in 0..=seg { + let ph = TAU * (i % seg) as f32 / seg as f32; + let n = [st * ph.sin(), ct, st * ph.cos()]; + b.v([n[0] * r, n[1] * r, n[2] * r], n, [i as f32 / seg as f32, j as f32 / rings as f32]); + } + } + let row = seg + 1; + for j in 0..rings { + for i in 0..seg { + let a = j * row + i; + let (b0, c, d) = (a + row, a + row + 1, a + 1); + if j != 0 { + b.tri(a, b0, d); + } + if j != rings - 1 { + b.tri(d, b0, c); + } + } + } + } + PrimitiveKind::Torus => { + let (rr, tr) = (pr.radius.abs(), pr.tube.abs()); + for j in 0..=rings { + let v = TAU * (j % rings) as f32 / rings as f32; + for i in 0..=seg { + let u = TAU * (i % seg) as f32 / seg as f32; + // Ring in the XZ plane. + let (cu, su, cv, sv) = (u.cos(), u.sin(), v.cos(), v.sin()); + let n = [cv * su, sv, cv * cu]; + let p = [(rr + tr * cv) * su, tr * sv, (rr + tr * cv) * cu]; + b.v(p, n, [i as f32 / seg as f32, j as f32 / rings as f32]); + } + } + let row = seg + 1; + for j in 0..rings { + for i in 0..seg { + let a = j * row + i; + b.quad(a, a + 1, a + row + 1, a + row); + } + } + } + PrimitiveKind::Cylinder | PrimitiveKind::Cone => { + let r = pr.radius.abs(); + let h = pr.height.abs() * 0.5; + let top_r = if kind == PrimitiveKind::Cone { 0.0 } else { r }; + // Side normal tilt for the cone (slope). + let (ny, nk) = { + let s = (r - top_r) / (2.0 * h).max(1e-6); + let l = (1.0 + s * s).sqrt(); + (s / l, 1.0 / l) + }; + let base = b.p.positions.len() as u32; + for i in 0..=seg { + let a = TAU * (i % seg) as f32 / seg as f32; + let (s, c) = a.sin_cos(); + let n = [s * nk, ny, c * nk]; + let u = i as f32 / seg as f32; + b.v([s * top_r, h, c * top_r], n, [u, 0.0]); + b.v([s * r, -h, c * r], n, [u, 1.0]); + } + for i in 0..seg { + let t0 = base + 2 * i; + let (b0, t1, b1) = (t0 + 1, t0 + 2, t0 + 3); + if kind == PrimitiveKind::Cone { + b.tri(t0, b0, b1); + } else { + b.quad(t0, b0, b1, t1); + } + } + let caps: &[(f32, f32)] = if kind == PrimitiveKind::Cone { &[(-1.0, r)] } else { &[(1.0, r), (-1.0, r)] }; + for &(side, rad) in caps { + let n = [0.0, side, 0.0]; + let c = b.v([0.0, h * side, 0.0], n, [0.5, 0.5]); + let first = b.p.positions.len() as u32; + for i in 0..=seg { + let a = TAU * (i % seg) as f32 / seg as f32; + let (s, co) = a.sin_cos(); + b.v([s * rad, h * side, co * rad], n, [0.5 + 0.5 * s, 0.5 - 0.5 * co * side]); + } + for i in 0..seg { + let (p0, p1) = (first + i, first + i + 1); + if side > 0.0 { b.tri(c, p0, p1) } else { b.tri(c, p1, p0) } + } + } + } + } + b.p.compute_tangents(); + b.p +} + +/// A primitive as a one-mesh model with one material. +pub fn model(kind: PrimitiveKind, pr: &PrimParams, material: Material) -> Model { + let mut material = material; + if kind == PrimitiveKind::Plane { + material.double_sided = true; + } + Model::single(Mesh { name: kind.label().to_string(), primitives: vec![mesh(kind, pr)] }, vec![material]) +} diff --git a/crates/model/src/tests.rs b/crates/model/src/tests.rs new file mode 100644 index 00000000..31040b7c --- /dev/null +++ b/crates/model/src/tests.rs @@ -0,0 +1,318 @@ +use std::collections::HashMap; + +use super::extrude::{BevelStyle, ExtrudeParams, Outline, extrude, group_contours}; +use super::prim::{PrimParams, PrimitiveKind, mesh}; +use super::triangulate::{signed_area, triangulate}; +use super::*; + +fn fixture(name: &str) -> Vec { + std::fs::read(format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn resolver(name: &str) -> Option> { + std::fs::read(format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"))).ok() +} + +fn check_quad(m: &Model) { + assert_eq!(m.nodes.len(), 2); + assert_eq!(m.roots, vec![0]); + assert_eq!(m.nodes[0].children, vec![1]); + assert_eq!(m.meshes.len(), 1); + let p = &m.meshes[0].primitives[0]; + assert_eq!(p.positions.len(), 4); + assert_eq!(p.indices, vec![0, 1, 2, 0, 2, 3]); + assert_eq!(p.uvs[0], [0.0, 1.0]); + assert_eq!(p.normals[2], [0.0, 0.0, 1.0]); + let mat = &m.materials[0]; + assert_eq!(mat.name, "Painted"); + assert_eq!(mat.base_color, [1.0, 0.5, 0.25, 1.0]); + assert!((mat.metallic - 0.25).abs() < 1e-6 && (mat.roughness - 0.75).abs() < 1e-6); + assert_eq!(mat.emissive, [0.1, 0.2, 0.3]); + assert!(mat.double_sided); + let t = &m.textures[mat.base_color_tex.unwrap().texture]; + assert_eq!((t.width, t.height), (2, 2)); + assert_eq!(t.data[0], [1.0, 0.0, 0.0, 1.0]); + assert_eq!(t.data[3], [1.0, 1.0, 1.0, 1.0]); + assert_eq!(t.wrap, [Wrap::Clamp, Wrap::Repeat]); + // Hierarchy: scale 2 under a +1 Y translation. + let (lo, hi) = m.bounds().unwrap(); + assert!((lo[0] + 1.0).abs() < 1e-9 && (hi[0] - 1.0).abs() < 1e-9); + assert!((lo[1] - 0.0).abs() < 1e-9 && (hi[1] - 2.0).abs() < 1e-9); + // Animation: root translation (0,1,0) → (2,1,0) over 1 s. + assert_eq!(m.animations.len(), 1); + assert_eq!(m.animations[0].name, "Move"); + assert!((m.animations[0].duration() - 1.0).abs() < 1e-9); + let inst = m.instances(Some(0), 0.5); + assert_eq!(inst.len(), 1); + let c = inst[0].world.apply(vec3(0.0, 0.0, 0.0)); + assert!((c.x - 1.0).abs() < 1e-6 && (c.y - 1.0).abs() < 1e-6, "{c:?}"); +} + +#[test] +fn gltf_with_data_uris() { + let m = gltf::parse(&fixture("quad.gltf"), &|_| None).unwrap(); + check_quad(&m); +} + +#[test] +fn glb_matches_gltf() { + let m = load("quad.glb", &fixture("quad.glb"), &resolver).unwrap(); + check_quad(&m); + let g = gltf::parse(&fixture("quad.gltf"), &|_| None).unwrap(); + assert_eq!(m.meshes, g.meshes); + assert_eq!(m.materials, g.materials); +} + +#[test] +fn gltf_errors_are_reported() { + assert!(gltf::parse(b"{\"asset\":{\"version\":\"1.0\"}}", &|_| None).is_err()); + assert!(gltf::parse(b"not json", &|_| None).is_err()); + let missing = br#"{"asset":{"version":"2.0"},"buffers":[{"uri":"nowhere.bin","byteLength":4}]}"#; + assert!(matches!(gltf::parse(missing, &|_| None), Err(ModelError::Missing(_)))); + assert!(gltf::split_glb(b"glTF\x01\0\0\0").is_err()); +} + +#[test] +fn base64_and_uri_decoding() { + assert_eq!(gltf::base64_decode("aGVsbG8=").unwrap(), b"hello"); + assert_eq!(gltf::base64_decode("aGVsbG8").unwrap(), b"hello"); + assert_eq!(gltf::uri_decode("my%20file.bin"), "my file.bin"); +} + +#[test] +fn obj_with_mtl() { + let m = load("cube.obj", &fixture("cube.obj"), &resolver).unwrap(); + assert_eq!(m.materials.len(), 2); + let red = &m.materials[0]; + assert_eq!(red.name, "Red"); + assert_eq!(red.base_color, [1.0, 0.0, 0.0, 1.0]); + assert!((red.roughness - (2.0f32 / 100.0).sqrt()).abs() < 1e-6); + assert_eq!(red.metallic, 0.0); + let gold = &m.materials[1]; + assert_eq!((gold.metallic, gold.roughness), (1.0, 0.3)); + let prims = &m.meshes[0].primitives; + assert_eq!(prims.len(), 2); + assert_eq!(prims[0].material, Some(0)); + assert_eq!(prims.iter().map(|p| p.triangle_count()).sum::(), 12); + // Four unique (v, vt, vn) corners per face. + assert_eq!(m.vertex_count(), 24); + // Every face is wound counter-clockwise from outside (face normal agrees with vn). + for p in prims { + for t in p.indices.chunks(3) { + let (a, b, c) = (p.positions[t[0] as usize], p.positions[t[1] as usize], p.positions[t[2] as usize]); + let f = cross(sub(b, a), sub(c, a)); + assert!(dot(f, p.normals[t[0] as usize]) > 0.0); + } + } + let (lo, hi) = m.bounds().unwrap(); + assert_eq!((lo, hi), ([-0.5; 3], [0.5; 3])); + // V is flipped to the image's top-down rows. + assert_eq!(prims[0].uvs[0], [0.0, 1.0]); +} + +#[test] +fn obj_without_normals_gets_them() { + let src = b"v 0 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n"; + let m = obj::parse(src, &|_| None).unwrap(); + let p = &m.meshes[0].primitives[0]; + assert_eq!(p.normals, vec![[0.0, 0.0, 1.0]; 3]); + assert!(obj::parse(b"v 0 0 0\n", &|_| None).is_err()); +} + +#[test] +fn channel_interpolation_modes() { + let mut c = Channel { node: 0, path: ChannelPath::Translation, interp: Interp::Step, times: vec![0.0, 1.0], values: vec![0.0, 0.0, 0.0, 10.0, 0.0, 0.0] }; + assert_eq!(c.sample(0.7).unwrap()[0], 0.0); + c.interp = Interp::Linear; + assert!((c.sample(0.7).unwrap()[0] - 7.0).abs() < 1e-9); + assert_eq!(c.sample(5.0).unwrap()[0], 10.0); + // Cubic spline with zero tangents: smoothstep. + c.interp = Interp::CubicSpline; + c.values = vec![0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 10.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + assert!((c.sample(0.5).unwrap()[0] - 5.0).abs() < 1e-9); + assert!((c.sample(0.25).unwrap()[0] - 10.0 * (3.0 * 0.0625 - 2.0 * 0.015625)).abs() < 1e-9); + // Rotation: slerp keeps unit length, half way around Z. + let r = + Channel { node: 0, path: ChannelPath::Rotation, interp: Interp::Linear, times: vec![0.0, 1.0], values: vec![0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0] }; + let q = r.sample(0.5).unwrap(); + assert!((q[2] - (0.5f64).sqrt()).abs() < 1e-9 && (q[3] - (0.5f64).sqrt()).abs() < 1e-9); +} + +#[test] +fn skinning_moves_vertices_with_joints() { + let mut p = mesh(PrimitiveKind::Plane, &PrimParams { size: [2.0, 2.0, 0.0], ..Default::default() }); + p.joints = vec![[0, 0, 0, 0], [1, 0, 0, 0], [1, 0, 0, 0], [0, 0, 0, 0]]; + p.weights = vec![[1.0, 0.0, 0.0, 0.0]; 4]; + let mut m = Model::single(Mesh { name: "s".into(), primitives: vec![p] }, vec![Material::default()]); + m.nodes[0].skin = Some(0); + m.nodes.push(Node::default()); + m.nodes.push(Node { translation: [0.0, 0.0, 5.0], ..Node::default() }); + m.roots = vec![0, 1, 2]; + m.skins.push(Skin { joints: vec![1, 2], inverse_bind: vec![Mat4::IDENTITY; 2] }); + let inst = m.instances(None, 0.0); + let pr = &inst[0].mesh.primitives[0]; + assert_eq!(skin_point(&inst[0], pr, 0, pr.positions[0])[2], 0.0); + assert_eq!(skin_point(&inst[0], pr, 1, pr.positions[1])[2], 5.0); +} + +/// Edges of a mesh welded by position: (undirected edge → uses, directed edge → uses). +fn edge_stats(p: &Primitive) -> (HashMap<(u32, u32), usize>, HashMap<(u32, u32), usize>) { + let mut weld: HashMap<[u32; 3], u32> = HashMap::new(); + let ids: Vec = p + .positions + .iter() + .map(|q| { + let k = q.map(|v| (v + 0.0).to_bits()); + let n = weld.len() as u32; + *weld.entry(k).or_insert(n) + }) + .collect(); + let mut und = HashMap::new(); + let mut dir = HashMap::new(); + for t in p.indices.chunks(3) { + let v = [ids[t[0] as usize], ids[t[1] as usize], ids[t[2] as usize]]; + if v[0] == v[1] || v[1] == v[2] || v[0] == v[2] { + continue; + } + for k in 0..3 { + let (a, b) = (v[k], v[(k + 1) % 3]); + *und.entry((a.min(b), a.max(b))).or_insert(0) += 1; + *dir.entry((a, b)).or_insert(0) += 1; + } + } + (und, dir) +} + +fn assert_watertight(p: &Primitive) { + let (und, dir) = edge_stats(p); + for (e, n) in &und { + assert_eq!(*n, 2, "edge {e:?} used {n} times"); + } + for (e, n) in &dir { + assert_eq!(*n, 1, "directed edge {e:?} used {n} times (inconsistent winding)"); + } +} + +/// Signed volume (positive when faces wind outward). +fn volume(p: &Primitive) -> f64 { + p.indices + .chunks(3) + .map(|t| { + let [a, b, c] = [0, 1, 2].map(|k| p.positions[t[k] as usize].map(|v| v as f64)); + (a[0] * (b[1] * c[2] - b[2] * c[1]) - a[1] * (b[0] * c[2] - b[2] * c[0]) + a[2] * (b[0] * c[1] - b[1] * c[0])) / 6.0 + }) + .sum() +} + +/// Vertex normals agree with the faces they belong to. +fn assert_normals_outward(p: &Primitive) { + for t in p.indices.chunks(3) { + let (a, b, c) = (p.positions[t[0] as usize], p.positions[t[1] as usize], p.positions[t[2] as usize]); + let f = cross(sub(b, a), sub(c, a)); + if dot(f, f) < 1e-10 { + continue; + } + for &i in t { + let n = p.normals[i as usize]; + assert!((dot(n, n) - 1.0).abs() < 1e-3, "normal not unit"); + assert!(dot(f, n) > 0.0, "normal {n:?} faces away from triangle {f:?}"); + } + } +} + +#[test] +fn primitive_vertex_counts_and_normals() { + let pr = PrimParams { segments: 16, rings: 8, ..Default::default() }; + let cases = [ + (PrimitiveKind::Cube, 24, 36), + (PrimitiveKind::Plane, 4, 6), + (PrimitiveKind::Sphere, 17 * 9, 6 * 16 * 7), + (PrimitiveKind::Torus, 17 * 9, 6 * 16 * 8), + (PrimitiveKind::Cylinder, 2 * 17 + 2 * 18, 6 * 16 + 2 * 3 * 16), + (PrimitiveKind::Cone, 2 * 17 + 18, 3 * 16 + 3 * 16), + ]; + for (k, v, i) in cases { + let m = mesh(k, &pr); + assert_eq!((m.positions.len(), m.indices.len()), (v, i), "{k:?}"); + assert_eq!(m.normals.len(), v); + assert_eq!(m.uvs.len(), v); + assert_eq!(m.tangents.len(), v); + assert_normals_outward(&m); + if k != PrimitiveKind::Plane { + assert!(volume(&m) > 0.0, "{k:?} winds inward"); + assert_watertight(&m); + } + } + // Volumes approach the analytic ones. + let cube = mesh(PrimitiveKind::Cube, &PrimParams { size: [2.0, 3.0, 4.0], ..Default::default() }); + assert!((volume(&cube) - 24.0).abs() < 1e-4); + let s = mesh(PrimitiveKind::Sphere, &PrimParams { radius: 1.0, segments: 128, rings: 64, ..Default::default() }); + assert!((volume(&s) - 4.0 / 3.0 * std::f64::consts::PI).abs() < 0.01); +} + +#[test] +fn triangulation_covers_polygon_with_hole() { + let outer = vec![[0.0, 0.0], [10.0, 0.0], [10.0, 10.0], [5.0, 12.0], [0.0, 10.0]]; + let hole = vec![[3.0, 3.0], [3.0, 7.0], [7.0, 7.0], [7.0, 3.0]]; + let pts: Vec<[f64; 2]> = outer.iter().chain(hole.iter()).copied().collect(); + let tris = triangulate(&outer, std::slice::from_ref(&hole)); + let area: f64 = tris.iter().map(|t| signed_area(&[pts[t[0]], pts[t[1]], pts[t[2]]]) * 0.5).sum(); + let expect = signed_area(&outer).abs() * 0.5 - 16.0; + assert!((area - expect).abs() < 1e-9, "{area} vs {expect}"); + assert!(tris.iter().all(|t| signed_area(&[pts[t[0]], pts[t[1]], pts[t[2]]]) >= 0.0)); + // A concave "C". + let c = vec![[0.0, 0.0], [6.0, 0.0], [6.0, 2.0], [2.0, 2.0], [2.0, 4.0], [6.0, 4.0], [6.0, 6.0], [0.0, 6.0]]; + let tris = triangulate(&c, &[]); + let area: f64 = tris.iter().map(|t| signed_area(&[c[t[0]], c[t[1]], c[t[2]]]) * 0.5).sum(); + assert!((area - 28.0).abs() < 1e-9); +} + +fn ring_outline() -> Outline { + let circle = + |r: f64, n: usize| -> Vec<[f64; 2]> { (0..n).map(|i| (i as f64 / n as f64) * std::f64::consts::TAU).map(|a| [r * a.cos(), r * a.sin()]).collect() }; + Outline { outer: circle(100.0, 48), holes: vec![circle(50.0, 32)] } +} + +#[test] +fn extrusion_is_watertight_with_outward_normals() { + let square = Outline { outer: vec![[0.0, 0.0], [100.0, 0.0], [100.0, 100.0], [0.0, 100.0]], holes: vec![] }; + for bevel in BevelStyle::ALL { + for o in [square.clone(), ring_outline()] { + let p = ExtrudeParams { bevel, bevel_depth: 8.0, hole_bevel: 0.5, depth: 40.0 }; + let m = extrude(std::slice::from_ref(&o), &p); + assert_watertight(&m); + assert_normals_outward(&m); + assert!(volume(&m) > 0.0, "{bevel:?}: inward winding"); + let (lo, hi) = m.bounds().unwrap(); + assert!(lo[2].abs() < 1e-6 && (hi[2] - 40.0).abs() < 1e-4); + } + } + // Without a bevel the square block's volume is exact. + let m = extrude(&[square], &ExtrudeParams { bevel: BevelStyle::None, bevel_depth: 0.0, hole_bevel: 1.0, depth: 10.0 }); + assert!((volume(&m) - 100_000.0).abs() < 1e-2); + // Front cap faces the camera (−z), back cap away. + assert!(m.normals.iter().any(|n| n[2] < -0.99) && m.normals.iter().any(|n| n[2] > 0.99)); +} + +#[test] +fn bevels_shrink_the_caps() { + let square = Outline { outer: vec![[0.0, 0.0], [100.0, 0.0], [100.0, 100.0], [0.0, 100.0]], holes: vec![] }; + let m = extrude(std::slice::from_ref(&square), &ExtrudeParams { bevel: BevelStyle::Angular, bevel_depth: 10.0, hole_bevel: 1.0, depth: 50.0 }); + // Front cap vertices (z = 0) are inset by the bevel. + let front: Vec<_> = m.positions.iter().filter(|p| p[2] == 0.0).collect(); + assert!(front.iter().all(|p| p[0] >= 9.999 && p[0] <= 90.001)); + // The widest ring (the walls) still reaches the outline. + assert!(m.positions.iter().any(|p| p[0] == 0.0 && p[2] == 10.0)); + // Convex bevels have more profile steps than angular ones. + let c = extrude(std::slice::from_ref(&square), &ExtrudeParams { bevel: BevelStyle::Convex, bevel_depth: 10.0, hole_bevel: 1.0, depth: 50.0 }); + assert!(c.positions.len() > m.positions.len()); +} + +#[test] +fn contours_group_into_outlines_by_nesting() { + let sq = |x: f64, y: f64, s: f64| vec![[x, y], [x + s, y], [x + s, y + s], [x, y + s]]; + // "O" (outer + hole) next to an island inside another hole. + let g = group_contours(vec![sq(0.0, 0.0, 10.0), sq(2.0, 2.0, 6.0), sq(20.0, 0.0, 10.0), sq(22.0, 2.0, 6.0), sq(24.0, 4.0, 2.0)]); + assert_eq!(g.len(), 3); + assert_eq!(g.iter().map(|o| o.holes.len()).sum::(), 2); +} diff --git a/crates/model/src/triangulate.rs b/crates/model/src/triangulate.rs new file mode 100644 index 00000000..7505f5f9 --- /dev/null +++ b/crates/model/src/triangulate.rs @@ -0,0 +1,187 @@ +//! Polygon triangulation with holes: holes are bridged into the outer contour (each hole's +//! rightmost vertex joined to a visible outer vertex), then ears are clipped. + +type P = [f64; 2]; + +/// Twice the signed area (positive = counter-clockwise in a Y-up frame). +pub fn signed_area(c: &[P]) -> f64 { + let n = c.len(); + (0..n) + .map(|i| { + let (a, b) = (c[i], c[(i + 1) % n]); + a[0] * b[1] - b[0] * a[1] + }) + .sum() +} + +#[inline] +fn cross(o: P, a: P, b: P) -> f64 { + (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) +} + +fn in_triangle(p: P, a: P, b: P, c: P) -> bool { + let d1 = cross(a, b, p); + let d2 = cross(b, c, p); + let d3 = cross(c, a, p); + d1 >= 0.0 && d2 >= 0.0 && d3 >= 0.0 +} + +/// Triangulate an outer contour with holes. Contours are closed polylines (last point not +/// repeated); orientation is normalised internally. Returns triangles as indices into the +/// contours concatenated in order (`outer`, then `holes`), counter-clockwise (Y up). +pub fn triangulate(outer: &[P], holes: &[Vec

]) -> Vec<[usize; 3]> { + if outer.len() < 3 { + return vec![]; + } + let mut pts: Vec

= outer.to_vec(); + for h in holes { + pts.extend_from_slice(h); + } + // Ring of global indices, outer counter-clockwise. + let mut ring: Vec = (0..outer.len()).collect(); + if signed_area(outer) < 0.0 { + ring.reverse(); + } + // Holes clockwise, rightmost first. + let mut hs: Vec> = vec![]; + let mut off = outer.len(); + for h in holes { + let mut r: Vec = (off..off + h.len()).collect(); + off += h.len(); + if h.len() < 3 { + continue; + } + if signed_area(h) > 0.0 { + r.reverse(); + } + hs.push(r); + } + hs.sort_by(|a, b| { + let mx = |r: &Vec| r.iter().map(|&i| pts[i][0]).fold(f64::NEG_INFINITY, f64::max); + mx(b).total_cmp(&mx(a)) + }); + for h in hs { + bridge(&pts, &mut ring, &h); + } + ear_clip(&pts, ring) +} + +/// Join hole `h` into `ring` through a bridge from the hole's rightmost vertex. +fn bridge(pts: &[P], ring: &mut Vec, h: &[usize]) { + let (hk, &m) = h.iter().enumerate().max_by(|a, b| pts[*a.1][0].total_cmp(&pts[*b.1][0])).unwrap_or((0, &h[0])); + let mp = pts[m]; + // Nearest ring edge hit by the ray +x from M. + let n = ring.len(); + let mut best: Option<(f64, usize)> = None; + for i in 0..n { + let (a, b) = (pts[ring[i]], pts[ring[(i + 1) % n]]); + if (a[1] > mp[1]) == (b[1] > mp[1]) && a[1] != mp[1] && b[1] != mp[1] { + continue; + } + if a[1] == b[1] { + continue; + } + let t = (mp[1] - a[1]) / (b[1] - a[1]); + if !(0.0..=1.0).contains(&t) { + continue; + } + let x = a[0] + t * (b[0] - a[0]); + if x >= mp[0] && best.is_none_or(|(bx, _)| x < bx) { + best = Some((x, i)); + } + } + let Some((x, i)) = best else { return }; + let (a, b) = (ring[i], ring[(i + 1) % n]); + // Candidate: the edge endpoint with the larger x. + let mut cand = if pts[a][0] > pts[b][0] { (i, a) } else { ((i + 1) % n, b) }; + let ip = [x, mp[1]]; + // Reflex ring vertices inside triangle (M, I, P) block the view: take the one with the + // smallest angle to the ray (closest on ties). + let pp = pts[cand.1]; + let (t0, t1, t2) = if cross(mp, ip, pp) >= 0.0 { (mp, ip, pp) } else { (mp, pp, ip) }; + let mut best_ang = f64::INFINITY; + for k in 0..n { + let v = ring[k]; + if v == cand.1 { + continue; + } + let q = pts[v]; + let prev = pts[ring[(k + n - 1) % n]]; + let next = pts[ring[(k + 1) % n]]; + let reflex = cross(prev, q, next) <= 0.0; + if reflex && in_triangle(q, t0, t1, t2) { + let ang = (q[1] - mp[1]).abs().atan2(q[0] - mp[0]); + let d = (q[0] - mp[0]).hypot(q[1] - mp[1]); + let cd = (pts[cand.1][0] - mp[0]).hypot(pts[cand.1][1] - mp[1]); + if ang < best_ang || (ang == best_ang && d < cd) { + best_ang = ang; + cand = (k, v); + } + } + } + let (k, _) = cand; + // ring[..=k], hole from M around back to M, ring[k] again, rest. + let mut out = Vec::with_capacity(ring.len() + h.len() + 2); + out.extend_from_slice(&ring[..=k]); + for j in 0..=h.len() { + out.push(h[(hk + j) % h.len()]); + } + out.extend_from_slice(&ring[k..]); + *ring = out; +} + +fn ear_clip(pts: &[P], mut ring: Vec) -> Vec<[usize; 3]> { + let mut tris = vec![]; + let mut guard = 0usize; + let mut i = 0usize; + while ring.len() > 3 { + let n = ring.len(); + if guard > 2 * n { + // No ear found (degenerate/self-intersecting input): clip anyway to finish. + let (a, b, c) = (ring[(i + n - 1) % n], ring[i % n], ring[(i + 1) % n]); + tris.push([a, b, c]); + ring.remove(i % n); + guard = 0; + continue; + } + let ii = i % n; + let (a, b, c) = (ring[(ii + n - 1) % n], ring[ii], ring[(ii + 1) % n]); + let (pa, pb, pc) = (pts[a], pts[b], pts[c]); + let cr = cross(pa, pb, pc); + let scale = 1.0 + pa[0].abs() + pa[1].abs() + pc[0].abs() + pc[1].abs(); + // A straight-through vertex is clipped as a zero-area triangle (keeps every contour edge + // in the triangulation, so caps meet walls without T-junctions). + let straight = cr.abs() <= 1e-12 * scale * scale && (pa[0] - pb[0]) * (pc[0] - pb[0]) + (pa[1] - pb[1]) * (pc[1] - pb[1]) <= 0.0; + let convex = cr > 0.0 && !straight; + let mut ear = convex || straight; + if convex { + for &v in &ring { + if v == a || v == b || v == c { + continue; + } + let q = pts[v]; + // Bridged duplicates share positions with the ear's corners. + if q == pa || q == pb || q == pc { + continue; + } + if in_triangle(q, pa, pb, pc) { + ear = false; + break; + } + } + } + if ear { + tris.push([a, b, c]); + ring.remove(ii); + guard = 0; + i = if ii == 0 { 0 } else { ii - 1 }; + } else { + i = ii + 1; + guard += 1; + } + } + if ring.len() == 3 { + tris.push([ring[0], ring[1], ring[2]]); + } + tris +} diff --git a/crates/model/tests/fixtures/cube.mtl b/crates/model/tests/fixtures/cube.mtl new file mode 100644 index 00000000..9ea4c99c --- /dev/null +++ b/crates/model/tests/fixtures/cube.mtl @@ -0,0 +1,11 @@ +# EffectCraft test fixture: materials for cube.obj (original work). +newmtl Red +Kd 1.0 0.0 0.0 +Ns 98 +d 1.0 + +newmtl Gold +Kd 1.0 0.766 0.336 +Ke 0.0 0.0 0.0 +Pr 0.3 +Pm 1.0 diff --git a/crates/model/tests/fixtures/cube.mtl.attribution b/crates/model/tests/fixtures/cube.mtl.attribution new file mode 100644 index 00000000..7af62d2d --- /dev/null +++ b/crates/model/tests/fixtures/cube.mtl.attribution @@ -0,0 +1,7 @@ +asset: cube.mtl +title: MTL fixture: the two materials of cube.obj (Phong exponent and PBR extension keys) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/model/tests/fixtures/cube.obj b/crates/model/tests/fixtures/cube.obj new file mode 100644 index 00000000..e50ef30c --- /dev/null +++ b/crates/model/tests/fixtures/cube.obj @@ -0,0 +1,29 @@ +# EffectCraft test fixture: a unit cube with texture coordinates and normals (original work). +mtllib cube.mtl +o Cube +v -0.5 -0.5 0.5 +v 0.5 -0.5 0.5 +v 0.5 0.5 0.5 +v -0.5 0.5 0.5 +v -0.5 -0.5 -0.5 +v 0.5 -0.5 -0.5 +v 0.5 0.5 -0.5 +v -0.5 0.5 -0.5 +vt 0 0 +vt 1 0 +vt 1 1 +vt 0 1 +vn 0 0 1 +vn 0 0 -1 +vn 1 0 0 +vn -1 0 0 +vn 0 1 0 +vn 0 -1 0 +usemtl Red +f 1/1/1 2/2/1 3/3/1 4/4/1 +f 6/1/2 5/2/2 8/3/2 7/4/2 +f 2/1/3 6/2/3 7/3/3 3/4/3 +usemtl Gold +f 5/1/4 1/2/4 4/3/4 8/4/4 +f 4/1/5 3/2/5 7/3/5 8/4/5 +f -4/1/6 -3/2/6 -7/3/6 -8/4/6 diff --git a/crates/model/tests/fixtures/cube.obj.attribution b/crates/model/tests/fixtures/cube.obj.attribution new file mode 100644 index 00000000..faf95fc0 --- /dev/null +++ b/crates/model/tests/fixtures/cube.obj.attribution @@ -0,0 +1,7 @@ +asset: cube.obj +title: Wavefront OBJ fixture: a unit cube with texture coordinates, normals and two materials +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/model/tests/fixtures/quad.glb b/crates/model/tests/fixtures/quad.glb new file mode 100644 index 0000000000000000000000000000000000000000..ab978546e3201331109e7eebb5503a071e2628df GIT binary patch literal 2028 zcmbVNJ#X7a7``Mx+8FiHspwMS&>;mFm!c%g){@$?k|?kxR}zc@g3+F+lXM7qM;wo~ z14BTPE&n0^AV9{f>DaAnenf_DO}D=9k&z8J?+uQrqr40{?7IG|ngj66E zX1EqC0L(lMoo!@Pq$M9RAq~gWqme$~h6RD}dn%dEB#&+E5sP_K1xi0+zHP)XWupd+ zcr~O3XbCgq!p0s^z3{WzuGbv8^f=(Qi&!CK&d)GasDg(gj)92<3R+OcU@$5WYpjw{ z!24nhX*Y~cIQG({uF~3Km1sX}J`1COgM5GlkV3jjtxh$I`ao61tpk;~8UU$ob-RwH zB4pF77o!A}r;h|brHxIfRfK%-JVL=(qaPzVr3hgX)~K4)e(#(r~WA9*j}p;?#lD~=$!i}(dg zd2aBs6#YWy|C?XSJ?d=5XugxS`J?BCcPA73Hif3*a-Yj7Dx$izl1rt!y$SD&1wogf zX#0P$+rUX4TR~=go+3*gEXNtdgIinp#bE=2OGS6SB~k9<+}+Nu#*6LM60 z)h)PjrO)u4+iiE?pM1y$&L0-$xW*saEarBDkQcf2b#pw=kIaRA+#>kRy-n^E{4Ra| zw21Zku=99*?b|glTyHsc5;zl3i!(eKy4hsW3Ly`90?8~^j4zyI~!S1+sI5YqnR W-rxVceZLN;q}}T5z1#im75NvFRaM9U literal 0 HcmV?d00001 diff --git a/crates/model/tests/fixtures/quad.glb.attribution b/crates/model/tests/fixtures/quad.glb.attribution new file mode 100644 index 00000000..c1b0e631 --- /dev/null +++ b/crates/model/tests/fixtures/quad.glb.attribution @@ -0,0 +1,7 @@ +asset: quad.glb +title: Binary glTF (GLB) fixture: the same textured, animated quad with its PNG texture in the BIN chunk +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/model/tests/fixtures/quad.gltf b/crates/model/tests/fixtures/quad.gltf new file mode 100644 index 00000000..82afef9b --- /dev/null +++ b/crates/model/tests/fixtures/quad.gltf @@ -0,0 +1,210 @@ +{ + "asset": { + "version": "2.0", + "generator": "EffectCraft fixture script (original work)" + }, + "scene": 0, + "scenes": [ + { + "nodes": [ + 0 + ] + } + ], + "nodes": [ + { + "name": "Root", + "translation": [ + 0.0, + 1.0, + 0.0 + ], + "children": [ + 1 + ] + }, + { + "name": "Quad", + "mesh": 0, + "scale": [ + 2.0, + 2.0, + 2.0 + ] + } + ], + "meshes": [ + { + "name": "Quad", + "primitives": [ + { + "attributes": { + "POSITION": 0, + "NORMAL": 1, + "TEXCOORD_0": 2 + }, + "indices": 3, + "material": 0 + } + ] + } + ], + "materials": [ + { + "name": "Painted", + "pbrMetallicRoughness": { + "baseColorFactor": [ + 1.0, + 0.5, + 0.25, + 1.0 + ], + "metallicFactor": 0.25, + "roughnessFactor": 0.75, + "baseColorTexture": { + "index": 0 + } + }, + "emissiveFactor": [ + 0.1, + 0.2, + 0.3 + ], + "doubleSided": true + } + ], + "textures": [ + { + "source": 0, + "sampler": 0 + } + ], + "samplers": [ + { + "wrapS": 33071, + "wrapT": 10497 + } + ], + "images": [ + { + "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAEklEQVR4nGP4z8DwHwyBNBgAAEnICff5q7YNAAAAAElFTkSuQmCC" + } + ], + "animations": [ + { + "name": "Move", + "samplers": [ + { + "input": 4, + "output": 5, + "interpolation": "LINEAR" + } + ], + "channels": [ + { + "sampler": 0, + "target": { + "node": 0, + "path": "translation" + } + } + ] + } + ], + "accessors": [ + { + "bufferView": 0, + "componentType": 5126, + "count": 4, + "type": "VEC3", + "min": [ + -0.5, + -0.5, + 0.0 + ], + "max": [ + 0.5, + 0.5, + 0.0 + ] + }, + { + "bufferView": 1, + "componentType": 5126, + "count": 4, + "type": "VEC3" + }, + { + "bufferView": 2, + "componentType": 5126, + "count": 4, + "type": "VEC2" + }, + { + "bufferView": 3, + "componentType": 5123, + "count": 6, + "type": "SCALAR" + }, + { + "bufferView": 4, + "componentType": 5126, + "count": 2, + "type": "SCALAR", + "min": [ + 0.0 + ], + "max": [ + 1.0 + ] + }, + { + "bufferView": 5, + "componentType": 5126, + "count": 2, + "type": "VEC3" + } + ], + "bufferViews": [ + { + "buffer": 0, + "byteOffset": 0, + "byteLength": 48, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 48, + "byteLength": 48, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 96, + "byteLength": 32, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 128, + "byteLength": 12, + "target": 34963 + }, + { + "buffer": 0, + "byteOffset": 140, + "byteLength": 8 + }, + { + "buffer": 0, + "byteOffset": 148, + "byteLength": 24 + } + ], + "buffers": [ + { + "byteLength": 172, + "uri": "data:application/octet-stream;base64,AAAAvwAAAL8AAAAAAAAAPwAAAL8AAAAAAAAAPwAAAD8AAAAAAAAAvwAAAD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAgD8AAIA/AACAPwAAgD8AAAAAAAAAAAAAAAAAAAEAAgAAAAIAAwAAAAAAAACAPwAAAAAAAIA/AAAAAAAAAEAAAIA/AAAAAA==" + } + ] +} diff --git a/crates/model/tests/fixtures/quad.gltf.attribution b/crates/model/tests/fixtures/quad.gltf.attribution new file mode 100644 index 00000000..1ccf03b7 --- /dev/null +++ b/crates/model/tests/fixtures/quad.gltf.attribution @@ -0,0 +1,7 @@ +asset: quad.gltf +title: glTF 2.0 fixture: a textured quad under a parent node with a translation animation (all data embedded as data URIs) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/project/src/build.rs b/crates/project/src/build.rs index 12e384d6..a201879d 100644 --- a/crates/project/src/build.rs +++ b/crates/project/src/build.rs @@ -584,6 +584,13 @@ pub fn light_options(ids: &mut Ids, kind: LightKind) -> PropGroup { .with(ids.prop("coneAngle", "Cone Angle", Value::Scalar(90.0)).with_ui(ParamUi::Angle)) .with(ids.prop("coneFeather", "Cone Feather", Value::Scalar(50.0)).with_ui(ParamUi::Percent)); } + if kind == LightKind::Environment { + // The environment image: a layer (footage, comp…) mapped as an equirectangular panorama; + // none = the comp's Environment Layer. + return g + .with(ids.prop("source", "Source", Value::Layer(None)).with_ui(ParamUi::Layer)) + .with(ids.prop("rotation", "Environment Rotation", Value::Scalar(0.0)).with_ui(ParamUi::Angle)); + } if kind != LightKind::Ambient { g = g .with(ids.prop("falloff", "Falloff", Value::Enum(0)).with_ui(popup(&["None", "Smooth", "Inverse Square Clamped"]))) @@ -609,6 +616,70 @@ pub fn material_options(ids: &mut Ids) -> PropGroup { .with(ids.prop("metal", "Metal", Value::Scalar(100.0)).with_ui(ParamUi::Percent)) } +/// Material Options of a 3D model or primitive layer (Advanced 3D): shadows and lights, plus +/// the primitive's own metallic-roughness material (models keep their file's materials). +pub fn model_material_options(ids: &mut Ids, primitive: bool) -> PropGroup { + let mut g = ids + .group("materialOptions", "Material Options") + .with(ids.prop("castsShadows", "Casts Shadows", Value::Enum(0)).with_ui(popup(&["Off", "On", "Only"]))) + .with(ids.prop("acceptsShadows", "Accepts Shadows", Value::Enum(1)).with_ui(popup(&["Off", "On", "Only"]))) + .with(ids.prop("acceptsLights", "Accepts Lights", Value::Bool(true)).with_ui(ParamUi::Checkbox)) + .with(ids.prop("appearsInReflections", "Appears in Reflections", Value::Bool(true)).with_ui(ParamUi::Checkbox)); + if primitive { + g = g + .with(ids.prop("baseColor", "Base Color", Value::Color([0.8, 0.8, 0.8, 1.0])).with_ui(ParamUi::Color)) + .with(ids.prop("metallic", "Metallic", Value::Scalar(0.0)).with_ui(slider(0.0, 100.0, 0.0, 100.0, 1))) + .with(ids.prop("roughness", "Roughness", Value::Scalar(50.0)).with_ui(slider(0.0, 100.0, 0.0, 100.0, 1))) + .with(ids.prop("emissive", "Emissive", Value::Color([0.0, 0.0, 0.0, 1.0])).with_ui(ParamUi::Color)); + } + g +} + +/// Geometry Options of an imported model layer: model units → pixels, and the animation clip +/// played along layer time. +pub fn model_geometry_options(ids: &mut Ids, unit_scale: f64, clips: &[String]) -> PropGroup { + let mut opts: Vec<&str> = vec!["None"]; + opts.extend(clips.iter().map(String::as_str)); + ids.group("geometryOptions", "Geometry Options") + .with(ids.prop("unitScale", "Model Scale", Value::Scalar(unit_scale)).with_ui(slider(0.0, 1.0e6, 0.0, 1000.0, 2))) + .with(ids.prop("animation", "Animation", Value::Enum(if clips.is_empty() { 0 } else { 1 })).with_ui(popup(&opts))) + .with(ids.prop("loopAnimation", "Loop Animation", Value::Bool(true)).with_ui(ParamUi::Checkbox)) + .with(ids.prop("animationSpeed", "Animation Speed", Value::Scalar(100.0)).with_ui(ParamUi::Percent)) +} + +/// Geometry Options of a primitive layer (pixels). +pub fn primitive_geometry_options(ids: &mut Ids, kind: crate::PrimitiveKind) -> PropGroup { + use crate::PrimitiveKind as K; + let px = |ids: &mut Ids, m: &str, n: &str, v: f64| ids.prop(m, n, Value::Scalar(v)).with_ui(slider(0.0, 100_000.0, 0.0, 2000.0, 1)); + let count = |ids: &mut Ids, m: &str, n: &str, v: f64| ids.prop(m, n, Value::Scalar(v)).with_ui(slider(3.0, 512.0, 3.0, 128.0, 0)); + let mut g = ids.group("geometryOptions", "Geometry Options"); + let props = match kind { + K::Cube => vec![px(ids, "width", "Width", 300.0), px(ids, "height", "Height", 300.0), px(ids, "depth", "Depth", 300.0)], + K::Plane => vec![px(ids, "width", "Width", 400.0), px(ids, "height", "Height", 400.0)], + K::Sphere => vec![px(ids, "radius", "Radius", 150.0), count(ids, "segments", "Segments", 48.0), count(ids, "rings", "Rings", 24.0)], + K::Torus => vec![ + px(ids, "radius", "Radius", 150.0), + px(ids, "tubeRadius", "Tube Radius", 50.0), + count(ids, "segments", "Segments", 48.0), + count(ids, "rings", "Sides", 24.0), + ], + K::Cone | K::Cylinder => vec![px(ids, "radius", "Radius", 150.0), px(ids, "height", "Height", 300.0), count(ids, "segments", "Segments", 48.0)], + }; + for p in props { + g = g.with(p); + } + g +} + +/// Geometry Options of a text or shape layer in an Advanced 3D comp (extrusion and bevels). +pub fn extrusion_geometry_options(ids: &mut Ids) -> PropGroup { + ids.group("geometryOptions", "Geometry Options") + .with(ids.prop("bevelStyle", "Bevel Style", Value::Enum(0)).with_ui(popup(&["None", "Angular", "Concave", "Convex"]))) + .with(ids.prop("bevelDepth", "Bevel Depth", Value::Scalar(2.0)).with_ui(slider(0.0, 1000.0, 0.0, 100.0, 1))) + .with(ids.prop("holeBevelDepth", "Hole Bevel Depth", Value::Scalar(100.0)).with_ui(slider(0.0, 100.0, 0.0, 100.0, 1))) + .with(ids.prop("extrusionDepth", "Extrusion Depth", Value::Scalar(0.0)).with_ui(slider(0.0, 10_000.0, 0.0, 1000.0, 1))) +} + pub fn audio(ids: &mut Ids) -> PropGroup { ids.group("audio", "Audio").with(ids.prop("levels", "Audio Levels", Value::Vec2([0.0, 0.0]))) } @@ -629,6 +700,7 @@ pub fn default_label(src: &LayerSource, project: &Project) -> Label { LayerSource::Shape => Label::Blue, LayerSource::Null => Label::Red, LayerSource::Camera | LayerSource::Light { .. } => Label::Pink, + LayerSource::Model { .. } | LayerSource::Primitive { .. } => Label::Aqua, } } @@ -640,7 +712,12 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource let (w, h) = (size.0 as f64, size.1 as f64); let (cw, ch) = (comp.width as f64, comp.height as f64); let anchor = match source { - LayerSource::Text | LayerSource::Shape | LayerSource::Camera | LayerSource::Light { .. } => [0.0, 0.0], + LayerSource::Text + | LayerSource::Shape + | LayerSource::Camera + | LayerSource::Light { .. } + | LayerSource::Model { .. } + | LayerSource::Primitive { .. } => [0.0, 0.0], LayerSource::Null => [50.0, 50.0], _ => [w / 2.0, h / 2.0], }; @@ -654,7 +731,9 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource } _ => {} } - if source.is_av() { + if source.is_model() { + // Model layers have no masks or effects (as in After Effects). + } else if source.is_av() { root.children.push(masks(&mut ids).into()); root.children.push(effects(&mut ids).into()); } else if matches!(source, LayerSource::Null) { @@ -694,8 +773,17 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource // Nulls are 100×100. } root.children.push(std::mem::replace(&mut tr, PropGroup::new(0, "", "")).into()); - if source.is_av() { - root.children.push(material_options(&mut ids).into()); + match source { + LayerSource::Primitive { kind } => { + root.children.push(primitive_geometry_options(&mut ids, kind).into()); + root.children.push(model_material_options(&mut ids, true).into()); + } + LayerSource::Model { .. } => { + root.children.push(model_geometry_options(&mut ids, 1.0, &[]).into()); + root.children.push(model_material_options(&mut ids, false).into()); + } + _ if source.is_av() => root.children.push(material_options(&mut ids).into()), + _ => {} } } let has_audio = matches!(&source, LayerSource::Footage { item } if matches!(project.item(*item).map(|i| &i.kind), Some(crate::ItemKind::Footage(f)) if f.has_audio)) @@ -711,6 +799,7 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource } else { crate::AutoOrient::Off }; + let switches = Switches { three_d: source.is_model(), ..Switches::default() }; Layer { id, name: comp.unique_layer_name(name), @@ -721,13 +810,14 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource in_point: Tick::ZERO, out_point: out, stretch: 100.0, - switches: Switches::default(), + switches, blend_mode: BlendMode::Normal, preserve_transparency: false, track_matte: None, parent: None, markers: vec![], markers_locked: false, + environment: false, auto_orient, props: root, } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 1ed35c09..4a00c413 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -310,6 +310,36 @@ impl LightKind { } } +/// A parametric 3D primitive (Layer ▸ New ▸ Cube/Sphere/Plane/Torus/Cone/Cylinder). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum PrimitiveKind { + #[default] + Cube, + Sphere, + Plane, + Torus, + Cone, + Cylinder, +} + +impl PrimitiveKind { + pub const ALL: [PrimitiveKind; 6] = + [PrimitiveKind::Cube, PrimitiveKind::Sphere, PrimitiveKind::Plane, PrimitiveKind::Torus, PrimitiveKind::Cone, PrimitiveKind::Cylinder]; + pub fn label(self) -> &'static str { + match self { + PrimitiveKind::Cube => "Cube", + PrimitiveKind::Sphere => "Sphere", + PrimitiveKind::Plane => "Plane", + PrimitiveKind::Torus => "Torus", + PrimitiveKind::Cone => "Cone", + PrimitiveKind::Cylinder => "Cylinder", + } + } + pub fn parse(s: &str) -> Option { + PrimitiveKind::ALL.into_iter().find(|k| k.label().eq_ignore_ascii_case(s.trim())) + } +} + /// What a layer shows. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] @@ -333,12 +363,20 @@ pub enum LayerSource { Light { kind: LightKind, }, + /// An imported 3D model (a footage item of kind [`FootageKind::Model`]; Advanced 3D). + Model { + item: ItemId, + }, + /// A parametric 3D primitive (Advanced 3D). + Primitive { + kind: PrimitiveKind, + }, } impl LayerSource { pub fn item(&self) -> Option { match self { - LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } => Some(*item), + LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } | LayerSource::Model { item } => Some(*item), _ => None, } } @@ -352,12 +390,18 @@ impl LayerSource { LayerSource::Null => "Null", LayerSource::Camera => "Camera", LayerSource::Light { .. } => "Light", + LayerSource::Model { .. } => "3D Model", + LayerSource::Primitive { .. } => "3D Primitive", } } /// Layers with pixels (cameras, lights and nulls have none). pub fn is_av(&self) -> bool { !matches!(self, LayerSource::Camera | LayerSource::Light { .. } | LayerSource::Null) } + /// A 3D model or primitive layer (meshes; drawn by the Advanced 3D renderer). + pub fn is_model(&self) -> bool { + matches!(self, LayerSource::Model { .. } | LayerSource::Primitive { .. }) + } } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -489,6 +533,10 @@ pub struct Layer { pub markers_locked: bool, #[serde(default)] pub auto_orient: AutoOrient, + /// Layer ▸ Environment Layer: the layer is the environment (image-based light and + /// reflections) of Advanced 3D comps instead of being drawn. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub environment: bool, /// The property tree (Masks, Effects, Transform, Text, Contents, Camera/Light options…). pub props: PropGroup, } @@ -540,6 +588,8 @@ pub enum FootageKind { Still, Sequence, Audio, + /// A 3D model (glTF 2.0 / OBJ), placed as a model layer. + Model, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -625,6 +675,7 @@ impl Item { FootageKind::Still => "Image", FootageKind::Sequence => "Image Sequence", FootageKind::Audio => "Audio", + FootageKind::Model => "3D Model", }, ItemKind::Solid(_) => "Solid", } diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 4c8a9787..80e30455 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -19,6 +19,7 @@ effectcraft-raster = { workspace = true } effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } +effectcraft-model = { workspace = true } kurbo = { workspace = true } rayon = { workspace = true } web-time = { workspace = true } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 14a81575..4ec70cfa 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -39,6 +39,10 @@ pub trait FootageSource: Send + Sync { fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option> { None } + /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D). + fn model(&self, _item: ItemId, _footage: &Footage) -> Option> { + None + } /// Set the decoded-frame cache budget in bytes (Settings ▸ Memory & CPU); sources without /// a cache ignore it. fn set_cache_budget(&self, _bytes: usize) {} diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index da5eaec2..22b2bd40 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -716,7 +716,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index 1824fec5..d13f3768 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -22,6 +22,7 @@ fn item_icon(it: &Item) -> Icon { ItemKind::Footage(f) => match f.kind { effectcraft_engine::project::FootageKind::Still | effectcraft_engine::project::FootageKind::Sequence => Icon::Image, effectcraft_engine::project::FootageKind::Audio => Icon::Audio, + effectcraft_engine::project::FootageKind::Model => Icon::Cube, _ => Icon::Footage, }, } diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 5b6c085f..c9a42698 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -433,6 +433,7 @@ fn layer_icon(l: &Layer, project: &effectcraft_engine::project::Project) -> Icon LayerSource::Null => Icon::Null, LayerSource::Camera => Icon::Camera, LayerSource::Light { .. } => Icon::Light, + LayerSource::Model { .. } | LayerSource::Primitive { .. } => Icon::Cube, LayerSource::Comp { .. } => Icon::Comp, LayerSource::Solid { .. } => { if l.switches.adjustment { diff --git a/docs/architecture.md b/docs/architecture.md index c3d478e3..7e2921e0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,6 +22,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `model` | 3D models for Advanced 3D: glTF 2.0 (`.gltf`/`.glb`) and OBJ/MTL import (meshes, PBR metallic-roughness materials and textures, node hierarchy, skins, animations), parametric primitives, extruded/bevelled outline meshes and polygon triangulation | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | | L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ae38adf9..7e265297 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -31,6 +31,7 @@ const LAYERS: &[(&str, u8)] = &[ ("text", 2), ("effects", 2), ("track", 2), + ("model", 2), ("render", 3), ("media", 3), ("expr", 3), From 9645ff9a6610cd8dc2d9594f63a3fea16d51a0e0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:14:29 -0400 Subject: [PATCH 2/4] M6.6: mask tracking + mask interpolation; M12.5: Warp Stabilizer Mask tracking (track.mask, background job with progress/cancel, one undo step, Mask Path keys; Tracker panel mask mode; Layer > Mask > Track Mask), Mask Interpolation panel + mask.interpolate, and the Warp Stabilizer effect (Effect > Distort, AE parameter groups, warp.analyze/cancel/status, analysis stored in the effect, invalidation on source/in/out/remap edits, viewer banner, crop/auto-scale/synthesize edges, track points). Adds the effects -> track same-layer edge. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/effects/Cargo.toml | 1 + crates/effects/src/lib.rs | 35 +- crates/effects/src/warp_stab.rs | 493 +++++++++++++++++++ crates/engine/src/commands/mask_interp.rs | 390 +++++++++++++++ crates/engine/src/commands/mod.rs | 4 + crates/engine/src/commands/stubs.rs | 2 - crates/engine/src/commands/track.rs | 58 ++- crates/engine/src/commands/warp_cmds.rs | 131 +++++ crates/engine/src/lib.rs | 30 ++ crates/engine/src/mask_track.rs | 313 ++++++++++++ crates/engine/src/menus.rs | 3 +- crates/engine/src/tests_mask_warp.rs | 384 +++++++++++++++ crates/engine/src/tests_track.rs | 20 +- crates/engine/src/tracking.rs | 4 +- crates/engine/src/warp.rs | 404 +++++++++++++++ crates/ui-egui/src/dock.rs | 5 +- crates/ui-egui/src/lib.rs | 9 + crates/ui-egui/src/panels/anim_tools.rs | 110 ++++- crates/ui-egui/src/panels/effect_controls.rs | 29 ++ crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/tracker.rs | 90 +++- crates/ui-egui/src/panels/viewer.rs | 20 + docs/agents.md | 46 ++ docs/architecture.md | 20 +- docs/parity.md | 6 +- xtask/src/main.rs | 1 + 27 files changed, 2560 insertions(+), 50 deletions(-) create mode 100644 crates/effects/src/warp_stab.rs create mode 100644 crates/engine/src/commands/mask_interp.rs create mode 100644 crates/engine/src/commands/warp_cmds.rs create mode 100644 crates/engine/src/mask_track.rs create mode 100644 crates/engine/src/tests_mask_warp.rs create mode 100644 crates/engine/src/warp.rs diff --git a/Cargo.lock b/Cargo.lock index 635fb0f5..46f34e63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1182,6 +1182,7 @@ dependencies = [ "effectcraft-keyframe", "effectcraft-project", "effectcraft-raster", + "effectcraft-track", "rayon", "serde_json", ] diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index db74bf27..e321fa2f 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -15,6 +15,7 @@ effectcraft-color = { workspace = true } effectcraft-keyframe = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } +effectcraft-track = { workspace = true } rayon = { workspace = true } [dev-dependencies] diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 2e29061a..6d11e19a 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -42,6 +42,7 @@ mod transition2; pub mod util; mod utility; mod utility2; +pub mod warp_stab; use std::collections::HashMap; use std::sync::OnceLock; @@ -352,6 +353,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(audio_fx::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); + v.extend(warp_stab::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -396,14 +398,20 @@ pub fn instantiate(spec: &EffectSpec, ids: &mut Ids, instance_name: &str, layer_ g.kind = GroupKind::Effect { effect: spec.id.to_string() }; for ps in &spec.params { let default = default_value(ps, layer_size); - let mut pr = Property::new(ids.alloc(), ps.id, ps.name, default).with_ui(ps.ui.clone()); + // `group/sub/param` ids nest the parameter in twirl-down groups (Warp Stabilizer's + // Stabilization / Borders / Advanced). + let (path, leaf) = ps.id.rsplit_once('/').map(|(g, l)| (Some(g), l)).unwrap_or((None, ps.id)); + let mut pr = Property::new(ids.alloc(), leaf, ps.name, default).with_ui(ps.ui.clone()); if matches!(ps.ui, ParamUi::Point) { pr.spatial = true; } if matches!(ps.ui, ParamUi::Checkbox | ParamUi::Popup { .. } | ParamUi::Layer) { pr.hold_only = true; } - g.children.push(pr.into()); + match path { + Some(path) => nested_group(&mut g, ids, path).children.push(pr.into()), + None => g.children.push(pr.into()), + } // Layer parameters carry a source choice (Source / Masks / Effects & Masks) unless the // effect declares its own (with its own default). let sid = layer_source_id(ps.id); @@ -416,6 +424,27 @@ pub fn instantiate(spec: &EffectSpec, ids: &mut Ids, instance_name: &str, layer_ g } +/// Display names of nested parameter groups by match id. +fn group_name(m: &str) -> &str { + warp_stab::GROUPS.iter().find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) +} + +/// The nested group at `path` (`borders/autoScale`) under `g`, created on first use. +fn nested_group<'a>(g: &'a mut PropGroup, ids: &mut Ids, path: &str) -> &'a mut PropGroup { + let (first, rest) = match path.split_once('/') { + Some((a, b)) => (a, Some(b)), + None => (path, None), + }; + if g.sub(first).is_none() { + g.children.push(ids.group(first, group_name(first)).into()); + } + let sub = g.sub_mut(first).expect("just added"); + match rest { + Some(r) => nested_group(sub, ids, r), + None => sub, + } +} + /// Effects the GPU compositor (`effectcraft-gpu`) implements with the CPU effect's semantics /// (Mercury GPU Acceleration; the Effects & Presets GPU badge). Expression controls are /// pass-throughs and count as GPU effects too. @@ -463,6 +492,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.ccforcemotionblur", "ec.time.ccwidetime", "ec.time.pixelmotionblur", + // Each frame gets its own stabilizing warp. + warp_stab::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/warp_stab.rs b/crates/effects/src/warp_stab.rs new file mode 100644 index 00000000..19171051 --- /dev/null +++ b/crates/effects/src/warp_stab.rs @@ -0,0 +1,493 @@ +//! Warp Stabilizer (Effect ▸ Distort ▸ Warp Stabilizer). +//! +//! The analysis (feature tracking and per-frame motion fits) runs as a background job in the +//! engine (`warp.analyze`) and is stored, as JSON, in the instance's hidden **Analysis** +//! parameter ([`effectcraft_track::stabilize::WarpAnalysis`]); **Analysis Key** records what +//! the analysis was made from (the layer's source, in/out points and time mapping) so edits that +//! change the frames invalidate it. Rendering derives the stabilization plan from the analysis +//! and the Stabilization / Borders / Advanced settings ([`effectcraft_track::stabilize::plan`], +//! cached per analysis and settings) and warps each frame by its corrective transform: +//! +//! - **Stabilize Only** shows the moving frame edges, **Stabilize, Crop** crops to the region +//! valid on every frame, **Stabilize, Crop, Auto-scale** scales that region back up to the +//! frame (up to Maximum Scale, minus the Action-safe Margin), Additional Scale scales on top; +//! - **Stabilize, Synthesize Edges** fills the uncovered borders from neighbouring frames +//! (within Synthesize Input Range), mapped through the analysed frame-to-frame motion and read +//! with [`crate::EffectHost::self_at`]; Synthesize Edge Feather blends the seam and Synthesize +//! Edge Cropping ignores bad edge pixels of the input frames; +//! - **Show Track Points** draws the analysed background features. +//! +//! *Subspace Warp* is approximated by a per-frame perspective warp (no content-preserving +//! mesh warp yet), and *Rolling Shutter Ripple* is stored but has no effect. +//! +//! Without an analysis (or with one made for another layer size) the input passes through. + +use std::collections::HashMap; +use std::hash::Hasher; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::Homography; +use effectcraft_track::stabilize::{Framing, Method, Plan, StabResult, StabSettings, WarpAnalysis, plan}; +use rayon::prelude::*; + +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.distort.warpstabilizer"; + +/// Hidden parameters: the stored analysis (JSON) and its key (see the module docs). +pub const ANALYSIS: &str = "analysis"; +pub const ANALYSIS_KEY: &str = "analysisKey"; + +/// Display names of the instance's parameter groups (by match id). +pub const GROUPS: &[(&str, &str)] = &[ + ("stabilization", "Stabilization"), + ("borders", "Borders"), + ("autoScale", "Auto-scale"), + ("advanced", "Advanced"), + ("edgeCropping", "Synthesize Edge Cropping"), +]; + +pub const RESULTS: [&str; 2] = ["Smooth Motion", "No Motion"]; +pub const METHODS: [&str; 4] = ["Position", "Position, Scale, Rotation", "Perspective", "Subspace Warp"]; +pub const FRAMINGS: [&str; 4] = ["Stabilize Only", "Stabilize, Crop", "Stabilize, Crop, Auto-scale", "Stabilize, Synthesize Edges"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "Warp Stabilizer", + category: "Distort", + params: vec![ + p("stabilization/result", "Result", Value::Enum(0), popup(&RESULTS)), + p("stabilization/smoothness", "Smoothness", num(50.0), slider(0.0, 1000.0, 0.0, 100.0, 0)), + p("stabilization/method", "Method", Value::Enum(3), popup(&METHODS)), + p("stabilization/preserveScale", "Preserve Scale", Value::Bool(false), ParamUi::Checkbox), + p("borders/framing", "Framing", Value::Enum(2), popup(&FRAMINGS)), + p("borders/autoScale/maximumScale", "Maximum Scale", num(150.0), slider(100.0, 1000.0, 100.0, 200.0, 0)), + p("borders/autoScale/actionSafeMargin", "Action-safe Margin", num(0.0), slider(0.0, 45.0, 0.0, 20.0, 0)), + p("borders/additionalScale", "Additional Scale", num(100.0), slider(1.0, 1000.0, 50.0, 150.0, 0)), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/rollingShutterRipple", "Rolling Shutter Ripple", Value::Enum(0), popup(&["Automatic Reduction", "Enhanced Reduction"])), + p("advanced/cropLessSmoothMore", "Crop Less <-> Smooth More", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("advanced/synthesizeInputRange", "Synthesize Input Range (seconds)", num(1.0), slider(0.0, 10.0, 0.0, 5.0, 2)), + p("advanced/synthesizeEdgeFeather", "Synthesize Edge Feather", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("advanced/edgeCropping/left", "Left", num(0.0), slider(0.0, 1000.0, 0.0, 50.0, 0)), + p("advanced/edgeCropping/top", "Top", num(0.0), slider(0.0, 1000.0, 0.0, 50.0, 0)), + p("advanced/edgeCropping/right", "Right", num(0.0), slider(0.0, 1000.0, 0.0, 50.0, 0)), + p("advanced/edgeCropping/bottom", "Bottom", num(0.0), slider(0.0, 1000.0, 0.0, 50.0, 0)), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p("advanced/showTrackPoints", "Show Track Points", Value::Bool(false), ParamUi::Checkbox), + p("advanced/trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 200.0, 0)), + p(ANALYSIS, "Analysis", Value::Str(String::new()), ParamUi::Hidden), + p(ANALYSIS_KEY, "Analysis Key", Value::Str(String::new()), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +/// A parameter by its spec id (`stabilization/smoothness`), from either flattened instance +/// params (nested groups become `#n/` prefixes, see [`crate::flatten_params`]) or plain ids. +pub fn param<'a>(params: &'a Params, id: &str) -> Option<&'a Value> { + if let Some(v) = params.get(id) { + return Some(v); + } + let mut pre = String::new(); + let segs: Vec<&str> = id.split('/').collect(); + let (last, groups) = segs.split_last()?; + for g in groups { + pre = params.group(&pre, g)?; + } + params.get(&format!("{pre}{last}")) +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Stabilization settings from the instance's parameters (`fps`: the analysis' frame rate). +pub fn settings(params: &Params, fps: f64) -> StabSettings { + StabSettings { + result: if e(params, "stabilization/result") == 1 { StabResult::NoMotion } else { StabResult::SmoothMotion }, + smoothness: f(params, "stabilization/smoothness"), + method: match e(params, "stabilization/method") { + 0 => Method::Position, + 1 => Method::Similarity, + 2 => Method::Perspective, + _ => Method::SubspaceWarp, + }, + preserve_scale: b(params, "stabilization/preserveScale"), + framing: match e(params, "borders/framing") { + 0 => Framing::StabilizeOnly, + 1 => Framing::StabilizeCrop, + 3 => Framing::SynthesizeEdges, + _ => Framing::StabilizeCropAutoScale, + }, + max_scale: f(params, "borders/autoScale/maximumScale"), + action_safe: f(params, "borders/autoScale/actionSafeMargin"), + additional_scale: f(params, "borders/additionalScale"), + // AE's 0…100 slider (50 = balanced) onto the planner's −100…100. + crop_less_smooth_more: (f(params, "advanced/cropLessSmoothMore") - 50.0) * 2.0, + fps, + } +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +struct KeyHasher(u64); +impl Hasher for KeyHasher { + fn finish(&self) -> u64 { + self.0 + } + fn write(&mut self, bytes: &[u8]) { + for b in bytes { + self.0 ^= *b as u64; + self.0 = self.0.wrapping_mul(0x0100_0000_01b3); + } + } +} + +fn settings_key(s: &StabSettings) -> u64 { + let mut h = KeyHasher(0x9e37_79b9_7f4a_7c15); + for v in [s.smoothness, s.max_scale, s.action_safe, s.additional_scale, s.crop_less_smooth_more, s.fps] { + h.write_u64(v.to_bits()); + } + h.write(format!("{:?}{:?}{:?}{:?}", s.result, s.method, s.preserve_scale, s.framing).as_bytes()); + h.finish() +} + +type AnalysisCache = Mutex>>; +type PlanCache = Mutex>>; + +/// Parsed analyses by content hash (parsing a long clip's JSON every frame would dominate). +fn analysis_cached(json: &str) -> Option<(u64, Arc)> { + static C: OnceLock = OnceLock::new(); + let key = fnv(json.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some((key, a.clone())); + } + let a = Arc::new(WarpAnalysis::from_json(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some((key, a)) +} + +fn plan_cached(akey: u64, a: &WarpAnalysis, s: &StabSettings) -> Arc { + static C: OnceLock = OnceLock::new(); + let key = (akey, settings_key(s)); + let c = C.get_or_init(Default::default); + if let Some(p) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return p.clone(); + } + let p = Arc::new(plan(a, s)); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 32 { + m.clear(); + } + m.insert(key, p.clone()); + p +} + +/// The stored analysis of an instance (`None` when not analysed). +pub fn analysis(params: &Params) -> Option> { + let j = s(params, ANALYSIS); + if j.is_empty() { + return None; + } + analysis_cached(j).map(|(_, a)| a).filter(|a| !a.is_empty()) +} + +/// The stabilization of an instance at layer time `time`: (frame index, plan). +pub fn plan_at(params: &Params, time: f64) -> Option<(usize, Arc, Arc)> { + let j = s(params, ANALYSIS); + if j.is_empty() { + return None; + } + let (key, a) = analysis_cached(j)?; + let k = a.frame_at(time)?; + let fps = if a.frame_duration > 0.0 { 1.0 / a.frame_duration } else { 30.0 }; + let pl = plan_cached(key, &a, &settings(params, fps)); + (k < pl.warps.len()).then_some((k, a, pl)) +} + +/// Motion from frame `k` to frame `j` (scene points of frame k → frame j), from the analysed +/// frame-to-frame homographies. +pub fn motion_between(a: &WarpAnalysis, k: usize, j: usize) -> Homography { + let mut m = Homography::IDENTITY; + if j > k { + for i in k + 1..=j { + m = a.frames[i].motion(Method::Perspective).then_after(&m); + } + } else { + for i in (j + 1..=k).rev() { + let inv = a.frames[i].motion(Method::Perspective).inverse().unwrap_or(Homography::IDENTITY); + m = inv.then_after(&m); + } + } + m +} + +fn inside(r: &[f64; 4], p: [f64; 2]) -> bool { + p[0] >= r[0] && p[0] <= r[2] && p[1] >= r[1] && p[1] <= r[3] +} + +/// Distance from `p` to the nearest edge of `r` (positive inside). +fn inset(r: &[f64; 4], p: [f64; 2]) -> f64 { + (p[0] - r[0]).min(r[2] - p[0]).min(p[1] - r[1]).min(r[3] - p[1]) +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn render(ctx: &EffectCtx, buf: Buf) -> Buf { + let Some((k, a, pl)) = plan_at(ctx.params, ctx.time) else { return buf }; + let size = a.size; + if (size[0] - ctx.layer_size[0]).abs() > 0.5 || (size[1] - ctx.layer_size[1]).abs() > 0.5 || ctx.adjustment { + return buf; + } + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let Some(inv) = pl.warps[k].inverse() else { return buf }; + let (ow, oh) = (((size[0] * sc).ceil() as u32).max(1), ((size[1] * sc).ceil() as u32).max(1)); + let synth = matches!(settings(ctx.params, 30.0).framing, Framing::SynthesizeEdges); + let crop = [ + f(ctx.params, "advanced/edgeCropping/left").max(0.0), + f(ctx.params, "advanced/edgeCropping/top").max(0.0), + size[0] - f(ctx.params, "advanced/edgeCropping/right").max(0.0), + size[1] - f(ctx.params, "advanced/edgeCropping/bottom").max(0.0), + ]; + let valid = if synth { crop } else { [0.0, 0.0, size[0], size[1]] }; + let feather = if synth { f(ctx.params, "advanced/synthesizeEdgeFeather").max(0.0) } else { 0.0 }; + let src = &buf; + // Inside the frame, edge pixels extend to the frame's edge; outside it is transparent. + let sample = |b: &Buf, q: [f64; 2]| { + if q[0] < 0.0 || q[1] < 0.0 || q[0] > size[0] || q[1] > size[1] { + return [0.0; 4]; + } + b.img.sample_bilinear_clamped(q[0] * b.scale + b.offset[0], q[1] * b.scale + b.offset[1]) + }; + let mut out = Image::new(ow, oh); + // Source position (frame k layer pixels) of every output pixel, and how much of the pixel + // still needs synthesizing (0 = none). + let need_m = Mutex::new(if synth { vec![0.0f32; (ow * oh) as usize] } else { vec![] }); + out.rows_mut().for_each(|(y, row)| { + let mut local = vec![]; + for (x, px) in row.iter_mut().enumerate() { + let p = [(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]; + if let Some(c) = &pl.crop + && !inside(c, p) + { + continue; + } + let q = inv.apply(p); + if !synth { + *px = sample(src, q); + continue; + } + let d = inset(&valid, q); + if d >= feather.max(0.5) { + *px = sample(src, q); + } else { + let w = if d <= 0.0 { + 1.0 + } else if feather > 0.0 { + (1.0 - d / feather) as f32 + } else { + 1.0 + }; + if d > 0.0 { + *px = sample(src, q); + } + local.push((x, w)); + } + } + if !local.is_empty() { + let mut n = need_m.lock().unwrap_or_else(|e| e.into_inner()); + for (x, w) in local { + n[y * ow as usize + x] = w; + } + } + }); + let need = need_m.into_inner().unwrap_or_else(|e| e.into_inner()); + if synth && need.iter().any(|w| *w > 0.0) { + synthesize(ctx, &a, k, &inv, &crop, sc, &mut out, &need); + } + if b(ctx.params, "advanced/showTrackPoints") { + let r = (2.5 * f(ctx.params, "advanced/trackPointSize") / 100.0 * sc).max(0.5); + for q in &a.frames[k].points { + let o = pl.warps[k].apply([q[0] as f64, q[1] as f64]); + dot(&mut out, [o[0] * sc, o[1] * sc], r, [0.25, 0.75, 1.0, 1.0]); + } + } + Buf { img: out, offset: [0.0; 2], scale: sc } +} + +/// Fill the pixels flagged in `need` (weight = how much synthesized content they take) from +/// neighbouring frames, nearest first. +#[allow(clippy::too_many_arguments)] +fn synthesize(ctx: &EffectCtx, a: &WarpAnalysis, k: usize, inv: &Homography, crop: &[f64; 4], sc: f64, out: &mut Image, need: &[f32]) { + let Some(host) = ctx.env.host else { return }; + let n = a.frames.len(); + let range = ((f(ctx.params, "advanced/synthesizeInputRange").max(0.0) / a.frame_duration.max(1e-6)).round() as usize).min(60); + let ow = out.width as usize; + let mut filled = vec![false; need.len()]; + let todo: Vec = (0..need.len()).filter(|i| need[*i] > 0.0).collect(); + let mut left = todo.len(); + for d in 1..=range { + if left == 0 { + break; + } + for j in [k as i64 + d as i64, k as i64 - d as i64] { + if j < 0 || j as usize >= n || left == 0 { + continue; + } + let j = j as usize; + let Some(nb) = host.self_at(a.start + j as f64 * a.frame_duration, ctx.env.effect_index) else { continue }; + let m = motion_between(a, k, j); + let hits: Vec<(usize, Px)> = todo + .par_iter() + .filter(|i| !filled[**i]) + .filter_map(|i| { + let (x, y) = (i % ow, i / ow); + let q = inv.apply([(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]); + let qj = m.apply(q); + inside(crop, qj).then(|| (*i, nb.img.sample_bilinear_clamped(qj[0] * nb.scale + nb.offset[0], qj[1] * nb.scale + nb.offset[1]))) + }) + .collect(); + for (i, px) in hits { + filled[i] = true; + left -= 1; + out.data[i] = lerp_px(out.data[i], px, need[i]); + } + } + } +} + +fn dot(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + let (x0, x1) = ((c[0] - r - 1.0).floor().max(0.0) as i64, (c[0] + r + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r - 1.0).floor().max(0.0) as i64, (c[1] + r + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]) - r).clamp(-0.5, 0.5); + let cov = (0.5 - d) as f32; + if cov > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, cov); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::stabilize::FrameMotion; + + fn analysis_json(n: usize, w: f64, h: f64, shift: impl Fn(usize) -> [f64; 2]) -> String { + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 25.0, size: [w, h], detailed: false, frames: vec![] }; + for k in 0..n { + let mut fm = FrameMotion::default(); + if k > 0 { + let (p, q) = (shift(k - 1), shift(k)); + let d = [q[0] - p[0], q[1] - p[1]]; + fm.t = d; + fm.s = [1.0, 0.0, d[0], d[1]]; + fm.h = [1.0, 0.0, d[0], 0.0, 1.0, d[1], 0.0, 0.0]; + } + a.frames.push(fm); + } + a.to_json() + } + + fn params(json: &str, framing: u32, result: u32) -> Params { + let s = &specs()[0]; + let mut p = Params { values: s.params.iter().map(|p| (p.id.to_string(), p.default.clone())).collect() }; + p.values.insert(ANALYSIS.into(), Value::Str(json.into())); + p.values.insert("borders/framing".into(), Value::Enum(framing)); + p.values.insert("stabilization/result".into(), Value::Enum(result)); + p + } + + /// A white dot at `c` on black. + fn frame(c: [f64; 2]) -> Image { + let mut im = Image::filled(64, 48, [0.0, 0.0, 0.0, 1.0]); + dot(&mut im, c, 3.0, [1.0; 4]); + im + } + + fn centroid(im: &Image) -> [f64; 2] { + let (mut sx, mut sy, mut sw) = (0.0, 0.0, 0.0); + for y in 0..im.height { + for x in 0..im.width { + let v = im.get(x as i64, y as i64)[0] as f64; + sx += v * (x as f64 + 0.5); + sy += v * (y as f64 + 0.5); + sw += v; + } + } + [sx / sw, sy / sw] + } + + #[test] + fn no_motion_locks_the_scene_and_unanalysed_passes_through() { + let shift = |k: usize| [((k * 7) % 5) as f64 - 2.0, ((k * 3) % 4) as f64 - 1.5]; + let json = analysis_json(10, 64.0, 48.0, shift); + let ps = params(&json, 0, 1); + let mut cs = vec![]; + for k in 0..10 { + let d = shift(k); + let img = frame([32.0 + d[0], 24.0 + d[1]]); + let ctx = EffectCtx { params: &ps, time: k as f64 / 25.0, layer_size: [64.0, 48.0], seed: 1, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img, offset: [0.0; 2], scale: 1.0 }); + cs.push(centroid(&out.img)); + } + for c in &cs { + assert!((c[0] - cs[0][0]).abs() < 0.05 && (c[1] - cs[0][1]).abs() < 0.05, "{cs:?}"); + } + // Not analysed: untouched. + let ps = params("", 0, 1); + let img = frame([10.0, 10.0]); + let ctx = EffectCtx { params: &ps, time: 0.0, layer_size: [64.0, 48.0], seed: 1, adjustment: false, env: EffectEnv::default() }; + assert_eq!(render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }).img.data, img.data); + } + + #[test] + fn nested_param_lookup() { + let mut p = Params::default(); + p.values.insert("#2/@match".into(), Value::Str("borders".into())); + p.values.insert("#1/@match".into(), Value::Str("stabilization".into())); + p.values.insert("#2/#1/@match".into(), Value::Str("autoScale".into())); + p.values.insert("#2/#1/maximumScale".into(), num(120.0)); + p.values.insert("#1/smoothness".into(), num(12.0)); + assert_eq!(f(&p, "borders/autoScale/maximumScale"), 120.0); + assert_eq!(f(&p, "stabilization/smoothness"), 12.0); + } +} diff --git a/crates/engine/src/commands/mask_interp.rs b/crates/engine/src/commands/mask_interp.rs new file mode 100644 index 00000000..5df40c6d --- /dev/null +++ b/crates/engine/src/commands/mask_interp.rs @@ -0,0 +1,390 @@ +//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode) and Smart Mask +//! Interpolation (Window ▸ Mask Interpolation). + +use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV}; +use effectcraft_path::interp::{self, AddVertices, InterpOpts, Matching}; +use effectcraft_project::{Layer, LayerId, Uid}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_mut, layer_p, str_p}; +use crate::mask_track::{MaskMethod, MaskWork}; +use crate::tracking::Direction; +use crate::{EngineError, Result, Session, cmd}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.mask", + "Track Mask", + ["Animation"], + None, + "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", + has_mask, + track_mask + ), + cmd!( + "track.maskMethod", + "Mask Tracking Method", + [], + None, + "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective}", + super::always, + mask_method + ), + cmd!( + "mask.interpolate", + "Apply Mask Interpolation", + [], + None, + "{layer?, mask?, times?: [from, to] (s; default the selected Mask Path keys), keyframeRate?: n|auto, keyframeFields?, linearVertexPaths?, bendingResistance? (0-100), quality? (0-100), addVertices?: n|false, addVerticesUnit?: pixels|total|percent, matchingMethod?: auto|curve|polyline, oneToOne?, firstVerticesMatch?}", + has_comp, + interpolate + ), + cmd!( + "mask.interpolationOptions", + "Mask Interpolation Options", + [], + None, + "{keyframeRate?: n|auto, keyframeFields?, linearVertexPaths?, bendingResistance?, quality?, addVertices?: n|false, addVerticesUnit?: pixels|total|percent, matchingMethod?: auto|curve|polyline, oneToOne?, firstVerticesMatch?}", + super::always, + interp_options + ), + ] +} + +// ---------------------------------------------------------------- mask tracking + +/// The mask a command works on: `mask` (uid, 1-based index or name), else the selected mask +/// (Timeline selection or selected vertices), else the layer's only mask. +pub(crate) fn mask_uid(s: &Session, layer: &Layer, p: &Value) -> Option { + let masks = layer.masks()?; + if let Some(k) = p.get("mask") { + return match k { + Value::Number(n) => { + let n = n.as_u64()?; + masks.groups().find(|g| g.uid == n).or_else(|| masks.groups().nth((n as usize).checked_sub(1)?)).map(|g| g.uid) + } + Value::String(name) => masks.groups().find(|g| &g.name == name).map(|g| g.uid), + _ => None, + }; + } + let is_mask = |u: Uid| masks.groups().any(|g| g.uid == u); + if let Some(u) = s.state.selected_props.iter().filter(|(l, _)| *l == layer.id).map(|(_, u)| *u).find(|u| is_mask(*u)) { + return Some(u); + } + // A selected Mask Path property selects its mask. + if let Some(u) = s + .state + .selected_props + .iter() + .filter(|(l, _)| *l == layer.id) + .find_map(|(_, u)| masks.groups().find(|g| g.get("path").is_some_and(|pr| pr.uid == *u)).map(|g| g.uid)) + { + return Some(u); + } + if let Some(v) = s.state.selected_vertices.iter().find(|v| v.layer == layer.id && is_mask(v.mask)) { + return Some(v.mask); + } + let mut it = masks.groups(); + match (it.next(), it.next()) { + (Some(g), None) => Some(g.uid), + _ => None, + } +} + +/// The selected layer and mask, if any (the Tracker panel's mask mode). +pub fn selected_mask(s: &Session) -> Option<(LayerId, Uid)> { + let comp = s.active_comp()?; + for lid in &s.state.selected_layers { + let Some(l) = comp.layer(*lid) else { continue }; + let explicit = + s.state.selected_props.iter().any(|(pl, u)| *pl == *lid && l.masks().is_some_and(|m| m.find_group(*u).is_some() || m.find(*u).is_some())) + || s.state.selected_vertices.iter().any(|v| v.layer == *lid); + if explicit && let Some(u) = mask_uid(s, l, &json!({})) { + return Some((*lid, u)); + } + } + None +} + +fn has_mask(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + selected_mask(s).map(|_| ()).ok_or_else(|| "select a mask to track".into()) +} + +fn method_p(p: &Value, cmd: &str) -> Result> { + match str_p(p, "method") { + Some(m) => MaskMethod::from_name(m) + .map(Some) + .ok_or_else(|| bad(cmd, format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective)"))), + None => Ok(None), + } +} + +fn mask_method(s: &mut Session, p: &Value) -> Result { + let m = method_p(p, "track.maskMethod")?.ok_or_else(|| bad("track.maskMethod", "missing `method`"))?; + s.state.mask_track_method = m; + s.bump(); + Ok(json!(m.id())) +} + +fn track_mask(s: &mut Session, p: &Value) -> Result { + const CMD: &str = "track.mask"; + let (cid, lid) = match p.get("layer") { + Some(_) => layer_p(s, p, CMD)?, + None => match selected_mask(s) { + Some((l, _)) => (s.active_comp_id().ok_or(EngineError::NoComp)?, l), + None => layer_p(s, p, CMD)?, + }, + }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let uid = mask_uid(s, layer, p).ok_or_else(|| bad(CMD, format!("layer `{}` has no such mask (select one or pass `mask`)", layer.name)))?; + if !layer.has_video() || matches!(layer.source, effectcraft_project::LayerSource::Text | effectcraft_project::LayerSource::Shape) { + return Err(bad(CMD, "the layer has no pixels to track")); + } + let method = method_p(p, CMD)?.unwrap_or(s.state.mask_track_method); + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad(CMD, "direction: forward|backward|frameForward|frameBackward"))?, + None => Direction::Forward, + }; + let now = comp.frame_rate.snap_nearest(if s.state.active_comp == Some(cid) { s.time() } else { Tick::ZERO }); + let start = f_p(p, "start").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v))); + let end = f_p(p, "end").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v))); + let times = super::track::analysis_times(comp, layer, dir, now, start, end); + if times.len() < 2 { + return Err(bad(CMD, "nothing to track in that direction (at the layer's end)")); + } + let mg = layer.props.find_group(uid).ok_or_else(|| bad(CMD, "no mask"))?; + let KV::Path(path) = mg.get("path").map(|pr| pr.value_at(layer.layer_time(times[0]))).unwrap_or(KV::Scalar(0.0)) else { + return Err(bad(CMD, "the mask has no path")); + }; + if path.len() < 2 { + return Err(bad(CMD, "the mask needs at least two vertices")); + } + let work = MaskWork { + project: s.project.clone(), + footage: s.footage.clone(), + expr: s.expr.clone(), + cache: s.layer_cache.clone(), + comp: cid, + layer: lid, + path, + method, + times: times.clone(), + }; + s.state.mask_track_method = method; + let wait = b_p(p, "wait").unwrap_or(false); + s.start_mask_track(work, uid, dir, wait).map_err(EngineError::Other)?; + let prog = s.mask_track_progress(); + Ok(json!({ + "mask": uid, + "method": method.id(), + "frames": times.len() - 1, + "start": times[0].seconds(), + "end": times.last().map(|t| t.seconds()), + "running": s.is_mask_tracking(), + "progress": prog, + })) +} + +// ---------------------------------------------------------------- mask interpolation + +/// Mask Interpolation panel options (kept in the editor state, serde for agents). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct MaskInterpOptions { + /// Keyframes per second; `None` = Auto (the comp frame rate). + pub keyframe_rate: Option, + /// Keyframe Fields (doubles the rate). + pub keyframe_fields: bool, + pub linear_vertex_paths: bool, + pub bending_resistance: f64, + pub quality: f64, + /// Add Mask Shape Vertices (value, unit: pixels | total | percent). + pub add_vertices: Option, + pub add_vertices_unit: String, + pub matching_method: String, + pub one_to_one: bool, + pub first_vertices_match: bool, +} + +impl Default for MaskInterpOptions { + fn default() -> Self { + MaskInterpOptions { + keyframe_rate: None, + keyframe_fields: false, + linear_vertex_paths: false, + bending_resistance: 50.0, + quality: 50.0, + add_vertices: Some(5.0), + add_vertices_unit: "pixels".into(), + matching_method: "auto".into(), + one_to_one: false, + first_vertices_match: true, + } + } +} + +impl MaskInterpOptions { + pub fn interp(&self) -> InterpOpts { + InterpOpts { + linear: self.linear_vertex_paths, + bending_resistance: self.bending_resistance.clamp(0.0, 100.0), + quality: self.quality.clamp(0.0, 100.0), + add_vertices: self.add_vertices.map(|v| match self.add_vertices_unit.as_str() { + "total" => AddVertices::Total(v.round().max(2.0) as usize), + "percent" => AddVertices::Percent(v), + _ => AddVertices::Pixels(v), + }), + matching: match self.matching_method.as_str() { + "curve" => Matching::Curve, + "polyline" => Matching::Polyline, + _ => Matching::Auto, + }, + one_to_one: self.one_to_one, + first_vertices_match: self.first_vertices_match, + } + } +} + +/// Merge option parameters into `o`. +fn apply_options(o: &mut MaskInterpOptions, p: &Value, cmd: &str) -> Result<()> { + match p.get("keyframeRate") { + Some(Value::String(s)) if s.eq_ignore_ascii_case("auto") => o.keyframe_rate = None, + Some(Value::Null) => o.keyframe_rate = None, + Some(v) => o.keyframe_rate = Some(v.as_f64().filter(|r| *r > 0.0).ok_or_else(|| bad(cmd, "keyframeRate: a positive number or \"auto\""))?.min(1000.0)), + None => {} + } + for (k, f) in [ + ("keyframeFields", &mut o.keyframe_fields), + ("linearVertexPaths", &mut o.linear_vertex_paths), + ("oneToOne", &mut o.one_to_one), + ("firstVerticesMatch", &mut o.first_vertices_match), + ] { + if let Some(v) = b_p(p, k) { + *f = v; + } + } + if let Some(v) = f_p(p, "bendingResistance") { + o.bending_resistance = v.clamp(0.0, 100.0); + } + if let Some(v) = f_p(p, "quality") { + o.quality = v.clamp(0.0, 100.0); + } + match p.get("addVertices") { + Some(Value::Bool(false)) | Some(Value::Null) => o.add_vertices = None, + Some(Value::Bool(true)) => o.add_vertices = Some(o.add_vertices.unwrap_or(5.0)), + Some(v) => o.add_vertices = Some(v.as_f64().filter(|x| *x > 0.0).ok_or_else(|| bad(cmd, "addVertices: a positive number or false"))?), + None => {} + } + if let Some(u) = str_p(p, "addVerticesUnit") { + let u = u.to_ascii_lowercase(); + o.add_vertices_unit = match u.as_str() { + "pixels" | "px" | "pixelsbetweenvertices" => "pixels", + "total" | "totalvertices" => "total", + "percent" | "%" | "percentageofoutline" => "percent", + _ => return Err(bad(cmd, "addVerticesUnit: pixels|total|percent")), + } + .into(); + } + if let Some(m) = str_p(p, "matchingMethod") { + let m = m.to_ascii_lowercase(); + if !["auto", "curve", "polyline"].contains(&m.as_str()) { + return Err(bad(cmd, "matchingMethod: auto|curve|polyline")); + } + o.matching_method = m; + } + Ok(()) +} + +fn interp_options(s: &mut Session, p: &Value) -> Result { + let mut o = s.state.mask_interp.clone(); + apply_options(&mut o, p, "mask.interpolationOptions")?; + s.state.mask_interp = o.clone(); + s.bump(); + Ok(serde_json::to_value(o).unwrap_or(Value::Null)) +} + +fn interpolate(s: &mut Session, p: &Value) -> Result { + const CMD: &str = "mask.interpolate"; + let mut o = s.state.mask_interp.clone(); + apply_options(&mut o, p, CMD)?; + // The layer: given, else the layer of selected Mask Path keys, else the selection. + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lid = match p.get("layer") { + Some(v) => super::resolve_layer(comp, v).ok_or_else(|| bad(CMD, format!("no layer {v}")))?, + None => match s.state.selected_keys.iter().find(|k| comp.layer(k.layer).is_some_and(|l| l.masks().is_some_and(|m| m.find(k.prop).is_some()))) { + Some(k) => k.layer, + None => layer_p(s, p, CMD)?.1, + }, + }; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let masks = layer.masks().ok_or_else(|| bad(CMD, "the layer has no masks"))?; + let uid = match p.get("mask") { + Some(_) => mask_uid(s, layer, p), + None => s + .state + .selected_keys + .iter() + .filter(|k| k.layer == lid) + .find_map(|k| masks.groups().find(|g| g.get("path").is_some_and(|pr| pr.uid == k.prop)).map(|g| g.uid)) + .or_else(|| mask_uid(s, layer, p)), + } + .ok_or_else(|| bad(CMD, "no such mask (select two Mask Path keyframes or pass `mask`)"))?; + let mg = layer.props.find_group(uid).ok_or_else(|| bad(CMD, "no mask"))?; + let path_prop = mg.get("path").ok_or_else(|| bad(CMD, "the mask has no path"))?; + // Key times (layer time): explicit, else selected keys on this Mask Path. + let mut times: Vec = match p.get("times").and_then(Value::as_array) { + Some(a) => { + let ts: Vec = a.iter().filter_map(Value::as_f64).map(|t| layer.layer_time(comp.frame_rate.snap_nearest(Tick::from_seconds_f64(t)))).collect(); + ts.into_iter().filter_map(|t| path_prop.keys.iter().find(|k| (k.time.0 - t.0).abs() <= comp.frame_duration().0 / 2).map(|k| k.time)).collect() + } + None => s.state.selected_keys.iter().filter(|k| k.layer == lid && k.prop == path_prop.uid).map(|k| k.time).collect(), + }; + times.sort(); + times.dedup(); + if times.len() < 2 { + return Err(bad(CMD, "select two or more Mask Path keyframes (or pass `times` of existing keys)")); + } + let rate = o.keyframe_rate.unwrap_or_else(|| comp.frame_rate.as_f64()) * if o.keyframe_fields { 2.0 } else { 1.0 }; + let step = Tick::from_seconds_f64(1.0 / rate.max(0.01)); + let opts = o.interp(); + let shapes: Vec = times.iter().filter_map(|t| path_prop.keys.iter().find(|k| k.time == *t).and_then(|k| k.value.as_path().cloned())).collect(); + if shapes.len() != times.len() { + return Err(bad(CMD, "the selected keys must be Mask Path keys")); + } + // (time, shape) for every new or replaced key. + let mut out: Vec<(Tick, ShapePath)> = vec![]; + let mut vertices = 0; + for i in 0..times.len() - 1 { + let (t0, t1) = (times[i], times[i + 1]); + let (a, b) = interp::correspond(&shapes[i], &shapes[i + 1], &opts); + vertices = vertices.max(a.len()); + out.push((t0, a.clone())); + let mut t = t0 + step; + while t1.0 - t.0 > step.0 / 4 { + let u = (t.0 - t0.0) as f64 / (t1.0 - t0.0) as f64; + out.push((t, interp::interpolate(&a, &b, u, &opts))); + t += step; + } + out.push((t1, b)); + } + let created = out.len() - times.len().min(out.len()); + let (first, last) = (times[0], times[times.len() - 1]); + s.state.mask_interp = o; + s.edit("Mask Interpolation", None, |proj, _| { + let l = layer_mut(proj, cid, lid)?; + let pr = l.props.find_group_mut(uid).and_then(|g| g.get_mut("path")).ok_or_else(|| bad(CMD, "no mask path"))?; + // Keys strictly inside the interpolated span are replaced. + pr.keys.retain(|k| k.time <= first || k.time >= last || times.contains(&k.time)); + for (t, sp) in &out { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*t, KV::Path(sp.clone()))); + } + Ok(()) + })?; + Ok(json!({"mask": uid, "keys": created, "vertices": vertices, "rate": rate})) +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 26db85c9..4537752f 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -20,6 +20,7 @@ mod link; mod lottie; mod markers; mod mask; +pub mod mask_interp; pub mod paint; mod paths; mod project_items; @@ -37,6 +38,7 @@ mod time; mod track; mod view; pub mod viewer_cmds; +mod warp_cmds; #[cfg(test)] pub(crate) use mask::split_segment as split_segment_for_tests; @@ -125,6 +127,8 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(comp_more::specs()); v.extend(frontend::specs()); v.extend(track::specs()); + v.extend(mask_interp::specs()); + v.extend(warp_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f7868d9c..f670ffd7 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -60,8 +60,6 @@ pub fn specs() -> Vec { stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), stub!("track.camera", "Track Camera", ["Animation"], "{}"), - stub!("track.warpStabilizer", "Warp Stabilizer VFX", ["Animation"], "{}"), - stub!("track.mask", "Track Mask", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/commands/track.rs b/crates/engine/src/commands/track.rs index 2588c061..928bf63b 100644 --- a/crates/engine/src/commands/track.rs +++ b/crates/engine/src/commands/track.rs @@ -69,7 +69,7 @@ pub fn specs() -> Vec { has_track, analyze ), - cmd!("track.stop", "Stop Analysis", [], None, "{}", is_tracking, |s, _| Ok(json!({"stopped": s.stop_track()}))), + cmd!("track.stop", "Stop Analysis", [], None, "{}", is_tracking, |s, _| Ok(json!({"stopped": s.stop_track() | s.stop_mask_track() | s.stop_warp()}))), cmd!("track.apply", "Apply", [], None, "{layer?, tracker?, dimensions?: xy|x|y}", has_track, apply), cmd!("track.reset", "Reset", [], None, "{layer?, tracker?}", has_track, reset), cmd!("track.delete", "Delete Tracker", [], None, "{layer?, tracker?}", has_track, delete), @@ -105,7 +105,7 @@ fn has_track(s: &Session) -> std::result::Result<(), String> { } fn is_tracking(s: &Session) -> std::result::Result<(), String> { - if s.is_tracking() { Ok(()) } else { Err("no track analysis is running".into()) } + if s.is_tracking() || s.is_mask_tracking() || s.is_warp_analyzing() { Ok(()) } else { Err("no track analysis is running".into()) } } // ---------- addressing ---------- @@ -432,31 +432,10 @@ fn analyze(s: &mut Session, p: &V) -> Result { let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; let (tg, settings) = layer.tracker(uid).ok_or_else(|| bad("track.analyze", "no tracker"))?; - let fd = comp.frame_duration(); let now = comp.frame_rate.snap_nearest(if s.state.active_comp == Some(cid) { s.time() } else { Tick::ZERO }); let start = f_p(p, "start").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v))); let end = f_p(p, "end").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v))); - let lo = layer.in_point.max(Tick::ZERO); - let hi = layer.out_point.min(comp.duration) - fd; - let mut times = vec![]; - match dir { - Direction::Forward | Direction::FrameForward => { - let mut t = start.unwrap_or(now).clamp(lo, hi); - let stop = if dir == Direction::FrameForward { t + fd } else { end.unwrap_or(hi).min(hi) }; - while t <= stop && t <= hi { - times.push(t); - t += fd; - } - } - Direction::Backward | Direction::FrameBackward => { - let mut t = end.unwrap_or(now).clamp(lo, hi); - let stop = if dir == Direction::FrameBackward { t - fd } else { start.unwrap_or(lo).max(lo) }; - while t >= stop && t >= lo { - times.push(t); - t -= fd; - } - } - } + let times = analysis_times(comp, layer, dir, now, start, end); if times.len() < 2 { return Err(bad("track.analyze", "nothing to analyze in that direction (at the layer's end)")); } @@ -489,6 +468,34 @@ fn analyze(s: &mut Session, p: &V) -> Result { })) } +/// Comp times to analyse from the CTI (`now`) in `dir`, bounded by `start` / `end` and the +/// layer's In/Out points; the first is the start frame. +pub(crate) fn analysis_times(comp: &effectcraft_project::Comp, layer: &Layer, dir: Direction, now: Tick, start: Option, end: Option) -> Vec { + let fd = comp.frame_duration(); + let lo = layer.in_point.max(Tick::ZERO); + let hi = layer.out_point.min(comp.duration) - fd; + let mut times = vec![]; + match dir { + Direction::Forward | Direction::FrameForward => { + let mut t = start.unwrap_or(now).clamp(lo, hi); + let stop = if dir == Direction::FrameForward { t + fd } else { end.unwrap_or(hi).min(hi) }; + while t <= stop && t <= hi { + times.push(t); + t += fd; + } + } + Direction::Backward | Direction::FrameBackward => { + let mut t = end.unwrap_or(now).clamp(lo, hi); + let stop = if dir == Direction::FrameBackward { t - fd } else { start.unwrap_or(lo).max(lo) }; + while t >= stop && t >= lo { + times.push(t); + t -= fd; + } + } + } + times +} + fn apply(s: &mut Session, p: &V) -> Result { let (cid, lid, uid) = current(s, p, "track.apply")?; let dims = match str_p(p, "dimensions") { @@ -560,9 +567,10 @@ fn delete(s: &mut Session, p: &V) -> Result { fn status(s: &mut Session, p: &V) -> Result { s.poll_track(); + s.poll_mask_track(); let progress = s.track_progress(); let cur = current(s, p, "track.status").ok(); - let mut out = json!({"running": s.is_tracking(), "progress": progress}); + let mut out = json!({"running": s.is_tracking() || s.is_mask_tracking(), "progress": progress, "mask": s.mask_track_progress(), "maskMethod": s.state.mask_track_method.id()}); if let Some((cid, lid, uid)) = cur && let Some(layer) = s.project.comp(cid).and_then(|c| c.layer(lid)) && let Some((g, st)) = layer.tracker(uid) diff --git a/crates/engine/src/commands/warp_cmds.rs b/crates/engine/src/commands/warp_cmds.rs new file mode 100644 index 00000000..14f0ee4b --- /dev/null +++ b/crates/engine/src/commands/warp_cmds.rs @@ -0,0 +1,131 @@ +//! Warp Stabilizer commands (Effect ▸ Distort ▸ Warp Stabilizer's Analyze / Cancel, Animation ▸ +//! Warp Stabilizer VFX, the Tracker panel button) and its status query. + +use effectcraft_project::{GroupKind, ItemId, Layer, LayerId, PropGroup, Uid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, has_comp, layer_p}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!("track.warpStabilizer", "Warp Stabilizer VFX", ["Animation"], None, "{layer?, wait?}", can_stabilize, warp_apply), + cmd!("warp.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, warp_analyze), + cmd!("warp.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_warp()}))), + query!("warp.status", "Warp Stabilizer Status", "{layer?, effect?}", warp_status), + ] +} + +// ---------------------------------------------------------------- Warp Stabilizer + +fn can_stabilize(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment => Ok(()), + Some(_) => Err("Warp Stabilizer needs a footage or composition layer".into()), + None => Err("select a layer to stabilize".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_warp_analyzing() { Ok(()) } else { Err("no Warp Stabilizer analysis is running".into()) } +} + +fn is_warp(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::warp_stab::ID) +} + +/// (comp, layer, effect uid) of a Warp Stabilizer: `layer` / `effect` (uid, name or 1-based +/// index among the layer's effects), else a selected one, else the layer's first. +fn warp_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_warp(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_warp(g))), + }; + if let Some(g) = g.filter(|g| is_warp(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no Warp Stabilizer found (apply Effect ▸ Distort ▸ Warp Stabilizer first)")) +} + +fn warp_analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = warp_p(s, p, "warp.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_warp(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_warp_analyzing(), "progress": s.warp_progress()}); + if wait { + out["status"] = warp_status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +/// Animation ▸ Warp Stabilizer VFX / the Tracker panel button: apply the effect to the selected +/// layer and start analysing it. +fn warp_apply(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.warpStabilizer")?; + let r = s.execute("effect.apply", json!({"effect": effectcraft_effects::warp_stab::ID, "layers": [lid.0], "comp": cid.0}))?; + let uid = r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_warp(cid, lid, uid, wait).map_err(EngineError::Other)?; + Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_warp_analyzing(), "progress": s.warp_progress()})) +} + +fn warp_status(s: &mut Session, p: &Value) -> Result { + s.poll_warp(false); + let mut out = json!({"running": s.is_warp_analyzing(), "progress": s.warp_progress(), "banner": s.warp_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = warp_p(s, p, "warp.status") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, s.time_of(cid)); + let params = effectcraft_effects::flatten_params(g, &mut |pr| ctx.value(layer, pr)); + let analysis = effectcraft_effects::warp_stab::analysis(¶ms); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["analyzed"] = json!(analysis.is_some()); + out["pending"] = json!(s.warp_pending.contains(&(cid, lid, uid))); + if let Some(a) = analysis { + out["frames"] = json!(a.frames.len()); + out["size"] = json!(a.size); + let lt = layer.layer_time(s.time_of(cid)).seconds(); + if let Some((k, _, plan)) = effectcraft_effects::warp_stab::plan_at(¶ms, lt) { + let m = plan.warps[k].0; + out["frame"] = json!(k); + out["autoScale"] = json!(plan.auto_scale * 100.0); + out["crop"] = json!(plan.crop); + out["validFraction"] = json!(plan.valid_fraction); + out["warp"] = json!(m); + let pts = &a.frames[k]; + out["features"] = json!(pts.features); + out["inliers"] = json!(pts.inliers); + } + } + Ok(out) +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f64ce817..ba34488a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -13,6 +13,7 @@ pub mod commands; pub mod config; pub mod demo; pub mod links; +pub mod mask_track; pub mod menus; pub mod prefs; pub mod render_queue; @@ -20,6 +21,7 @@ mod session_settings; pub mod shortcuts; pub mod tracking; pub mod viewer; +pub mod warp; use std::sync::Arc; @@ -165,6 +167,12 @@ pub struct EditorState { /// Tracker panel ▸ Current Track: (tracked layer, tracker group uid) in the active comp. #[serde(default)] pub current_track: Option<(LayerId, Uid)>, + /// Tracker panel ▸ Method in mask mode (Track Mask). + #[serde(default)] + pub mask_track_method: mask_track::MaskMethod, + /// Mask Interpolation panel options. + #[serde(default)] + pub mask_interp: commands::mask_interp::MaskInterpOptions, /// Paint and Brushes panel options (Brush, Clone Stamp and Eraser tools). #[serde(default)] pub paint: commands::paint::PaintOptions, @@ -249,6 +257,12 @@ pub struct Session { pub render_job: Option, /// The running (or finished, not yet polled) track analysis. pub track_job: Option, + /// The running (or finished, not yet polled) mask track. + pub mask_job: Option, + /// The running (or finished, not yet polled) Warp Stabilizer analysis. + pub warp_job: Option, + /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). + pub warp_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -291,6 +305,9 @@ impl Default for Session { exporter: None, render_job: None, track_job: None, + mask_job: None, + warp_job: None, + warp_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -351,6 +368,12 @@ impl Session { let r = f(&mut p, &mut st)?; // Layer styles: one Global Light per comp, whichever layer edited it. effectcraft_project::styles::sync_global_light(&before, &mut p); + // Warp Stabilizer analyses made from other frames are cleared (and queued again). + for w in warp::invalidate(&before, &mut p) { + if !self.warp_pending.contains(&w) { + self.warp_pending.push(w); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -523,6 +546,11 @@ impl Session { self.render_job = None; self.stop_track(); self.track_job = None; + self.stop_mask_track(); + self.mask_job = None; + self.stop_warp(); + self.warp_job = None; + self.warp_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -554,6 +582,8 @@ mod tests_lottie; #[cfg(test)] mod tests_markers; #[cfg(test)] +mod tests_mask_warp; +#[cfg(test)] mod tests_menu_cmds; #[cfg(test)] mod tests_project_items; diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs new file mode 100644 index 00000000..1cdeda1e --- /dev/null +++ b/crates/engine/src/mask_track.rs @@ -0,0 +1,313 @@ +//! Mask tracking runtime (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode): follow +//! the pixels inside a mask from frame to frame and key its Mask Path on every analysed frame. +//! +//! Analysis runs on a background thread (progress, cancel; blocking with `wait` and on wasm32) +//! like the point tracker, over the layer's source frames; one undo step covers it. The +//! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the +//! chosen motion model), and every frame's motion is applied to the mask's vertices and +//! tangents, so the shape's Bezier structure is kept. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_keyframe::{Keyframe, ShapePath, Value}; +use effectcraft_project::{ItemId, LayerId, Project, Uid}; +use effectcraft_render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track as trk; +use effectcraft_track::fit::Model; +use serde::{Deserialize, Serialize}; + +use crate::tracking::{Direction, TrackProgress, lock, source_frame}; +use crate::{Event, Session}; + +/// Tracker panel ▸ Method (mask tracking). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MaskMethod { + Position, + PositionScale, + #[default] + PositionScaleRotation, + PositionScaleRotationSkew, + Perspective, +} + +impl MaskMethod { + pub const ALL: [MaskMethod; 5] = + [MaskMethod::Position, MaskMethod::PositionScale, MaskMethod::PositionScaleRotation, MaskMethod::PositionScaleRotationSkew, MaskMethod::Perspective]; + pub fn label(self) -> &'static str { + match self { + MaskMethod::Position => "Position", + MaskMethod::PositionScale => "Position & Scale", + MaskMethod::PositionScaleRotation => "Position, Scale & Rotation", + MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew", + MaskMethod::Perspective => "Perspective", + } + } + pub fn id(self) -> &'static str { + match self { + MaskMethod::Position => "position", + MaskMethod::PositionScale => "positionScale", + MaskMethod::PositionScaleRotation => "positionScaleRotation", + MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew", + MaskMethod::Perspective => "perspective", + } + } + pub fn from_name(s: &str) -> Option { + let n: String = s.chars().filter(|c| c.is_ascii_alphanumeric()).collect::().to_ascii_lowercase(); + MaskMethod::ALL + .into_iter() + .find(|m| m.id().to_ascii_lowercase() == n || m.label().chars().filter(|c| c.is_ascii_alphanumeric()).collect::().to_ascii_lowercase() == n) + } + pub fn model(self) -> Model { + match self { + MaskMethod::Position => Model::Translation, + MaskMethod::PositionScale => Model::TranslationScale, + MaskMethod::PositionScaleRotation => Model::Similarity, + MaskMethod::PositionScaleRotationSkew => Model::Affine, + MaskMethod::Perspective => Model::Homography, + } + } +} + +/// A closed polyline through a Bezier path (`steps` samples per segment), in layer pixels. +pub fn flatten(p: &ShapePath, steps: usize) -> Vec<[f64; 2]> { + let n = p.len(); + if n == 0 { + return vec![]; + } + let segs = if p.closed { n } else { n.saturating_sub(1) }; + let mut out = Vec::with_capacity(segs * steps + 1); + for i in 0..segs { + let j = (i + 1) % n; + let (a, b) = (p.vertices[i], p.vertices[j]); + let o = p.out_tangents.get(i).copied().unwrap_or([0.0; 2]); + let it = p.in_tangents.get(j).copied().unwrap_or([0.0; 2]); + let c = [a, [a[0] + o[0], a[1] + o[1]], [b[0] + it[0], b[1] + it[1]], b]; + for k in 0..steps { + let t = k as f64 / steps as f64; + let u = 1.0 - t; + let w = [u * u * u, 3.0 * u * u * t, 3.0 * u * t * t, t * t * t]; + out.push([0, 1].map(|d| w[0] * c[0][d] + w[1] * c[1][d] + w[2] * c[2][d] + w[3] * c[3][d])); + } + } + if !p.closed { + out.push(p.vertices[n - 1]); + } + out +} + +/// A path moved by a frame-to-frame motion (vertices and tangent handles). +pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath { + ShapePath { + vertices: p.vertices.iter().map(|v| h.apply(*v)).collect(), + in_tangents: p.vertices.iter().zip(&p.in_tangents).map(|(v, t)| h.apply_tangent(*v, *t)).collect(), + out_tangents: p.vertices.iter().zip(&p.out_tangents).map(|(v, t)| h.apply_tangent(*v, *t)).collect(), + closed: p.closed, + } +} + +/// One tracked frame: (layer time, comp time, mask path). +type MaskFrame = (Tick, Tick, ShapePath); + +#[derive(Default)] +pub struct MaskTrackShared { + pub cancel: AtomicBool, + pub state: Mutex, + frames: Mutex>, +} + +/// A running (or finished, not yet polled) mask track. +pub struct MaskTrackJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub mask: Uid, + pub direction: Direction, +} + +impl MaskTrackJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> TrackProgress { + lock(&self.shared.state).clone() + } +} + +/// What the worker needs, detached from the session. +pub(crate) struct MaskWork { + pub project: Arc, + pub footage: Arc, + pub expr: Option>, + pub cache: Arc, + pub comp: ItemId, + pub layer: LayerId, + /// The mask path on the first frame. + pub path: ShapePath, + pub method: MaskMethod, + /// Comp times; the first is the start frame. + pub times: Vec, +} + +fn run_work(w: MaskWork, shared: &MaskTrackShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); + lock(&shared.state).total = w.times.len().saturating_sub(1) as u64; + let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 }); + let mut path = w.path.clone(); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone())); + let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + for k in 1..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + lock(&shared.state).cancelled = true; + break; + } + let Some(cur) = next.take() else { + lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); + break; + }; + let (res, nf) = + rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + next = nf; + let Some(step) = res else { + lock(&shared.state).error = Some(format!("lost the mask's pixels at {:.3} s (too little texture inside the mask)", w.times[k].seconds())); + break; + }; + path = transform_path(&path, &step.motion); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone())); + let mut s = lock(&shared.state); + s.done = k as u64; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 }; + s.time = w.times[k].seconds(); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) { + let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return }; + let Some(g) = layer.props.find_group_mut(job.mask) else { return }; + let Some(pr) = g.get_mut("path") else { return }; + for (lt, _, path) in frames { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + } +} + +impl Session { + /// Whether a mask track is running. + pub fn is_mask_tracking(&self) -> bool { + self.mask_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Live progress of the running (or just finished) mask track. + pub fn mask_track_progress(&self) -> Option { + self.mask_job.as_ref().map(|j| j.progress()) + } + + /// Cancel the running mask track (frames tracked so far are kept). + pub fn stop_mask_track(&mut self) -> bool { + match &self.mask_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + pub(crate) fn start_mask_track(&mut self, work: MaskWork, mask: Uid, direction: Direction, wait: bool) -> Result<(), String> { + if self.is_mask_tracking() || self.is_tracking() { + return Err("a track analysis is already running".into()); + } + self.poll_mask_track(); + let wait = wait || cfg!(target_arch = "wasm32"); + self.history.undo.push(("Track Mask".into(), self.project.clone())); + self.history.redo.clear(); + self.history.merge_key = None; + let shared = Arc::new(MaskTrackShared::default()); + let (comp, layer) = (work.comp, work.layer); + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("mask-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.mask_job = Some(MaskTrackJob { shared, thread, comp, layer, mask, direction }); + self.poll_mask_track(); + Ok(()) + } + + /// Key newly tracked frames and move the CTI with them; drop the job once it finished. + /// Frontends call this every frame while tracking. + pub fn poll_mask_track(&mut self) -> bool { + let Some(job) = &self.mask_job else { return false }; + let frames = std::mem::take(&mut *lock(&job.shared.frames)); + let finished = job.is_finished(); + let changed = !frames.is_empty(); + if changed { + write_frames(Arc::make_mut(&mut self.project), job, &frames); + if self.state.active_comp == Some(job.comp) + && let Some(last) = frames.last() + { + self.state.times.insert(job.comp, last.1); + } + self.bump(); + } + if finished { + let mut job = self.mask_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let msg = match (&st.error, st.cancelled) { + (Some(e), _) => format!("Mask tracking stopped: {e}"), + (None, true) => format!("Mask tracking stopped after {} frame(s)", st.done), + _ => format!("Tracked the mask over {} frame(s) in {:.1} s", st.done, st.elapsed), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + } + changed || finished + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn methods_parse_and_paths_transform() { + assert_eq!(MaskMethod::from_name("positionScaleRotationSkew"), Some(MaskMethod::PositionScaleRotationSkew)); + assert_eq!(MaskMethod::from_name("Position & Scale"), Some(MaskMethod::PositionScale)); + assert_eq!(MaskMethod::from_name("perspective"), Some(MaskMethod::Perspective)); + let p = ShapePath::ellipse([50.0, 40.0], 40.0, 20.0); + let poly = flatten(&p, 8); + assert_eq!(poly.len(), p.len() * 8); + assert!(poly.iter().all(|q| ((q[0] - 50.0) / 20.0).powi(2) + ((q[1] - 40.0) / 10.0).powi(2) < 1.01)); + let h = trk::Homography([[0.0, -2.0, 10.0], [2.0, 0.0, 0.0], [0.0, 0.0, 1.0]]); + let q = transform_path(&p, &h); + for i in 0..p.len() { + let v = h.apply(p.vertices[i]); + assert!((q.vertices[i][0] - v[0]).abs() < 1e-9); + let t = p.out_tangents[i]; + assert!((q.out_tangents[i][0] + 2.0 * t[1]).abs() < 1e-9 && (q.out_tangents[i][1] - 2.0 * t[0]).abs() < 1e-9); + } + } +} diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index a935ebc0..38252a88 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -524,6 +524,7 @@ Layer Reset Mask | layer.mask.reset Remove Mask | layer.mask.remove Remove All Masks | layer.mask.removeAll + Track Mask | track.mask Mode None | layer.mask.mode {"mode":"None"} Add | layer.mask.mode {"mode":"Add"} @@ -932,7 +933,7 @@ Window Effects & Presets | window.panel {"panel":"effectsPresets"} | Cmd+5 Info | window.panel {"panel":"info"} | Cmd+2 Lumetri Scopes | window.unavailablePanel {"panel":"lumetriScopes"} - Mask Interpolation | window.unavailablePanel {"panel":"maskInterpolation"} + Mask Interpolation | window.panel {"panel":"maskInterpolation"} Media Browser | window.unavailablePanel {"panel":"mediaBrowser"} Metadata | window.unavailablePanel {"panel":"metadata"} Motion Sketch | window.panel {"panel":"motionSketch"} diff --git a/crates/engine/src/tests_mask_warp.rs b/crates/engine/src/tests_mask_warp.rs new file mode 100644 index 00000000..7f915e42 --- /dev/null +++ b/crates/engine/src/tests_mask_warp.rs @@ -0,0 +1,384 @@ +//! Mask tracking, Mask Interpolation and the Warp Stabilizer end to end through the commands, on +//! synthetic footage generated in-test (a rotating/scaling textured patch, a shaky camera). + +use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV}; +use effectcraft_path::interp::paths_cross; +use effectcraft_project::{LayerId, Uid}; +use effectcraft_raster::Image; +use effectcraft_render::RenderOpts; +use effectcraft_time::Tick; +use effectcraft_track::Homography; +use serde_json::json; + +use crate::Session; +use crate::tests_track::{FPS, H, W, frame_time, patch_frames, rotating_pose, setup, texture}; + +fn pose_h(f: u32) -> Homography { + let (c, rot, sc) = rotating_pose(f); + let (sn, cs) = rot.to_radians().sin_cos(); + Homography([[sc * cs, -sc * sn, c[0]], [sc * sn, sc * cs, c[1]], [0.0, 0.0, 1.0]]) +} + +fn mask_path(s: &Session, layer: LayerId, mask: Uid) -> effectcraft_project::Property { + let l = s.active_comp().unwrap().layer(layer).unwrap(); + l.props.find_group(mask).unwrap().get("path").unwrap().clone() +} + +fn new_mask(s: &mut Session, layer: LayerId, path: &ShapePath) -> Uid { + let ins: Vec<[f64; 2]> = path.in_tangents.clone(); + let outs: Vec<[f64; 2]> = path.out_tangents.clone(); + let r = s.execute("mask.new", json!({"layer": layer.0, "vertices": path.vertices, "inTangents": ins, "outTangents": outs, "closed": true})).unwrap(); + r["mask"].as_u64().unwrap() +} + +#[test] +fn track_mask_follows_a_rotating_patch() { + let (mut s, clip, _) = setup(patch_frames(rotating_pose)); + let (c0, _, _) = rotating_pose(0); + // An ellipse inside the textured patch. + let shape = ShapePath::ellipse(c0, 64.0, 52.0); + let uid = new_mask(&mut s, clip, &shape); + // Selecting the mask enables Track Mask (the Tracker panel's mask mode). + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + s.state.selected_props = vec![(clip, uid)]; + assert!(s.is_enabled("track.mask")); + assert_eq!(crate::commands::mask_interp::selected_mask(&s), Some((clip, uid))); + s.execute("time.set", json!({"time": 0})).unwrap(); + let r = s.execute_checked("track.mask", json!({"method": "positionScaleRotation", "direction": "forward", "wait": true})).unwrap(); + assert_eq!(r["frames"], 24, "{r}"); + assert!(s.mask_job.is_none()); + let pr = mask_path(&s, clip, uid); + assert_eq!(pr.keys.len(), 25); + let inv0 = pose_h(0).inverse().unwrap(); + let mut worst: f64 = 0.0; + for f in 0..25 { + let KV::Path(p) = pr.value_at(frame_time(f)) else { panic!() }; + let m = pose_h(f).then_after(&inv0); + for (v, o) in p.vertices.iter().zip(&shape.vertices) { + let want = m.apply(*o); + worst = worst.max((v[0] - want[0]).hypot(v[1] - want[1])); + } + // Tangents rotate and scale with the patch. + let want_t = m.apply_tangent(shape.vertices[0], shape.out_tangents[0]); + assert!((p.out_tangents[0][0] - want_t[0]).hypot(p.out_tangents[0][1] - want_t[1]) < 1.5); + } + assert!(worst < 1.5, "worst vertex error {worst} px"); + assert_eq!(s.time(), frame_time(24)); + // One undo step; redo restores the keys. + assert!(s.undo()); + assert!(mask_path(&s, clip, uid).keys.is_empty()); + assert!(s.redo()); + assert_eq!(mask_path(&s, clip, uid).keys.len(), 25); + // Track backward one frame from the end, with another method. + let r = s.execute("track.mask", json!({"layer": clip.0, "mask": uid, "method": "position", "direction": "frameBackward", "wait": true})).unwrap(); + assert_eq!(r["frames"], 1); + assert_eq!(s.state.mask_track_method, crate::mask_track::MaskMethod::Position); + // Serde keeps the keys. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let l = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap(); + assert_eq!(l.props.find_group(uid).unwrap().get("path").unwrap().keys.len(), 25); + // Bad parameters are reported. + assert!(s.execute_checked("track.mask", json!({"layer": clip.0, "mask": uid, "method": "warp"})).is_err()); + assert!(s.execute_checked("track.mask", json!({"layer": clip.0, "mask": uid, "bogus": 1})).is_err()); +} + +#[test] +fn track_mask_in_background_and_cancel() { + let (mut s, clip, _) = setup(patch_frames(rotating_pose)); + let uid = new_mask(&mut s, clip, &ShapePath::rect(rotating_pose(0).0, 60.0, 60.0)); + s.execute("time.set", json!({"time": 0})).unwrap(); + let r = s.execute("track.mask", json!({"layer": clip.0, "mask": uid, "method": "perspective"})).unwrap(); + assert_eq!(r["running"], true); + // The analysis keeps running in the background; Stop keeps what was tracked. + assert!(s.is_enabled("track.stop")); + s.execute("track.stop", json!({})).unwrap(); + let t0 = std::time::Instant::now(); + while s.mask_job.is_some() { + assert!(t0.elapsed().as_secs() < 60); + std::thread::sleep(std::time::Duration::from_millis(2)); + s.poll_mask_track(); + } + let n = mask_path(&s, clip, uid).keys.len(); + assert!((1..25).contains(&n), "{n} keys"); + // A full background run. + s.execute("time.set", json!({"time": 0})).unwrap(); + s.execute("track.mask", json!({"layer": clip.0, "mask": uid, "method": "perspective"})).unwrap(); + while s.mask_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_mask_track(); + } + let pr = mask_path(&s, clip, uid); + assert_eq!(pr.keys.len(), 25); + let KV::Path(p) = pr.value_at(frame_time(20)) else { panic!() }; + let m = pose_h(20).then_after(&pose_h(0).inverse().unwrap()); + let want = m.apply(ShapePath::rect(rotating_pose(0).0, 60.0, 60.0).vertices[2]); + assert!((p.vertices[2][0] - want[0]).hypot(p.vertices[2][1] - want[1]) < 2.0); +} + +/// A five-pointed star (10 vertices) around `c`, rotated by `rot` degrees. +fn star(c: [f64; 2], r: f64, rot: f64) -> ShapePath { + let pts: Vec<[f64; 2]> = (0..10) + .map(|i| { + let a = (rot + i as f64 * 36.0 - 90.0).to_radians(); + let rr = if i % 2 == 0 { r } else { r * 0.45 }; + [c[0] + rr * a.cos(), c[1] + rr * a.sin()] + }) + .collect(); + ShapePath::polygon(&pts, true) +} + +#[test] +fn mask_interpolation_creates_matching_keys() { + let (mut s, clip, _) = setup(patch_frames(rotating_pose)); + let a = ShapePath::rect([100.0, 100.0], 80.0, 60.0); + let b = star([200.0, 120.0], 50.0, 30.0); + let uid = new_mask(&mut s, clip, &a); + let cid = s.active_comp_id().unwrap(); + let (t0, t1) = (frame_time(0), frame_time(20)); + s.edit("keys", None, |p, _| { + let pr = p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().props.find_group_mut(uid).unwrap().get_mut("path").unwrap(); + pr.keys = vec![Keyframe::new(t0, KV::Path(a.clone())), Keyframe::new(t1, KV::Path(b.clone()))]; + Ok(()) + }) + .unwrap(); + let prop = mask_path(&s, clip, uid).uid; + s.execute("keys.select", json!({"keys": [{"layer": clip.0, "prop": prop, "time": 0.0}, {"layer": clip.0, "prop": prop, "time": 0.8}]})).unwrap(); + assert_eq!(s.state.selected_keys.len(), 2); + let r = s + .execute_checked( + "mask.interpolate", + json!({"keyframeRate": "auto", "bendingResistance": 50, "quality": 50, "addVertices": 10, "addVerticesUnit": "pixels", "matchingMethod": "auto", "firstVerticesMatch": false}), + ) + .unwrap(); + // 19 in-betweens at the comp rate (frames 1…19). + assert_eq!(r["keys"], 19, "{r}"); + let pr = mask_path(&s, clip, uid); + assert_eq!(pr.keys.len(), 21); + let shapes: Vec = pr.keys.iter().map(|k| k.value.as_path().unwrap().clone()).collect(); + let n = shapes[0].len(); + assert!(n >= 28, "{n} vertices"); + assert!(shapes.iter().all(|p| p.len() == n)); + // Ends keep their outlines; vertex paths between consecutive keys don't cross. + for (i, w) in shapes.windows(2).enumerate() { + assert_eq!(paths_cross(&w[0], &w[1]), 0, "keys {i}/{}", i + 1); + } + assert!( + shapes[0] + .vertices + .iter() + .all(|v| (v[0] - 60.0).abs() < 1e-6 || (v[0] - 140.0).abs() < 1e-6 || (v[1] - 70.0).abs() < 1e-6 || (v[1] - 130.0).abs() < 1e-6) + ); + // Options persist; Keyframe Fields doubles the rate; undo restores. + assert_eq!(s.state.mask_interp.add_vertices, Some(10.0)); + assert!(s.undo()); + assert_eq!(mask_path(&s, clip, uid).keys.len(), 2); + let r = s + .execute( + "mask.interpolate", + json!({"layer": clip.0, "mask": uid, "times": [0.0, 0.8], "keyframeRate": 10, "keyframeFields": true, "linearVertexPaths": true}), + ) + .unwrap(); + assert_eq!(r["keys"], 15, "{r}"); + assert!(s.execute("mask.interpolate", json!({"layer": clip.0, "mask": uid, "times": [0.0]})).is_err()); + assert!(s.execute_checked("mask.interpolate", json!({"matchingMethod": "magic", "times": [0, 0.8], "layer": clip.0})).is_err()); +} + +// ---------------------------------------------------------------- Warp Stabilizer + +/// Camera offset on frame `f`: a slow pan plus jitter. +fn cam(f: u32) -> [f64; 2] { + let j = [((f * 7919) % 9) as f64 - 4.0, ((f * 104_729) % 7) as f64 - 3.0]; + [1.2 * f as f64 + j[0], 0.5 * f as f64 + j[1]] +} + +fn shaky(f: u32) -> Image { + let c = cam(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let v = texture(x as f64 + 0.5 + c[0], y as f64 + 0.5 + c[1], 11); + img.set(x, y, [v, v * 0.9, v * 0.8, 1.0]); + } + } + img +} + +fn jitter(p: &[[f64; 2]]) -> f64 { + let mut s = 0.0; + for k in 1..p.len() - 1 { + s += (p[k + 1][0] - 2.0 * p[k][0] + p[k - 1][0]).hypot(p[k + 1][1] - 2.0 * p[k][1] + p[k - 1][1]); + } + s / (p.len() - 2) as f64 +} + +fn warp_at(s: &mut Session, f: u32) -> Homography { + s.execute("time.set", json!({"time": frame_time(f).seconds()})).unwrap(); + let st = s.execute("warp.status", json!({})).unwrap(); + let m: [[f64; 3]; 3] = serde_json::from_value(st["warp"].clone()).unwrap_or_else(|_| panic!("{st}")); + Homography(m) +} + +/// Output positions of the scene point that sits at the frame centre on frame 0. +fn scene_track(s: &mut Session) -> (Vec<[f64; 2]>, Vec<[f64; 2]>) { + let x0 = [160.0 + cam(0)[0], 120.0 + cam(0)[1]]; + let before: Vec<[f64; 2]> = (0..25).map(|f| [x0[0] - cam(f)[0], x0[1] - cam(f)[1]]).collect(); + let after = (0..25).map(|f| warp_at(s, f).apply(before[f as usize])).collect(); + (before, after) +} + +fn render(s: &Session, f: u32) -> Image { + s.render(s.active_comp_id().unwrap(), frame_time(f), RenderOpts::default()) +} + +fn transparent(img: &Image) -> usize { + img.data.iter().filter(|p| p[3] < 0.5).count() +} + +fn set_param(s: &mut Session, layer: LayerId, path: &str, v: serde_json::Value) { + s.execute("prop.set", json!({"layer": layer.0, "path": format!("effects/#1/{path}"), "value": v})).unwrap(); +} + +#[test] +fn warp_stabilizer_analyzes_and_stabilizes() { + let (mut s, clip, solid) = setup(shaky); + s.execute("edit.clear", json!({"layers": [solid.0]})).unwrap(); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + // Effect ▸ Distort ▸ Warp Stabilizer, then analyse (blocking). + let r = s.execute("effect.apply", json!({"effect": "Warp Stabilizer"})).unwrap(); + let fx = r["effects"][0].as_u64().unwrap(); + assert!(s.warp_pending.contains(&(s.active_comp_id().unwrap(), clip, fx)), "applying queues an analysis"); + let st = s.execute_checked("warp.status", json!({})).unwrap(); + assert_eq!(st["analyzed"], false); + let r = s.execute_checked("warp.analyze", json!({"layer": clip.0, "wait": true})).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert_eq!(r["status"]["analyzed"], true, "{r}"); + assert!(s.warp_pending.is_empty() && s.warp_job.is_none()); + // Groups and AE parameter names. + let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(fx).unwrap().clone(); + let names: Vec<&str> = g.groups().map(|g| g.name.as_str()).collect(); + assert_eq!(names, ["Stabilization", "Borders", "Advanced"]); + assert!(g.prop("borders/autoScale/maximumScale").is_some()); + assert_eq!(g.prop("stabilization/smoothness").unwrap().name, "Smoothness"); + + // Smooth Motion: the jitter of a scene point drops by more than 80 %. + set_param(&mut s, clip, "borders/framing", json!(0)); + let (before, after) = scene_track(&mut s); + let (j0, j1) = (jitter(&before), jitter(&after)); + assert!(j1 < 0.2 * j0, "jitter {j0} → {j1}"); + // Rendering applies the frame's correction: output(p) = source(W⁻¹ p). + let w = warp_at(&mut s, 7); + let out = render(&s, 7); + let src = shaky(7); + let inv = w.inverse().unwrap(); + for p in [[160.0, 120.0], [100.0, 80.0], [230.0, 170.0]] { + let q = inv.apply([p[0] + 0.5, p[1] + 0.5]); + let want = src.sample_bilinear(q[0], q[1]); + let got = out.get(p[0] as i64, p[1] as i64); + assert!((want[0] - got[0]).abs() < 0.03, "{p:?}: {want:?} vs {got:?}"); + } + // Stabilize Only shows the moving borders. + let shown: usize = (0..25).map(|f| transparent(&render(&s, f))).sum(); + assert!(shown > 500, "{shown}"); + + // No Motion keeps a feature fixed. + set_param(&mut s, clip, "stabilization/result", json!(1)); + let (_, fixed) = scene_track(&mut s); + for q in &fixed { + assert!((q[0] - fixed[12][0]).hypot(q[1] - fixed[12][1]) < 0.6, "{q:?} vs {:?}", fixed[12]); + } + set_param(&mut s, clip, "stabilization/result", json!(0)); + + // Stabilize, Crop, Auto-scale (the default) hides the borders; Crop leaves an opaque centre. + set_param(&mut s, clip, "borders/framing", json!(2)); + let st = s.execute("warp.status", json!({})).unwrap(); + assert!(st["autoScale"].as_f64().unwrap() > 100.5, "{st}"); + for f in 0..25 { + assert_eq!(transparent(&render(&s, f)), 0, "frame {f}"); + } + set_param(&mut s, clip, "borders/framing", json!(1)); + let st = s.execute("warp.status", json!({})).unwrap(); + let crop: [f64; 4] = serde_json::from_value(st["crop"].clone()).unwrap(); + for f in [0, 9, 24] { + let img = render(&s, f); + for y in (crop[1].ceil() as u32 + 1)..(crop[3].floor() as u32 - 1) { + for x in (crop[0].ceil() as u32 + 1)..(crop[2].floor() as u32 - 1) { + assert!(img.get(x as i64, y as i64)[3] > 0.99, "frame {f} ({x}, {y})"); + } + } + assert!(img.get(0, 0)[3] < 0.01); + } + // Synthesize Edges fills most of the borders from neighbouring frames. + set_param(&mut s, clip, "borders/framing", json!(3)); + let synth: usize = (0..25).map(|f| transparent(&render(&s, f))).sum(); + assert!(synth * 10 < shown, "synthesized {synth} vs {shown}"); + // Show Track Points draws the features. + set_param(&mut s, clip, "borders/framing", json!(0)); + let plain = render(&s, 5); + set_param(&mut s, clip, "advanced/showTrackPoints", json!(true)); + assert_ne!(render(&s, 5).data, plain.data); + + // Serde keeps the analysis; undo / redo of the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let bg = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap().props.find_group(fx).unwrap().clone(); + assert_eq!(bg, s.active_comp().unwrap().layer(clip).unwrap().props.find_group(fx).unwrap().clone()); + while s.history.undo.last().is_some_and(|(l, _)| l != "Warp Stabilizer Analysis") { + s.undo(); + } + assert!(s.undo()); + assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], false); + assert!(s.redo()); + assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], true); +} + +#[test] +fn warp_analysis_invalidates_and_cancels() { + let (mut s, clip, solid) = setup(shaky); + s.execute("edit.clear", json!({"layers": [solid.0]})).unwrap(); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + // Animation ▸ Warp Stabilizer VFX applies and analyses. + assert!(s.is_enabled("track.warpStabilizer")); + let r = s.execute_checked("track.warpStabilizer", json!({"wait": true})).unwrap(); + let fx = r["effect"].as_u64().unwrap(); + let cid = s.active_comp_id().unwrap(); + assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], true); + // Trimming the layer clears the analysis and queues it again. + s.execute("layer.timing", json!({"layers": [clip.0], "out": 0.8})).unwrap(); + let st = s.execute("warp.status", json!({})).unwrap(); + assert_eq!(st["analyzed"], false, "{st}"); + assert_eq!(st["pending"], true); + // Without an analysis the layer renders unstabilized. + let img = s.render(cid, frame_time(3), RenderOpts::default()); + assert_eq!(transparent(&img), 0); + // Frontends start queued analyses in the background (poll_warp with auto). + assert!(s.poll_warp(true)); + assert!(s.warp_job.is_some()); + let t0 = std::time::Instant::now(); + while s.warp_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + let p = s.warp_progress().unwrap(); + assert!(p.banner().starts_with("Analyzing in background (step 1 of 2)") || p.banner() == "Stabilizing..."); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_warp(true); + } + assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], true); + // Undoing the slip brings back the old analysis (the key matches again). + // Cancel: nothing is written. + s.execute("effect.apply", json!({"effect": "ec.distort.warpstabilizer", "layers": [clip.0]})).unwrap(); + let fx2 = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().nth(1).unwrap().uid; + let r = s.execute("warp.analyze", json!({"layer": clip.0, "effect": fx2})).unwrap(); + assert_eq!(r["running"], true); + assert!(s.execute("warp.analyze", json!({"layer": clip.0, "effect": fx})).is_err(), "one analysis at a time"); + s.execute("warp.cancel", json!({})).unwrap(); + while s.warp_job.is_some() { + std::thread::sleep(std::time::Duration::from_millis(2)); + s.poll_warp(false); + } + let st = s.execute("warp.status", json!({"layer": clip.0, "effect": fx2})).unwrap(); + assert_eq!(st["analyzed"], false, "{st}"); + // Time Remap changes invalidate too. + s.execute("layer.enableTimeRemap", json!({"layers": [clip.0]})).unwrap(); + assert_eq!(s.execute("warp.status", json!({"layer": clip.0, "effect": fx})).unwrap()["analyzed"], false); + let _ = FPS; + let _ = Tick::ZERO; +} diff --git a/crates/engine/src/tests_track.rs b/crates/engine/src/tests_track.rs index 7ea69cfa..adcf7df0 100644 --- a/crates/engine/src/tests_track.rs +++ b/crates/engine/src/tests_track.rs @@ -17,11 +17,11 @@ use serde_json::json; use crate::Session; -const W: u32 = 320; -const H: u32 = 240; -const FPS: u32 = 25; +pub(crate) const W: u32 = 320; +pub(crate) const H: u32 = 240; +pub(crate) const FPS: u32 = 25; -fn texture(x: f64, y: f64, seed: u32) -> f32 { +pub(crate) fn texture(x: f64, y: f64, seed: u32) -> f32 { let mut v = 0.0f64; let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345); let mut rnd = || { @@ -41,7 +41,7 @@ fn texture(x: f64, y: f64, seed: u32) -> f32 { } /// Footage computed per frame (cached). -struct Synth { +pub(crate) struct Synth { make: Box Image + Send + Sync>, cache: Mutex>>, } @@ -55,9 +55,9 @@ impl FootageSource for Synth { } /// Patch pose on frame `f`: centre, rotation (degrees), scale. -type Pose = fn(u32) -> ([f64; 2], f64, f64); +pub(crate) type Pose = fn(u32) -> ([f64; 2], f64, f64); -fn patch_frames(pose: Pose) -> impl Fn(u32) -> Image + Send + Sync { +pub(crate) fn patch_frames(pose: Pose) -> impl Fn(u32) -> Image + Send + Sync { move |f| { let (c, rot, sc) = pose(f); let (sn, cs) = rot.to_radians().sin_cos(); @@ -76,7 +76,7 @@ fn patch_frames(pose: Pose) -> impl Fn(u32) -> Image + Send + Sync { } /// A comp with the synthetic clip (layer `clip`) and a 100 × 100 solid above it (layer `solid`). -fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, LayerId, LayerId) { +pub(crate) fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, LayerId, LayerId) { let mut s = Session::default(); s.execute("comp.new", json!({"name": "Track", "width": W, "height": H, "frameRate": FPS, "duration": 1})).unwrap(); let cid = s.active_comp_id().unwrap(); @@ -121,7 +121,7 @@ fn tracker_points(s: &Session, layer: LayerId) -> Vec Tick { +pub(crate) fn frame_time(f: u32) -> Tick { FrameRate::new(FPS as i64, 1).tick_of(f as i64) } @@ -220,7 +220,7 @@ fn stabilize_keeps_the_feature_fixed() { assert!(worst < 0.25, "feature drifts by {worst} px"); } -fn rotating_pose(f: u32) -> ([f64; 2], f64, f64) { +pub(crate) fn rotating_pose(f: u32) -> ([f64; 2], f64, f64) { let t = f as f64; ([160.0 + 1.2 * t, 120.0 - 0.7 * t], 1.5 * t, 1.0 + 0.008 * t) } diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs index 43e24cb8..c406edb9 100644 --- a/crates/engine/src/tracking.rs +++ b/crates/engine/src/tracking.rs @@ -155,7 +155,7 @@ pub struct TrackShared { pub frames: Mutex>, } -fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { +pub(crate) fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { m.lock().unwrap_or_else(|e| e.into_inner()) } @@ -197,7 +197,7 @@ pub(crate) struct Work { /// The layer's source frame at comp time `t` at 100 % (layer pixels + offset = image pixels). /// Footage at its native size is used straight from the footage source (no copy). -fn source_frame( +pub(crate) fn source_frame( r: &Renderer, project: &Project, cid: ItemId, diff --git a/crates/engine/src/warp.rs b/crates/engine/src/warp.rs new file mode 100644 index 00000000..0b2dd03d --- /dev/null +++ b/crates/engine/src/warp.rs @@ -0,0 +1,404 @@ +//! Warp Stabilizer runtime: the analysis job (`warp.analyze`) and keeping analyses valid. +//! +//! The analysis renders the layer's input to the effect (source → masks → the effects above +//! the Warp Stabilizer, `Renderer::layer_input`) for every frame between the layer's In and Out +//! points on a background thread, and feeds the frames to `effectcraft_track::stabilize`'s +//! analyzer (step 1 of 2, "Analyzing in background"); step 2 ("Stabilizing") derives the +//! stabilization plan once to validate the result. When the job finishes, the analysis is +//! written into the effect's hidden Analysis parameter as one undo step, together with a key of +//! what it was made from (the layer's source, In/Out, start, stretch and Time Remap). Edits that +//! change those clear the analysis ([`invalidate`], run after every edit) and queue the effect +//! for re-analysis, which frontends start with [`Session::poll_warp`] (After Effects re-analyses +//! automatically too). + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::warp_stab::{self, ANALYSIS, ANALYSIS_KEY}; +use effectcraft_keyframe::Value; +use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, LayerSource, Project, PropGroup, Uid}; +use effectcraft_raster::Image; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::stabilize::{AnalyzeOpts, Analyzer, StabSettings, WarpAnalysis}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WarpProgress { + /// 1 = analyzing, 2 = stabilizing. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, +} + +impl WarpProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Stabilizing...".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +#[derive(Default)] +pub struct WarpShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) Warp Stabilizer analysis. +pub struct WarpJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the analysis is made from. + key: String, +} + +impl WarpJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> WarpProgress { + lock(&self.shared.state).clone() + } +} + +/// What the worker needs, detached from the session. +pub(crate) struct WarpWork { + pub project: Arc, + pub footage: Arc, + pub expr: Option>, + pub cache: Arc, + pub comp: ItemId, + pub layer: LayerId, + /// Position of the effect in the layer's stack (the effects before it are analysed). + pub effect_index: usize, + pub size: [f64; 2], + pub detailed: bool, + /// Comp times of the analysed frames. + pub times: Vec, +} + +fn fnv(s: &str) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in s.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +/// What an analysis of `layer` depends on (its frames): source (and the footage item's +/// interpretation), In/Out, start time, stretch, Time Remap and the comp frame rate. +pub fn signature(p: &Project, comp: ItemId, layer: &Layer) -> String { + let src = match &layer.source { + LayerSource::Footage { item } | LayerSource::Solid { item } | LayerSource::Comp { item } => match p.item(*item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) => format!("{f:?}"), + Some(ItemKind::Solid(s)) => format!("{s:?}"), + Some(ItemKind::Comp(c)) => format!("{}x{}@{:?}/{:?}", c.width, c.height, c.frame_rate, c.duration), + _ => String::new(), + }, + _ => String::new(), + }; + let rate = p.comp(comp).map(|c| format!("{:?}", c.frame_rate)).unwrap_or_default(); + let s = format!( + "{:?}|{src}|{:?}|{:?}|{:?}|{}|{:?}|{rate}", + layer.source, + layer.in_point, + layer.out_point, + layer.start_time, + layer.stretch, + layer.props.get("timeRemap") + ); + format!("{:016x}", fnv(&s)) +} + +fn is_warp(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == warp_stab::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.get(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// Clear analyses that no longer match their layer, after an edit. Returns the Warp +/// Stabilizers that need (re-)analysis: cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clear = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_warp) { + continue; + } + let sig = signature(after, *cid, l); + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_warp(g)) { + let analysed = !str_of(g, ANALYSIS).is_empty(); + if analysed && str_of(g, ANALYSIS_KEY) != sig { + clear.push((*cid, l.id, g.uid)); + todo.push((*cid, l.id, g.uid)); + } else if !analysed && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid) in clear { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + for m in [ANALYSIS, ANALYSIS_KEY] { + if let Some(p) = g.get_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + } + } + todo +} + +/// The layer's input to effect `effect_index` at comp time `t`, in layer pixels at 100 %. +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: WarpWork, shared: &WarpShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = Analyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + return; + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + lock(&shared.state).step = 2; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let a = an.finish(lt0, fd); + // Step 2: derive a plan once (validates the motion and warms nothing else up). + let plan = effectcraft_track::stabilize::plan(&a, &StabSettings { fps: if fd > 0.0 { 1.0 / fd } else { 30.0 }, ..Default::default() }); + if plan.warps.len() != a.frames.len() || plan.warps.iter().any(|h| h.0.iter().flatten().any(|v| !v.is_finite())) { + return fail("the stabilization could not be computed"); + } + *lock(&shared.result) = Some(a); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// Frames to analyse: every comp frame from the layer's In to its Out point (inside the comp). +pub(crate) fn analysis_times(comp: &effectcraft_project::Comp, layer: &Layer) -> Vec { + let fd = comp.frame_duration(); + let lo = comp.frame_rate.snap_nearest(layer.in_point.max(Tick::ZERO)); + let hi = layer.out_point.min(comp.duration); + let mut t = lo; + let mut v = vec![]; + while t < hi { + v.push(t); + t += fd; + } + v +} + +impl Session { + pub fn is_warp_analyzing(&self) -> bool { + self.warp_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn warp_progress(&self) -> Option { + self.warp_job.as_ref().map(|j| j.progress()) + } + + /// The effect being analysed: (comp, layer, effect uid). + pub fn warp_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.warp_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_warp(&mut self) -> bool { + match &self.warp_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing Warp Stabilizer `effect` on `layer`. `wait`: block until done (wasm32 + /// always does). Returns the number of frames. + pub fn start_warp(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_warp_analyzing() { + return Err("a Warp Stabilizer analysis is already running".into()); + } + self.poll_warp_job(); + self.warp_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_warp(g)).ok_or("not a Warp Stabilizer")?; + if l.switches.adjustment { + return Err("Warp Stabilizer needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let detailed = g.prop("advanced/detailedAnalysis").is_some_and(|p| p.value.as_bool()); + let n = times.len(); + let work = WarpWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed, + times, + }; + let key = signature(&self.project, comp, l); + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(WarpShared::default()); + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("warp-analyze".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.warp_job = Some(WarpJob { shared, thread, comp, layer, effect, key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_warp_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_warp_job(&mut self) -> bool { + let Some(job) = &self.warp_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.warp_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(a), None) if !st.cancelled => { + let json = a.to_json(); + let (cid, lid, uid, key) = (job.comp, job.layer, job.effect, job.key.clone()); + let r = self.edit("Warp Stabilizer Analysis", None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the Warp Stabilizer was removed during analysis".into()))?; + for (m, v) in [(ANALYSIS, json), (ANALYSIS_KEY, key)] { + if let Some(pr) = g.get_mut(m) { + pr.value = Value::Str(v); + } + } + Ok(()) + }); + let msg = match r { + Ok(()) => format!("Warp Stabilizer: analysed {} frame(s) in {:.1} s", a.frames.len(), st.elapsed), + Err(e) => format!("Warp Stabilizer: {e}"), + }; + self.events.push(Event::Toast { message: msg, error: false }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("Warp Stabilizer analysis failed: {e}"), + None => "Warp Stabilizer analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// re-analyses in the background, like After Effects. + pub fn poll_warp(&mut self, auto: bool) -> bool { + let changed = self.poll_warp_job(); + if auto && self.warp_job.is_none() { + while let Some((c, l, e)) = self.warp_pending.first().copied() { + self.warp_pending.remove(0); + if self.start_warp(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 5bb9c8b6..99bc7c5e 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -33,10 +33,11 @@ pub enum PanelKind { Flowchart, History, Markers, + MaskInterpolation, } impl PanelKind { - pub const ALL: [PanelKind; 23] = [ + pub const ALL: [PanelKind; 24] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -60,6 +61,7 @@ impl PanelKind { PanelKind::Flowchart, PanelKind::History, PanelKind::Markers, + PanelKind::MaskInterpolation, ]; pub fn title(self) -> &'static str { match self { @@ -86,6 +88,7 @@ impl PanelKind { PanelKind::Flowchart => "Flowchart", PanelKind::History => "History", PanelKind::Markers => "Markers", + PanelKind::MaskInterpolation => "Mask Interpolation", } } pub fn id(self) -> String { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 65880e33..071c1948 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -718,6 +718,15 @@ impl EffectcraftApp { self.session.poll_track(); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } + if self.session.mask_job.is_some() { + self.session.poll_mask_track(); + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } + // Warp Stabilizer: finish analyses and start queued (re-)analyses in the background. + if self.session.warp_job.is_some() || !self.session.warp_pending.is_empty() { + self.session.poll_warp(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/panels/anim_tools.rs b/crates/ui-egui/src/panels/anim_tools.rs index aa4b6368..5da48415 100644 --- a/crates/ui-egui/src/panels/anim_tools.rs +++ b/crates/ui-egui/src/panels/anim_tools.rs @@ -1,4 +1,4 @@ -//! Wiggler, Smoother and Motion Sketch panels (Window menu). Their settings live in +//! Wiggler, Smoother, Motion Sketch and Mask Interpolation panels (Window menu). Their settings live in //! [`crate::state::AnimToolsState`] (serde, so agents can read and set them); Apply runs the //! engine commands `keys.wiggle`, `keys.smooth` and `motion.sketch`. //! @@ -294,3 +294,111 @@ pub fn sketch_overlay(app: &mut EffectcraftApp, ui: &mut egui::Ui) { } } } + +/// Mask Interpolation panel: Keyframe Rate / Fields, Linear Vertex Paths, Bending Resistance, +/// Quality, Add Mask Shape Vertices, Matching Method, 1:1 Vertex Matches, First Vertices Match; +/// Apply runs `mask.interpolate` on the selected Mask Path keyframes. The options live in the +/// editor state (`mask.interpolationOptions`). +pub fn mask_interpolation(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + let x0 = rect.min.x + 12.0; + let xv = x0 + 130.0; + let w = (rect.max.x - xv - 12.0).clamp(80.0, 200.0); + let mut y = rect.min.y + 10.0; + let o = app.session.state.mask_interp.clone(); + let mut changes = serde_json::Map::new(); + label(&p, x0, y, "Keyframe Rate:", &t); + let auto = o.keyframe_rate.is_none(); + if let Some(i) = choice(app, ui, Rect::from_min_size(pos2(xv, y), vec2(70.0, 20.0)), "maskInterp.rateMode", &["Auto", "Custom"], usize::from(!auto)) { + let fps = app.session.active_comp().map(|c| c.frame_rate.as_f64()).unwrap_or(30.0); + changes.insert("keyframeRate".into(), if i == 0 { json!("auto") } else { json!(fps) }); + } + if let Some(r) = o.keyframe_rate { + let nr = number(app, ui, pos2(xv + 78.0, y), "maskInterp.rate", r, 0.5, (0.1, 1000.0), 1, " per second"); + if (nr - r).abs() > 1e-9 { + changes.insert("keyframeRate".into(), json!(nr)); + } + } else { + p.text(pos2(xv + 78.0, y + 9.0), Align2::LEFT_CENTER, "per second", Tokens::ui(12.0), t.text_dim); + } + y += ROW; + let mut check = |app: &mut EffectcraftApp, y: f32, key: &str, text: &str, on: bool| { + if checkbox(app, ui, pos2(x0, y + 2.0), &format!("maskInterp.{key}"), text, on) != on { + changes.insert(key.into(), json!(!on)); + } + }; + check(app, y, "keyframeFields", "Keyframe Fields (doubles rate)", o.keyframe_fields); + y += ROW; + check(app, y, "linearVertexPaths", "Use \"Linear\" Vertex Paths", o.linear_vertex_paths); + y += ROW; + label(&p, x0, y, "Bending Resistance:", &t); + let br = number(app, ui, pos2(xv, y), "maskInterp.bendingResistance", o.bending_resistance, 0.5, (0.0, 100.0), 0, ""); + if (br - o.bending_resistance).abs() > 1e-9 { + changes.insert("bendingResistance".into(), json!(br)); + } + y += ROW; + label(&p, x0, y, "Quality:", &t); + let q = number(app, ui, pos2(xv, y), "maskInterp.quality", o.quality, 0.5, (0.0, 100.0), 0, ""); + if (q - o.quality).abs() > 1e-9 { + changes.insert("quality".into(), json!(q)); + } + y += ROW; + let add_on = o.add_vertices.is_some(); + if checkbox(app, ui, pos2(x0, y + 2.0), "maskInterp.addVerticesOn", "Add Mask Shape Vertices", add_on) != add_on { + changes.insert("addVertices".into(), if add_on { json!(false) } else { json!(5.0) }); + } + y += ROW; + if let Some(v) = o.add_vertices { + let nv = number(app, ui, pos2(x0 + 22.0, y), "maskInterp.addVertices", v, 0.2, (0.1, 2000.0), 1, ""); + if (nv - v).abs() > 1e-9 { + changes.insert("addVertices".into(), json!(nv)); + } + let units = ["pixels", "total", "percent"]; + let cur = units.iter().position(|u| *u == o.add_vertices_unit).unwrap_or(0); + if let Some(i) = choice( + app, + ui, + Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), + "maskInterp.addVerticesUnit", + &["Pixels Between Vertices", "Total Vertices", "Percentage of Outline"], + cur, + ) { + changes.insert("addVerticesUnit".into(), json!(units[i])); + } + } + y += ROW; + label(&p, x0, y, "Matching Method:", &t); + let methods = ["auto", "curve", "polyline"]; + let cur = methods.iter().position(|m| *m == o.matching_method).unwrap_or(0); + if let Some(i) = choice(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "maskInterp.matchingMethod", &["Auto", "Curve", "Polyline"], cur) { + changes.insert("matchingMethod".into(), json!(methods[i])); + } + y += ROW; + let mut check = |app: &mut EffectcraftApp, y: f32, key: &str, text: &str, on: bool| { + if checkbox(app, ui, pos2(x0, y + 2.0), &format!("maskInterp.{key}"), text, on) != on { + changes.insert(key.into(), json!(!on)); + } + }; + check(app, y, "oneToOne", "Use 1:1 Vertex Matches", o.one_to_one); + y += ROW; + check(app, y, "firstVerticesMatch", "First Vertices Match", o.first_vertices_match); + y += ROW + 8.0; + if !changes.is_empty() + && let Err(e) = app.session.execute("mask.interpolationOptions", serde_json::Value::Object(changes)) + { + app.ui.status = e.to_string(); + } + let b = Rect::from_min_size(pos2(rect.max.x - 92.0, y), vec2(80.0, 24.0)); + let enabled = app.session.state.selected_keys.len() >= 2; + if widgets::text_button(ui, b, "Apply", enabled, &t, egui::Id::new("mi-apply")).clicked() { + match app.session.execute("mask.interpolate", json!({})) { + Ok(r) => app.ui.status = format!("Mask Interpolation: {} keyframes added ({} vertices)", r["keys"], r["vertices"]), + Err(e) => app.ui.status = e.to_string(), + } + } + app.auto.add("maskInterp.apply", b, "Apply"); + let msg = if enabled { "Applies to the selected Mask Path keyframes" } else { "Select two or more Mask Path keyframes" }; + p.text(pos2(x0, b.max.y + 16.0), Align2::LEFT_CENTER, msg, Tokens::ui(11.0), t.text_faint); + let _ = (Color32::WHITE, Sense::hover(), Stroke::NONE); +} diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 78facbcc..0a19d57f 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -510,6 +510,33 @@ fn group_rows( } } +/// Warp Stabilizer's Analyze / Cancel buttons and analysis status. +fn warp_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) { + let t = app.tokens; + let running = app.session.warp_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_warp_analyzing(); + let busy = app.session.is_warp_analyzing(); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("warp-analyze", g.uid))).clicked() && !busy { + actions.push(("warp.analyze".into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.warp.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("warp-cancel", g.uid))).clicked() { + actions.push(("warp.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.warp.{}.cancel", g.uid), b2, "Cancel"); + } + let analysed = matches!(g.get(effectcraft_engine::effects::warp_stab::ANALYSIS).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let status = match app.session.warp_progress() { + Some(pr) if running => pr.banner(), + _ if analysed => "Analysis complete".to_string(), + _ if app.session.warp_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".to_string(), + _ => "Not analyzed: click Analyze".to_string(), + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); +} + // --------------------------------------------------------------------------------------------- // Curves / Levels editors @@ -517,6 +544,7 @@ fn group_rows( fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, + effectcraft_engine::effects::warp_stab::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -891,6 +919,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if er.max.y >= body.min.y && er.min.y <= body.max.y { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), + effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 39cf4c5c..3f737502 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -81,6 +81,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Audio => misc::audio(app, ui, rect), PanelKind::History => misc::history(app, ui, rect), PanelKind::Markers => misc::markers(app, ui, rect), + PanelKind::MaskInterpolation => anim_tools::mask_interpolation(app, ui, rect), PanelKind::Wiggler => anim_tools::wiggler(app, ui, rect), PanelKind::Smoother => anim_tools::smoother(app, ui, rect), PanelKind::MotionSketch => anim_tools::motion_sketch(app, ui, rect), diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index ad84665b..d1f456b2 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -133,12 +133,15 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); + let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); - button(app, ui, r2, "Warp Stabilizer", false, "tracker.warpStabilizer"); + let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); + if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { + run(app, &ctx, "track.warpStabilizer", json!({})); + } y += 28.0; let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); - let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); if button(app, ui, r1, "Track Motion", can_track, "tracker.trackMotion") { run(app, &ctx, "track.motion", src_p.clone()); } @@ -150,6 +153,11 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Open a composition to track motion.", Tokens::ui(12.0), t.text_faint); return; }; + // A selected mask switches the panel to mask tracking. + if let Some((ml, mu)) = effectcraft_engine::commands::mask_interp::selected_mask(&app.session) { + mask_mode(app, ui, &ctx, &p, rect, y, ml, mu); + return; + } let field_x = x0 + 100.0; let field_w = (w - 100.0).max(80.0); @@ -334,6 +342,84 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } +/// The Tracker panel in mask mode: Method and the four track buttons (◀| ◀ ▶ |▶). +#[allow(clippy::too_many_arguments)] +fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p: &egui::Painter, rect: Rect, mut y: f32, layer: LayerId, mask: Uid) { + use effectcraft_engine::mask_track::MaskMethod; + let t = app.tokens; + let x0 = rect.min.x + 10.0; + let w = (rect.width() - 20.0).max(120.0); + let field_x = x0 + 100.0; + let field_w = (w - 100.0).max(80.0); + let name = app + .session + .active_comp() + .and_then(|c| c.layer(layer)) + .and_then(|l| l.props.find_group(mask).map(|g| format!("{} ({})", g.name, l.name))) + .unwrap_or_default(); + label(p, pos2(x0, y + 10.0), "Track Mask:", &t); + p.text(pos2(field_x, y + 10.0), Align2::LEFT_CENTER, name, Tokens::ui(12.0), t.text); + y += ROW; + label(p, pos2(x0, y + 10.0), "Method:", &t); + let method = app.session.state.mask_track_method; + let dd = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0)); + let did = egui::Id::new("tracker-mask-method-dd"); + if widgets::dropdown(ui, dd, method.label(), &t, did).clicked() { + widgets::open_popup(ui, did); + } + app.auto.add("tracker.mask.method", dd, "Method"); + let labels: Vec = MaskMethod::ALL.iter().map(|m| m.label().to_string()).collect(); + if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &labels, MaskMethod::ALL.iter().position(|m| *m == method)) { + run(app, ctx, "track.maskMethod", json!({"method": MaskMethod::ALL[i].id()})); + } + y += ROW + 6.0; + label(p, pos2(x0, y + 11.0), "Track:", &t); + let running = app.session.is_mask_tracking(); + let running_dir = app.session.mask_job.as_ref().map(|j| j.direction); + let mut bx = field_x; + for (dir, fwd, frame, id, tip) in [ + (effectcraft_engine::tracking::Direction::FrameBackward, false, true, "frameBackward", "Track selected masks 1 frame backward"), + (effectcraft_engine::tracking::Direction::Backward, false, false, "backward", "Track selected masks backward"), + (effectcraft_engine::tracking::Direction::Forward, true, false, "forward", "Track selected masks forward"), + (effectcraft_engine::tracking::Direction::FrameForward, true, true, "frameForward", "Track selected masks 1 frame forward"), + ] { + let r = Rect::from_min_size(pos2(bx, y), vec2(30.0, 22.0)); + let resp = ui.interact(r, egui::Id::new(("tracker-mask-track", id)), Sense::click()).on_hover_text(tip); + let stop = running && running_dir == Some(dir) && !frame; + let on = !running || stop; + p.rect_filled(r, 4.0, if resp.hovered() && on { t.hover } else { t.field_bg }); + p.rect_stroke(r, 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + analyze_glyph(p, r, fwd, frame, stop, if on { t.text } else { t.text_faint }); + app.auto.add(&format!("tracker.mask.track.{id}"), r, tip); + if resp.clicked() && on { + if stop { + run(app, ctx, "track.stop", json!({})); + } else { + run(app, ctx, "track.mask", json!({"layer": layer.0, "mask": mask, "direction": id})); + } + } + bx += 34.0; + } + y += 34.0; + if let Some(pr) = app.session.mask_track_progress() { + let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0)); + p.rect_filled(bar, 3.0, t.field_bg); + let f = if pr.total > 0 { pr.done as f32 / pr.total as f32 } else { 0.0 }; + p.rect_filled(Rect::from_min_size(bar.min, vec2(bar.width() * f.clamp(0.0, 1.0), 6.0)), 3.0, t.accent); + app.auto.add("tracker.mask.progress", bar, "Mask tracking progress"); + p.text( + pos2(x0, y + 18.0), + Align2::LEFT_CENTER, + format!("Tracking {} / {} frames ({:.0} fps)", pr.done, pr.total, pr.fps), + Tokens::ui(11.0), + t.text_dim, + ); + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } else { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Tracks the pixels inside the mask and keys its Mask Path.", Tokens::ui(11.0), t.text_faint); + } +} + // ---------------------------------------------------------------- dialogs pub fn open_options(app: &mut EffectcraftApp) -> Result<(), String> { diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index e82c4d55..d847edb8 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -562,6 +562,26 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { track_hits = super::tracker::draw_overlay(app, &painter, &map, &m, layer, tu, time); } + // Warp Stabilizer banner over the layer being analysed ("Analyzing in background (step 1 + // of 2)…" / "Stabilizing…"). + if let Some((wc, wl, _)) = app.session.warp_target() + && wc == cid + && let Some(pr) = app.session.warp_progress() + && let Some(layer) = comp.layer(wl) + && layer.is_active_at(time) + { + let (m, _) = l2c(&ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(effectcraft_engine::geom::vec2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let text = pr.banner(); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0)); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add("viewer.warpBanner", r, &text); + } + // Free transform box of a path (double-click a path with the Selection tool). let ft = ov::draw_free_transform(app, &ctx, &painter, &map, &paths); diff --git a/docs/agents.md b/docs/agents.md index fd2cc8ea..510bc356 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -100,6 +100,52 @@ effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \ track.analyze '{"wait":true}' track.apply '{}' --save ``` +### Mask tracking and Mask Interpolation + +`track.mask` follows the pixels inside a mask and keys its Mask Path on every frame (vertices and +tangents move with the fitted motion). `method` is `position`, `positionScale`, +`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew` +or `perspective`; `direction` is `forward|backward|frameForward|frameBackward` from the current +time (or `start` / `end` in seconds). Like `track.analyze` it runs in the background in the app +(progress in `track.status` under `mask`, `track.stop` cancels and keeps what was tracked) and is +one undo step; `"wait": true` blocks. + +`mask.interpolate` (Window ▸ Mask Interpolation ▸ Apply) adds in-between Mask Path keys between +each pair of selected Mask Path keys (or the existing keys at `times`, in seconds), giving both +ends the same vertex count with a matched correspondence. Options: `keyframeRate` (number or +`"auto"` = the comp rate), `keyframeFields`, `linearVertexPaths`, `bendingResistance`, `quality`, +`addVertices` (number or `false`) with `addVerticesUnit` `pixels|total|percent`, `matchingMethod` +`auto|curve|polyline`, `oneToOne`, `firstVerticesMatch`. `mask.interpolationOptions` sets the +panel's defaults. + +```sh +effectcraft-cli exec track.mask '{"layer":"#2","mask":1,"method":"perspective","wait":true}' clip.ecproj --save +effectcraft-cli exec mask.interpolate '{"layer":"#2","mask":"Mask 1","times":[0,2],"addVertices":10}' clip.ecproj --save --json +``` + +### Warp Stabilizer + +Effect ▸ Distort ▸ Warp Stabilizer (`effect.apply {"effect":"Warp Stabilizer"}`), Animation ▸ Warp +Stabilizer VFX and the Tracker panel's button (`track.warpStabilizer`, which applies the effect and +starts analysing) stabilize a layer. `warp.analyze {layer?, effect?, wait?}` analyses every frame +between the layer's In and Out points in the background ("Analyzing in background (step 1 of 2)", +then "Stabilizing..."; `warp.status` reports `progress`, `banner`, whether the effect is +`analyzed`, and the current frame's `warp` matrix, `autoScale` and `crop`); `warp.cancel` stops it +without writing anything. The analysis is stored in the effect (saved with the project) and is +one undo step; trimming, slipping, time-remapping or replacing the layer's source clears it and +the app re-analyses automatically (headless: run `warp.analyze` again). Settings are ordinary +properties under the instance's groups, e.g. `effects/#1/stabilization/result` (0 Smooth Motion, +1 No Motion), `effects/#1/stabilization/smoothness`, `effects/#1/stabilization/method`, +`effects/#1/borders/framing` (0 Stabilize Only … 3 Stabilize, Synthesize Edges), +`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. + +```sh +effectcraft-cli run shaky.ecproj track.warpStabilizer '{"layer":"#1","wait":true}' \ + prop.set '{"layer":"#1","path":"effects/#1/stabilization/result","value":1}' --save +effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --save --json +effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index c3d478e3..033ed6cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,7 +35,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L5 | `automation` | The MCP server, headless or bridged to the running app | | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | -Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `effects → project, text, path`, +Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `effects → project, text, path, track`, `media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine`. ## 2. Time @@ -158,6 +158,24 @@ queue; blocking with `wait`), and `effectcraft-track` matches each point in para covers an analysis. `track.apply` keys the target's Position/Rotation/Scale, the tracked layer's Anchor Point and Position (Stabilize), or a Corner Pin effect (Parallel / Perspective). +**Mask tracking** (`track.mask`, `engine::mask_track`) runs the same kind of background job over +the layer's source frames: `effectcraft-track`'s mask tracker detects features inside the mask, +tracks them with pyramidal Lucas–Kanade and fits the chosen model (position … perspective) with +RANSAC; the motion moves the Mask Path's vertices and tangents and is keyed per frame. **Mask +Interpolation** (`mask.interpolate`) uses `effectcraft-path`'s smart interpolation (arc-length +vertex insertion, shape-context matching, rigid in-betweens) to key in-between shapes. + +**Warp Stabilizer** (`effects::warp_stab`, `engine::warp`): `warp.analyze` renders the layer's +input to the effect (`Renderer::layer_input`: source, masks and the effects above it) for every +frame on a background thread and stores `effectcraft-track`'s `WarpAnalysis` (per-frame +translation / similarity / homography fits) as JSON in the effect's hidden Analysis parameter, +with a key of the layer's source, In/Out, start, stretch and Time Remap. `Session::edit` clears +analyses whose key no longer matches and queues them; the desktop app re-analyses them in the +background. Rendering derives the stabilization plan (smoothed camera path or No Motion, framing, +auto-scale; cached per analysis and settings) and warps each frame; Synthesize Edges fills the +borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the +effects crate, hence the `effects → track` edge. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index af0c9420..d259fec2 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -97,15 +97,15 @@ wasm threads. | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | -| Masks & roto | 44% | 13.0 | mask tracking, Roto Brush, variable-width mask feather points | +| Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | 0% | 13.0 | point tracker, Warp Stabilizer, mask and face tracking | +| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five -OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Warp Stabilizer, Compressor, +OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX Mocha and Cineware are third-party and not counted. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ae38adf9..76ff6fa3 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -55,6 +55,7 @@ const SAME_LAYER: &[(&str, &str)] = &[ ("effects", "project"), ("effects", "text"), ("effects", "path"), + ("effects", "track"), ("media", "render"), ("expr", "render"), ("export", "render"), From dec091e57539cf1642025df10736ea4b0eb196e4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:27:59 -0400 Subject: [PATCH 3/4] M7.5: advanced 3D renderer Advanced 3D comps render their 3D runs as one scene of world-space triangles: model and primitive layers (meshes, skinning, animation clips by layer time), extruded text/shape layers, and textured cards for every other layer with their Material Options. Shading is physically based in linear light: Cook-Torrance GGX + Lambert for parallel/spot/point lights (AE falloff and cone), ambient lights, image-based light from an Environment light's layer (equirectangular; cosine irradiance map + roughness mip chain + split-sum BRDF fit), and PCF shadow maps (spot/parallel/point cube faces). 2x2 SSAA, depth of field as a gather blur by circle of confusion, back to the working encoding. The software rasteriser (depth buffer, top-left rule, perspective-correct, opaque visibility then sorted transparent blending) is the reference and CI path; effectcraft-gpu implements Accelerator::raster_3d with a wgpu render pipeline (advanced3d.wgsl) and matches it within tolerance. Environment layers are hidden from both renderers. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/model3d.rs | 2 +- crates/gpu/src/adv3d.rs | 318 +++++++++ crates/gpu/src/context.rs | 3 + crates/gpu/src/lib.rs | 10 + crates/gpu/src/shaders/advanced3d.wgsl | 429 ++++++++++++ crates/gpu/src/tests_adv3d.rs | 167 +++++ crates/host/src/lib.rs | 33 + crates/render/src/lib.rs | 8 +- crates/render/src/shapes.rs | 21 + crates/render/src/three_d/adv/mod.rs | 165 +++++ crates/render/src/three_d/adv/raster.rs | 492 +++++++++++++ crates/render/src/three_d/adv/scene.rs | 872 ++++++++++++++++++++++++ crates/render/src/three_d/adv/shade.rs | 408 +++++++++++ crates/render/src/three_d/adv/tests.rs | 314 +++++++++ crates/render/src/three_d/compose.rs | 6 + crates/render/src/three_d/light.rs | 13 +- crates/render/src/three_d/mod.rs | 1 + 17 files changed, 3259 insertions(+), 3 deletions(-) create mode 100644 crates/gpu/src/adv3d.rs create mode 100644 crates/gpu/src/shaders/advanced3d.wgsl create mode 100644 crates/gpu/src/tests_adv3d.rs create mode 100644 crates/render/src/three_d/adv/mod.rs create mode 100644 crates/render/src/three_d/adv/raster.rs create mode 100644 crates/render/src/three_d/adv/scene.rs create mode 100644 crates/render/src/three_d/adv/shade.rs create mode 100644 crates/render/src/three_d/adv/tests.rs diff --git a/crates/engine/src/commands/model3d.rs b/crates/engine/src/commands/model3d.rs index 1ad9cc32..5de5a2c2 100644 --- a/crates/engine/src/commands/model3d.rs +++ b/crates/engine/src/commands/model3d.rs @@ -349,7 +349,7 @@ pub fn specs() -> Vec { "3D Model Layer", [], None, - "{item?: id|name (a 3D model footage item) | path?: .gltf|.glb|.obj file, name?, time? (s)}", + "{item?: id or name of a 3D model footage item, path?: a .gltf, .glb or .obj file to import, name?, time? (s)}", has_comp, new_model ), diff --git a/crates/gpu/src/adv3d.rs b/crates/gpu/src/adv3d.rs new file mode 100644 index 00000000..6e428c84 --- /dev/null +++ b/crates/gpu/src/adv3d.rs @@ -0,0 +1,318 @@ +//! The Advanced 3D rasteriser on wgpu: a render pipeline with a reversed-Z depth buffer +//! (`Depth32Float`, Greater), the scene's materials, textures, lights and shadow maps in +//! storage buffers, and `advanced3d.wgsl`'s physically based fragment shader. Opaque triangles +//! are drawn with depth writes, then the sorted transparent ones blended over (premultiplied). +//! Colour (`Rgba16Float`) and camera depth (`R32Float`) are read back for the CPU's resolve, +//! depth of field and encoding. + +use effectcraft_render::three_d::adv::{Scene, Target}; +use wgpu::util::DeviceExt; + +use crate::context::{Enc, GpuContext}; + +/// Pipelines (built on first use). +pub(crate) struct Pipes { + bgl: wgpu::BindGroupLayout, + opaque: wgpu::RenderPipeline, + transparent: wgpu::RenderPipeline, +} + +const COLOR: wgpu::TextureFormat = wgpu::TextureFormat::Rgba16Float; +const DEPTH_OUT: wgpu::TextureFormat = wgpu::TextureFormat::R32Float; +const DEPTH: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float; +const VERTEX_SIZE: u64 = 52; + +fn storage(binding: u32) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only: true }, has_dynamic_offset: false, min_binding_size: None }, + count: None, + } +} + +impl Pipes { + fn new(device: &wgpu::Device) -> Pipes { + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("effectcraft advanced 3d"), + source: wgpu::ShaderSource::Wgsl(include_str!("shaders/advanced3d.wgsl").into()), + }); + let mut entries = vec![wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }]; + entries.extend((1..=6).map(storage)); + let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("advanced 3d"), entries: &entries }); + let layout = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("advanced 3d"), bind_group_layouts: &[Some(&bgl)], immediate_size: 0 }); + let attrs = [ + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x3, offset: 0, shader_location: 0 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x3, offset: 12, shader_location: 1 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x2, offset: 24, shader_location: 2 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x4, offset: 32, shader_location: 3 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Uint32, offset: 48, shader_location: 4 }, + ]; + let make = |transparent: bool| { + let blend = transparent.then_some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(if transparent { "advanced 3d transparent" } else { "advanced 3d opaque" }), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs"), + compilation_options: Default::default(), + buffers: &[Some(wgpu::VertexBufferLayout { array_stride: VERTEX_SIZE, step_mode: wgpu::VertexStepMode::Vertex, attributes: &attrs })], + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + ..Default::default() + }, + depth_stencil: Some(wgpu::DepthStencilState { + format: DEPTH, + depth_write_enabled: Some(!transparent), + depth_compare: Some(wgpu::CompareFunction::Greater), + stencil: Default::default(), + bias: Default::default(), + }), + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs"), + compilation_options: Default::default(), + targets: &[ + Some(wgpu::ColorTargetState { format: COLOR, blend, write_mask: wgpu::ColorWrites::ALL }), + Some(wgpu::ColorTargetState { + format: DEPTH_OUT, + blend: None, + write_mask: if transparent { wgpu::ColorWrites::empty() } else { wgpu::ColorWrites::ALL }, + }), + ], + }), + multiview_mask: None, + cache: None, + }) + }; + Pipes { opaque: make(false), transparent: make(true), bgl } + } +} + +fn f32s(v: impl IntoIterator) -> Vec { + let mut b: Vec = v.into_iter().flat_map(f32::to_le_bytes).collect(); + while b.len() < 16 { + b.push(0); + } + b +} + +/// Pack the scene's arrays as the shader's storage buffers. +fn pack(s: &Scene) -> [Vec; 7] { + let mut globals: Vec = vec![]; + // Column-major clip matrix. + for c in 0..4 { + for r in 0..4 { + globals.push(s.clip[r][c]); + } + } + globals.extend(s.view[2]); + globals.extend([s.eye[0], s.eye[1], s.eye[2], s.ortho as u32 as f32]); + globals.extend([s.cam_fwd[0], s.cam_fwd[1], s.cam_fwd[2], s.lights.len() as f32]); + globals.extend([s.ambient[0], s.ambient[1], s.ambient[2], s.env.is_some() as u32 as f32]); + let e = s.env.unwrap_or(effectcraft_render::three_d::adv::EnvInfo { radiance: 0, mips: 1, irradiance: 0, intensity: 0.0, rotation: 0.0 }); + globals.extend([e.radiance as f32, e.mips as f32, e.irradiance as f32, e.intensity]); + let lit = !(s.lights.is_empty() && s.env.is_none() && s.ambient == [0.0; 3]); + globals.extend([e.rotation, lit as u32 as f32, 0.0, 0.0]); + let mats = s.materials.iter().flat_map(|m| { + let flags = (m.double_sided as u32) | (m.unlit as u32) << 1 | (m.accepts_lights as u32) << 2 | (m.receives_shadows as u32) << 3 | (m.alpha_mode << 8); + [ + m.base[0], + m.base[1], + m.base[2], + m.base[3], + m.emissive[0], + m.emissive[1], + m.emissive[2], + m.metallic, + m.roughness, + m.normal_scale, + m.occlusion_strength, + m.alpha_cutoff, + m.tex_base as f32, + m.tex_mr as f32, + m.tex_normal as f32, + m.tex_occlusion as f32, + m.tex_emissive as f32, + flags as f32, + m.diffuse_k, + m.specular_k, + m.ambient_k, + m.opacity, + 0.0, + 0.0, + ] + }); + let mut tex: Vec = s.textures.iter().flat_map(|t| [t.offset, t.width, t.height, t.wrap_u | t.wrap_v << 8]).flat_map(u32::to_le_bytes).collect(); + while tex.len() < 16 { + tex.push(0); + } + let texels = f32s(s.texels.iter().flatten().copied()); + let lights = s.lights.iter().flat_map(|l| { + [ + l.pos[0], + l.pos[1], + l.pos[2], + l.kind as f32, + l.dir[0], + l.dir[1], + l.dir[2], + l.cos_outer, + l.color[0], + l.color[1], + l.color[2], + l.cos_inner, + l.falloff as f32, + l.radius, + l.falloff_distance, + l.shadow as f32, + l.shadow_darkness, + 0.0, + 0.0, + 0.0, + ] + }); + let shadows = s.shadows.iter().flat_map(|m| { + let mut v: Vec = m.view.iter().flatten().copied().collect(); + v.extend([m.focal, m.ortho as u32 as f32, f32::from_bits(m.size), f32::from_bits(m.offset), m.bias, m.radius, 0.0, 0.0]); + v + }); + [f32s(globals), f32s(mats), tex, texels, f32s(lights), f32s(shadows), f32s(s.shadow_texels.iter().copied())] +} + +fn half_to_f32(h: u16) -> f32 { + let s = if h & 0x8000 != 0 { -1.0 } else { 1.0 }; + let e = ((h >> 10) & 0x1f) as i32; + let m = (h & 0x3ff) as f32; + match e { + 0 => s * m * 2f32.powi(-24), + 31 => { + if m == 0.0 { + s * f32::INFINITY + } else { + f32::NAN + } + } + _ => s * (1.0 + m / 1024.0) * 2f32.powi(e - 15), + } +} + +/// Rasterise a scene. `None` when the device can't (no readback, size or buffer limits). +pub(crate) fn render(g: &GpuContext, s: &Scene) -> Option { + if !g.can_readback() || !g.fits(s.width, s.height) || s.indices.is_empty() { + return None; + } + let limits = g.device.limits(); + let bufs = pack(s); + if bufs.iter().any(|b| b.len() as u64 > limits.max_storage_buffer_binding_size) { + return None; + } + let p = g.adv3d.get_or_init(|| Pipes::new(&g.device)); + let dev = &g.device; + let vb: Vec = s + .vertices + .iter() + .flat_map(|v| { + let mut b = Vec::with_capacity(VERTEX_SIZE as usize); + for x in v.pos.iter().chain(&v.normal).chain(&v.uv).chain(&v.tangent) { + b.extend_from_slice(&x.to_le_bytes()); + } + b.extend_from_slice(&v.material.to_le_bytes()); + b + }) + .collect(); + let vbuf = dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d vertices"), contents: &vb, usage: wgpu::BufferUsages::VERTEX }); + let ib: Vec = s.indices.iter().flat_map(|i| i.to_le_bytes()).collect(); + let ibuf = dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d indices"), contents: &ib, usage: wgpu::BufferUsages::INDEX }); + let ub = dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d globals"), contents: &bufs[0], usage: wgpu::BufferUsages::UNIFORM }); + let sbs: Vec = bufs[1..] + .iter() + .map(|b| dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d data"), contents: b, usage: wgpu::BufferUsages::STORAGE })) + .collect(); + let mut entries = vec![wgpu::BindGroupEntry { binding: 0, resource: ub.as_entire_binding() }]; + for (i, b) in sbs.iter().enumerate() { + entries.push(wgpu::BindGroupEntry { binding: i as u32 + 1, resource: b.as_entire_binding() }); + } + let bg = dev.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("adv3d"), layout: &p.bgl, entries: &entries }); + let tex = |format: wgpu::TextureFormat, usage: wgpu::TextureUsages| { + dev.create_texture(&wgpu::TextureDescriptor { + label: Some("adv3d target"), + size: wgpu::Extent3d { width: s.width, height: s.height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage, + view_formats: &[], + }) + }; + let rt = wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC; + let color = tex(COLOR, rt); + let zout = tex(DEPTH_OUT, rt); + let depth = tex(DEPTH, wgpu::TextureUsages::RENDER_ATTACHMENT); + let (cv, zv, dv) = (color.create_view(&Default::default()), zout.create_view(&Default::default()), depth.create_view(&Default::default())); + let mut enc = dev.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("adv3d") }); + { + let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("adv3d"), + color_attachments: &[ + Some(wgpu::RenderPassColorAttachment { + view: &cv, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), store: wgpu::StoreOp::Store }, + }), + Some(wgpu::RenderPassColorAttachment { + view: &zv, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color { r: -1.0, g: 0.0, b: 0.0, a: 0.0 }), store: wgpu::StoreOp::Store }, + }), + ], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &dv, + depth_ops: Some(wgpu::Operations { load: wgpu::LoadOp::Clear(0.0), store: wgpu::StoreOp::Store }), + stencil_ops: None, + }), + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_bind_group(0, &bg, &[]); + pass.set_vertex_buffer(0, vbuf.slice(..)); + pass.set_index_buffer(ibuf.slice(..), wgpu::IndexFormat::Uint32); + let n = s.indices.len() as u32; + if s.opaque_count > 0 { + pass.set_pipeline(&p.opaque); + pass.draw_indexed(0..s.opaque_count, 0, 0..1); + } + if n > s.opaque_count { + pass.set_pipeline(&p.transparent); + pass.draw_indexed(s.opaque_count..n, 0, 0..1); + } + } + g.queue.submit([enc.finish()]); + let mut e = Enc::new(g); + let cb = e.read_texture(&color, s.width, s.height, 8)?; + let zb = e.read_texture(&zout, s.width, s.height, 4)?; + let color = cb.chunks_exact(8).map(|c| [0, 1, 2, 3].map(|k| half_to_f32(u16::from_le_bytes([c[2 * k], c[2 * k + 1]])))).collect(); + let depth = zb + .chunks_exact(4) + .map(|c| { + let z = f32::from_le_bytes([c[0], c[1], c[2], c[3]]); + if z < 0.0 { f32::INFINITY } else { z } + }) + .collect(); + Some(Target { width: s.width, height: s.height, color, depth }) +} diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index 9f814377..1b141301 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -94,6 +94,8 @@ pub struct GpuContext { dummy_tex: wgpu::TextureView, dummy_buf: wgpu::Buffer, uploads: Mutex, + /// Advanced 3D render pipelines (built on first use). + pub(crate) adv3d: std::sync::OnceLock, } fn tex_entry(binding: u32) -> wgpu::BindGroupLayoutEntry { @@ -225,6 +227,7 @@ impl GpuContext { dummy_tex: dummy.create_view(&Default::default()), dummy_buf, uploads: Mutex::new(Uploads::default()), + adv3d: std::sync::OnceLock::new(), }) } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index e7b54ab2..c299cf5d 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -9,6 +9,9 @@ //! clamping and quantisation, and converts colour spaces. 3D runs, adjustment layers and //! wireframes run on the CPU between GPU steps (read back, draw, upload). //! +//! Advanced 3D comps rasterise on a render pipeline (`advanced3d.wgsl`: depth buffer, PBR, +//! image-based light, shadow maps), see [`Accelerator::raster_3d`]. +//! //! GPU effects ([`effectcraft_effects::GPU_EFFECTS`]) run as compute kernels with the CPU //! effect's exact steps (padding, box-blur radii, parameter conversions); chains of them are //! uploaded and read back once. @@ -18,6 +21,7 @@ //! skip readback entirely with [`Gpu::render_display`], which leaves an RGBA8 texture for //! egui-wgpu to draw. +mod adv3d; mod context; mod effects; mod ops; @@ -121,7 +125,13 @@ impl Accelerator for Gpu { fn effects(&self, chain: &[FxStep], buf: &Buf, levels: Option) -> Option { effects::run_chain(&mut Enc::new(&self.ctx), chain, buf, levels) } + + fn raster_3d(&self, scene: &effectcraft_render::three_d::adv::Scene) -> Option { + adv3d::render(&self.ctx, scene) + } } #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_adv3d; diff --git a/crates/gpu/src/shaders/advanced3d.wgsl b/crates/gpu/src/shaders/advanced3d.wgsl new file mode 100644 index 00000000..9ae81ff5 --- /dev/null +++ b/crates/gpu/src/shaders/advanced3d.wgsl @@ -0,0 +1,429 @@ +// Advanced 3D: rasterised meshes with physically based shading. A step-for-step port of +// `effectcraft_render::three_d::adv::shade` (Cook–Torrance GGX + Lambert, image-based light +// from an equirectangular environment, PCF shadow maps); the CPU rasteriser is the reference. + +struct Globals { + clip: mat4x4, + // Camera-space depth row (x, y, z, w). + view_z: vec4, + // eye.xyz, ortho (0/1) + eye: vec4, + // cam_fwd.xyz, light count + fwd: vec4, + // ambient.rgb, environment present (0/1) + ambient: vec4, + // radiance first texture, mip count, irradiance texture, intensity + env: vec4, + // rotation, any lighting (0/1), -, - + env2: vec4, +}; + +@group(0) @binding(0) var g: Globals; +@group(0) @binding(1) var materials: array>; +@group(0) @binding(2) var texinfo: array>; +@group(0) @binding(3) var texels: array>; +@group(0) @binding(4) var lights: array>; +@group(0) @binding(5) var shadows: array>; +@group(0) @binding(6) var shadow_texels: array; + +const PI: f32 = 3.14159265358979; +const MAT_STRIDE: u32 = 6u; +const LIGHT_STRIDE: u32 = 5u; +const SHADOW_STRIDE: u32 = 5u; + +struct VIn { + @location(0) pos: vec3, + @location(1) normal: vec3, + @location(2) uv: vec2, + @location(3) tangent: vec4, + @location(4) material: u32, +}; + +struct VOut { + @builtin(position) clip: vec4, + @location(0) pos: vec3, + @location(1) normal: vec3, + @location(2) uv: vec2, + @location(3) tangent: vec4, + @location(4) @interpolate(flat) material: u32, +}; + +@vertex +fn vs(v: VIn) -> VOut { + var o: VOut; + o.clip = g.clip * vec4(v.pos, 1.0); + o.pos = v.pos; + o.normal = v.normal; + o.uv = v.uv; + o.tangent = v.tangent; + o.material = v.material; + return o; +} + +fn norm3(a: vec3) -> vec3 { + let l = sqrt(dot(a, a)); + if (l > 1e-20) { + return a / l; + } + return vec3(0.0, 0.0, -1.0); +} + +fn srgb_to_linear(c: f32) -> f32 { + if (c <= 0.04045) { + return c / 12.92; + } + return pow((c + 0.055) / 1.055, 2.4); +} + +fn wrap_i(i: i32, n: i32, mode: u32) -> i32 { + if (mode == 1u) { + return clamp(i, 0, n - 1); + } + if (mode == 2u) { + let p = 2 * n; + var m = i % p; + if (m < 0) { + m = m + p; + } + if (m >= n) { + return p - 1 - m; + } + return m; + } + var m = i % n; + if (m < 0) { + m = m + n; + } + return m; +} + +fn texel(t: vec4, x: i32, y: i32) -> vec4 { + let w = i32(t.y); + let h = i32(t.z); + let xi = wrap_i(x, w, t.w & 255u); + let yi = wrap_i(y, h, (t.w >> 8u) & 255u); + return texels[t.x + u32(yi * w + xi)]; +} + +fn sample_tex(i: i32, uv: vec2) -> vec4 { + let t = texinfo[u32(i)]; + if (t.y == 0u || t.z == 0u) { + return vec4(1.0); + } + let x = uv.x * f32(t.y) - 0.5; + let y = uv.y * f32(t.z) - 0.5; + let x0 = floor(x); + let y0 = floor(y); + let fx = x - x0; + let fy = y - y0; + let xi = i32(x0); + let yi = i32(y0); + let a = texel(t, xi, yi); + let b = texel(t, xi + 1, yi); + let c = texel(t, xi, yi + 1); + let d = texel(t, xi + 1, yi + 1); + let top = a + (b - a) * fx; + let bot = c + (d - c) * fx; + return top + (bot - top) * fy; +} + +fn d_ggx(n_h: f32, a: f32) -> f32 { + let a2 = a * a; + let d = n_h * n_h * (a2 - 1.0) + 1.0; + return a2 / max(PI * d * d, 1e-12); +} + +fn g1(x: f32, k: f32) -> f32 { + return x / (x * (1.0 - k) + k); +} + +fn brdf(base: vec3, metallic: f32, rough_in: f32, spec_k: f32, diff_k: f32, n: vec3, v: vec3, l: vec3) -> vec3 { + let n_l = dot(n, l); + if (n_l <= 0.0) { + return vec3(0.0); + } + let n_v = max(dot(n, v), 1e-4); + let h = norm3(v + l); + let n_h = max(dot(n, h), 0.0); + let v_h = max(dot(v, h), 0.0); + let r = clamp(rough_in, 0.03, 1.0); + let f0 = mix(vec3(0.04 * spec_k), base, metallic); + let fk = pow(clamp(1.0 - v_h, 0.0, 1.0), 5.0); + let f = f0 + (vec3(1.0) - f0) * fk; + let d = d_ggx(n_h, r * r); + let k = (r + 1.0) * (r + 1.0) / 8.0; + let gg = g1(n_v, k) * g1(n_l, k); + let spec = d * gg / max(4.0 * n_v * n_l, 1e-4); + let kd = (vec3(1.0) - f) * (1.0 - metallic); + return (kd * base / PI * diff_k + f * spec) * n_l * PI; +} + +fn env_brdf(n_v: f32, rough: f32) -> vec2 { + let c0 = vec4(-1.0, -0.0275, -0.572, 0.022); + let c1 = vec4(1.0, 0.0425, 1.04, -0.04); + let r = rough * c0 + c1; + let a004 = min(r.x * r.x, exp2(-9.28 * n_v)) * r.x + r.y; + return vec2(-1.04 * a004 + r.z, 1.04 * a004 + r.w); +} + +fn equirect_uv(d: vec3, rot: f32) -> vec2 { + let u = 0.5 + (atan2(d.x, d.z) + rot) / (2.0 * PI); + let v = acos(clamp(-d.y, -1.0, 1.0)) / PI; + return vec2(u - floor(u), v); +} + +fn shadow_factor(m: u32, p: vec3, n_l: f32) -> f32 { + let r0 = shadows[m]; + let r1 = shadows[m + 1u]; + let r2 = shadows[m + 2u]; + let info = shadows[m + 3u]; + let extra = shadows[m + 4u]; + let q = vec3(dot(r0.xyz, p) + r0.w, dot(r1.xyz, p) + r1.w, dot(r2.xyz, p) + r2.w); + let focal = info.x; + let ortho = info.y > 0.5; + let size = bitcast(info.z); + let offset = bitcast(info.w); + let c = f32(size) * 0.5; + var u: f32; + var v: f32; + if (ortho) { + u = q.x * focal + c; + v = q.y * focal + c; + } else { + if (q.z <= 1e-3) { + return 1.0; + } + u = q.x * focal / q.z + c; + v = q.y * focal / q.z + c; + } + let z = q.z; + let bias = extra.x * (1.0 + 2.0 * (1.0 - clamp(n_l, 0.0, 1.0))) + abs(z) * 0.0015; + let step = extra.y / 2.0; + let sz = i32(size); + var lit = 0.0; + for (var j = -2; j <= 2; j = j + 1) { + for (var i = -2; i <= 2; i = i + 1) { + let x = i32(floor(u + f32(i) * step)); + let y = i32(floor(v + f32(j) * step)); + if (x < 0 || y < 0 || x >= sz || y >= sz) { + lit = lit + 1.0; + continue; + } + let d = shadow_texels[offset + u32(y * sz + x)]; + if (z - bias <= d) { + lit = lit + 1.0; + } + } + } + return lit / 25.0; +} + +fn light_shadow(li: u32, kind: u32, lpos: vec3, shadow: i32, darkness: f32, p: vec3, n_l: f32) -> f32 { + if (shadow < 0) { + return 1.0; + } + var face = 0u; + if (kind == 2u) { + let d = p - lpos; + let a = abs(d); + if (a.x >= a.y && a.x >= a.z) { + face = select(1u, 0u, d.x > 0.0); + } else if (a.y >= a.z) { + face = select(3u, 2u, d.y > 0.0); + } else { + face = select(5u, 4u, d.z > 0.0); + } + } + let f = shadow_factor((u32(shadow) + face) * SHADOW_STRIDE, p, n_l); + return 1.0 - darkness * (1.0 - f); +} + +fn attenuation(kind: u32, falloff: u32, radius: f32, falloff_distance: f32, cos_inner: f32, cos_outer: f32, dir: vec3, to_l: vec3, dist: f32) -> f32 { + var k = 1.0; + if (falloff == 1u && kind != 0u) { + if (dist <= radius) { + k = 1.0; + } else if (falloff_distance <= 0.0) { + k = 0.0; + } else { + let t = clamp((dist - radius) / falloff_distance, 0.0, 1.0); + k = 1.0 - t * t * (3.0 - 2.0 * t); + } + } else if (falloff == 2u && kind != 0u) { + if (dist <= radius || (radius <= 0.0 && dist <= 1.0)) { + k = 1.0; + } else { + let q = max(radius, 1.0) / dist; + k = q * q; + } + } + if (kind == 1u) { + let cos_a = clamp(dot(-to_l, dir), -1.0, 1.0); + if (cos_a >= cos_inner) { + k = k * 1.0; + } else if (cos_a <= cos_outer) { + k = 0.0; + } else { + let a = acos(cos_a); + let ai = acos(cos_inner); + let ao = acos(cos_outer); + let t = clamp((a - ai) / max(ao - ai, 1e-6), 0.0, 1.0); + k = k * (1.0 - t * t * (3.0 - 2.0 * t)); + } + } + return k; +} + +struct FOut { + @location(0) color: vec4, + @location(1) depth: f32, +}; + +@fragment +fn fs(i: VOut, @builtin(front_facing) front: bool) -> FOut { + let mb = i.material * MAT_STRIDE; + let m0 = materials[mb]; + let m1 = materials[mb + 1u]; + let m2 = materials[mb + 2u]; + let m3 = materials[mb + 3u]; + let m4 = materials[mb + 4u]; + let m5 = materials[mb + 5u]; + let flags = u32(m4.y); + let double_sided = (flags & 1u) != 0u; + let unlit = (flags & 2u) != 0u; + let accepts_lights = (flags & 4u) != 0u; + let receives_shadows = (flags & 8u) != 0u; + let alpha_mode = (flags >> 8u) & 3u; + if (!front && !double_sided) { + discard; + } + var base = m0; + let tex_base = i32(m3.x); + if (tex_base >= 0) { + base = base * sample_tex(tex_base, i.uv); + } + let opacity = m5.y; + var alpha = base.a * opacity; + if (alpha_mode == 1u && alpha < m2.w) { + discard; + } + if (alpha <= 1e-5 && alpha_mode != 0u) { + discard; + } + if (alpha_mode == 0u) { + alpha = opacity; + } + let rgb = base.rgb; + var emissive = m1.xyz; + let tex_em = i32(m4.x); + if (tex_em >= 0) { + emissive = emissive * sample_tex(tex_em, i.uv).rgb; + } + var out: FOut; + out.depth = dot(g.view_z.xyz, i.pos) + g.view_z.w; + if (unlit || !accepts_lights || g.env2.y < 0.5) { + let c = rgb + emissive; + out.color = vec4(c * alpha, alpha); + return out; + } + var metallic = m1.w; + var rough = m2.x; + let tex_mr = i32(m3.y); + if (tex_mr >= 0) { + let t = sample_tex(tex_mr, i.uv); + rough = rough * t.y; + metallic = metallic * t.z; + } + var n = norm3(i.normal); + if (!front) { + n = -n; + } + let tex_n = i32(m3.z); + if (tex_n >= 0) { + let t = sample_tex(tex_n, i.uv); + let ns = m2.y; + let tn = vec3((t.x * 2.0 - 1.0) * ns, (t.y * 2.0 - 1.0) * ns, t.z * 2.0 - 1.0); + let tg = norm3(i.tangent.xyz - n * dot(n, i.tangent.xyz)); + let sign = select(-i.tangent.w, i.tangent.w, front); + let bt = cross(n, tg) * sign; + n = norm3(tg * tn.x + bt * tn.y + n * tn.z); + } + var ao = 1.0; + let tex_ao = i32(m3.w); + if (tex_ao >= 0) { + ao = 1.0 + m2.z * (sample_tex(tex_ao, i.uv).x - 1.0); + } + var v: vec3; + if (g.eye.w > 0.5) { + v = -g.fwd.xyz; + } else { + v = norm3(g.eye.xyz - i.pos); + } + let diff_k = m4.z; + let spec_k = m4.w; + let amb_k = m5.x; + var acc = vec3(0.0); + let nl_count = u32(g.fwd.w); + for (var li = 0u; li < nl_count; li = li + 1u) { + let lb = li * LIGHT_STRIDE; + let l0 = lights[lb]; + let l1 = lights[lb + 1u]; + let l2 = lights[lb + 2u]; + let l3 = lights[lb + 3u]; + let l4 = lights[lb + 4u]; + let kind = u32(l0.w); + var to_l: vec3; + var dist: f32; + if (kind == 0u) { + to_l = -l1.xyz; + dist = 1e30; + } else { + let d = l0.xyz - i.pos; + let len = sqrt(dot(d, d)); + if (len < 1e-6) { + to_l = vec3(0.0, 0.0, -1.0); + dist = 0.0; + } else { + to_l = d / len; + dist = len; + } + } + let k = attenuation(kind, u32(l3.x), l3.y, l3.z, l2.w, l1.w, l1.xyz, to_l, dist); + if (k <= 0.0) { + continue; + } + let n_l = dot(n, to_l); + if (n_l <= 0.0) { + continue; + } + var sh = 1.0; + if (receives_shadows) { + sh = light_shadow(li, kind, l0.xyz, i32(l3.w), l4.x, i.pos, n_l); + } + let b = brdf(rgb, metallic, rough, spec_k, diff_k, n, v, to_l); + acc = acc + b * l2.xyz * k * sh; + } + acc = acc + g.ambient.xyz * rgb * amb_k * ao; + if (g.ambient.w > 0.5) { + let n_v = clamp(dot(n, v), 1e-4, 1.0); + let r = n * (2.0 * dot(n, v)) - v; + let rot = g.env2.x; + let irr = sample_tex(i32(g.env.z), equirect_uv(n, rot)).rgb * g.env.w; + let mips = u32(g.env.y); + let lv = clamp(rough, 0.0, 1.0) * f32(max(mips, 1u) - 1u); + let lo = u32(floor(lv)); + let hi = min(lo + 1u, max(mips, 1u) - 1u); + let f = lv - f32(lo); + let ruv = equirect_uv(r, rot); + let sa = sample_tex(i32(g.env.x) + i32(lo), ruv).rgb; + let sb = sample_tex(i32(g.env.x) + i32(hi), ruv).rgb; + let spec = (sa + (sb - sa) * f) * g.env.w; + let f0 = mix(vec3(0.04 * spec_k), rgb, metallic); + let ab = env_brdf(n_v, rough); + let fs = f0 * ab.x + vec3(ab.y); + acc = acc + (irr * rgb * (vec3(1.0) - fs) * (1.0 - metallic) * diff_k + spec * fs) * ao; + } + let c = acc + emissive; + out.color = vec4(c * alpha, alpha); + return out; +} diff --git a/crates/gpu/src/tests_adv3d.rs b/crates/gpu/src/tests_adv3d.rs new file mode 100644 index 00000000..067e4c8a --- /dev/null +++ b/crates/gpu/src/tests_adv3d.rs @@ -0,0 +1,167 @@ +//! Advanced 3D: the GPU rasteriser against the CPU reference on the same scenes. +//! +//! Tolerances: the GPU's colour target is half float (relative error ≤ 1e-3) and its +//! rasteriser and interpolators differ from the CPU's in the last bits, so pixels on triangle +//! edges may be covered by the other neighbour. Interior pixels agree within 2/255; at most +//! 1.5 % of pixels (silhouettes) may differ more. Skipped without an adapter. + +use effectcraft_keyframe::Value; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, LightKind, PrimitiveKind, Project, Renderer as R3, Solid, build}; +use effectcraft_render::three_d::adv::{self, Scene}; +use effectcraft_render::{Accelerator, EvalCtx, NoFootage, RenderOpts, Renderer}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::Gpu; + +fn gpu() -> Option { + let g = Gpu::headless(); + if g.is_none() { + eprintln!("effectcraft-gpu adv3d tests: no GPU adapter, skipping"); + } + g +} + +fn add(p: &mut Project, cid: ItemId, src: LayerSource, edit: impl FnOnce(&mut effectcraft_project::Layer)) { + let comp = p.comp(cid).unwrap().clone(); + let mut l = build::layer(p, &comp, "L", src, (comp.width, comp.height), None); + edit(&mut l); + p.comp_mut(cid).unwrap().layers.insert(0, l); +} + +fn set(l: &mut effectcraft_project::Layer, path: &str, v: Value) { + l.props.prop_mut(path).unwrap_or_else(|| panic!("{path}")).value = v; +} + +/// Primitives with metal/dielectric materials, a shadow-casting spot light and a point light, +/// an environment light, a semi-transparent textured card. +fn scene_project() -> (Project, ItemId) { + let mut p = Project::default(); + let mut c = Comp::new(240, 160, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + c.renderer = R3::Advanced3D; + let cid = p.add_item("C", Default::default(), None, ItemKind::Comp(c.into())); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Plane }, |l| { + set(l, "geometryOptions/width", Value::Scalar(900.0)); + set(l, "geometryOptions/height", Value::Scalar(900.0)); + set(l, "transform/position", Value::Vec3([120.0, 130.0, 0.0])); + set(l, "transform/rotationX", Value::Scalar(90.0)); + }); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Sphere }, |l| { + set(l, "geometryOptions/radius", Value::Scalar(35.0)); + set(l, "transform/position", Value::Vec3([80.0, 95.0, 0.0])); + set(l, "materialOptions/baseColor", Value::Color([0.9, 0.2, 0.2, 1.0])); + set(l, "materialOptions/roughness", Value::Scalar(35.0)); + set(l, "materialOptions/castsShadows", Value::Enum(1)); + }); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Torus }, |l| { + set(l, "geometryOptions/radius", Value::Scalar(35.0)); + set(l, "geometryOptions/tubeRadius", Value::Scalar(12.0)); + set(l, "transform/position", Value::Vec3([165.0, 90.0, 0.0])); + set(l, "transform/rotationX", Value::Scalar(60.0)); + set(l, "materialOptions/metallic", Value::Scalar(100.0)); + set(l, "materialOptions/roughness", Value::Scalar(20.0)); + set(l, "materialOptions/castsShadows", Value::Enum(1)); + }); + let sid = p.add_item("S", Default::default(), None, ItemKind::Solid(Solid { color: [0.2, 0.6, 0.9], width: 60, height: 40, pixel_aspect: 1.0 })); + add(&mut p, cid, LayerSource::Solid { item: sid }, |l| { + l.switches.three_d = true; + set(l, "transform/opacity", Value::Scalar(60.0)); + set(l, "transform/position", Value::Vec3([120.0, 80.0, -60.0])); + set(l, "transform/rotationY", Value::Scalar(30.0)); + }); + let env = p.add_item("E", Default::default(), None, ItemKind::Solid(Solid { color: [0.6, 0.7, 1.0], width: 32, height: 16, pixel_aspect: 1.0 })); + add(&mut p, cid, LayerSource::Solid { item: env }, |l| { + l.environment = true; + l.switches.three_d = true; + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Environment }, |l| set(l, "lightOptions/intensity", Value::Scalar(50.0))); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Spot }, |l| { + set(l, "transform/position", Value::Vec3([40.0, -120.0, -200.0])); + set(l, "transform/poi", Value::Vec3([120.0, 100.0, 0.0])); + set(l, "lightOptions/castsShadows", Value::Bool(true)); + set(l, "lightOptions/coneAngle", Value::Scalar(110.0)); + set(l, "lightOptions/shadowDiffusion", Value::Scalar(4.0)); + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Point }, |l| { + set(l, "transform/position", Value::Vec3([200.0, 20.0, -100.0])); + set(l, "lightOptions/intensity", Value::Scalar(40.0)); + set(l, "lightOptions/falloff", Value::Enum(1)); + }); + (p, cid) +} + +fn scene(p: &Project, cid: ItemId) -> Scene { + let r = Renderer::new(p, &NoFootage, RenderOpts::default()); + let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect(); + adv::scene_of(&r, &ctx, &run, (ctx.comp.width, ctx.comp.height)) +} + +fn compare(s: &Scene, g: &Gpu) { + let cpu = adv::raster::render(s); + let gt = g.raster_3d(s).expect("gpu raster"); + assert_eq!((gt.width, gt.height), (cpu.width, cpu.height)); + let n = cpu.color.len(); + let mut bad = 0; + let mut sum = 0.0f64; + for (a, b) in cpu.color.iter().zip(>.color) { + let d = (0..4).map(|k| (a[k] - b[k]).abs() / a[k].abs().max(1.0)).fold(0.0f32, f32::max); + sum += d as f64; + if d > 2.0 / 255.0 { + bad += 1; + } + } + let frac = bad as f64 / n as f64; + eprintln!("adv3d gpu vs cpu: {bad}/{n} pixels beyond 2/255 ({:.3} %), mean {:.5}", frac * 100.0, sum / n as f64); + assert!(frac < 0.015, "{:.3} % of pixels differ", frac * 100.0); + assert!(sum / (n as f64) < 0.002); + // Depth agrees where both see a surface. + let mut dz = 0; + for (a, b) in cpu.depth.iter().zip(>.depth) { + if a.is_finite() != b.is_finite() || (a.is_finite() && (a - b).abs() > 0.01 * a.abs().max(1.0)) { + dz += 1; + } + } + assert!((dz as f64) < 0.015 * n as f64, "{dz} depth mismatches"); +} + +#[test] +fn gpu_matches_cpu_lit_scene() { + let Some(g) = gpu() else { return }; + let (p, cid) = scene_project(); + let s = scene(&p, cid); + assert!(!s.shadows.is_empty() && s.env.is_some() && s.opaque_count < s.indices.len() as u32); + compare(&s, &g); +} + +#[test] +fn gpu_matches_cpu_unlit_and_full_frames() { + let Some(g) = gpu() else { return }; + let (mut p, cid) = scene_project(); + // Unlit: remove the lights. + p.comp_mut(cid).unwrap().layers.retain(|l| !l.is_light()); + compare(&scene(&p, cid), &g); + // Whole frames through the renderer with the GPU attached. + let (p, cid) = scene_project(); + let cpu = Renderer::new(&p, &NoFootage, RenderOpts::default()).comp_frame(cid, Tick::ZERO); + let mut r = Renderer::new(&p, &NoFootage, RenderOpts { backend: effectcraft_render::Backend::Gpu, ..Default::default() }); + r.accel = Some(&g); + let gimg = r.comp_frame(cid, Tick::ZERO); + let diff = cpu.data.iter().zip(&gimg.data).filter(|(a, b)| (0..4).any(|k| (a[k] - b[k]).abs() > 3.0 / 255.0)).count(); + assert!((diff as f64) < 0.02 * cpu.data.len() as f64, "{diff} pixels differ"); +} + +#[test] +fn gpu_raster_perf_smoke() { + let Some(g) = gpu() else { return }; + let (p, cid) = scene_project(); + let s = scene(&p, cid); + let t0 = std::time::Instant::now(); + let _ = adv::raster::render(&s); + let cpu = t0.elapsed(); + // First use compiles the pipelines. + let _ = g.raster_3d(&s); + let t0 = std::time::Instant::now(); + let _ = g.raster_3d(&s); + let gpu = t0.elapsed(); + eprintln!("adv3d 480x320 (2x SSAA of 240x160), {} triangles: cpu {:?}, gpu {:?}", s.indices.len() / 3, cpu, gpu); +} diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index 16199dfb..d8906a1a 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -74,6 +74,39 @@ mod tests { assert!(img.data.iter().any(|p| p[3] > 0.5)); } + /// File ▸ Import of glTF/GLB/OBJ models through the media layer, placed as model layers and + /// drawn by the Advanced 3D renderer. + #[test] + fn imported_models_render_in_advanced_3d() { + let fx = |n: &str| format!("{}/../model/tests/fixtures/{n}", env!("CARGO_MANIFEST_DIR")); + let mut s = super::session(); + s.execute("comp.new", json!({"name": "M", "width": 160, "height": 120, "renderer": "advanced3d"})).unwrap(); + let r = s.execute("file.import", json!({"paths": [fx("cube.obj"), fx("quad.glb"), fx("quad.gltf")]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let items: Vec = r["items"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect(); + assert_eq!(items.len(), 3); + let cube = s.execute("layer.addItem", json!({"item": items[0]})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.set", json!({"layer": cube, "path": "transform/rotationY", "value": 30})).unwrap(); + s.execute("prop.set", json!({"layer": cube, "path": "transform/rotationX", "value": 20})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let opts = effectcraft_engine::render::RenderOpts::default(); + let img = s.render(cid, s.time(), opts); + let covered = img.data.iter().filter(|p| p[3] > 0.99).count(); + assert!(covered > 1000, "the cube covers part of the frame: {covered}"); + // Red faces (material Red) are visible. + assert!(img.data.iter().any(|p| p[3] > 0.99 && p[0] > 0.5 && p[1] < 0.2)); + // The textured binary glTF quad. + for _ in 0..3 { + s.execute("edit.undo", json!({})).unwrap(); + } + s.execute("layer.addItem", json!({"item": items[1]})).unwrap(); + let img = s.render(cid, s.time(), opts); + assert!(img.data.iter().filter(|p| p[3] > 0.99).count() > 1000); + // Classic 3D doesn't draw models. + s.execute("comp.renderer", json!({"renderer": "classic3d"})).unwrap(); + assert!(s.render(cid, s.time(), opts).data.iter().all(|p| p[3] == 0.0)); + } + /// The web app's path: outputs go to a sink (downloads), never to the file system. #[test] fn render_queue_exports_to_a_sink() { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 4ec70cfa..d557357b 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -178,6 +178,12 @@ pub trait Accelerator: Send + Sync { /// Run a chain of supported effects on `buf`, clamping and quantising to `levels` after /// every effect when set (8/16 bpc). `None` = not handled. fn effects(&self, chain: &[FxStep], buf: &Buf, levels: Option) -> Option; + /// Rasterise and shade an Advanced 3D scene at its raster size (depth buffer, PBR, image + /// based light, shadow maps), with the CPU rasteriser's semantics + /// ([`three_d::adv::raster::render`]). `None` = not handled (the CPU renders it). + fn raster_3d(&self, _scene: &three_d::adv::Scene) -> Option { + None + } } #[derive(Clone, Copy, Debug)] @@ -1185,7 +1191,7 @@ impl<'a> Renderer<'a> { comp.layers .iter() .rev() - .filter(|l| l.is_active_at(t) && l.has_video() && (!any_solo || l.switches.solo) && (self.opts.guides || !l.switches.guide)) + .filter(|l| l.is_active_at(t) && l.has_video() && !l.environment && (!any_solo || l.switches.solo) && (self.opts.guides || !l.switches.guide)) .collect() } diff --git a/crates/render/src/shapes.rs b/crates/render/src/shapes.rs index 25eb6090..7cb59135 100644 --- a/crates/render/src/shapes.rs +++ b/crates/render/src/shapes.rs @@ -508,6 +508,27 @@ pub fn render(ctx: &EvalCtx, layer: &Layer, contents: &PropGroup, s: f64) -> Buf #[cfg(test)] mod tests; +/// The filled outlines of a shape layer in layer space with their straight colours (gradient +/// fills: the colour half way along), for Advanced 3D extrusion. Strokes are not extruded. +pub fn fill_outlines(ctx: &EvalCtx, layer: &Layer, contents: &PropGroup) -> Vec<(Vec, [f32; 4])> { + let mut arena = Vec::new(); + let (draws, _) = collect(ctx, layer, contents, &mut arena); + draws + .iter() + .rev() + .filter(|d| d.opacity > 0.0) + .filter_map(|d| { + let c = match &d.paint { + Paint::Fill { color, .. } => *color, + Paint::Gradient { g, stroke: None, .. } => g.sample(0.5), + _ => return None, + }; + let paths: Vec = d.slots.iter().map(|&i| arena[i].clone()).filter(|p| !p.elements().is_empty()).collect(); + (!paths.is_empty()).then_some((paths, [c[0], c[1], c[2], c[3] * d.opacity])) + }) + .collect() +} + /// Layer-space bounds of a shape layer's painted contents (strokes included), without /// rasterising. pub fn content_bounds(ctx: &EvalCtx, layer: &Layer, contents: &PropGroup) -> Option { diff --git a/crates/render/src/three_d/adv/mod.rs b/crates/render/src/three_d/adv/mod.rs new file mode 100644 index 00000000..04f23885 --- /dev/null +++ b/crates/render/src/three_d/adv/mod.rs @@ -0,0 +1,165 @@ +//! The Advanced 3D renderer (Composition Settings ▸ 3D Renderer ▸ Advanced 3D). +//! +//! A run of 3D layers becomes one [`Scene`] of world-space triangles (models, primitives, +//! extruded text and shapes, and textured cards for every other layer), lit physically +//! ([`shade`]) by the comp's lights, an Environment light's image and shadow maps. The scene +//! is rasterised with a depth buffer on the GPU when an [`crate::Accelerator`] implements +//! [`crate::Accelerator::raster_3d`], otherwise by the software rasteriser ([`raster`]), at 2×2 +//! supersampling. The resolve (box filter), depth of field (a gather blur by each pixel's +//! circle of confusion) and the conversion back to the working encoding run on the CPU for +//! both paths, and the result is composited over the layers below the run. + +pub mod raster; +pub mod scene; +pub mod shade; + +use effectcraft_project::Layer; +use rayon::prelude::*; + +pub use raster::Target; +pub use scene::{EnvInfo, Light, Material, Scene, ShadowMap, TexInfo, Vertex}; + +use super::camera::Dof; +use super::compose::camera_for; +use crate::{EvalCtx, Image, Renderer}; + +/// Box-filter a supersampled target down by `k`: (premultiplied colour, mean depth of the +/// covered subsamples, ∞ where none). +pub fn resolve(t: &Target, k: u32) -> (Image, Vec) { + let (w, h) = (t.width / k, t.height / k); + let mut img = Image::new(w, h); + let mut depth = vec![f32::INFINITY; (w * h) as usize]; + let n = (k * k) as f32; + img.data.par_chunks_mut(w as usize).zip(depth.par_chunks_mut(w as usize)).enumerate().for_each(|(y, (row, drow))| { + for x in 0..w as usize { + let mut acc = [0.0f32; 4]; + let (mut dz, mut dn) = (0.0f32, 0.0f32); + for j in 0..k as usize { + for i in 0..k as usize { + let s = (y * k as usize + j) * t.width as usize + x * k as usize + i; + let c = t.color[s]; + for q in 0..4 { + acc[q] += c[q]; + } + if t.depth[s].is_finite() { + dz += t.depth[s]; + dn += 1.0; + } + } + } + row[x] = acc.map(|v| v / n); + if dn > 0.0 { + drow[x] = dz / dn; + } + } + }); + (img, depth) +} + +/// Depth of field: a gather blur where each pixel collects the neighbours whose circle of +/// confusion (from their depth, in output pixels) reaches it; nearer in-focus surfaces are not +/// smeared by the blur of what is behind them. +pub fn depth_of_field(img: &Image, depth: &[f32], dof: &Dof, scale: f64) -> Image { + let (w, h) = (img.width as i64, img.height as i64); + let max_r = 48.0f32; + let radius: Vec = depth + .iter() + .map(|&z| { + let z = if z.is_finite() { z as f64 } else { 1.0e9 }; + ((dof.coc(z) * scale * 0.5) as f32).min(max_r) + }) + .collect(); + let rmax = radius.iter().fold(0.0f32, |a, &b| a.max(b)); + if rmax < 0.5 { + return img.clone(); + } + // Disk samples (golden-angle spiral) over the largest radius. + const N: usize = 64; + let golden = std::f32::consts::PI * (3.0 - 5.0f32.sqrt()); + let taps: Vec<(f32, f32, f32)> = (1..=N) + .map(|i| { + let r = (i as f32 / N as f32).sqrt() * rmax; + let a = i as f32 * golden; + (r * a.cos(), r * a.sin(), r) + }) + .collect(); + let mut out = Image::new(img.width, img.height); + out.data.par_chunks_mut(w as usize).enumerate().for_each(|(y, row)| { + for x in 0..w as usize { + let i = y * w as usize + x; + let (rp, zp) = (radius[i], depth[i]); + let cw = 1.0 / (std::f32::consts::PI * rp * rp).max(1.0); + let c = img.data[i]; + let mut acc = [c[0] * cw, c[1] * cw, c[2] * cw, c[3] * cw]; + let mut wsum = cw; + for &(dx, dy, d) in &taps { + let (sx, sy) = (x as i64 + dx.round() as i64, y as i64 + dy.round() as i64); + if sx < 0 || sy < 0 || sx >= w || sy >= h { + continue; + } + let j = (sy * w + sx) as usize; + // A sample behind this pixel blurs no further than this pixel's own circle. + let rq = if depth[j] > zp { radius[j].min(rp) } else { radius[j] }; + let cov = (rq - d + 0.5).clamp(0.0, 1.0); + if cov <= 0.0 { + continue; + } + // Each tap stands for an annulus of area ≈ π·rmax²/N. + let wq = cov / (std::f32::consts::PI * rq * rq).max(1.0) * (std::f32::consts::PI * rmax * rmax / N as f32); + let q = img.data[j]; + for k in 0..4 { + acc[k] += q[k] * wq; + } + wsum += wq; + } + row[x] = acc.map(|v| v / wsum); + } + }); + out +} + +/// Linear → sRGB-encoded premultiplied pixels (the working encoding of non-linear projects). +pub fn encode(img: &mut Image) { + img.data.par_iter_mut().for_each(|p| { + let a = p[3]; + if a > 1e-6 { + for c in 0..3 { + p[c] = shade::linear_to_srgb(p[c] / a) * a; + } + } + }); +} + +/// Render a scene on the CPU, resolved: (premultiplied image at output size, depth). +pub fn render_cpu(s: &Scene) -> (Image, Vec) { + resolve(&raster::render(s), s.ssaa.max(1)) +} + +/// Build the scene of a run (for tests and tools). +pub fn scene_of(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> Scene { + scene::build(r, ctx, run, out) +} + +/// Draw a run of 3D layers with the Advanced 3D renderer into `canvas`. +pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut Image) { + let s = scene::build(r, ctx, run, (canvas.width, canvas.height)); + if s.indices.is_empty() { + return; + } + let target = r.active_accel().and_then(|a| a.raster_3d(&s)).filter(|t| t.width == s.width && t.height == s.height).unwrap_or_else(|| raster::render(&s)); + let (mut img, depth) = resolve(&target, s.ssaa.max(1)); + let cam = camera_for(r, ctx); + if let Some(dof) = cam.dof.filter(|_| !ctx.comp.draft_3d && !r.opts.draft) { + img = depth_of_field(&img, &depth, &dof, r.opts.scale); + } + if !s.linear_io { + encode(&mut img); + } + canvas.data.par_iter_mut().zip(img.data.par_iter()).for_each(|(d, s)| { + let k = 1.0 - s[3]; + *d = [s[0] + d[0] * k, s[1] + d[1] * k, s[2] + d[2] * k, s[3] + d[3] * k]; + }); +} + +#[cfg(test)] +mod tests; diff --git a/crates/render/src/three_d/adv/raster.rs b/crates/render/src/three_d/adv/raster.rs new file mode 100644 index 00000000..36cec108 --- /dev/null +++ b/crates/render/src/three_d/adv/raster.rs @@ -0,0 +1,492 @@ +//! The software rasteriser of the Advanced 3D renderer (the CPU reference and the fallback +//! when no GPU is available) and the shadow-map rasteriser shared by both paths. +//! +//! Triangles are clipped against the depth range (`0 ≤ z ≤ w`, as GPUs do), set up in screen +//! space and scanned per band of rows in parallel with edge functions (pixel centres, top-left +//! fill rule) and perspective-correct barycentrics. Opaque triangles resolve visibility first +//! (reversed depth: larger ndc z is nearer) and each pixel is shaded once; transparent triangles +//! (already sorted back to front) are then shaded per fragment and blended over, depth-tested +//! against the opaque surfaces. + +use rayon::prelude::*; + +use super::scene::{Scene, ShadowMap, Vertex}; +use super::shade::{Frag, sample, shade}; + +/// A rendered target at raster size. +pub struct Target { + pub width: u32, + pub height: u32, + /// Premultiplied linear RGBA. + pub color: Vec<[f32; 4]>, + /// Camera-space depth of the nearest opaque surface (∞ where empty). + pub depth: Vec, +} + +/// A clipped triangle in screen space; `w` holds, per corner, its weights over the original +/// triangle's three vertices. +#[derive(Clone, Copy)] +struct STri { + p: [[f64; 2]; 3], + z: [f32; 3], + inv_w: [f32; 3], + w: [[f32; 3]; 3], + tri: u32, + front: bool, + bbox: [i32; 4], +} + +#[inline] +fn mat_mul(m: &[[f32; 4]; 4], p: [f32; 3]) -> [f32; 4] { + let mut o = [0.0; 4]; + for r in 0..4 { + o[r] = m[r][0] * p[0] + m[r][1] * p[1] + m[r][2] * p[2] + m[r][3]; + } + o +} + +/// Clip a polygon (clip coords + original weights) against `0 ≤ z ≤ w`. +fn clip_poly(poly: Vec<([f32; 4], [f32; 3])>) -> Vec<([f32; 4], [f32; 3])> { + let mut p = poly; + for plane in 0..2 { + if p.is_empty() { + break; + } + let d = |c: &[f32; 4]| if plane == 0 { c[2] } else { c[3] - c[2] }; + let mut out = Vec::with_capacity(p.len() + 2); + for i in 0..p.len() { + let (a, b) = (p[i], p[(i + 1) % p.len()]); + let (da, db) = (d(&a.0), d(&b.0)); + if da >= 0.0 { + out.push(a); + } + if (da >= 0.0) != (db >= 0.0) { + let t = da / (da - db); + let mut c = [0.0; 4]; + let mut w = [0.0; 3]; + for k in 0..4 { + c[k] = a.0[k] + (b.0[k] - a.0[k]) * t; + } + for k in 0..3 { + w[k] = a.1[k] + (b.1[k] - a.1[k]) * t; + } + out.push((c, w)); + } + } + p = out; + } + p +} + +/// Set up the screen triangles of index range `range`. +fn setup(s: &Scene, clip: &[[f32; 4]], range: std::ops::Range) -> Vec { + let (w, h) = (s.width as f64, s.height as f64); + let mut out = vec![]; + for t in range.step_by(3) { + let ids = [s.indices[t] as usize, s.indices[t + 1] as usize, s.indices[t + 2] as usize]; + let mat = s.vertices[ids[0]].material as usize; + if !s.materials.get(mat).is_some_and(|m| m.visible) { + continue; + } + let poly = vec![(clip[ids[0]], [1.0, 0.0, 0.0]), (clip[ids[1]], [0.0, 1.0, 0.0]), (clip[ids[2]], [0.0, 0.0, 1.0])]; + let poly = clip_poly(poly); + if poly.len() < 3 { + continue; + } + let scr: Vec<([f64; 2], f32, f32, [f32; 3])> = poly + .iter() + .map(|(c, wt)| { + let iw = 1.0 / c[3]; + let (nx, ny) = (c[0] * iw, c[1] * iw); + ([(nx as f64 + 1.0) * 0.5 * w, (1.0 - ny as f64) * 0.5 * h], c[2] * iw, iw, *wt) + }) + .collect(); + // Facing from the polygon's screen-space winding (y down: negative = front). + let mut area = 0.0; + for i in 0..scr.len() { + let (a, b) = (scr[i].0, scr[(i + 1) % scr.len()].0); + area += a[0] * b[1] - b[0] * a[1]; + } + if area == 0.0 || !area.is_finite() { + continue; + } + let front = area < 0.0; + for k in 1..scr.len() - 1 { + let c = [scr[0], scr[k], scr[k + 1]]; + let mut x0 = f64::INFINITY; + let mut y0 = f64::INFINITY; + let mut x1 = f64::NEG_INFINITY; + let mut y1 = f64::NEG_INFINITY; + for v in &c { + x0 = x0.min(v.0[0]); + y0 = y0.min(v.0[1]); + x1 = x1.max(v.0[0]); + y1 = y1.max(v.0[1]); + } + let bbox = [ + (x0.floor() as i64).max(0) as i32, + (y0.floor() as i64).max(0) as i32, + (x1.ceil() as i64).min(s.width as i64) as i32, + (y1.ceil() as i64).min(s.height as i64) as i32, + ]; + if bbox[0] >= bbox[2] || bbox[1] >= bbox[3] { + continue; + } + out.push(STri { + p: [c[0].0, c[1].0, c[2].0], + z: [c[0].1, c[1].1, c[2].1], + inv_w: [c[0].2, c[1].2, c[2].2], + w: [c[0].3, c[1].3, c[2].3], + tri: (t / 3) as u32, + front, + bbox, + }); + } + } + out +} + +/// Coverage of a pixel centre: (ndc z, perspective-correct weights over the screen corners). +#[inline] +fn cover(t: &STri, px: f64, py: f64) -> Option<(f32, [f32; 3])> { + let [a, b, c] = t.p; + let area = (b[0] - a[0]) * (c[1] - a[1]) - (b[1] - a[1]) * (c[0] - a[0]); + // Orient so the area is positive. + let (b, c, swap) = if area < 0.0 { (c, b, true) } else { (b, c, false) }; + let area = area.abs(); + let edge = |p: [f64; 2], q: [f64; 2]| -> f64 { (q[0] - p[0]) * (py - p[1]) - (q[1] - p[1]) * (px - p[0]) }; + let top_left = |p: [f64; 2], q: [f64; 2]| -> bool { + let (dx, dy) = (q[0] - p[0], q[1] - p[1]); + dy < 0.0 || (dy == 0.0 && dx > 0.0) + }; + let e0 = edge(b, c); // opposite a + let e1 = edge(c, a); // opposite b + let e2 = edge(a, b); // opposite c + let inside = |e: f64, p: [f64; 2], q: [f64; 2]| e > 0.0 || (e == 0.0 && top_left(p, q)); + if !(inside(e0, b, c) && inside(e1, c, a) && inside(e2, a, b)) { + return None; + } + let (l0, l1, l2) = ((e0 / area) as f32, (e1 / area) as f32, (e2 / area) as f32); + // Back to the stored corner order. + let l = if swap { [l0, l2, l1] } else { [l0, l1, l2] }; + let z = l[0] * t.z[0] + l[1] * t.z[1] + l[2] * t.z[2]; + let q = [l[0] * t.inv_w[0], l[1] * t.inv_w[1], l[2] * t.inv_w[2]]; + let sum = q[0] + q[1] + q[2]; + if sum <= 0.0 { + return None; + } + Some((z, [q[0] / sum, q[1] / sum, q[2] / sum])) +} + +/// Interpolate the original triangle's vertex attributes at screen-corner weights `l`. +fn frag(s: &Scene, t: &STri, l: [f32; 3]) -> Frag { + let base = t.tri as usize * 3; + let v: [&Vertex; 3] = [0, 1, 2].map(|k| &s.vertices[s.indices[base + k] as usize]); + // Weights over the original vertices. + let mut wv = [0.0f32; 3]; + for c in 0..3 { + for k in 0..3 { + wv[k] += l[c] * t.w[c][k]; + } + } + let mut f = Frag { pos: [0.0; 3], normal: [0.0; 3], uv: [0.0; 2], tangent: [0.0; 4], material: v[0].material, front: t.front }; + for k in 0..3 { + let x = wv[k]; + for i in 0..3 { + f.pos[i] += v[k].pos[i] * x; + f.normal[i] += v[k].normal[i] * x; + } + for i in 0..2 { + f.uv[i] += v[k].uv[i] * x; + } + for i in 0..4 { + f.tangent[i] += v[k].tangent[i] * x; + } + } + f.tangent[3] = v[0].tangent[3]; + f +} + +const BAND: i32 = 16; + +/// Rasterise and shade the scene. +pub fn render(s: &Scene) -> Target { + let (w, h) = (s.width as usize, s.height as usize); + let clip: Vec<[f32; 4]> = s.vertices.iter().map(|v| mat_mul(&s.clip, v.pos)).collect(); + let opaque = setup(s, &clip, 0..s.opaque_count as usize); + let trans = setup(s, &clip, s.opaque_count as usize..s.indices.len()); + let bands = (h as i32 + BAND - 1) / BAND; + let bin = |ts: &[STri]| -> Vec> { + let mut b = vec![vec![]; bands.max(0) as usize]; + for (i, t) in ts.iter().enumerate() { + for band in (t.bbox[1] / BAND)..((t.bbox[3] + BAND - 1) / BAND).min(bands) { + b[band as usize].push(i as u32); + } + } + b + }; + let ob = bin(&opaque); + let tb = bin(&trans); + let view = s.view; + let cam_z = |p: [f32; 3]| view[2][0] * p[0] + view[2][1] * p[1] + view[2][2] * p[2] + view[2][3]; + let mut color = vec![[0.0f32; 4]; w * h]; + let mut depth = vec![f32::INFINITY; w * h]; + color.par_chunks_mut(w * BAND as usize).zip(depth.par_chunks_mut(w * BAND as usize)).enumerate().for_each(|(band, (crow, drow))| { + let y0 = band as i32 * BAND; + let rows = crow.len() / w; + // Visibility: (screen tri, ndc z, weights). + let mut vis: Vec<(u32, f32, [f32; 3])> = vec![(u32::MAX, 0.0, [0.0; 3]); rows * w]; + for &ti in &ob[band] { + let t = &opaque[ti as usize]; + for y in t.bbox[1].max(y0)..t.bbox[3].min(y0 + rows as i32) { + let r = (y - y0) as usize * w; + for x in t.bbox[0]..t.bbox[2] { + if let Some((z, l)) = cover(t, x as f64 + 0.5, y as f64 + 0.5) { + let v = &mut vis[r + x as usize]; + if z > v.1 && z <= 1.0 { + *v = (ti, z, l); + } + } + } + } + } + // A discarded opaque fragment (alpha test, back face) lets the next nearer-than-nothing + // surface through only on a second look; keep it simple: resolve again without it. + for i in 0..rows * w { + let (ti, _, l) = vis[i]; + if ti == u32::MAX { + continue; + } + let t = &opaque[ti as usize]; + let f = frag(s, t, l); + match shade(s, &f) { + Some(c) => { + crow[i] = c; + drow[i] = cam_z(f.pos); + } + None => { + // Find the nearest surviving fragment under this pixel. + let (x, y) = ((i % w) as f64 + 0.5, (y0 as usize + i / w) as f64 + 0.5); + let mut best: Option<(f32, [f32; 4], f32)> = None; + for &tj in &ob[band] { + if tj == ti { + continue; + } + let t2 = &opaque[tj as usize]; + if (x as i32) < t2.bbox[0] || (x as i32) >= t2.bbox[2] || (y as i32) < t2.bbox[1] || (y as i32) >= t2.bbox[3] { + continue; + } + if let Some((z, l2)) = cover(t2, x, y) + && best.is_none_or(|b| z > b.0) + { + let f2 = frag(s, t2, l2); + if let Some(c) = shade(s, &f2) { + best = Some((z, c, cam_z(f2.pos))); + } + } + } + if let Some((z, c, d)) = best { + crow[i] = c; + drow[i] = d; + vis[i].1 = z; + } else { + vis[i].1 = 0.0; + } + } + } + } + // Transparent fragments in order, over the opaque result. + for &ti in &tb[band] { + let t = &trans[ti as usize]; + for y in t.bbox[1].max(y0)..t.bbox[3].min(y0 + rows as i32) { + let r = (y - y0) as usize * w; + for x in t.bbox[0]..t.bbox[2] { + let i = r + x as usize; + if let Some((z, l)) = cover(t, x as f64 + 0.5, y as f64 + 0.5) + && z > vis[i].1 + && z <= 1.0 + && let Some(c) = shade(s, &frag(s, t, l)) + { + let d = crow[i]; + let k = 1.0 - c[3]; + crow[i] = [c[0] + d[0] * k, c[1] + d[1] * k, c[2] + d[2] * k, c[3] + d[3] * k]; + } + } + } + } + }); + Target { width: s.width, height: s.height, color, depth } +} + +// ---------------------------------------------------------------- shadow maps + +fn basis(eye: [f64; 3], fwd: [f64; 3], down: [f64; 3]) -> [[f32; 4]; 3] { + use effectcraft_geom::vec3; + let m = crate::three_d::camera::basis_view(vec3(eye[0], eye[1], eye[2]), vec3(fwd[0], fwd[1], fwd[2]), vec3(down[0], down[1], down[2])); + [0, 1, 2].map(|r| m.0[r].map(|v| v as f32)) +} + +/// Rasterise the shadow-casting triangles' light-space depth into `m`'s texels. +fn shadow_raster(s: &Scene, m: &ShadowMap, casters: &[usize]) -> Vec { + let n = m.size as usize; + let mut d = vec![f32::INFINITY; n * n]; + let c = n as f32 * 0.5; + const NEAR: f32 = 1.0; + for &t in casters { + let ids = [s.indices[3 * t] as usize, s.indices[3 * t + 1] as usize, s.indices[3 * t + 2] as usize]; + let mat = &s.materials[s.vertices[ids[0]].material as usize]; + let alpha_tex = (mat.alpha_mode != 0 && mat.tex_base >= 0).then(|| &s.textures[mat.tex_base as usize]); + // Light space, clipped against the near plane (perspective). + let q: Vec<([f32; 3], [f32; 2])> = ids + .iter() + .map(|&i| { + let p = s.vertices[i].pos; + let v = &m.view; + ([0, 1, 2].map(|r| v[r][0] * p[0] + v[r][1] * p[1] + v[r][2] * p[2] + v[r][3]), s.vertices[i].uv) + }) + .collect(); + let mut poly = q; + if !m.ortho { + let mut out = vec![]; + for i in 0..poly.len() { + let (a, b) = (poly[i], poly[(i + 1) % poly.len()]); + let (da, db) = (a.0[2] - NEAR, b.0[2] - NEAR); + if da >= 0.0 { + out.push(a); + } + if (da >= 0.0) != (db >= 0.0) { + let k = da / (da - db); + let lerp = |x: f32, y: f32| x + (y - x) * k; + out.push(([lerp(a.0[0], b.0[0]), lerp(a.0[1], b.0[1]), lerp(a.0[2], b.0[2])], [lerp(a.1[0], b.1[0]), lerp(a.1[1], b.1[1])])); + } + } + poly = out; + } + if poly.len() < 3 { + continue; + } + // Screen: (x, y, 1/z or 1, depth) per corner. + let scr: Vec<([f64; 2], f32, f32, [f32; 2])> = poly + .iter() + .map(|(p, uv)| { + if m.ortho { + ([(p[0] * m.focal + c) as f64, (p[1] * m.focal + c) as f64], 1.0, p[2], *uv) + } else { + let iz = 1.0 / p[2]; + ([(p[0] * m.focal * iz + c) as f64, (p[1] * m.focal * iz + c) as f64], iz, p[2], *uv) + } + }) + .collect(); + for k in 1..scr.len() - 1 { + let tri = [scr[0], scr[k], scr[k + 1]]; + let (a, b, cc) = (tri[0].0, tri[1].0, tri[2].0); + let area = (b[0] - a[0]) * (cc[1] - a[1]) - (b[1] - a[1]) * (cc[0] - a[0]); + if area.abs() < 1e-12 { + continue; + } + let x0 = a[0].min(b[0]).min(cc[0]).floor().max(0.0) as usize; + let x1 = (a[0].max(b[0]).max(cc[0]).ceil().max(0.0) as usize).min(n); + let y0 = a[1].min(b[1]).min(cc[1]).floor().max(0.0) as usize; + let y1 = (a[1].max(b[1]).max(cc[1]).ceil().max(0.0) as usize).min(n); + for y in y0..y1 { + for x in x0..x1 { + let (px, py) = (x as f64 + 0.5, y as f64 + 0.5); + let e0 = ((cc[0] - b[0]) * (py - b[1]) - (cc[1] - b[1]) * (px - b[0])) / area; + let e1 = ((a[0] - cc[0]) * (py - cc[1]) - (a[1] - cc[1]) * (px - cc[0])) / area; + let e2 = 1.0 - e0 - e1; + if e0 < 0.0 || e1 < 0.0 || e2 < 0.0 { + continue; + } + let l = [e0 as f32, e1 as f32, e2 as f32]; + let (z, uv) = if m.ortho { + (l[0] * tri[0].2 + l[1] * tri[1].2 + l[2] * tri[2].2, [0, 1].map(|i| l[0] * tri[0].3[i] + l[1] * tri[1].3[i] + l[2] * tri[2].3[i])) + } else { + let iz = l[0] * tri[0].1 + l[1] * tri[1].1 + l[2] * tri[2].1; + let z = 1.0 / iz; + (z, [0, 1].map(|i| (l[0] * tri[0].3[i] * tri[0].1 + l[1] * tri[1].3[i] * tri[1].1 + l[2] * tri[2].3[i] * tri[2].1) * z)) + }; + let px = &mut d[y * n + x]; + if z >= *px { + continue; + } + if let Some(t) = alpha_tex + && sample(t, &s.texels, uv[0], uv[1])[3] * mat.base[3] * mat.opacity < 0.5 + { + continue; + } + *px = z; + } + } + } + } + d +} + +/// Build the shadow maps of every shadow-casting light (lights marked `shadow == 0`). +pub(crate) fn shadow_maps(s: &mut Scene, draft: bool) { + let casters: Vec = (0..s.indices.len() / 3) + .filter(|&t| s.materials.get(s.vertices[s.indices[3 * t] as usize].material as usize).is_some_and(|m| m.casts_shadows)) + .collect(); + if casters.is_empty() { + for l in &mut s.lights { + l.shadow = -1; + } + return; + } + // Bounding sphere of the scene (parallel lights' orthographic frustum). + let (mut lo, mut hi) = ([f64::INFINITY; 3], [f64::NEG_INFINITY; 3]); + for v in &s.vertices { + for k in 0..3 { + lo[k] = lo[k].min(v.pos[k] as f64); + hi[k] = hi[k].max(v.pos[k] as f64); + } + } + let centre = [0, 1, 2].map(|k| (lo[k] + hi[k]) * 0.5); + let radius = ((hi[0] - lo[0]).powi(2) + (hi[1] - lo[1]).powi(2) + (hi[2] - lo[2]).powi(2)).sqrt() * 0.5 + 1.0; + let size = if draft { 512 } else { 1024 }; + let mut maps: Vec = vec![]; + for li in 0..s.lights.len() { + if s.lights[li].shadow != 0 { + s.lights[li].shadow = -1; + continue; + } + let l = s.lights[li]; + let first = maps.len() as i32; + let soft = 1.0 + (l.shadow_diffusion * 0.5).min(24.0); + let pos = l.pos.map(|v| v as f64); + let dir = l.dir.map(|v| v as f64); + let hint = if dir[1].abs() < 0.9 { [0.0, 1.0, 0.0] } else { [0.0, 0.0, 1.0] }; + match l.kind { + 0 => { + let eye = [centre[0] - dir[0] * radius * 2.0, centre[1] - dir[1] * radius * 2.0, centre[2] - dir[2] * radius * 2.0]; + let focal = (size as f64 * 0.5 / radius) as f32; + maps.push(ShadowMap { view: basis(eye, dir, hint), focal, ortho: true, size, offset: 0, bias: 1.5 / focal, radius: soft }); + } + 1 => { + let half = (l.cos_outer.clamp(-1.0, 1.0).acos() as f64 + 0.05).min(1.45); + let focal = (size as f64 * 0.5 / half.tan()) as f32; + maps.push(ShadowMap { view: basis(pos, dir, hint), focal, ortho: false, size, offset: 0, bias: 1.0, radius: soft }); + } + _ => { + let fs = size / 2; + for (f, d) in [ + ([1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([0.0, 1.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, -1.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 1.0], [0.0, 1.0, 0.0]), + ([0.0, 0.0, -1.0], [0.0, 1.0, 0.0]), + ] { + maps.push(ShadowMap { view: basis(pos, f, d), focal: fs as f32 * 0.5, ortho: false, size: fs, offset: 0, bias: 1.0, radius: soft }); + } + } + } + s.lights[li].shadow = first; + } + let texels: Vec> = maps.par_iter().map(|m| shadow_raster(s, m, &casters)).collect(); + for (m, t) in maps.iter_mut().zip(texels) { + m.offset = s.shadow_texels.len() as u32; + s.shadow_texels.extend(t); + } + s.shadows = maps; +} diff --git a/crates/render/src/three_d/adv/scene.rs b/crates/render/src/three_d/adv/scene.rs new file mode 100644 index 00000000..1d3acc3c --- /dev/null +++ b/crates/render/src/three_d/adv/scene.rs @@ -0,0 +1,872 @@ +//! The Advanced 3D scene: everything a rasteriser needs, flattened into world-space triangles +//! and plain arrays (identical input for the CPU rasteriser and the GPU pipeline). +//! +//! - Model and primitive layers: their meshes (glTF/OBJ models are +Y up in model units and +//! are mapped into the layer by `Model Scale` with Y and Z flipped, a 180° turn about X). +//! - Extruded text and shape layers (Geometry Options ▸ Extrusion Depth > 0): bevelled meshes. +//! - Every other 3D layer (footage, solids, comps, flat text and shapes): a textured card of +//! its processed buffer, lit with its Material Options. +//! - Lights from the comp's light layers, the Environment light's image (prefiltered for image +//! based lighting), and shadow maps of the shadow-casting lights. +//! +//! Colours are linear light; textures and layer buffers are linearised from the sRGB curve +//! unless the project already blends in linear light. + +use std::collections::HashMap; +use std::sync::Arc; + +use effectcraft_effects::Buf; +use effectcraft_geom::{Mat4, Vec3, vec3}; +use effectcraft_model::{self as model, Primitive}; +use effectcraft_project::{ItemKind, Layer, LayerSource, LightKind, PrimitiveKind}; + +use super::shade::srgb_to_linear; +use crate::three_d::camera::{CameraState, NEAR}; +use crate::three_d::compose::camera_for; +use crate::three_d::light::{LightState, Material as ClassicMaterial}; +use crate::{EvalCtx, Renderer}; + +/// One vertex in world space. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +#[repr(C)] +pub struct Vertex { + pub pos: [f32; 3], + pub normal: [f32; 3], + pub uv: [f32; 2], + pub tangent: [f32; 4], + pub material: u32, +} + +/// A material, resolved for shading. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Material { + /// Linear RGBA factor. + pub base: [f32; 4], + pub metallic: f32, + pub roughness: f32, + pub emissive: [f32; 3], + pub normal_scale: f32, + pub occlusion_strength: f32, + /// 0 opaque, 1 mask, 2 blend. + pub alpha_mode: u32, + pub alpha_cutoff: f32, + /// Layer opacity (multiplies alpha). + pub opacity: f32, + /// Texture indices (−1 = none). + pub tex_base: i32, + pub tex_mr: i32, + pub tex_normal: i32, + pub tex_occlusion: i32, + pub tex_emissive: i32, + pub double_sided: bool, + pub unlit: bool, + pub accepts_lights: bool, + pub receives_shadows: bool, + pub casts_shadows: bool, + /// Drawn (false for "Casts Shadows: Only"). + pub visible: bool, + /// Material Options weights (cards: Diffuse, Specular Intensity / 50%, Ambient). + pub diffuse_k: f32, + pub specular_k: f32, + pub ambient_k: f32, +} + +impl Default for Material { + fn default() -> Self { + Material { + base: [1.0; 4], + metallic: 0.0, + roughness: 0.5, + emissive: [0.0; 3], + normal_scale: 1.0, + occlusion_strength: 1.0, + alpha_mode: 0, + alpha_cutoff: 0.5, + opacity: 1.0, + tex_base: -1, + tex_mr: -1, + tex_normal: -1, + tex_occlusion: -1, + tex_emissive: -1, + double_sided: false, + unlit: false, + accepts_lights: true, + receives_shadows: true, + casts_shadows: false, + visible: true, + diffuse_k: 1.0, + specular_k: 1.0, + ambient_k: 1.0, + } + } +} + +impl Material { + /// Drawn in the transparent (sorted, blended) pass. + pub fn transparent(&self) -> bool { + self.alpha_mode == 2 || self.opacity < 0.999 + } +} + +/// A texture in [`Scene::texels`]: linear straight RGBA. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct TexInfo { + pub offset: u32, + pub width: u32, + pub height: u32, + /// 0 repeat, 1 clamp, 2 mirror. + pub wrap_u: u32, + pub wrap_v: u32, +} + +/// A light (ambient lights are summed into [`Scene::ambient`]). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Light { + /// 0 parallel, 1 spot, 2 point. + pub kind: u32, + pub pos: [f32; 3], + /// Direction the light travels. + pub dir: [f32; 3], + /// Linear colour × intensity. + pub color: [f32; 3], + pub cos_inner: f32, + pub cos_outer: f32, + /// 0 none, 1 smooth, 2 inverse square clamped. + pub falloff: u32, + pub radius: f32, + pub falloff_distance: f32, + /// First shadow map (point lights use six), −1 = none. + pub shadow: i32, + pub shadow_darkness: f32, + /// Shadow Diffusion (pixels). + pub shadow_diffusion: f32, +} + +/// A depth map seen from a light: `view` maps world → light space (+z forward); pixels +/// `u = focal·x/z + size/2` (perspective) or `focal·x + size/2` (orthographic); texels store +/// the light-space depth of the nearest caster. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ShadowMap { + pub view: [[f32; 4]; 3], + pub focal: f32, + pub ortho: bool, + pub size: u32, + pub offset: u32, + /// Depth bias (world units). + pub bias: f32, + /// PCF spread (texels). + pub radius: f32, +} + +/// The environment image: radiance mip levels `radiance..radiance + mips` and an irradiance +/// map, in [`Scene::textures`]. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EnvInfo { + pub radiance: u32, + pub mips: u32, + pub irradiance: u32, + pub intensity: f32, + /// Radians about the vertical axis. + pub rotation: f32, +} + +#[derive(Clone, Debug, Default)] +pub struct Scene { + /// Raster size (output × `ssaa`). + pub width: u32, + pub height: u32, + pub ssaa: u32, + /// World → clip (row-major; reversed depth: ndc z = near / camera z). + pub clip: [[f32; 4]; 4], + /// World → camera (row-major; camera z = depth for depth of field). + pub view: [[f32; 4]; 4], + pub eye: [f32; 3], + pub cam_fwd: [f32; 3], + pub ortho: bool, + pub vertices: Vec, + /// Triangles: the first `opaque_count` indices are opaque, the rest transparent, sorted + /// back to front. + pub indices: Vec, + pub opaque_count: u32, + pub materials: Vec, + pub textures: Vec, + pub texels: Vec<[f32; 4]>, + pub lights: Vec, + pub shadows: Vec, + pub shadow_texels: Vec, + pub env: Option, + pub ambient: [f32; 3], + /// Shading inputs and outputs are linear (no sRGB conversion needed). + pub linear_io: bool, +} + +/// Row-major f32 copy of a matrix. +pub fn m32(m: &Mat4) -> [[f32; 4]; 4] { + m.0.map(|r| r.map(|v| v as f32)) +} + +/// Builds a [`Scene`]: texture uploads deduplicated by source. +pub(crate) struct Builder { + pub s: Scene, + tex_keys: HashMap<(usize, bool), i32>, + /// Per triangle: transparent and its sort depth. + tri_depth: Vec<(bool, f32)>, + cam: CameraState, +} + +impl Builder { + fn texture_from(&mut self, w: u32, h: u32, wrap: [u32; 2], data: impl Iterator) -> i32 { + let offset = self.s.texels.len() as u32; + self.s.texels.extend(data); + self.s.textures.push(TexInfo { offset, width: w, height: h, wrap_u: wrap[0], wrap_v: wrap[1] }); + (self.s.textures.len() - 1) as i32 + } + + /// Upload a model texture (`srgb`: colour data to linearise). + fn model_texture(&mut self, t: &Arc, srgb: bool) -> i32 { + let key = (Arc::as_ptr(t) as usize, srgb); + if let Some(&i) = self.tex_keys.get(&key) { + return i; + } + let lin = srgb && !self.s.linear_io; + let wrap = t.wrap.map(|w| match w { + model::Wrap::Repeat => 0, + model::Wrap::Clamp => 1, + model::Wrap::Mirror => 2, + }); + let i = self.texture_from( + t.width, + t.height, + wrap, + t.data.iter().map(|p| if lin { [srgb_to_linear(p[0]), srgb_to_linear(p[1]), srgb_to_linear(p[2]), p[3]] } else { *p }), + ); + self.tex_keys.insert(key, i); + i + } + + /// Upload a premultiplied layer buffer as a linear straight-alpha texture (clamped edges). + fn buf_texture(&mut self, b: &Buf) -> (i32, bool) { + let lin = !self.s.linear_io; + let mut opaque = true; + let data: Vec<[f32; 4]> = b + .img + .data + .iter() + .map(|p| { + let a = p[3]; + if a < 0.999 { + opaque = false; + } + if a <= 1e-6 { + return [0.0; 4]; + } + let s = [p[0] / a, p[1] / a, p[2] / a]; + if lin { [srgb_to_linear(s[0]), srgb_to_linear(s[1]), srgb_to_linear(s[2]), a] } else { [s[0], s[1], s[2], a] } + }) + .collect(); + (self.texture_from(b.img.width, b.img.height, [1, 1], data.into_iter()), opaque) + } + + fn material(&mut self, m: Material) -> u32 { + self.s.materials.push(m); + (self.s.materials.len() - 1) as u32 + } + + /// Append a primitive with per-vertex matrices (model/layer space → world). + fn emit(&mut self, p: &Primitive, mat: u32, xf: &dyn Fn(usize) -> Mat4) { + if p.positions.is_empty() || p.indices.len() < 3 { + return; + } + let base = self.s.vertices.len() as u32; + let transparent = self.s.materials[mat as usize].transparent(); + let uniform = xf(usize::MAX); + let normal_of = |m: &Mat4| -> (Mat4, f64) { + let a = &m.0; + let det = a[0][0] * (a[1][1] * a[2][2] - a[1][2] * a[2][1]) - a[0][1] * (a[1][0] * a[2][2] - a[1][2] * a[2][0]) + + a[0][2] * (a[1][0] * a[2][1] - a[1][1] * a[2][0]); + let inv = Mat4([[a[0][0], a[0][1], a[0][2], 0.0], [a[1][0], a[1][1], a[1][2], 0.0], [a[2][0], a[2][1], a[2][2], 0.0], [0.0, 0.0, 0.0, 1.0]]) + .inverse() + .unwrap_or(Mat4::IDENTITY); + (inv.transpose(), det) + }; + let (un, udet) = normal_of(&uniform); + let mut flip_count = 0usize; + for k in 0..p.positions.len() { + let (m, n, det) = if p.joints.is_empty() { + (uniform, un, udet) + } else { + let m = xf(k); + let (n, d) = normal_of(&m); + (m, n, d) + }; + if det < 0.0 { + flip_count += 1; + } + let q = p.positions[k]; + let w = m.apply(vec3(q[0] as f64, q[1] as f64, q[2] as f64)); + let nn = p.normals.get(k).copied().unwrap_or([0.0, 0.0, 1.0]); + let wn = n.apply_vec(vec3(nn[0] as f64, nn[1] as f64, nn[2] as f64)).normalize(); + let t = p.tangents.get(k).copied().unwrap_or([1.0, 0.0, 0.0, 1.0]); + let wt = m.apply_vec(vec3(t[0] as f64, t[1] as f64, t[2] as f64)).normalize(); + self.s.vertices.push(Vertex { + pos: [w.x as f32, w.y as f32, w.z as f32], + normal: [wn.x as f32, wn.y as f32, wn.z as f32], + uv: p.uvs.get(k).copied().unwrap_or([0.0, 0.0]), + tangent: [wt.x as f32, wt.y as f32, wt.z as f32, if det < 0.0 { -t[3] } else { t[3] }], + material: mat, + }); + } + // Mirroring transforms reverse the winding. + let flip = flip_count * 2 > p.positions.len(); + for t in p.indices.chunks_exact(3) { + let (a, b, c) = (t[0] + base, t[1] + base, t[2] + base); + if flip { + self.s.indices.extend([a, c, b]) + } else { + self.s.indices.extend([a, b, c]) + } + let z = [a, b, c].iter().map(|&i| self.cam.depth(Vec3::from(self.s.vertices[i as usize].pos.map(|v| v as f64)))).sum::() / 3.0; + self.tri_depth.push((transparent, z as f32)); + } + } + + /// Opaque triangles first, then transparent ones back to front. + fn sort_triangles(&mut self) { + let tris: Vec<[u32; 3]> = self.s.indices.chunks_exact(3).map(|t| [t[0], t[1], t[2]]).collect(); + let mut order: Vec = (0..tris.len()).collect(); + order.sort_by(|&a, &b| { + let (ta, za) = self.tri_depth[a]; + let (tb, zb) = self.tri_depth[b]; + ta.cmp(&tb).then_with(|| if ta { zb.total_cmp(&za) } else { std::cmp::Ordering::Equal }) + }); + self.s.opaque_count = 3 * self.tri_depth.iter().filter(|t| !t.0).count() as u32; + self.s.indices = order.into_iter().flat_map(|i| tris[i]).collect(); + } +} + +/// The Material Options of a card (classic properties). +fn card_material(ctx: &EvalCtx, layer: &Layer) -> Material { + let c = ClassicMaterial::of(ctx, layer); + Material { + roughness: (1.0 - c.shininess).clamp(0.05, 1.0), + double_sided: true, + accepts_lights: c.accepts_lights, + receives_shadows: c.accepts_shadows != 0, + casts_shadows: c.casts_shadows != 0, + visible: c.casts_shadows != 2, + diffuse_k: c.diffuse, + specular_k: c.specular * 2.0, + ambient_k: c.ambient, + ..Material::default() + } +} + +/// Shadow/light switches and the primitive material of a model or primitive layer. +fn model_options(ctx: &EvalCtx, layer: &Layer, lin: bool) -> (Material, Option) { + let g = layer.props.sub("materialOptions"); + let e = |m: &str, d: u32| g.and_then(|g| g.get(m)).map_or(d, |p| ctx.value(layer, p).as_enum()); + let b = |m: &str| g.and_then(|g| g.get(m)).is_none_or(|p| ctx.value(layer, p).as_bool()); + let casts = e("castsShadows", 0); + let opts = Material { + accepts_lights: b("acceptsLights"), + receives_shadows: e("acceptsShadows", 1) != 0, + casts_shadows: casts != 0, + visible: casts != 2, + ..Material::default() + }; + let prim = match (layer.source.clone(), g) { + (LayerSource::Primitive { .. }, Some(g)) => { + let c = ctx.color(layer, g, "baseColor"); + let em = ctx.color(layer, g, "emissive"); + let l = |v: f32| if lin { v } else { srgb_to_linear(v) }; + let mut m = model::Material::plain( + [l(c[0]), l(c[1]), l(c[2]), 1.0], + (ctx.f(layer, g, "metallic", 0.0) / 100.0).clamp(0.0, 1.0) as f32, + (ctx.f(layer, g, "roughness", 50.0) / 100.0).clamp(0.0, 1.0) as f32, + ); + m.emissive = [l(em[0]), l(em[1]), l(em[2])]; + Some(m) + } + _ => None, + }; + (opts, prim) +} + +fn prim_kind(k: PrimitiveKind) -> model::prim::PrimitiveKind { + use model::prim::PrimitiveKind as M; + match k { + PrimitiveKind::Cube => M::Cube, + PrimitiveKind::Sphere => M::Sphere, + PrimitiveKind::Plane => M::Plane, + PrimitiveKind::Torus => M::Torus, + PrimitiveKind::Cone => M::Cone, + PrimitiveKind::Cylinder => M::Cylinder, + } +} + +/// The primitive model of a primitive layer at the context time. +pub fn primitive_model(ctx: &EvalCtx, layer: &Layer, kind: PrimitiveKind, mat: model::Material) -> model::Model { + let g = layer.props.sub("geometryOptions"); + let f = |m: &str, d: f64| g.map_or(d, |g| ctx.f(layer, g, m, d)); + let pr = model::prim::PrimParams { + size: [f("width", 300.0) as f32, f("height", 300.0) as f32, f("depth", 300.0) as f32], + radius: f("radius", 150.0) as f32, + tube: f("tubeRadius", 50.0) as f32, + height: f("height", 300.0) as f32, + segments: f("segments", 48.0).round().clamp(3.0, 512.0) as u32, + rings: f("rings", 24.0).round().clamp(2.0, 512.0) as u32, + }; + model::prim::model(prim_kind(kind), &pr, mat) +} + +/// Model space → layer space: model units to pixels, Y up → Y down (a 180° turn about X). +pub fn model_to_layer(unit_scale: f64) -> Mat4 { + Mat4::scale(vec3(unit_scale, -unit_scale, -unit_scale)) +} + +fn model_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, b: &mut Builder) { + let lin = b.s.linear_io; + let (opts, prim_mat) = model_options(ctx, layer, lin); + let opacity = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); + let geo = layer.props.sub("geometryOptions"); + let (mdl, unit, clip, t): (Arc, f64, Option, f64) = match &layer.source { + LayerSource::Primitive { kind } => (Arc::new(primitive_model(ctx, layer, *kind, prim_mat.unwrap_or_default())), 1.0, None, 0.0), + LayerSource::Model { item } => { + let Some(it) = r.project.item(*item) else { return }; + let ItemKind::Footage(f) = &it.kind else { return }; + let Some(m) = r.footage.model(*item, f) else { return }; + let unit = geo.map_or(1.0, |g| ctx.f(layer, g, "unitScale", 1.0)); + let clip = geo.map_or(0, |g| ctx.e(layer, g, "animation")) as usize; + let clip = clip.checked_sub(1).filter(|c| *c < m.animations.len()); + let speed = geo.map_or(100.0, |g| ctx.f(layer, g, "animationSpeed", 100.0)) / 100.0; + let mut t = layer.layer_time(ctx.time).seconds() * speed; + if let Some(c) = clip { + let d = m.animations[c].duration(); + let looping = geo.is_none_or(|g| ctx.b(layer, g, "loopAnimation")); + if looping && d > 0.0 { + t = t.rem_euclid(d); + } + } + (m, unit, clip, t) + } + _ => return, + }; + let world = ctx.world_matrix(layer) * model_to_layer(unit); + // Materials of the model (one default when it has none). + let mut mats: Vec = vec![]; + let defaults = [model::Material::plain([0.8, 0.8, 0.8, 1.0], 0.0, 0.5)]; + let src: &[model::Material] = if mdl.materials.is_empty() { &defaults } else { &mdl.materials }; + for mm in src { + let tex = |b: &mut Builder, t: Option, srgb: bool| -> i32 { + t.and_then(|t| mdl.textures.get(t.texture)).map_or(-1, |t| b.model_texture(t, srgb)) + }; + let m = Material { + base: mm.base_color, + metallic: mm.metallic, + roughness: mm.roughness, + emissive: mm.emissive, + normal_scale: mm.normal_scale, + occlusion_strength: mm.occlusion_strength, + alpha_mode: match mm.alpha_mode { + model::AlphaMode::Opaque => 0, + model::AlphaMode::Mask => 1, + model::AlphaMode::Blend => 2, + }, + alpha_cutoff: mm.alpha_cutoff, + opacity, + tex_base: tex(b, mm.base_color_tex, true), + tex_mr: tex(b, mm.metallic_roughness_tex, false), + tex_normal: tex(b, mm.normal_tex, false), + tex_occlusion: tex(b, mm.occlusion_tex, false), + tex_emissive: tex(b, mm.emissive_tex, true), + double_sided: mm.double_sided, + unlit: mm.unlit, + ..opts + }; + mats.push(b.material(m)); + } + for inst in mdl.instances(clip, t) { + for p in &inst.mesh.primitives { + let mat = mats[p.material.unwrap_or(0).min(mats.len() - 1)]; + b.emit(p, mat, &|k| if k == usize::MAX { world * inst.world } else { world * model::vertex_matrix(&inst, p, k) }); + } + } +} + +/// Closed polylines of a path (flattened to `tol` pixels). +pub fn contours(path: &kurbo::BezPath, tol: f64) -> Vec> { + let mut out: Vec> = vec![]; + let mut cur: Vec<[f64; 2]> = vec![]; + kurbo::flatten(path, tol, |el| match el { + kurbo::PathEl::MoveTo(p) => { + if cur.len() >= 3 { + out.push(std::mem::take(&mut cur)); + } + cur = vec![[p.x, p.y]]; + } + kurbo::PathEl::LineTo(p) => cur.push([p.x, p.y]), + kurbo::PathEl::ClosePath => { + if cur.len() >= 3 { + out.push(std::mem::take(&mut cur)); + } + cur.clear(); + } + _ => {} + }); + if cur.len() >= 3 { + out.push(cur); + } + out +} + +/// Geometry Options of a text/shape layer when it is extruded. +pub fn extrusion_params(ctx: &EvalCtx, layer: &Layer) -> Option { + let g = layer.props.sub("geometryOptions")?; + let depth = ctx.f(layer, g, "extrusionDepth", 0.0); + (depth > 0.0).then(|| model::extrude::ExtrudeParams { + bevel: model::extrude::BevelStyle::from_index(ctx.e(layer, g, "bevelStyle")), + bevel_depth: ctx.f(layer, g, "bevelDepth", 2.0).max(0.0), + hole_bevel: (ctx.f(layer, g, "holeBevelDepth", 100.0) / 100.0).clamp(0.0, 1.0), + depth, + }) +} + +/// Extruded meshes of a text or shape layer: (mesh in layer space, straight colour). +pub fn extruded_meshes(ctx: &EvalCtx, layer: &Layer, p: &model::extrude::ExtrudeParams) -> Vec<(Primitive, [f32; 4])> { + let mut groups: Vec<(Vec, [f32; 4])> = vec![]; + match layer.source { + LayerSource::Text => { + if let Some(tg) = crate::text::text_geom(ctx, layer) { + for g in tg.glyphs.iter().filter(|g| g.apply_fill && g.fill[3] > 0.0) { + groups.push((vec![g.path()], g.fill)); + } + } + } + LayerSource::Shape => { + if let Some(c) = layer.props.sub("contents") { + groups = crate::shapes::fill_outlines(ctx, layer, c); + } + } + _ => {} + } + groups + .into_iter() + .filter_map(|(paths, color)| { + let cs: Vec> = paths.iter().flat_map(|bp| contours(bp, 0.25)).collect(); + let outlines = model::extrude::group_contours(cs); + if outlines.is_empty() { + return None; + } + Some((model::extrude::extrude(&outlines, p), color)) + }) + .collect() +} + +fn extruded_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, p: &model::extrude::ExtrudeParams, b: &mut Builder) { + let base = card_material(ctx, layer); + let opacity = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); + let world = ctx.world_matrix(layer); + let lin = b.s.linear_io; + for (mesh, c) in extruded_meshes(ctx, layer, p) { + let l = |v: f32| if lin { v } else { srgb_to_linear(v) }; + let m = b.material(Material { + base: [l(c[0]), l(c[1]), l(c[2]), c[3]], + alpha_mode: if c[3] < 0.999 { 2 } else { 0 }, + opacity, + double_sided: false, + ..base + }); + b.emit(&mesh, m, &|_| world); + } +} + +fn card(b: &mut Builder, buf: &Buf, world: Mat4, mut mat: Material) { + if buf.img.width == 0 || buf.img.height == 0 { + return; + } + let (tex, opaque) = b.buf_texture(buf); + mat.tex_base = tex; + mat.alpha_mode = if opaque { 0 } else { 2 }; + let m = b.material(mat); + let s = buf.scale.max(1e-9); + let (x0, y0) = (-buf.offset[0] / s, -buf.offset[1] / s); + let (x1, y1) = ((buf.img.width as f64 - buf.offset[0]) / s, (buf.img.height as f64 - buf.offset[1]) / s); + // Faces the camera (−z) in layer space; uv (0,0) at the buffer's top-left texel corner. + let p = Primitive { + positions: vec![[x0 as f32, y0 as f32, 0.0], [x1 as f32, y0 as f32, 0.0], [x1 as f32, y1 as f32, 0.0], [x0 as f32, y1 as f32, 0.0]], + normals: vec![[0.0, 0.0, -1.0]; 4], + uvs: vec![[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]], + tangents: vec![[1.0, 0.0, 0.0, 1.0]; 4], + indices: vec![0, 2, 1, 0, 3, 2], + material: Some(0), + ..Default::default() + }; + b.emit(&p, m, &|_| world); +} + +fn card_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, b: &mut Builder) { + let mut mat = card_material(ctx, layer); + mat.opacity = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); + let world = ctx.world_matrix(layer); + if crate::text::per_char_3d(ctx, layer) { + for (buf, m) in crate::text::per_char_planes(ctx, layer, r.opts.scale) { + let buf = r.to_blend(Arc::new(buf), None); + card(b, &buf, world * m, mat); + } + return; + } + if let Some(buf) = r.blend_layer_buf(ctx, layer) { + card(b, &buf, world, mat); + } +} + +/// Evaluated lights: (scene lights, ambient sum, environment light (layer, intensity × colour, +/// rotation)). +fn lights(ctx: &EvalCtx, lin: bool) -> (Vec, [f32; 3], Option<(Option, [f32; 3], f32)>) { + let mut out = vec![]; + let mut ambient = [0.0f32; 3]; + let mut env = None; + let l = |v: f32| if lin { v } else { srgb_to_linear(v) }; + for layer in ctx.comp.layers.iter().filter(|l| l.is_light() && l.switches.video && l.is_active_at(ctx.time)) { + let LayerSource::Light { kind } = layer.source else { continue }; + let Some(g) = layer.props.sub("lightOptions") else { continue }; + let k = (ctx.f(layer, g, "intensity", 100.0) / 100.0) as f32; + let c = ctx.color(layer, g, "color"); + let color = [l(c[0]) * k, l(c[1]) * k, l(c[2]) * k]; + match kind { + LightKind::Ambient => { + for i in 0..3 { + ambient[i] += color[i]; + } + } + LightKind::Environment => { + if env.is_none() { + let src = g.get("source").and_then(|p| ctx.value(layer, p).as_layer()); + let rot = ctx.f(layer, g, "rotation", 0.0).to_radians() as f32; + env = Some((src, color, rot)); + } + } + _ => { + let Some(st) = LightState::from_layer(ctx, layer) else { continue }; + let f = |v: Vec3| [v.x as f32, v.y as f32, v.z as f32]; + out.push(Light { + kind: match kind { + LightKind::Parallel => 0, + LightKind::Spot => 1, + _ => 2, + }, + pos: f(st.pos), + dir: f(st.dir.normalize()), + color, + cos_inner: (st.cone_half * (1.0 - st.feather)).cos() as f32, + cos_outer: st.cone_half.cos() as f32, + falloff: st.falloff, + radius: st.radius as f32, + falloff_distance: st.falloff_distance as f32, + // Marked; `raster::shadow_maps` assigns the maps. + shadow: if st.casts_shadows { 0 } else { -1 }, + shadow_darkness: st.shadow_darkness as f32, + shadow_diffusion: st.shadow_diffusion as f32, + }); + } + } + } + (out, ambient, env) +} + +/// Box-downsample a straight RGBA image by 2 (odd sizes round up). +fn half(w: u32, h: u32, d: &[[f32; 4]]) -> (u32, u32, Vec<[f32; 4]>) { + let (nw, nh) = (w.div_ceil(2).max(1), h.div_ceil(2).max(1)); + let mut out = vec![[0.0f32; 4]; (nw * nh) as usize]; + for y in 0..nh { + for x in 0..nw { + let mut acc = [0.0f32; 4]; + let mut n = 0.0; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (sx, sy) = (2 * x + dx, 2 * y + dy); + if sx < w && sy < h { + let p = d[(sy * w + sx) as usize]; + for k in 0..4 { + acc[k] += p[k]; + } + n += 1.0; + } + } + out[(y * nw + x) as usize] = acc.map(|v| v / n); + } + } + (nw, nh, out) +} + +/// Cosine-weighted irradiance (`w`×`h` equirectangular) of an equirectangular radiance image: +/// each texel is the cosine-weighted mean radiance over the hemisphere around its direction +/// (a constant environment gives the same constant). +pub fn irradiance_map(sw: u32, sh: u32, src: &[[f32; 4]], w: u32, h: u32) -> Vec<[f32; 4]> { + use super::shade::{dot, equirect_dir}; + // Source directions and solid-angle weights (∝ sin θ). + let dirs: Vec<([f32; 3], f32, [f32; 3])> = (0..sh) + .flat_map(|y| (0..sw).map(move |x| (x, y))) + .map(|(x, y)| { + let (u, v) = ((x as f32 + 0.5) / sw as f32, (y as f32 + 0.5) / sh as f32); + let p = src[(y * sw + x) as usize]; + (equirect_dir(u, v, 0.0), (v * std::f32::consts::PI).sin(), [p[0], p[1], p[2]]) + }) + .collect(); + let mut out = vec![[0.0f32; 4]; (w * h) as usize]; + for y in 0..h { + for x in 0..w { + let n = equirect_dir((x as f32 + 0.5) / w as f32, (y as f32 + 0.5) / h as f32, 0.0); + let (mut acc, mut wsum) = ([0.0f32; 3], 0.0f32); + for (d, sa, c) in &dirs { + let k = dot(n, *d).max(0.0) * sa; + if k > 0.0 { + for i in 0..3 { + acc[i] += c[i] * k; + } + wsum += k; + } + } + let inv = if wsum > 0.0 { 1.0 / wsum } else { 0.0 }; + out[(y * w + x) as usize] = [acc[0] * inv, acc[1] * inv, acc[2] * inv, 1.0]; + } + } + out +} + +/// Prefilter an environment image into the scene's textures. +pub(crate) fn add_environment(b: &mut Builder, w: u32, h: u32, data: Vec<[f32; 4]>, intensity: [f32; 3], rotation: f32) { + // Radiance level 0 at most 1024 wide. + let (mut w, mut h, mut d) = (w, h, data); + while w > 1024 { + (w, h, d) = half(w, h, &d); + } + let first = b.s.textures.len() as u32; + let mut levels = 0; + let mut small: Option<(u32, u32, Vec<[f32; 4]>)> = None; + loop { + b.texture_from(w, h, [0, 1], d.iter().copied()); + levels += 1; + if w <= 64 && small.is_none() { + small = Some((w, h, d.clone())); + } + if w <= 8 || h <= 4 { + break; + } + (w, h, d) = half(w, h, &d); + } + let (sw, sh, sd) = small.unwrap_or((w, h, d)); + let irr = irradiance_map(sw, sh, &sd, 32, 16); + let ii = b.texture_from(32, 16, [0, 1], irr.into_iter()) as u32; + // Intensity × colour folded into the radiance (one scalar on the GPU): tint the texels. + let k = (intensity[0] + intensity[1] + intensity[2]) / 3.0; + if k > 0.0 && (intensity[0] != intensity[1] || intensity[1] != intensity[2]) { + let tint = intensity.map(|c| c / k); + for t in first..=ii { + let ti = b.s.textures[t as usize]; + for p in &mut b.s.texels[ti.offset as usize..(ti.offset + ti.width * ti.height) as usize] { + for c in 0..3 { + p[c] *= tint[c]; + } + } + } + } + b.s.env = Some(EnvInfo { radiance: first, mips: levels, irradiance: ii, intensity: k, rotation }); +} + +/// The environment layer's pixels as linear straight RGBA. +fn env_pixels(r: &Renderer, ctx: &EvalCtx, src: Option, lin: bool) -> Option<(u32, u32, Vec<[f32; 4]>)> { + let layer = match src { + Some(id) => ctx.comp.layer(effectcraft_project::LayerId(id))?, + None => ctx.comp.layers.iter().find(|l| l.environment && l.is_active_at(ctx.time))?, + }; + let buf = r.blend_layer_buf(ctx, layer)?; + let img = &buf.img; + let data = img + .data + .iter() + .map(|p| { + let a = p[3]; + if a <= 1e-6 { + return [0.0, 0.0, 0.0, 1.0]; + } + let s = [p[0] / a, p[1] / a, p[2] / a]; + if lin { [s[0], s[1], s[2], 1.0] } else { [srgb_to_linear(s[0]), srgb_to_linear(s[1]), srgb_to_linear(s[2]), 1.0] } + }) + .collect(); + Some((img.width, img.height, data)) +} + +/// Supersampling factor: 2×2 unless draft or very large. +fn ssaa_for(r: &Renderer, out: (u32, u32)) -> u32 { + if r.opts.draft || (out.0 as u64 * out.1 as u64) > 8_000_000 { 1 } else { 2 } +} + +/// Build the scene of a run of 3D layers. +pub(crate) fn build(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> Scene { + let cam = camera_for(r, ctx); + let lin = r.pipe.linear || r.pipe.linear_blend; + let ssaa = ssaa_for(r, out); + let mut b = Builder { + s: Scene { width: out.0 * ssaa, height: out.1 * ssaa, ssaa, linear_io: lin, ..Scene::default() }, + tex_keys: HashMap::new(), + tri_depth: vec![], + cam, + }; + for layer in run { + if layer.environment || layer.switches.adjustment || ctx.opacity(layer) <= 0.0 || !layer.has_video() { + continue; + } + if layer.source.is_model() { + model_layer(r, ctx, layer, &mut b); + } else if matches!(layer.source, LayerSource::Text | LayerSource::Shape) + && let Some(p) = extrusion_params(ctx, layer) + { + extruded_layer(r, ctx, layer, &p, &mut b); + } else { + card_layer(r, ctx, layer, &mut b); + } + } + b.sort_triangles(); + // Lights, environment, shadows (Draft 3D: unlit, like After Effects). + if !ctx.comp.draft_3d { + let (ls, amb, env) = lights(ctx, lin); + b.s.lights = ls; + b.s.ambient = amb; + if let Some((src, k, rot)) = env + && let Some((w, h, d)) = env_pixels(r, ctx, src, lin) + { + add_environment(&mut b, w, h, d, k, rot); + } + super::raster::shadow_maps(&mut b.s, r.opts.draft); + } + // Camera matrices. + let s = r.opts.scale; + let (cw, ch) = (ctx.comp.width as f64, ctx.comp.height as f64); + let (ow, oh) = (out.0 as f64, out.1 as f64); + let (a, bb) = (2.0 * s * cam.zoom / ow, s * cw / ow - 1.0); + let (c, d) = (2.0 * s * cam.zoom / oh, s * ch / oh - 1.0); + let proj = if cam.ortho { + let (mut zmin, mut zmax) = (f64::INFINITY, f64::NEG_INFINITY); + for v in &b.s.vertices { + let z = cam.depth(Vec3::from(v.pos.map(|x| x as f64))); + zmin = zmin.min(z); + zmax = zmax.max(z); + } + if !zmin.is_finite() { + (zmin, zmax) = (-1.0, 1.0); + } + let pad = (zmax - zmin).abs() * 0.01 + 1.0; + let (zmin, zmax) = (zmin - pad, zmax + pad); + let zr = zmax - zmin; + Mat4([[a, 0.0, 0.0, bb], [0.0, -c, 0.0, -d], [0.0, 0.0, -1.0 / zr, zmax / zr], [0.0, 0.0, 0.0, 1.0]]) + } else { + Mat4([[a, 0.0, bb, 0.0], [0.0, -c, -d, 0.0], [0.0, 0.0, 0.0, NEAR], [0.0, 0.0, 1.0, 0.0]]) + }; + b.s.clip = m32(&(proj * cam.view)); + b.s.view = m32(&cam.view); + let f = cam.forward(); + b.s.cam_fwd = [f.x as f32, f.y as f32, f.z as f32]; + b.s.eye = [cam.eye.x as f32, cam.eye.y as f32, cam.eye.z as f32]; + b.s.ortho = cam.ortho; + b.s +} diff --git a/crates/render/src/three_d/adv/shade.rs b/crates/render/src/three_d/adv/shade.rs new file mode 100644 index 00000000..a8ab5e70 --- /dev/null +++ b/crates/render/src/three_d/adv/shade.rs @@ -0,0 +1,408 @@ +//! Physically based shading for the Advanced 3D renderer (CPU reference; `crates/gpu`'s +//! `advanced3d.wgsl` repeats it step for step). +//! +//! - Direct light: Cook–Torrance microfacet specular with the GGX/Trowbridge–Reitz normal +//! distribution, the Smith–Schlick-GGX geometry term (`k = (r + 1)² / 8`) and Schlick's +//! Fresnel (`F0` = 0.04 for dielectrics, the base colour for metals), plus a Lambertian +//! diffuse lobe weighted by `(1 − F)(1 − metallic)` (Karis 2013, "Real Shading in Unreal +//! Engine 4"; the glTF 2.0 metallic-roughness model). Light colours are the irradiance on a +//! surface facing the light (a white 100% light shows a white diffuse surface at its base +//! colour), i.e. π is folded into the light intensity. +//! - Image-based light: a cosine-convolved irradiance map for diffuse, a box-filtered mip +//! chain of the equirectangular environment sampled by roughness for specular, and the +//! analytic split-sum environment BRDF fit (Karis 2014, "Physically Based Shading on +//! Mobile"). +//! - Shadows: depth maps per light, percentage-closer filtered. + +use super::scene::{EnvInfo, Light, Material, Scene, ShadowMap, TexInfo}; + +pub const PI: f32 = std::f32::consts::PI; + +#[inline] +pub fn dot(a: [f32; 3], b: [f32; 3]) -> f32 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +#[inline] +pub fn cross(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +#[inline] +pub fn norm(a: [f32; 3]) -> [f32; 3] { + let l = dot(a, a).sqrt(); + if l > 1e-20 { [a[0] / l, a[1] / l, a[2] / l] } else { [0.0, 0.0, -1.0] } +} +#[inline] +fn add(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +#[inline] +fn sub(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +#[inline] +fn scale(a: [f32; 3], k: f32) -> [f32; 3] { + [a[0] * k, a[1] * k, a[2] * k] +} +#[inline] +fn mix(a: f32, b: f32, t: f32) -> f32 { + a + (b - a) * t +} + +/// sRGB-encoded → linear light. +#[inline] +pub fn srgb_to_linear(c: f32) -> f32 { + if c <= 0.04045 { c / 12.92 } else { ((c + 0.055) / 1.055).powf(2.4) } +} +/// Linear light → sRGB-encoded (over-range values continue the curve). +#[inline] +pub fn linear_to_srgb(c: f32) -> f32 { + if c <= 0.0031308 { c * 12.92 } else { 1.055 * c.powf(1.0 / 2.4) - 0.055 } +} + +/// GGX / Trowbridge–Reitz normal distribution, `a = roughness²`. +#[inline] +pub fn d_ggx(n_h: f32, a: f32) -> f32 { + let a2 = a * a; + let d = n_h * n_h * (a2 - 1.0) + 1.0; + a2 / (PI * d * d).max(1e-12) +} + +/// Smith–Schlick-GGX geometry term for direct light (`k = (r + 1)² / 8`). +#[inline] +pub fn g_smith(n_v: f32, n_l: f32, rough: f32) -> f32 { + let k = (rough + 1.0) * (rough + 1.0) / 8.0; + let g1 = |x: f32| x / (x * (1.0 - k) + k); + g1(n_v) * g1(n_l) +} + +/// Schlick's Fresnel approximation. +#[inline] +pub fn f_schlick(f0: [f32; 3], v_h: f32) -> [f32; 3] { + let k = (1.0 - v_h).clamp(0.0, 1.0).powi(5); + [f0[0] + (1.0 - f0[0]) * k, f0[1] + (1.0 - f0[1]) * k, f0[2] + (1.0 - f0[2]) * k] +} + +/// Cook–Torrance BRDF × N·L × π for one light: (diffuse colour factor, specular colour). +/// Returned as the reflected fraction of the light's irradiance. +pub fn brdf(base: [f32; 3], metallic: f32, rough: f32, spec_k: f32, diff_k: f32, n: [f32; 3], v: [f32; 3], l: [f32; 3]) -> [f32; 3] { + let n_l = dot(n, l); + if n_l <= 0.0 { + return [0.0; 3]; + } + let n_v = dot(n, v).max(1e-4); + let h = norm(add(v, l)); + let n_h = dot(n, h).max(0.0); + let v_h = dot(v, h).max(0.0); + let r = rough.clamp(0.03, 1.0); + let f0 = [mix(0.04 * spec_k, base[0], metallic), mix(0.04 * spec_k, base[1], metallic), mix(0.04 * spec_k, base[2], metallic)]; + let f = f_schlick(f0, v_h); + let d = d_ggx(n_h, r * r); + let g = g_smith(n_v, n_l, r); + let spec = d * g / (4.0 * n_v * n_l).max(1e-4); + let mut out = [0.0; 3]; + for c in 0..3 { + let kd = (1.0 - f[c]) * (1.0 - metallic); + out[c] = (kd * base[c] / PI * diff_k + f[c] * spec) * n_l * PI; + } + out +} + +/// Split-sum environment BRDF (scale, bias) for `F0·A + B` (analytic fit). +#[inline] +pub fn env_brdf(n_v: f32, rough: f32) -> (f32, f32) { + let c0 = [-1.0, -0.0275, -0.572, 0.022]; + let c1 = [1.0, 0.0425, 1.04, -0.04]; + let r = [rough * c0[0] + c1[0], rough * c0[1] + c1[1], rough * c0[2] + c1[2], rough * c0[3] + c1[3]]; + let a004 = (r[0] * r[0]).min((-9.28 * n_v).exp2()) * r[0] + r[1]; + (-1.04 * a004 + r[2], 1.04 * a004 + r[3]) +} + +/// Wrap a texel index. +#[inline] +fn wrap(i: i64, n: i64, mode: u32) -> i64 { + match mode { + 1 => i.clamp(0, n - 1), + 2 => { + let m = i.rem_euclid(2 * n); + if m >= n { 2 * n - 1 - m } else { m } + } + _ => i.rem_euclid(n), + } +} + +/// Bilinear texture sample at (u, v) (0..1 across the texture, texel centres at +0.5). +pub fn sample(t: &TexInfo, texels: &[[f32; 4]], u: f32, v: f32) -> [f32; 4] { + let (w, h) = (t.width as i64, t.height as i64); + if w == 0 || h == 0 { + return [1.0; 4]; + } + let x = u * w as f32 - 0.5; + let y = v * h as f32 - 0.5; + let (x0, y0) = (x.floor(), y.floor()); + let (fx, fy) = (x - x0, y - y0); + let (x0, y0) = (x0 as i64, y0 as i64); + let get = |xi: i64, yi: i64| -> [f32; 4] { + let xi = wrap(xi, w, t.wrap_u); + let yi = wrap(yi, h, t.wrap_v); + texels.get(t.offset as usize + (yi * w + xi) as usize).copied().unwrap_or([0.0; 4]) + }; + let (a, b, c, d) = (get(x0, y0), get(x0 + 1, y0), get(x0, y0 + 1), get(x0 + 1, y0 + 1)); + let mut o = [0.0; 4]; + for k in 0..4 { + let top = a[k] + (b[k] - a[k]) * fx; + let bot = c[k] + (d[k] - c[k]) * fx; + o[k] = top + (bot - top) * fy; + } + o +} + +/// Equirectangular coordinates of a world direction (AE world: +Y down; the environment's up +/// is −Y), rotated about the vertical axis by `rot` radians. +#[inline] +pub fn equirect_uv(d: [f32; 3], rot: f32) -> (f32, f32) { + let u = 0.5 + (d[0].atan2(d[2]) + rot) / (2.0 * PI); + let v = (-d[1]).clamp(-1.0, 1.0).acos() / PI; + (u - u.floor(), v) +} + +/// Inverse of [`equirect_uv`] (texel centre → direction). +pub fn equirect_dir(u: f32, v: f32, rot: f32) -> [f32; 3] { + let phi = (u - 0.5) * 2.0 * PI - rot; + let th = v * PI; + [th.sin() * phi.sin(), -th.cos(), th.sin() * phi.cos()] +} + +/// Diffuse (irradiance) and roughness-filtered specular radiance of the environment. +pub fn env_light(s: &Scene, e: &EnvInfo, n: [f32; 3], r: [f32; 3], rough: f32) -> ([f32; 3], [f32; 3]) { + let (u, v) = equirect_uv(n, e.rotation); + let irr = s.textures.get(e.irradiance as usize).map_or([0.0; 4], |t| sample(t, &s.texels, u, v)); + let (u, v) = equirect_uv(r, e.rotation); + let lv = rough.clamp(0.0, 1.0) * (e.mips.saturating_sub(1)) as f32; + let l0 = lv.floor() as u32; + let l1 = (l0 + 1).min(e.mips.saturating_sub(1)); + let f = lv - l0 as f32; + let a = s.textures.get((e.radiance + l0) as usize).map_or([0.0; 4], |t| sample(t, &s.texels, u, v)); + let b = s.textures.get((e.radiance + l1) as usize).map_or([0.0; 4], |t| sample(t, &s.texels, u, v)); + let spec = [mix(a[0], b[0], f), mix(a[1], b[1], f), mix(a[2], b[2], f)]; + (scale([irr[0], irr[1], irr[2]], e.intensity), scale(spec, e.intensity)) +} + +/// Light-space position (u, v in shadow-map pixels, depth) of a world point. +#[inline] +pub fn shadow_coord(m: &ShadowMap, p: [f32; 3]) -> Option<(f32, f32, f32)> { + let v = &m.view; + let q = [ + v[0][0] * p[0] + v[0][1] * p[1] + v[0][2] * p[2] + v[0][3], + v[1][0] * p[0] + v[1][1] * p[1] + v[1][2] * p[2] + v[1][3], + v[2][0] * p[0] + v[2][1] * p[1] + v[2][2] * p[2] + v[2][3], + ]; + let c = m.size as f32 * 0.5; + if m.ortho { + Some((q[0] * m.focal + c, q[1] * m.focal + c, q[2])) + } else if q[2] > 1e-3 { + Some((q[0] * m.focal / q[2] + c, q[1] * m.focal / q[2] + c, q[2])) + } else { + None + } +} + +/// Fraction of light reaching `p` through shadow map `m` (1 = lit), PCF over a 5×5 grid +/// spread over `radius` texels. +pub fn shadow_factor(m: &ShadowMap, texels: &[f32], p: [f32; 3], n_l: f32) -> f32 { + let Some((u, v, z)) = shadow_coord(m, p) else { return 1.0 }; + let size = m.size as i64; + // Slope-scaled bias (world units are pixels). + let bias = m.bias * (1.0 + 2.0 * (1.0 - n_l.clamp(0.0, 1.0))) + z.abs() * 0.0015; + let step = m.radius / 2.0; + let mut lit = 0.0; + for j in -2..=2 { + for i in -2..=2 { + let x = (u + i as f32 * step).floor() as i64; + let y = (v + j as f32 * step).floor() as i64; + if x < 0 || y < 0 || x >= size || y >= size { + lit += 1.0; + continue; + } + let d = texels[m.offset as usize + (y * size + x) as usize]; + if z - bias <= d { + lit += 1.0; + } + } + } + lit / 25.0 +} + +/// The visibility of light `li` at `p` (1 = unshadowed), darkened by Shadow Darkness. +pub fn light_shadow(s: &Scene, l: &Light, p: [f32; 3], n_l: f32) -> f32 { + if l.shadow < 0 { + return 1.0; + } + let first = l.shadow as usize; + let m = if l.kind == 2 { + // Point light: the cube face of the major axis of the light → point direction. + let d = sub(p, l.pos); + let (ax, ay, az) = (d[0].abs(), d[1].abs(), d[2].abs()); + let face = if ax >= ay && ax >= az { + if d[0] > 0.0 { 0 } else { 1 } + } else if ay >= az { + if d[1] > 0.0 { 2 } else { 3 } + } else if d[2] > 0.0 { + 4 + } else { + 5 + }; + s.shadows.get(first + face) + } else { + s.shadows.get(first) + }; + let Some(m) = m else { return 1.0 }; + let f = shadow_factor(m, &s.shadow_texels, p, n_l); + 1.0 - l.shadow_darkness * (1.0 - f) +} + +/// Distance attenuation × spot cone of a light at `p` (AE falloff modes). +pub fn attenuation(l: &Light, p: [f32; 3], to_l: [f32; 3], dist: f32) -> f32 { + let mut k = match l.falloff { + 1 if l.kind != 0 => { + if dist <= l.radius { + 1.0 + } else if l.falloff_distance <= 0.0 { + 0.0 + } else { + let t = ((dist - l.radius) / l.falloff_distance).clamp(0.0, 1.0); + 1.0 - t * t * (3.0 - 2.0 * t) + } + } + 2 if l.kind != 0 => { + if dist <= l.radius || (l.radius <= 0.0 && dist <= 1.0) { + 1.0 + } else { + (l.radius.max(1.0) / dist).powi(2) + } + } + _ => 1.0, + }; + let _ = p; + if l.kind == 1 { + let cos_a = dot(scale(to_l, -1.0), l.dir).clamp(-1.0, 1.0); + k *= if cos_a >= l.cos_inner { + 1.0 + } else if cos_a <= l.cos_outer { + 0.0 + } else { + // Smoothstep across the feather in angle. + let a = cos_a.acos(); + let (ai, ao) = (l.cos_inner.acos(), l.cos_outer.acos()); + let t = ((a - ai) / (ao - ai).max(1e-6)).clamp(0.0, 1.0); + 1.0 - t * t * (3.0 - 2.0 * t) + }; + } + k +} + +/// Inputs of one fragment. +#[derive(Clone, Copy, Debug)] +pub struct Frag { + pub pos: [f32; 3], + /// Interpolated (unnormalised) vertex normal. + pub normal: [f32; 3], + pub uv: [f32; 2], + pub tangent: [f32; 4], + pub material: u32, + pub front: bool, +} + +/// Shade a fragment: premultiplied linear RGBA, or `None` when discarded (alpha test, back +/// face of a single-sided material). +pub fn shade(s: &Scene, f: &Frag) -> Option<[f32; 4]> { + let m: &Material = s.materials.get(f.material as usize)?; + if !f.front && !m.double_sided { + return None; + } + let tex = |i: i32| -> Option<[f32; 4]> { (i >= 0).then(|| s.textures.get(i as usize).map(|t| sample(t, &s.texels, f.uv[0], f.uv[1]))).flatten() }; + let mut base = m.base; + if let Some(t) = tex(m.tex_base) { + for k in 0..4 { + base[k] *= t[k]; + } + } + let alpha = base[3] * m.opacity; + if m.alpha_mode == 1 && alpha < m.alpha_cutoff { + return None; + } + if alpha <= 1e-5 && m.alpha_mode != 0 { + return None; + } + let alpha = if m.alpha_mode == 0 { m.opacity } else { alpha }; + let rgb = [base[0], base[1], base[2]]; + let mut emissive = m.emissive; + if let Some(t) = tex(m.tex_emissive) { + emissive = [emissive[0] * t[0], emissive[1] * t[1], emissive[2] * t[2]]; + } + if m.unlit || !m.accepts_lights || (s.lights.is_empty() && s.env.is_none() && s.ambient == [0.0; 3]) { + let c = add(rgb, emissive); + return Some([c[0] * alpha, c[1] * alpha, c[2] * alpha, alpha]); + } + let (mut metallic, mut rough) = (m.metallic, m.roughness); + if let Some(t) = tex(m.tex_mr) { + rough *= t[1]; + metallic *= t[2]; + } + let mut n = norm(f.normal); + if !f.front { + n = scale(n, -1.0); + } + if let Some(t) = tex(m.tex_normal) { + let tn = [(t[0] * 2.0 - 1.0) * m.normal_scale, (t[1] * 2.0 - 1.0) * m.normal_scale, t[2] * 2.0 - 1.0]; + let tg = [f.tangent[0], f.tangent[1], f.tangent[2]]; + // Gram-Schmidt against the shading normal. + let tg = norm(sub(tg, scale(n, dot(n, tg)))); + let sign = if f.front { f.tangent[3] } else { -f.tangent[3] }; + let bt = scale(cross(n, tg), sign); + n = norm(add(add(scale(tg, tn[0]), scale(bt, tn[1])), scale(n, tn[2]))); + } + let ao = match tex(m.tex_occlusion) { + Some(t) => 1.0 + m.occlusion_strength * (t[0] - 1.0), + None => 1.0, + }; + let v = if s.ortho { scale(s.cam_fwd, -1.0) } else { norm(sub(s.eye, f.pos)) }; + let mut out = [0.0f32; 3]; + for l in &s.lights { + let (to_l, dist) = if l.kind == 0 { + (scale(l.dir, -1.0), f32::INFINITY) + } else { + let d = sub(l.pos, f.pos); + let len = dot(d, d).sqrt(); + if len < 1e-6 { ([0.0, 0.0, -1.0], 0.0) } else { (scale(d, 1.0 / len), len) } + }; + let k = attenuation(l, f.pos, to_l, dist); + if k <= 0.0 { + continue; + } + let n_l = dot(n, to_l); + if n_l <= 0.0 { + continue; + } + let sh = if m.receives_shadows { light_shadow(s, l, f.pos, n_l) } else { 1.0 }; + let b = brdf(rgb, metallic, rough, m.specular_k, m.diffuse_k, n, v, to_l); + for c in 0..3 { + out[c] += b[c] * l.color[c] * k * sh; + } + } + for c in 0..3 { + out[c] += s.ambient[c] * rgb[c] * m.ambient_k * ao; + } + if let Some(e) = &s.env { + let n_v = dot(n, v).clamp(1e-4, 1.0); + let r = sub(scale(n, 2.0 * dot(n, v)), v); + let (irr, spec) = env_light(s, e, n, r, rough); + let f0 = [mix(0.04 * m.specular_k, rgb[0], metallic), mix(0.04 * m.specular_k, rgb[1], metallic), mix(0.04 * m.specular_k, rgb[2], metallic)]; + let (a, b) = env_brdf(n_v, rough); + for c in 0..3 { + let fs = f0[c] * a + b; + out[c] += (irr[c] * rgb[c] * (1.0 - fs) * (1.0 - metallic) * m.diffuse_k + spec[c] * fs) * ao; + } + } + let c = add(out, emissive); + Some([c[0] * alpha, c[1] * alpha, c[2] * alpha, alpha]) +} diff --git a/crates/render/src/three_d/adv/tests.rs b/crates/render/src/three_d/adv/tests.rs new file mode 100644 index 00000000..23db2cbc --- /dev/null +++ b/crates/render/src/three_d/adv/tests.rs @@ -0,0 +1,314 @@ +use effectcraft_keyframe::Value; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, LightKind, PrimitiveKind, Project, Renderer as R3, build}; +use effectcraft_time::{FrameRate, Tick}; + +use super::scene::{Material, Scene, TexInfo, Vertex, irradiance_map}; +use super::shade::{self, PI}; +use super::*; +use crate::{NoFootage, RenderOpts, Renderer}; + +// ---------------------------------------------------------------- shading + +#[test] +fn brdf_matches_cook_torrance_formula() { + // Reference: f = kd·c/π + D·G·F/(4·NV·NL), returned × NL × π. + let n = [0.0, 0.0, 1.0]; + let v = shade::norm([0.3, 0.1, 1.0]); + let l = shade::norm([-0.4, 0.2, 1.0]); + let (base, metal, rough) = ([0.8, 0.4, 0.2], 0.3f32, 0.45f32); + let got = shade::brdf(base, metal, rough, 1.0, 1.0, n, v, l); + let h = shade::norm([v[0] + l[0], v[1] + l[1], v[2] + l[2]]); + let (nv, nl, nh, vh) = (shade::dot(n, v), shade::dot(n, l), shade::dot(n, h), shade::dot(v, h)); + let a2 = (rough * rough).powi(2); + let d = a2 / (PI * ((nh * nh) * (a2 - 1.0) + 1.0).powi(2)); + let k = (rough + 1.0).powi(2) / 8.0; + let g = nv / (nv * (1.0 - k) + k) * nl / (nl * (1.0 - k) + k); + for c in 0..3 { + let f0 = 0.04 + (base[c] - 0.04) * metal; + let f = f0 + (1.0 - f0) * (1.0 - vh).powi(5); + let expect = ((1.0 - f) * (1.0 - metal) * base[c] / PI + d * g * f / (4.0 * nv * nl)) * nl * PI; + assert!((got[c] - expect).abs() < 1e-5, "channel {c}: {} vs {expect}", got[c]); + } + // No light from behind the surface. + assert_eq!(shade::brdf(base, metal, rough, 1.0, 1.0, n, v, [0.0, 0.0, -1.0]), [0.0; 3]); + // A white rough dielectric lit head-on reflects about its base colour (π folded in). + let w = shade::brdf([1.0; 3], 0.0, 1.0, 1.0, 1.0, n, n, n); + assert!(w[0] > 0.95 && w[0] < 1.05, "{w:?}"); +} + +#[test] +fn ggx_distribution_is_normalised() { + // ∫ D(h)·(n·h) dω = 1 over the hemisphere. + for &rough in &[0.2f32, 0.5, 0.9] { + let a = rough * rough; + let n = 4000; + let mut sum = 0.0f64; + for i in 0..n { + let th = (i as f64 + 0.5) / n as f64 * std::f64::consts::FRAC_PI_2; + let c = th.cos() as f32; + sum += shade::d_ggx(c, a) as f64 * c as f64 * th.sin() * std::f64::consts::FRAC_PI_2 / n as f64 * 2.0 * std::f64::consts::PI; + } + assert!((sum - 1.0).abs() < 0.02, "roughness {rough}: {sum}"); + } +} + +#[test] +fn constant_environment_has_constant_irradiance() { + let src = vec![[0.25, 0.5, 0.75, 1.0]; 64 * 32]; + let irr = irradiance_map(64, 32, &src, 32, 16); + for p in &irr { + for c in 0..3 { + assert!((p[c] - src[0][c]).abs() < 1e-4, "{p:?}"); + } + } + // And a sky brighter than the ground lights upward-facing normals more. + let sky: Vec<[f32; 4]> = (0..32).flat_map(|y| (0..64).map(move |_| if y < 16 { [1.0, 1.0, 1.0, 1.0] } else { [0.0, 0.0, 0.0, 1.0] })).collect(); + let irr = irradiance_map(64, 32, &sky, 32, 16); + assert!(irr[16][0] > 0.9 && irr[15 * 32 + 16][0] < 0.1, "{:?} {:?}", irr[16], irr[15 * 32 + 16]); +} + +#[test] +fn equirect_round_trip_and_env_brdf() { + for (u, v) in [(0.1f32, 0.2f32), (0.5, 0.5), (0.9, 0.7)] { + let d = shade::equirect_dir(u, v, 0.3); + let (u2, v2) = shade::equirect_uv(d, 0.3); + assert!((u - u2).abs() < 1e-4 && (v - v2).abs() < 1e-4); + } + // Up (−Y in AE space) is the top row. + assert!(shade::equirect_uv([0.0, -1.0, 0.0], 0.0).1 < 1e-3); + let (a, b) = shade::env_brdf(1.0, 0.0); + assert!(a + b > 0.9 && a + b <= 1.05); + let (a, b) = shade::env_brdf(0.1, 1.0); + assert!(a >= 0.0 && b > -0.01 && a + b < 1.0); +} + +// ---------------------------------------------------------------- rasteriser + +/// A scene of axis-aligned quads (z = camera depth) seen by an orthographic identity camera +/// mapping world x/y straight to raster pixels. +fn quad_scene(w: u32, h: u32, quads: &[([f32; 4], f32, [f32; 4])]) -> Scene { + let mut s = Scene { width: w, height: h, ssaa: 1, linear_io: true, ..Scene::default() }; + // clip: x → 2x/w − 1, y → 1 − 2y/h, z: depth 0..1000 → 1..0 (nearer = larger). + s.clip = [[2.0 / w as f32, 0.0, 0.0, -1.0], [0.0, -2.0 / h as f32, 0.0, 1.0], [0.0, 0.0, -0.001, 1.0], [0.0, 0.0, 0.0, 1.0]]; + s.view = [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]; + s.ortho = true; + s.cam_fwd = [0.0, 0.0, 1.0]; + for (rect, z, color) in quads { + let m = s.materials.len() as u32; + s.materials.push(Material { base: *color, unlit: true, double_sided: true, alpha_mode: if color[3] < 1.0 { 2 } else { 0 }, ..Material::default() }); + let b = s.vertices.len() as u32; + for (x, y) in [(rect[0], rect[1]), (rect[2], rect[1]), (rect[2], rect[3]), (rect[0], rect[3])] { + s.vertices.push(Vertex { pos: [x, y, *z], normal: [0.0, 0.0, -1.0], uv: [0.0; 2], tangent: [1.0, 0.0, 0.0, 1.0], material: m }); + } + s.indices.extend([b, b + 2, b + 1, b, b + 3, b + 2]); + } + // Opaque first, transparent (in given order) after. + let tris: Vec<[u32; 3]> = s.indices.chunks(3).map(|t| [t[0], t[1], t[2]]).collect(); + let (o, t): (Vec<_>, Vec<_>) = tris.into_iter().partition(|t| !s.materials[s.vertices[t[0] as usize].material as usize].transparent()); + s.opaque_count = 3 * o.len() as u32; + s.indices = o.into_iter().chain(t).flatten().collect(); + s +} + +#[test] +fn depth_buffer_keeps_the_nearest_surface() { + let red = [1.0, 0.0, 0.0, 1.0]; + let blue = [0.0, 0.0, 1.0, 1.0]; + // Blue is listed first but is farther away. + let s = quad_scene(40, 30, &[([0.0, 0.0, 30.0, 20.0], 500.0, blue), ([10.0, 10.0, 40.0, 30.0], 100.0, red)]); + let t = raster::render(&s); + let px = |x: usize, y: usize| t.color[y * 40 + x]; + assert_eq!(px(5, 5), blue); + assert_eq!(px(15, 15), red, "overlap: the nearer quad wins"); + assert_eq!(px(35, 25), red); + assert_eq!(px(35, 5), [0.0; 4]); + assert_eq!(t.depth[15 * 40 + 15], 100.0); + assert!(t.depth[5 * 40 + 35].is_infinite()); +} + +#[test] +fn coverage_is_exact_for_pixel_aligned_quads() { + // Shared diagonal edges are filled once (top-left rule): exact pixel counts. + let s = quad_scene(32, 32, &[([4.0, 4.0, 20.0, 12.0], 10.0, [1.0, 1.0, 1.0, 0.5])]); + let t = raster::render(&s); + let covered = t.color.iter().filter(|c| c[3] > 0.0).count(); + assert_eq!(covered, 16 * 8); + // Blended once (no double coverage along the diagonal). + assert!(t.color.iter().all(|c| c[3] == 0.0 || (c[3] - 0.5).abs() < 1e-6)); + // Two transparent layers composite in order. + let s = quad_scene(8, 8, &[([0.0, 0.0, 8.0, 8.0], 100.0, [1.0, 0.0, 0.0, 0.5]), ([0.0, 0.0, 8.0, 8.0], 50.0, [0.0, 0.0, 1.0, 0.5])]); + let c = raster::render(&s).color[0]; + assert!((c[0] - 0.25).abs() < 1e-6 && (c[2] - 0.5).abs() < 1e-6 && (c[3] - 0.75).abs() < 1e-6, "{c:?}"); +} + +#[test] +fn textures_sample_bilinear_with_wrap_modes() { + let t = TexInfo { offset: 0, width: 2, height: 1, wrap_u: 0, wrap_v: 1 }; + let texels = vec![[0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 1.0, 1.0]]; + assert_eq!(shade::sample(&t, &texels, 0.25, 0.5)[0], 0.0); + assert_eq!(shade::sample(&t, &texels, 0.5, 0.5)[0], 0.5); + // Repeat wraps the right edge back to texel 0. + assert!((shade::sample(&t, &texels, 1.0, 0.5)[0] - 0.5).abs() < 1e-6); + let c = TexInfo { wrap_u: 1, ..t }; + assert_eq!(shade::sample(&c, &texels, 1.0, 0.5)[0], 1.0); +} + +// ---------------------------------------------------------------- full renderer + +fn project(w: u32, h: u32) -> (Project, ItemId) { + let mut p = Project::default(); + let mut c = Comp::new(w, h, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + c.renderer = R3::Advanced3D; + let id = p.add_item("C", Default::default(), None, ItemKind::Comp(c.into())); + (p, id) +} + +fn add(p: &mut Project, cid: ItemId, src: LayerSource, edit: impl FnOnce(&mut effectcraft_project::Layer)) { + let comp = p.comp(cid).unwrap().clone(); + let mut l = build::layer(p, &comp, "L", src, (comp.width, comp.height), None); + edit(&mut l); + p.comp_mut(cid).unwrap().layers.insert(0, l); +} + +fn set(l: &mut effectcraft_project::Layer, path: &str, v: Value) { + l.props.prop_mut(path).unwrap_or_else(|| panic!("{path}")).value = v; +} + +fn render(p: &Project, cid: ItemId) -> crate::Image { + Renderer::new(p, &NoFootage, RenderOpts::default()).comp_frame(cid, Tick::ZERO) +} + +#[test] +fn lit_cube_renders_with_depth_and_shading() { + let (mut p, cid) = project(160, 120); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Cube }, |l| { + set(l, "geometryOptions/width", Value::Scalar(60.0)); + set(l, "geometryOptions/height", Value::Scalar(60.0)); + set(l, "geometryOptions/depth", Value::Scalar(60.0)); + set(l, "transform/rotationY", Value::Scalar(35.0)); + set(l, "transform/rotationX", Value::Scalar(-25.0)); + set(l, "materialOptions/baseColor", Value::Color([1.0, 1.0, 1.0, 1.0])); + }); + let img = render(&p, cid); + let centre = img.data[60 * 160 + 80]; + assert!(centre[3] > 0.99, "cube covers the centre: {centre:?}"); + assert_eq!(img.data[2 * 160 + 2][3], 0.0, "corner is empty"); + // No lights: unlit (base colour) — add a light and faces get different shades. + add(&mut p, cid, LayerSource::Light { kind: LightKind::Parallel }, |_| {}); + let lit = render(&p, cid); + let mut shades: Vec = lit.data.iter().filter(|q| q[3] > 0.99).map(|q| (q[0] * 20.0) as i32).collect(); + shades.sort(); + shades.dedup(); + assert!(shades.len() >= 3, "faces are shaded differently: {shades:?}"); + // Classic 3D draws no meshes. + p.comp_mut(cid).unwrap().renderer = R3::Classic3D; + assert!(render(&p, cid).data.iter().all(|q| q[3] == 0.0)); +} + +#[test] +fn shadows_darken_the_receiver() { + let (mut p, cid) = project(200, 150); + // A floor plane facing the camera, a sphere in front of it, a spot light behind the camera. + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Plane }, |l| { + set(l, "geometryOptions/width", Value::Scalar(400.0)); + set(l, "geometryOptions/height", Value::Scalar(400.0)); + set(l, "transform/position", Value::Vec3([100.0, 75.0, 200.0])); + set(l, "materialOptions/baseColor", Value::Color([1.0, 1.0, 1.0, 1.0])); + }); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Sphere }, |l| { + set(l, "geometryOptions/radius", Value::Scalar(30.0)); + set(l, "transform/position", Value::Vec3([100.0, 75.0, 50.0])); + set(l, "materialOptions/castsShadows", Value::Enum(2)); + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Point }, |l| { + set(l, "transform/position", Value::Vec3([100.0, 75.0, -300.0])); + set(l, "lightOptions/castsShadows", Value::Bool(true)); + }); + let img = render(&p, cid); + // The sphere casts only (invisible); its shadow lands on the plane behind it. + let mid = img.data[75 * 200 + 100]; + let side = img.data[75 * 200 + 20]; + assert!(mid[3] > 0.99 && side[3] > 0.99); + assert!(mid[0] < side[0] * 0.5, "shadowed centre {mid:?} vs lit side {side:?}"); + // Without shadow casting the centre is lit. + let l = &mut p.comp_mut(cid).unwrap().layers[0]; + set(l, "lightOptions/castsShadows", Value::Bool(false)); + let img = render(&p, cid); + assert!(img.data[75 * 200 + 100][0] > side[0] * 0.8); +} + +#[test] +fn cards_and_environment_light() { + let (mut p, cid) = project(120, 90); + let sid = p.add_item( + "S", + Default::default(), + None, + ItemKind::Solid(effectcraft_project::Solid { color: [0.5, 0.5, 0.5], width: 60, height: 40, pixel_aspect: 1.0 }), + ); + add(&mut p, cid, LayerSource::Solid { item: sid }, |l| l.switches.three_d = true); + let unlit = render(&p, cid); + let c = unlit.data[20 * 120 + 30]; + assert!((c[0] - 0.5).abs() < 0.01 && c[3] > 0.99, "unlit card keeps its colour: {c:?}"); + assert_eq!(unlit.data[80 * 120 + 110][3], 0.0); + // An Environment light sourcing a white environment layer brightens the card. + let env = p.add_item( + "E", + Default::default(), + None, + ItemKind::Solid(effectcraft_project::Solid { color: [1.0, 1.0, 1.0], width: 64, height: 32, pixel_aspect: 1.0 }), + ); + add(&mut p, cid, LayerSource::Solid { item: env }, |l| { + l.environment = true; + l.switches.three_d = true; + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Environment }, |_| {}); + let lit = render(&p, cid); + let c2 = lit.data[20 * 120 + 30]; + assert!(c2[0] > 0.3, "{c2:?}"); + // The environment layer itself is not drawn. + assert_eq!(lit.data[80 * 120 + 110][3], 0.0); +} + +#[test] +fn depth_of_field_blurs_out_of_focus_surfaces() { + let mut img = Image::new(64, 64); + for y in 0..64 { + for x in 0..64 { + let v = if (x / 4 + y / 4) % 2 == 0 { 1.0 } else { 0.0 }; + img.data[y * 64 + x] = [v, v, v, 1.0]; + } + } + let var = |im: &Image| { + let m: f32 = im.data.iter().map(|p| p[0]).sum::() / im.data.len() as f32; + im.data.iter().map(|p| (p[0] - m).powi(2)).sum::() / im.data.len() as f32 + }; + let dof = crate::three_d::camera::Dof { focus: 1000.0, aperture: 40.0, blur_level: 1.0 }; + let sharp = depth_of_field(&img, &vec![1000.0; 64 * 64], &dof, 1.0); + assert!((var(&sharp) - var(&img)).abs() < 1e-6, "in focus: unchanged"); + let blurred = depth_of_field(&img, &vec![3000.0; 64 * 64], &dof, 1.0); + assert!(var(&blurred) < var(&img) * 0.3, "out of focus: blurred"); +} + +#[test] +fn extruded_text_renders_as_a_mesh() { + let (mut p, cid) = project(200, 100); + add(&mut p, cid, LayerSource::Text, |l| { + l.switches.three_d = true; + let doc = effectcraft_keyframe::TextDoc { text: "IO".into(), size: 60.0, fill: [1.0, 1.0, 1.0, 1.0], apply_fill: true, ..Default::default() }; + set(l, "text/sourceText", Value::Text(Box::new(doc))); + let mut next = 10_000u64; + l.props.children.push(build::extrusion_geometry_options(&mut build::Ids(&mut next)).into()); + set(l, "geometryOptions/extrusionDepth", Value::Scalar(30.0)); + set(l, "geometryOptions/bevelStyle", Value::Enum(1)); + set(l, "transform/rotationY", Value::Scalar(40.0)); + }); + let ctx = crate::EvalCtx { project: &p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let l = &ctx.comp.layers[0]; + let params = scene::extrusion_params(&ctx, l).unwrap(); + let meshes = scene::extruded_meshes(&ctx, l, ¶ms); + assert!(!meshes.is_empty()); + assert!(meshes.iter().all(|(m, _)| m.positions.iter().any(|q| (q[2] - 30.0).abs() < 1e-3))); + let img = render(&p, cid); + assert!(img.data.iter().filter(|q| q[3] > 0.5).count() > 200); +} diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index 1ef7412b..e6fc9209 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -323,6 +323,12 @@ pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut if run.is_empty() { return; } + // Advanced 3D comps: meshes, physically based lights, shadow maps (collapsed precomps keep + // the parent's Classic 3D planes). + if ctx.comp.renderer == effectcraft_project::Renderer::Advanced3D && r.collapse3d.is_none() { + super::adv::draw_run(r, ctx, run, canvas); + return; + } let out = (canvas.width, canvas.height); // A collapsed precomp's 3D layers are lit by the outer comp's lights. let lights = scene_lights(&r.collapse3d.map_or(*ctx, |c| c.parent)); diff --git a/crates/render/src/three_d/light.rs b/crates/render/src/three_d/light.rs index 9476f98d..5a7963b4 100644 --- a/crates/render/src/three_d/light.rs +++ b/crates/render/src/three_d/light.rs @@ -117,7 +117,18 @@ impl LightState { /// All lights switched on and active at the context time. pub fn lights_at(ctx: &EvalCtx) -> Vec { - ctx.comp.layers.iter().filter(|l| l.is_light() && l.switches.video && l.is_active_at(ctx.time)).filter_map(|l| LightState::from_layer(ctx, l)).collect() + // Environment lights only light Advanced 3D comps (image-based light). + ctx.comp + .layers + .iter() + .filter(|l| { + l.is_light() + && l.switches.video + && l.is_active_at(ctx.time) + && l.source != (effectcraft_project::LayerSource::Light { kind: LightKind::Environment }) + }) + .filter_map(|l| LightState::from_layer(ctx, l)) + .collect() } /// Material Options of a 3D layer. diff --git a/crates/render/src/three_d/mod.rs b/crates/render/src/three_d/mod.rs index 8bef7360..e97c67fe 100644 --- a/crates/render/src/three_d/mod.rs +++ b/crates/render/src/three_d/mod.rs @@ -5,6 +5,7 @@ //! consecutive 3D layers is handed to [`compose::draw_run`] as a group (2D layers break 3D //! groups, as in After Effects). +pub mod adv; pub mod camera; pub(crate) mod compose; pub mod light; From 79ad7e184382f3e60d1205ad6d6e980f3f079f60 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:36:59 -0400 Subject: [PATCH 4/4] M7.6: extrusion + primitives Extruded text keeps per-character 3D: each glyph's outline is extruded in its own space and placed by its animated matrix; shape layers extrude their fill outlines (gradients use their mid colour). Shadow-map bias scales with the texel footprint at the receiver and the PCF reach (no acne on curved casters), identically on CPU and GPU. Docs: Advanced 3D in architecture.md, parity table updated (3D, Text). Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/shaders/advanced3d.wgsl | 3 ++- crates/render/src/three_d/adv/raster.rs | 6 ++--- crates/render/src/three_d/adv/scene.rs | 23 ++++++++++--------- crates/render/src/three_d/adv/shade.rs | 4 +++- crates/render/src/three_d/adv/tests.rs | 2 +- docs/architecture.md | 30 ++++++++++++++++++++++++- docs/parity.md | 8 +++---- 7 files changed, 55 insertions(+), 21 deletions(-) diff --git a/crates/gpu/src/shaders/advanced3d.wgsl b/crates/gpu/src/shaders/advanced3d.wgsl index 9ae81ff5..12100cc3 100644 --- a/crates/gpu/src/shaders/advanced3d.wgsl +++ b/crates/gpu/src/shaders/advanced3d.wgsl @@ -197,7 +197,8 @@ fn shadow_factor(m: u32, p: vec3, n_l: f32) -> f32 { v = q.y * focal / q.z + c; } let z = q.z; - let bias = extra.x * (1.0 + 2.0 * (1.0 - clamp(n_l, 0.0, 1.0))) + abs(z) * 0.0015; + let texel = select(abs(z) / focal, 1.0 / focal, ortho); + let bias = extra.x * texel * (1.0 + extra.y) * (1.0 + 3.0 * (1.0 - clamp(n_l, 0.0, 1.0))); let step = extra.y / 2.0; let sz = i32(size); var lit = 0.0; diff --git a/crates/render/src/three_d/adv/raster.rs b/crates/render/src/three_d/adv/raster.rs index 36cec108..972a627b 100644 --- a/crates/render/src/three_d/adv/raster.rs +++ b/crates/render/src/three_d/adv/raster.rs @@ -460,12 +460,12 @@ pub(crate) fn shadow_maps(s: &mut Scene, draft: bool) { 0 => { let eye = [centre[0] - dir[0] * radius * 2.0, centre[1] - dir[1] * radius * 2.0, centre[2] - dir[2] * radius * 2.0]; let focal = (size as f64 * 0.5 / radius) as f32; - maps.push(ShadowMap { view: basis(eye, dir, hint), focal, ortho: true, size, offset: 0, bias: 1.5 / focal, radius: soft }); + maps.push(ShadowMap { view: basis(eye, dir, hint), focal, ortho: true, size, offset: 0, bias: 1.5, radius: soft }); } 1 => { let half = (l.cos_outer.clamp(-1.0, 1.0).acos() as f64 + 0.05).min(1.45); let focal = (size as f64 * 0.5 / half.tan()) as f32; - maps.push(ShadowMap { view: basis(pos, dir, hint), focal, ortho: false, size, offset: 0, bias: 1.0, radius: soft }); + maps.push(ShadowMap { view: basis(pos, dir, hint), focal, ortho: false, size, offset: 0, bias: 1.5, radius: soft }); } _ => { let fs = size / 2; @@ -477,7 +477,7 @@ pub(crate) fn shadow_maps(s: &mut Scene, draft: bool) { ([0.0, 0.0, 1.0], [0.0, 1.0, 0.0]), ([0.0, 0.0, -1.0], [0.0, 1.0, 0.0]), ] { - maps.push(ShadowMap { view: basis(pos, f, d), focal: fs as f32 * 0.5, ortho: false, size: fs, offset: 0, bias: 1.0, radius: soft }); + maps.push(ShadowMap { view: basis(pos, f, d), focal: fs as f32 * 0.5, ortho: false, size: fs, offset: 0, bias: 1.5, radius: soft }); } } } diff --git a/crates/render/src/three_d/adv/scene.rs b/crates/render/src/three_d/adv/scene.rs index 1d3acc3c..14251745 100644 --- a/crates/render/src/three_d/adv/scene.rs +++ b/crates/render/src/three_d/adv/scene.rs @@ -152,7 +152,7 @@ pub struct ShadowMap { pub ortho: bool, pub size: u32, pub offset: u32, - /// Depth bias (world units). + /// Depth bias in texels (scaled by the texel footprint at the receiver). pub bias: f32, /// PCF spread (texels). pub radius: f32, @@ -531,33 +531,35 @@ pub fn extrusion_params(ctx: &EvalCtx, layer: &Layer) -> Option Vec<(Primitive, [f32; 4])> { - let mut groups: Vec<(Vec, [f32; 4])> = vec![]; +/// Extruded meshes of a text or shape layer: (mesh, straight colour, mesh space → layer +/// space). Text extrudes each character in its own space, so per-character 3D animation +/// (position, rotation, scale in Z) carries over. +pub fn extruded_meshes(ctx: &EvalCtx, layer: &Layer, p: &model::extrude::ExtrudeParams) -> Vec<(Primitive, [f32; 4], Mat4)> { + let mut groups: Vec<(Vec, [f32; 4], Mat4)> = vec![]; match layer.source { LayerSource::Text => { if let Some(tg) = crate::text::text_geom(ctx, layer) { for g in tg.glyphs.iter().filter(|g| g.apply_fill && g.fill[3] > 0.0) { - groups.push((vec![g.path()], g.fill)); + groups.push((vec![g.local.clone()], g.fill, g.m)); } } } LayerSource::Shape => { if let Some(c) = layer.props.sub("contents") { - groups = crate::shapes::fill_outlines(ctx, layer, c); + groups = crate::shapes::fill_outlines(ctx, layer, c).into_iter().map(|(ps, c)| (ps, c, Mat4::IDENTITY)).collect(); } } _ => {} } groups .into_iter() - .filter_map(|(paths, color)| { + .filter_map(|(paths, color, m)| { let cs: Vec> = paths.iter().flat_map(|bp| contours(bp, 0.25)).collect(); let outlines = model::extrude::group_contours(cs); if outlines.is_empty() { return None; } - Some((model::extrude::extrude(&outlines, p), color)) + Some((model::extrude::extrude(&outlines, p), color, m)) }) .collect() } @@ -567,7 +569,7 @@ fn extruded_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, p: &model::extrude let opacity = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); let world = ctx.world_matrix(layer); let lin = b.s.linear_io; - for (mesh, c) in extruded_meshes(ctx, layer, p) { + for (mesh, c, local) in extruded_meshes(ctx, layer, p) { let l = |v: f32| if lin { v } else { srgb_to_linear(v) }; let m = b.material(Material { base: [l(c[0]), l(c[1]), l(c[2]), c[3]], @@ -576,7 +578,8 @@ fn extruded_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, p: &model::extrude double_sided: false, ..base }); - b.emit(&mesh, m, &|_| world); + let xf = world * local; + b.emit(&mesh, m, &|_| xf); } } diff --git a/crates/render/src/three_d/adv/shade.rs b/crates/render/src/three_d/adv/shade.rs index a8ab5e70..16d05287 100644 --- a/crates/render/src/three_d/adv/shade.rs +++ b/crates/render/src/three_d/adv/shade.rs @@ -212,7 +212,9 @@ pub fn shadow_factor(m: &ShadowMap, texels: &[f32], p: [f32; 3], n_l: f32) -> f3 let Some((u, v, z)) = shadow_coord(m, p) else { return 1.0 }; let size = m.size as i64; // Slope-scaled bias (world units are pixels). - let bias = m.bias * (1.0 + 2.0 * (1.0 - n_l.clamp(0.0, 1.0))) + z.abs() * 0.0015; + // Texel footprint (world units) × PCF reach, steeper for grazing light. + let texel = if m.ortho { 1.0 / m.focal } else { z.abs() / m.focal }; + let bias = m.bias * texel * (1.0 + m.radius) * (1.0 + 3.0 * (1.0 - n_l.clamp(0.0, 1.0))); let step = m.radius / 2.0; let mut lit = 0.0; for j in -2..=2 { diff --git a/crates/render/src/three_d/adv/tests.rs b/crates/render/src/three_d/adv/tests.rs index 23db2cbc..47ad6178 100644 --- a/crates/render/src/three_d/adv/tests.rs +++ b/crates/render/src/three_d/adv/tests.rs @@ -308,7 +308,7 @@ fn extruded_text_renders_as_a_mesh() { let params = scene::extrusion_params(&ctx, l).unwrap(); let meshes = scene::extruded_meshes(&ctx, l, ¶ms); assert!(!meshes.is_empty()); - assert!(meshes.iter().all(|(m, _)| m.positions.iter().any(|q| (q[2] - 30.0).abs() < 1e-3))); + assert!(meshes.iter().all(|(m, _, _)| m.positions.iter().any(|q| (q[2] - 30.0).abs() < 1e-3))); let img = render(&p, cid); assert!(img.data.iter().filter(|q| q[3] > 0.5).count() > 200); } diff --git a/docs/architecture.md b/docs/architecture.md index 7e2921e0..c9dfc023 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,7 +69,35 @@ the **source** (solid, footage frame, text, shape contents, or a nested comp), a then **effects** in order, then **layer styles**, then the **transform** into comp space (with motion blur sub-samples when enabled), the **track matte**, and finally **blends** into the accumulator. Adjustment layers apply their effects to the accumulator. Runs of 3D layers are composited per pixel through the -active camera, with lights, shadows and depth of field. +active camera, with lights, shadows and depth of field (Classic 3D), or rasterised as meshes by +the Advanced 3D renderer (below). + +**Advanced 3D** (Composition Settings ▸ 3D Renderer ▸ Advanced 3D, `comp.renderer`; +`crates/render/src/three_d/adv`, `crates/model`). A run of 3D layers becomes one scene of +world-space triangles: 3D model layers (glTF 2.0 / OBJ footage, mapped from +Y-up model units +into the layer by Geometry Options ▸ Model Scale; skins and animation clips played along layer +time), primitive layers (Layer ▸ New ▸ Cube/Sphere/Plane/Torus/Cone/Cylinder, parametric in +Geometry Options), text and shape layers with Geometry Options ▸ Extrusion Depth (glyph and +fill outlines flattened, triangulated with holes and extruded with Angular/Concave/Convex +bevels, per character so per-character 3D carries over), and a textured card for every other +3D layer. Shading is physically based in linear light (glTF metallic-roughness; Cook–Torrance +GGX specular, Smith–Schlick geometry, Schlick Fresnel, Lambert diffuse, after Karis 2013): +parallel, spot and point lights with After Effects' falloffs, ambient lights, image-based +light from an Environment light whose Source (or the comp's Environment Layer, Layer ▸ +Environment Layer) is an equirectangular image (cosine irradiance map; roughness-indexed mip +chain with the split-sum environment BRDF fit), and percentage-closer-filtered shadow maps +(spot: perspective, parallel: orthographic over the scene, point: six cube faces). Cards keep +Material Options (Accepts Lights/Shadows, Casts Shadows On/Only, Ambient, Diffuse, Specular); +models and primitives add Base Color, Metallic, Roughness, Emissive. Rasterisation runs at 2×2 +supersampling with a reversed-Z depth buffer; opaque triangles resolve visibility first, the +transparent ones are sorted back to front and blended. The software rasteriser +(`adv::raster`) is the reference and the headless/CI path; `effectcraft-gpu` implements +`Accelerator::raster_3d` with a wgpu render pipeline (`advanced3d.wgsl`) that repeats the +shading step for step, and tests compare the two. Resolve, depth of field (a gather blur by +each pixel's circle of confusion from the camera's Depth of Field settings) and the conversion +back to the working encoding run on the CPU for both. Not yet: motion blur, track mattes and +blend modes inside an Advanced 3D run, collapsed 3D precomps (drawn flattened), morph targets, +extruded strokes. A **layer cache** keeps each layer's finished pixels (source, masks and effects) keyed by a hash of its evaluated inputs, excluding the transform. Static and transform-only layers render once; diff --git a/docs/parity.md b/docs/parity.md index af0c9420..3a1b0932 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -74,7 +74,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, -Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of +the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and wasm threads. @@ -91,9 +91,9 @@ wasm threads. | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | | Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | -| Text | ≈ 85% | 1.5 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no text on 3D bevels; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | +| Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | +| 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | @@ -132,4 +132,4 @@ Mocha and Cineware are third-party and not counted. The engine underneath (keyframes, expressions, shape operators, text animators, Classic 3D, all 38 blend modes, the render queue) is deep. Most of what is missing is interactive tooling in the viewer, settings that are stored but not yet rendered, about 69 menu entries that are still -disabled, and the large systems: tracking, puppet, paint, roto, Advanced 3D and GPU rendering. +disabled, and the large systems: tracking, puppet, paint and roto (Advanced 3D landed in M7.4–M7.6).