Merge branch 'worktree-agent-ad0c2eed1a8890262'
This commit is contained in:
Brandon Thomas
commit
cb64ba6fa9
|
|
@ -1185,6 +1185,7 @@ dependencies = [
|
|||
"effectcraft-raster",
|
||||
"effectcraft-track",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,6 @@ Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 Octobe
|
|||
- **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in
|
||||
parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours**
|
||||
by the audit's conservative per-feature figures.
|
||||
- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model.
|
||||
- Hardest remaining pieces: the 3D camera tracker, Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7).
|
||||
|
||||
Come tell us what matters most to you on [Discord](https://discord.gg/artcraft).
|
||||
|
|
|
|||
|
|
@ -17,5 +17,6 @@ effectcraft-project = { workspace = true }
|
|||
effectcraft-raster = { workspace = true }
|
||||
effectcraft-track = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ pub mod paint;
|
|||
mod perspective;
|
||||
mod perspective2;
|
||||
pub mod puppet;
|
||||
pub mod roto;
|
||||
mod sim;
|
||||
mod sim2;
|
||||
mod sim3;
|
||||
|
|
@ -381,6 +382,7 @@ pub fn registry() -> &'static [EffectSpec] {
|
|||
v.extend(paint::specs());
|
||||
v.extend(puppet::specs());
|
||||
v.extend(warp_stab::specs());
|
||||
v.extend(roto::specs());
|
||||
v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name)));
|
||||
for s in v.iter_mut() {
|
||||
if GPU_EFFECTS.contains(&s.id) {
|
||||
|
|
@ -456,7 +458,7 @@ pub fn instantiate(spec: &EffectSpec, ids: &mut Ids, instance_name: &str, layer_
|
|||
|
||||
/// Display names of nested parameter groups by match id.
|
||||
fn group_name(m: &str) -> &str {
|
||||
warp_stab::GROUPS.iter().find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m)
|
||||
warp_stab::GROUPS.iter().chain(roto::GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m)
|
||||
}
|
||||
|
||||
/// The nested group at `path` (`borders/autoScale`) under `g`, created on first use.
|
||||
|
|
@ -528,6 +530,8 @@ pub const TIME_DEPENDENT: &[&str] = &[
|
|||
"ec.obsolete.mochashape",
|
||||
// Each frame gets its own stabilizing warp.
|
||||
warp_stab::ID,
|
||||
// Each frame has its own segmentation.
|
||||
roto::ID,
|
||||
];
|
||||
|
||||
/// See [`TIME_DEPENDENT`].
|
||||
|
|
|
|||
|
|
@ -0,0 +1,698 @@
|
|||
//! Roto Brush & Refine Edge (applied by the Roto Brush tool; Effect ▸ Matte).
|
||||
//!
|
||||
//! The instance stores the user's strokes, base frame and segmentation span as JSON in its hidden
|
||||
//! **Strokes** parameter ([`RotoData`]); the segmentation itself is *derived* data: each frame's
|
||||
//! result is a deterministic function of its chain key ([`RotoData::chain_keys`] seeded with
|
||||
//! [`seed`]: what the frames are, the Roto Brush settings and the strokes from the base frame
|
||||
//! out to the frame), so it lives in a process-wide content-keyed cache ([`cached`]) that the
|
||||
//! engine's propagation job fills at full resolution and rendering fills on demand (reading
|
||||
//! neighbouring frames with [`crate::EffectHost::self_at`]). Undo, redo and reopening a project
|
||||
//! therefore never see stale mattes. **Freeze** stores the final mattes in the hidden **Frozen**
|
||||
//! parameter ([`FrozenCache`]); they are used while their key still matches.
|
||||
//!
|
||||
//! Rendering: binary segmentation → Roto Brush Matte (Shift Edge, Reduce Chatter, Feather,
|
||||
//! Contrast) → Refine Edge matting inside the Refine Edge band (guided filter + closed-form
|
||||
//! matting with Higher Quality) → Refine Edge Matte adjustments → matte motion blur →
|
||||
//! decontamination of the edge colours. Outside the segmentation span the layer is transparent;
|
||||
//! without strokes the effect passes its input through.
|
||||
//!
|
||||
//! *Version* is stored but all versions use the same (classic, graph-cut) engine; *Quality ▸
|
||||
//! Best* segments at a higher working resolution with more iterations.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap, VecDeque};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::ParamUi;
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_track::roto::refine::{self, MatteParams};
|
||||
use effectcraft_track::roto::{self as rb, FrameSeg, RotoData, SegOpts, rle};
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::warp_stab::param;
|
||||
use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider};
|
||||
|
||||
pub use effectcraft_track::roto::{Stroke, StrokeKind};
|
||||
|
||||
/// The effect's id.
|
||||
pub const ID: &str = "ec.matte.rotobrush";
|
||||
/// Hidden parameters.
|
||||
pub const STROKES: &str = "strokes";
|
||||
pub const INPUT_KEY: &str = "inputKey";
|
||||
pub const FROZEN: &str = "frozen";
|
||||
|
||||
/// Default segmentation span each side of the base frame (frames).
|
||||
pub const DEFAULT_SPAN: i64 = 20;
|
||||
|
||||
/// Display names of the instance's parameter groups.
|
||||
pub const GROUPS: &[(&str, &str)] = &[("rotoBrushMatte", "Roto Brush Matte"), ("refineEdgeMatte", "Refine Edge Matte")];
|
||||
|
||||
pub const VERSIONS: [&str; 3] = ["1.0", "2.0", "3.0"];
|
||||
pub const QUALITIES: [&str; 2] = ["Standard", "Best"];
|
||||
|
||||
pub fn specs() -> Vec<EffectSpec> {
|
||||
vec![EffectSpec {
|
||||
id: ID,
|
||||
name: "Roto Brush & Refine Edge",
|
||||
category: "Matte",
|
||||
params: vec![
|
||||
p("version", "Version", Value::Enum(2), popup(&VERSIONS)),
|
||||
p("quality", "Quality", Value::Enum(0), popup(&QUALITIES)),
|
||||
p("rotoBrushMatte/searchRadius", "Search Radius", num(15.0), slider(0.0, 100.0, 0.0, 50.0, 1)),
|
||||
p("rotoBrushMatte/motionThreshold", "Motion Threshold", num(10.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("rotoBrushMatte/motionDamping", "Motion Damping", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("rotoBrushMatte/viewSearchRegion", "View Search Region", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("rotoBrushMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)),
|
||||
p("rotoBrushMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("rotoBrushMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)),
|
||||
p("rotoBrushMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("refineEdgeMatte/smooth", "Smooth", num(1.0), slider(0.0, 100.0, 0.0, 10.0, 1)),
|
||||
p("refineEdgeMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)),
|
||||
p("refineEdgeMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("refineEdgeMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)),
|
||||
p("refineEdgeMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("refineEdgeMatte/useMotionBlur", "Use Motion Blur", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("refineEdgeMatte/motionBlurSamples", "Motion Blur Samples", num(11.0), slider(1.0, 64.0, 1.0, 32.0, 0)),
|
||||
p("refineEdgeMatte/shutterAngle", "Shutter Angle", num(180.0), slider(0.0, 720.0, 0.0, 360.0, 0)),
|
||||
p("refineEdgeMatte/higherQuality", "Higher Quality", Value::Bool(true), ParamUi::Checkbox),
|
||||
p("refineEdgeMatte/decontaminateEdgeColors", "Decontaminate Edge Colors", Value::Bool(true), ParamUi::Checkbox),
|
||||
p("refineEdgeMatte/decontaminationAmount", "Decontamination Amount", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("refineEdgeMatte/extendWhereSmoothed", "Extend Where Smoothed", Value::Bool(true), ParamUi::Checkbox),
|
||||
p("refineEdgeMatte/increaseDecontaminationRadius", "Increase Decontamination Radius", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)),
|
||||
p("refineEdgeMatte/viewDecontaminationMap", "View Decontamination Map", Value::Bool(false), ParamUi::Checkbox),
|
||||
p(STROKES, "Strokes", Value::Str(String::new()), ParamUi::Hidden),
|
||||
p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden),
|
||||
p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden),
|
||||
],
|
||||
render,
|
||||
gpu: false,
|
||||
float: true,
|
||||
}]
|
||||
}
|
||||
|
||||
fn f(params: &Params, id: &str) -> f64 {
|
||||
param(params, id).map(Value::as_f64).unwrap_or(0.0)
|
||||
}
|
||||
fn e(params: &Params, id: &str) -> u32 {
|
||||
param(params, id).map(Value::as_enum).unwrap_or(0)
|
||||
}
|
||||
fn b(params: &Params, id: &str) -> bool {
|
||||
param(params, id).map(Value::as_bool).unwrap_or(false)
|
||||
}
|
||||
fn s<'a>(params: &'a Params, id: &str) -> &'a str {
|
||||
match param(params, id) {
|
||||
Some(Value::Str(s)) => s,
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
fn fnv(h: &mut u64, bytes: &[u8]) {
|
||||
for b in bytes {
|
||||
*h ^= *b as u64;
|
||||
*h = h.wrapping_mul(0x0100_0000_01b3);
|
||||
}
|
||||
}
|
||||
|
||||
/// The instance's strokes (parsed once per content).
|
||||
pub fn data(params: &Params) -> Arc<RotoData> {
|
||||
static C: OnceLock<Mutex<HashMap<u64, Arc<RotoData>>>> = OnceLock::new();
|
||||
let j = s(params, STROKES);
|
||||
let mut h = 0xcbf2_9ce4_8422_2325;
|
||||
fnv(&mut h, j.as_bytes());
|
||||
let c = C.get_or_init(Default::default);
|
||||
if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) {
|
||||
return d.clone();
|
||||
}
|
||||
let d = Arc::new(RotoData::from_json(j));
|
||||
let mut m = c.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if m.len() > 64 {
|
||||
m.clear();
|
||||
}
|
||||
m.insert(h, d.clone());
|
||||
d
|
||||
}
|
||||
|
||||
/// Segmentation settings at `scale` (pixels per layer pixel).
|
||||
pub fn seg_opts(params: &Params, scale: f64) -> SegOpts {
|
||||
SegOpts {
|
||||
scale,
|
||||
search_radius: f(params, "rotoBrushMatte/searchRadius").max(0.0),
|
||||
motion_threshold: f(params, "rotoBrushMatte/motionThreshold"),
|
||||
motion_damping: f(params, "rotoBrushMatte/motionDamping"),
|
||||
best: e(params, "quality") == 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Matte settings.
|
||||
pub fn matte_params(params: &Params) -> MatteParams {
|
||||
MatteParams {
|
||||
rb_feather: f(params, "rotoBrushMatte/feather").max(0.0),
|
||||
rb_contrast: f(params, "rotoBrushMatte/contrast"),
|
||||
rb_shift: f(params, "rotoBrushMatte/shiftEdge"),
|
||||
rb_chatter: f(params, "rotoBrushMatte/reduceChatter"),
|
||||
smooth: f(params, "refineEdgeMatte/smooth").max(0.0),
|
||||
feather: f(params, "refineEdgeMatte/feather").max(0.0),
|
||||
contrast: f(params, "refineEdgeMatte/contrast"),
|
||||
shift: f(params, "refineEdgeMatte/shiftEdge"),
|
||||
chatter: f(params, "refineEdgeMatte/reduceChatter"),
|
||||
motion_blur: b(params, "refineEdgeMatte/useMotionBlur"),
|
||||
mb_samples: f(params, "refineEdgeMatte/motionBlurSamples").round().clamp(1.0, 64.0) as usize,
|
||||
shutter_angle: f(params, "refineEdgeMatte/shutterAngle"),
|
||||
higher_quality: b(params, "refineEdgeMatte/higherQuality"),
|
||||
decontaminate: b(params, "refineEdgeMatte/decontaminateEdgeColors"),
|
||||
decontamination: f(params, "refineEdgeMatte/decontaminationAmount"),
|
||||
extend_where_smoothed: b(params, "refineEdgeMatte/extendWhereSmoothed"),
|
||||
increase_radius: f(params, "refineEdgeMatte/increaseDecontaminationRadius").max(0.0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed of the chain keys: what the frames are (the engine-maintained Input Key), the layer
|
||||
/// size, the frame rate and the segmentation settings.
|
||||
pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 {
|
||||
let mut h = 0xcbf2_9ce4_8422_2325;
|
||||
fnv(&mut h, s(params, INPUT_KEY).as_bytes());
|
||||
for v in [
|
||||
layer_size[0],
|
||||
layer_size[1],
|
||||
fps,
|
||||
f(params, "rotoBrushMatte/searchRadius"),
|
||||
f(params, "rotoBrushMatte/motionThreshold"),
|
||||
f(params, "rotoBrushMatte/motionDamping"),
|
||||
] {
|
||||
fnv(&mut h, &v.to_bits().to_le_bytes());
|
||||
}
|
||||
fnv(&mut h, &[e(params, "quality") as u8, e(params, "version") as u8]);
|
||||
h
|
||||
}
|
||||
|
||||
/// The layer frame shown at layer time `t` seconds.
|
||||
pub fn frame_of(t: f64, fps: f64) -> i64 {
|
||||
(t * fps + 1e-6).floor() as i64
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ derived-data cache
|
||||
|
||||
struct SegCache {
|
||||
map: HashMap<(u64, u64), Arc<FrameSeg>>,
|
||||
order: VecDeque<(u64, u64)>,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
const CACHE_BYTES: usize = 768 << 20;
|
||||
|
||||
fn cache() -> &'static Mutex<SegCache> {
|
||||
static C: OnceLock<Mutex<SegCache>> = OnceLock::new();
|
||||
C.get_or_init(|| Mutex::new(SegCache { map: HashMap::new(), order: VecDeque::new(), bytes: 0 }))
|
||||
}
|
||||
|
||||
/// A cached segmentation (chain key, scale).
|
||||
pub fn cached(key: u64, scale: f64) -> Option<Arc<FrameSeg>> {
|
||||
cache().lock().unwrap_or_else(|e| e.into_inner()).map.get(&(key, scale.to_bits())).cloned()
|
||||
}
|
||||
|
||||
pub fn store(key: u64, scale: f64, seg: Arc<FrameSeg>) {
|
||||
let mut c = cache().lock().unwrap_or_else(|e| e.into_inner());
|
||||
let k = (key, scale.to_bits());
|
||||
if c.map.contains_key(&k) {
|
||||
return;
|
||||
}
|
||||
c.bytes += seg.matte.len() + seg.refine.len();
|
||||
c.map.insert(k, seg);
|
||||
c.order.push_back(k);
|
||||
while c.bytes > CACHE_BYTES {
|
||||
let Some(old) = c.order.pop_front() else { break };
|
||||
if let Some(s) = c.map.remove(&old) {
|
||||
c.bytes -= s.matte.len() + s.refine.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the cached segmentations of these chain keys (every scale).
|
||||
pub fn forget(keys: impl IntoIterator<Item = u64>) {
|
||||
let keys: std::collections::HashSet<u64> = keys.into_iter().collect();
|
||||
let mut c = cache().lock().unwrap_or_else(|e| e.into_inner());
|
||||
let gone: Vec<(u64, u64)> = c.map.keys().filter(|k| keys.contains(&k.0)).copied().collect();
|
||||
for k in gone {
|
||||
if let Some(s) = c.map.remove(&k) {
|
||||
c.bytes -= s.matte.len() + s.refine.len();
|
||||
}
|
||||
}
|
||||
c.order.retain(|k| !keys.contains(&k.0));
|
||||
}
|
||||
|
||||
/// Drop every cached segmentation (Edit ▸ Purge).
|
||||
pub fn purge() {
|
||||
let mut c = cache().lock().unwrap_or_else(|e| e.into_inner());
|
||||
c.map.clear();
|
||||
c.order.clear();
|
||||
c.bytes = 0;
|
||||
}
|
||||
|
||||
/// The layer-space grid (`round(size · scale)` pixels) of a buffer.
|
||||
pub fn layer_grid(buf: &Buf, layer_size: [f64; 2]) -> Image {
|
||||
let s = if buf.scale > 0.0 { buf.scale } else { 1.0 };
|
||||
let (w, h) = (((layer_size[0] * s).round() as u32).max(1), ((layer_size[1] * s).round() as u32).max(1));
|
||||
let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64);
|
||||
let mut img = Image::new(w, h);
|
||||
img.rows_mut().for_each(|(y, row)| {
|
||||
for (x, px) in row.iter_mut().enumerate() {
|
||||
*px = buf.img.get(x as i64 + ox, y as i64 + oy);
|
||||
}
|
||||
});
|
||||
img
|
||||
}
|
||||
|
||||
/// Everything needed to compute an instance's segmentations.
|
||||
pub struct Chain {
|
||||
pub data: Arc<RotoData>,
|
||||
pub keys: BTreeMap<i64, u64>,
|
||||
pub opts: SegOpts,
|
||||
pub refine_radius: f64,
|
||||
}
|
||||
|
||||
impl Chain {
|
||||
pub fn new(params: &Params, layer_size: [f64; 2], fps: f64, scale: f64) -> Chain {
|
||||
let data = data(params);
|
||||
let keys = data.chain_keys(seed(params, layer_size, fps));
|
||||
Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale) }
|
||||
}
|
||||
|
||||
/// Frame `f`'s segmentation at this chain's scale: cached, or computed from the nearest
|
||||
/// cached frame towards the base (full-resolution results serve smaller scales).
|
||||
/// `frame(g)`: the effect's input on frame g on the layer grid at this scale.
|
||||
pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>, cancel: &dyn Fn() -> bool) -> Option<Arc<FrameSeg>> {
|
||||
let sc = self.opts.scale;
|
||||
let lookup = |g: i64| -> Option<Arc<FrameSeg>> {
|
||||
let k = *self.keys.get(&g)?;
|
||||
if let Some(s) = cached(k, sc) {
|
||||
return Some(s);
|
||||
}
|
||||
if sc != 1.0 {
|
||||
let full = cached(k, 1.0)?;
|
||||
let s = Arc::new(downscale(&full, sc));
|
||||
store(k, sc, s.clone());
|
||||
return Some(s);
|
||||
}
|
||||
None
|
||||
};
|
||||
if let Some(s) = lookup(f) {
|
||||
return Some(s);
|
||||
}
|
||||
let base = self.data.base?;
|
||||
if !self.data.in_span(f) {
|
||||
return None;
|
||||
}
|
||||
// Walk towards the base to the nearest available frame.
|
||||
let mut path = vec![f];
|
||||
let mut g = f;
|
||||
let mut start: Option<Arc<FrameSeg>> = None;
|
||||
while let Some(src) = self.data.source_of(g) {
|
||||
if let Some(s) = lookup(src) {
|
||||
start = Some(s);
|
||||
break;
|
||||
}
|
||||
path.push(src);
|
||||
g = src;
|
||||
}
|
||||
path.reverse();
|
||||
let mut prev_img: Option<Arc<Image>> = None;
|
||||
let mut cur = start;
|
||||
for g in path {
|
||||
if cancel() {
|
||||
return None;
|
||||
}
|
||||
let img = frame(g)?;
|
||||
let strokes = self.data.strokes_at(g);
|
||||
let seg = match (&cur, g == base) {
|
||||
(_, true) | (None, _) => rb::segment(&img, &strokes, None, &self.opts, self.refine_radius),
|
||||
(Some(p), false) => {
|
||||
let src = self.data.source_of(g)?;
|
||||
let pimg = match prev_img.take() {
|
||||
Some(i) => i,
|
||||
None => frame(src)?,
|
||||
};
|
||||
if pimg.width != img.width || pimg.height != img.height {
|
||||
return None;
|
||||
}
|
||||
rb::propagate(&pimg, p, &img, &strokes, &self.opts, self.refine_radius)
|
||||
}
|
||||
};
|
||||
let seg = Arc::new(seg);
|
||||
store(*self.keys.get(&g)?, sc, seg.clone());
|
||||
cur = Some(seg);
|
||||
prev_img = Some(img);
|
||||
}
|
||||
cur
|
||||
}
|
||||
|
||||
/// Whether frame `f` is computed (at full resolution or at `scale`).
|
||||
pub fn is_cached(&self, f: i64, scale: f64) -> bool {
|
||||
self.keys.get(&f).is_some_and(|k| cached(*k, 1.0).is_some() || cached(*k, scale).is_some())
|
||||
}
|
||||
}
|
||||
|
||||
/// Nearest-neighbour resampling of a segmentation to another scale.
|
||||
pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg {
|
||||
let (w, h) = (((s.w as f64) * scale).round().max(1.0) as usize, ((s.h as f64) * scale).round().max(1.0) as usize);
|
||||
resample(s, w, h)
|
||||
}
|
||||
|
||||
fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {
|
||||
if s.w == w && s.h == h {
|
||||
return s.clone();
|
||||
}
|
||||
let mut out = FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], fg: s.fg.clone(), bg: s.bg.clone() };
|
||||
if s.w == 0 || s.h == 0 {
|
||||
return out;
|
||||
}
|
||||
for y in 0..h {
|
||||
let sy = (((y as f64 + 0.5) * s.h as f64 / h as f64) as usize).min(s.h - 1);
|
||||
for x in 0..w {
|
||||
let sx = (((x as f64 + 0.5) * s.w as f64 / w as f64) as usize).min(s.w - 1);
|
||||
out.matte[y * w + x] = s.matte[sy * s.w + sx];
|
||||
out.refine[y * w + x] = s.refine.get(sy * s.w + sx).copied().unwrap_or(0);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The final matte of frame `seg` (with its neighbours), on its own grid.
|
||||
pub fn final_alpha(
|
||||
img: &Image,
|
||||
seg: &FrameSeg,
|
||||
prev: Option<&FrameSeg>,
|
||||
next: Option<&FrameSeg>,
|
||||
mp: &MatteParams,
|
||||
refine_radius: f64,
|
||||
scale: f64,
|
||||
) -> Vec<f32> {
|
||||
let rgb = rb::rgb_of(img);
|
||||
refine::matte_alpha(&rgb, seg, prev, next, mp, refine_radius * scale, scale)
|
||||
}
|
||||
|
||||
fn needs_neighbours(mp: &MatteParams) -> bool {
|
||||
mp.rb_chatter > 0.0 || mp.chatter > 0.0 || mp.motion_blur
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Freeze
|
||||
|
||||
/// Frozen mattes (Freeze): the final 8-bit alpha and the Refine Edge band of every frame of
|
||||
/// the span at full resolution, valid while `key` matches [`frozen_key`].
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct FrozenCache {
|
||||
pub key: String,
|
||||
pub size: [usize; 2],
|
||||
pub frames: BTreeMap<i64, FrozenFrame>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct FrozenFrame {
|
||||
/// Alpha (RLE, base64).
|
||||
pub a: String,
|
||||
/// Refine Edge band (RLE, base64).
|
||||
pub b: String,
|
||||
}
|
||||
|
||||
impl FrozenCache {
|
||||
pub fn from_json(s: &str) -> Option<FrozenCache> {
|
||||
if s.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
serde_json::from_str(s).ok()
|
||||
}
|
||||
pub fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// What frozen mattes depend on: every chain key and the matte settings.
|
||||
pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64) -> String {
|
||||
let ch = Chain::new(params, layer_size, fps, 1.0);
|
||||
let mut h = 0xcbf2_9ce4_8422_2325;
|
||||
for (f, k) in &ch.keys {
|
||||
fnv(&mut h, &f.to_le_bytes());
|
||||
fnv(&mut h, &k.to_le_bytes());
|
||||
}
|
||||
fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes());
|
||||
format!("{h:016x}")
|
||||
}
|
||||
|
||||
/// Whether the instance is frozen (has frozen mattes).
|
||||
pub fn is_frozen(params: &Params) -> bool {
|
||||
!s(params, FROZEN).is_empty()
|
||||
}
|
||||
|
||||
type FrozenParsed = Mutex<HashMap<u64, Arc<FrozenCache>>>;
|
||||
|
||||
fn frozen(params: &Params) -> Option<Arc<FrozenCache>> {
|
||||
static C: OnceLock<FrozenParsed> = OnceLock::new();
|
||||
let j = s(params, FROZEN);
|
||||
if j.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut h = 0xcbf2_9ce4_8422_2325;
|
||||
fnv(&mut h, j.as_bytes());
|
||||
let c = C.get_or_init(Default::default);
|
||||
if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) {
|
||||
return Some(d.clone());
|
||||
}
|
||||
let d = Arc::new(FrozenCache::from_json(j)?);
|
||||
let mut m = c.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if m.len() > 8 {
|
||||
m.clear();
|
||||
}
|
||||
m.insert(h, d.clone());
|
||||
Some(d)
|
||||
}
|
||||
|
||||
/// Encode one frame for [`FrozenCache`].
|
||||
pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame {
|
||||
let a: Vec<u8> = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect();
|
||||
FrozenFrame { a: rle::encode(&a), b: rle::encode(band) }
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ render
|
||||
|
||||
/// A frame's matte (alpha and band) on the layer grid of `img` at `scale`.
|
||||
fn matte_for(ctx: &EffectCtx, f: i64, img: &Arc<Image>, scale: f64) -> Option<(Vec<f32>, Vec<u8>, f64)> {
|
||||
let (w, h) = (img.width as usize, img.height as usize);
|
||||
// Frozen mattes.
|
||||
if let Some(fc) = frozen(ctx.params)
|
||||
&& fc.key == frozen_key(ctx.params, ctx.layer_size, ctx.fps())
|
||||
&& let Some(fr) = fc.frames.get(&f)
|
||||
{
|
||||
let a = rle::decode(&fr.a)?;
|
||||
let bnd = rle::decode(&fr.b)?;
|
||||
let [fw, fh] = fc.size;
|
||||
if a.len() == fw * fh && bnd.len() == fw * fh {
|
||||
let af: Vec<f32> = a.iter().map(|v| *v as f32 / 255.0).collect();
|
||||
let (alpha, band) = if fw == w && fh == h {
|
||||
(af, bnd)
|
||||
} else {
|
||||
let mut al = vec![0.0f32; w * h];
|
||||
let mut bd = vec![0u8; w * h];
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64);
|
||||
al[y * w + x] = rb::matting::sample(&af, fw, fh, sx, sy);
|
||||
bd[y * w + x] = bnd[(sy as usize).min(fh - 1) * fw + (sx as usize).min(fw - 1)];
|
||||
}
|
||||
}
|
||||
(al, bd)
|
||||
};
|
||||
let rr = data(ctx.params).refine_radius();
|
||||
return Some((alpha, band, rr));
|
||||
}
|
||||
}
|
||||
let host = ctx.env.host;
|
||||
let chain = Chain::new(ctx.params, ctx.layer_size, ctx.fps(), scale);
|
||||
let fps = ctx.fps();
|
||||
let effects = ctx.env.effect_index;
|
||||
let layer_size = ctx.layer_size;
|
||||
let me = img.clone();
|
||||
let mut frame = |g: i64| -> Option<Arc<Image>> {
|
||||
if g == f {
|
||||
return Some(me.clone());
|
||||
}
|
||||
let buf = host?.self_at(g as f64 / fps, effects)?;
|
||||
let im = layer_grid(&buf, layer_size);
|
||||
(im.width as usize == w && im.height as usize == h).then(|| Arc::new(im))
|
||||
};
|
||||
let seg = chain.ensure(f, &mut frame, &|| false)?;
|
||||
let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { resample(&seg, w, h) };
|
||||
let mp = matte_params(ctx.params);
|
||||
let (mut prev, mut next) = (None, None);
|
||||
if needs_neighbours(&mp) {
|
||||
for (slot, g) in [(&mut prev, f - 1), (&mut next, f + 1)] {
|
||||
if chain.data.in_span(g)
|
||||
&& let Some(s) = chain.ensure(g, &mut frame, &|| false)
|
||||
{
|
||||
*slot = Some(if s.w == w && s.h == h { (*s).clone() } else { resample(&s, w, h) });
|
||||
}
|
||||
}
|
||||
}
|
||||
let alpha = final_alpha(img, &seg, prev.as_ref(), next.as_ref(), &mp, chain.refine_radius, scale);
|
||||
Some((alpha, seg.refine, chain.refine_radius))
|
||||
}
|
||||
|
||||
fn render(ctx: &EffectCtx, buf: Buf) -> Buf {
|
||||
let d = data(ctx.params);
|
||||
if d.is_empty() || ctx.adjustment {
|
||||
return buf;
|
||||
}
|
||||
let f = frame_of(ctx.time, ctx.fps());
|
||||
let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 };
|
||||
let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale };
|
||||
if !d.in_span(f) {
|
||||
return out;
|
||||
}
|
||||
let img = Arc::new(layer_grid(&buf, ctx.layer_size));
|
||||
let Some((alpha, band, rr)) = matte_for(ctx, f, &img, sc) else { return out };
|
||||
let (w, h) = (img.width as usize, img.height as usize);
|
||||
let mp = matte_params(ctx.params);
|
||||
let mut rgb = rb::rgb_of(&img);
|
||||
let map = refine::decontaminate(&mut rgb, &alpha, &band, w, h, &mp, rr * sc, sc);
|
||||
let view_map = b(ctx.params, "refineEdgeMatte/viewDecontaminationMap");
|
||||
let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let (bx, by) = (x as i64 + ox, y as i64 + oy);
|
||||
if bx < 0 || by < 0 || bx >= out.img.width as i64 || by >= out.img.height as i64 {
|
||||
continue;
|
||||
}
|
||||
let i = y * w + x;
|
||||
let a_in = img.data[i][3];
|
||||
let px = if view_map {
|
||||
let v = map[i];
|
||||
[v, v, v, 1.0]
|
||||
} else {
|
||||
let a = alpha[i] * a_in;
|
||||
[rgb[i][0] * a, rgb[i][1] * a, rgb[i][2] * a, a]
|
||||
};
|
||||
out.img.set(bx as u32, by as u32, px);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The final matte of the effect at layer frame `f` on the full-resolution layer grid of
|
||||
/// `input` (the effect's input buffer at scale 1), computing what is missing. For Freeze and
|
||||
/// the Layer panel overlays.
|
||||
pub fn frame_matte(
|
||||
params: &Params,
|
||||
layer_size: [f64; 2],
|
||||
fps: f64,
|
||||
f: i64,
|
||||
frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>,
|
||||
cancel: &dyn Fn() -> bool,
|
||||
) -> Option<(Vec<f32>, Vec<u8>)> {
|
||||
let chain = Chain::new(params, layer_size, fps, 1.0);
|
||||
let img = frame(f)?;
|
||||
let seg = chain.ensure(f, frame, cancel)?;
|
||||
let mp = matte_params(params);
|
||||
let (mut prev, mut next) = (None, None);
|
||||
if needs_neighbours(&mp) {
|
||||
if chain.data.in_span(f - 1) {
|
||||
prev = chain.ensure(f - 1, frame, cancel);
|
||||
}
|
||||
if chain.data.in_span(f + 1) {
|
||||
next = chain.ensure(f + 1, frame, cancel);
|
||||
}
|
||||
}
|
||||
let alpha = final_alpha(&img, &seg, prev.as_deref(), next.as_deref(), &mp, chain.refine_radius, 1.0);
|
||||
Some((alpha, seg.refine.clone()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{EffectEnv, EffectHost, LayerPixels};
|
||||
|
||||
/// A host serving a moving red disk over a blue/green background as the effect's input.
|
||||
struct Seq;
|
||||
fn frame(t: f64) -> Image {
|
||||
let fi = (t * 25.0 + 1e-6).floor();
|
||||
let c = [40.0 + 2.0 * fi, 40.0];
|
||||
let mut img = Image::new(100, 80);
|
||||
for y in 0..80 {
|
||||
for x in 0..100 {
|
||||
let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0;
|
||||
let a = (0.5 - d).clamp(0.0, 1.0) as f32;
|
||||
let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0;
|
||||
let fg = [0.85 + tex, 0.2, 0.1];
|
||||
let bg = [0.1, 0.4 + tex, 0.7];
|
||||
img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
}
|
||||
impl EffectHost for Seq {
|
||||
fn layer(&self, _: u64, _: bool) -> Option<LayerPixels> {
|
||||
None
|
||||
}
|
||||
fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option<Vec<f32>> {
|
||||
None
|
||||
}
|
||||
fn self_at(&self, t: f64, _: usize) -> Option<Buf> {
|
||||
Some(Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 })
|
||||
}
|
||||
}
|
||||
|
||||
fn params_with(d: &RotoData) -> Params {
|
||||
let s = &specs()[0];
|
||||
let mut params = Params { values: s.params.iter().map(|p| (p.id.to_string(), p.default.clone())).collect() };
|
||||
params.values.insert(STROKES.into(), Value::Str(d.to_json()));
|
||||
params
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_the_segmented_disk_on_demand() {
|
||||
let mut d = RotoData::default();
|
||||
d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0], [45.0, 40.0]] }, 20, [0, 24]);
|
||||
d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[5.0, 5.0], [95.0, 5.0], [95.0, 75.0]] }, 20, [0, 24]);
|
||||
let params = params_with(&d);
|
||||
let host = Seq;
|
||||
let run = |t: f64| {
|
||||
let ctx = EffectCtx {
|
||||
params: ¶ms,
|
||||
time: t,
|
||||
layer_size: [100.0, 80.0],
|
||||
seed: 1,
|
||||
adjustment: false,
|
||||
env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() },
|
||||
};
|
||||
crate::apply(&specs()[0], &ctx, Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 }).img
|
||||
};
|
||||
// Frame 5 (propagated through frames 1–4).
|
||||
let out = run(5.0 / 25.0 + 0.001);
|
||||
let c = [50.0, 40.0];
|
||||
let mut wrong = 0;
|
||||
for y in 0..80u32 {
|
||||
for x in 0..100u32 {
|
||||
let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]);
|
||||
let a = out.data[(y * 100 + x) as usize][3];
|
||||
if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) {
|
||||
wrong += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(wrong < 20, "{wrong} wrong pixels");
|
||||
// Outside the span: transparent.
|
||||
let mut d2 = (*data(¶ms)).clone();
|
||||
d2.span = [0, 3];
|
||||
let p2 = params_with(&d2);
|
||||
let ctx = EffectCtx {
|
||||
params: &p2,
|
||||
time: 0.3,
|
||||
layer_size: [100.0, 80.0],
|
||||
seed: 1,
|
||||
adjustment: false,
|
||||
env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() },
|
||||
};
|
||||
let o = crate::apply(&specs()[0], &ctx, Buf { img: frame(0.3), offset: [0.0; 2], scale: 1.0 }).img;
|
||||
assert!(o.data.iter().all(|p| p[3] == 0.0));
|
||||
}
|
||||
}
|
||||
|
|
@ -525,6 +525,9 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
s.snapshot = None;
|
||||
s.state.viewer.show_snapshot = false;
|
||||
}
|
||||
if matches!(what.as_str(), "all" | "image" | "memory" | "memoryAndDisk") {
|
||||
effectcraft_effects::roto::purge();
|
||||
}
|
||||
s.events.push(crate::Event::PurgeCaches);
|
||||
s.toast(format!("Purged {what} cache"));
|
||||
Ok(json!({"purged": what}))
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ mod prop_groups;
|
|||
pub mod puppet;
|
||||
mod query;
|
||||
mod render_queue;
|
||||
pub mod roto_cmds;
|
||||
mod settings;
|
||||
mod stubs;
|
||||
mod styles;
|
||||
|
|
@ -134,6 +135,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
|
|||
v.extend(track::specs());
|
||||
v.extend(mask_interp::specs());
|
||||
v.extend(warp_cmds::specs());
|
||||
v.extend(roto_cmds::specs());
|
||||
v.extend(paint::specs());
|
||||
v.extend(puppet::specs());
|
||||
v.extend(liquify::specs());
|
||||
|
|
|
|||
|
|
@ -0,0 +1,424 @@
|
|||
//! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer
|
||||
//! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status.
|
||||
//!
|
||||
//! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the
|
||||
//! effect on the first stroke, as After Effects does); the first foreground stroke sets the base
|
||||
//! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze
|
||||
//! first). Every edit is one undo step.
|
||||
|
||||
use effectcraft_effects::roto::{self as fx, FROZEN, Stroke, StrokeKind};
|
||||
use effectcraft_keyframe::Value as KV;
|
||||
use effectcraft_project::build::Ids;
|
||||
use effectcraft_project::{ItemId, Layer, LayerId, PropGroup, Uid};
|
||||
use effectcraft_track::roto::{RotoData, iou, rle};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{CommandSpec, always, b_p, bad, f_p, has_comp, layer_mut, layer_p, str_p};
|
||||
use crate::roto::{self as rt, Direction, RotoOptions, RotoTask, VIEWS, is_roto, params_static};
|
||||
use crate::{EngineError, Result, Session, cmd, query};
|
||||
|
||||
pub fn specs() -> Vec<CommandSpec> {
|
||||
vec![
|
||||
cmd!(
|
||||
"roto.stroke",
|
||||
"Roto Brush Stroke",
|
||||
[],
|
||||
None,
|
||||
"{layer?, kind?: fg|bg|refine|refineErase, points: [[x, y], …] (layer pixels), frame? (layer frame; default current), radius? (layer px; default the tool's diameter / 2), effect?}",
|
||||
has_comp,
|
||||
stroke
|
||||
),
|
||||
cmd!(
|
||||
"roto.propagate",
|
||||
"Propagate Roto Brush",
|
||||
[],
|
||||
None,
|
||||
"{layer?, effect?, direction?: forward|backward|both, to? (layer frame: extend the span), wait?: block until done}",
|
||||
has_comp,
|
||||
propagate
|
||||
),
|
||||
cmd!("roto.span", "Set Segmentation Span", [], None, "{layer?, effect?, start?, end? (layer frames)}", has_comp, span),
|
||||
cmd!("roto.freeze", "Freeze", [], None, "{layer?, effect?, wait?}", has_comp, freeze),
|
||||
cmd!("roto.unfreeze", "Unfreeze", [], None, "{layer?, effect?}", has_comp, unfreeze),
|
||||
cmd!("roto.clearStrokes", "Remove Roto Brush Strokes", [], None, "{layer?, effect?, frame? (only this layer frame), kind?}", has_comp, clear_strokes),
|
||||
cmd!("roto.cancel", "Stop Roto Brush", [], None, "{}", is_running, |s, _| Ok(json!({"stopped": s.stop_roto()}))),
|
||||
cmd!(
|
||||
"roto.options",
|
||||
"Roto Brush Options",
|
||||
[],
|
||||
None,
|
||||
"{diameter?, refineDiameter?, view?: alphaBoundary|alpha|alphaOverlay|none, overlayColor?: [r,g,b], overlayOpacity?, boundaryColor?: [r,g,b], autoPropagate?}",
|
||||
always,
|
||||
options
|
||||
),
|
||||
query!(
|
||||
"roto.status",
|
||||
"Roto Brush Status",
|
||||
"{layer?, effect?, frame? (layer frame: also report its matte), matte?: include the matte (RLE, base64), compute?: compute the frame if needed, compareTo?: RLE matte to report the IoU against}",
|
||||
status
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn is_running(s: &Session) -> std::result::Result<(), String> {
|
||||
if s.is_roto_running() { Ok(()) } else { Err("no Roto Brush job is running".into()) }
|
||||
}
|
||||
|
||||
/// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index),
|
||||
/// else a selected one, else the first on the selected (or any) layer.
|
||||
pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
|
||||
let cid = super::comp_id(s, p)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let candidates: Vec<LayerId> = match p.get("layer") {
|
||||
Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?],
|
||||
None => {
|
||||
let mut v: Vec<LayerId> = s.state.selected_props.iter().map(|(l, _)| *l).collect();
|
||||
v.extend(s.state.selected_layers.iter().copied());
|
||||
v.extend(comp.layers.iter().map(|l| l.id));
|
||||
v
|
||||
}
|
||||
};
|
||||
for lid in candidates {
|
||||
let Some(fxg) = comp.layer(lid).and_then(Layer::effects) else { continue };
|
||||
let g = match p.get("effect") {
|
||||
Some(Value::Number(n)) => {
|
||||
let n = n.as_u64().unwrap_or(0);
|
||||
fxg.groups().find(|g| g.uid == n).or_else(|| fxg.groups().nth((n as usize).saturating_sub(1)))
|
||||
}
|
||||
Some(Value::String(name)) => fxg.groups().find(|g| &g.name == name),
|
||||
_ => s
|
||||
.state
|
||||
.selected_props
|
||||
.iter()
|
||||
.find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten())
|
||||
.or_else(|| fxg.groups().find(|g| is_roto(g))),
|
||||
};
|
||||
if let Some(g) = g.filter(|g| is_roto(g)) {
|
||||
return Ok((cid, lid, g.uid));
|
||||
}
|
||||
if p.get("layer").is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)"))
|
||||
}
|
||||
|
||||
fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> {
|
||||
let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?;
|
||||
let g = l.props.find_group(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?;
|
||||
Ok((l, g))
|
||||
}
|
||||
|
||||
fn data_of(g: &PropGroup) -> RotoData {
|
||||
fx::data(¶ms_static(g)).as_ref().clone()
|
||||
}
|
||||
|
||||
fn frozen_of(g: &PropGroup) -> bool {
|
||||
matches!(g.get(FROZEN).map(|p| &p.value), Some(KV::Str(s)) if !s.is_empty())
|
||||
}
|
||||
|
||||
/// Write an instance's data (one undo step).
|
||||
fn write_data(s: &mut Session, label: &str, cid: ItemId, lid: LayerId, uid: Uid, d: &RotoData) -> Result<()> {
|
||||
let j = d.to_json();
|
||||
s.edit(label, None, |proj, _| {
|
||||
let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?;
|
||||
if let Some(pr) = g.get_mut(fx::STROKES) {
|
||||
pr.value = KV::Str(j);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Current layer frame of `layer` (or `frame` from the params).
|
||||
fn frame_p(s: &Session, p: &Value, cid: ItemId, layer: &Layer) -> i64 {
|
||||
if let Some(f) = f_p(p, "frame") {
|
||||
return f.round() as i64;
|
||||
}
|
||||
let comp = s.project.comp(cid);
|
||||
let fps = comp.map(|c| c.frame_rate.as_f64()).unwrap_or(30.0);
|
||||
fx::frame_of(layer.layer_time(s.time_of(cid)).seconds(), fps)
|
||||
}
|
||||
|
||||
fn points_p(p: &Value, cmd: &str) -> Result<Vec<[f64; 2]>> {
|
||||
let raw = p.get("points").and_then(Value::as_array).ok_or_else(|| bad(cmd, "missing `points` [[x, y], …] (layer pixels)"))?;
|
||||
let mut v = vec![];
|
||||
for q in raw {
|
||||
let a = q.as_array().ok_or_else(|| bad(cmd, "each point is [x, y]"))?;
|
||||
let (Some(x), Some(y)) = (a.first().and_then(Value::as_f64), a.get(1).and_then(Value::as_f64)) else {
|
||||
return Err(bad(cmd, "each point is [x, y]"));
|
||||
};
|
||||
v.push([x, y]);
|
||||
}
|
||||
if v.is_empty() {
|
||||
return Err(bad(cmd, "`points` is empty"));
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
fn stroke(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "roto.stroke";
|
||||
let (cid, lid) = match roto_p(s, p, cmd) {
|
||||
Ok((c, l, _)) => (c, l),
|
||||
Err(_) => layer_p(s, p, cmd)?,
|
||||
};
|
||||
let kind = match str_p(p, "kind") {
|
||||
Some(k) => StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}` (fg | bg | refine | refineErase)")))?,
|
||||
None => StrokeKind::Fg,
|
||||
};
|
||||
let points = points_p(p, cmd)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?;
|
||||
if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() {
|
||||
return Err(bad(cmd, "the Roto Brush works on footage, solid and composition layers"));
|
||||
}
|
||||
let frame = frame_p(s, p, cid, layer);
|
||||
let limits = rt::frame_limits(comp, layer);
|
||||
if frame < limits[0] || frame > limits[1] {
|
||||
return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1])));
|
||||
}
|
||||
let o = &s.state.roto;
|
||||
let def_r = if matches!(kind, StrokeKind::Refine | StrokeKind::RefineErase) { o.refine_diameter / 2.0 } else { o.diameter / 2.0 };
|
||||
let radius = f_p(p, "radius").unwrap_or(def_r).clamp(0.5, 2500.0);
|
||||
let size = rt::layer_size(&s.project, comp, layer);
|
||||
// The instance: the requested / first one, else a new one.
|
||||
let existing = roto_p(s, p, cmd).ok().filter(|(c, l, _)| *c == cid && *l == lid).map(|x| x.2);
|
||||
if let Some(uid) = existing {
|
||||
let (_, g) = group(s, cid, lid, uid)?;
|
||||
if frozen_of(g) {
|
||||
return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes"));
|
||||
}
|
||||
}
|
||||
if existing.is_none() && !kind.is_segmentation() {
|
||||
return Err(bad(cmd, "draw a foreground stroke with the Roto Brush tool first"));
|
||||
}
|
||||
let st = Stroke { kind, frame, radius, points };
|
||||
let label = if kind.is_segmentation() { "Roto Brush Stroke" } else { "Refine Edge Stroke" };
|
||||
let (uid, d) = s.edit(label, None, |proj, _| {
|
||||
let mut next = proj.next_id;
|
||||
let l = layer_mut(proj, cid, lid)?;
|
||||
let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?;
|
||||
let uid = match existing {
|
||||
Some(u) => u,
|
||||
None => {
|
||||
let spec = effectcraft_effects::find(fx::ID).ok_or_else(|| EngineError::Other("Roto Brush effect missing".into()))?;
|
||||
let n = fxg.groups().filter(|g| is_roto(g)).count();
|
||||
let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) };
|
||||
let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size);
|
||||
let u = g.uid;
|
||||
fxg.children.push(g.into());
|
||||
u
|
||||
}
|
||||
};
|
||||
let g = fxg.find_group_mut(uid).ok_or_else(|| bad(cmd, "effect gone"))?;
|
||||
let mut d = data_of(g);
|
||||
d.add(st, fx::DEFAULT_SPAN, limits);
|
||||
if let Some(pr) = g.get_mut(fx::STROKES) {
|
||||
pr.value = KV::Str(d.to_json());
|
||||
}
|
||||
proj.next_id = proj.next_id.max(next);
|
||||
Ok((uid, d))
|
||||
})?;
|
||||
if !s.state.selected_props.iter().any(|(l, u)| *l == lid && *u == uid) {
|
||||
s.state.selected_layers = vec![lid];
|
||||
}
|
||||
Ok(json!({
|
||||
"layer": lid.0, "effect": uid, "frame": frame, "kind": kind, "radius": radius,
|
||||
"base": d.base, "span": d.span, "strokes": d.strokes.len(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn propagate(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "roto.propagate";
|
||||
let (cid, lid, uid) = roto_p(s, p, cmd)?;
|
||||
let dir = match str_p(p, "direction") {
|
||||
Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?,
|
||||
None => Direction::Both,
|
||||
};
|
||||
if let Some(to) = f_p(p, "to") {
|
||||
let to = to.round() as i64;
|
||||
let (l, g) = group(s, cid, lid, uid)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let lim = rt::frame_limits(comp, l);
|
||||
let mut d = data_of(g);
|
||||
let to = to.clamp(lim[0], lim[1]);
|
||||
let before = d.span;
|
||||
d.span[0] = d.span[0].min(to);
|
||||
d.span[1] = d.span[1].max(to);
|
||||
if d.span != before {
|
||||
if frozen_of(g) {
|
||||
return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span"));
|
||||
}
|
||||
write_data(s, "Extend Segmentation Span", cid, lid, uid, &d)?;
|
||||
}
|
||||
}
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?;
|
||||
let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()});
|
||||
if wait {
|
||||
out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn span(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "roto.span";
|
||||
let (cid, lid, uid) = roto_p(s, p, cmd)?;
|
||||
let (l, g) = group(s, cid, lid, uid)?;
|
||||
if frozen_of(g) {
|
||||
return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span"));
|
||||
}
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let lim = rt::frame_limits(comp, l);
|
||||
let mut d = data_of(g);
|
||||
let base = d.base.ok_or_else(|| bad(cmd, "no strokes yet"))?;
|
||||
let first = d.strokes.iter().map(|s| s.frame).min().unwrap_or(base);
|
||||
let last = d.strokes.iter().map(|s| s.frame).max().unwrap_or(base);
|
||||
if let Some(a) = f_p(p, "start") {
|
||||
d.span[0] = (a.round() as i64).clamp(lim[0], first);
|
||||
}
|
||||
if let Some(b) = f_p(p, "end") {
|
||||
d.span[1] = (b.round() as i64).clamp(last, lim[1]);
|
||||
}
|
||||
write_data(s, "Segmentation Span", cid, lid, uid, &d)?;
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "span": d.span, "base": base}))
|
||||
}
|
||||
|
||||
fn freeze(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "roto.freeze";
|
||||
let (cid, lid, uid) = roto_p(s, p, cmd)?;
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?;
|
||||
let (_, g) = group(s, cid, lid, uid)?;
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()}))
|
||||
}
|
||||
|
||||
fn unfreeze(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "roto.unfreeze";
|
||||
let (cid, lid, uid) = roto_p(s, p, cmd)?;
|
||||
let (_, g) = group(s, cid, lid, uid)?;
|
||||
if !frozen_of(g) {
|
||||
return Ok(json!({"layer": lid.0, "effect": uid, "frozen": false}));
|
||||
}
|
||||
s.edit("Unfreeze", None, |proj, _| {
|
||||
let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?;
|
||||
if let Some(pr) = g.get_mut(FROZEN) {
|
||||
pr.value = KV::Str(String::new());
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "frozen": false}))
|
||||
}
|
||||
|
||||
fn clear_strokes(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "roto.clearStrokes";
|
||||
let (cid, lid, uid) = roto_p(s, p, cmd)?;
|
||||
let (_, g) = group(s, cid, lid, uid)?;
|
||||
if frozen_of(g) {
|
||||
return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes"));
|
||||
}
|
||||
let mut d = data_of(g);
|
||||
let frame = f_p(p, "frame").map(|f| f.round() as i64);
|
||||
let kind = match str_p(p, "kind") {
|
||||
Some(k) => Some(StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}`")))?),
|
||||
None => None,
|
||||
};
|
||||
let n0 = d.strokes.len();
|
||||
d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k)));
|
||||
if d.strokes.iter().all(|s| !s.kind.is_segmentation()) {
|
||||
d.base = None;
|
||||
} else if d.base.is_some_and(|b| !d.strokes.iter().any(|s| s.frame == b && s.kind.is_segmentation())) {
|
||||
// The base moves to the earliest remaining stroked frame.
|
||||
d.base = d.strokes.iter().filter(|s| s.kind.is_segmentation()).map(|s| s.frame).min();
|
||||
}
|
||||
let removed = n0 - d.strokes.len();
|
||||
if removed > 0 {
|
||||
write_data(s, "Remove Roto Brush Strokes", cid, lid, uid, &d)?;
|
||||
}
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "removed": removed, "strokes": d.strokes.len(), "base": d.base}))
|
||||
}
|
||||
|
||||
fn options(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "roto.options";
|
||||
let mut o: RotoOptions = s.state.roto.clone();
|
||||
for (k, f) in [("diameter", &mut o.diameter as &mut f64), ("refineDiameter", &mut o.refine_diameter), ("overlayOpacity", &mut o.overlay_opacity)] {
|
||||
if let Some(v) = f_p(p, k) {
|
||||
*f = v;
|
||||
}
|
||||
}
|
||||
o.diameter = o.diameter.clamp(1.0, 5000.0);
|
||||
o.refine_diameter = o.refine_diameter.clamp(1.0, 5000.0);
|
||||
o.overlay_opacity = o.overlay_opacity.clamp(0.0, 100.0);
|
||||
if let Some(v) = str_p(p, "view") {
|
||||
let v = VIEWS.iter().find(|x| x.eq_ignore_ascii_case(v)).ok_or_else(|| bad(cmd, format!("unknown view `{v}` ({})", VIEWS.join(" | "))))?;
|
||||
o.view = v.to_string();
|
||||
}
|
||||
for (k, c) in [("overlayColor", &mut o.overlay_color), ("boundaryColor", &mut o.boundary_color)] {
|
||||
if let Some(a) = p.get(k).and_then(Value::as_array) {
|
||||
if a.len() < 3 {
|
||||
return Err(bad(cmd, format!("`{k}` is [r, g, b] (0–1)")));
|
||||
}
|
||||
for (i, v) in a.iter().take(3).enumerate() {
|
||||
c[i] = v.as_f64().unwrap_or(0.0).clamp(0.0, 1.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(b) = b_p(p, "autoPropagate") {
|
||||
o.auto_propagate = b;
|
||||
}
|
||||
s.state.roto = o.clone();
|
||||
Ok(serde_json::to_value(o).unwrap_or(Value::Null))
|
||||
}
|
||||
|
||||
fn status(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
s.poll_roto(false);
|
||||
let mut out =
|
||||
json!({"running": s.is_roto_running(), "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner()), "options": s.state.roto});
|
||||
let Ok((cid, lid, uid)) = roto_p(s, p, "roto.status") else { return Ok(out) };
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone();
|
||||
let (layer, g) = group(s, cid, lid, uid)?;
|
||||
let (layer, g) = (layer.clone(), g.clone());
|
||||
let d = data_of(&g);
|
||||
out["layer"] = json!(lid.0);
|
||||
out["effect"] = json!(uid);
|
||||
out["base"] = json!(d.base);
|
||||
out["span"] = json!(d.span);
|
||||
out["strokes"] = json!(d.strokes.len());
|
||||
out["strokeFrames"] = json!(d.stroke_frames());
|
||||
out["frozen"] = json!(frozen_of(&g));
|
||||
out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid)));
|
||||
out["limits"] = json!(rt::frame_limits(&comp, &layer));
|
||||
let frame = frame_p(s, p, cid, &layer);
|
||||
out["frame"] = json!(frame);
|
||||
let want_matte = b_p(p, "matte").unwrap_or(false);
|
||||
let compute = b_p(p, "compute").unwrap_or(want_matte);
|
||||
if (p.get("frame").is_some() || want_matte) && d.in_span(frame) {
|
||||
let size = rt::layer_size(&s.project, &comp, &layer);
|
||||
let fps = comp.frame_rate.as_f64();
|
||||
let ps = params_static(&g);
|
||||
let chain = fx::Chain::new(&ps, size, fps, 1.0);
|
||||
let seg = match chain.keys.get(&frame).and_then(|k| fx::cached(*k, 1.0)) {
|
||||
Some(sg) => Some(sg),
|
||||
None if compute => {
|
||||
let index = layer.effects().and_then(|f| f.groups().position(|x| x.uid == uid)).unwrap_or(0);
|
||||
let mut src = rt::FrameSource::new(&s.project, s.footage.as_ref(), s.expr.as_deref(), &s.layer_cache, cid, &comp, &layer, index, size);
|
||||
chain.ensure(frame, &mut |f| src.get(f), &|| false)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
if let Some(sg) = seg {
|
||||
out["area"] = json!(sg.area());
|
||||
out["centroid"] = json!(sg.centroid());
|
||||
out["size"] = json!([sg.w, sg.h]);
|
||||
out["refineArea"] = json!(sg.refine.iter().filter(|v| **v != 0).count());
|
||||
if want_matte {
|
||||
out["matte"] = json!(rle::encode(&sg.matte));
|
||||
}
|
||||
if let Some(gt) = p.get("compareTo").and_then(Value::as_str).and_then(rle::decode) {
|
||||
out["iou"] = json!(iou(&sg.matte, >));
|
||||
}
|
||||
}
|
||||
}
|
||||
let computed = s.roto_computed(cid, lid, uid);
|
||||
out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::<Vec<_>>());
|
||||
Ok(out)
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ pub mod mask_track;
|
|||
pub mod menus;
|
||||
pub mod prefs;
|
||||
pub mod render_queue;
|
||||
pub mod roto;
|
||||
mod session_settings;
|
||||
pub mod shortcuts;
|
||||
pub mod tracking;
|
||||
|
|
@ -41,6 +42,7 @@ pub use effectcraft_project as project;
|
|||
pub use effectcraft_render as render;
|
||||
pub use effectcraft_text as text;
|
||||
pub use effectcraft_time as time;
|
||||
pub use effectcraft_track as track;
|
||||
pub use render_queue::{ExportJob, ExportResult, Exporter, JobState};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
|
|
@ -176,6 +178,9 @@ pub struct EditorState {
|
|||
/// Paint and Brushes panel options (Brush, Clone Stamp and Eraser tools).
|
||||
#[serde(default)]
|
||||
pub paint: commands::paint::PaintOptions,
|
||||
/// Roto Brush and Refine Edge tool options.
|
||||
#[serde(default)]
|
||||
pub roto: roto::RotoOptions,
|
||||
/// Puppet tool options for new meshes.
|
||||
#[serde(default)]
|
||||
pub puppet: commands::puppet::PuppetOptions,
|
||||
|
|
@ -263,6 +268,10 @@ pub struct Session {
|
|||
pub warp_job: Option<warp::WarpJob>,
|
||||
/// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid).
|
||||
pub warp_pending: Vec<(ItemId, LayerId, Uid)>,
|
||||
/// The running (or finished, not yet polled) Roto Brush propagation / Freeze.
|
||||
pub roto_job: Option<roto::RotoJob>,
|
||||
/// Roto Brush instances edited since their last propagation: (comp, layer, effect uid).
|
||||
pub roto_pending: Vec<(ItemId, LayerId, Uid)>,
|
||||
pub events: Vec<Event>,
|
||||
/// Commands executed: (id, params).
|
||||
pub journal: Vec<(String, Value)>,
|
||||
|
|
@ -325,6 +334,8 @@ impl Default for Session {
|
|||
mask_job: None,
|
||||
warp_job: None,
|
||||
warp_pending: vec![],
|
||||
roto_job: None,
|
||||
roto_pending: vec![],
|
||||
events: vec![],
|
||||
journal: vec![],
|
||||
layer_cache: Arc::new(LayerCache::default()),
|
||||
|
|
@ -392,6 +403,12 @@ impl Session {
|
|||
self.warp_pending.push(w);
|
||||
}
|
||||
}
|
||||
// Roto Brush Input Keys follow the frames; edited instances propagate again.
|
||||
for r in roto::sync(&before, &mut p) {
|
||||
if !self.roto_pending.contains(&r) {
|
||||
self.roto_pending.push(r);
|
||||
}
|
||||
}
|
||||
let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key;
|
||||
if !same {
|
||||
self.history.undo.push((label.to_string(), before));
|
||||
|
|
@ -569,6 +586,9 @@ impl Session {
|
|||
self.stop_warp();
|
||||
self.warp_job = None;
|
||||
self.warp_pending.clear();
|
||||
self.stop_roto();
|
||||
self.roto_job = None;
|
||||
self.roto_pending.clear();
|
||||
p.fix_next_id();
|
||||
self.project = Arc::new(p);
|
||||
self.history = History::default();
|
||||
|
|
@ -635,3 +655,5 @@ pub fn text_families() -> Vec<String> {
|
|||
}
|
||||
#[cfg(test)]
|
||||
mod tests_paint;
|
||||
#[cfg(test)]
|
||||
mod tests_roto;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,569 @@
|
|||
//! Roto Brush & Refine Edge runtime: propagation and Freeze jobs, Input Key maintenance and the
|
||||
//! tool options.
|
||||
//!
|
||||
//! Segmentations are derived data in `effectcraft_effects::roto`'s content-keyed cache (see that
|
||||
//! module). `roto.propagate` fills it at full resolution in the background (or blocking with
|
||||
//! `wait`), frame by frame outward from the base frame, rendering the effect's input
|
||||
//! (`Renderer::layer_input`, source → masks → the effects above it). `roto.freeze` runs the same
|
||||
//! pass and stores the final mattes in the effect (one undo step). [`sync`], run after every
|
||||
//! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made
|
||||
//! of (the layer's source and timing, its masks and the effects above it) and drops frozen
|
||||
//! mattes made from other frames.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY};
|
||||
use effectcraft_effects::{Params, flatten_params};
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::tracking::lock;
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Roto Brush / Refine Edge tool options (serde: `roto.options`).
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct RotoOptions {
|
||||
/// Roto Brush tip diameter (layer pixels).
|
||||
pub diameter: f64,
|
||||
/// Refine Edge tip diameter (layer pixels).
|
||||
pub refine_diameter: f64,
|
||||
/// Layer panel view: `alphaBoundary` (pink outline), `alpha`, `alphaOverlay`, `none`.
|
||||
pub view: String,
|
||||
/// Alpha Overlay colour (straight RGB 0–1) and opacity (percent).
|
||||
pub overlay_color: [f64; 3],
|
||||
pub overlay_opacity: f64,
|
||||
/// Alpha Boundary colour.
|
||||
pub boundary_color: [f64; 3],
|
||||
/// Start propagating in the background after each stroke.
|
||||
pub auto_propagate: bool,
|
||||
}
|
||||
|
||||
impl Default for RotoOptions {
|
||||
fn default() -> Self {
|
||||
RotoOptions {
|
||||
diameter: 20.0,
|
||||
refine_diameter: 10.0,
|
||||
view: "alphaBoundary".into(),
|
||||
overlay_color: [1.0, 0.0, 0.0],
|
||||
overlay_opacity: 50.0,
|
||||
boundary_color: [1.0, 0.25, 0.75],
|
||||
auto_propagate: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const VIEWS: [&str; 4] = ["alphaBoundary", "alpha", "alphaOverlay", "none"];
|
||||
|
||||
/// What a job does.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum RotoTask {
|
||||
Propagate,
|
||||
Freeze,
|
||||
}
|
||||
|
||||
/// Which way to propagate.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Direction {
|
||||
Forward,
|
||||
Backward,
|
||||
#[default]
|
||||
Both,
|
||||
}
|
||||
|
||||
impl Direction {
|
||||
pub fn from_name(s: &str) -> Option<Direction> {
|
||||
Some(match s.to_ascii_lowercase().as_str() {
|
||||
"forward" | "fwd" => Direction::Forward,
|
||||
"backward" | "back" => Direction::Backward,
|
||||
"both" | "all" => Direction::Both,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Live job progress (serde for agents / the control channel).
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RotoProgress {
|
||||
pub task: Option<RotoTask>,
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub elapsed: f64,
|
||||
pub fps: f64,
|
||||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl RotoProgress {
|
||||
pub fn banner(&self) -> String {
|
||||
let pc = (100 * self.done).checked_div(self.total).unwrap_or(0);
|
||||
match self.task {
|
||||
Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"),
|
||||
_ => format!("Propagating Roto Brush: {pc}%"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct RotoShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<RotoProgress>,
|
||||
frozen: Mutex<Option<FrozenCache>>,
|
||||
}
|
||||
|
||||
pub struct RotoJob {
|
||||
pub shared: Arc<RotoShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub effect: Uid,
|
||||
pub task: RotoTask,
|
||||
/// The strokes the job was started with (a Freeze result is dropped when they changed).
|
||||
strokes: String,
|
||||
}
|
||||
|
||||
impl RotoJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> RotoProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_roto(g: &PropGroup) -> bool {
|
||||
matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID)
|
||||
}
|
||||
|
||||
fn str_of(g: &PropGroup, m: &str) -> String {
|
||||
match g.get(m).map(|p| &p.value) {
|
||||
Some(Value::Str(s)) => s.clone(),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_str(g: &mut PropGroup, m: &str, v: String) {
|
||||
if let Some(p) = g.get_mut(m) {
|
||||
p.value = Value::Str(v);
|
||||
p.keys.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fn fnv(s: &str) -> u64 {
|
||||
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
for b in s.as_bytes() {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(0x0100_0000_01b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
/// What the effect's input frames are made of: the layer's source and timing, its masks and the
|
||||
/// effects above the Roto Brush.
|
||||
pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String {
|
||||
let mut s = crate::warp::signature(p, comp, layer);
|
||||
if let Some(m) = layer.masks() {
|
||||
s.push_str(&format!("{m:?}"));
|
||||
}
|
||||
if let Some(fx) = layer.effects() {
|
||||
for g in fx.groups() {
|
||||
if g.uid == uid {
|
||||
break;
|
||||
}
|
||||
s.push_str(&format!("{g:?}"));
|
||||
}
|
||||
}
|
||||
format!("{:016x}", fnv(&s))
|
||||
}
|
||||
|
||||
/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped.
|
||||
/// Returns the instances whose segmentation changed (to propagate again).
|
||||
pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
|
||||
let mut todo = vec![];
|
||||
let mut set = vec![];
|
||||
for (cid, it) in &after.items {
|
||||
let ItemKind::Comp(c) = &it.kind else { continue };
|
||||
for l in &c.layers {
|
||||
let Some(fxg) = l.effects() else { continue };
|
||||
for g in fxg.groups().filter(|g| is_roto(g)) {
|
||||
let key = input_key(after, *cid, l, g.uid);
|
||||
let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid));
|
||||
if str_of(g, INPUT_KEY) != key {
|
||||
set.push((*cid, l.id, g.uid, key));
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
} else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) {
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (cid, lid, uid, key) in set {
|
||||
if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) {
|
||||
let had = !str_of(g, INPUT_KEY).is_empty();
|
||||
set_str(g, INPUT_KEY, key);
|
||||
if had {
|
||||
set_str(g, FROZEN, String::new());
|
||||
}
|
||||
}
|
||||
}
|
||||
todo.retain(|(c, l, u)| {
|
||||
after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty())
|
||||
});
|
||||
todo
|
||||
}
|
||||
|
||||
/// Parameters of an instance with static values (strokes and keys are never animated).
|
||||
pub(crate) fn params_static(g: &PropGroup) -> Params {
|
||||
flatten_params(g, &mut |p| p.value.clone())
|
||||
}
|
||||
|
||||
/// Evaluated parameters of an instance at comp time `t`.
|
||||
pub(crate) fn params_at(p: &Project, cid: ItemId, layer: &Layer, g: &PropGroup, t: Tick, expr: Option<&dyn ExprHost>) -> Params {
|
||||
match p.comp(cid) {
|
||||
Some(c) => {
|
||||
let ctx = EvalCtx { expr, ..EvalCtx::new(p, cid, c, t) };
|
||||
flatten_params(g, &mut |pr| ctx.value(layer, pr))
|
||||
}
|
||||
None => params_static(g),
|
||||
}
|
||||
}
|
||||
|
||||
/// The layer's source size (layer pixels).
|
||||
pub(crate) fn layer_size(p: &Project, comp: &effectcraft_project::Comp, layer: &Layer) -> [f64; 2] {
|
||||
let (w, h) = effectcraft_render::source_size(p, layer);
|
||||
if w == 0 { [comp.width as f64, comp.height as f64] } else { [w as f64, h as f64] }
|
||||
}
|
||||
|
||||
/// First and last layer frame of the layer inside the comp (inclusive).
|
||||
pub fn frame_limits(comp: &effectcraft_project::Comp, layer: &Layer) -> [i64; 2] {
|
||||
let fps = comp.frame_rate.as_f64().max(1e-6);
|
||||
let lo = layer.layer_time(layer.in_point.max(Tick::ZERO)).seconds();
|
||||
let hi = layer.layer_time(layer.out_point.min(comp.duration)).seconds();
|
||||
let a = fx::frame_of(lo, fps);
|
||||
let b = (fx::frame_of(hi, fps) - 1).max(a);
|
||||
[a, b]
|
||||
}
|
||||
|
||||
/// Comp time of layer frame `f`.
|
||||
pub(crate) fn comp_time_of(comp: &effectcraft_project::Comp, layer: &Layer, f: i64) -> Tick {
|
||||
layer.comp_time(comp.frame_rate.tick_of(f))
|
||||
}
|
||||
|
||||
/// What the worker needs, detached from the session.
|
||||
struct Work {
|
||||
project: Arc<Project>,
|
||||
footage: Arc<dyn FootageSource>,
|
||||
expr: Option<Arc<dyn ExprHost>>,
|
||||
cache: Arc<LayerCache>,
|
||||
comp: ItemId,
|
||||
layer: LayerId,
|
||||
effect_index: usize,
|
||||
params: Params,
|
||||
size: [f64; 2],
|
||||
fps: f64,
|
||||
frames: Vec<i64>,
|
||||
task: RotoTask,
|
||||
}
|
||||
|
||||
/// Renders the effect's input frames for a worker, keeping the last few.
|
||||
pub(crate) struct FrameSource<'a> {
|
||||
r: Renderer<'a>,
|
||||
ctx0: EvalCtx<'a>,
|
||||
comp: &'a effectcraft_project::Comp,
|
||||
layer: &'a Layer,
|
||||
index: usize,
|
||||
size: [f64; 2],
|
||||
memo: HashMap<i64, Arc<Image>>,
|
||||
}
|
||||
|
||||
impl<'a> FrameSource<'a> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn new(
|
||||
project: &'a Project,
|
||||
footage: &'a dyn FootageSource,
|
||||
expr: Option<&'a dyn ExprHost>,
|
||||
cache: &'a LayerCache,
|
||||
cid: ItemId,
|
||||
comp: &'a effectcraft_project::Comp,
|
||||
layer: &'a Layer,
|
||||
index: usize,
|
||||
size: [f64; 2],
|
||||
) -> FrameSource<'a> {
|
||||
let mut r = Renderer::new(project, footage, RenderOpts { scale: 1.0, ..Default::default() });
|
||||
r.expr = expr;
|
||||
r.cache = Some(cache);
|
||||
let ctx0 = EvalCtx { expr, ..EvalCtx::new(project, cid, comp, Tick::ZERO) };
|
||||
FrameSource { r, ctx0, comp, layer, index, size, memo: HashMap::new() }
|
||||
}
|
||||
|
||||
pub(crate) fn get(&mut self, f: i64) -> Option<Arc<Image>> {
|
||||
if let Some(i) = self.memo.get(&f) {
|
||||
return Some(i.clone());
|
||||
}
|
||||
let t = comp_time_of(self.comp, self.layer, f);
|
||||
let buf = self.r.layer_input(&self.ctx0.at(t), self.layer, self.index)?;
|
||||
let buf = if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
|
||||
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
|
||||
effectcraft_effects::Buf {
|
||||
img: effectcraft_raster::resample(&buf.img, w, h),
|
||||
offset: [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale],
|
||||
scale: 1.0,
|
||||
}
|
||||
} else {
|
||||
(*buf).clone()
|
||||
};
|
||||
let img = Arc::new(fx::layer_grid(&buf, self.size));
|
||||
if self.memo.len() > 4 {
|
||||
self.memo.clear();
|
||||
}
|
||||
self.memo.insert(f, img.clone());
|
||||
Some(img)
|
||||
}
|
||||
}
|
||||
|
||||
fn run_work(w: Work, shared: &RotoShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let mut src = FrameSource::new(&w.project, w.footage.as_ref(), w.expr.as_deref(), &w.cache, w.comp, comp, layer, w.effect_index, w.size);
|
||||
{
|
||||
let mut s = lock(&shared.state);
|
||||
s.task = Some(w.task);
|
||||
s.total = w.frames.len() as u64;
|
||||
}
|
||||
let cancel = || shared.cancel.load(Ordering::Relaxed);
|
||||
let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0);
|
||||
let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() };
|
||||
for (k, f) in w.frames.iter().enumerate() {
|
||||
if cancel() {
|
||||
let mut s = lock(&shared.state);
|
||||
s.cancelled = true;
|
||||
s.finished = true;
|
||||
return;
|
||||
}
|
||||
let mut get = |g: i64| src.get(g);
|
||||
let ok = match w.task {
|
||||
RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(),
|
||||
RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) {
|
||||
Some((a, b)) => {
|
||||
if let Some(img) = src.get(*f) {
|
||||
frozen.size = [img.width as usize, img.height as usize];
|
||||
}
|
||||
frozen.frames.insert(*f, fx::freeze_frame(&a, &b));
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
},
|
||||
};
|
||||
if !ok && !cancel() {
|
||||
return fail(&format!("frame {f} could not be segmented"));
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64 + 1;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 };
|
||||
}
|
||||
if w.task == RotoTask::Freeze {
|
||||
*lock(&shared.frozen) = Some(frozen);
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Frames to compute, outward from the base frame.
|
||||
pub(crate) fn job_frames(d: &effectcraft_track::roto::RotoData, dir: Direction) -> Vec<i64> {
|
||||
let Some(base) = d.base else { return vec![] };
|
||||
let mut v = vec![base];
|
||||
if dir != Direction::Backward {
|
||||
v.extend(base + 1..=d.span[1]);
|
||||
}
|
||||
if dir != Direction::Forward {
|
||||
v.extend((d.span[0]..base).rev());
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub fn is_roto_running(&self) -> bool {
|
||||
self.roto_job.as_ref().is_some_and(|j| !j.is_finished())
|
||||
}
|
||||
|
||||
pub fn roto_progress(&self) -> Option<RotoProgress> {
|
||||
self.roto_job.as_ref().map(|j| j.progress())
|
||||
}
|
||||
|
||||
/// (comp, layer, effect uid, task) of the running job.
|
||||
pub fn roto_target(&self) -> Option<(ItemId, LayerId, Uid, RotoTask)> {
|
||||
self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task))
|
||||
}
|
||||
|
||||
pub fn stop_roto(&mut self) -> bool {
|
||||
match &self.roto_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.shared.cancel.store(true, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start propagating (or freezing) Roto Brush `effect` on `layer`. `wait`: block until done
|
||||
/// (wasm32 always does). Returns the number of frames.
|
||||
pub fn start_roto(&mut self, comp: ItemId, layer: LayerId, effect: Uid, task: RotoTask, dir: Direction, wait: bool) -> Result<usize, String> {
|
||||
if self.is_roto_running() {
|
||||
if !wait {
|
||||
return Err("a Roto Brush job is already running".into());
|
||||
}
|
||||
self.stop_roto();
|
||||
if let Some(mut j) = self.roto_job.take()
|
||||
&& let Some(t) = j.thread.take()
|
||||
{
|
||||
let _ = t.join();
|
||||
}
|
||||
}
|
||||
self.poll_roto_job();
|
||||
self.roto_pending.retain(|x| *x != (comp, layer, effect));
|
||||
let c = self.project.comp(comp).ok_or("no composition")?;
|
||||
let l = c.layer(layer).ok_or("no layer")?;
|
||||
let fxg = l.effects().ok_or("the layer has no effects")?;
|
||||
let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?;
|
||||
let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?;
|
||||
let size = layer_size(&self.project, c, l);
|
||||
let fps = c.frame_rate.as_f64();
|
||||
let pstat = params_static(g);
|
||||
let d = fx::data(&pstat);
|
||||
if d.is_empty() {
|
||||
return Err("draw a foreground stroke with the Roto Brush tool first".into());
|
||||
}
|
||||
let t = comp_time_of(c, l, d.base.unwrap_or(0));
|
||||
let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
|
||||
let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir });
|
||||
let n = frames.len();
|
||||
let work = Work {
|
||||
project: self.project.clone(),
|
||||
footage: self.footage.clone(),
|
||||
expr: self.expr.clone(),
|
||||
cache: self.layer_cache.clone(),
|
||||
comp,
|
||||
layer,
|
||||
effect_index: index,
|
||||
params,
|
||||
size,
|
||||
fps,
|
||||
frames,
|
||||
task,
|
||||
};
|
||||
let strokes = str_of(g, fx::STROKES);
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
let shared = Arc::new(RotoShared::default());
|
||||
let thread = if wait {
|
||||
run_work(work, &shared);
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(std::thread::Builder::new().name("roto".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?)
|
||||
};
|
||||
self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes });
|
||||
self.events.push(Event::ProjectChanged { revision: self.revision });
|
||||
self.poll_roto_job();
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Finish a done job: Freeze writes the frozen mattes (one undo step).
|
||||
fn poll_roto_job(&mut self) -> bool {
|
||||
let Some(job) = &self.roto_job else { return false };
|
||||
if !job.is_finished() {
|
||||
return false;
|
||||
}
|
||||
let mut job = self.roto_job.take().expect("job");
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
if let Some(e) = &st.error {
|
||||
self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true });
|
||||
self.bump();
|
||||
return true;
|
||||
}
|
||||
if st.cancelled {
|
||||
self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false });
|
||||
self.bump();
|
||||
return true;
|
||||
}
|
||||
match job.task {
|
||||
RotoTask::Propagate => {
|
||||
self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false });
|
||||
// Rendered frames depend on the cache: redraw.
|
||||
self.bump();
|
||||
}
|
||||
RotoTask::Freeze => {
|
||||
let Some(fc) = lock(&job.shared.frozen).take() else { return true };
|
||||
let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone());
|
||||
let r = self.edit("Freeze", None, |p, _| {
|
||||
let g = p
|
||||
.comp_mut(cid)
|
||||
.and_then(|c| c.layer_mut(lid))
|
||||
.and_then(|l| l.props.find_group_mut(uid))
|
||||
.ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?;
|
||||
if str_of(g, fx::STROKES) != strokes {
|
||||
return Err(crate::EngineError::Other("the strokes changed while freezing".into()));
|
||||
}
|
||||
set_str(g, FROZEN, fc.to_json());
|
||||
Ok(())
|
||||
});
|
||||
let msg = match r {
|
||||
Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed),
|
||||
Err(e) => format!("Roto Brush: {e}"),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: false });
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Frontends call this every frame: finishes jobs and (with `auto`) propagates edited
|
||||
/// instances in the background.
|
||||
pub fn poll_roto(&mut self, auto: bool) -> bool {
|
||||
let changed = self.poll_roto_job();
|
||||
if auto && self.roto_job.is_none() && self.state.roto.auto_propagate {
|
||||
while let Some((c, l, e)) = self.roto_pending.first().copied() {
|
||||
self.roto_pending.remove(0);
|
||||
if self.start_roto(c, l, e, RotoTask::Propagate, Direction::Both, false).is_ok() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
/// The computed frames of an instance (full resolution), by layer frame.
|
||||
pub fn roto_computed(&self, comp: ItemId, layer: LayerId, effect: Uid) -> BTreeMap<i64, bool> {
|
||||
let mut out = BTreeMap::new();
|
||||
let Some(c) = self.project.comp(comp) else { return out };
|
||||
let Some(l) = c.layer(layer) else { return out };
|
||||
let Some(g) = l.props.find_group(effect) else { return out };
|
||||
let ps = params_static(g);
|
||||
let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0);
|
||||
for f in ch.keys.keys() {
|
||||
out.insert(*f, ch.is_cached(*f, 1.0));
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,288 @@
|
|||
//! Roto Brush & Refine Edge end to end through the commands, on synthetic footage: a textured
|
||||
//! disk moving over a textured background.
|
||||
|
||||
use effectcraft_project::LayerId;
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::RenderOpts;
|
||||
use effectcraft_track::roto::rle;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::Session;
|
||||
use crate::tests_track::{H, W, frame_time, setup};
|
||||
|
||||
const R: f64 = 40.0;
|
||||
|
||||
fn noise(x: f64, y: f64, seed: u32) -> f64 {
|
||||
fn h(i: i64, j: i64, s: u32) -> f64 {
|
||||
let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64;
|
||||
v ^= v >> 13;
|
||||
v = v.wrapping_mul(0x5bd1e995);
|
||||
v ^= v >> 15;
|
||||
(v % 10_000) as f64 / 10_000.0
|
||||
}
|
||||
let mut out = 0.0;
|
||||
for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] {
|
||||
let (fx, fy) = (x / cell, y / cell);
|
||||
let (i, j) = (fx.floor() as i64, fy.floor() as i64);
|
||||
let (tx, ty) = (fx - i as f64, fy - j as f64);
|
||||
let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty));
|
||||
let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx;
|
||||
let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx;
|
||||
out += amp * (a + (b - a) * sy);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Disk centre on frame `f`.
|
||||
fn center(f: u32) -> [f64; 2] {
|
||||
let t = f as f64;
|
||||
[90.0 + 4.0 * t, 110.0 + 1.5 * t + 5.0 * (t * 0.3).sin()]
|
||||
}
|
||||
|
||||
/// Disk centre with a jump at frame 8 (propagation loses it there).
|
||||
fn jumping(f: u32) -> [f64; 2] {
|
||||
if f < 8 { center(f) } else { [230.0 + 2.0 * (f - 8) as f64, 150.0] }
|
||||
}
|
||||
|
||||
fn draw(c: [f64; 2]) -> Image {
|
||||
let mut img = Image::new(W, H);
|
||||
for y in 0..H {
|
||||
for x in 0..W {
|
||||
let p = [x as f64 + 0.5, y as f64 + 0.5];
|
||||
let d = (p[0] - c[0]).hypot(p[1] - c[1]);
|
||||
let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32;
|
||||
let n = noise(p[0] - c[0], p[1] - c[1], 11);
|
||||
let f = [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32];
|
||||
let (bn, bm) = (noise(p[0], p[1], 5), noise(p[0] * 1.7, p[1] * 1.7, 9));
|
||||
let b = [(0.05 + 0.25 * bm) as f32, (0.3 + 0.4 * bn) as f32, (0.45 + 0.4 * bm) as f32];
|
||||
img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
}
|
||||
|
||||
fn gt(c: [f64; 2]) -> String {
|
||||
let m: Vec<u8> = (0..W * H).map(|i| (((i % W) as f64 + 0.5 - c[0]).hypot((i / W) as f64 + 0.5 - c[1]) <= R) as u8).collect();
|
||||
rle::encode(&m)
|
||||
}
|
||||
|
||||
fn line(a: [f64; 2], b: [f64; 2]) -> Value {
|
||||
json!((0..=8).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 8.0, a[1] + (b[1] - a[1]) * k as f64 / 8.0]).collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
/// A session with the clip (the Track solid removed) and foreground / background strokes on
|
||||
/// frame 0. `name`: the footage file name (segmentations are cached process-wide by content key,
|
||||
/// which includes the footage's identity: different clips need different names).
|
||||
fn painted(path: fn(u32) -> [f64; 2], name: &str) -> (Session, LayerId, u64) {
|
||||
let (mut s, clip, solid) = setup(move |f| draw(path(f)));
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
s.edit("remove solid", None, |p, _| {
|
||||
p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
|
||||
for it in p.items.values_mut() {
|
||||
if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
|
||||
f.path = name.into();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
s.execute("time.set", json!({"time": 0})).unwrap();
|
||||
let c = path(0);
|
||||
let r =
|
||||
s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": line([c[0] - 22.0, c[1] - 8.0], [c[0] + 22.0, c[1] + 8.0]), "radius": 6})).unwrap();
|
||||
let uid = r["effect"].as_u64().unwrap();
|
||||
assert_eq!(r["base"], 0);
|
||||
assert_eq!(r["span"], json!([0, 20]));
|
||||
for (a, b) in [([c[0] - 60.0, c[1] - 58.0], [c[0] + 70.0, c[1] - 58.0]), ([c[0] + 70.0, c[1] + 58.0], [c[0] - 60.0, c[1] + 58.0])] {
|
||||
s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "bg", "points": line(a, b), "radius": 6})).unwrap();
|
||||
}
|
||||
(s, clip, uid)
|
||||
}
|
||||
|
||||
/// Drop an instance's derived segmentations from the process-wide cache.
|
||||
fn forget(s: &Session, clip: LayerId, uid: u64) {
|
||||
let comp = s.active_comp().unwrap();
|
||||
let l = comp.layer(clip).unwrap();
|
||||
let g = l.props.find_group(uid).unwrap();
|
||||
let size = crate::roto::layer_size(&s.project, comp, l);
|
||||
let ch = effectcraft_effects::roto::Chain::new(&crate::roto::params_static(g), size, comp.frame_rate.as_f64(), 1.0);
|
||||
effectcraft_effects::roto::forget(ch.keys.values().copied());
|
||||
}
|
||||
|
||||
fn iou_at(s: &mut Session, clip: LayerId, f: u32, c: [f64; 2]) -> f64 {
|
||||
let st = s.execute("roto.status", json!({"layer": clip.0, "frame": f, "compute": true, "compareTo": gt(c)})).unwrap();
|
||||
st["iou"].as_f64().unwrap_or_else(|| panic!("no matte on frame {f}: {st}"))
|
||||
}
|
||||
|
||||
fn alpha_at(s: &Session, f: u32) -> Vec<f32> {
|
||||
let img = s.render(s.active_comp_id().unwrap(), frame_time(f), RenderOpts { scale: 1.0, ..Default::default() });
|
||||
img.data.iter().map(|p| p[3]).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roto_brush_segments_and_propagates() {
|
||||
let (mut s, clip, uid) = painted(center, "disk-a.mov");
|
||||
let l = s.active_comp().unwrap().layer(clip).unwrap().clone();
|
||||
assert_eq!(l.effects().unwrap().groups().filter(|g| crate::roto::is_roto(g)).count(), 1, "one effect for all strokes");
|
||||
let j0 = iou_at(&mut s, clip, 0, center(0));
|
||||
assert!(j0 > 0.95, "base frame IoU {j0}");
|
||||
// Propagate the span (blocking).
|
||||
let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap();
|
||||
assert_eq!(r["frames"], 21, "{r}");
|
||||
assert_eq!(r["status"]["computed"].as_array().unwrap().len(), 21);
|
||||
let mut worst: f64 = 1.0;
|
||||
for f in 0..=20 {
|
||||
worst = worst.min(iou_at(&mut s, clip, f, center(f)));
|
||||
}
|
||||
assert!(worst > 0.9, "worst IoU over the span {worst}");
|
||||
// The render applies the matte; outside the span the layer is transparent.
|
||||
let a = alpha_at(&s, 12);
|
||||
let c = center(12);
|
||||
let mut wrong = 0;
|
||||
for (i, v) in a.iter().enumerate() {
|
||||
let d = (((i as u32 % W) as f64 + 0.5 - c[0]).hypot((i as u32 / W) as f64 + 0.5 - c[1])) - R;
|
||||
if (d < -3.0 && *v < 0.5) || (d > 3.0 && *v > 0.5) {
|
||||
wrong += 1;
|
||||
}
|
||||
}
|
||||
assert!(wrong < 150, "{wrong} wrong pixels in the render");
|
||||
assert!(alpha_at(&s, 23).iter().all(|v| *v == 0.0));
|
||||
// Extending the span propagates further.
|
||||
let r = s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "to": 24, "wait": true})).unwrap();
|
||||
assert_eq!(r["status"]["span"], json!([0, 24]));
|
||||
assert!(iou_at(&mut s, clip, 24, center(24)) > 0.9);
|
||||
// Undo / redo of strokes and the span.
|
||||
let n = |s: &Session| {
|
||||
let st = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().get(effectcraft_effects::roto::STROKES).unwrap().value.clone();
|
||||
match st {
|
||||
effectcraft_keyframe::Value::Str(j) => effectcraft_track::roto::RotoData::from_json(&j),
|
||||
_ => panic!(),
|
||||
}
|
||||
};
|
||||
assert_eq!(n(&s).span, [0, 24]);
|
||||
assert!(s.undo());
|
||||
assert_eq!(n(&s).span, [0, 20]);
|
||||
assert_eq!(n(&s).strokes.len(), 3);
|
||||
assert!(s.undo());
|
||||
assert_eq!(n(&s).strokes.len(), 2);
|
||||
assert!(s.redo());
|
||||
assert!(s.redo());
|
||||
assert_eq!(n(&s).strokes.len(), 3);
|
||||
assert_eq!(n(&s).span, [0, 24]);
|
||||
// Serde keeps the strokes; the reopened project renders the same matte.
|
||||
let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap();
|
||||
let g = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone();
|
||||
assert_eq!(
|
||||
crate::roto::params_static(&g).get("strokes"),
|
||||
crate::roto::params_static(s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap()).get("strokes")
|
||||
);
|
||||
// Bad parameters are reported.
|
||||
assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "sideways", "points": [[1, 1]]})).is_err());
|
||||
assert!(s.execute_checked("roto.propagate", json!({"layer": clip.0, "bogus": 1})).is_err());
|
||||
assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": [[1, 1]], "frame": 99})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correction_strokes_restart_propagation() {
|
||||
let (mut s, clip, _) = painted(jumping, "disk-jump.mov");
|
||||
s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap();
|
||||
assert!(iou_at(&mut s, clip, 6, jumping(6)) > 0.9);
|
||||
let bad = iou_at(&mut s, clip, 8, jumping(8));
|
||||
assert!(bad < 0.5, "the jump should break propagation ({bad})");
|
||||
// Correct frame 8: paint the disk in and its old place out.
|
||||
let (c, old) = (jumping(8), center(8));
|
||||
s.execute("roto.stroke", json!({"layer": clip.0, "frame": 8, "points": line([c[0] - 20.0, c[1]], [c[0] + 20.0, c[1]]), "radius": 6})).unwrap();
|
||||
s.execute(
|
||||
"roto.stroke",
|
||||
json!({"layer": clip.0, "frame": 8, "kind": "bg", "points": line([old[0] - 15.0, old[1] - 15.0], [old[0] + 15.0, old[1] + 15.0]), "radius": 8}),
|
||||
)
|
||||
.unwrap();
|
||||
let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap();
|
||||
let computed: Vec<i64> = serde_json::from_value(st["computed"].clone()).unwrap();
|
||||
assert!(computed.contains(&7) && !computed.contains(&8) && !computed.contains(&9), "frames from the correction on are recomputed: {computed:?}");
|
||||
assert_eq!(st["strokeFrames"], json!([0, 8]));
|
||||
s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "wait": true})).unwrap();
|
||||
for f in 8..=20 {
|
||||
let j = iou_at(&mut s, clip, f, jumping(f));
|
||||
assert!(j > 0.9, "frame {f}: IoU {j}");
|
||||
}
|
||||
// Clearing the correction brings the bad frame back.
|
||||
let r = s.execute("roto.clearStrokes", json!({"layer": clip.0, "frame": 8})).unwrap();
|
||||
assert_eq!(r["removed"], 2);
|
||||
assert!(iou_at(&mut s, clip, 8, jumping(8)) < 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn freeze_caches_the_mattes() {
|
||||
let (mut s, clip, uid) = painted(center, "disk-freeze.mov");
|
||||
// A refine band along the edge on the base frame.
|
||||
let c = center(0);
|
||||
let ring: Vec<[f64; 2]> = (0..=40).map(|k| k as f64 / 40.0 * std::f64::consts::TAU).map(|a| [c[0] + R * a.cos(), c[1] + R * a.sin()]).collect();
|
||||
s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "refine", "points": ring, "radius": 5})).unwrap();
|
||||
s.execute("roto.span", json!({"layer": clip.0, "end": 3})).unwrap();
|
||||
let live: Vec<Vec<f32>> = (0..=3).map(|f| alpha_at(&s, f)).collect();
|
||||
// Freeze in the background and wait for it.
|
||||
let r = s.execute_checked("roto.freeze", json!({"layer": clip.0})).unwrap();
|
||||
assert_eq!(r["frames"], 4);
|
||||
let t0 = std::time::Instant::now();
|
||||
while s.roto_job.is_some() {
|
||||
assert!(t0.elapsed().as_secs() < 300);
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
s.poll_roto(false);
|
||||
}
|
||||
let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap();
|
||||
assert_eq!(st["frozen"], true, "{st}");
|
||||
// Frozen renders match the live ones (8-bit alpha), even without the derived segmentations.
|
||||
forget(&s, clip, uid);
|
||||
s.layer_cache = Default::default();
|
||||
for (f, want) in live.iter().enumerate() {
|
||||
let got = alpha_at(&s, f as u32);
|
||||
let worst = got.iter().zip(want).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);
|
||||
assert!(worst <= 1.0 / 255.0 + 1e-4, "frame {f}: {worst}");
|
||||
}
|
||||
assert!(st["computed"].as_array().unwrap().len() >= 4);
|
||||
// Frozen effects refuse strokes; Unfreeze is undoable.
|
||||
assert!(s.execute("roto.stroke", json!({"layer": clip.0, "points": [[10, 10]]})).is_err());
|
||||
s.execute("roto.unfreeze", json!({"layer": clip.0})).unwrap();
|
||||
assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], false);
|
||||
assert!(s.undo());
|
||||
assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], true);
|
||||
// Changing the source frames (the layer's timing) drops the frozen mattes.
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
s.edit("shift", None, |p, _| {
|
||||
let l = p.comp_mut(cid).unwrap().layer_mut(clip).unwrap();
|
||||
l.start_time = frame_time(1);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(s.execute("roto.status", json!({"layer": clip.0, "effect": uid})).unwrap()["frozen"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn propagation_is_deterministic_and_runs_in_background() {
|
||||
let (mut s, clip, uid) = painted(center, "disk-bg.mov");
|
||||
s.execute("roto.span", json!({"layer": clip.0, "end": 5})).unwrap();
|
||||
s.execute("roto.propagate", json!({"layer": clip.0})).unwrap();
|
||||
let t0 = std::time::Instant::now();
|
||||
while s.roto_job.is_some() {
|
||||
assert!(t0.elapsed().as_secs() < 300);
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
s.poll_roto(false);
|
||||
}
|
||||
let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap();
|
||||
assert_eq!(st["computed"], json!([0, 1, 2, 3, 4, 5]), "{st}");
|
||||
let a = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone();
|
||||
// Recomputing from scratch gives the same matte.
|
||||
forget(&s, clip, uid);
|
||||
let b = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone();
|
||||
assert!(a.is_string());
|
||||
assert_eq!(a, b);
|
||||
// Tool options.
|
||||
let o = s.execute_checked("roto.options", json!({"diameter": 30, "view": "alphaOverlay"})).unwrap();
|
||||
assert_eq!(o["view"], "alphaOverlay");
|
||||
assert_eq!(s.state.roto.diameter, 30.0);
|
||||
assert!(s.execute("roto.options", json!({"view": "sideways"})).is_err());
|
||||
// The default radius follows the tool diameter.
|
||||
let r = s.execute("roto.stroke", json!({"layer": clip.0, "points": [[90, 110]]})).unwrap();
|
||||
assert_eq!(r["radius"], 15.0);
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "effectcraft-track"
|
||||
description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves"
|
||||
description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and Roto Brush segmentation (graph cut, matting)"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ pub mod fit;
|
|||
pub mod klt;
|
||||
pub mod mask;
|
||||
pub mod plane;
|
||||
pub mod roto;
|
||||
pub mod solve;
|
||||
pub mod stabilize;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,228 @@
|
|||
//! Gaussian mixture colour models (full 3 × 3 covariances), as used by GrabCut (C. Rother,
|
||||
//! V. Kolmogorov and A. Blake, "GrabCut: Interactive Foreground Extraction using Iterated Graph
|
||||
//! Cuts", SIGGRAPH 2004).
|
||||
//!
|
||||
//! Fitting is deterministic: components are initialised by recursive splitting of the cluster
|
||||
//! with the largest variance along its principal axis (Orchard & Bouman, "Color Quantization of
|
||||
//! Images", IEEE TSP 1991), then refined by a few hard-assignment EM iterations.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// One Gaussian: weight, mean, inverse covariance and log normaliser.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Gauss {
|
||||
pub weight: f64,
|
||||
pub mean: [f64; 3],
|
||||
pub inv: [[f64; 3]; 3],
|
||||
/// −log(weight) + ½·log(det Σ) (the 2π term is dropped: it cancels between models).
|
||||
pub log_norm: f64,
|
||||
}
|
||||
|
||||
/// A colour model.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Gmm {
|
||||
pub comps: Vec<Gauss>,
|
||||
}
|
||||
|
||||
/// Regularisation added to every covariance's diagonal (colour noise floor).
|
||||
const REG: f64 = 2e-4;
|
||||
|
||||
fn det3(m: &[[f64; 3]; 3]) -> f64 {
|
||||
m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0])
|
||||
}
|
||||
|
||||
fn inv3(m: &[[f64; 3]; 3]) -> Option<[[f64; 3]; 3]> {
|
||||
let d = det3(m);
|
||||
if d.abs() < 1e-30 {
|
||||
return None;
|
||||
}
|
||||
let mut r = [[0.0; 3]; 3];
|
||||
r[0][0] = (m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d;
|
||||
r[0][1] = (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d;
|
||||
r[0][2] = (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d;
|
||||
r[1][0] = (m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d;
|
||||
r[1][1] = (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d;
|
||||
r[1][2] = (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d;
|
||||
r[2][0] = (m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d;
|
||||
r[2][1] = (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d;
|
||||
r[2][2] = (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d;
|
||||
Some(r)
|
||||
}
|
||||
|
||||
/// Sufficient statistics of a cluster.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct Stats {
|
||||
n: f64,
|
||||
s: [f64; 3],
|
||||
ss: [[f64; 3]; 3],
|
||||
}
|
||||
|
||||
impl Stats {
|
||||
fn add(&mut self, c: [f32; 3]) {
|
||||
let c = [c[0] as f64, c[1] as f64, c[2] as f64];
|
||||
self.n += 1.0;
|
||||
for i in 0..3 {
|
||||
self.s[i] += c[i];
|
||||
for j in 0..3 {
|
||||
self.ss[i][j] += c[i] * c[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
fn mean(&self) -> [f64; 3] {
|
||||
let n = self.n.max(1.0);
|
||||
[self.s[0] / n, self.s[1] / n, self.s[2] / n]
|
||||
}
|
||||
fn cov(&self) -> [[f64; 3]; 3] {
|
||||
let n = self.n.max(1.0);
|
||||
let m = self.mean();
|
||||
let mut c = [[0.0; 3]; 3];
|
||||
for i in 0..3 {
|
||||
for j in 0..3 {
|
||||
c[i][j] = self.ss[i][j] / n - m[i] * m[j];
|
||||
}
|
||||
c[i][i] += REG;
|
||||
}
|
||||
c
|
||||
}
|
||||
}
|
||||
|
||||
/// Principal eigenvector and eigenvalue of a symmetric 3 × 3 matrix (power iteration).
|
||||
fn principal(c: &[[f64; 3]; 3]) -> ([f64; 3], f64) {
|
||||
let mut v = [0.577, 0.577, 0.577];
|
||||
let mut lambda = 0.0;
|
||||
for _ in 0..40 {
|
||||
let w = [
|
||||
c[0][0] * v[0] + c[0][1] * v[1] + c[0][2] * v[2],
|
||||
c[1][0] * v[0] + c[1][1] * v[1] + c[1][2] * v[2],
|
||||
c[2][0] * v[0] + c[2][1] * v[1] + c[2][2] * v[2],
|
||||
];
|
||||
let n = (w[0] * w[0] + w[1] * w[1] + w[2] * w[2]).sqrt();
|
||||
if n < 1e-30 {
|
||||
break;
|
||||
}
|
||||
lambda = n;
|
||||
v = [w[0] / n, w[1] / n, w[2] / n];
|
||||
}
|
||||
(v, lambda)
|
||||
}
|
||||
|
||||
impl Gmm {
|
||||
/// Fit `k` components to `samples` (at most `max_samples` are used, taken at a fixed stride).
|
||||
pub fn fit(samples: &[[f32; 3]], k: usize, max_samples: usize) -> Gmm {
|
||||
if samples.is_empty() {
|
||||
return Gmm::default();
|
||||
}
|
||||
let stride = samples.len().div_ceil(max_samples.max(1)).max(1);
|
||||
let xs: Vec<[f32; 3]> = samples.iter().step_by(stride).copied().collect();
|
||||
// Orchard–Bouman splitting.
|
||||
let mut label = vec![0usize; xs.len()];
|
||||
let mut ncl = 1;
|
||||
while ncl < k {
|
||||
let mut st = vec![Stats::default(); ncl];
|
||||
for (x, l) in xs.iter().zip(&label) {
|
||||
st[*l].add(*x);
|
||||
}
|
||||
let (mut best, mut best_l, mut axis) = (usize::MAX, 0.0, [0.0; 3]);
|
||||
for (c, s) in st.iter().enumerate() {
|
||||
if s.n < 2.0 {
|
||||
continue;
|
||||
}
|
||||
let (v, l) = principal(&s.cov());
|
||||
if l > best_l {
|
||||
best = c;
|
||||
best_l = l;
|
||||
axis = v;
|
||||
}
|
||||
}
|
||||
if best == usize::MAX || best_l <= REG * 1.01 {
|
||||
break;
|
||||
}
|
||||
let m = st[best].mean();
|
||||
let thr = axis[0] * m[0] + axis[1] * m[1] + axis[2] * m[2];
|
||||
for (x, l) in xs.iter().zip(label.iter_mut()) {
|
||||
if *l == best && axis[0] * x[0] as f64 + axis[1] * x[1] as f64 + axis[2] * x[2] as f64 > thr {
|
||||
*l = ncl;
|
||||
}
|
||||
}
|
||||
ncl += 1;
|
||||
}
|
||||
let mut g = Gmm::from_labels(&xs, &label, ncl);
|
||||
// Hard EM.
|
||||
for _ in 0..3 {
|
||||
for (x, l) in xs.iter().zip(label.iter_mut()) {
|
||||
*l = g.best(*x);
|
||||
}
|
||||
let ng = Gmm::from_labels(&xs, &label, g.comps.len());
|
||||
if ng.comps.is_empty() {
|
||||
break;
|
||||
}
|
||||
g = ng;
|
||||
}
|
||||
g
|
||||
}
|
||||
|
||||
fn from_labels(xs: &[[f32; 3]], label: &[usize], k: usize) -> Gmm {
|
||||
let mut st = vec![Stats::default(); k];
|
||||
for (x, l) in xs.iter().zip(label) {
|
||||
st[*l].add(*x);
|
||||
}
|
||||
let total: f64 = st.iter().map(|s| s.n).sum::<f64>().max(1.0);
|
||||
let comps = st
|
||||
.iter()
|
||||
.filter(|s| s.n >= 1.0)
|
||||
.filter_map(|s| {
|
||||
let c = s.cov();
|
||||
let inv = inv3(&c)?;
|
||||
let det = det3(&c).max(1e-300);
|
||||
let weight = s.n / total;
|
||||
Some(Gauss { weight, mean: s.mean(), inv, log_norm: -weight.ln() + 0.5 * det.ln() })
|
||||
})
|
||||
.collect();
|
||||
Gmm { comps }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn comp_cost(g: &Gauss, c: [f32; 3]) -> f64 {
|
||||
let d = [c[0] as f64 - g.mean[0], c[1] as f64 - g.mean[1], c[2] as f64 - g.mean[2]];
|
||||
let mut q = 0.0;
|
||||
for i in 0..3 {
|
||||
q += d[i] * (g.inv[i][0] * d[0] + g.inv[i][1] * d[1] + g.inv[i][2] * d[2]);
|
||||
}
|
||||
g.log_norm + 0.5 * q
|
||||
}
|
||||
|
||||
/// The most likely component of colour `c`.
|
||||
pub fn best(&self, c: [f32; 3]) -> usize {
|
||||
let mut b = 0;
|
||||
let mut bv = f64::INFINITY;
|
||||
for (i, g) in self.comps.iter().enumerate() {
|
||||
let v = Self::comp_cost(g, c);
|
||||
if v < bv {
|
||||
bv = v;
|
||||
b = i;
|
||||
}
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// −log p(c) (up to a constant shared by all models), clamped to a finite range.
|
||||
pub fn cost(&self, c: [f32; 3]) -> f64 {
|
||||
if self.comps.is_empty() {
|
||||
return 10.0;
|
||||
}
|
||||
// log-sum-exp over components.
|
||||
let mut m = f64::INFINITY;
|
||||
let mut costs = [0.0f64; 16];
|
||||
let n = self.comps.len().min(16);
|
||||
for (i, g) in self.comps.iter().take(n).enumerate() {
|
||||
costs[i] = Self::comp_cost(g, c);
|
||||
m = m.min(costs[i]);
|
||||
}
|
||||
let s: f64 = costs[..n].iter().map(|v| (m - v).exp()).sum();
|
||||
(m - s.ln()).clamp(-20.0, 60.0)
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.comps.is_empty()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,465 @@
|
|||
//! Matte utilities: exact Euclidean distance transforms (Felzenszwalb & Huttenlocher,
|
||||
//! "Distance Transforms of Sampled Functions", Theory of Computing 8, 2012), box and Gaussian
|
||||
//! filters, the colour **guided filter** (K. He, J. Sun and X. Tang, "Guided Image Filtering",
|
||||
//! ECCV 2010 / IEEE PAMI 2013) used for edge matting, foreground colour estimation
|
||||
//! (decontamination, from the compositing equation I = αF + (1 − α)B with B and F estimated from
|
||||
//! nearby pure background / foreground pixels), and the Refine Edge matte adjustments.
|
||||
|
||||
use rayon::prelude::*;
|
||||
|
||||
const BIG: f32 = 1e12;
|
||||
|
||||
/// 1-D squared distance transform of `f` (lower envelope of parabolas).
|
||||
fn dt1(f: &[f32], d: &mut [f32], v: &mut [usize], z: &mut [f64]) {
|
||||
let n = f.len();
|
||||
if n == 0 {
|
||||
return;
|
||||
}
|
||||
let ff = |q: usize| f[q] as f64 + (q * q) as f64;
|
||||
let mut k = 0usize;
|
||||
v[0] = 0;
|
||||
z[0] = f64::NEG_INFINITY;
|
||||
z[1] = f64::INFINITY;
|
||||
for q in 1..n {
|
||||
let mut s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64));
|
||||
while s <= z[k] {
|
||||
if k == 0 {
|
||||
break;
|
||||
}
|
||||
k -= 1;
|
||||
s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64));
|
||||
}
|
||||
k += 1;
|
||||
v[k] = q;
|
||||
z[k] = s;
|
||||
z[k + 1] = f64::INFINITY;
|
||||
}
|
||||
k = 0;
|
||||
for (q, dq) in d.iter_mut().enumerate().take(n) {
|
||||
while z[k + 1] < q as f64 {
|
||||
k += 1;
|
||||
}
|
||||
let p = v[k];
|
||||
let dd = q as f64 - p as f64;
|
||||
*dq = (dd * dd + f[p] as f64) as f32;
|
||||
}
|
||||
}
|
||||
|
||||
/// Euclidean distance from every pixel to the nearest pixel where `seed` is true (BIG.sqrt()
|
||||
/// when there is none).
|
||||
pub fn distance(seed: &[bool], w: usize, h: usize) -> Vec<f32> {
|
||||
if w == 0 || h == 0 {
|
||||
return vec![];
|
||||
}
|
||||
let mut g: Vec<f32> = seed.iter().map(|s| if *s { 0.0 } else { BIG }).collect();
|
||||
// Columns.
|
||||
let cols: Vec<Vec<f32>> = (0..w)
|
||||
.into_par_iter()
|
||||
.map(|x| {
|
||||
let f: Vec<f32> = (0..h).map(|y| g[y * w + x]).collect();
|
||||
let mut d = vec![0.0; h];
|
||||
let (mut v, mut z) = (vec![0usize; h], vec![0.0f64; h + 1]);
|
||||
dt1(&f, &mut d, &mut v, &mut z);
|
||||
d
|
||||
})
|
||||
.collect();
|
||||
for (x, c) in cols.iter().enumerate() {
|
||||
for y in 0..h {
|
||||
g[y * w + x] = c[y];
|
||||
}
|
||||
}
|
||||
// Rows.
|
||||
g.par_chunks_mut(w).for_each(|row| {
|
||||
let f = row.to_vec();
|
||||
let (mut v, mut z) = (vec![0usize; w], vec![0.0f64; w + 1]);
|
||||
dt1(&f, row, &mut v, &mut z);
|
||||
});
|
||||
g.par_iter_mut().for_each(|v| *v = v.sqrt());
|
||||
g
|
||||
}
|
||||
|
||||
/// Signed distance to a binary matte's edge: positive inside (distance to the nearest
|
||||
/// background pixel centre, minus ½), negative outside.
|
||||
pub fn signed_distance(m: &[u8], w: usize, h: usize) -> Vec<f32> {
|
||||
let fg: Vec<bool> = m.iter().map(|v| *v != 0).collect();
|
||||
let bg: Vec<bool> = m.iter().map(|v| *v == 0).collect();
|
||||
let din = distance(&bg, w, h);
|
||||
let dout = distance(&fg, w, h);
|
||||
m.iter().enumerate().map(|(i, v)| if *v != 0 { din[i] - 0.5 } else { 0.5 - dout[i] }).collect()
|
||||
}
|
||||
|
||||
/// Mean over the (2r+1)² window (clamped at the edges: averages only the pixels inside).
|
||||
pub fn box_mean(src: &[f32], w: usize, h: usize, r: usize) -> Vec<f32> {
|
||||
if w == 0 || h == 0 {
|
||||
return vec![];
|
||||
}
|
||||
// Horizontal window sums per row.
|
||||
let mut tmp = vec![0.0f64; w * h];
|
||||
tmp.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
|
||||
let s = &src[y * w..y * w + w];
|
||||
let mut acc = 0.0f64;
|
||||
for x in 0..r.min(w) {
|
||||
acc += s[x] as f64;
|
||||
}
|
||||
for (x, o) in row.iter_mut().enumerate() {
|
||||
if x + r < w {
|
||||
acc += s[x + r] as f64;
|
||||
}
|
||||
if x > r {
|
||||
acc -= s[x - r - 1] as f64;
|
||||
}
|
||||
*o = acc;
|
||||
}
|
||||
});
|
||||
// Vertical prefix sums (row by row, cache friendly), then window differences.
|
||||
let mut pre = vec![0.0f64; w * (h + 1)];
|
||||
for y in 0..h {
|
||||
let (done, rest) = pre.split_at_mut((y + 1) * w);
|
||||
let prev = &done[y * w..];
|
||||
let cur = &mut rest[..w];
|
||||
let t = &tmp[y * w..y * w + w];
|
||||
for x in 0..w {
|
||||
cur[x] = prev[x] + t[x];
|
||||
}
|
||||
}
|
||||
let mut out = vec![0.0f32; w * h];
|
||||
out.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
|
||||
let a = y.saturating_sub(r);
|
||||
let b = (y + r + 1).min(h);
|
||||
let cy = (b - a) as f64;
|
||||
for (x, o) in row.iter_mut().enumerate() {
|
||||
let cx = ((x + r + 1).min(w) - x.saturating_sub(r)) as f64;
|
||||
*o = ((pre[b * w + x] - pre[a * w + x]) / (cx * cy)) as f32;
|
||||
}
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
/// Approximate Gaussian blur (three box passes) with standard deviation `sigma` pixels.
|
||||
pub fn gauss(src: &[f32], w: usize, h: usize, sigma: f64) -> Vec<f32> {
|
||||
if sigma < 0.3 {
|
||||
return src.to_vec();
|
||||
}
|
||||
// Box radius whose three passes match the variance: 3·((2r+1)² − 1)/12 = σ².
|
||||
let r = (((4.0 * sigma * sigma + 1.0).sqrt() - 1.0) / 2.0).round().max(1.0) as usize;
|
||||
let a = box_mean(src, w, h, r);
|
||||
let b = box_mean(&a, w, h, r);
|
||||
box_mean(&b, w, h, r)
|
||||
}
|
||||
|
||||
fn inv3(m: [[f32; 3]; 3]) -> [[f32; 3]; 3] {
|
||||
let d = m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0])
|
||||
+ m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]);
|
||||
let d = if d.abs() < 1e-20 { 1e-20 } else { d };
|
||||
[
|
||||
[(m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d, (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d, (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d],
|
||||
[(m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d, (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d, (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d],
|
||||
[(m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d, (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d, (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d],
|
||||
]
|
||||
}
|
||||
|
||||
/// The colour guided filter of `p` with guide `img` (RGB, w × h), window radius `r`,
|
||||
/// regularisation `eps` (He et al. 2010, section 3.5).
|
||||
pub fn guided_filter(img: &[[f32; 3]], p: &[f32], w: usize, h: usize, r: usize, eps: f32) -> Vec<f32> {
|
||||
let n = w * h;
|
||||
let ch = |c: usize| -> Vec<f32> { img.iter().map(|v| v[c]).collect() };
|
||||
let (ir, ig, ib) = (ch(0), ch(1), ch(2));
|
||||
let mul = |a: &[f32], b: &[f32]| -> Vec<f32> { a.iter().zip(b).map(|(x, y)| x * y).collect() };
|
||||
let bm = |v: &[f32]| box_mean(v, w, h, r);
|
||||
let (mr, mg, mb, mp) = (bm(&ir), bm(&ig), bm(&ib), bm(p));
|
||||
let (mrp, mgp, mbp) = (bm(&mul(&ir, p)), bm(&mul(&ig, p)), bm(&mul(&ib, p)));
|
||||
let (vrr, vrg, vrb) = (bm(&mul(&ir, &ir)), bm(&mul(&ir, &ig)), bm(&mul(&ir, &ib)));
|
||||
let (vgg, vgb, vbb) = (bm(&mul(&ig, &ig)), bm(&mul(&ig, &ib)), bm(&mul(&ib, &ib)));
|
||||
let mut a = vec![[0.0f32; 3]; n];
|
||||
let mut b = vec![0.0f32; n];
|
||||
a.par_iter_mut().zip(b.par_iter_mut()).enumerate().for_each(|(i, (ai, bi))| {
|
||||
let cov = [mrp[i] - mr[i] * mp[i], mgp[i] - mg[i] * mp[i], mbp[i] - mb[i] * mp[i]];
|
||||
let s = [
|
||||
[vrr[i] - mr[i] * mr[i] + eps, vrg[i] - mr[i] * mg[i], vrb[i] - mr[i] * mb[i]],
|
||||
[vrg[i] - mr[i] * mg[i], vgg[i] - mg[i] * mg[i] + eps, vgb[i] - mg[i] * mb[i]],
|
||||
[vrb[i] - mr[i] * mb[i], vgb[i] - mg[i] * mb[i], vbb[i] - mb[i] * mb[i] + eps],
|
||||
];
|
||||
let inv = inv3(s);
|
||||
let av = [
|
||||
cov[0] * inv[0][0] + cov[1] * inv[1][0] + cov[2] * inv[2][0],
|
||||
cov[0] * inv[0][1] + cov[1] * inv[1][1] + cov[2] * inv[2][1],
|
||||
cov[0] * inv[0][2] + cov[1] * inv[1][2] + cov[2] * inv[2][2],
|
||||
];
|
||||
*ai = av;
|
||||
*bi = mp[i] - av[0] * mr[i] - av[1] * mg[i] - av[2] * mb[i];
|
||||
});
|
||||
let (a0, a1, a2): (Vec<f32>, Vec<f32>, Vec<f32>) = (a.iter().map(|v| v[0]).collect(), a.iter().map(|v| v[1]).collect(), a.iter().map(|v| v[2]).collect());
|
||||
let (ma0, ma1, ma2, mbb) = (bm(&a0), bm(&a1), bm(&a2), bm(&b));
|
||||
(0..n).into_par_iter().map(|i| ma0[i] * ir[i] + ma1[i] * ig[i] + ma2[i] * ib[i] + mbb[i]).collect()
|
||||
}
|
||||
|
||||
/// Closed-form matting (A. Levin, D. Lischinski and Y. Weiss, "A Closed-Form Solution to Natural
|
||||
/// Image Matting", IEEE PAMI 30(2), 2008): the alpha that minimises αᵀLα with the matting
|
||||
/// Laplacian `L` of the colour image, with `known` pixels fixed to `init`. Solved by conjugate
|
||||
/// gradients over the unknown pixels, starting from `init`; `L·x` is evaluated with box filters
|
||||
/// (K. He, J. Sun and X. Tang, "Fast Matting Using Large Kernel Matting Laplacian Matrices",
|
||||
/// CVPR 2010): `(L·x)ᵢ = Σ_{k∋i} (xᵢ − a_kᵀIᵢ − b_k)` with `(a_k, b_k)` the ridge-regression
|
||||
/// fit of `x` to the colours in window k.
|
||||
pub fn closed_form(img: &[[f32; 3]], init: &[f32], known: &[bool], w: usize, h: usize, r: usize, eps: f32, iters: usize) -> Vec<f32> {
|
||||
let n = w * h;
|
||||
if n == 0 || known.iter().all(|k| *k) {
|
||||
return init.to_vec();
|
||||
}
|
||||
let ch = |c: usize| -> Vec<f32> { img.iter().map(|v| v[c]).collect() };
|
||||
let ic = [ch(0), ch(1), ch(2)];
|
||||
let bm = |v: &[f32]| box_mean(v, w, h, r);
|
||||
let mu = [bm(&ic[0]), bm(&ic[1]), bm(&ic[2])];
|
||||
let prod = |a: usize, b: usize| -> Vec<f32> { ic[a].iter().zip(&ic[b]).map(|(x, y)| x * y).collect() };
|
||||
let mut cov = [[vec![], vec![], vec![]], [vec![], vec![], vec![]], [vec![], vec![], vec![]]];
|
||||
for a in 0..3 {
|
||||
for b in a..3 {
|
||||
let m = bm(&prod(a, b));
|
||||
cov[a][b] = m.clone();
|
||||
cov[b][a] = m;
|
||||
}
|
||||
}
|
||||
let inv: Vec<[[f32; 3]; 3]> = (0..n)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let mut s = [[0.0f32; 3]; 3];
|
||||
for a in 0..3 {
|
||||
for b in 0..3 {
|
||||
s[a][b] = cov[a][b][i] - mu[a][i] * mu[b][i] + if a == b { eps } else { 0.0 };
|
||||
}
|
||||
}
|
||||
inv3(s)
|
||||
})
|
||||
.collect();
|
||||
// Pixels per window (= windows per pixel).
|
||||
let cnt: Vec<f32> = {
|
||||
let (rr, mut out) = (r as i64, vec![0.0f32; n]);
|
||||
for (i, o) in out.iter_mut().enumerate() {
|
||||
let (x, y) = ((i % w) as i64, (i / w) as i64);
|
||||
let cx = (x + rr).min(w as i64 - 1) - (x - rr).max(0) + 1;
|
||||
let cy = (y + rr).min(h as i64 - 1) - (y - rr).max(0) + 1;
|
||||
*o = (cx * cy) as f32;
|
||||
}
|
||||
out
|
||||
};
|
||||
let lap = |x: &[f32]| -> Vec<f32> {
|
||||
let pm = bm(x);
|
||||
let ipm: Vec<Vec<f32>> = (0..3).map(|c| bm(&ic[c].iter().zip(x).map(|(a, b)| a * b).collect::<Vec<_>>())).collect();
|
||||
let ab: Vec<[f32; 4]> = (0..n)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let cv = [ipm[0][i] - mu[0][i] * pm[i], ipm[1][i] - mu[1][i] * pm[i], ipm[2][i] - mu[2][i] * pm[i]];
|
||||
let m = &inv[i];
|
||||
let a = [
|
||||
m[0][0] * cv[0] + m[0][1] * cv[1] + m[0][2] * cv[2],
|
||||
m[1][0] * cv[0] + m[1][1] * cv[1] + m[1][2] * cv[2],
|
||||
m[2][0] * cv[0] + m[2][1] * cv[1] + m[2][2] * cv[2],
|
||||
];
|
||||
[a[0], a[1], a[2], pm[i] - a[0] * mu[0][i] - a[1] * mu[1][i] - a[2] * mu[2][i]]
|
||||
})
|
||||
.collect();
|
||||
let sa: Vec<Vec<f32>> = (0..3).map(|c| bm(&ab.iter().map(|v| v[c]).collect::<Vec<_>>())).collect();
|
||||
let sb = bm(&ab.iter().map(|v| v[3]).collect::<Vec<_>>());
|
||||
(0..n).into_par_iter().map(|i| cnt[i] * (x[i] - sa[0][i] * ic[0][i] - sa[1][i] * ic[1][i] - sa[2][i] * ic[2][i] - sb[i])).collect()
|
||||
};
|
||||
// L_UU x = −(L α_K)_U.
|
||||
let a0: Vec<f32> = (0..n).map(|i| if known[i] { init[i] } else { 0.0 }).collect();
|
||||
let la0 = lap(&a0);
|
||||
let mut x: Vec<f32> = (0..n).map(|i| if known[i] { 0.0 } else { init[i] }).collect();
|
||||
let lx = lap(&x);
|
||||
let mut res: Vec<f64> = (0..n).map(|i| if known[i] { 0.0 } else { (-la0[i] - lx[i]) as f64 }).collect();
|
||||
let mut p = res.clone();
|
||||
let mut rs: f64 = res.iter().map(|v| v * v).sum();
|
||||
let rs0 = rs.max(1e-30);
|
||||
for _ in 0..iters {
|
||||
if rs <= rs0 * 1e-8 {
|
||||
break;
|
||||
}
|
||||
let pf: Vec<f32> = p.iter().map(|v| *v as f32).collect();
|
||||
let ap: Vec<f64> = lap(&pf).iter().enumerate().map(|(i, v)| if known[i] { 0.0 } else { *v as f64 }).collect();
|
||||
let pap: f64 = p.iter().zip(&ap).map(|(a, b)| a * b).sum();
|
||||
if pap <= 1e-30 {
|
||||
break;
|
||||
}
|
||||
let alpha = rs / pap;
|
||||
for i in 0..n {
|
||||
x[i] += (alpha * p[i]) as f32;
|
||||
res[i] -= alpha * ap[i];
|
||||
}
|
||||
let rs_new: f64 = res.iter().map(|v| v * v).sum();
|
||||
let beta = rs_new / rs;
|
||||
rs = rs_new;
|
||||
for i in 0..n {
|
||||
p[i] = res[i] + beta * p[i];
|
||||
}
|
||||
}
|
||||
(0..n).map(|i| if known[i] { init[i] } else { x[i].clamp(0.0, 1.0) }).collect()
|
||||
}
|
||||
|
||||
/// Contrast (0–100 %) around ½ on a soft alpha.
|
||||
pub fn contrast(a: &mut [f32], pc: f64) {
|
||||
if pc <= 0.0 {
|
||||
return;
|
||||
}
|
||||
let k = 1.0 / (1.0 - 0.99 * (pc / 100.0).clamp(0.0, 1.0)) as f32;
|
||||
a.par_iter_mut().for_each(|v| *v = (0.5 + (*v - 0.5) * k).clamp(0.0, 1.0));
|
||||
}
|
||||
|
||||
/// Shift Edge (−100–100 %) on a soft alpha: positive values grow the matte (a gamma curve on α).
|
||||
pub fn shift_soft(a: &mut [f32], pc: f64) {
|
||||
if pc.abs() < 1e-9 {
|
||||
return;
|
||||
}
|
||||
let g = 2f32.powf((-pc / 50.0) as f32);
|
||||
a.par_iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0).powf(g));
|
||||
}
|
||||
|
||||
/// Estimate the pure foreground colour of every pixel listed in `region` (alpha strictly
|
||||
/// between 0 and 1) and mix it in by `amount` (0–1). `radius`: neighbourhood (pixels) that pure
|
||||
/// background / foreground estimates are taken from. Returns the decontamination map (how much
|
||||
/// each pixel's colour changed, 0–1).
|
||||
pub fn decontaminate(img: &mut [[f32; 3]], alpha: &[f32], region: &[bool], w: usize, h: usize, radius: usize, amount: f32) -> Vec<f32> {
|
||||
let n = w * h;
|
||||
let mut map = vec![0.0f32; n];
|
||||
if !region.iter().any(|r| *r) || amount <= 0.0 {
|
||||
return map;
|
||||
}
|
||||
let lo: Vec<f32> = alpha.iter().map(|a| if *a < 0.02 { 1.0 } else { 0.0 }).collect();
|
||||
let hi: Vec<f32> = alpha.iter().map(|a| if *a > 0.98 { 1.0 } else { 0.0 }).collect();
|
||||
// Windowed means of pure background / foreground colours, widening where none is near.
|
||||
let est = |m: &[f32]| -> Vec<Option<[f32; 3]>> {
|
||||
let mut out: Vec<Option<[f32; 3]>> = vec![None; n];
|
||||
let mut r = radius.max(2);
|
||||
for _ in 0..4 {
|
||||
let cnt = box_mean(m, w, h, r);
|
||||
let sums: Vec<Vec<f32>> = (0..3).map(|c| box_mean(&img.iter().zip(m).map(|(p, k)| p[c] * k).collect::<Vec<_>>(), w, h, r)).collect();
|
||||
let mut missing = false;
|
||||
for i in 0..n {
|
||||
if region[i] && out[i].is_none() {
|
||||
if cnt[i] > 1e-4 {
|
||||
out[i] = Some([sums[0][i] / cnt[i], sums[1][i] / cnt[i], sums[2][i] / cnt[i]]);
|
||||
} else {
|
||||
missing = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !missing {
|
||||
break;
|
||||
}
|
||||
r *= 2;
|
||||
}
|
||||
out
|
||||
};
|
||||
let bgc = est(&lo);
|
||||
let fgc = est(&hi);
|
||||
for i in 0..n {
|
||||
if !region[i] {
|
||||
continue;
|
||||
}
|
||||
let a = alpha[i].clamp(0.0, 1.0);
|
||||
if a <= 0.0 || a >= 1.0 {
|
||||
continue;
|
||||
}
|
||||
let c = img[i];
|
||||
let f = match (bgc[i], fgc[i]) {
|
||||
(Some(b), fbar) => {
|
||||
let mut f = [0.0f32; 3];
|
||||
for k in 0..3 {
|
||||
f[k] = ((c[k] - (1.0 - a) * b[k]) / a).clamp(0.0, 1.0);
|
||||
}
|
||||
// Low alphas amplify noise: lean on the nearby foreground colour there.
|
||||
if let Some(fb) = fbar {
|
||||
let t = ((0.25 - a) / 0.25).clamp(0.0, 1.0);
|
||||
for k in 0..3 {
|
||||
f[k] += (fb[k] - f[k]) * t;
|
||||
}
|
||||
}
|
||||
f
|
||||
}
|
||||
(None, Some(fb)) => fb,
|
||||
(None, None) => c,
|
||||
};
|
||||
let mut d = 0.0f32;
|
||||
for k in 0..3 {
|
||||
let v = c[k] + (f[k] - c[k]) * amount;
|
||||
d = d.max((v - c[k]).abs());
|
||||
img[i][k] = v;
|
||||
}
|
||||
map[i] = d.min(1.0);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// Sample a float plane bilinearly at continuous coordinates (pixel centres at +0.5), clamped.
|
||||
pub fn sample(p: &[f32], w: usize, h: usize, x: f64, y: f64) -> f32 {
|
||||
if w == 0 || h == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let fx = (x - 0.5).clamp(0.0, (w - 1) as f64);
|
||||
let fy = (y - 0.5).clamp(0.0, (h - 1) as f64);
|
||||
let (x0, y0) = (fx.floor() as usize, fy.floor() as usize);
|
||||
let (x1, y1) = ((x0 + 1).min(w - 1), (y0 + 1).min(h - 1));
|
||||
let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32);
|
||||
let g = |x: usize, y: usize| p[y * w + x];
|
||||
let top = g(x0, y0) + (g(x1, y0) - g(x0, y0)) * tx;
|
||||
let bot = g(x0, y1) + (g(x1, y1) - g(x0, y1)) * tx;
|
||||
top + (bot - top) * ty
|
||||
}
|
||||
|
||||
/// Average of `a` sampled along the segment `[-v/2, v/2]` with `samples` taps (motion blur of a
|
||||
/// matte moving by `v` pixels during the shutter).
|
||||
pub fn motion_blur(a: &[f32], w: usize, h: usize, v: [f64; 2], samples: usize) -> Vec<f32> {
|
||||
let n = samples.max(2);
|
||||
if v[0].hypot(v[1]) < 0.25 {
|
||||
return a.to_vec();
|
||||
}
|
||||
(0..w * h)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5);
|
||||
let mut s = 0.0;
|
||||
for k in 0..n {
|
||||
let t = k as f64 / (n - 1) as f64 - 0.5;
|
||||
s += sample(a, w, h, x - v[0] * t, y - v[1] * t);
|
||||
}
|
||||
s / n as f32
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn distance_matches_brute_force() {
|
||||
let (w, h) = (23, 17);
|
||||
let seed: Vec<bool> = (0..w * h).map(|i| (i * 7919) % 31 == 0).collect();
|
||||
let d = distance(&seed, w, h);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let mut b = f32::INFINITY;
|
||||
for j in 0..h {
|
||||
for i in 0..w {
|
||||
if seed[j * w + i] {
|
||||
b = b.min(((x as f32 - i as f32).powi(2) + (y as f32 - j as f32).powi(2)).sqrt());
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!((d[y * w + x] - b).abs() < 1e-3, "{x},{y}: {} vs {b}", d[y * w + x]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guided_filter_recovers_a_linear_blend() {
|
||||
// I = αF + (1 − α)B with constant F, B: the filtered binary matte is close to α.
|
||||
let (w, h) = (64, 8);
|
||||
let alpha: Vec<f32> = (0..w * h).map(|i| (((i % w) as f32 - 26.0) / 12.0).clamp(0.0, 1.0)).collect();
|
||||
let (f, b) = ([0.9f32, 0.2, 0.1], [0.1f32, 0.3, 0.8]);
|
||||
let img: Vec<[f32; 3]> = alpha.iter().map(|a| [a * f[0] + (1.0 - a) * b[0], a * f[1] + (1.0 - a) * b[1], a * f[2] + (1.0 - a) * b[2]]).collect();
|
||||
let p: Vec<f32> = alpha.iter().map(|a| if *a >= 0.5 { 1.0 } else { 0.0 }).collect();
|
||||
let q = guided_filter(&img, &p, w, h, 8, 1e-5);
|
||||
let err: f32 = q.iter().zip(&alpha).map(|(a, b)| (a.clamp(0.0, 1.0) - b).abs()).sum::<f32>() / (w * h) as f32;
|
||||
assert!(err < 0.03, "mean error {err}");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,438 @@
|
|||
//! Min-cut / max-flow on sparse graphs with source and sink terminals, by the
|
||||
//! Boykov–Kolmogorov algorithm (Y. Boykov and V. Kolmogorov, "An Experimental Comparison of
|
||||
//! Min-Cut/Max-Flow Algorithms for Energy Minimization in Vision", IEEE PAMI 26(9), 2004).
|
||||
//!
|
||||
//! Two search trees grow from the terminals (S from the source, T from the sink) through
|
||||
//! non-saturated edges; when they touch, the path found is augmented; saturated tree edges turn
|
||||
//! the nodes below them into *orphans*, which the adoption stage re-attaches to their tree (or
|
||||
//! frees). Trees are reused between augmentations, which is what makes the method fast on the
|
||||
//! grid graphs of image segmentation. The timestamp/distance heuristic of section 3.2.3 keeps
|
||||
//! adoption paths short.
|
||||
//!
|
||||
//! Capacities are integers (callers scale their energies), so saturation tests are exact and
|
||||
//! the result is deterministic.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
|
||||
const NONE: u32 = u32::MAX;
|
||||
const TERMINAL: u32 = u32::MAX - 1;
|
||||
const ORPHAN: u32 = u32::MAX - 2;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct Node {
|
||||
first: u32,
|
||||
/// Arc to the parent (from this node), or TERMINAL / ORPHAN / NONE (free).
|
||||
parent: u32,
|
||||
/// Residual terminal capacity: > 0 towards the source, < 0 towards the sink.
|
||||
tr_cap: i64,
|
||||
is_sink: bool,
|
||||
active: bool,
|
||||
ts: u32,
|
||||
dist: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct Arc {
|
||||
head: u32,
|
||||
next: u32,
|
||||
r_cap: i32,
|
||||
}
|
||||
|
||||
/// A graph for one min-cut computation.
|
||||
pub struct Graph {
|
||||
nodes: Vec<Node>,
|
||||
arcs: Vec<Arc>,
|
||||
flow: i64,
|
||||
time: u32,
|
||||
queue: VecDeque<u32>,
|
||||
orphans: VecDeque<u32>,
|
||||
}
|
||||
|
||||
/// Which side of the cut a node ended on.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Segment {
|
||||
Source,
|
||||
Sink,
|
||||
}
|
||||
|
||||
impl Graph {
|
||||
pub fn new(nodes: usize, edges_hint: usize) -> Graph {
|
||||
Graph {
|
||||
nodes: vec![Node { first: NONE, parent: NONE, tr_cap: 0, is_sink: false, active: false, ts: 0, dist: 0 }; nodes],
|
||||
arcs: Vec::with_capacity(edges_hint * 2),
|
||||
flow: 0,
|
||||
time: 0,
|
||||
queue: VecDeque::new(),
|
||||
orphans: VecDeque::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.nodes.len()
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.nodes.is_empty()
|
||||
}
|
||||
|
||||
/// Terminal capacities of node `i`: `source` is paid when `i` ends on the sink side, `sink`
|
||||
/// when it ends on the source side.
|
||||
pub fn add_tweights(&mut self, i: usize, source: i64, sink: i64) {
|
||||
let n = &mut self.nodes[i];
|
||||
let d = n.tr_cap;
|
||||
let (s, t) = if d > 0 { (source + d, sink) } else { (source, sink - d) };
|
||||
self.flow += s.min(t);
|
||||
n.tr_cap = s - t;
|
||||
}
|
||||
|
||||
/// An edge between `i` and `j` with capacity `cap` from i to j and `rev` from j to i.
|
||||
pub fn add_edge(&mut self, i: usize, j: usize, cap: i32, rev: i32) {
|
||||
debug_assert!(i != j);
|
||||
let a = self.arcs.len() as u32;
|
||||
self.arcs.push(Arc { head: j as u32, next: self.nodes[i].first, r_cap: cap });
|
||||
self.nodes[i].first = a;
|
||||
self.arcs.push(Arc { head: i as u32, next: self.nodes[j].first, r_cap: rev });
|
||||
self.nodes[j].first = a + 1;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn set_active(&mut self, i: u32) {
|
||||
let n = &mut self.nodes[i as usize];
|
||||
if !n.active {
|
||||
n.active = true;
|
||||
self.queue.push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the maximum flow (= the minimum cut's cost).
|
||||
pub fn maxflow(&mut self) -> i64 {
|
||||
for i in 0..self.nodes.len() {
|
||||
let n = &mut self.nodes[i];
|
||||
if n.tr_cap != 0 {
|
||||
n.is_sink = n.tr_cap < 0;
|
||||
n.parent = TERMINAL;
|
||||
n.ts = 0;
|
||||
n.dist = 1;
|
||||
self.set_active(i as u32);
|
||||
}
|
||||
}
|
||||
let mut current: Option<u32> = None;
|
||||
loop {
|
||||
let i = match current.take() {
|
||||
Some(i) if self.nodes[i as usize].parent != NONE => i,
|
||||
_ => loop {
|
||||
let Some(i) = self.queue.pop_front() else { return self.flow };
|
||||
self.nodes[i as usize].active = false;
|
||||
if self.nodes[i as usize].parent != NONE {
|
||||
break i;
|
||||
}
|
||||
},
|
||||
};
|
||||
// Growth.
|
||||
let mut found = NONE;
|
||||
let ni = self.nodes[i as usize];
|
||||
let mut a = ni.first;
|
||||
if !ni.is_sink {
|
||||
while a != NONE {
|
||||
let arc = self.arcs[a as usize];
|
||||
if arc.r_cap > 0 {
|
||||
let j = arc.head as usize;
|
||||
let nj = self.nodes[j];
|
||||
if nj.parent == NONE {
|
||||
let n = &mut self.nodes[j];
|
||||
n.is_sink = false;
|
||||
n.parent = a ^ 1;
|
||||
n.ts = ni.ts;
|
||||
n.dist = ni.dist + 1;
|
||||
self.set_active(j as u32);
|
||||
} else if nj.is_sink {
|
||||
found = a;
|
||||
break;
|
||||
} else if nj.ts <= ni.ts && nj.dist > ni.dist {
|
||||
let n = &mut self.nodes[j];
|
||||
n.parent = a ^ 1;
|
||||
n.ts = ni.ts;
|
||||
n.dist = ni.dist + 1;
|
||||
}
|
||||
}
|
||||
a = arc.next;
|
||||
}
|
||||
} else {
|
||||
while a != NONE {
|
||||
let arc = self.arcs[a as usize];
|
||||
if self.arcs[(a ^ 1) as usize].r_cap > 0 {
|
||||
let j = arc.head as usize;
|
||||
let nj = self.nodes[j];
|
||||
if nj.parent == NONE {
|
||||
let n = &mut self.nodes[j];
|
||||
n.is_sink = true;
|
||||
n.parent = a ^ 1;
|
||||
n.ts = ni.ts;
|
||||
n.dist = ni.dist + 1;
|
||||
self.set_active(j as u32);
|
||||
} else if !nj.is_sink {
|
||||
found = a ^ 1;
|
||||
break;
|
||||
} else if nj.ts <= ni.ts && nj.dist > ni.dist {
|
||||
let n = &mut self.nodes[j];
|
||||
n.parent = a ^ 1;
|
||||
n.ts = ni.ts;
|
||||
n.dist = ni.dist + 1;
|
||||
}
|
||||
}
|
||||
a = arc.next;
|
||||
}
|
||||
}
|
||||
self.time = self.time.wrapping_add(1);
|
||||
if found != NONE {
|
||||
// Keep `i` current: it may have more paths.
|
||||
current = Some(i);
|
||||
self.augment(found);
|
||||
self.adopt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn augment(&mut self, middle: u32) {
|
||||
let mut b = self.arcs[middle as usize].r_cap as i64;
|
||||
// Source side.
|
||||
let mut i = self.arcs[(middle ^ 1) as usize].head;
|
||||
loop {
|
||||
let a = self.nodes[i as usize].parent;
|
||||
if a == TERMINAL {
|
||||
break;
|
||||
}
|
||||
b = b.min(self.arcs[(a ^ 1) as usize].r_cap as i64);
|
||||
i = self.arcs[a as usize].head;
|
||||
}
|
||||
b = b.min(self.nodes[i as usize].tr_cap);
|
||||
// Sink side.
|
||||
let mut i = self.arcs[middle as usize].head;
|
||||
loop {
|
||||
let a = self.nodes[i as usize].parent;
|
||||
if a == TERMINAL {
|
||||
break;
|
||||
}
|
||||
b = b.min(self.arcs[a as usize].r_cap as i64);
|
||||
i = self.arcs[a as usize].head;
|
||||
}
|
||||
b = b.min(-self.nodes[i as usize].tr_cap);
|
||||
let bi = b as i32;
|
||||
self.arcs[(middle ^ 1) as usize].r_cap += bi;
|
||||
self.arcs[middle as usize].r_cap -= bi;
|
||||
// Source side.
|
||||
let mut i = self.arcs[(middle ^ 1) as usize].head;
|
||||
loop {
|
||||
let a = self.nodes[i as usize].parent;
|
||||
if a == TERMINAL {
|
||||
break;
|
||||
}
|
||||
self.arcs[a as usize].r_cap += bi;
|
||||
self.arcs[(a ^ 1) as usize].r_cap -= bi;
|
||||
if self.arcs[(a ^ 1) as usize].r_cap == 0 {
|
||||
self.make_orphan(i);
|
||||
}
|
||||
i = self.arcs[a as usize].head;
|
||||
}
|
||||
self.nodes[i as usize].tr_cap -= b;
|
||||
if self.nodes[i as usize].tr_cap == 0 {
|
||||
self.make_orphan(i);
|
||||
}
|
||||
// Sink side.
|
||||
let mut i = self.arcs[middle as usize].head;
|
||||
loop {
|
||||
let a = self.nodes[i as usize].parent;
|
||||
if a == TERMINAL {
|
||||
break;
|
||||
}
|
||||
self.arcs[(a ^ 1) as usize].r_cap += bi;
|
||||
self.arcs[a as usize].r_cap -= bi;
|
||||
if self.arcs[a as usize].r_cap == 0 {
|
||||
self.make_orphan(i);
|
||||
}
|
||||
i = self.arcs[a as usize].head;
|
||||
}
|
||||
self.nodes[i as usize].tr_cap += b;
|
||||
if self.nodes[i as usize].tr_cap == 0 {
|
||||
self.make_orphan(i);
|
||||
}
|
||||
self.flow += b;
|
||||
}
|
||||
|
||||
fn make_orphan(&mut self, i: u32) {
|
||||
self.nodes[i as usize].parent = ORPHAN;
|
||||
self.orphans.push_back(i);
|
||||
}
|
||||
|
||||
/// Whether node `j`'s path to its terminal is intact; returns its distance (marking the
|
||||
/// path with the current time) or None.
|
||||
fn origin_dist(&mut self, j: u32) -> Option<u32> {
|
||||
let mut d = 0u32;
|
||||
let mut k = j;
|
||||
loop {
|
||||
let n = self.nodes[k as usize];
|
||||
if n.ts == self.time {
|
||||
d += n.dist;
|
||||
break;
|
||||
}
|
||||
let a = n.parent;
|
||||
d += 1;
|
||||
if a == TERMINAL {
|
||||
let n = &mut self.nodes[k as usize];
|
||||
n.ts = self.time;
|
||||
n.dist = 1;
|
||||
break;
|
||||
}
|
||||
if a == ORPHAN || a == NONE {
|
||||
return None;
|
||||
}
|
||||
k = self.arcs[a as usize].head;
|
||||
}
|
||||
// Mark the path.
|
||||
let total = d;
|
||||
let mut k = j;
|
||||
let mut dd = d;
|
||||
while self.nodes[k as usize].ts != self.time {
|
||||
let n = &mut self.nodes[k as usize];
|
||||
n.ts = self.time;
|
||||
n.dist = dd;
|
||||
dd -= 1;
|
||||
k = self.arcs[n.parent as usize].head;
|
||||
}
|
||||
Some(total)
|
||||
}
|
||||
|
||||
fn adopt(&mut self) {
|
||||
while let Some(i) = self.orphans.pop_front() {
|
||||
let sink = self.nodes[i as usize].is_sink;
|
||||
let mut best = NONE;
|
||||
let mut best_d = u32::MAX;
|
||||
let mut a = self.nodes[i as usize].first;
|
||||
while a != NONE {
|
||||
let arc = self.arcs[a as usize];
|
||||
// S orphan: the parent must push flow into i (residual on j→i = sister of i→j).
|
||||
// T orphan: i must push flow into the parent (residual on i→j).
|
||||
let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap };
|
||||
if cap > 0 {
|
||||
let j = arc.head;
|
||||
let nj = self.nodes[j as usize];
|
||||
if nj.is_sink == sink
|
||||
&& nj.parent != NONE
|
||||
&& let Some(d) = self.origin_dist(j)
|
||||
&& d < best_d
|
||||
{
|
||||
best = a;
|
||||
best_d = d;
|
||||
}
|
||||
}
|
||||
a = arc.next;
|
||||
}
|
||||
if best != NONE {
|
||||
let n = &mut self.nodes[i as usize];
|
||||
n.parent = best;
|
||||
n.ts = self.time;
|
||||
n.dist = best_d + 1;
|
||||
continue;
|
||||
}
|
||||
// No parent: free the node; its children become orphans, neighbours that could
|
||||
// reach it become active.
|
||||
self.nodes[i as usize].ts = 0;
|
||||
let mut a = self.nodes[i as usize].first;
|
||||
while a != NONE {
|
||||
let arc = self.arcs[a as usize];
|
||||
let j = arc.head;
|
||||
let nj = self.nodes[j as usize];
|
||||
if nj.is_sink == sink && nj.parent != NONE {
|
||||
let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap };
|
||||
if cap > 0 {
|
||||
self.set_active(j);
|
||||
}
|
||||
let pa = nj.parent;
|
||||
if pa != TERMINAL && pa != ORPHAN && self.arcs[pa as usize].head == i {
|
||||
self.make_orphan(j);
|
||||
}
|
||||
}
|
||||
a = arc.next;
|
||||
}
|
||||
self.nodes[i as usize].parent = NONE;
|
||||
}
|
||||
}
|
||||
|
||||
/// The side of node `i` after [`Graph::maxflow`] (free nodes go to the sink).
|
||||
pub fn segment(&self, i: usize) -> Segment {
|
||||
let n = &self.nodes[i];
|
||||
if n.parent != NONE && !n.is_sink { Segment::Source } else { Segment::Sink }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Brute-force min cut over all labelings of a small graph.
|
||||
fn brute(n: usize, t: &[(i64, i64)], e: &[(usize, usize, i32, i32)]) -> i64 {
|
||||
let mut best = i64::MAX;
|
||||
for m in 0..(1u32 << n) {
|
||||
let src = |i: usize| m & (1 << i) != 0;
|
||||
let mut c = 0i64;
|
||||
for (i, (s, k)) in t.iter().enumerate() {
|
||||
c += if src(i) { *k } else { *s };
|
||||
}
|
||||
for (i, j, a, b) in e {
|
||||
if src(*i) && !src(*j) {
|
||||
c += *a as i64;
|
||||
}
|
||||
if src(*j) && !src(*i) {
|
||||
c += *b as i64;
|
||||
}
|
||||
}
|
||||
best = best.min(c);
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_brute_force_on_random_graphs() {
|
||||
let mut s = 12345u64;
|
||||
let mut rnd = |m: u64| {
|
||||
s ^= s << 13;
|
||||
s ^= s >> 7;
|
||||
s ^= s << 17;
|
||||
s % m
|
||||
};
|
||||
for _ in 0..300 {
|
||||
let n = 2 + rnd(9) as usize;
|
||||
let t: Vec<(i64, i64)> = (0..n).map(|_| (rnd(20) as i64, rnd(20) as i64)).collect();
|
||||
let mut e = vec![];
|
||||
for _ in 0..rnd(20) {
|
||||
let (i, j) = (rnd(n as u64) as usize, rnd(n as u64) as usize);
|
||||
if i != j {
|
||||
e.push((i, j, rnd(15) as i32, rnd(15) as i32));
|
||||
}
|
||||
}
|
||||
let mut g = Graph::new(n, e.len());
|
||||
for (i, (a, b)) in t.iter().enumerate() {
|
||||
g.add_tweights(i, *a, *b);
|
||||
}
|
||||
for (i, j, a, b) in &e {
|
||||
g.add_edge(*i, *j, *a, *b);
|
||||
}
|
||||
let f = g.maxflow();
|
||||
assert_eq!(f, brute(n, &t, &e));
|
||||
// The labeling realises the cut.
|
||||
let src = |i: usize| g.segment(i) == Segment::Source;
|
||||
let mut c = 0i64;
|
||||
for (i, (a, b)) in t.iter().enumerate() {
|
||||
c += if src(i) { *b } else { *a };
|
||||
}
|
||||
for (i, j, a, b) in &e {
|
||||
if src(*i) && !src(*j) {
|
||||
c += *a as i64;
|
||||
}
|
||||
if src(*j) && !src(*i) {
|
||||
c += *b as i64;
|
||||
}
|
||||
}
|
||||
assert_eq!(c, f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,754 @@
|
|||
//! Roto Brush segmentation and propagation, clean-room from published methods:
|
||||
//!
|
||||
//! - **Interactive graph-cut segmentation** (Y. Boykov and M.-P. Jolly, "Interactive Graph Cuts
|
||||
//! for Optimal Boundary & Region Segmentation of Objects in N-D Images", ICCV 2001): the user's
|
||||
//! foreground / background strokes are hard constraints, every other pixel gets a data cost
|
||||
//! from colour models and neighbouring pixels a contrast-sensitive smoothness cost; the
|
||||
//! minimum s–t cut ([`maxflow`], Boykov–Kolmogorov) is the segmentation.
|
||||
//! - **Iterated colour models** (C. Rother, V. Kolmogorov and A. Blake, "GrabCut", SIGGRAPH
|
||||
//! 2004): Gaussian mixtures ([`gmm`]) are fitted to the strokes, then re-fitted to the
|
||||
//! segmentation and the cut repeated; the 8-connected n-link weight is
|
||||
//! `γ/dist · exp(−β‖Iₚ − I_q‖²)` with `β = 1 / (2⟨‖Iₚ − I_q‖²⟩)`.
|
||||
//! - **Coarse to fine**: large frames are segmented at a reduced size, then re-cut at full
|
||||
//! resolution in a narrow band around the upsampled boundary (Lombaert et al., "A Multilevel
|
||||
//! Banded Graph Cuts Method for Fast Image Segmentation", ICCV 2005).
|
||||
//! - **Propagation** (in the spirit of X. Bai et al., "Video SnapCut", SIGGRAPH 2009): the
|
||||
//! previous frame's matte is warped to the next frame by optical flow
|
||||
//! (`effectcraft_raster::flow::block_flow`), and the frame re-segmented by graph cut only in a
|
||||
//! band of **Search Radius** pixels around the warped boundary, with the previous frame's
|
||||
//! colour models and the warped matte as a shape prior. Correction strokes on a frame re-cut
|
||||
//! the whole frame with the warped matte as a soft prior, and propagation restarts from there.
|
||||
//!
|
||||
//! Mattes are in layer pixels at a given scale (1 = full resolution). Strokes are stored in
|
||||
//! layer pixels ([`RotoData`], serde) and keyed per frame so that each frame's result is a
|
||||
//! deterministic function of its *chain*: the base frame's strokes, then every frame's strokes
|
||||
//! on the way out to it ([`RotoData::chain_keys`]).
|
||||
|
||||
pub mod gmm;
|
||||
pub mod matting;
|
||||
pub mod maxflow;
|
||||
pub mod refine;
|
||||
pub mod rle;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use effectcraft_raster::Image;
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use gmm::Gmm;
|
||||
use maxflow::{Graph, Segment};
|
||||
|
||||
/// What a stroke paints.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum StrokeKind {
|
||||
/// Roto Brush (green): foreground.
|
||||
Fg,
|
||||
/// Roto Brush with Alt (red): background.
|
||||
Bg,
|
||||
/// Refine Edge tool: mark a band for edge matting.
|
||||
Refine,
|
||||
/// Refine Edge tool with Alt: remove from the band.
|
||||
RefineErase,
|
||||
}
|
||||
|
||||
impl StrokeKind {
|
||||
pub fn from_name(s: &str) -> Option<StrokeKind> {
|
||||
Some(match s.to_ascii_lowercase().as_str() {
|
||||
"fg" | "foreground" => StrokeKind::Fg,
|
||||
"bg" | "background" => StrokeKind::Bg,
|
||||
"refine" | "refineedge" => StrokeKind::Refine,
|
||||
"refineerase" | "refine-erase" | "erase" => StrokeKind::RefineErase,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
pub fn is_segmentation(self) -> bool {
|
||||
matches!(self, StrokeKind::Fg | StrokeKind::Bg)
|
||||
}
|
||||
}
|
||||
|
||||
/// One stroke: a polyline swept by a disc of `radius` layer pixels, on layer frame `frame`.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Stroke {
|
||||
pub kind: StrokeKind,
|
||||
pub frame: i64,
|
||||
pub radius: f64,
|
||||
pub points: Vec<[f64; 2]>,
|
||||
}
|
||||
|
||||
/// A Roto Brush instance's user data: strokes, the base frame and the segmentation span
|
||||
/// (inclusive layer frames).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct RotoData {
|
||||
pub base: Option<i64>,
|
||||
pub span: [i64; 2],
|
||||
pub strokes: Vec<Stroke>,
|
||||
}
|
||||
|
||||
fn fnv(h: &mut u64, bytes: &[u8]) {
|
||||
for b in bytes {
|
||||
*h ^= *b as u64;
|
||||
*h = h.wrapping_mul(0x0100_0000_01b3);
|
||||
}
|
||||
}
|
||||
|
||||
impl RotoData {
|
||||
pub fn from_json(s: &str) -> RotoData {
|
||||
if s.trim().is_empty() {
|
||||
return RotoData::default();
|
||||
}
|
||||
serde_json::from_str(s).unwrap_or_default()
|
||||
}
|
||||
pub fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_default()
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.base.is_none() || !self.strokes.iter().any(|s| s.kind.is_segmentation())
|
||||
}
|
||||
pub fn strokes_at(&self, frame: i64) -> Vec<&Stroke> {
|
||||
self.strokes.iter().filter(|s| s.frame == frame).collect()
|
||||
}
|
||||
/// Frames that carry strokes.
|
||||
pub fn stroke_frames(&self) -> Vec<i64> {
|
||||
let mut v: Vec<i64> = self.strokes.iter().map(|s| s.frame).collect();
|
||||
v.sort_unstable();
|
||||
v.dedup();
|
||||
v
|
||||
}
|
||||
pub fn in_span(&self, f: i64) -> bool {
|
||||
self.base.is_some() && f >= self.span[0] && f <= self.span[1]
|
||||
}
|
||||
/// Add a stroke: the first segmentation stroke sets the base frame and a span of
|
||||
/// `default_span` frames each side (clamped to `limits`); strokes outside the span extend it.
|
||||
pub fn add(&mut self, s: Stroke, default_span: i64, limits: [i64; 2]) {
|
||||
let f = s.frame;
|
||||
match self.base {
|
||||
None if s.kind.is_segmentation() => {
|
||||
self.base = Some(f);
|
||||
self.span = [(f - default_span).max(limits[0]), (f + default_span).min(limits[1]).max(f)];
|
||||
}
|
||||
Some(_) => {
|
||||
self.span[0] = self.span[0].min(f);
|
||||
self.span[1] = self.span[1].max(f);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
self.strokes.push(s);
|
||||
}
|
||||
/// Largest Refine Edge stroke radius (layer pixels).
|
||||
pub fn refine_radius(&self) -> f64 {
|
||||
self.strokes.iter().filter(|s| matches!(s.kind, StrokeKind::Refine)).map(|s| s.radius).fold(0.0, f64::max)
|
||||
}
|
||||
/// Content key of every frame in the span: the base frame's from `seed` (what the frames
|
||||
/// and settings are) and its strokes, every other frame's from its neighbour towards the
|
||||
/// base and its own strokes.
|
||||
pub fn chain_keys(&self, seed: u64) -> BTreeMap<i64, u64> {
|
||||
let mut out = BTreeMap::new();
|
||||
let Some(base) = self.base else { return out };
|
||||
let key = |prev: u64, f: i64| {
|
||||
let mut h = prev ^ 0x9e37_79b9_7f4a_7c15;
|
||||
fnv(&mut h, &f.to_le_bytes());
|
||||
for s in self.strokes.iter().filter(|s| s.frame == f) {
|
||||
fnv(&mut h, serde_json::to_string(s).unwrap_or_default().as_bytes());
|
||||
}
|
||||
h
|
||||
};
|
||||
let k0 = key(seed, base);
|
||||
out.insert(base, k0);
|
||||
let mut k = k0;
|
||||
for f in base + 1..=self.span[1] {
|
||||
k = key(k, f);
|
||||
out.insert(f, k);
|
||||
}
|
||||
k = k0;
|
||||
for f in (self.span[0]..base).rev() {
|
||||
k = key(k, f);
|
||||
out.insert(f, k);
|
||||
}
|
||||
out
|
||||
}
|
||||
/// The neighbour a frame is propagated from (None for the base frame and outside the span).
|
||||
pub fn source_of(&self, f: i64) -> Option<i64> {
|
||||
let base = self.base?;
|
||||
if !self.in_span(f) || f == base {
|
||||
return None;
|
||||
}
|
||||
Some(if f > base { f - 1 } else { f + 1 })
|
||||
}
|
||||
}
|
||||
|
||||
/// Segmentation settings (the effect's Roto Brush parameters, in frame pixels).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct SegOpts {
|
||||
/// Pixels of the matte per layer pixel.
|
||||
pub scale: f64,
|
||||
/// Search Radius (layer pixels).
|
||||
pub search_radius: f64,
|
||||
/// Motion Threshold / Motion Damping, percent.
|
||||
pub motion_threshold: f64,
|
||||
pub motion_damping: f64,
|
||||
/// Quality ▸ Best.
|
||||
pub best: bool,
|
||||
}
|
||||
|
||||
impl Default for SegOpts {
|
||||
fn default() -> Self {
|
||||
SegOpts { scale: 1.0, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false }
|
||||
}
|
||||
}
|
||||
|
||||
/// One frame's segmentation: the binary matte, the Refine Edge band and the colour models.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct FrameSeg {
|
||||
pub w: usize,
|
||||
pub h: usize,
|
||||
pub matte: Vec<u8>,
|
||||
pub refine: Vec<u8>,
|
||||
pub fg: Gmm,
|
||||
pub bg: Gmm,
|
||||
}
|
||||
|
||||
impl FrameSeg {
|
||||
pub fn empty(w: usize, h: usize) -> FrameSeg {
|
||||
FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], ..Default::default() }
|
||||
}
|
||||
pub fn area(&self) -> usize {
|
||||
self.matte.iter().filter(|v| **v != 0).count()
|
||||
}
|
||||
/// Centroid of the matte (pixels), if any.
|
||||
pub fn centroid(&self) -> Option<[f64; 2]> {
|
||||
let (mut sx, mut sy, mut n) = (0.0, 0.0, 0.0);
|
||||
for (i, v) in self.matte.iter().enumerate() {
|
||||
if *v != 0 {
|
||||
sx += (i % self.w) as f64 + 0.5;
|
||||
sy += (i / self.w) as f64 + 0.5;
|
||||
n += 1.0;
|
||||
}
|
||||
}
|
||||
(n > 0.0).then(|| [sx / n, sy / n])
|
||||
}
|
||||
}
|
||||
|
||||
/// Straight RGB of an image (transparent pixels are black).
|
||||
pub fn rgb_of(img: &Image) -> Vec<[f32; 3]> {
|
||||
img.data.par_iter().map(|p| if p[3] > 1e-6 { [p[0] / p[3], p[1] / p[3], p[2] / p[3]] } else { [0.0; 3] }).collect()
|
||||
}
|
||||
|
||||
/// Pixels covered by `strokes` (of the given kinds) at `scale`.
|
||||
pub fn rasterize(strokes: &[&Stroke], kinds: &[StrokeKind], scale: f64, w: usize, h: usize) -> Vec<bool> {
|
||||
let mut m = vec![false; w * h];
|
||||
for s in strokes.iter().filter(|s| kinds.contains(&s.kind)) {
|
||||
let r = (s.radius * scale).max(0.5);
|
||||
let pts: Vec<[f64; 2]> = s.points.iter().map(|p| [p[0] * scale, p[1] * scale]).collect();
|
||||
let segs: Vec<([f64; 2], [f64; 2])> = if pts.len() == 1 { vec![(pts[0], pts[0])] } else { pts.windows(2).map(|w| (w[0], w[1])).collect() };
|
||||
for (a, b) in segs {
|
||||
let x0 = (a[0].min(b[0]) - r).floor().max(0.0) as usize;
|
||||
let x1 = ((a[0].max(b[0]) + r).ceil().max(0.0) as usize).min(w);
|
||||
let y0 = (a[1].min(b[1]) - r).floor().max(0.0) as usize;
|
||||
let y1 = ((a[1].max(b[1]) + r).ceil().max(0.0) as usize).min(h);
|
||||
let d = [b[0] - a[0], b[1] - a[1]];
|
||||
let l2 = d[0] * d[0] + d[1] * d[1];
|
||||
for y in y0..y1 {
|
||||
for x in x0..x1 {
|
||||
let p = [x as f64 + 0.5, y as f64 + 0.5];
|
||||
let t = if l2 > 0.0 { (((p[0] - a[0]) * d[0] + (p[1] - a[1]) * d[1]) / l2).clamp(0.0, 1.0) } else { 0.0 };
|
||||
let q = [a[0] + d[0] * t, a[1] + d[1] * t];
|
||||
if (p[0] - q[0]).hypot(p[1] - q[1]) <= r {
|
||||
m[y * w + x] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
/// Hard constraints: 1 = foreground, 2 = background, 0 = free.
|
||||
const HARD_FG: u8 = 1;
|
||||
const HARD_BG: u8 = 2;
|
||||
|
||||
/// Graph-cut energy weights.
|
||||
const GAMMA: f64 = 50.0;
|
||||
const PRIOR: f64 = 1.0;
|
||||
/// Fixed-point scale of the energies.
|
||||
const K: f64 = 16.0;
|
||||
const INF: i64 = 1 << 40;
|
||||
|
||||
const NB: [(i64, i64, bool); 8] = [(1, 0, true), (0, 1, true), (1, 1, true), (-1, 1, true), (-1, 0, false), (0, -1, false), (-1, -1, false), (1, -1, false)];
|
||||
|
||||
fn beta(img: &[[f32; 3]], free: &[bool], w: usize, h: usize) -> f64 {
|
||||
let (mut s, mut n) = (0.0f64, 0.0f64);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let i = y * w + x;
|
||||
if !free[i] {
|
||||
continue;
|
||||
}
|
||||
let c = img[i];
|
||||
for (dx, dy) in [(1usize, 0usize), (0, 1)] {
|
||||
let (nx, ny) = (x + dx, y + dy);
|
||||
if nx < w && ny < h {
|
||||
let q = img[ny * w + nx];
|
||||
s += ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64;
|
||||
n += 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if s <= 1e-12 { 1.0 } else { n / (2.0 * s) }
|
||||
}
|
||||
|
||||
/// One min-cut over the `free` pixels; others keep `labels`. `prior`: P(foreground) per pixel.
|
||||
fn cut(img: &[[f32; 3]], w: usize, h: usize, labels: &mut [u8], free: &[bool], hard: &[u8], fg: &Gmm, bg: &Gmm, prior: Option<&[f32]>) {
|
||||
let mut idx = vec![u32::MAX; w * h];
|
||||
let mut nodes = vec![];
|
||||
for i in 0..w * h {
|
||||
if free[i] {
|
||||
idx[i] = nodes.len() as u32;
|
||||
nodes.push(i);
|
||||
}
|
||||
}
|
||||
if nodes.is_empty() {
|
||||
return;
|
||||
}
|
||||
let b = beta(img, free, w, h);
|
||||
let costs: Vec<(f64, f64)> = nodes
|
||||
.par_iter()
|
||||
.map(|&i| {
|
||||
let c = img[i];
|
||||
let (mut cf, mut cb) = (fg.cost(c), bg.cost(c));
|
||||
if let Some(p) = prior {
|
||||
let pf = (p[i] as f64).clamp(0.02, 0.98);
|
||||
cf += PRIOR * -pf.ln();
|
||||
cb += PRIOR * -(1.0 - pf).ln();
|
||||
}
|
||||
(cf, cb)
|
||||
})
|
||||
.collect();
|
||||
let mut g = Graph::new(nodes.len(), nodes.len() * 4);
|
||||
for (n, &i) in nodes.iter().enumerate() {
|
||||
let (x, y) = ((i % w) as i64, (i / w) as i64);
|
||||
let (cf, cb) = costs[n];
|
||||
let m = cf.min(cb);
|
||||
let (mut src, mut snk) = (((cb - m) * K).round() as i64, ((cf - m) * K).round() as i64);
|
||||
match hard[i] {
|
||||
HARD_FG => (src, snk) = (INF, 0),
|
||||
HARD_BG => (src, snk) = (0, INF),
|
||||
_ => {}
|
||||
}
|
||||
g.add_tweights(n, src, snk);
|
||||
let c = img[i];
|
||||
for (dx, dy, fwd) in NB {
|
||||
let (nx, ny) = (x + dx, y + dy);
|
||||
if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 {
|
||||
continue;
|
||||
}
|
||||
let j = ny as usize * w + nx as usize;
|
||||
let q = img[j];
|
||||
let d2 = ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64;
|
||||
let dist = if dx != 0 && dy != 0 { std::f64::consts::SQRT_2 } else { 1.0 };
|
||||
let wt = ((GAMMA / dist * (-b * d2).exp()) * K).round() as i32;
|
||||
if wt <= 0 {
|
||||
continue;
|
||||
}
|
||||
if idx[j] == u32::MAX {
|
||||
// Fixed neighbour: disagreeing with it costs the edge.
|
||||
if labels[j] != 0 {
|
||||
g.add_tweights(n, wt as i64, 0);
|
||||
} else {
|
||||
g.add_tweights(n, 0, wt as i64);
|
||||
}
|
||||
} else if fwd {
|
||||
g.add_edge(n, idx[j] as usize, wt, wt);
|
||||
}
|
||||
}
|
||||
}
|
||||
g.maxflow();
|
||||
for (n, &i) in nodes.iter().enumerate() {
|
||||
labels[i] = (g.segment(n) == Segment::Source) as u8;
|
||||
}
|
||||
}
|
||||
|
||||
/// Colour samples of the pixels where `sel` holds.
|
||||
fn samples(img: &[[f32; 3]], sel: impl Fn(usize) -> bool) -> Vec<[f32; 3]> {
|
||||
img.iter().enumerate().filter(|(i, _)| sel(*i)).map(|(_, c)| *c).collect()
|
||||
}
|
||||
|
||||
const COMPONENTS: usize = 5;
|
||||
const MAX_SAMPLES: usize = 24_000;
|
||||
|
||||
fn fit_models(img: &[[f32; 3]], labels: &[u8]) -> (Gmm, Gmm) {
|
||||
let fg = samples(img, |i| labels[i] != 0);
|
||||
let bg = samples(img, |i| labels[i] == 0);
|
||||
(Gmm::fit(&fg, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bg, COMPONENTS, MAX_SAMPLES))
|
||||
}
|
||||
|
||||
/// Keep the foreground components (8-connected) that touch `seeds`; fill small background
|
||||
/// holes that touch neither the frame edge nor a background seed.
|
||||
fn clean(labels: &mut [u8], seeds: &[bool], bg_seeds: &[bool], w: usize, h: usize) {
|
||||
let n = w * h;
|
||||
let mut comp = vec![u32::MAX; n];
|
||||
let mut stack = vec![];
|
||||
let mut fg_area = 0usize;
|
||||
for start in 0..n {
|
||||
if comp[start] != u32::MAX {
|
||||
continue;
|
||||
}
|
||||
let lab = labels[start];
|
||||
let id = start as u32;
|
||||
comp[start] = id;
|
||||
stack.push(start);
|
||||
let mut members = vec![];
|
||||
let (mut seeded, mut edge, mut bgs) = (false, false, false);
|
||||
while let Some(i) = stack.pop() {
|
||||
members.push(i);
|
||||
seeded |= seeds[i];
|
||||
bgs |= bg_seeds[i];
|
||||
let (x, y) = ((i % w) as i64, (i / w) as i64);
|
||||
edge |= x == 0 || y == 0 || x == w as i64 - 1 || y == h as i64 - 1;
|
||||
for (dx, dy, _) in NB {
|
||||
let (nx, ny) = (x + dx, y + dy);
|
||||
if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 {
|
||||
continue;
|
||||
}
|
||||
let j = ny as usize * w + nx as usize;
|
||||
if comp[j] == u32::MAX && labels[j] == lab {
|
||||
comp[j] = id;
|
||||
stack.push(j);
|
||||
}
|
||||
}
|
||||
}
|
||||
if lab != 0 {
|
||||
if seeded {
|
||||
fg_area += members.len();
|
||||
} else {
|
||||
for i in members {
|
||||
labels[i] = 0;
|
||||
}
|
||||
}
|
||||
} else if !edge && !bgs {
|
||||
// Remember holes as negative candidates (resolved below once fg_area is known).
|
||||
for i in members {
|
||||
comp[i] = u32::MAX - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fill holes smaller than 2 % of the foreground.
|
||||
let mut seen = vec![false; n];
|
||||
for start in 0..n {
|
||||
if comp[start] != u32::MAX - 1 || seen[start] || labels[start] != 0 {
|
||||
continue;
|
||||
}
|
||||
let mut members = vec![start];
|
||||
seen[start] = true;
|
||||
let mut k = 0;
|
||||
while k < members.len() {
|
||||
let i = members[k];
|
||||
k += 1;
|
||||
let (x, y) = ((i % w) as i64, (i / w) as i64);
|
||||
for (dx, dy, _) in NB {
|
||||
let (nx, ny) = (x + dx, y + dy);
|
||||
if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 {
|
||||
continue;
|
||||
}
|
||||
let j = ny as usize * w + nx as usize;
|
||||
if !seen[j] && comp[j] == u32::MAX - 1 && labels[j] == 0 {
|
||||
seen[j] = true;
|
||||
members.push(j);
|
||||
}
|
||||
}
|
||||
}
|
||||
// A hole is enclosed by foreground that survived the component filter.
|
||||
let enclosed = members.iter().all(|&i| {
|
||||
let (x, y) = ((i % w) as i64, (i / w) as i64);
|
||||
NB.iter().all(|(dx, dy, _)| {
|
||||
let (nx, ny) = (x + dx, y + dy);
|
||||
if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 {
|
||||
return false;
|
||||
}
|
||||
let j = ny as usize * w + nx as usize;
|
||||
labels[j] != 0 || comp[j] == u32::MAX - 1
|
||||
})
|
||||
});
|
||||
if enclosed && (members.len() as f64) < 0.02 * fg_area as f64 {
|
||||
for i in members {
|
||||
labels[i] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn downsample_rgb(img: &[[f32; 3]], w: usize, h: usize, f: usize) -> (Vec<[f32; 3]>, usize, usize) {
|
||||
let (cw, ch) = (w.div_ceil(f), h.div_ceil(f));
|
||||
let out = (0..cw * ch)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let (cx, cy) = (i % cw, i / cw);
|
||||
let mut s = [0.0f32; 3];
|
||||
let mut n = 0.0;
|
||||
for y in cy * f..((cy + 1) * f).min(h) {
|
||||
for x in cx * f..((cx + 1) * f).min(w) {
|
||||
let c = img[y * w + x];
|
||||
s[0] += c[0];
|
||||
s[1] += c[1];
|
||||
s[2] += c[2];
|
||||
n += 1.0;
|
||||
}
|
||||
}
|
||||
[s[0] / n, s[1] / n, s[2] / n]
|
||||
})
|
||||
.collect();
|
||||
(out, cw, ch)
|
||||
}
|
||||
|
||||
/// What a frame is segmented with besides its strokes: the matte propagated from its neighbour.
|
||||
pub struct Prior {
|
||||
/// Warped neighbour matte, soft (0–1).
|
||||
pub soft: Vec<f32>,
|
||||
/// The neighbour's colour models.
|
||||
pub fg: Gmm,
|
||||
pub bg: Gmm,
|
||||
/// Warped Refine Edge band.
|
||||
pub refine: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Segment a frame from its strokes (and a propagated prior): GrabCut iterations on a reduced
|
||||
/// copy, then a full-resolution cut in a band around the boundary.
|
||||
pub fn segment(img: &Image, strokes: &[&Stroke], prior: Option<&Prior>, opts: &SegOpts, refine_radius: f64) -> FrameSeg {
|
||||
let (w, h) = (img.width as usize, img.height as usize);
|
||||
if w == 0 || h == 0 {
|
||||
return FrameSeg::empty(w, h);
|
||||
}
|
||||
let rgb = rgb_of(img);
|
||||
let fgs = rasterize(strokes, &[StrokeKind::Fg], opts.scale, w, h);
|
||||
let bgs = rasterize(strokes, &[StrokeKind::Bg], opts.scale, w, h);
|
||||
let hard: Vec<u8> = fgs
|
||||
.iter()
|
||||
.zip(&bgs)
|
||||
.map(|(f, b)| {
|
||||
if *b {
|
||||
HARD_BG
|
||||
} else if *f {
|
||||
HARD_FG
|
||||
} else {
|
||||
0
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let soft_prior: Option<Vec<f32>> = prior.map(|p| matting::gauss(&p.soft, w, h, (opts.search_radius * opts.scale / 4.0).max(1.0)));
|
||||
// Initial colour models: the strokes (and the propagated matte).
|
||||
let (fg0, bg0) = match prior {
|
||||
Some(p) => {
|
||||
let fgc = samples(&rgb, |i| (hard[i] == HARD_FG) || (hard[i] == 0 && p.soft[i] >= 0.5));
|
||||
let bgc = samples(&rgb, |i| (hard[i] == HARD_BG) || (hard[i] == 0 && p.soft[i] < 0.5));
|
||||
let f = Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES);
|
||||
let b = Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES);
|
||||
(if f.is_empty() { p.fg.clone() } else { f }, if b.is_empty() { p.bg.clone() } else { b })
|
||||
}
|
||||
None => {
|
||||
let fgc = samples(&rgb, |i| hard[i] == HARD_FG);
|
||||
let mut bgc = samples(&rgb, |i| hard[i] == HARD_BG);
|
||||
if bgc.is_empty() {
|
||||
// No background strokes: what is far from the foreground strokes.
|
||||
let d = matting::distance(&fgs, w, h);
|
||||
let r = strokes.iter().filter(|s| s.kind == StrokeKind::Fg).map(|s| s.radius).fold(1.0, f64::max) * opts.scale;
|
||||
let far = (4.0 * r).max(0.25 * w.max(h) as f64) as f32;
|
||||
bgc = samples(&rgb, |i| d[i] > far);
|
||||
if bgc.is_empty() {
|
||||
bgc = samples(&rgb, |i| hard[i] != HARD_FG);
|
||||
}
|
||||
}
|
||||
(Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES))
|
||||
}
|
||||
};
|
||||
let iters = if opts.best { 4 } else { 3 };
|
||||
let work = if opts.best { 960 } else { 480 };
|
||||
let f = w.max(h).div_ceil(work).max(1);
|
||||
let mut labels: Vec<u8> = match &soft_prior {
|
||||
Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(),
|
||||
None => fgs.iter().map(|v| *v as u8).collect(),
|
||||
};
|
||||
for (l, hd) in labels.iter_mut().zip(&hard) {
|
||||
if *hd != 0 {
|
||||
*l = (*hd == HARD_FG) as u8;
|
||||
}
|
||||
}
|
||||
let (mut fgm, mut bgm) = (fg0, bg0);
|
||||
if f > 1 {
|
||||
let (crgb, cw, ch) = downsample_rgb(&rgb, w, h, f);
|
||||
let mut chard = vec![0u8; cw * ch];
|
||||
let mut cprior = soft_prior.as_ref().map(|_| vec![0.0f32; cw * ch]);
|
||||
for cy in 0..ch {
|
||||
for cx in 0..cw {
|
||||
let (mut anyf, mut anyb, mut s, mut n) = (false, false, 0.0f32, 0.0f32);
|
||||
for y in cy * f..((cy + 1) * f).min(h) {
|
||||
for x in cx * f..((cx + 1) * f).min(w) {
|
||||
let i = y * w + x;
|
||||
anyf |= hard[i] == HARD_FG;
|
||||
anyb |= hard[i] == HARD_BG;
|
||||
if let Some(p) = &soft_prior {
|
||||
s += p[i];
|
||||
}
|
||||
n += 1.0;
|
||||
}
|
||||
}
|
||||
chard[cy * cw + cx] = match (anyf, anyb) {
|
||||
(true, false) => HARD_FG,
|
||||
(false, true) => HARD_BG,
|
||||
_ => 0,
|
||||
};
|
||||
if let Some(cp) = &mut cprior {
|
||||
cp[cy * cw + cx] = s / n;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut cl: Vec<u8> = match &cprior {
|
||||
Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(),
|
||||
None => chard.iter().map(|v| (*v == HARD_FG) as u8).collect(),
|
||||
};
|
||||
let free = vec![true; cw * ch];
|
||||
for it in 0..iters {
|
||||
cut(&crgb, cw, ch, &mut cl, &free, &chard, &fgm, &bgm, cprior.as_deref());
|
||||
if it + 1 < iters {
|
||||
let (a, b) = fit_models(&crgb, &cl);
|
||||
if !a.is_empty() && !b.is_empty() {
|
||||
(fgm, bgm) = (a, b);
|
||||
}
|
||||
}
|
||||
}
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let i = y * w + x;
|
||||
labels[i] = if hard[i] != 0 { (hard[i] == HARD_FG) as u8 } else { cl[(y / f) * cw + x / f] };
|
||||
}
|
||||
}
|
||||
let (a, b) = fit_models(&rgb, &labels);
|
||||
if !a.is_empty() && !b.is_empty() {
|
||||
(fgm, bgm) = (a, b);
|
||||
}
|
||||
// Full-resolution band.
|
||||
let sd = matting::signed_distance(&labels, w, h);
|
||||
let band = f as f32 + 1.5;
|
||||
let free: Vec<bool> = sd.iter().map(|d| d.abs() <= band).collect();
|
||||
cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref());
|
||||
} else {
|
||||
let free = vec![true; w * h];
|
||||
for it in 0..iters {
|
||||
cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref());
|
||||
if it + 1 < iters {
|
||||
let (a, b) = fit_models(&rgb, &labels);
|
||||
if !a.is_empty() && !b.is_empty() {
|
||||
(fgm, bgm) = (a, b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Keep what the strokes (or the prior's core) select.
|
||||
let mut seeds = fgs.clone();
|
||||
if let Some(p) = &soft_prior {
|
||||
let core = core_of(p, w, h, opts.search_radius * opts.scale);
|
||||
for (s, c) in seeds.iter_mut().zip(core) {
|
||||
*s |= c;
|
||||
}
|
||||
}
|
||||
clean(&mut labels, &seeds, &bgs, w, h);
|
||||
finish(img, &rgb, labels, strokes, prior.map(|p| p.refine.as_slice()), opts, refine_radius)
|
||||
}
|
||||
|
||||
/// Pixels deep inside a soft matte (more than `r` from its edge), else the matte itself.
|
||||
fn core_of(soft: &[f32], w: usize, h: usize, r: f64) -> Vec<bool> {
|
||||
let bin: Vec<u8> = soft.iter().map(|v| (*v >= 0.5) as u8).collect();
|
||||
let sd = matting::signed_distance(&bin, w, h);
|
||||
let core: Vec<bool> = sd.iter().map(|d| *d as f64 > r).collect();
|
||||
if core.iter().any(|c| *c) { core } else { bin.iter().map(|v| *v != 0).collect() }
|
||||
}
|
||||
|
||||
/// Final models and Refine Edge band of a segmented frame.
|
||||
fn finish(_img: &Image, rgb: &[[f32; 3]], labels: Vec<u8>, strokes: &[&Stroke], warped_refine: Option<&[u8]>, opts: &SegOpts, refine_radius: f64) -> FrameSeg {
|
||||
let (w, h) = (_img.width as usize, _img.height as usize);
|
||||
let (fg, bg) = fit_models(rgb, &labels);
|
||||
let add = rasterize(strokes, &[StrokeKind::Refine], opts.scale, w, h);
|
||||
let erase = rasterize(strokes, &[StrokeKind::RefineErase], opts.scale, w, h);
|
||||
let near: Option<Vec<f32>> = warped_refine.filter(|r| r.iter().any(|v| *v != 0)).map(|_| matting::signed_distance(&labels, w, h));
|
||||
let rr = (refine_radius * opts.scale + 2.0) as f32;
|
||||
let refine: Vec<u8> = (0..w * h)
|
||||
.map(|i| {
|
||||
let carried = match (warped_refine, &near) {
|
||||
(Some(r), Some(sd)) => r[i] != 0 && sd[i].abs() <= rr,
|
||||
_ => false,
|
||||
};
|
||||
((carried || add[i]) && !erase[i]) as u8
|
||||
})
|
||||
.collect();
|
||||
FrameSeg { w, h, matte: labels, refine, fg, bg }
|
||||
}
|
||||
|
||||
/// Warp the previous frame's segmentation onto the next frame by optical flow (next → prev),
|
||||
/// with Motion Threshold / Damping applied to the vectors.
|
||||
pub fn warp_prior(prev_img: &Image, prev: &FrameSeg, next_img: &Image, opts: &SegOpts) -> Prior {
|
||||
let (w, h) = (next_img.width as usize, next_img.height as usize);
|
||||
let flow = effectcraft_raster::flow::block_flow(next_img, prev_img, 8, 4);
|
||||
let thr = (opts.motion_threshold / 100.0 * opts.search_radius * opts.scale).max(0.0);
|
||||
let damp = (1.0 - opts.motion_damping / 100.0).clamp(0.0, 1.0);
|
||||
let src: Vec<f32> = prev.matte.iter().map(|v| *v as f32).collect();
|
||||
let pw = prev.w.min(w);
|
||||
let ph = prev.h.min(h);
|
||||
let res: Vec<(f32, u8)> = (0..w * h)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5);
|
||||
let v = flow.at(x, y);
|
||||
let (mut vx, mut vy) = (v[0] as f64, v[1] as f64);
|
||||
if vx.hypot(vy) < thr {
|
||||
vx *= damp;
|
||||
vy *= damp;
|
||||
}
|
||||
let (sx, sy) = (x + vx, y + vy);
|
||||
let soft = if pw == prev.w && ph == prev.h { matting::sample(&src, prev.w, prev.h, sx, sy) } else { 0.0 };
|
||||
let (ix, iy) = (sx.floor().clamp(0.0, (prev.w.max(1) - 1) as f64) as usize, sy.floor().clamp(0.0, (prev.h.max(1) - 1) as f64) as usize);
|
||||
let r = prev.refine.get(iy * prev.w + ix).copied().unwrap_or(0);
|
||||
(soft, r)
|
||||
})
|
||||
.collect();
|
||||
Prior { soft: res.iter().map(|r| r.0).collect(), fg: prev.fg.clone(), bg: prev.bg.clone(), refine: res.iter().map(|r| r.1).collect() }
|
||||
}
|
||||
|
||||
/// Propagate the previous frame's segmentation to the next frame: warp it, then re-cut in a
|
||||
/// band of Search Radius pixels around the warped boundary. Strokes on the next frame re-cut
|
||||
/// the whole frame with the warped matte as a soft prior instead.
|
||||
pub fn propagate(prev_img: &Image, prev: &FrameSeg, next_img: &Image, strokes: &[&Stroke], opts: &SegOpts, refine_radius: f64) -> FrameSeg {
|
||||
let (w, h) = (next_img.width as usize, next_img.height as usize);
|
||||
let prior = warp_prior(prev_img, prev, next_img, opts);
|
||||
if strokes.iter().any(|s| s.kind.is_segmentation()) {
|
||||
return segment(next_img, strokes, Some(&prior), opts, refine_radius);
|
||||
}
|
||||
let rgb = rgb_of(next_img);
|
||||
let mut labels: Vec<u8> = prior.soft.iter().map(|v| (*v >= 0.5) as u8).collect();
|
||||
let sd = matting::signed_distance(&labels, w, h);
|
||||
let r = (opts.search_radius * opts.scale).max(2.0) as f32;
|
||||
let free: Vec<bool> = sd.iter().map(|d| d.abs() <= r).collect();
|
||||
let hard = vec![0u8; w * h];
|
||||
let soft = matting::gauss(&prior.soft, w, h, (r as f64 / 4.0).max(1.0));
|
||||
let (fg, bg) = if prev.fg.is_empty() || prev.bg.is_empty() { fit_models(&rgb, &labels) } else { (prev.fg.clone(), prev.bg.clone()) };
|
||||
cut(&rgb, w, h, &mut labels, &free, &hard, &fg, &bg, Some(&soft));
|
||||
let seeds = core_of(&prior.soft, w, h, r as f64);
|
||||
clean(&mut labels, &seeds, &vec![false; w * h], w, h);
|
||||
finish(next_img, &rgb, labels, strokes, Some(&prior.refine), opts, refine_radius)
|
||||
}
|
||||
|
||||
/// Intersection over union of two binary mattes.
|
||||
pub fn iou(a: &[u8], b: &[u8]) -> f64 {
|
||||
let (mut i, mut u) = (0usize, 0usize);
|
||||
for (x, y) in a.iter().zip(b) {
|
||||
let (x, y) = (*x != 0, *y != 0);
|
||||
i += (x && y) as usize;
|
||||
u += (x || y) as usize;
|
||||
}
|
||||
if u == 0 { 1.0 } else { i as f64 / u as f64 }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
//! From a frame's binary segmentation to the final soft matte: the Roto Brush Matte and Refine
|
||||
//! Edge Matte adjustments, edge matting inside the Refine Edge band (colour guided filter),
|
||||
//! Reduce Chatter (temporal averaging with the neighbouring frames), matte motion blur and edge
|
||||
//! colour decontamination.
|
||||
|
||||
use super::FrameSeg;
|
||||
use super::matting::{self, contrast, gauss, guided_filter, shift_soft, signed_distance};
|
||||
|
||||
/// The effect's matte parameters (After Effects' units: pixels and percent).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct MatteParams {
|
||||
// Roto Brush Matte.
|
||||
pub rb_feather: f64,
|
||||
pub rb_contrast: f64,
|
||||
pub rb_shift: f64,
|
||||
pub rb_chatter: f64,
|
||||
// Refine Edge Matte.
|
||||
pub smooth: f64,
|
||||
pub feather: f64,
|
||||
pub contrast: f64,
|
||||
pub shift: f64,
|
||||
pub chatter: f64,
|
||||
pub motion_blur: bool,
|
||||
pub mb_samples: usize,
|
||||
pub shutter_angle: f64,
|
||||
/// Higher Quality: closed-form matting after the guided-filter estimate.
|
||||
pub higher_quality: bool,
|
||||
pub decontaminate: bool,
|
||||
pub decontamination: f64,
|
||||
pub extend_where_smoothed: bool,
|
||||
pub increase_radius: f64,
|
||||
}
|
||||
|
||||
impl Default for MatteParams {
|
||||
fn default() -> Self {
|
||||
MatteParams {
|
||||
rb_feather: 0.0,
|
||||
rb_contrast: 0.0,
|
||||
rb_shift: 0.0,
|
||||
rb_chatter: 0.0,
|
||||
smooth: 1.0,
|
||||
feather: 0.0,
|
||||
contrast: 0.0,
|
||||
shift: 0.0,
|
||||
chatter: 0.0,
|
||||
motion_blur: false,
|
||||
mb_samples: 11,
|
||||
shutter_angle: 180.0,
|
||||
higher_quality: true,
|
||||
decontaminate: true,
|
||||
decontamination: 100.0,
|
||||
extend_where_smoothed: true,
|
||||
increase_radius: 0.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn shifted(seg: &FrameSeg, px: f64) -> Vec<f32> {
|
||||
if px.abs() < 0.25 {
|
||||
return seg.matte.iter().map(|v| *v as f32).collect();
|
||||
}
|
||||
let sd = signed_distance(&seg.matte, seg.w, seg.h);
|
||||
sd.iter().map(|d| ((*d as f64) > -px) as u8 as f32).collect()
|
||||
}
|
||||
|
||||
/// The soft matte of `seg` (all planes `seg.w × seg.h`), with `rgb` the frame's straight colour.
|
||||
/// `prev` / `next`: the neighbouring frames' segmentations (Reduce Chatter, motion blur).
|
||||
/// `band_radius`: the Refine Edge brush radius in these pixels; `scale`: pixels per layer pixel.
|
||||
pub fn matte_alpha(
|
||||
rgb: &[[f32; 3]],
|
||||
seg: &FrameSeg,
|
||||
prev: Option<&FrameSeg>,
|
||||
next: Option<&FrameSeg>,
|
||||
p: &MatteParams,
|
||||
band_radius: f64,
|
||||
scale: f64,
|
||||
) -> Vec<f32> {
|
||||
let (w, h) = (seg.w, seg.h);
|
||||
let n = w * h;
|
||||
let same = |s: &&FrameSeg| s.w == w && s.h == h;
|
||||
let (prev, next) = (prev.filter(same), next.filter(same));
|
||||
// Roto Brush Matte: Shift Edge (100 % = 10 px), Reduce Chatter, Feather, Contrast.
|
||||
let shift_px = p.rb_shift / 100.0 * 10.0 * scale;
|
||||
let mut a = shifted(seg, shift_px);
|
||||
let nb: Vec<Vec<f32>> = [prev, next].into_iter().flatten().map(|s| shifted(s, shift_px)).collect();
|
||||
if p.rb_chatter > 0.0 && !nb.is_empty() {
|
||||
let k = (p.rb_chatter / 100.0).clamp(0.0, 1.0) as f32 * (nb.len() as f32 / (nb.len() as f32 + 1.0));
|
||||
for i in 0..n {
|
||||
let m = nb.iter().map(|v| v[i]).sum::<f32>() / nb.len() as f32;
|
||||
a[i] += (m - a[i]) * k;
|
||||
}
|
||||
}
|
||||
if p.rb_feather > 0.0 {
|
||||
a = gauss(&a, w, h, p.rb_feather * scale / 2.0);
|
||||
}
|
||||
contrast(&mut a, p.rb_contrast);
|
||||
// Refine Edge band: guided-filter matting.
|
||||
if seg.refine.iter().any(|v| *v != 0) && seg.refine.len() == n {
|
||||
let r = (band_radius * 1.0).round().max(2.0) as usize;
|
||||
let eps = (2e-4 * (1.0 + p.smooth.max(0.0))) as f32;
|
||||
// Work on the band's bounding box (plus the filter's reach).
|
||||
let (mut x0, mut y0, mut x1, mut y1) = (w, h, 0, 0);
|
||||
for (i, v) in seg.refine.iter().enumerate() {
|
||||
if *v != 0 {
|
||||
let (x, y) = (i % w, i / w);
|
||||
x0 = x0.min(x);
|
||||
y0 = y0.min(y);
|
||||
x1 = x1.max(x + 1);
|
||||
y1 = y1.max(y + 1);
|
||||
}
|
||||
}
|
||||
let pad = 2 * r + 2;
|
||||
let (x0, y0, x1, y1) = (x0.saturating_sub(pad), y0.saturating_sub(pad), (x1 + pad).min(w), (y1 + pad).min(h));
|
||||
let (cw, ch) = (x1 - x0, y1 - y0);
|
||||
let crop = |src: &dyn Fn(usize) -> f32| -> Vec<f32> { (0..cw * ch).map(|j| src((y0 + j / cw) * w + x0 + j % cw)).collect() };
|
||||
let cimg: Vec<[f32; 3]> = (0..cw * ch).map(|j| rgb[(y0 + j / cw) * w + x0 + j % cw]).collect();
|
||||
let cp = crop(&|i| a[i]);
|
||||
let band: Vec<bool> = crop(&|i| seg.refine[i] as f32).iter().map(|v| *v > 0.0).collect();
|
||||
let outside: Vec<bool> = band.iter().map(|b| !b).collect();
|
||||
// Guided filter estimate, then closed-form matting with everything outside the band known.
|
||||
let mut q = guided_filter(&cimg, &cp, cw, ch, r, eps);
|
||||
for j in 0..cw * ch {
|
||||
q[j] = if band[j] { q[j].clamp(0.0, 1.0) } else { cp[j] };
|
||||
}
|
||||
if p.higher_quality {
|
||||
q = matting::closed_form(&cimg, &q, &outside, cw, ch, 1, 1e-5 * (1.0 + p.smooth.max(0.0)) as f32, 50);
|
||||
}
|
||||
// Refine Edge Matte adjustments.
|
||||
if p.smooth > 1.0 {
|
||||
q = gauss(&q, cw, ch, (p.smooth - 1.0) * 0.5 * scale);
|
||||
}
|
||||
if p.feather > 0.0 {
|
||||
q = gauss(&q, cw, ch, p.feather * scale / 2.0);
|
||||
}
|
||||
contrast(&mut q, p.contrast);
|
||||
shift_soft(&mut q, p.shift);
|
||||
// Blend in, fading over the band's last two pixels.
|
||||
let din = matting::distance(&outside, cw, ch);
|
||||
for j in 0..cw * ch {
|
||||
if !band[j] {
|
||||
continue;
|
||||
}
|
||||
let wgt = (din[j] / 2.0).clamp(0.0, 1.0);
|
||||
let i = (y0 + j / cw) * w + x0 + j % cw;
|
||||
a[i] += (q[j] - a[i]) * wgt;
|
||||
}
|
||||
// Refine-stage Reduce Chatter: towards the neighbours' mattes inside the band.
|
||||
if p.chatter > 0.0 && !nb.is_empty() {
|
||||
let k = (p.chatter / 100.0).clamp(0.0, 1.0) as f32 * 0.5;
|
||||
let soft: Vec<Vec<f32>> = nb.iter().map(|v| gauss(v, w, h, (r as f64 / 2.0).max(1.0))).collect();
|
||||
for i in 0..n {
|
||||
if seg.refine[i] != 0 {
|
||||
let m = soft.iter().map(|v| v[i]).sum::<f32>() / soft.len() as f32;
|
||||
a[i] += (m - a[i]) * k;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Motion blur: the matte's motion between the neighbouring frames.
|
||||
if p.motion_blur {
|
||||
let c = seg.centroid();
|
||||
let v = match (prev.and_then(FrameSeg::centroid), next.and_then(FrameSeg::centroid), c) {
|
||||
(Some(a), Some(b), _) => Some([(b[0] - a[0]) / 2.0, (b[1] - a[1]) / 2.0]),
|
||||
(Some(a), None, Some(c)) => Some([c[0] - a[0], c[1] - a[1]]),
|
||||
(None, Some(b), Some(c)) => Some([b[0] - c[0], b[1] - c[1]]),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(v) = v {
|
||||
let s = p.shutter_angle / 360.0;
|
||||
a = matting::motion_blur(&a, w, h, [v[0] * s, v[1] * s], p.mb_samples);
|
||||
}
|
||||
}
|
||||
a.iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0));
|
||||
a
|
||||
}
|
||||
|
||||
/// Decontaminate edge colours of `rgb` in place for the matte `alpha` (see
|
||||
/// [`matting::decontaminate`]); returns the decontamination map. `band`: the Refine Edge band
|
||||
/// (pixels where decontamination always applies).
|
||||
pub fn decontaminate(rgb: &mut [[f32; 3]], alpha: &[f32], band: &[u8], w: usize, h: usize, p: &MatteParams, band_radius: f64, scale: f64) -> Vec<f32> {
|
||||
if !p.decontaminate {
|
||||
return vec![0.0; w * h];
|
||||
}
|
||||
let grow = p.increase_radius * scale;
|
||||
let in_band: Vec<bool> = if band.len() == w * h && band.iter().any(|v| *v != 0) {
|
||||
if grow > 0.5 {
|
||||
let d = matting::distance(&band.iter().map(|v| *v != 0).collect::<Vec<_>>(), w, h);
|
||||
d.iter().map(|v| (*v as f64) <= grow).collect()
|
||||
} else {
|
||||
band.iter().map(|v| *v != 0).collect()
|
||||
}
|
||||
} else {
|
||||
vec![false; w * h]
|
||||
};
|
||||
let region: Vec<bool> = (0..w * h).map(|i| alpha[i] > 0.004 && alpha[i] < 0.996 && (in_band[i] || p.extend_where_smoothed)).collect();
|
||||
let radius = (band_radius + grow).max(4.0 * scale).round() as usize;
|
||||
matting::decontaminate(rgb, alpha, ®ion, w, h, radius, (p.decontamination / 100.0).clamp(0.0, 1.0) as f32)
|
||||
}
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
//! Compact text encodings of mattes for project storage: run-length coding (LEB128 varints)
|
||||
//! wrapped in standard base64 (RFC 4648 §4).
|
||||
|
||||
const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
pub fn base64_encode(data: &[u8]) -> String {
|
||||
let mut out = String::with_capacity(data.len().div_ceil(3) * 4);
|
||||
for c in data.chunks(3) {
|
||||
let b = [c[0], *c.get(1).unwrap_or(&0), *c.get(2).unwrap_or(&0)];
|
||||
let n = (b[0] as u32) << 16 | (b[1] as u32) << 8 | b[2] as u32;
|
||||
out.push(B64[(n >> 18) as usize & 63] as char);
|
||||
out.push(B64[(n >> 12) as usize & 63] as char);
|
||||
out.push(if c.len() > 1 { B64[(n >> 6) as usize & 63] as char } else { '=' });
|
||||
out.push(if c.len() > 2 { B64[n as usize & 63] as char } else { '=' });
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn base64_decode(s: &str) -> Option<Vec<u8>> {
|
||||
let val = |c: u8| -> Option<u32> {
|
||||
Some(match c {
|
||||
b'A'..=b'Z' => (c - b'A') as u32,
|
||||
b'a'..=b'z' => (c - b'a' + 26) as u32,
|
||||
b'0'..=b'9' => (c - b'0' + 52) as u32,
|
||||
b'+' => 62,
|
||||
b'/' => 63,
|
||||
_ => return None,
|
||||
})
|
||||
};
|
||||
let bytes: Vec<u8> = s.bytes().filter(|c| !c.is_ascii_whitespace()).collect();
|
||||
if !bytes.len().is_multiple_of(4) {
|
||||
return None;
|
||||
}
|
||||
let mut out = Vec::with_capacity(bytes.len() / 4 * 3);
|
||||
for c in bytes.chunks(4) {
|
||||
let pad = c.iter().rev().take_while(|b| **b == b'=').count();
|
||||
let mut n = 0u32;
|
||||
for (i, b) in c.iter().enumerate() {
|
||||
n |= if *b == b'=' { 0 } else { val(*b)? } << (18 - 6 * i);
|
||||
}
|
||||
out.push((n >> 16) as u8);
|
||||
if pad < 2 {
|
||||
out.push((n >> 8) as u8);
|
||||
}
|
||||
if pad < 1 {
|
||||
out.push(n as u8);
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn put(out: &mut Vec<u8>, mut v: u64) {
|
||||
loop {
|
||||
let b = (v & 0x7f) as u8;
|
||||
v >>= 7;
|
||||
if v == 0 {
|
||||
out.push(b);
|
||||
return;
|
||||
}
|
||||
out.push(b | 0x80);
|
||||
}
|
||||
}
|
||||
|
||||
fn get(data: &[u8], pos: &mut usize) -> Option<u64> {
|
||||
let mut v = 0u64;
|
||||
let mut shift = 0;
|
||||
loop {
|
||||
let b = *data.get(*pos)?;
|
||||
*pos += 1;
|
||||
v |= ((b & 0x7f) as u64) << shift;
|
||||
if b & 0x80 == 0 {
|
||||
return Some(v);
|
||||
}
|
||||
shift += 7;
|
||||
if shift > 63 {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a byte plane as (value, run length) pairs.
|
||||
pub fn encode(values: &[u8]) -> String {
|
||||
let mut out = vec![];
|
||||
put(&mut out, values.len() as u64);
|
||||
let mut i = 0;
|
||||
while i < values.len() {
|
||||
let v = values[i];
|
||||
let mut j = i + 1;
|
||||
while j < values.len() && values[j] == v {
|
||||
j += 1;
|
||||
}
|
||||
out.push(v);
|
||||
put(&mut out, (j - i) as u64);
|
||||
i = j;
|
||||
}
|
||||
base64_encode(&out)
|
||||
}
|
||||
|
||||
/// Decode [`encode`]'s output (None when malformed).
|
||||
pub fn decode(s: &str) -> Option<Vec<u8>> {
|
||||
let data = base64_decode(s)?;
|
||||
let mut pos = 0;
|
||||
let n = get(&data, &mut pos)? as usize;
|
||||
if n > 1 << 30 {
|
||||
return None;
|
||||
}
|
||||
let mut out = Vec::with_capacity(n);
|
||||
while out.len() < n {
|
||||
let v = *data.get(pos)?;
|
||||
pos += 1;
|
||||
let run = get(&data, &mut pos)? as usize;
|
||||
if out.len() + run > n {
|
||||
return None;
|
||||
}
|
||||
out.resize(out.len() + run, v);
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn round_trips() {
|
||||
for n in [0usize, 1, 2, 3, 4, 5, 100, 1000] {
|
||||
let v: Vec<u8> = (0..n).map(|i| ((i / 7) % 3) as u8 * 100).collect();
|
||||
assert_eq!(decode(&encode(&v)).unwrap(), v);
|
||||
let raw: Vec<u8> = (0..n).map(|i| (i * 31) as u8).collect();
|
||||
assert_eq!(base64_decode(&base64_encode(&raw)).unwrap(), raw);
|
||||
}
|
||||
assert_eq!(base64_encode(b"Man"), "TWFu");
|
||||
assert_eq!(base64_encode(b"Ma"), "TWE=");
|
||||
assert!(decode("!!!!").is_none());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,331 @@
|
|||
//! Roto Brush on synthetic sequences: a textured disk moving over a textured background.
|
||||
|
||||
use super::refine::{MatteParams, decontaminate, matte_alpha};
|
||||
use super::*;
|
||||
|
||||
pub(crate) const W: u32 = 200;
|
||||
pub(crate) const H: u32 = 150;
|
||||
pub(crate) const R: f64 = 32.0;
|
||||
|
||||
/// Smooth value noise in 0..1.
|
||||
pub(crate) fn noise(x: f64, y: f64, seed: u32) -> f64 {
|
||||
fn h(i: i64, j: i64, s: u32) -> f64 {
|
||||
let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64;
|
||||
v ^= v >> 13;
|
||||
v = v.wrapping_mul(0x5bd1e995);
|
||||
v ^= v >> 15;
|
||||
(v % 10_000) as f64 / 10_000.0
|
||||
}
|
||||
let mut out = 0.0;
|
||||
for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] {
|
||||
let (fx, fy) = (x / cell, y / cell);
|
||||
let (i, j) = (fx.floor() as i64, fy.floor() as i64);
|
||||
let (tx, ty) = (fx - i as f64, fy - j as f64);
|
||||
let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty));
|
||||
let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx;
|
||||
let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx;
|
||||
out += amp * (a + (b - a) * sy);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(crate) fn fg_color(x: f64, y: f64) -> [f32; 3] {
|
||||
let n = noise(x, y, 11);
|
||||
[(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32]
|
||||
}
|
||||
|
||||
pub(crate) fn bg_color(x: f64, y: f64) -> [f32; 3] {
|
||||
let n = noise(x, y, 5);
|
||||
let m = noise(x * 1.7, y * 1.7, 9);
|
||||
[(0.05 + 0.25 * m) as f32, (0.3 + 0.4 * n) as f32, (0.45 + 0.4 * m) as f32]
|
||||
}
|
||||
|
||||
pub(crate) fn center(f: i64) -> [f64; 2] {
|
||||
let t = f as f64;
|
||||
[60.0 + 3.1 * t, 70.0 + 1.3 * t + 4.0 * (t * 0.3).sin()]
|
||||
}
|
||||
|
||||
/// Frame `f`: the disk (texture moving with it) over the static background, with a 1-pixel
|
||||
/// anti-aliased edge; `c` overrides the centre.
|
||||
pub(crate) fn frame_at(c: [f64; 2]) -> (Image, Vec<u8>) {
|
||||
let mut img = Image::new(W, H);
|
||||
let mut gt = vec![0u8; (W * H) as usize];
|
||||
for y in 0..H {
|
||||
for x in 0..W {
|
||||
let p = [x as f64 + 0.5, y as f64 + 0.5];
|
||||
let d = (p[0] - c[0]).hypot(p[1] - c[1]);
|
||||
let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32;
|
||||
let f = fg_color(p[0] - c[0], p[1] - c[1]);
|
||||
let b = bg_color(p[0], p[1]);
|
||||
img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]);
|
||||
gt[(y * W + x) as usize] = (d <= R) as u8;
|
||||
}
|
||||
}
|
||||
(img, gt)
|
||||
}
|
||||
|
||||
fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke {
|
||||
Stroke { kind, frame, radius, points: (0..=10).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 10.0, a[1] + (b[1] - a[1]) * k as f64 / 10.0]).collect() }
|
||||
}
|
||||
|
||||
/// The strokes an artist would paint on frame `f`: one through the disk, one around it.
|
||||
pub(crate) fn base_strokes(f: i64) -> Vec<Stroke> {
|
||||
let c = center(f);
|
||||
vec![
|
||||
line(StrokeKind::Fg, f, [c[0] - 18.0, c[1] - 6.0], [c[0] + 18.0, c[1] + 6.0], 5.0),
|
||||
line(StrokeKind::Bg, f, [c[0] - 50.0, c[1] - 45.0], [c[0] + 50.0, c[1] - 45.0], 5.0),
|
||||
line(StrokeKind::Bg, f, [c[0] + 50.0, c[1] + 45.0], [c[0] - 45.0, c[1] + 45.0], 5.0),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strokes_segment_the_base_frame() {
|
||||
let (img, gt) = frame_at(center(0));
|
||||
let st = base_strokes(0);
|
||||
let refs: Vec<&Stroke> = st.iter().collect();
|
||||
let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0);
|
||||
let j = iou(&seg.matte, >);
|
||||
assert!(j > 0.95, "IoU {j}");
|
||||
// Deterministic.
|
||||
let again = segment(&img, &refs, None, &SegOpts::default(), 0.0);
|
||||
assert_eq!(seg, again);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coarse_to_fine_segments_large_frames() {
|
||||
// A 2× upscaled frame goes through the reduced-size path (Standard works at 480 px).
|
||||
let (small, _) = frame_at(center(0));
|
||||
let img = effectcraft_raster::resample(&small, W * 3, H * 3);
|
||||
let c = center(0);
|
||||
let st: Vec<Stroke> = base_strokes(0)
|
||||
.into_iter()
|
||||
.map(|mut s| {
|
||||
s.points.iter_mut().for_each(|p| *p = [p[0] * 3.0, p[1] * 3.0]);
|
||||
s.radius *= 3.0;
|
||||
s
|
||||
})
|
||||
.collect();
|
||||
let refs: Vec<&Stroke> = st.iter().collect();
|
||||
let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0);
|
||||
let gt: Vec<u8> =
|
||||
(0..W * H * 9).map(|i| ((((i % (W * 3)) as f64 + 0.5) / 3.0 - c[0]).hypot((((i / (W * 3)) as f64 + 0.5) / 3.0) - c[1]) <= R) as u8).collect();
|
||||
let j = iou(&seg.matte, >);
|
||||
assert!(j > 0.95, "IoU {j}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn propagation_follows_the_disk_for_20_frames() {
|
||||
let (img0, _) = frame_at(center(0));
|
||||
let st = base_strokes(0);
|
||||
let refs: Vec<&Stroke> = st.iter().collect();
|
||||
let opts = SegOpts::default();
|
||||
let mut seg = segment(&img0, &refs, None, &opts, 0.0);
|
||||
let mut prev = img0;
|
||||
let mut worst: f64 = 1.0;
|
||||
for f in 1..=20 {
|
||||
let (img, gt) = frame_at(center(f));
|
||||
seg = propagate(&prev, &seg, &img, &[], &opts, 0.0);
|
||||
let j = iou(&seg.matte, >);
|
||||
worst = worst.min(j);
|
||||
prev = img;
|
||||
}
|
||||
assert!(worst > 0.9, "worst IoU {worst}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_correction_stroke_fixes_a_bad_frame() {
|
||||
let opts = SegOpts::default();
|
||||
let (img0, _) = frame_at(center(0));
|
||||
let st = base_strokes(0);
|
||||
let refs: Vec<&Stroke> = st.iter().collect();
|
||||
let seg0 = segment(&img0, &refs, None, &opts, 0.0);
|
||||
// The disk jumps far away on frame 1: propagation cannot follow.
|
||||
let jump = [150.0, 90.0];
|
||||
let (img1, gt1) = frame_at(jump);
|
||||
let bad = propagate(&img0, &seg0, &img1, &[], &opts, 0.0);
|
||||
assert!(iou(&bad.matte, >1) < 0.5, "the jump should break propagation");
|
||||
// Paint the disk in and the old place out.
|
||||
let c0 = center(0);
|
||||
let fix = [
|
||||
line(StrokeKind::Fg, 1, [jump[0] - 15.0, jump[1]], [jump[0] + 15.0, jump[1]], 5.0),
|
||||
line(StrokeKind::Bg, 1, [c0[0] - 10.0, c0[1] - 10.0], [c0[0] + 10.0, c0[1] + 10.0], 6.0),
|
||||
];
|
||||
let fr: Vec<&Stroke> = fix.iter().collect();
|
||||
let good = propagate(&img0, &seg0, &img1, &fr, &opts, 0.0);
|
||||
let j = iou(&good.matte, >1);
|
||||
assert!(j > 0.95, "corrected IoU {j}");
|
||||
// And propagation continues from the corrected frame.
|
||||
let c2 = [jump[0] + 3.0, jump[1] + 1.0];
|
||||
let (img2, gt2) = frame_at(c2);
|
||||
let next = propagate(&img1, &good, &img2, &[], &opts, 0.0);
|
||||
assert!(iou(&next.matte, >2) > 0.93);
|
||||
}
|
||||
|
||||
/// A soft-edged disk (alpha ramp over `width` pixels) over a background.
|
||||
fn soft_frame(c: [f64; 2], width: f64, fg: [f32; 3], bg: [f32; 3]) -> (Image, Vec<f32>) {
|
||||
let mut img = Image::new(W, H);
|
||||
let mut gt = vec![0.0f32; (W * H) as usize];
|
||||
for y in 0..H {
|
||||
for x in 0..W {
|
||||
let p = [x as f64 + 0.5, y as f64 + 0.5];
|
||||
let d = (p[0] - c[0]).hypot(p[1] - c[1]);
|
||||
let a = ((R + width / 2.0 - d) / width).clamp(0.0, 1.0) as f32;
|
||||
// Faint texture on both layers.
|
||||
let tf = (noise(p[0], p[1], 3) as f32 - 0.5) * 0.04;
|
||||
let tb = (noise(p[0], p[1], 4) as f32 - 0.5) * 0.04;
|
||||
let f = [fg[0] + tf, fg[1] + tf, fg[2] + tf];
|
||||
let b = [bg[0] + tb, bg[1] + tb, bg[2] + tb];
|
||||
img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]);
|
||||
gt[(y * W + x) as usize] = a;
|
||||
}
|
||||
}
|
||||
(img, gt)
|
||||
}
|
||||
|
||||
fn ring(frame: i64, c: [f64; 2], r: f64, radius: f64) -> Stroke {
|
||||
Stroke {
|
||||
kind: StrokeKind::Refine,
|
||||
frame,
|
||||
radius,
|
||||
points: (0..=48).map(|k| k as f64 / 48.0 * std::f64::consts::TAU).map(|a| [c[0] + r * a.cos(), c[1] + r * a.sin()]).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refine_edge_recovers_a_soft_edge_and_decontaminates() {
|
||||
let c = [100.0, 75.0];
|
||||
let (fgc, bgc) = ([0.9f32, 0.25, 0.1], [0.1f32, 0.3, 0.9]);
|
||||
let (img, gt) = soft_frame(c, 10.0, fgc, bgc);
|
||||
let mut st = vec![
|
||||
line(StrokeKind::Fg, 0, [c[0] - 15.0, c[1]], [c[0] + 15.0, c[1]], 5.0),
|
||||
line(StrokeKind::Bg, 0, [c[0] - 60.0, c[1] - 55.0], [c[0] + 60.0, c[1] - 55.0], 5.0),
|
||||
];
|
||||
st.push(ring(0, c, R, 9.0));
|
||||
let refs: Vec<&Stroke> = st.iter().collect();
|
||||
let seg = segment(&img, &refs, None, &SegOpts::default(), 9.0);
|
||||
assert!(seg.refine.iter().filter(|v| **v != 0).count() > 1000);
|
||||
let p = MatteParams::default();
|
||||
let rgb = rgb_of(&img);
|
||||
let a = matte_alpha(&rgb, &seg, None, None, &p, 9.0, 1.0);
|
||||
// Inside the ramp the matte is fractional and close to the truth.
|
||||
let (mut err, mut n, mut frac) = (0.0f64, 0usize, 0usize);
|
||||
let (mut err_bin, mut n_bin) = (0.0f64, 0usize);
|
||||
for i in 0..a.len() {
|
||||
if gt[i] > 0.0 && gt[i] < 1.0 {
|
||||
err += (a[i] - gt[i]).abs() as f64;
|
||||
err_bin += (seg.matte[i] as f32 - gt[i]).abs() as f64;
|
||||
n += 1;
|
||||
n_bin += 1;
|
||||
frac += (a[i] > 0.05 && a[i] < 0.95) as usize;
|
||||
}
|
||||
}
|
||||
let (err, err_bin) = (err / n as f64, err_bin / n_bin as f64);
|
||||
assert!(err < 0.03, "mean alpha error {err} (binary {err_bin})");
|
||||
assert!(err < err_bin * 0.4);
|
||||
assert!(frac as f64 > 0.6 * n as f64, "{frac}/{n} fractional");
|
||||
// Decontamination: edge pixels' colour is the foreground's, not a blue blend.
|
||||
let mut col = rgb.clone();
|
||||
let map = decontaminate(&mut col, &a, &seg.refine, W as usize, H as usize, &p, 9.0, 1.0);
|
||||
let (mut blue_before, mut blue_after, mut m) = (0.0f64, 0.0f64, 0usize);
|
||||
for i in 0..a.len() {
|
||||
if a[i] > 0.2 && a[i] < 0.8 {
|
||||
blue_before += rgb[i][2] as f64;
|
||||
blue_after += col[i][2] as f64;
|
||||
m += 1;
|
||||
}
|
||||
}
|
||||
let (bb, ba) = (blue_before / m as f64, blue_after / m as f64);
|
||||
assert!(ba < 0.2 && bb > 0.35, "blue before {bb}, after {ba}");
|
||||
assert!(map.iter().any(|v| *v > 0.2));
|
||||
// Without the band, the binary matte passes through.
|
||||
let mut seg2 = seg.clone();
|
||||
seg2.refine.iter_mut().for_each(|v| *v = 0);
|
||||
let a2 = matte_alpha(&rgb, &seg2, None, None, &p, 9.0, 1.0);
|
||||
assert!(a2.iter().zip(&seg.matte).all(|(x, y)| (*x - *y as f32).abs() < 1e-6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduce_chatter_and_motion_blur_use_the_neighbours() {
|
||||
let (w, h) = (40usize, 20usize);
|
||||
let mk = |x0: usize| FrameSeg {
|
||||
w,
|
||||
h,
|
||||
matte: (0..w * h).map(|i| (i % w >= x0 && i % w < x0 + 10) as u8).collect(),
|
||||
refine: vec![0; w * h],
|
||||
..Default::default()
|
||||
};
|
||||
let (a, b, c) = (mk(10), mk(14), mk(18));
|
||||
let rgb = vec![[0.5f32; 3]; w * h];
|
||||
let p = MatteParams { rb_chatter: 100.0, ..Default::default() };
|
||||
let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0);
|
||||
// Pixel 12 is only in the previous frame: partially on.
|
||||
assert!(al[12] > 0.2 && al[12] < 0.5, "{}", al[12]);
|
||||
let p = MatteParams { motion_blur: true, ..Default::default() };
|
||||
let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0);
|
||||
assert!(al[14] > 0.3 && al[14] < 0.9, "{}", al[14]);
|
||||
assert!(al[19] > 0.99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_keys_restart_at_correction_frames() {
|
||||
let mut d = RotoData::default();
|
||||
for s in base_strokes(5) {
|
||||
d.add(s, 20, [0, 30]);
|
||||
}
|
||||
assert_eq!(d.base, Some(5));
|
||||
assert_eq!(d.span, [0, 25]);
|
||||
let k1 = d.chain_keys(1);
|
||||
d.add(line(StrokeKind::Fg, 12, [0.0, 0.0], [1.0, 1.0], 3.0), 20, [0, 30]);
|
||||
let k2 = d.chain_keys(1);
|
||||
for f in 0..=11 {
|
||||
assert_eq!(k1[&f], k2[&f], "frame {f}");
|
||||
}
|
||||
for f in 12..=25 {
|
||||
assert_ne!(k1[&f], k2[&f], "frame {f}");
|
||||
}
|
||||
assert_ne!(d.chain_keys(2)[&5], k2[&5]);
|
||||
let back = RotoData::from_json(&d.to_json());
|
||||
assert_eq!(back, d);
|
||||
assert_eq!(d.source_of(12), Some(11));
|
||||
assert_eq!(d.source_of(3), Some(4));
|
||||
assert_eq!(d.source_of(5), None);
|
||||
}
|
||||
|
||||
/// `cargo test -p effectcraft-track --release roto_perf -- --ignored --nocapture`
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn roto_perf_1080p() {
|
||||
let up = |c: [f64; 2]| {
|
||||
let (img, _) = frame_at(c);
|
||||
effectcraft_raster::resample(&img, 1920, 1080)
|
||||
};
|
||||
let s = 1920.0 / W as f64;
|
||||
let st: Vec<Stroke> = base_strokes(0)
|
||||
.into_iter()
|
||||
.map(|mut k| {
|
||||
k.points.iter_mut().for_each(|p| *p = [p[0] * s, p[1] * s]);
|
||||
k.radius *= s;
|
||||
k
|
||||
})
|
||||
.collect();
|
||||
let refs: Vec<&Stroke> = st.iter().collect();
|
||||
let opts = SegOpts { search_radius: 15.0 * s / 4.0, ..Default::default() };
|
||||
let a = up(center(0));
|
||||
let t = std::time::Instant::now();
|
||||
let seg = segment(&a, &refs, None, &opts, 0.0);
|
||||
eprintln!("segment 1080p: {:?}", t.elapsed());
|
||||
let b = up(center(1));
|
||||
let t = std::time::Instant::now();
|
||||
let next = propagate(&a, &seg, &b, &[], &opts, 0.0);
|
||||
eprintln!("propagate 1080p: {:?} (area {})", t.elapsed(), next.area());
|
||||
let mut seg2 = next.clone();
|
||||
let c = center(1);
|
||||
let ring: Vec<[f64; 2]> = (0..=64).map(|k| k as f64 / 64.0 * std::f64::consts::TAU).map(|t| [(c[0] + R * t.cos()) * s, (c[1] + R * t.sin()) * s]).collect();
|
||||
let st = [Stroke { kind: StrokeKind::Refine, frame: 1, radius: 20.0, points: ring }];
|
||||
let r = rasterize(&st.iter().collect::<Vec<_>>(), &[StrokeKind::Refine], 1.0, 1920, 1080);
|
||||
seg2.refine = r.iter().map(|v| *v as u8).collect();
|
||||
let rgb = rgb_of(&b);
|
||||
let t = std::time::Instant::now();
|
||||
let _ = matte_alpha(&rgb, &seg2, None, None, &MatteParams::default(), 20.0, 1.0);
|
||||
eprintln!("refine 1080p: {:?}", t.elapsed());
|
||||
}
|
||||
|
|
@ -33,6 +33,7 @@ pub enum Icon {
|
|||
Clone,
|
||||
Eraser,
|
||||
RotoBrush,
|
||||
RefineEdge,
|
||||
Puppet,
|
||||
// switches & timeline
|
||||
Eye,
|
||||
|
|
@ -282,6 +283,15 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
|||
pen.ellipse(6.0, 10.0, 3.6, 3.6);
|
||||
pen.dot(6.0, 10.0, 1.0);
|
||||
}
|
||||
RefineEdge => {
|
||||
// A brush over a soft (dashed) edge.
|
||||
pen.line(&[(13.5, 2.5), (9.0, 7.0)]);
|
||||
pen.ellipse(7.0, 9.0, 2.6, 2.6);
|
||||
pen.line(&[(2.0, 14.0), (3.5, 12.5)]);
|
||||
pen.line(&[(5.0, 14.0), (6.5, 12.5)]);
|
||||
pen.line(&[(8.0, 14.0), (9.5, 12.5)]);
|
||||
pen.line(&[(11.0, 14.0), (12.5, 12.5)]);
|
||||
}
|
||||
Puppet => {
|
||||
pen.circle(8.0, 3.6, 1.6);
|
||||
pen.line(&[(8.0, 5.2), (8.0, 10.0)]);
|
||||
|
|
|
|||
|
|
@ -725,6 +725,7 @@ impl EffectcraftApp {
|
|||
// Warp Stabilizer: finish analyses and start queued (re-)analyses in the background.
|
||||
if self.session.warp_job.is_some() || !self.session.warp_pending.is_empty() {
|
||||
self.session.poll_warp(true);
|
||||
self.session.poll_roto(true);
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(100));
|
||||
}
|
||||
self.apply_prefs(&ctx);
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ pub const UI_COMMANDS: &[UiCommand] = &[
|
|||
uic!("tool.pen", "Pen Tool", [], Some("G")),
|
||||
uic!("tool.type", "Type Tools", [], Some("Cmd+T")),
|
||||
uic!("tool.brush", "Brush Tools", [], Some("Cmd+B")),
|
||||
uic!("tool.rotoBrush", "Roto Brush Tool", [], Some("Alt+W")),
|
||||
uic!("tool.rotoBrush", "Roto Brush & Refine Edge Tools", [], Some("Alt+W")),
|
||||
uic!("tool.puppet", "Puppet Tools", [], Some("Cmd+P")),
|
||||
uic!("app.newComp", "New Composition...", [], None),
|
||||
uic!("app.compSettings", "Composition Settings...", [], None),
|
||||
|
|
@ -139,6 +139,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V
|
|||
"pen" => Some(9),
|
||||
"type" => Some(10),
|
||||
"brush" => Some(11),
|
||||
"rotoBrush" => Some(14),
|
||||
"puppet" => Some(15),
|
||||
_ => None,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -537,6 +537,35 @@ fn warp_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l
|
|||
p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim);
|
||||
}
|
||||
|
||||
/// Roto Brush & Refine Edge: Freeze / Unfreeze and Propagate buttons and the segmentation status.
|
||||
fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) {
|
||||
use effectcraft_engine::effects::roto as fx;
|
||||
let t = app.tokens;
|
||||
let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone());
|
||||
let frozen = fx::is_frozen(¶ms);
|
||||
let d = fx::data(¶ms);
|
||||
let running = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid) && app.session.is_roto_running();
|
||||
let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0));
|
||||
let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0));
|
||||
let label = if frozen { "Unfreeze" } else { "Freeze" };
|
||||
if widgets::text_button(ui, b1, label, frozen, &t, egui::Id::new(("roto-freeze-ec", g.uid))).clicked() && !running {
|
||||
let id = if frozen { "roto.unfreeze" } else { "roto.freeze" };
|
||||
actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid})));
|
||||
}
|
||||
app.auto.add(&format!("effectControls.roto.{}.freeze", g.uid), b1, label);
|
||||
let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Propagate", "roto.propagate") };
|
||||
if widgets::text_button(ui, b2, l2, false, &t, egui::Id::new(("roto-prop-ec", g.uid))).clicked() && !frozen {
|
||||
actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid})));
|
||||
}
|
||||
app.auto.add(&format!("effectControls.roto.{}.propagate", g.uid), b2, l2);
|
||||
let status = match (app.session.roto_progress(), d.base) {
|
||||
(Some(pr), _) if running => pr.banner(),
|
||||
(_, None) => "No strokes: paint with the Roto Brush tool (Alt+W) in the Layer panel".to_string(),
|
||||
(_, Some(b)) => format!("Base frame {b}, span {}–{}{}", d.span[0], d.span[1], if frozen { " (frozen)" } else { "" }),
|
||||
};
|
||||
p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Curves / Levels editors
|
||||
|
||||
|
|
@ -545,6 +574,7 @@ fn editor_height(effect: &str, width: f32) -> f32 {
|
|||
match effect {
|
||||
"ec.color.curves" => 34.0 + curves_size(width) + 26.0,
|
||||
effectcraft_engine::effects::warp_stab::ID => 50.0,
|
||||
effectcraft_engine::effects::roto::ID => 50.0,
|
||||
"ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0,
|
||||
"ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0,
|
||||
_ => 0.0,
|
||||
|
|
@ -920,6 +950,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
match effect.as_str() {
|
||||
"ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions),
|
||||
effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions),
|
||||
effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions),
|
||||
"ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions),
|
||||
_ => histogram_only(app, ui, &bp, g, er),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,13 @@
|
|||
//!
|
||||
//! The image is the layer's source and masks followed by its effects up to the View menu's
|
||||
//! choice (by default the last Paint effect, as After Effects shows "Paint"). Dragging with a
|
||||
//! paint tool records the stroke's points in layer space and, on release, runs `paint.stroke`
|
||||
//! paint tools records the stroke's points in layer space and, on release, runs `paint.stroke`
|
||||
//! at the current time (the drag's duration feeds Write On). Alt/Option-click with the Clone
|
||||
//! Stamp sets the clone source.
|
||||
//!
|
||||
//! With a Roto Brush & Refine Edge effect the panel shows the effect's input with the matte
|
||||
//! overlay, the segmentation span bar and the view / Freeze buttons; the Roto Brush and Refine
|
||||
//! Edge tools paint their strokes here (see [`super::roto_tool`]).
|
||||
|
||||
use effectcraft_engine::effects::paint;
|
||||
use effectcraft_engine::project::{Layer, LayerId};
|
||||
|
|
@ -46,9 +50,15 @@ pub fn current_layer(app: &EffectcraftApp) -> Option<Layer> {
|
|||
comp.layer(id).cloned()
|
||||
}
|
||||
|
||||
/// Number of effects shown by default: through the last Paint effect (none when there isn't one).
|
||||
fn default_view(layer: &Layer) -> usize {
|
||||
layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last()).unwrap_or(0)
|
||||
/// Number of effects shown by default: the Roto Brush's input while a Roto Brush tool is active
|
||||
/// (or when there is no Paint effect), else through the last Paint effect (none when there
|
||||
/// isn't one).
|
||||
fn default_view(layer: &Layer, tool: Tool) -> usize {
|
||||
let paint = layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last());
|
||||
match super::roto_tool::roto_index(layer) {
|
||||
Some(r) if tool.roto_kind(false).is_some() || paint.is_none() => r,
|
||||
_ => paint.unwrap_or(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
||||
|
|
@ -64,7 +74,8 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
let comp = app.session.project.comp(cid).cloned().unwrap_or_else(|| effectcraft_engine::project::Comp::new(1, 1, Default::default(), Default::default()));
|
||||
let time = app.session.time();
|
||||
let fx_count = layer.effects().map(|f| f.groups().count()).unwrap_or(0);
|
||||
let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer));
|
||||
let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer, app.ui.tool));
|
||||
let roto = super::roto_tool::roto_index(&layer);
|
||||
|
||||
// Title strip.
|
||||
let top = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0));
|
||||
|
|
@ -90,9 +101,17 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
}
|
||||
let tc = super::timecode(&app.session, &comp, time);
|
||||
p.text(pos2(bar.max.x - 10.0, bar.center().y), Align2::RIGHT_CENTER, tc, Tokens::mono(12.0), t.timecode);
|
||||
// Roto Brush: view / Freeze buttons and the segmentation span bar.
|
||||
let mut canvas_bottom = bar.min.y;
|
||||
if roto.is_some() {
|
||||
super::roto_tool::bar_buttons(app, ui, &p, bar, vr.max.x + 12.0, &layer);
|
||||
let sb = Rect::from_min_max(pos2(rect.min.x, bar.min.y - 18.0), pos2(rect.max.x, bar.min.y));
|
||||
super::roto_tool::span_bar(app, ui, &p, sb, &comp, &layer);
|
||||
canvas_bottom = sb.min.y;
|
||||
}
|
||||
|
||||
// Canvas.
|
||||
let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, bar.min.y));
|
||||
let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, canvas_bottom));
|
||||
p.rect_filled(area, 0.0, t.pasteboard);
|
||||
let (w, h) = effectcraft_engine::render::source_size(&app.session.project, &layer);
|
||||
let (lw, lh) = if w == 0 { (comp.width as f64, comp.height as f64) } else { (w as f64, h as f64) };
|
||||
|
|
@ -139,10 +158,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
let r = Rect::from_min_max(to_screen([tx.rect[0], tx.rect[1]]), to_screen([tx.rect[2], tx.rect[3]]));
|
||||
painter.image(tx.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE);
|
||||
}
|
||||
if roto.is_some_and(|r| r == view) {
|
||||
super::roto_tool::draw_overlay(app, &ctx, &painter, &comp, &layer, scale, &to_screen);
|
||||
}
|
||||
painter.rect_stroke(frame, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside);
|
||||
app.auto.add("layerPanel.canvas", frame, &layer.name);
|
||||
app.auto.add("layerPanel.area", area, "Layer panel");
|
||||
|
||||
// Interaction: Roto Brush / Refine Edge.
|
||||
if app.ui.tool.roto_kind(false).is_some() {
|
||||
super::roto_tool::interact(app, ui, &painter, area, &layer, zoom, &to_screen, &to_layer);
|
||||
return;
|
||||
}
|
||||
// Interaction: paint tools.
|
||||
let resp = ui.interact(area, egui::Id::new("layer-panel-interact"), Sense::click_and_drag());
|
||||
let kind = app.ui.tool.paint_kind();
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ pub mod project;
|
|||
pub mod properties;
|
||||
pub mod puppet_tool;
|
||||
pub mod render_queue;
|
||||
pub mod roto_tool;
|
||||
pub mod script_console;
|
||||
pub mod settings;
|
||||
pub mod shortcut_editor;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,331 @@
|
|||
//! Roto Brush and Refine Edge tools in the Layer panel: green foreground strokes, Alt/Option red
|
||||
//! background strokes, Refine Edge strokes along soft edges; the matte overlay (Alpha Boundary in
|
||||
//! pink, Alpha, Alpha Overlay), the segmentation span bar and the Freeze button.
|
||||
|
||||
use effectcraft_engine::effects::roto as fx;
|
||||
use effectcraft_engine::project::{Comp, Layer};
|
||||
use effectcraft_engine::render::{EvalCtx, RenderOpts, Renderer};
|
||||
use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::theme::Tokens;
|
||||
use crate::{EffectcraftApp, widgets};
|
||||
|
||||
/// Position of the layer's first Roto Brush & Refine Edge effect.
|
||||
pub fn roto_index(layer: &Layer) -> Option<usize> {
|
||||
layer.effects()?.groups().position(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == fx::ID))
|
||||
}
|
||||
|
||||
fn roto_group(layer: &Layer) -> Option<&effectcraft_engine::project::PropGroup> {
|
||||
layer.effects()?.groups().nth(roto_index(layer)?)
|
||||
}
|
||||
|
||||
/// A stroke being drawn.
|
||||
#[derive(Clone, Debug)]
|
||||
struct Drag {
|
||||
layer: u64,
|
||||
kind: &'static str,
|
||||
points: Vec<[f64; 2]>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Overlay {
|
||||
key: (u64, u64, i64, u32, String),
|
||||
tex: egui::TextureHandle,
|
||||
rect: [f64; 4],
|
||||
}
|
||||
|
||||
fn c32(c: [f64; 3], a: u8) -> Color32 {
|
||||
Color32::from_rgba_unmultiplied((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8, a)
|
||||
}
|
||||
|
||||
/// Draw the matte overlay of the Roto Brush on `layer` (the panel shows the effect's input).
|
||||
pub fn draw_overlay(
|
||||
app: &mut EffectcraftApp,
|
||||
ctx: &egui::Context,
|
||||
painter: &egui::Painter,
|
||||
comp: &Comp,
|
||||
layer: &Layer,
|
||||
scale: f64,
|
||||
to_screen: &dyn Fn([f64; 2]) -> Pos2,
|
||||
) {
|
||||
let Some(idx) = roto_index(layer) else { return };
|
||||
let o = app.session.state.roto.clone();
|
||||
if o.view == "none" {
|
||||
return;
|
||||
}
|
||||
let Some(cid) = app.session.active_comp_id() else { return };
|
||||
let time = app.session.time();
|
||||
let key = (
|
||||
app.session.revision,
|
||||
layer.id.0,
|
||||
time.0,
|
||||
(scale * 1000.0) as u32,
|
||||
format!("{}{:?}{:?}{}", o.view, o.overlay_color, o.boundary_color, o.overlay_opacity),
|
||||
);
|
||||
let id = egui::Id::new("roto-overlay");
|
||||
let cached: Option<Overlay> = ctx.data(|d| d.get_temp(id));
|
||||
let ov = match cached {
|
||||
Some(c) if c.key == key => Some(c),
|
||||
_ => {
|
||||
let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time, expr: app.session.expr.as_deref() };
|
||||
let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() });
|
||||
r.expr = app.session.expr.as_deref();
|
||||
r.cache = Some(&app.session.layer_cache);
|
||||
let (Some(inp), Some(out)) = (r.layer_input(&ectx, layer, idx), r.layer_input(&ectx, layer, idx + 1)) else { return };
|
||||
let (w, h) = (inp.img.width as usize, inp.img.height as usize);
|
||||
let a_at = |x: i64, y: i64| -> f32 {
|
||||
// Output pixel at the same layer point.
|
||||
let lx = (x as f64 + 0.5 - inp.offset[0]) / inp.scale;
|
||||
let ly = (y as f64 + 0.5 - inp.offset[1]) / inp.scale;
|
||||
let (ox, oy) = out.to_px([lx, ly]);
|
||||
out.img.get(ox.floor() as i64, oy.floor() as i64)[3]
|
||||
};
|
||||
let alpha: Vec<f32> = (0..w * h).map(|i| a_at((i % w) as i64, (i / w) as i64)).collect();
|
||||
let inside = |x: i64, y: i64| x >= 0 && y >= 0 && x < w as i64 && y < h as i64 && alpha[y as usize * w + x as usize] >= 0.5;
|
||||
let mut px = vec![Color32::TRANSPARENT; w * h];
|
||||
let bc = c32(o.boundary_color, 255);
|
||||
let oc = c32(o.overlay_color, (o.overlay_opacity / 100.0 * 255.0).clamp(0.0, 255.0) as u8);
|
||||
// View Search Region: the band around the boundary (yellow).
|
||||
let search = roto_group(layer).filter(|g| g.prop("rotoBrushMatte/viewSearchRegion").is_some_and(|p| p.value.as_bool())).map(|g| {
|
||||
let r = g.prop("rotoBrushMatte/searchRadius").map(|p| p.value.as_f64()).unwrap_or(15.0) * scale;
|
||||
let bin: Vec<u8> = alpha.iter().map(|a| (*a >= 0.5) as u8).collect();
|
||||
let sd = effectcraft_engine::track::roto::matting::signed_distance(&bin, w, h);
|
||||
(sd, r as f32)
|
||||
});
|
||||
for y in 0..h as i64 {
|
||||
for x in 0..w as i64 {
|
||||
let i = y as usize * w + x as usize;
|
||||
let a = alpha[i];
|
||||
px[i] = match o.view.as_str() {
|
||||
"alpha" => {
|
||||
let v = (a.clamp(0.0, 1.0) * 255.0) as u8;
|
||||
Color32::from_rgb(v, v, v)
|
||||
}
|
||||
"alphaOverlay" if a < 0.5 => oc,
|
||||
"alphaOverlay" => Color32::TRANSPARENT,
|
||||
_ => {
|
||||
let me = a >= 0.5;
|
||||
let edge = me && [(-1, 0), (1, 0), (0, -1), (0, 1)].iter().any(|(dx, dy)| !inside(x + dx, y + dy));
|
||||
if edge { bc } else { Color32::TRANSPARENT }
|
||||
}
|
||||
};
|
||||
if let Some((sd, r)) = &search {
|
||||
let d = sd[i].abs();
|
||||
if (d - r).abs() < 0.75 {
|
||||
px[i] = Color32::from_rgb(255, 220, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let img = egui::ColorImage { size: [w, h], pixels: px, source_size: egui::Vec2::new(w as f32, h as f32) };
|
||||
let x0 = -inp.offset[0] / inp.scale;
|
||||
let y0 = -inp.offset[1] / inp.scale;
|
||||
let rect = [x0, y0, x0 + w as f64 / inp.scale, y0 + h as f64 / inp.scale];
|
||||
let tex = ctx.load_texture("roto-overlay", img, egui::TextureOptions::NEAREST);
|
||||
let c = Overlay { key, tex, rect };
|
||||
ctx.data_mut(|d| d.insert_temp(id, c.clone()));
|
||||
Some(c)
|
||||
}
|
||||
};
|
||||
if let Some(ov) = ov {
|
||||
let r = Rect::from_min_max(to_screen([ov.rect[0], ov.rect[1]]), to_screen([ov.rect[2], ov.rect[3]]));
|
||||
painter.image(ov.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE);
|
||||
}
|
||||
}
|
||||
|
||||
/// Pointer handling for the Roto Brush / Refine Edge tools. Returns true when it handled input.
|
||||
pub fn interact(
|
||||
app: &mut EffectcraftApp,
|
||||
ui: &mut egui::Ui,
|
||||
painter: &egui::Painter,
|
||||
area: Rect,
|
||||
layer: &Layer,
|
||||
zoom: f64,
|
||||
to_screen: &dyn Fn([f64; 2]) -> Pos2,
|
||||
to_layer: &dyn Fn(Pos2) -> [f64; 2],
|
||||
) -> bool {
|
||||
let ctx = ui.ctx().clone();
|
||||
let mods = ui.input(|i| i.modifiers);
|
||||
let Some(kind) = app.ui.tool.roto_kind(mods.alt) else { return false };
|
||||
let resp = ui.interact(area, egui::Id::new("layer-panel-roto"), Sense::click_and_drag());
|
||||
let o = app.session.state.roto.clone();
|
||||
let refine = kind.starts_with("refine");
|
||||
let dia = if refine { o.refine_diameter } else { o.diameter };
|
||||
let col = match kind {
|
||||
"fg" => Color32::from_rgb(40, 220, 60),
|
||||
"bg" => Color32::from_rgb(235, 50, 40),
|
||||
"refine" => Color32::from_rgb(110, 200, 255),
|
||||
_ => Color32::from_rgb(255, 160, 60),
|
||||
};
|
||||
if let Some(hp) = resp.hover_pos() {
|
||||
let lp = to_layer(hp);
|
||||
app.pointer_comp = Some([lp[0] as f32, lp[1] as f32]);
|
||||
ctx.set_cursor_icon(egui::CursorIcon::Crosshair);
|
||||
let r = (dia * 0.5 * zoom) as f32;
|
||||
painter.circle_stroke(hp, r, Stroke::new(1.5, col));
|
||||
let sign = if kind == "bg" || kind == "refineErase" { "−" } else { "+" };
|
||||
painter.text(hp, Align2::CENTER_CENTER, sign, Tokens::semibold(12.0), col);
|
||||
}
|
||||
let did = egui::Id::new("layer-panel-roto-drag");
|
||||
if resp.drag_started()
|
||||
&& let Some(pos) = ui.input(|i| i.pointer.press_origin()).or(resp.interact_pointer_pos())
|
||||
{
|
||||
ctx.data_mut(|d| d.insert_temp(did, Drag { layer: layer.id.0, kind, points: vec![to_layer(pos)] }));
|
||||
}
|
||||
if let Some(mut d) = ctx.data(|d| d.get_temp::<Drag>(did)) {
|
||||
if let Some(pos) = resp.interact_pointer_pos() {
|
||||
let lp = to_layer(pos);
|
||||
let last = d.points.last().copied().unwrap_or(lp);
|
||||
if (lp[0] - last[0]).hypot(lp[1] - last[1]) * zoom >= 1.5 {
|
||||
d.points.push(lp);
|
||||
ctx.data_mut(|dd| dd.insert_temp(did, d.clone()));
|
||||
}
|
||||
}
|
||||
let width = ((dia * zoom) as f32).max(1.0);
|
||||
let pts: Vec<Pos2> = d.points.iter().map(|q| to_screen(*q)).collect();
|
||||
let c = col.gamma_multiply(0.6);
|
||||
if pts.len() == 1 {
|
||||
painter.circle_filled(pts[0], width / 2.0, c);
|
||||
} else {
|
||||
painter.add(egui::Shape::line(pts, Stroke::new(width, c)));
|
||||
}
|
||||
if resp.drag_stopped() {
|
||||
ctx.data_mut(|dd| dd.remove::<Drag>(did));
|
||||
run_stroke(app, d.layer, d.kind, &d.points, dia / 2.0);
|
||||
}
|
||||
} else if resp.clicked()
|
||||
&& let Some(pos) = resp.interact_pointer_pos()
|
||||
{
|
||||
run_stroke(app, layer.id.0, kind, &[to_layer(pos)], dia / 2.0);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn run_stroke(app: &mut EffectcraftApp, layer: u64, kind: &str, points: &[[f64; 2]], radius: f64) {
|
||||
let r = app.session.execute("roto.stroke", json!({"layer": layer, "kind": kind, "points": points, "radius": radius}));
|
||||
if let Err(e) = r {
|
||||
app.ui.status = e.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
/// The segmentation span bar (above the panel's bottom bar): the layer's frames, the span, the
|
||||
/// base frame (yellow), computed frames (green) and stroke frames. Click to go to a frame; drag
|
||||
/// the span's ends to change it.
|
||||
pub fn span_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, r: Rect, comp: &Comp, layer: &Layer) {
|
||||
let t = app.tokens;
|
||||
let Some(g) = roto_group(layer) else { return };
|
||||
let Some(cid) = app.session.active_comp_id() else { return };
|
||||
let uid = g.uid;
|
||||
let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone());
|
||||
let d = fx::data(¶ms);
|
||||
p.rect_filled(r, 0.0, t.panel_bg);
|
||||
let lim = effectcraft_engine::roto::frame_limits(comp, layer);
|
||||
let n = (lim[1] - lim[0] + 1).max(1) as f32;
|
||||
let track = Rect::from_min_max(pos2(r.min.x + 10.0, r.min.y + 4.0), pos2(r.max.x - 10.0, r.max.y - 4.0));
|
||||
let x_of = |f: f64| track.min.x + ((f - lim[0] as f64) / n as f64) as f32 * track.width();
|
||||
let f_of = |x: f32| (lim[0] as f64 + ((x - track.min.x) / track.width()).clamp(0.0, 0.9999) as f64 * n as f64).floor() as i64;
|
||||
p.rect_filled(track, 2.0, t.pasteboard);
|
||||
if let Some(base) = d.base {
|
||||
let sp = Rect::from_min_max(pos2(x_of(d.span[0] as f64), track.min.y), pos2(x_of(d.span[1] as f64 + 1.0), track.max.y));
|
||||
let frozen = fx::is_frozen(¶ms);
|
||||
p.rect_filled(sp, 2.0, if frozen { Color32::from_rgb(60, 110, 190) } else { Color32::from_rgb(90, 90, 96) });
|
||||
for (f, done) in app.session.roto_computed(cid, layer.id, uid) {
|
||||
if done {
|
||||
let cr = Rect::from_min_max(pos2(x_of(f as f64), track.max.y - 4.0), pos2(x_of(f as f64 + 1.0), track.max.y));
|
||||
p.rect_filled(cr, 0.0, Color32::from_rgb(70, 200, 90));
|
||||
}
|
||||
}
|
||||
for f in d.stroke_frames() {
|
||||
let x = x_of(f as f64 + 0.5);
|
||||
p.line_segment([pos2(x, track.min.y), pos2(x, track.min.y + 5.0)], Stroke::new(2.0, Color32::WHITE));
|
||||
}
|
||||
let bx = x_of(base as f64 + 0.5);
|
||||
p.rect_filled(Rect::from_center_size(pos2(bx, track.center().y), vec2(6.0, track.height())), 1.0, Color32::from_rgb(240, 200, 40));
|
||||
}
|
||||
// Current time.
|
||||
let fps = comp.frame_rate.as_f64();
|
||||
let cur = fx::frame_of(layer.layer_time(app.session.time()).seconds(), fps);
|
||||
let cx = x_of(cur as f64 + 0.5);
|
||||
p.line_segment([pos2(cx, r.min.y), pos2(cx, r.max.y)], Stroke::new(1.0, t.timecode));
|
||||
p.rect_stroke(track, 2.0, Stroke::new(1.0, Color32::from_black_alpha(140)), StrokeKind::Outside);
|
||||
app.auto.add("layerPanel.rotoSpan", track, "Segmentation span");
|
||||
let resp = ui.interact(track, egui::Id::new("roto-span-bar"), Sense::click_and_drag());
|
||||
let hid = egui::Id::new("roto-span-handle");
|
||||
if resp.drag_started()
|
||||
&& let (Some(pos), Some(_)) = (resp.interact_pointer_pos(), d.base)
|
||||
{
|
||||
let (a, b) = (x_of(d.span[0] as f64), x_of(d.span[1] as f64 + 1.0));
|
||||
let h = if (pos.x - a).abs() < 6.0 {
|
||||
1
|
||||
} else if (pos.x - b).abs() < 6.0 {
|
||||
2
|
||||
} else {
|
||||
0
|
||||
};
|
||||
ui.ctx().data_mut(|m| m.insert_temp(hid, h));
|
||||
}
|
||||
let handle: i32 = ui.ctx().data(|m| m.get_temp(hid)).unwrap_or(0);
|
||||
if let Some(pos) = resp.interact_pointer_pos() {
|
||||
let f = f_of(pos.x);
|
||||
if handle == 0 && (resp.clicked() || resp.dragged()) {
|
||||
let t = layer.comp_time(comp.frame_rate.tick_of(f));
|
||||
let _ = app.session.execute("time.set", json!({"time": t.seconds()}));
|
||||
}
|
||||
if handle != 0 && resp.dragged() {
|
||||
p.line_segment([pos2(pos.x, track.min.y), pos2(pos.x, track.max.y)], Stroke::new(2.0, Color32::from_rgb(240, 200, 40)));
|
||||
}
|
||||
if handle != 0 && resp.drag_stopped() {
|
||||
let k = if handle == 1 { "start" } else { "end" };
|
||||
if let Err(e) = app.session.execute("roto.span", json!({"layer": layer.id.0, "effect": uid, k: f})) {
|
||||
app.ui.status = e.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
if resp.drag_stopped() {
|
||||
ui.ctx().data_mut(|m| m.remove::<i32>(hid));
|
||||
}
|
||||
}
|
||||
|
||||
/// View-mode toggles and Freeze / Propagate buttons in the Layer panel's bottom bar, from `x`.
|
||||
pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, bar: Rect, x: f32, layer: &Layer) {
|
||||
let t = app.tokens;
|
||||
let mut x = x;
|
||||
let view = app.session.state.roto.view.clone();
|
||||
for (id, label) in [("alpha", "Alpha"), ("alphaBoundary", "Boundary"), ("alphaOverlay", "Overlay")] {
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(66.0, 20.0));
|
||||
let on = view == id;
|
||||
if widgets::text_button(ui, r, label, on, &t, egui::Id::new(("roto-view", id))).clicked() {
|
||||
let v = if on { "none" } else { id };
|
||||
let _ = app.session.execute("roto.options", json!({"view": v}));
|
||||
}
|
||||
app.auto.add(&format!("layerPanel.roto.view.{id}"), r, label);
|
||||
x += 70.0;
|
||||
}
|
||||
let Some(g) = roto_group(layer) else { return };
|
||||
let uid = g.uid;
|
||||
let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone());
|
||||
let frozen = fx::is_frozen(¶ms);
|
||||
let running = app.session.is_roto_running();
|
||||
x += 8.0;
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(74.0, 20.0));
|
||||
let label = if frozen { "Unfreeze" } else { "Freeze" };
|
||||
if widgets::text_button(ui, r, label, frozen, &t, egui::Id::new(("roto-freeze", uid))).clicked() {
|
||||
let id = if frozen { "roto.unfreeze" } else { "roto.freeze" };
|
||||
if let Err(e) = app.session.execute(id, json!({"layer": layer.id.0, "effect": uid})) {
|
||||
app.ui.status = e.to_string();
|
||||
}
|
||||
}
|
||||
app.auto.add("layerPanel.roto.freeze", r, label);
|
||||
x += 80.0;
|
||||
if running {
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0));
|
||||
if widgets::text_button(ui, r, "Stop", false, &t, egui::Id::new("roto-stop")).clicked() {
|
||||
let _ = app.session.execute("roto.cancel", json!({}));
|
||||
}
|
||||
app.auto.add("layerPanel.roto.stop", r, "Stop");
|
||||
if let Some(pr) = app.session.roto_progress() {
|
||||
p.text(pos2(r.max.x + 8.0, bar.center().y), Align2::LEFT_CENTER, pr.banner(), Tokens::ui(11.5), t.text_dim);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -34,6 +34,7 @@ pub enum Tool {
|
|||
Clone,
|
||||
Eraser,
|
||||
RotoBrush,
|
||||
RefineEdge,
|
||||
Puppet,
|
||||
PuppetStarch,
|
||||
PuppetBend,
|
||||
|
|
@ -58,10 +59,10 @@ impl Tool {
|
|||
&[Tool::Brush],
|
||||
&[Tool::Clone],
|
||||
&[Tool::Eraser],
|
||||
&[Tool::RotoBrush],
|
||||
&[Tool::RotoBrush, Tool::RefineEdge],
|
||||
&[Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, Tool::PuppetAdvanced, Tool::PuppetOverlap],
|
||||
];
|
||||
pub const ALL: [Tool; 29] = [
|
||||
pub const ALL: [Tool; 30] = [
|
||||
Tool::Selection,
|
||||
Tool::Hand,
|
||||
Tool::Zoom,
|
||||
|
|
@ -86,6 +87,7 @@ impl Tool {
|
|||
Tool::Clone,
|
||||
Tool::Eraser,
|
||||
Tool::RotoBrush,
|
||||
Tool::RefineEdge,
|
||||
Tool::Puppet,
|
||||
Tool::PuppetStarch,
|
||||
Tool::PuppetBend,
|
||||
|
|
@ -118,6 +120,7 @@ impl Tool {
|
|||
Tool::Clone => "Clone Stamp Tool",
|
||||
Tool::Eraser => "Eraser Tool",
|
||||
Tool::RotoBrush => "Roto Brush Tool",
|
||||
Tool::RefineEdge => "Refine Edge Tool",
|
||||
Tool::Puppet => "Puppet Position Pin Tool",
|
||||
Tool::PuppetStarch => "Puppet Starch Pin Tool",
|
||||
Tool::PuppetBend => "Puppet Bend Pin Tool",
|
||||
|
|
@ -139,7 +142,7 @@ impl Tool {
|
|||
Tool::Pen | Tool::PenAdd | Tool::PenDelete | Tool::PenConvert | Tool::MaskFeather => Some("G"),
|
||||
Tool::Type | Tool::TypeVertical => Some("Cmd+T"),
|
||||
Tool::Brush | Tool::Clone | Tool::Eraser => Some("Cmd+B"),
|
||||
Tool::RotoBrush => Some("Alt+W"),
|
||||
Tool::RotoBrush | Tool::RefineEdge => Some("Alt+W"),
|
||||
Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Some("Cmd+P"),
|
||||
}
|
||||
}
|
||||
|
|
@ -169,6 +172,7 @@ impl Tool {
|
|||
Tool::Clone => Icon::Clone,
|
||||
Tool::Eraser => Icon::Eraser,
|
||||
Tool::RotoBrush => Icon::RotoBrush,
|
||||
Tool::RefineEdge => Icon::RefineEdge,
|
||||
Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Icon::Puppet,
|
||||
}
|
||||
}
|
||||
|
|
@ -187,6 +191,17 @@ impl Tool {
|
|||
_ => return None,
|
||||
})
|
||||
}
|
||||
/// Roto Brush / Refine Edge: the stroke kind they paint (`roto.stroke` kind), without and
|
||||
/// with Alt/Option.
|
||||
pub fn roto_kind(self, alt: bool) -> Option<&'static str> {
|
||||
Some(match (self, alt) {
|
||||
(Tool::RotoBrush, false) => "fg",
|
||||
(Tool::RotoBrush, true) => "bg",
|
||||
(Tool::RefineEdge, false) => "refine",
|
||||
(Tool::RefineEdge, true) => "refineErase",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
/// Paint tools, with their stroke kind (`paint.stroke` kind).
|
||||
pub fn paint_kind(self) -> Option<&'static str> {
|
||||
Some(match self {
|
||||
|
|
|
|||
|
|
@ -147,6 +147,28 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa
|
|||
effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json
|
||||
```
|
||||
|
||||
### Roto Brush & Refine Edge
|
||||
|
||||
The Roto Brush tool (Alt+W cycles Roto Brush / Refine Edge) paints in the Layer panel; agents use
|
||||
`roto.stroke {layer, kind: fg|bg|refine|refineErase, points: [[x, y], …], frame?, radius?}`
|
||||
(layer pixels, layer frames). The first foreground stroke applies the Roto Brush & Refine Edge
|
||||
effect and sets the base frame with a span of 20 frames each side. `roto.propagate {layer,
|
||||
direction?: forward|backward|both, to?, wait?}` segments the span (in the background unless
|
||||
`wait`); strokes on any other frame correct it and propagation restarts from there.
|
||||
`roto.span {start?, end?}`, `roto.freeze {wait?}` / `roto.unfreeze`, `roto.clearStrokes {frame?,
|
||||
kind?}`, `roto.cancel` and `roto.options {diameter?, refineDiameter?, view?: alphaBoundary|alpha|
|
||||
alphaOverlay|none}` complete the set; every edit is one undo step. `roto.status {layer, frame?,
|
||||
matte?, compute?, compareTo?}` reports the base frame, span, computed and stroked frames, frozen
|
||||
state, job progress and, for a frame, the matte's area, centroid, RLE matte and IoU against a
|
||||
reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/searchRadius`,
|
||||
`effects/#1/refineEdgeMatte/decontaminateEdgeColors`, …).
|
||||
|
||||
```sh
|
||||
effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[360,230]],"radius":10}' \
|
||||
roto.stroke '{"layer":"#1","kind":"bg","points":[[40,40],[600,40]],"radius":12}' \
|
||||
roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save
|
||||
```
|
||||
|
||||
### Text: styles and editing
|
||||
|
||||
Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the
|
||||
|
|
|
|||
|
|
@ -207,6 +207,19 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E
|
|||
borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the
|
||||
effects crate, hence the `effects → track` edge.
|
||||
|
||||
**Roto Brush & Refine Edge** (`effectcraft_track::roto`, `effects::roto`, `engine::roto`):
|
||||
strokes (foreground, background, Refine Edge) are stored as JSON in the effect's hidden Strokes
|
||||
parameter with the base frame and segmentation span. Each frame is segmented by graph cut
|
||||
(Boykov–Jolly hard constraints, GrabCut colour mixtures, our own Boykov–Kolmogorov max-flow,
|
||||
coarse to fine) and propagated to the next frame by warping the matte with block optical flow and
|
||||
re-cutting in a Search Radius band; frames with correction strokes are re-cut with the warped
|
||||
matte as a soft prior. Refine Edge bands get guided-filter + closed-form matting and
|
||||
foreground-colour decontamination. Segmentations are derived data: each frame's chain key (Input
|
||||
Key, settings, strokes from the base frame out) indexes a process-wide cache that renders fill on
|
||||
demand (`self_at`) and `roto.propagate` fills in the background; `Session::edit` keeps the Input
|
||||
Key in step with the layer's source, masks and upstream effects. Freeze stores final 8-bit mattes
|
||||
in the effect.
|
||||
|
||||
## 5. Expressions
|
||||
|
||||
Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`,
|
||||
|
|
|
|||
|
|
@ -73,11 +73,11 @@ library groundwork for mask tracking and Warp Stabilizer.
|
|||
| Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 |
|
||||
| Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** |
|
||||
|
||||
What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker,
|
||||
What is left is concentrated in large systems: the 3D camera tracker,
|
||||
the Warp Stabilizer and mask-tracking UI on top of
|
||||
the new tracking library, SVG/PSD import, WebM, a disk cache,
|
||||
native macOS menus, and wasm threads. (The missing effect categories, 3D Channel, Immersive Video
|
||||
and OCIO, landed in M9.11; the scripting object model in M14.4.)
|
||||
and OCIO, landed in M9.11; the scripting object model in M14.4; Roto Brush and Refine Edge in M6.7.)
|
||||
|
||||
## By area
|
||||
|
||||
|
|
@ -97,7 +97,7 @@ and OCIO, landed in M9.11; the scripting object model in M14.4.)
|
|||
| Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) |
|
||||
| Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) |
|
||||
| Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves |
|
||||
| Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) |
|
||||
| Masks & roto | 68% | 6.0 | variable-width mask feather points, Roto Brush's learned (3.0) segmentation model (mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) |
|
||||
| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) |
|
||||
| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) |
|
||||
| Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser |
|
||||
|
|
|
|||
Loading…
Reference in New Issue