From b29d833b2d3b9cc69ab11105340829ffee44f71a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:12:36 -0400 Subject: [PATCH] M13.6: Lumetri Scopes, Footage, Media Browser, Metadata, Progress and Content-Aware Fill panels; Content-Aware Fill, Scene Edit Detection, Auto-trace, Align Video to Data Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/align_data.rs | 256 +++++++++ crates/engine/src/commands/autotrace.rs | 211 +++++++ crates/engine/src/commands/content_fill.rs | 344 ++++++++++++ crates/engine/src/commands/footage_panel.rs | 224 ++++++++ crates/engine/src/commands/frontend.rs | 2 +- crates/engine/src/commands/mod.rs | 14 +- crates/engine/src/commands/panels_cmds.rs | 214 ++++++++ crates/engine/src/commands/scene_detect.rs | 182 ++++++ crates/engine/src/commands/stubs.rs | 8 - crates/engine/src/jobs.rs | 320 +++++++++++ crates/engine/src/lib.rs | 23 + crates/engine/src/media_browser.rs | 270 +++++++++ crates/engine/src/menus.rs | 12 +- crates/engine/src/tests_panels.rs | 518 ++++++++++++++++++ crates/project/src/lib.rs | 4 + crates/ui-egui/src/dock.rs | 34 +- crates/ui-egui/src/lib.rs | 5 + .../ui-egui/src/panels/content_fill_panel.rs | 138 +++++ crates/ui-egui/src/panels/footage_panel.rs | 150 +++++ crates/ui-egui/src/panels/forms.rs | 54 ++ crates/ui-egui/src/panels/media_panels.rs | 366 +++++++++++++ crates/ui-egui/src/panels/mod.rs | 13 + crates/ui-egui/src/panels/panel_kit.rs | 70 +++ crates/ui-egui/src/panels/project.rs | 15 + crates/ui-egui/src/panels/scopes_panel.rs | 197 +++++++ crates/ui-egui/src/panels/timeline.rs | 1 + crates/ui-egui/src/state.rs | 4 + crates/ui-egui/tests/ui_panels.rs | 104 ++++ docs/architecture.md | 36 ++ docs/parity.md | 24 +- 30 files changed, 3786 insertions(+), 27 deletions(-) create mode 100644 crates/engine/src/commands/align_data.rs create mode 100644 crates/engine/src/commands/autotrace.rs create mode 100644 crates/engine/src/commands/content_fill.rs create mode 100644 crates/engine/src/commands/footage_panel.rs create mode 100644 crates/engine/src/commands/panels_cmds.rs create mode 100644 crates/engine/src/commands/scene_detect.rs create mode 100644 crates/engine/src/jobs.rs create mode 100644 crates/engine/src/media_browser.rs create mode 100644 crates/engine/src/tests_panels.rs create mode 100644 crates/ui-egui/src/panels/content_fill_panel.rs create mode 100644 crates/ui-egui/src/panels/footage_panel.rs create mode 100644 crates/ui-egui/src/panels/media_panels.rs create mode 100644 crates/ui-egui/src/panels/panel_kit.rs create mode 100644 crates/ui-egui/src/panels/scopes_panel.rs create mode 100644 crates/ui-egui/tests/ui_panels.rs diff --git a/crates/engine/src/commands/align_data.rs b/crates/engine/src/commands/align_data.rs new file mode 100644 index 00000000..1f18de5f --- /dev/null +++ b/crates/engine/src/commands/align_data.rs @@ -0,0 +1,256 @@ +//! Layer ▸ Time ▸ Align Video to Data: move a footage layer in time so its frames line up with +//! the samples of a data file (telemetry, GPS, sensor logs imported as data footage). +//! +//! The data's sample with timestamp `T` sits at comp time `dataStart + (T − T₀)` (`T₀` = the +//! first sample's timestamp; `dataStart` defaults to 0). A video frame at source time `s` was +//! recorded at `V₀ + s`, where `V₀` is the video's start: `videoStart` if given, else the +//! footage file's creation date (desktop). Aligning sets the layer's Start Time to +//! `dataStart + V₀ − T₀`, so the frame recorded at `T` plays when the data says `T`. +//! +//! **Supported data** (File ▸ Import as data footage; also in `docs/architecture.md`): +//! - JSON: an array of objects, or an object holding such an array (the first array of objects +//! found at the top level, e.g. `{"samples": [...]}`); +//! - CSV / TSV with a header row (quoted fields allowed). +//! +//! The time key is `key`, else the first field named `time`, `timestamp`, `t`, `date`, +//! `datetime`, `utc`, `gps_time`, `seconds` or `time_s` (any case). Timestamps may be ISO 8601 +//! date-times (`2026-05-01T10:15:30.5Z`, offsets `±hh:mm`), times of day (`10:15:30.5`), +//! timecode (`10:15:30:12` at the footage's rate), Unix seconds, or Unix milliseconds (numbers +//! above 10¹¹). When the data has only times of day, `V₀` is reduced to its UTC time of day. + +use effectcraft_project::{FootageKind, ItemId, ItemKind}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, f_p, has_layers, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// A parsed timestamp: seconds, and whether it carries a date (absolute) or is a time of day / +/// relative number. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Stamp { + pub secs: f64, + pub dated: bool, +} + +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let mp = (m + 9) % 12; + let doy = (153 * mp + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +/// `hh:mm:ss(.fff)` or timecode `hh:mm:ss:ff` / `hh:mm:ss;ff` → seconds. +fn clock(s: &str, fps: f64) -> Option { + let parts: Vec<&str> = s.split([':', ';']).collect(); + match parts.as_slice() { + [h, m, sec] => Some(h.trim().parse::().ok()? * 3600.0 + m.parse::().ok()? * 60.0 + sec.parse::().ok()?), + [h, m, sec, f] => { + Some(h.trim().parse::().ok()? * 3600.0 + m.parse::().ok()? * 60.0 + sec.parse::().ok()? + f.parse::().ok()? / fps.max(1.0)) + } + [m, sec] => Some(m.trim().parse::().ok()? * 60.0 + sec.parse::().ok()?), + _ => None, + } +} + +/// Parse a timestamp value (see the module docs). `fps` reads timecode frames. +pub fn parse_stamp(v: &Value, fps: f64) -> Option { + match v { + Value::Number(n) => { + let x = n.as_f64()?; + Some(if x > 1e11 { Stamp { secs: x / 1000.0, dated: true } } else { Stamp { secs: x, dated: x > 1e8 } }) + } + Value::String(s) => { + let s = s.trim(); + if let Ok(x) = s.parse::() { + return parse_stamp(&json!(x), fps); + } + // ISO 8601: date [T| ] time [zone]. + if s.len() >= 10 && s.as_bytes()[4] == b'-' && s.as_bytes()[7] == b'-' { + let (y, m, d) = (s[0..4].parse::().ok()?, s[5..7].parse::().ok()?, s[8..10].parse::().ok()?); + let mut secs = days_from_civil(y, m, d) as f64 * 86_400.0; + let rest = s[10..].trim_start_matches(['T', 't', ' ']); + if !rest.is_empty() { + // Zone: Z or ±hh:mm / ±hhmm. + let (time, zone) = match rest.find(['Z', 'z', '+']).or_else(|| rest.rfind('-')) { + Some(i) => (&rest[..i], &rest[i..]), + None => (rest, ""), + }; + secs += clock(time, fps)?; + if zone.len() > 1 { + let sign = if zone.starts_with('-') { -1.0 } else { 1.0 }; + let z = zone[1..].replace(':', ""); + let (zh, zm) = (z.get(0..2)?.parse::().ok()?, z.get(2..4).and_then(|m| m.parse::().ok()).unwrap_or(0.0)); + secs -= sign * (zh * 3600.0 + zm * 60.0); + } + } + return Some(Stamp { secs, dated: true }); + } + clock(s, fps).map(|secs| Stamp { secs, dated: false }) + } + _ => None, + } +} + +const TIME_KEYS: &[&str] = &["time", "timestamp", "t", "date", "datetime", "utc", "gps_time", "gpstime", "seconds", "time_s", "times"]; + +/// Split a CSV/TSV line (quotes allowed). +fn split_row(line: &str, sep: char) -> Vec { + let mut out = vec![]; + let mut cur = String::new(); + let mut q = false; + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '"' if q && chars.peek() == Some(&'"') => { + cur.push('"'); + chars.next(); + } + '"' => q = !q, + c if c == sep && !q => out.push(std::mem::take(&mut cur)), + c => cur.push(c), + } + } + out.push(cur); + out.into_iter().map(|s| s.trim().to_string()).collect() +} + +/// The time column of data footage: (key used, timestamps in row order). +pub fn data_times(text: &str, format: &str, key: Option<&str>) -> std::result::Result<(String, Vec), String> { + let pick = |names: &[String]| -> Option { + if let Some(k) = key { + return names.iter().find(|n| n.eq_ignore_ascii_case(k)).cloned(); + } + TIME_KEYS.iter().find_map(|t| names.iter().find(|n| n.eq_ignore_ascii_case(t)).cloned()) + }; + if format == "json" { + let v: Value = serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|e| format!("invalid JSON: {e}"))?; + let rows = match &v { + Value::Array(a) => a.clone(), + Value::Object(o) => { + o.values().find_map(|x| x.as_array().filter(|a| a.first().is_some_and(Value::is_object)).cloned()).ok_or("no array of samples in the JSON")? + } + _ => return Err("the JSON holds no samples".into()), + }; + let names: Vec = rows.first().and_then(Value::as_object).map(|o| o.keys().cloned().collect()).unwrap_or_default(); + let k = pick(&names) + .ok_or_else(|| format!("no time field (looked for {}); pass `key`", key.map(|k| k.to_string()).unwrap_or_else(|| TIME_KEYS.join(", "))))?; + let times = rows.iter().filter_map(|r| r.get(&k).cloned()).collect(); + return Ok((k, times)); + } + let sep = if format == "tsv" { '\t' } else { ',' }; + let mut lines = text.trim_start_matches('\u{feff}').lines().filter(|l| !l.trim().is_empty()); + let header = split_row(lines.next().ok_or("the file is empty")?, sep); + let k = pick(&header) + .ok_or_else(|| format!("no time column (looked for {}); pass `key`", key.map(|k| k.to_string()).unwrap_or_else(|| TIME_KEYS.join(", "))))?; + let col = header.iter().position(|h| *h == k).unwrap_or(0); + let times = lines.filter_map(|l| split_row(l, sep).get(col).map(|c| json!(c))).collect(); + Ok((k, times)) +} + +fn data_item(s: &Session, p: &Value) -> Result { + let c = "layer.alignVideoToData"; + let id = match p.get("data") { + Some(Value::Number(n)) => n.as_u64().map(ItemId), + Some(Value::String(name)) => s.project.find_by_name(name).map(|i| i.id), + _ => s.project.items.values().find(|i| matches!(&i.kind, ItemKind::Footage(f) if f.kind == FootageKind::Data)).map(|i| i.id), + } + .ok_or_else(|| bad(c, "no data footage: import a JSON/CSV/TSV file and pass `data`"))?; + Ok(id) +} + +fn run(s: &mut Session, p: &Value) -> Result { + let c = "layer.alignVideoToData"; + let (cid, lid) = layer_p(s, p, c)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let footage = match layer.source.item().and_then(|i| s.project.item(i)).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.kind != FootageKind::Data => f.clone(), + _ => return Err(bad(c, "select a footage layer")), + }; + let fps = footage.frame_rate.as_f64(); + let did = data_item(s, p)?; + let Some(ItemKind::Footage(df)) = s.project.item(did).map(|i| &i.kind) else { return Err(bad(c, "not data footage")) }; + let text = df.data.clone().ok_or_else(|| bad(c, "the data item holds no data"))?; + let format = df.codec.to_ascii_lowercase(); + let (key, raw) = data_times(&text, &format, str_p(p, "key").filter(|k| !k.is_empty())).map_err(|e| bad(c, e))?; + let stamps: Vec = raw.iter().filter_map(|v| parse_stamp(v, fps)).collect(); + let first = *stamps.first().ok_or_else(|| bad(c, format!("no readable timestamps in `{key}`")))?; + let video = match p.get("videoStart").filter(|v| v.as_str() != Some("")) { + Some(v) => parse_stamp(v, fps).ok_or_else(|| bad(c, "videoStart: ISO 8601 date-time, hh:mm:ss, timecode or seconds"))?, + None => { + let (created, _, _) = crate::media_browser::file_dates(&footage.path); + Stamp { secs: created.ok_or_else(|| bad(c, "the footage has no recording time; pass `videoStart`"))? as f64, dated: true } + } + }; + // Compare like with like: times of day when the data has no dates. + let v0 = if video.dated && !first.dated { video.secs.rem_euclid(86_400.0) } else { video.secs }; + let data_start = f_p(p, "dataStart").unwrap_or(0.0); + let fr = comp.frame_rate; + let start = fr.snap_nearest(Tick::from_seconds_f64(data_start + v0 - first.secs)); + let shift = start - layer.start_time; + s.edit("Align Video to Data", None, |proj, _| { + let l = super::layer_mut(proj, cid, lid)?; + l.start_time += shift; + l.in_point += shift; + l.out_point += shift; + Ok(()) + })?; + Ok(json!({ + "layer": lid.0, + "startTime": start.seconds(), + "shift": shift.seconds(), + "key": key, + "samples": stamps.len(), + "firstSample": first.secs, + "videoStart": v0, + "dataStart": data_start, + })) +} + +pub fn specs() -> Vec { + vec![cmd!( + "layer.alignVideoToData", + "Align Video to Data", + ["Layer", "Time"], + None, + "{layer?, data?: data footage id|name, key?: time field, videoStart?: ISO date-time | hh:mm:ss | timecode | seconds (default: file creation time), dataStart? (comp s of the first sample, 0)}", + has_layers, + run + )] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_timestamps() { + let p = |v: Value| parse_stamp(&v, 25.0).unwrap(); + assert_eq!(p(json!("1970-01-02T00:00:10Z")).secs, 86_410.0); + assert_eq!(p(json!("2000-03-01T00:00:00Z")).secs, 951_868_800.0); + assert_eq!(p(json!("1970-01-01T02:00:00+02:00")).secs, 0.0); + assert_eq!(p(json!("1970-01-01 01:00:00.5-01:00")).secs, 7200.5); + assert_eq!(p(json!("10:00:01.25")), Stamp { secs: 36_001.25, dated: false }); + assert_eq!(p(json!("10:00:01:05")).secs, 36_001.2); + assert_eq!(p(json!(1_700_000_000_500u64)), Stamp { secs: 1_700_000_000.5, dated: true }); + assert_eq!(p(json!(12.5)), Stamp { secs: 12.5, dated: false }); + assert_eq!(p(json!("3.5")).secs, 3.5); + assert!(parse_stamp(&json!("soon"), 25.0).is_none()); + } + + #[test] + fn finds_time_columns() { + let (k, t) = data_times("lat,Time,speed\n1,\"10:00:00\",3\n2,10:00:01,4\n", "csv", None).unwrap(); + assert_eq!(k, "Time"); + assert_eq!(t, vec![json!("10:00:00"), json!("10:00:01")]); + let (k, t) = data_times("a\tts\n1\t5\n", "tsv", Some("ts")).unwrap(); + assert_eq!((k.as_str(), t), ("ts", vec![json!("5")])); + let (k, t) = data_times(r#"{"meta": 1, "samples": [{"timestamp": 3, "v": 1}, {"timestamp": 4}]}"#, "json", None).unwrap(); + assert_eq!((k.as_str(), t), ("timestamp", vec![json!(3), json!(4)])); + assert!(data_times("x,y\n1,2\n", "csv", None).is_err()); + } +} diff --git a/crates/engine/src/commands/autotrace.rs b/crates/engine/src/commands/autotrace.rs new file mode 100644 index 00000000..093468b6 --- /dev/null +++ b/crates/engine/src/commands/autotrace.rs @@ -0,0 +1,211 @@ +//! Layer ▸ Auto-trace…: masks from a layer's alpha, a colour channel or its luminance, at the +//! current frame or over the work area (one Mask Path key per frame). The tracing is +//! `effectcraft_path::trace` (marching squares, Douglas–Peucker, Bezier fitting). + +use effectcraft_color::Label; +use effectcraft_keyframe::{Interp, Keyframe}; +use effectcraft_path::trace::{TraceOpts, Traced, trace}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{ItemKind, LayerSource, MaskMode, Solid, Value as KV}; +use effectcraft_raster::Image; +use effectcraft_render::{RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_layers, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Channel { + Alpha, + Red, + Green, + Blue, + Luminance, +} + +impl Channel { + fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "alpha" | "a" => Channel::Alpha, + "red" | "r" => Channel::Red, + "green" | "g" => Channel::Green, + "blue" | "b" => Channel::Blue, + "luminance" | "luma" | "l" => Channel::Luminance, + _ => return None, + }) + } +} + +/// The traced scalar field of a frame (straight channel values, 0…1). +pub fn channel_field(img: &Image, ch: Channel, invert: bool) -> Vec { + img.data + .iter() + .map(|p| { + let a = p[3]; + let s = |v: f32| if a > 1e-6 { v / a } else { 0.0 }; + let v = match ch { + Channel::Alpha => a, + Channel::Red => s(p[0]), + Channel::Green => s(p[1]), + Channel::Blue => s(p[2]), + Channel::Luminance => 0.2126 * s(p[0]) + 0.7152 * s(p[1]) + 0.0722 * s(p[2]), + } + .clamp(0.0, 1.0); + if invert { 1.0 - v } else { v } + }) + .collect() +} + +/// Trace one frame: outlines in layer space. +fn trace_frame(img: &Image, offset: [f64; 2], ch: Channel, invert: bool, blur: f64, o: &TraceOpts) -> Vec { + let src = if blur > 0.0 { effectcraft_raster::gaussian_blur(img, blur / 2.0, blur / 2.0, false) } else { img.clone() }; + let field = channel_field(&src, ch, invert); + let mut out = trace(&field, img.width as usize, img.height as usize, o); + for t in &mut out { + for v in &mut t.path.vertices { + v[0] += offset[0]; + v[1] += offset[1]; + } + } + out +} + +fn run(s: &mut Session, p: &Value) -> Result { + let c = "layer.autoTrace"; + let (cid, lid) = layer_p(s, p, c)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + if !layer.source.is_av() { + return Err(bad(c, "auto-trace needs a footage, solid, precomp, text or shape layer")); + } + let ch = match str_p(p, "channel") { + Some(x) => Channel::from_name(x).ok_or_else(|| bad(c, "channel: alpha|red|green|blue|luminance"))?, + None => Channel::Alpha, + }; + let invert = b_p(p, "invert").unwrap_or(false); + let blur = f_p(p, "blur").unwrap_or(1.0).max(0.0); + let o = TraceOpts { + threshold: (f_p(p, "threshold").unwrap_or(50.0) / 100.0).clamp(0.0, 1.0) as f32, + tolerance: f_p(p, "tolerance").unwrap_or(1.0).max(0.01), + min_area: f_p(p, "minimumArea").unwrap_or(10.0).max(0.0), + corner_roundness: (f_p(p, "cornerRoundness").unwrap_or(50.0) / 100.0).clamp(0.0, 1.0), + }; + let fr = comp.frame_rate; + let work_area = matches!(str_p(p, "timeSpan"), Some("workArea" | "work area" | "work_area")); + let times: Vec = if work_area { + let a = comp.work_area.0.max(layer.in_point); + let b = comp.work_area.1.min(layer.out_point); + let (fa, fb) = (fr.frame_at(fr.snap_nearest(a)), fr.frame_at(b - Tick(1))); + (fa..=fb).map(|f| fr.tick_of(f)).collect() + } else { + vec![s.time()] + }; + if times.is_empty() { + return Err(bad(c, "the work area doesn't overlap the layer")); + } + // Trace every frame (in parallel). + let frames: Vec> = { + let mut r = Renderer::new(&s.project, s.footage.as_ref(), RenderOpts::default()); + r.expr = s.expr.as_deref(); + let r = &r; + let (project, expr) = (&s.project, s.expr.as_deref()); + use rayon::prelude::*; + times + .par_iter() + .map(|t| match crate::tracking::source_frame(r, project, cid, &comp, &layer, *t, expr) { + Some((img, off)) => trace_frame(&img, off, ch, invert, blur, &o), + None => vec![], + }) + .collect() + }; + let n_masks = frames.iter().map(Vec::len).max().unwrap_or(0); + if n_masks == 0 { + return Err(bad(c, "nothing to trace: no pixels pass the threshold")); + } + let new_layer = b_p(p, "applyToNewLayer").unwrap_or(false); + let (sw, sh) = effectcraft_render::source_size(&s.project, &layer); + let layer_times: Vec = times.iter().map(|t| layer.layer_time(*t)).collect(); + let target = s.edit("Auto-trace", None, |proj, st| { + let target = if new_layer { + // A solid the size of the layer, in the same place, carries the masks. + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let name = format!("Auto-traced {}", layer.name); + let sid = proj.add_item( + &name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: sw.max(1), height: sh.max(1), pixel_aspect: comp.pixel_aspect }), + ); + let mut l = build::layer(proj, &comp, &name, LayerSource::Solid { item: sid }, (sw.max(1), sh.max(1)), None); + l.start_time = layer.start_time; + l.in_point = layer.in_point; + l.out_point = layer.out_point; + l.stretch = layer.stretch; + l.parent = layer.parent; + if let (Some(dst), Some(src)) = (l.props.sub_mut("transform"), layer.transform()) { + let mut tr = src.clone(); + tr.reassign_uids(&mut proj.next_id); + *dst = tr; + } + let id = l.id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = cm.layers.iter().position(|x| x.id == lid).unwrap_or(0); + cm.layers.insert(at, l); + id + } else { + lid + }; + let mut next = proj.next_id; + let l = super::layer_mut(proj, cid, target)?; + let masks = l.props.sub_mut("masks").ok_or_else(|| bad(c, "this layer can't have masks"))?; + let base = masks.children.len(); + for k in 0..n_masks { + // Depth decides the mode: outlines add, holes subtract. + let first = frames.iter().find_map(|f| f.get(k)).expect("mask"); + let mode = if first.depth % 2 == 0 { MaskMode::Add } else { MaskMode::Subtract }; + let mode = if k == 0 { MaskMode::Add } else { mode }; + let mut g = build::mask( + &mut Ids(&mut next), + &format!("Auto-trace {}", k + 1), + first.path.clone(), + mode, + build::MASK_COLORS[(base + k) % build::MASK_COLORS.len()], + ); + if frames.len() > 1 { + let path = g.get_mut("path").expect("path"); + for (f, lt) in frames.iter().zip(&layer_times) { + if let Some(tr) = f.get(k) { + effectcraft_keyframe::set_key(&mut path.keys, Keyframe::new(*lt, KV::Path(tr.path.clone()))); + } + } + let op = g.get_mut("opacity").expect("opacity"); + if frames.iter().any(|f| f.get(k).is_none()) { + for (f, lt) in frames.iter().zip(&layer_times) { + let mut key = Keyframe::new(*lt, KV::Scalar(if f.get(k).is_some() { 100.0 } else { 0.0 })); + key.in_interp = Interp::Hold; + key.out_interp = Interp::Hold; + effectcraft_keyframe::set_key(&mut op.keys, key); + } + } + } + masks.children.push(g.into()); + } + proj.next_id = next; + st.selected_layers = vec![target]; + Ok(target) + })?; + Ok(json!({"layer": target.0, "masks": n_masks, "frames": times.len()})) +} + +pub fn specs() -> Vec { + vec![cmd!( + "layer.autoTrace", + "Auto-trace...", + ["Layer"], + None, + "{layer?, timeSpan?: currentFrame|workArea, channel?: alpha|red|green|blue|luminance, invert?, blur? (px, 1), tolerance? (px, 1), threshold? (%, 50), minimumArea? (px, 10), cornerRoundness? (%, 50), applyToNewLayer?}", + has_layers, + run + )] +} diff --git a/crates/engine/src/commands/content_fill.rs b/crates/engine/src/commands/content_fill.rs new file mode 100644 index 00000000..b030a991 --- /dev/null +++ b/crates/engine/src/commands/content_fill.rs @@ -0,0 +1,344 @@ +//! Content-Aware Fill (Window ▸ Content-Aware Fill, Layer ▸ New ▸ Content-Aware Fill Layer): +//! fill the transparent area of a layer (what its masks cut away) across the work area or the +//! whole layer, render the result as a PNG sequence and add it as a "Fill" layer above the +//! source, like After Effects. The algorithms are `effectcraft_raster::inpaint` (Object: +//! flow-guided propagation + PatchMatch; Surface: propagation + one synthesised fill carried by +//! the flow; Edge Blend: membrane fill). It runs as a background job (Window ▸ Progress). + +use std::sync::Arc; + +use effectcraft_color::Label; +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource, build}; +use effectcraft_raster::Image; +use effectcraft_raster::inpaint::{FillInput, FillMethod, FillOpts, dilate, fill_sequence}; +use effectcraft_render::{EvalCtx, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use super::scene_detect::LayerFrames; +use super::{CommandSpec, always, b_p, bad, f_p, has_layers, layer_p, str_p}; +use crate::jobs::Apply; +use crate::{EngineError, Result, Session, cmd}; + +/// Content-Aware Fill panel settings (serde, in [`crate::EditorState`]). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct FillSettings { + /// `object`, `surface` or `edgeBlend`. + pub method: String, + /// `workArea` or `entire` (the layer's duration). + pub range: String, + /// Grow the fill area by this many pixels. + pub alpha_expansion: f64, + /// `none`, `subtle`, `moderate`, `strong`. + pub lighting_correction: String, + /// Reference frame: a layer of the comp and the comp time of the frame to use. + pub reference_layer: Option, + pub reference_time: Option, +} + +impl Default for FillSettings { + fn default() -> Self { + FillSettings { + method: "object".into(), + range: "workArea".into(), + alpha_expansion: 0.0, + lighting_correction: "none".into(), + reference_layer: None, + reference_time: None, + } + } +} + +fn lighting(s: &str) -> Option { + Some(match s { + "none" | "off" => 0.0, + "subtle" => 0.35, + "moderate" => 0.65, + "strong" => 1.0, + _ => return None, + }) +} + +fn set(s: &mut Session, p: &Value) -> Result { + let c = "contentFill.set"; + let mut st = s.state.content_fill.clone(); + if let Some(m) = str_p(p, "method") { + st.method = FillMethod::from_name(m).ok_or_else(|| bad(c, "method: object|surface|edgeBlend"))?.name().into(); + } + if let Some(r) = str_p(p, "range") { + st.range = match r { + "workArea" | "work area" => "workArea".into(), + "entire" | "entireDuration" | "layer" => "entire".into(), + _ => return Err(bad(c, "range: workArea|entire")), + }; + } + if let Some(a) = f_p(p, "alphaExpansion") { + st.alpha_expansion = a.clamp(0.0, 100.0); + } + if let Some(l) = str_p(p, "lightingCorrection") { + lighting(l).ok_or_else(|| bad(c, "lightingCorrection: none|subtle|moderate|strong"))?; + st.lighting_correction = l.into(); + } + match p.get("referenceLayer") { + Some(Value::Null) => st.reference_layer = None, + Some(v) => { + let comp = s.active_comp().ok_or(EngineError::NoComp)?; + st.reference_layer = Some(super::resolve_layer(comp, v).ok_or_else(|| bad(c, format!("no layer {v}")))?); + st.reference_time = Some(f_p(p, "referenceTime").unwrap_or(s.time().seconds())); + } + None => { + if let Some(t) = f_p(p, "referenceTime") { + st.reference_time = Some(t); + } + } + } + s.state.content_fill = st.clone(); + Ok(serde_json::to_value(st).unwrap_or_default()) +} + +/// A layer's masked source at comp time `t` on a `w × h` canvas in layer pixels (straight colour +/// with alpha 1 where kept), and the hole (`true` where the alpha is under one half). +fn masked_frame(r: &Renderer, ctx: &EvalCtx, layer: &effectcraft_project::Layer, w: u32, h: u32) -> (Image, Vec) { + let mut img = Image::new(w, h); + let mut hole = vec![true; (w * h) as usize]; + if let Some(buf) = r.layer_input(ctx, layer, 0) { + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for y in 0..h { + for x in 0..w { + let (bx, by) = ((x as f64 + 0.5) * sc - buf.offset[0], (y as f64 + 0.5) * sc - buf.offset[1]); + let p = if (sc - 1.0).abs() < 1e-9 { buf.img.get(bx.floor() as i64, by.floor() as i64) } else { buf.img.sample_bilinear(bx, by) }; + let i = (y * w + x) as usize; + if p[3] >= 0.5 { + img.data[i] = [p[0] / p[3], p[1] / p[3], p[2] / p[3], 1.0]; + hole[i] = false; + } + } + } + } + (img, hole) +} + +/// Encode a straight-alpha frame (alpha 1) as 8-bit PNG. +fn png(img: &Image) -> std::result::Result, String> { + let rgba: Vec = img.data.iter().flat_map(|p| [p[0], p[1], p[2], p[3]].map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8)).collect(); + super::comp_more::encode_png(&rgba, img.width, img.height) +} + +fn default_dir(s: &Session, name: &str) -> String { + let base = s + .path + .as_deref() + .and_then(|p| std::path::Path::new(p).parent().map(|d| d.join("Fill"))) + .unwrap_or_else(|| std::env::temp_dir().join("effectcraft-fill")); + let safe: String = name.chars().map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' }).collect(); + let mut k = 1; + loop { + let d = base.join(format!("{safe}_Fill_{k}")); + if !d.exists() { + return d.to_string_lossy().to_string(); + } + k += 1; + } +} + +fn generate(s: &mut Session, p: &Value) -> Result { + let c = "contentFill.generate"; + let (cid, lid) = layer_p(s, p, c)?; + // Parameters override the panel settings. + let mut st = s.state.content_fill.clone(); + { + let saved = s.state.content_fill.clone(); + let mut q = p.clone(); + if let Some(o) = q.as_object_mut() { + o.retain(|k, _| ["method", "range", "alphaExpansion", "lightingCorrection", "referenceLayer", "referenceTime"].contains(&k.as_str())); + } + set(s, &q)?; + std::mem::swap(&mut st, &mut s.state.content_fill); + s.state.content_fill = saved; + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + if !layer.source.is_av() || layer.source.item().is_none() { + return Err(bad(c, "Content-Aware Fill needs a footage, solid or precomp layer")); + } + let (w, h) = effectcraft_render::source_size(&s.project, &layer); + if w == 0 || h == 0 { + return Err(bad(c, "the layer has no pixels")); + } + let fr = comp.frame_rate; + let (a, b) = + if st.range == "entire" { (layer.in_point, layer.out_point) } else { (comp.work_area.0.max(layer.in_point), comp.work_area.1.min(layer.out_point)) }; + let (a, b) = (a.max(Tick::ZERO), b.min(comp.duration)); + if b <= a { + return Err(bad(c, "the range doesn't overlap the layer")); + } + let (fa, fb) = (fr.frame_at(fr.snap_nearest(a)), fr.frame_at(b - Tick(1))); + let times: Vec = (fa..=fb).map(|f| fr.tick_of(f)).collect(); + let opts = FillOpts { + method: FillMethod::from_name(&st.method).unwrap_or_default(), + lighting: lighting(&st.lighting_correction).unwrap_or(0.0), + ..Default::default() + }; + let expand = st.alpha_expansion.round().max(0.0) as usize; + let reference = st.reference_layer.filter(|r| comp.layer(*r).is_some()).map(|r| (r, Tick::from_seconds_f64(st.reference_time.unwrap_or(0.0)))); + let dir = str_p(p, "outputDir").map(str::to_string).unwrap_or_else(|| default_dir(s, &layer.name)); + let lf = LayerFrames::new(s, cid, lid); + let services = s.services.clone(); + let importer = s.importer.clone(); + let wait = b_p(p, "wait").unwrap_or(false); + let name = layer.name.clone(); + s.spawn_task("contentFill", format!("Content-Aware Fill: {name}"), wait, move |ctl| { + ctl.message("Reading frames"); + let project = lf.project.clone(); + let comp = project.comp(cid).ok_or("composition missing")?; + let layer = comp.layer(lid).ok_or("layer missing")?; + let mut r = Renderer::new(&project, lf.footage.as_ref(), RenderOpts::default()); + r.expr = lf.expr.as_deref(); + r.cache = Some(&lf.cache); + let ctx0 = EvalCtx { expr: lf.expr.as_deref(), ..EvalCtx::new(&project, cid, comp, Tick::ZERO) }; + let n = times.len() as u64; + let mut frames = Vec::with_capacity(times.len()); + let mut holes = Vec::with_capacity(times.len()); + for (k, t) in times.iter().enumerate() { + let (img, hole) = masked_frame(&r, &ctx0.at(*t), layer, w, h); + holes.push(dilate(&hole, w as usize, h as usize, expand)); + frames.push(img); + if !ctl.progress(k as u64 + 1, n * 3) { + return Err(crate::render_queue::CANCELLED.into()); + } + } + if !holes.iter().any(|h| h.iter().any(|b| *b)) { + return Err("the layer has no transparent area to fill: draw a mask (Subtract) around what to remove".into()); + } + let mut references = vec![]; + if let Some((rl, rt)) = reference + && let Some(l) = comp.layer(rl) + { + let (img, hole) = masked_frame(&r, &ctx0.at(rt), l, w, h); + if hole.iter().all(|b| !*b) { + let k = times.iter().enumerate().min_by_key(|(_, t)| (t.0 - rt.0).abs()).map(|(k, _)| k).unwrap_or(0); + references.push((k, img)); + } + } + ctl.message("Filling"); + let input = FillInput { frames, holes, references }; + let out = fill_sequence(&input, &opts, &mut |d, t| ctl.progress(n + (d as u64 * n / t.max(1) as u64), n * 3)).ok_or(crate::render_queue::CANCELLED)?; + ctl.message("Writing the fill layer"); + std::fs::create_dir_all(&dir).map_err(|e| format!("{dir}: {e}"))?; + let mut paths = vec![]; + for (k, img) in out.iter().enumerate() { + let path = std::path::Path::new(&dir).join(format!("fill_{k:05}.png")).to_string_lossy().to_string(); + services.write_file(&path, &png(img)?).map_err(|e| format!("{path}: {e}"))?; + paths.push(path); + if !ctl.progress(2 * n + k as u64 + 1, n * 3) { + return Err(crate::render_queue::CANCELLED.into()); + } + } + let start = times[0]; + let count = times.len(); + let apply: Apply = Box::new(move |s: &mut Session| add_fill_layer(s, cid, lid, &name, paths, (w, h), start, count, importer)); + Ok(apply) + }) +} + +/// Import the rendered sequence and put it in a layer above the source. +#[allow(clippy::too_many_arguments)] +fn add_fill_layer( + s: &mut Session, + cid: ItemId, + lid: LayerId, + name: &str, + paths: Vec, + size: (u32, u32), + start: Tick, + count: usize, + importer: Option>, +) -> Result { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or_else(|| bad("contentFill.generate", "the source layer is gone"))?.clone(); + let fr = comp.frame_rate; + let dur = fr.tick_of(count as i64); + let mut footage = importer.and_then(|i| i.probe(&paths[0]).ok()).filter(|f| f.has_video).unwrap_or_else(|| Footage { + path: paths[0].clone(), + kind: FootageKind::Sequence, + width: size.0, + height: size.1, + has_video: true, + codec: "PNG".into(), + ..Default::default() + }); + footage.kind = FootageKind::Sequence; + footage.sequence = paths.clone(); + footage.frame_rate = fr; + footage.duration = dur; + let (lname, item, layer) = s.edit("Content-Aware Fill", None, |proj, st| { + let folder = proj.folder_named("Fill").unwrap_or_else(|| proj.add_item("Fill", Label::Yellow, None, ItemKind::Folder)); + let n = proj.items.values().filter(|i| i.parent == Some(folder)).count() + 1; + let lname = format!("Fill {n} [{name}]"); + let iid = proj.add_item(&lname, Label::Lavender, Some(folder), ItemKind::Footage(footage)); + let mut l = build::layer(proj, &comp, &lname, LayerSource::Footage { item: iid }, size, Some(dur)); + l.start_time = start; + l.in_point = start; + l.out_point = (start + dur).min(comp.duration); + l.parent = src.parent; + l.switches.three_d = src.switches.three_d; + if let (Some(dst), Some(t)) = (l.props.sub_mut("transform"), src.transform()) { + let mut tr = t.clone(); + tr.reassign_uids(&mut proj.next_id); + *dst = tr; + } + let id = l.id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = cm.layers.iter().position(|x| x.id == lid).unwrap_or(0); + cm.layers.insert(at, l); + st.selected_layers = vec![id]; + Ok((lname, iid, id)) + })?; + Ok(json!({"layer": layer.0, "item": item.0, "name": lname, "frames": count, "files": paths})) +} + +fn open_panel(s: &mut Session, p: &Value) -> Result { + // Parameters given: generate straight away (agents); else show the panel. + if p.as_object().is_some_and(|o| !o.is_empty()) { + return generate(s, p); + } + s.events.push(crate::Event::Frontend { command: "window.panel".into(), params: json!({"panel": "contentAwareFill"}) }); + Ok(json!({"frontend": "window.panel"})) +} + +pub fn specs() -> Vec { + const P: &str = "{layer?, method?: object|surface|edgeBlend, range?: workArea|entire, alphaExpansion? (px), lightingCorrection?: none|subtle|moderate|strong, referenceLayer?, referenceTime? (s), outputDir?, wait?: bool}"; + vec![ + cmd!( + "contentFill.set", + "Content-Aware Fill Settings", + [], + None, + "{method?: object|surface|edgeBlend, range?: workArea|entire, alphaExpansion? (px), lightingCorrection?: none|subtle|moderate|strong, referenceLayer?: layer|null, referenceTime? (s)}", + always, + set + ), + CommandSpec { + id: "contentFill.generate", + label: "Generate Fill Layer", + menu: &[], + shortcut: None, + params: P, + enabled: has_layers, + run: generate, + journal: true, + }, + CommandSpec { + id: "layer.newContentAwareFill", + label: "Content-Aware Fill Layer...", + menu: &["Layer", "New"], + shortcut: None, + params: P, + enabled: always, + run: open_panel, + journal: true, + }, + ] +} diff --git a/crates/engine/src/commands/footage_panel.rs b/crates/engine/src/commands/footage_panel.rs new file mode 100644 index 00000000..4d11a35a --- /dev/null +++ b/crates/engine/src/commands/footage_panel.rs @@ -0,0 +1,224 @@ +//! The Footage panel (double-click footage in the Project panel): plays a footage item on its own +//! time ruler, marks In and Out points in source time, and edits the marked range into the active +//! composition with **Overlay Edit** (a new layer at the current time, on top) or **Ripple Insert +//! Edit** (the same, after pushing later layers — and the later part of layers crossing the +//! current time — back by the clip's duration). + +use effectcraft_project::{ItemId, ItemKind, LayerSource, build}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, comp_id, f_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// What the Footage panel shows (source time). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FootageView { + pub item: ItemId, + pub time: Tick, + #[serde(default)] + pub in_point: Option, + #[serde(default)] + pub out_point: Option, +} + +fn footage_item(s: &Session, p: &Value, cmd: &str) -> Result { + let id = match p.get("item") { + Some(Value::Number(n)) => n.as_u64().map(ItemId), + Some(Value::String(name)) => s.project.find_by_name(name).map(|i| i.id), + _ => s.state.footage_panel.as_ref().map(|f| f.item).or_else(|| s.state.project_selection.first().copied()), + } + .ok_or_else(|| bad(cmd, "no footage: pass `item` or open one in the Footage panel"))?; + match s.project.item(id).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.kind != effectcraft_project::FootageKind::Data => Ok(id), + Some(ItemKind::Solid(_)) => Ok(id), + _ => Err(bad(cmd, format!("item {} is not footage", id.0))), + } +} + +/// Source duration and frame rate of a footage / solid item (stills: `None`). +fn source_timing(s: &Session, item: ItemId) -> (Option, effectcraft_time::FrameRate) { + match s.project.item(item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.kind != effectcraft_project::FootageKind::Still && f.duration > Tick::ZERO => (Some(f.duration), f.frame_rate), + Some(ItemKind::Footage(f)) => (None, f.frame_rate), + _ => (None, s.active_comp().map(|c| c.frame_rate).unwrap_or(effectcraft_time::FrameRate::FPS_30)), + } +} + +fn source_time(s: &Session, p: &Value, item: ItemId) -> Option { + let (_, fr) = source_timing(s, item); + if let Some(t) = f_p(p, "time") { + return Some(fr.snap_nearest(Tick::from_seconds_f64(t.max(0.0)))); + } + p.get("frame").and_then(Value::as_i64).map(|f| fr.tick_of(f.max(0))) +} + +fn open(s: &mut Session, p: &Value) -> Result { + let item = footage_item(s, p, "footage.open")?; + if s.state.footage_panel.as_ref().is_none_or(|f| f.item != item) { + s.state.footage_panel = Some(FootageView { item, time: Tick::ZERO, in_point: None, out_point: None }); + } + s.events.push(crate::Event::Frontend { command: "window.panel".into(), params: json!({"panel": "footage"}) }); + Ok(json!(s.state.footage_panel)) +} + +fn view_mut<'a>(s: &'a mut Session, cmd: &str) -> Result<&'a mut FootageView> { + s.state.footage_panel.as_mut().ok_or_else(|| bad(cmd, "the Footage panel shows no footage (footage.open)")) +} + +fn clamp_time(s: &Session, item: ItemId, t: Tick) -> Tick { + let (dur, fr) = source_timing(s, item); + match dur { + Some(d) => t.clamp(Tick::ZERO, (d - fr.frame_duration()).max(Tick::ZERO)), + None => t.max(Tick::ZERO), + } +} + +fn set_time(s: &mut Session, p: &Value) -> Result { + let item = view_mut(s, "footage.setTime")?.item; + let t = source_time(s, p, item).ok_or_else(|| bad("footage.setTime", "missing `time` or `frame`"))?; + let t = clamp_time(s, item, t); + view_mut(s, "footage.setTime")?.time = t; + Ok(json!(t.seconds())) +} + +fn set_mark(s: &mut Session, p: &Value, out: bool) -> Result { + let c = if out { "footage.setOut" } else { "footage.setIn" }; + let v = view_mut(s, c)?.clone(); + let t = source_time(s, p, v.item).unwrap_or(v.time); + let t = clamp_time(s, v.item, t); + let (_, fr) = source_timing(s, v.item); + let view = view_mut(s, c)?; + if out { + // The Out point marks the last frame shown: the range ends after it. + let end = t + fr.frame_duration(); + view.out_point = Some(end); + if view.in_point.is_some_and(|i| i >= end) { + view.in_point = None; + } + } else { + view.in_point = Some(t); + if view.out_point.is_some_and(|o| o <= t) { + view.out_point = None; + } + } + Ok(json!(s.state.footage_panel)) +} + +fn clear_marks(s: &mut Session, _: &Value) -> Result { + let v = view_mut(s, "footage.clearInOut")?; + v.in_point = None; + v.out_point = None; + Ok(Value::Null) +} + +/// Overlay / Ripple Insert Edit of the marked range at the comp's current time. +fn edit_into_comp(s: &mut Session, p: &Value, ripple: bool) -> Result { + let c = if ripple { "footage.rippleInsertEdit" } else { "footage.overlayEdit" }; + let item = footage_item(s, p, c)?; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let view = s.state.footage_panel.clone().filter(|v| v.item == item); + let (dur, _) = source_timing(s, item); + let src_in = f_p(p, "in").map(Tick::from_seconds_f64).or(view.as_ref().and_then(|v| v.in_point)).unwrap_or(Tick::ZERO); + let fr = comp.frame_rate; + let at = fr.snap_nearest(f_p(p, "at").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid))).max(Tick::ZERO); + let src_out = f_p(p, "out").map(Tick::from_seconds_f64).or(view.as_ref().and_then(|v| v.out_point)).or(dur).unwrap_or(src_in + (comp.duration - at)); + if src_out <= src_in { + return Err(bad(c, "the Out point must be after the In point")); + } + let len = fr.snap_nearest(src_out - src_in).max(fr.frame_duration()); + if at >= comp.duration { + return Err(bad(c, "the current time is past the end of the composition")); + } + let it = s.project.item(item).cloned().ok_or_else(|| bad(c, "no such item"))?; + let (src, size) = match &it.kind { + ItemKind::Footage(f) => (LayerSource::Footage { item }, (f.width, f.height)), + ItemKind::Solid(so) => (LayerSource::Solid { item }, (so.width, so.height)), + _ => return Err(bad(c, "not footage")), + }; + let label = if ripple { "Ripple Insert Edit" } else { "Overlay Edit" }; + let id = s.edit(label, None, |proj, st| { + let mut next = proj.next_id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + if ripple { + let mut added = vec![]; + for i in 0..cm.layers.len() { + let l = &mut cm.layers[i]; + if l.in_point >= at { + l.start_time += len; + l.in_point += len; + l.out_point += len; + } else if l.out_point > at { + // Split: the part after the current time moves back. + let mut b = l.clone(); + super::edit::reid(&mut b, &mut next); + l.out_point = at; + b.in_point = at + len; + b.out_point += len; + b.start_time += len; + added.push((i, b)); + } + } + for (i, b) in added.into_iter().rev() { + cm.layers.insert(i, b); + } + } + proj.next_id = next; + let mut l = build::layer(proj, &comp, &it.name, src, size, Some(len)); + l.start_time = at - src_in; + l.in_point = at; + l.out_point = (at + len).min(comp.duration); + let lid = l.id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + cm.layers.insert(0, l); + st.selected_layers = vec![lid]; + st.selected_props.clear(); + st.selected_keys.clear(); + Ok(lid) + })?; + Ok(json!({"layer": id.0, "in": at.seconds(), "duration": len.seconds()})) +} + +fn has_view_or_footage(s: &Session) -> std::result::Result<(), String> { + if s.state.footage_panel.is_some() { Ok(()) } else { Err("open footage in the Footage panel first".into()) } +} + +fn can_edit(s: &Session) -> std::result::Result<(), String> { + has_view_or_footage(s)?; + super::has_comp(s) +} + +fn info(s: &mut Session, _: &Value) -> Result { + let Some(v) = &s.state.footage_panel else { return Ok(Value::Null) }; + let (dur, fr) = source_timing(s, v.item); + let name = s.project.item(v.item).map(|i| i.name.clone()).unwrap_or_default(); + Ok( + json!({"item": v.item.0, "name": name, "time": v.time.seconds(), "frame": fr.frame_at(v.time), "in": v.in_point.map(|t| t.seconds()), "out": v.out_point.map(|t| t.seconds()), "duration": dur.map(|d| d.seconds()), "frameRate": fr.as_f64()}), + ) +} + +pub fn specs() -> Vec { + vec![ + cmd!("footage.open", "Open in Footage Panel", [], None, "{item: id|name}", always, open), + crate::query!("footage.info", "Footage Panel State", "{}", info), + cmd!("footage.setTime", "Footage Panel Time", [], None, "{time? (s, source) | frame?}", has_view_or_footage, set_time), + cmd!("footage.setIn", "Set In Point", [], None, "{time? | frame? (default: the panel's time)}", has_view_or_footage, |s, p| set_mark(s, p, false)), + cmd!("footage.setOut", "Set Out Point", [], None, "{time? | frame? (default: the panel's time)}", has_view_or_footage, |s, p| set_mark(s, p, true)), + cmd!("footage.clearInOut", "Clear In and Out", [], None, "{}", has_view_or_footage, clear_marks), + cmd!("footage.overlayEdit", "Overlay Edit", [], None, "{item?, comp?, in? (s), out? (s), at? (comp s; default the current time)}", can_edit, |s, p| { + edit_into_comp(s, p, false) + }), + cmd!( + "footage.rippleInsertEdit", + "Ripple Insert Edit", + [], + None, + "{item?, comp?, in? (s), out? (s), at? (comp s; default the current time)}", + can_edit, + |s, p| edit_into_comp(s, p, true) + ), + ] +} diff --git a/crates/engine/src/commands/frontend.rs b/crates/engine/src/commands/frontend.rs index 168e77a5..7abd2923 100644 --- a/crates/engine/src/commands/frontend.rs +++ b/crates/engine/src/commands/frontend.rs @@ -63,7 +63,7 @@ pub fn specs() -> Vec { "Show Panel", [], None, - "{panel: project|effectControls|composition|layer|timeline|info|audio|preview|effectsPresets|properties|character|paragraph|align|tracker|wiggler|smoother|motionSketch|paint|brushes|renderQueue|flowchart|history|markers|tools}", + "{panel: project|effectControls|composition|layer|timeline|info|audio|preview|effectsPresets|properties|character|paragraph|align|tracker|wiggler|smoother|motionSketch|paint|brushes|renderQueue|flowchart|history|markers|tools|lumetriScopes|footage|mediaBrowser|metadata|progress|contentAwareFill}", always ), fe!( diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 39fa4234..12324cf4 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -1,12 +1,15 @@ //! The command registry. Ids follow After Effects' menu structure; every menu item, panel button, //! shortcut and viewer/timeline gesture maps to one of these. +mod align_data; mod anim; pub mod anim_tools; mod animation; +mod autotrace; mod camera_cmds; mod comp; -mod comp_more; +pub(crate) mod comp_more; +pub mod content_fill; mod create; mod edit; mod effect; @@ -15,6 +18,7 @@ pub mod expr_tools; mod file; pub(crate) mod file_more; mod focus; +pub mod footage_panel; mod frontend; mod help; mod key_labels; @@ -30,6 +34,7 @@ mod mask; pub mod mask_interp; pub(crate) mod model3d; pub mod paint; +pub mod panels_cmds; mod paths; mod project_items; mod prop; @@ -39,6 +44,7 @@ pub mod puppet; mod query; mod render_queue; pub mod roto_cmds; +mod scene_detect; mod settings; mod shape_stroke; mod stubs; @@ -156,6 +162,12 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(essential::specs()); v.extend(expr_tools::specs()); v.extend(proxy::specs()); + v.extend(panels_cmds::specs()); + v.extend(footage_panel::specs()); + v.extend(autotrace::specs()); + v.extend(scene_detect::specs()); + v.extend(align_data::specs()); + v.extend(content_fill::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/panels_cmds.rs b/crates/engine/src/commands/panels_cmds.rs new file mode 100644 index 00000000..eb056878 --- /dev/null +++ b/crates/engine/src/commands/panels_cmds.rs @@ -0,0 +1,214 @@ +//! Commands behind the Progress, Media Browser, Metadata and Lumetri Scopes panels. + +use effectcraft_project::ItemId; +use effectcraft_raster::Image; +use effectcraft_raster::scopes::{self, ColorStandard, ScopeKind, ScopeOpts}; +use effectcraft_render::RenderOpts; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, b_p, bad, f_p, str_p}; +use crate::{EngineError, Result, Session, cmd, media_browser as mb, query}; + +// ---------------------------------------------------------------- Progress + +fn jobs_list(s: &mut Session, _: &Value) -> Result { + s.poll_jobs(); + Ok(json!({"running": s.jobs(), "finished": s.job_log})) +} + +fn jobs_cancel(s: &mut Session, p: &Value) -> Result { + let id = str_p(p, "job").unwrap_or("all"); + Ok(json!({"cancelled": s.cancel_job(id)})) +} + +fn jobs_wait(s: &mut Session, _: &Value) -> Result { + s.wait_jobs(); + Ok(json!({"finished": s.job_log})) +} + +// ---------------------------------------------------------------- Media Browser + +fn browser_dir(s: &Session, p: &Value) -> String { + str_p(p, "path").map(str::to_string).or_else(|| s.state.media_browser.folder.clone()).unwrap_or_else(mb::home) +} + +fn browser_list(s: &mut Session, p: &Value) -> Result { + let dir = browser_dir(s, p); + let only = b_p(p, "importableOnly").unwrap_or(s.state.media_browser.importable_only); + let entries = mb::list(&dir, only).map_err(EngineError::Other)?; + Ok(json!({"path": dir, "parent": mb::parent(&dir), "entries": entries, "favorites": s.state.media_browser.favorites})) +} + +fn browser_enabled(_: &Session) -> std::result::Result<(), String> { + if mb::available() { Ok(()) } else { Err("the Media Browser needs the desktop app".into()) } +} + +fn browser_go(s: &mut Session, p: &Value) -> Result { + let dir = match str_p(p, "path") { + Some("..") => s.state.media_browser.folder.clone().and_then(|d| mb::parent(&d)).unwrap_or_else(mb::home), + Some(d) => d.to_string(), + None => mb::home(), + }; + if let Some(b) = b_p(p, "importableOnly") { + s.state.media_browser.importable_only = b; + } + // Validate before switching. + mb::list(&dir, false).map_err(EngineError::Other)?; + s.state.media_browser.folder = Some(dir.clone()); + browser_list(s, &json!({"path": dir})) +} + +fn browser_favorite(s: &mut Session, p: &Value, add: bool) -> Result { + let dir = browser_dir(s, p); + let favs = &mut s.state.media_browser.favorites; + favs.retain(|f| f != &dir); + if add { + favs.push(dir); + } + Ok(json!({"favorites": favs})) +} + +fn browser_import(s: &mut Session, p: &Value) -> Result { + let paths = p.get("paths").or(p.get("path")).cloned().ok_or_else(|| bad("mediaBrowser.import", "missing `paths`"))?; + let r = s.execute("file.import", json!({"paths": paths}))?; + // Add to the active comp (drag into the timeline). + if b_p(p, "addToComp").unwrap_or(false) + && s.active_comp_id().is_some() + && let Some(items) = r.get("items").and_then(Value::as_array) + { + for it in items { + s.execute("layer.addItem", json!({"item": it}))?; + } + } + Ok(r) +} + +fn file_info(s: &mut Session, p: &Value) -> Result { + let path = str_p(p, "path").ok_or_else(|| bad("mediaBrowser.fileInfo", "missing `path`"))?; + let (created, modified, size) = mb::file_dates(path); + let mut v = json!({"path": path, "kind": mb::kind_of(path), "size": size, "created": created.map(mb::iso_date), "modified": modified.map(mb::iso_date)}); + if let Some(imp) = s.importer.clone() + && matches!(mb::kind_of(path), Some("video" | "image" | "audio" | "model")) + && let Ok(f) = imp.probe(path) + { + v["footage"] = mb::footage_metadata(&f); + } + Ok(v) +} + +// ---------------------------------------------------------------- Metadata + +fn item_ref(s: &Session, p: &Value) -> Option { + match p.get("item") { + Some(Value::Number(n)) => n.as_u64().map(ItemId), + Some(Value::String(name)) => s.project.items.values().find(|i| &i.name == name).map(|i| i.id), + _ => s.state.project_selection.first().copied(), + } +} + +fn metadata(s: &mut Session, p: &Value) -> Result { + let project = mb::project_metadata(&s.project, s.path.as_deref()); + let item = match item_ref(s, p) { + Some(id) => Some(mb::item_metadata(&s.project, s.project.item(id).ok_or_else(|| bad("item.metadata", format!("no item {}", id.0)))?)), + None => None, + }; + Ok(json!({"project": project, "item": item})) +} + +fn project_comment(s: &mut Session, p: &Value) -> Result { + let text = str_p(p, "comment").ok_or_else(|| bad("project.setProjectComment", "missing `comment`"))?.to_string(); + s.edit("Project Comment", super::merge_p(p), |proj, _| { + proj.settings.comment = text; + Ok(()) + })?; + Ok(Value::Null) +} + +// ---------------------------------------------------------------- Lumetri Scopes + +/// The comp frame the scopes read (the viewer's image: over the comp background), at most +/// `max_side` pixels on the long side. +pub fn scope_frame(s: &Session, comp: ItemId, t: Tick, max_side: u32) -> Option { + let c = s.project.comp(comp)?; + let long = c.width.max(c.height).max(1) as f64; + let scale = (max_side as f64 / long).min(1.0); + let mut img = s.render(comp, t, RenderOpts { scale, ..Default::default() }); + let bg = c.background; + for p in &mut img.data { + let k = 1.0 - p[3].clamp(0.0, 1.0); + *p = [p[0] + bg[0] * k, p[1] + bg[1] * k, p[2] + bg[2] * k, 1.0]; + } + Some(img) +} + +/// Scope options from parameters. +pub fn scope_opts(p: &Value) -> std::result::Result<(ScopeKind, ScopeOpts), String> { + let kind = match str_p(p, "scope") { + Some(k) => ScopeKind::from_name(k).ok_or_else(|| format!("unknown scope `{k}`; one of {}", ScopeKind::ALL.map(|k| k.name()).join(", ")))?, + None => ScopeKind::WaveformRgb, + }; + let standard = match str_p(p, "standard") { + Some(x) => ColorStandard::from_name(x).ok_or_else(|| format!("standard: rec601|rec709|rec2020, not `{x}`"))?, + None => ColorStandard::Rec709, + }; + let size = f_p(p, "size").unwrap_or(256.0).clamp(16.0, 1024.0) as u32; + Ok((kind, ScopeOpts { standard, float: b_p(p, "float").unwrap_or(false), clamp: b_p(p, "clamp").unwrap_or(true), width: size, height: size })) +} + +fn scopes_analyze(s: &mut Session, p: &Value) -> Result { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = super::time_p(s, p, Some(comp)); + let (kind, o) = scope_opts(p).map_err(|e| bad("scopes.analyze", e))?; + let img = scope_frame(s, cid, t, 512).ok_or(EngineError::NoComp)?; + let sc = scopes::compute(&img, kind, &o); + // Signal statistics of the frame. + let n = img.data.len().max(1) as f64; + let mut mean = [0.0f64; 3]; + let (mut ymin, mut ymax, mut ysum) = (f64::MAX, f64::MIN, 0.0); + for px in &img.data { + for c in 0..3 { + mean[c] += px[c] as f64 / n; + } + let y = o.standard.ycbcr([px[0], px[1], px[2]])[0] as f64; + ymin = ymin.min(y); + ymax = ymax.max(y); + ysum += y; + } + let mut v = json!({ + "scope": kind.name(), + "width": sc.width, + "height": sc.height, + "peaks": [sc.peak(0), sc.peak(1), sc.peak(2)], + "totals": [sc.total(0), sc.total(1), sc.total(2)], + "mean": mean, + "luma": {"min": ymin, "max": ymax, "mean": ysum / n}, + }); + if kind == ScopeKind::Histogram { + v["histogram"] = json!(scopes::histogram_bars(&sc)); + } + Ok(v) +} + +pub fn specs() -> Vec { + vec![ + query!("jobs.list", "Background Jobs", "{}", jobs_list), + cmd!("jobs.cancel", "Cancel Job", [], None, "{job?: render|track|maskTrack|warp|camera|roto|task:|all}", always, jobs_cancel), + query!("jobs.wait", "Wait for Background Jobs", "{}", jobs_wait), + query!("mediaBrowser.list", "List Folder", "{path?, importableOnly?}", browser_list), + cmd!("mediaBrowser.go", "Go to Folder", [], None, "{path?: folder | \"..\", importableOnly?}", browser_enabled, browser_go), + cmd!("mediaBrowser.addFavorite", "Add to Favorites", [], None, "{path?}", browser_enabled, |s, p| browser_favorite(s, p, true)), + cmd!("mediaBrowser.removeFavorite", "Remove from Favorites", [], None, "{path?}", browser_enabled, |s, p| browser_favorite(s, p, false)), + cmd!("mediaBrowser.import", "Import", [], None, "{paths, addToComp?}", browser_enabled, browser_import), + query!("mediaBrowser.fileInfo", "File Info", "{path}", file_info), + query!("item.metadata", "Metadata", "{item?: id|name}", metadata), + cmd!("project.setProjectComment", "Project Comment", [], None, "{comment}", always, project_comment), + query!( + "scopes.analyze", + "Lumetri Scopes", + "{comp?, time?|frame?, scope?: waveformRgb|waveformLuma|waveformYc|vectorscopeYuv|vectorscopeHls|histogram|paradeRgb|paradeYuv, standard?: rec601|rec709|rec2020, float?, clamp?, size?}", + scopes_analyze + ), + ] +} diff --git a/crates/engine/src/commands/scene_detect.rs b/crates/engine/src/commands/scene_detect.rs new file mode 100644 index 00000000..8d523e21 --- /dev/null +++ b/crates/engine/src/commands/scene_detect.rs @@ -0,0 +1,182 @@ +//! Layer ▸ Scene Edit Detection…: find the cuts in a footage (or precomp) layer and Create +//! Markers, Split Layers, or Split and Precompose at them. The analysis +//! (`effectcraft_raster::cuts`: colour histograms + motion-compensated difference, adaptive peak +//! threshold) runs as a background job (Window ▸ Progress). + +use std::sync::Arc; + +use effectcraft_project::{ItemId, LayerId, Marker, Project}; +use effectcraft_raster::cuts::{self, CutOpts}; +use effectcraft_render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_layers, layer_p, str_p}; +use crate::jobs::Apply; +use crate::{EngineError, Result, Session, cmd}; + +/// What a background job needs to render a layer's frames. +#[derive(Clone)] +pub(crate) struct LayerFrames { + pub project: Arc, + pub footage: Arc, + pub expr: Option>, + pub cache: Arc, + pub comp: ItemId, + pub layer: LayerId, +} + +impl LayerFrames { + pub fn new(s: &Session, comp: ItemId, layer: LayerId) -> LayerFrames { + LayerFrames { project: s.project.clone(), footage: s.footage.clone(), expr: s.expr.clone(), cache: s.layer_cache.clone(), comp, layer } + } + /// Run `f` with a frame getter: comp time → (layer source pixels, offset in layer space). + pub fn with(&self, f: impl FnOnce(&(dyn Fn(Tick) -> Option<(Arc, [f64; 2])> + Sync)) -> R) -> Option { + let comp = self.project.comp(self.comp)?; + let layer = comp.layer(self.layer)?; + let mut r = Renderer::new(&self.project, self.footage.as_ref(), RenderOpts::default()); + r.expr = self.expr.as_deref(); + r.cache = Some(&self.cache); + let r = &r; + let get = |t: Tick| crate::tracking::source_frame(r, &self.project, self.comp, comp, layer, t, self.expr.as_deref()); + Some(f(&get)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Mode { + Markers, + Split, + SplitPrecompose, +} + +/// Cut times (comp time) of the layer's frames `times` (first frame never a cut). +pub(crate) fn analyse( + lf: &LayerFrames, + times: &[Tick], + o: &CutOpts, + progress: &dyn Fn(u64, u64) -> bool, +) -> std::result::Result<(Vec, Vec), String> { + lf.with(|get| { + let mut sigs = Vec::with_capacity(times.len()); + // Signatures in batches (parallel), with progress and cancel between batches. + for (k, chunk) in times.chunks(8).enumerate() { + use rayon::prelude::*; + let batch: Vec = chunk.par_iter().map(|t| get(*t).map(|(img, _)| cuts::signature(&img)).unwrap_or_default()).collect(); + sigs.extend(batch); + if !progress(((k + 1) * 8).min(times.len()) as u64, times.len() as u64) { + return Err(crate::render_queue::CANCELLED.to_string()); + } + } + let mut scores = vec![0.0f32]; + for i in 1..sigs.len() { + scores.push(cuts::dissimilarity(&sigs[i - 1], &sigs[i])); + } + let cuts = cuts::detect(&scores, o); + Ok((cuts.into_iter().map(|i| times[i]).collect(), scores)) + }) + .ok_or_else(|| "layer missing".to_string())? +} + +fn run(s: &mut Session, p: &Value) -> Result { + let c = "layer.sceneEditDetection"; + let (cid, lid) = layer_p(s, p, c)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + if layer.source.item().is_none() { + return Err(bad(c, "scene edit detection needs a footage or precomp layer")); + } + let mode = match str_p(p, "mode").unwrap_or("markers") { + "markers" | "createMarkers" => Mode::Markers, + "split" | "splitLayers" => Mode::Split, + "splitPrecompose" | "splitAndPrecompose" => Mode::SplitPrecompose, + x => return Err(bad(c, format!("mode: markers|split|splitPrecompose, not `{x}`"))), + }; + let fr = comp.frame_rate; + let (a, b) = (fr.frame_at(fr.snap_nearest(layer.in_point.max(Tick::ZERO))), fr.frame_at(layer.out_point.min(comp.duration) - Tick(1))); + let times: Vec = (a..=b).map(|f| fr.tick_of(f)).collect(); + if times.len() < 2 { + return Err(bad(c, "the layer is too short")); + } + let o = CutOpts { threshold: f_p(p, "threshold").map(|t| (t as f32).clamp(0.01, 1.0)).unwrap_or(CutOpts::default().threshold), ..Default::default() }; + let lf = LayerFrames::new(s, cid, lid); + let name = layer.name.clone(); + let wait = b_p(p, "wait").unwrap_or(false); + s.spawn_task("sceneDetect", format!("Scene Edit Detection: {name}"), wait, move |ctl| { + let (cut_times, _) = analyse(&lf, ×, &o, &|d, t| ctl.progress(d, t))?; + let apply: Apply = Box::new(move |s: &mut Session| apply(s, cid, lid, mode, &cut_times)); + Ok(apply) + }) +} + +fn apply(s: &mut Session, cid: ItemId, lid: LayerId, mode: Mode, cuts: &[Tick]) -> Result { + let secs: Vec = cuts.iter().map(|t| t.seconds()).collect(); + if cuts.is_empty() { + return Ok(json!({"cuts": secs, "layers": [lid.0]})); + } + if mode == Mode::Markers { + s.edit("Scene Edit Detection", None, |proj, _| { + let l = super::layer_mut(proj, cid, lid)?; + for (i, t) in cuts.iter().enumerate() { + let lt = l.layer_time(*t); + l.markers.retain(|m| m.time != lt); + l.markers.push(Marker { time: lt, comment: format!("Scene {}", i + 2), ..Default::default() }); + } + l.markers.sort_by_key(|m| m.time); + Ok(()) + })?; + return Ok(json!({"cuts": secs, "layers": [lid.0]})); + } + let snap = s.project.clone(); + // Split from the source layer: each piece ends at the next cut (pieces stacked like Split + // Layer: later pieces above). + let pieces = s.edit("Scene Edit Detection", None, |proj, st| { + let mut next = proj.next_id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let i = cm.layers.iter().position(|l| l.id == lid).ok_or(EngineError::NoComp)?; + let mut ids = vec![lid]; + for t in cuts { + let cur = cm.layers.iter().position(|l| l.id == *ids.last().expect("piece")).unwrap_or(i); + let l = &mut cm.layers[cur]; + if !(*t > l.in_point && *t < l.out_point) { + continue; + } + let mut b = l.clone(); + super::edit::reid(&mut b, &mut next); + b.in_point = *t; + l.out_point = *t; + ids.push(b.id); + cm.layers.insert(cur, b); + } + proj.next_id = next; + st.selected_layers = ids.clone(); + Ok(ids) + })?; + let mut out = pieces.clone(); + if mode == Mode::SplitPrecompose { + let base = s.project.comp(cid).and_then(|c| c.layer(lid)).map(|l| l.name.clone()).unwrap_or_default(); + out.clear(); + for (k, id) in pieces.iter().enumerate() { + let r = s.execute( + "layer.precompose", + json!({"comp": cid.0, "layers": [id.0], "mode": "move", "adjustDuration": true, "name": format!("{base} Scene {}", k + 1)}), + )?; + let new = r.get("layer").and_then(Value::as_u64).map(LayerId).unwrap_or(*id); + out.push(new); + } + s.collapse_undo("Scene Edit Detection", &snap); + } + Ok(json!({"cuts": secs, "layers": out.iter().map(|l| l.0).collect::>()})) +} + +pub fn specs() -> Vec { + vec![cmd!( + "layer.sceneEditDetection", + "Scene Edit Detection...", + ["Layer"], + None, + "{layer?, mode?: markers|split|splitPrecompose, threshold? (0…1, 0.25), wait?: bool}", + has_layers, + run + )] +} diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 273ecf99..440464c0 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -17,13 +17,7 @@ macro_rules! stub { pub fn specs() -> Vec { vec![ - // Layer ▸ New (3D / content-aware fill). - stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), // Mask options not in the mask model yet; pen / vertex editing (timeline milestone). - // Time (timeline milestone). - stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), - stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), - stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), // Cameras / lights / materials / 3D views (3D milestone). stub!("camera.stereoRig", "Create Stereo 3D Rig", ["Layer", "Camera"], "{}"), stub!("camera.orbitNull", "Create Orbit Null", ["Layer", "Camera"], "{}"), @@ -39,7 +33,5 @@ pub fn specs() -> Vec { // Keyframes / text / tracking. stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), - // Workspaces and panels that don't exist yet. - stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] } diff --git a/crates/engine/src/jobs.rs b/crates/engine/src/jobs.rs new file mode 100644 index 00000000..39a16915 --- /dev/null +++ b/crates/engine/src/jobs.rs @@ -0,0 +1,320 @@ +//! Background jobs: one list (Window ▸ Progress, `jobs.list`, `jobs.cancel`) over the render +//! queue, the analyses (tracker, mask tracker, Warp Stabilizer, 3D Camera Tracker, Roto Brush) and +//! the generic background tasks started here (Content-Aware Fill, Scene Edit Detection). +//! +//! A generic task runs a closure on a background thread (inline on wasm32 or when the caller +//! waits). The closure reports progress through [`TaskCtl`] and returns an [`Apply`]: a function +//! the session runs on the UI thread when [`Session::poll_jobs`] sees the task finish (that is +//! where the project is edited, as one undo step). + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tracking::lock; + +/// Applied to the session when a task finishes successfully. +pub type Apply = Box crate::Result + Send>; + +/// Progress reporting and cancellation for a running task. +#[derive(Default)] +pub struct TaskCtl { + cancel: AtomicBool, + state: Mutex, +} + +#[derive(Clone, Debug, Default)] +struct TaskState { + done: u64, + total: u64, + message: String, + finished: bool, +} + +impl TaskCtl { + /// Report progress; returns `false` once the task was cancelled. + pub fn progress(&self, done: u64, total: u64) -> bool { + { + let mut s = lock(&self.state); + s.done = done; + s.total = total; + } + crate::offload::report(done, total); + !self.cancelled() + } + pub fn message(&self, m: impl Into) { + lock(&self.state).message = m.into(); + } + pub fn cancelled(&self) -> bool { + self.cancel.load(Ordering::Relaxed) + } +} + +type Work = Box Result + Send>; + +/// A generic background task. +pub struct Task { + pub id: u64, + pub kind: &'static str, + pub label: String, + pub ctl: Arc, + started: web_time::Instant, + result: Arc>>>, + thread: Option>, +} + +/// A finished job (kept for the Progress panel and agents). +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JobRecord { + pub id: String, + pub kind: String, + pub label: String, + /// `done`, `cancelled` or `failed`. + pub status: String, + pub message: String, + pub result: Value, + pub seconds: f64, +} + +/// One row of the Progress panel. +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JobInfo { + /// `render`, `track`, `maskTrack`, `warp`, `camera`, `roto`, or `task:`. + pub id: String, + pub kind: String, + pub label: String, + pub done: u64, + pub total: u64, + /// 0…1 (`None` when the total is not known yet). + pub fraction: Option, + pub running: bool, + pub message: String, + pub cancellable: bool, +} + +fn info(id: &str, kind: &str, label: String, done: u64, total: u64, running: bool, message: String) -> JobInfo { + JobInfo { + id: id.into(), + kind: kind.into(), + label, + done, + total, + fraction: (total > 0).then(|| (done as f64 / total as f64).clamp(0.0, 1.0)), + running, + message, + cancellable: running, + } +} + +impl Session { + /// Start a background task. `wait` (and wasm32) runs it to completion and applies it before + /// returning; the result is then the apply's value. + pub fn spawn_task( + &mut self, + kind: &'static str, + label: impl Into, + wait: bool, + work: impl FnOnce(&TaskCtl) -> Result + Send + 'static, + ) -> crate::Result { + let id = self.next_task_id; + self.next_task_id += 1; + let ctl = Arc::new(TaskCtl::default()); + let result: Arc>>> = Arc::new(Mutex::new(None)); + let label = label.into(); + let wait = wait || cfg!(target_arch = "wasm32"); + let work: Work = Box::new(work); + let run = { + let (ctl, result) = (ctl.clone(), result.clone()); + move || { + let r = work(&ctl); + *lock(&result) = Some(r); + lock(&ctl.state).finished = true; + } + }; + let thread = if wait { + run(); + None + } else { + Some(std::thread::Builder::new().name(format!("task-{kind}")).spawn(run).map_err(|e| crate::EngineError::Other(e.to_string()))?) + }; + self.tasks.push(Task { id, kind, label, ctl, started: web_time::Instant::now(), result, thread }); + if wait { + let rec = self.finish_task(self.tasks.len() - 1); + return match rec.status.as_str() { + "done" => Ok(rec.result), + _ => Err(crate::EngineError::Other(if rec.message.is_empty() { rec.status } else { rec.message })), + }; + } + Ok(json!({"job": format!("task:{id}"), "started": true})) + } + + fn finish_task(&mut self, i: usize) -> JobRecord { + let mut t = self.tasks.remove(i); + if let Some(th) = t.thread.take() { + let _ = th.join(); + } + let r = lock(&t.result).take(); + let cancelled = t.ctl.cancelled(); + let (status, message, result) = match r { + Some(Ok(apply)) if !cancelled => match apply(self) { + Ok(v) => ("done", String::new(), v), + Err(e) => ("failed", e.to_string(), Value::Null), + }, + Some(Ok(_)) => ("cancelled", String::new(), Value::Null), + Some(Err(e)) if cancelled || e == crate::render_queue::CANCELLED => ("cancelled", String::new(), Value::Null), + Some(Err(e)) => ("failed", e, Value::Null), + None => ("failed", "the task stopped".into(), Value::Null), + }; + let rec = JobRecord { + id: format!("task:{}", t.id), + kind: t.kind.into(), + label: t.label.clone(), + status: status.into(), + message: message.clone(), + result, + seconds: t.started.elapsed().as_secs_f64(), + }; + let toast = match status { + "done" => format!("{} finished", t.label), + "cancelled" => format!("{} cancelled", t.label), + _ => format!("{} failed: {message}", t.label), + }; + self.events.push(crate::Event::Toast { message: toast, error: status == "failed" }); + self.job_log.push(rec.clone()); + if self.job_log.len() > 50 { + self.job_log.remove(0); + } + rec + } + + /// Apply finished tasks. Frontends call this every frame while [`Session::tasks`] is not empty. + pub fn poll_jobs(&mut self) -> bool { + let mut any = false; + while let Some(i) = self.tasks.iter().position(|t| lock(&t.ctl.state).finished) { + self.finish_task(i); + any = true; + } + any + } + + /// Wait for every running task (CLI, tests). + pub fn wait_jobs(&mut self) { + while !self.tasks.is_empty() { + if let Some(th) = self.tasks.iter_mut().find_map(|t| t.thread.take()) { + let _ = th.join(); + } + self.poll_jobs(); + if self.tasks.iter().all(|t| t.thread.is_none() && !lock(&t.ctl.state).finished) { + break; + } + } + } + + /// Every running background job. + pub fn jobs(&self) -> Vec { + let mut out = vec![]; + if let Some(st) = self.render_progress() + && !st.finished + { + let name = st + .current + .and_then(|id| self.project.render_queue.iter().find(|i| i.id == id)) + .and_then(|i| self.project.comp(i.comp).map(|_| i.comp)) + .and_then(|c| self.project.item(c)) + .map(|i| i.name.clone()); + out.push(info( + "render", + "render", + format!("Render Queue{}", name.map(|n| format!(": {n}")).unwrap_or_default()), + st.done, + st.total, + true, + format!("item {} of {}", (st.items_done + 1).min(st.items_total.max(1)), st.items_total), + )); + } + if let Some(p) = self.track_progress() + && !p.finished + { + out.push(info("track", "track", "Track Motion".into(), p.done, p.total, true, String::new())); + } + if let Some(p) = self.mask_track_progress() + && !p.finished + { + out.push(info("maskTrack", "maskTrack", "Track Mask".into(), p.done, p.total, true, String::new())); + } + if let Some(p) = self.warp_progress() + && !p.finished + { + out.push(info("warp", "warp", "Warp Stabilizer analysis".into(), p.done, p.total, true, format!("step {}", p.step))); + } + if let Some(p) = self.camera_progress() + && !p.finished + { + out.push(info("camera", "camera", "3D Camera Tracker analysis".into(), p.done, p.total, true, format!("step {}", p.step))); + } + if let Some(p) = self.roto_progress() + && !p.finished + { + out.push(info("roto", "roto", "Roto Brush propagation".into(), p.done, p.total, true, String::new())); + } + for t in &self.tasks { + let s = lock(&t.ctl.state).clone(); + out.push(info(&format!("task:{}", t.id), t.kind, t.label.clone(), s.done, s.total, !s.finished, s.message)); + } + out + } + + /// Cancel a job by id (`render`, `track`, …, `task:`; `all` cancels everything). + pub fn cancel_job(&mut self, id: &str) -> bool { + match id { + "render" => self.stop_render(), + "track" => self.stop_track(), + "maskTrack" => self.stop_mask_track(), + "warp" => self.stop_warp(), + "camera" => self.stop_camera(), + "roto" => self.stop_roto(), + "all" => { + let ids: Vec = self.jobs().into_iter().map(|j| j.id).collect(); + ids.iter().fold(false, |a, j| self.cancel_job(j) | a) + } + _ => { + let Some(n) = id.strip_prefix("task:").and_then(|n| n.parse::().ok()) else { return false }; + match self.tasks.iter().find(|t| t.id == n) { + Some(t) => { + t.ctl.cancel.store(true, Ordering::Relaxed); + true + } + None => false, + } + } + } + } + + /// Fold the undo steps recorded since the project was `snap` into one step called `name` + /// (commands built from several commands). + pub fn collapse_undo(&mut self, name: &str, snap: &Arc) { + let h = &mut self.history; + let Some(start) = h.undo.iter().rposition(|(_, p)| Arc::ptr_eq(p, snap)) else { return }; + let first = h.undo[start].1.clone(); + h.undo.truncate(start); + h.undo.push((name.to_string(), first)); + h.merge_key = None; + } + + /// Cancel and drop every task (new/open project). + pub(crate) fn drop_tasks(&mut self) { + for t in &self.tasks { + t.ctl.cancel.store(true, Ordering::Relaxed); + } + for mut t in std::mem::take(&mut self.tasks) { + if let Some(th) = t.thread.take() { + let _ = th.join(); + } + } + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 508fb72e..c8efb428 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -13,8 +13,10 @@ pub mod camera_track; pub mod commands; pub mod config; pub mod demo; +pub mod jobs; pub mod links; pub mod mask_track; +pub mod media_browser; pub mod menus; pub mod offload; pub mod prefs; @@ -43,6 +45,7 @@ pub use effectcraft_effects as effects; pub use effectcraft_geom as geom; pub use effectcraft_keyframe as keyframe; pub use effectcraft_project as project; +pub use effectcraft_raster as raster; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; @@ -216,6 +219,15 @@ pub struct EditorState { /// Essential Graphics can expose). #[serde(default)] pub essential_solo: bool, + /// The Footage panel: the footage shown, its time and In/Out marks (source time). + #[serde(default)] + pub footage_panel: Option, + /// Media Browser: the folder shown and the favourites. + #[serde(default)] + pub media_browser: media_browser::BrowserState, + /// Content-Aware Fill panel settings. + #[serde(default)] + pub content_fill: commands::content_fill::FillSettings, } fn one_view() -> u8 { @@ -322,6 +334,11 @@ pub struct Session { pub offload: Option>, /// Analyses running in the [`Session::offload`] worker. pub offloaded: Vec, + /// Generic background tasks (Content-Aware Fill, Scene Edit Detection), see [`jobs`]. + pub tasks: Vec, + /// Finished tasks, newest last (Progress panel, `jobs.list`). + pub job_log: Vec, + pub next_task_id: u64, } /// A script to run (see [`Session::script`]). @@ -378,6 +395,9 @@ impl Default for Session { script: None, offload: None, offloaded: vec![], + tasks: vec![], + job_log: vec![], + next_task_id: 1, } } } @@ -634,6 +654,7 @@ impl Session { self.stop_roto(); self.roto_job = None; self.roto_pending.clear(); + self.drop_tasks(); p.fix_next_id(); upgrade_effects(&mut p); self.project = Arc::new(p); @@ -743,4 +764,6 @@ pub fn text_families() -> Vec { #[cfg(test)] mod tests_paint; #[cfg(test)] +mod tests_panels; +#[cfg(test)] mod tests_roto; diff --git a/crates/engine/src/media_browser.rs b/crates/engine/src/media_browser.rs new file mode 100644 index 00000000..afc7dc34 --- /dev/null +++ b/crates/engine/src/media_browser.rs @@ -0,0 +1,270 @@ +//! Media Browser (Window ▸ Media Browser): browse the local file system for footage, keep +//! favourite folders, read file metadata, and import (`file.import`). Desktop only: the web build +//! has no file system to browse ([`available`] is false there). +//! +//! Also the metadata the Metadata panel shows for project items and files (codec, size, rate, +//! duration, colour profile, dates). + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use effectcraft_project::{Footage, FootageKind, Item, ItemKind, Project}; + +/// Media Browser state (in [`crate::EditorState`], serde for agents). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BrowserState { + /// The folder shown (`None` = the home folder). + #[serde(default)] + pub folder: Option, + /// Favourite folders. + #[serde(default)] + pub favorites: Vec, + /// Show only importable files. + #[serde(default)] + pub importable_only: bool, +} + +/// Whether this build can browse files. +pub fn available() -> bool { + !cfg!(target_arch = "wasm32") +} + +/// One directory entry. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Entry { + pub name: String, + pub path: String, + pub is_dir: bool, + pub size: u64, + /// Seconds since the Unix epoch. + pub modified: Option, + /// Footage kind guessed from the extension (`video`, `image`, `audio`, `data`, `model`, + /// `project`), `None` for other files and folders. + pub kind: Option<&'static str>, +} + +const VIDEO: &[&str] = &["mp4", "m4v", "mov", "webm", "mkv", "avi", "mxf", "gif"]; +const IMAGE: &[&str] = &["png", "jpg", "jpeg", "tif", "tiff", "exr", "bmp", "webp", "psd", "psb", "svg", "tga", "dpx", "hdr"]; +const AUDIO: &[&str] = &["wav", "aif", "aiff", "mp3", "m4a", "aac", "flac", "ogg", "opus"]; +const DATA: &[&str] = &["json", "csv", "tsv", "mgjson"]; +const MODEL: &[&str] = &["gltf", "glb", "obj"]; + +/// Kind of file from its extension. +pub fn kind_of(path: &str) -> Option<&'static str> { + let ext = std::path::Path::new(path).extension()?.to_str()?.to_ascii_lowercase(); + let e = ext.as_str(); + if VIDEO.contains(&e) { + Some("video") + } else if IMAGE.contains(&e) { + Some("image") + } else if AUDIO.contains(&e) { + Some("audio") + } else if DATA.contains(&e) { + Some("data") + } else if MODEL.contains(&e) { + Some("model") + } else if matches!(e, "ecproj" | "ectemplate" | "json5") { + Some("project") + } else { + None + } +} + +/// The user's home folder (or `/`). +pub fn home() -> String { + std::env::var("HOME").or_else(|_| std::env::var("USERPROFILE")).unwrap_or_else(|_| "/".into()) +} + +fn unix(t: std::io::Result) -> Option { + t.ok()?.duration_since(std::time::UNIX_EPOCH).ok().map(|d| d.as_secs()) +} + +/// List a folder: folders first, then files, by name (hidden entries skipped). +pub fn list(dir: &str, importable_only: bool) -> Result, String> { + if !available() { + return Err("the Media Browser needs the desktop app".into()); + } + let rd = std::fs::read_dir(dir).map_err(|e| format!("{dir}: {e}"))?; + let mut out: Vec = rd + .filter_map(|e| e.ok()) + .filter_map(|e| { + let name = e.file_name().to_string_lossy().to_string(); + if name.starts_with('.') { + return None; + } + let md = e.metadata().ok()?; + let path = e.path().to_string_lossy().to_string(); + let kind = if md.is_dir() { None } else { kind_of(&path) }; + if importable_only && !md.is_dir() && kind.is_none() { + return None; + } + Some(Entry { name, path, is_dir: md.is_dir(), size: if md.is_dir() { 0 } else { md.len() }, modified: unix(md.modified()), kind }) + }) + .collect(); + out.sort_by(|a, b| b.is_dir.cmp(&a.is_dir).then_with(|| a.name.to_lowercase().cmp(&b.name.to_lowercase()))); + Ok(out) +} + +/// The parent folder of `dir`. +pub fn parent(dir: &str) -> Option { + std::path::Path::new(dir).parent().map(|p| p.to_string_lossy().to_string()).filter(|p| !p.is_empty()) +} + +/// Format seconds since the epoch as an ISO 8601 UTC date-time. +pub fn iso_date(secs: u64) -> String { + let days = (secs / 86_400) as i64; + let rem = secs % 86_400; + // Civil-from-days (Howard Hinnant's algorithm). + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem % 3600 / 60, rem % 60) +} + +/// File dates and size: (created, modified, size). +pub fn file_dates(path: &str) -> (Option, Option, Option) { + if !available() || path.is_empty() { + return (None, None, None); + } + match std::fs::metadata(path) { + Ok(md) => (unix(md.created()), unix(md.modified()), Some(md.len())), + Err(_) => (None, None, None), + } +} + +fn kind_name(k: FootageKind) -> &'static str { + match k { + FootageKind::Video => "video", + FootageKind::Still => "still", + FootageKind::Sequence => "sequence", + FootageKind::Audio => "audio", + FootageKind::Model => "model", + FootageKind::Data => "data", + } +} + +/// Metadata of footage (the fields the Metadata panel lists). +pub fn footage_metadata(f: &Footage) -> Value { + let (created, modified, size) = file_dates(f.sequence.first().map(String::as_str).unwrap_or(&f.path)); + json!({ + "path": f.path, + "kind": kind_name(f.kind), + "codec": f.codec, + "width": f.width, + "height": f.height, + "pixelAspect": f.pixel_aspect, + "frameRate": f.frame_rate.as_f64(), + "duration": f.duration.seconds(), + "frames": if f.has_video && f.kind != FootageKind::Still { Some(f.frame_rate.frame_at(f.duration)) } else { None }, + "hasVideo": f.has_video, + "hasAudio": f.has_audio, + "alpha": format!("{:?}", f.alpha), + "colorProfile": f.color_profile.map(|c| format!("{c:?}")).unwrap_or_else(|| "sRGB (default)".into()), + "linearLight": f.linear_light, + "fields": format!("{:?}", f.fields), + "sequenceFiles": f.sequence.len(), + "dataRows": f.data.as_ref().map(|d| d.lines().count().saturating_sub(1)), + "created": created.map(iso_date), + "modified": modified.map(iso_date), + "fileSize": size, + "missing": f.missing, + }) +} + +/// Metadata of a project item. +pub fn item_metadata(p: &Project, it: &Item) -> Value { + let mut v = json!({ + "id": it.id.0, + "name": it.name, + "type": it.type_name(), + "comment": it.comment, + "label": format!("{:?}", it.label), + }); + let extra = match &it.kind { + ItemKind::Footage(f) => footage_metadata(f), + ItemKind::Comp(c) => json!({ + "width": c.width, + "height": c.height, + "pixelAspect": c.pixel_aspect, + "frameRate": c.frame_rate.as_f64(), + "duration": c.duration.seconds(), + "layers": c.layers.len(), + "workArea": [c.work_area.0.seconds(), c.work_area.1.seconds()], + }), + ItemKind::Solid(s) => json!({"width": s.width, "height": s.height, "color": s.color}), + ItemKind::Folder => json!({"items": p.items.values().filter(|i| i.parent == Some(it.id)).count()}), + }; + if let (Value::Object(a), Value::Object(b)) = (&mut v, extra) { + a.extend(b); + } + if let Some(px) = &it.proxy { + v["proxy"] = json!(px.footage.path); + } + v +} + +/// Project-level metadata. +pub fn project_metadata(p: &Project, path: Option<&str>) -> Value { + let st = &p.settings; + let count = |f: &dyn Fn(&ItemKind) -> bool| p.items.values().filter(|i| f(&i.kind)).count(); + json!({ + "path": path, + "comment": st.comment, + "bitDepth": st.bit_depth.label(), + "workingSpace": st.working_space.map(|c| format!("{c:?}")), + "linearize": st.linearize, + "timeDisplay": format!("{:?}", st.time_display), + "audioSampleRate": st.audio_sample_rate, + "items": p.items.len(), + "compositions": count(&|k| matches!(k, ItemKind::Comp(_))), + "footage": count(&|k| matches!(k, ItemKind::Footage(_))), + "solids": count(&|k| matches!(k, ItemKind::Solid(_))), + "folders": count(&|k| matches!(k, ItemKind::Folder)), + "renderQueue": p.render_queue.len(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lists_a_folder() { + let dir = std::env::temp_dir().join(format!("ec-mb-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("Shots")).unwrap(); + std::fs::write(dir.join("b.png"), b"x").unwrap(); + std::fs::write(dir.join("A.mov"), b"xy").unwrap(); + std::fs::write(dir.join("notes.txt"), b"xyz").unwrap(); + std::fs::write(dir.join(".hidden"), b"").unwrap(); + let d = dir.to_string_lossy().to_string(); + let all = list(&d, false).unwrap(); + let names: Vec<&str> = all.iter().map(|e| e.name.as_str()).collect(); + assert_eq!(names, ["Shots", "A.mov", "b.png", "notes.txt"]); + assert!(all[0].is_dir); + assert_eq!(all[1].kind, Some("video")); + assert_eq!(all[2].kind, Some("image")); + assert_eq!(all[1].size, 2); + let media = list(&d, true).unwrap(); + assert_eq!(media.len(), 3); + assert_eq!(parent(&dir.join("Shots").to_string_lossy()).as_deref(), Some(d.as_str())); + assert!(list(&dir.join("nope").to_string_lossy(), false).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn iso_dates() { + assert_eq!(iso_date(0), "1970-01-01T00:00:00Z"); + assert_eq!(iso_date(1_700_000_000), "2023-11-14T22:13:20Z"); + assert_eq!(iso_date(951_782_400), "2000-02-29T00:00:00Z"); + } +} diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 0470625c..2ae54aee 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -942,19 +942,19 @@ Window Audio | window.panel {"panel":"audio"} | Cmd+4 Brushes | window.panel {"panel":"brushes"} | Cmd+9 Character | window.panel {"panel":"character"} | Cmd+6 - Content-Aware Fill | window.unavailablePanel {"panel":"contentAwareFill"} + Content-Aware Fill | window.panel {"panel":"contentAwareFill"} Effects & Presets | window.panel {"panel":"effectsPresets"} | Cmd+5 Essential Graphics | window.panel {"panel":"essentialGraphics"} Info | window.panel {"panel":"info"} | Cmd+2 - Lumetri Scopes | window.unavailablePanel {"panel":"lumetriScopes"} + Lumetri Scopes | window.panel {"panel":"lumetriScopes"} Mask Interpolation | window.panel {"panel":"maskInterpolation"} - Media Browser | window.unavailablePanel {"panel":"mediaBrowser"} - Metadata | window.unavailablePanel {"panel":"metadata"} + Media Browser | window.panel {"panel":"mediaBrowser"} + Metadata | window.panel {"panel":"metadata"} Motion Sketch | window.panel {"panel":"motionSketch"} Paint | window.panel {"panel":"paint"} | Cmd+8 Paragraph | window.panel {"panel":"paragraph"} | Cmd+7 Preview | window.panel {"panel":"preview"} | Cmd+3 - Progress | window.unavailablePanel {"panel":"progress"} + Progress | window.panel {"panel":"progress"} Properties | window.panel {"panel":"properties"} Script Console | window.panel {"panel":"scriptConsole"} Smoother | window.panel {"panel":"smoother"} @@ -965,7 +965,7 @@ Window Composition | window.panel {"panel":"composition"} Effect Controls | window.panel {"panel":"effectControls"} | F3 Flowchart | window.panel {"panel":"flowchart"} - Footage | window.unavailablePanel {"panel":"footage"} + Footage | window.panel {"panel":"footage"} Layer | window.panel {"panel":"layer"} Project | window.panel {"panel":"project"} | Cmd+0 Render Queue | window.panel {"panel":"renderQueue"} | Cmd+Alt+0 diff --git a/crates/engine/src/tests_panels.rs b/crates/engine/src/tests_panels.rs new file mode 100644 index 00000000..d989b28e --- /dev/null +++ b/crates/engine/src/tests_panels.rs @@ -0,0 +1,518 @@ +//! M13.6 panels and tools through the commands: Progress (jobs), Media Browser, Metadata, +//! Lumetri Scopes, the Footage panel's Overlay / Ripple Insert edits, Content-Aware Fill, Scene +//! Edit Detection, Auto-trace and Align Video to Data. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use effectcraft_color::Label; +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource, build}; +use effectcraft_raster::Image; +use effectcraft_raster::inpaint::psnr; +use effectcraft_render::FootageSource; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::{Value, json}; + +use crate::Session; + +const FPS: u32 = 25; + +/// Footage computed per frame. +struct Gen { + make: Box Image + Send + Sync>, + cache: Mutex>>, +} + +impl FootageSource for Gen { + fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option> { + let f = (t.seconds() * FPS as f64).round().max(0.0) as u32; + let mut c = self.cache.lock().unwrap(); + Some(c.entry(f).or_insert_with(|| Arc::new((self.make)(f))).clone()) + } +} + +/// A comp `w × h`, `secs` long, with one footage layer drawn by `make` (returns session, comp, +/// layer, footage item). +fn setup(w: u32, h: u32, secs: f64, make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, ItemId, LayerId, ItemId) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Panels", "width": w, "height": h, "frameRate": FPS, "duration": secs})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let (lid, item) = s + .edit("clip", None, |p, _| { + let footage = Footage { + path: "synthetic.mov".into(), + kind: FootageKind::Video, + width: w, + height: h, + frame_rate: FrameRate::new(FPS as i64, 1), + duration: Tick::from_seconds_f64(secs), + has_video: true, + codec: "SYN".into(), + ..Default::default() + }; + let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); + let comp = p.comp(cid).unwrap().clone(); + let l = build::layer(p, &comp, "Clip", LayerSource::Footage { item }, (w, h), None); + let id = l.id; + p.comp_mut(cid).unwrap().layers.insert(0, l); + Ok((id, item)) + }) + .unwrap(); + s.footage = Arc::new(Gen { make: Box::new(make), cache: Mutex::default() }); + s.execute("layer.select", json!({"layers": [lid.0]})).unwrap(); + (s, cid, lid, item) +} + +fn tmp(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("ec-panels-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +// ---------------------------------------------------------------- Progress + +#[test] +fn progress_lists_and_cancels_jobs() { + let mut s = Session::default(); + let r = s + .spawn_task("test", "Endless job", false, |ctl| { + let mut k = 0; + while ctl.progress(k % 100, 100) { + std::thread::sleep(std::time::Duration::from_millis(2)); + k += 1; + } + Err(crate::render_queue::CANCELLED.into()) + }) + .unwrap(); + let id = r["job"].as_str().unwrap().to_string(); + let listed = s.execute("jobs.list", json!({})).unwrap(); + let running = listed["running"].as_array().unwrap(); + assert_eq!(running.len(), 1); + assert_eq!(running[0]["id"], json!(id)); + assert_eq!(running[0]["label"], json!("Endless job")); + assert_eq!(running[0]["cancellable"], json!(true)); + assert_eq!(s.execute("jobs.cancel", json!({"job": id})).unwrap()["cancelled"], json!(true)); + s.execute("jobs.wait", json!({})).unwrap(); + assert!(s.jobs().is_empty()); + assert_eq!(s.job_log.last().unwrap().status, "cancelled"); + // A finished job applies its edit (one undo step) and reports its result. + let v = s + .spawn_task("test", "Quick job", true, |ctl| { + ctl.progress(1, 1); + let apply: crate::jobs::Apply = Box::new(|s: &mut Session| { + s.edit("Quick", None, |p, _| { + p.settings.comment = "done".into(); + Ok(()) + })?; + Ok(json!(42)) + }); + Ok(apply) + }) + .unwrap(); + assert_eq!(v, json!(42)); + assert_eq!(s.project.settings.comment, "done"); + assert_eq!(s.job_log.last().unwrap().status, "done"); + assert!(s.undo()); + assert_eq!(s.project.settings.comment, ""); +} + +// ---------------------------------------------------------------- Media Browser & Metadata + +#[test] +fn media_browser_and_metadata() { + let dir = tmp("browser"); + std::fs::create_dir_all(dir.join("Footage")).unwrap(); + std::fs::write(dir.join("shot.mov"), b"0123").unwrap(); + std::fs::write(dir.join("data.csv"), "time,v\n0,1\n").unwrap(); + std::fs::write(dir.join("readme.txt"), "x").unwrap(); + let d = dir.to_string_lossy().to_string(); + let mut s = Session::default(); + let r = s.execute("mediaBrowser.go", json!({"path": d})).unwrap(); + let names: Vec<&str> = r["entries"].as_array().unwrap().iter().map(|e| e["name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Footage", "data.csv", "readme.txt", "shot.mov"]); + assert_eq!(s.state.media_browser.folder.as_deref(), Some(d.as_str())); + let r = s.execute("mediaBrowser.list", json!({"importableOnly": true})).unwrap(); + assert_eq!(r["entries"].as_array().unwrap().len(), 3); + s.execute("mediaBrowser.addFavorite", json!({})).unwrap(); + assert_eq!(s.state.media_browser.favorites, vec![d.clone()]); + s.execute("mediaBrowser.go", json!({"path": dir.join("Footage").to_string_lossy()})).unwrap(); + s.execute("mediaBrowser.go", json!({"path": ".."})).unwrap(); + assert_eq!(s.state.media_browser.folder.as_deref(), Some(d.as_str())); + s.execute("mediaBrowser.removeFavorite", json!({"path": d})).unwrap(); + assert!(s.state.media_browser.favorites.is_empty()); + // Import a data file from the browser; its metadata reads back. + let r = s.execute("mediaBrowser.import", json!({"paths": [dir.join("data.csv").to_string_lossy()]})).unwrap(); + let id = r["items"][0].as_u64().unwrap(); + let m = s.execute("item.metadata", json!({"item": id})).unwrap(); + assert_eq!(m["item"]["type"], json!("Data")); + assert_eq!(m["item"]["codec"], json!("CSV")); + assert_eq!(m["item"]["dataRows"], json!(1)); + assert!(m["item"]["modified"].as_str().unwrap().ends_with('Z')); + assert_eq!(m["project"]["footage"], json!(1)); + // Editable comments: item (undoable) and project. + s.execute("project.setComment", json!({"items": [id], "comment": "telemetry"})).unwrap(); + s.execute("project.setProjectComment", json!({"comment": "client cut"})).unwrap(); + let m = s.execute("item.metadata", json!({"item": id})).unwrap(); + assert_eq!(m["item"]["comment"], json!("telemetry")); + assert_eq!(m["project"]["comment"], json!("client cut")); + s.undo(); + assert_eq!(s.project.settings.comment, ""); + let fi = s.execute("mediaBrowser.fileInfo", json!({"path": dir.join("shot.mov").to_string_lossy()})).unwrap(); + assert_eq!(fi["size"], json!(4)); + assert_eq!(fi["kind"], json!("video")); + std::fs::remove_dir_all(&dir).unwrap(); +} + +// ---------------------------------------------------------------- Lumetri Scopes + +#[test] +fn scopes_read_the_comp_frame() { + let mut s = Session::default(); + s.execute("comp.new", json!({"width": 64, "height": 48, "frameRate": 25, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap(); + let v = s.execute("scopes.analyze", json!({"scope": "vectorscopeYuv", "size": 101})).unwrap(); + let st = effectcraft_raster::scopes::ColorStandard::Rec709; + let [_, cb, cr] = st.ycbcr([1.0, 0.0, 0.0]); + assert_eq!(v["peaks"][0], json!([((cb + 0.5) * 100.0).round() as u32, ((0.5 - cr) * 100.0).round() as u32])); + let h = s.execute("scopes.analyze", json!({"scope": "histogram", "standard": "rec601"})).unwrap(); + let bars = &h["histogram"]; + assert_eq!(bars[0][255], json!(64 * 48)); + assert_eq!(bars[1][0], json!(64 * 48)); + assert!((h["luma"]["mean"].as_f64().unwrap() - 0.299).abs() < 1e-3); + assert!(s.execute("scopes.analyze", json!({"scope": "bogus"})).is_err()); +} + +// ---------------------------------------------------------------- Footage panel + +#[test] +fn footage_panel_overlay_and_ripple_insert() { + let (mut s, cid, clip, item) = setup(32, 24, 8.0, |_| Image::filled(32, 24, [0.5, 0.5, 0.5, 1.0])); + s.execute("footage.open", json!({"item": item.0})).unwrap(); + assert!( + s.drain_events().iter().any(|e| matches!(e, crate::Event::Frontend { command, params } if command == "window.panel" && params["panel"] == "footage")) + ); + s.execute("footage.setTime", json!({"time": 1.0})).unwrap(); + s.execute("footage.setIn", json!({})).unwrap(); + s.execute("footage.setOut", json!({"frame": 49})).unwrap(); + let info = s.execute("footage.info", json!({})).unwrap(); + assert_eq!(info["in"], json!(1.0)); + assert_eq!(info["out"], json!(2.0)); + // Overlay Edit at 0.6 s: a 1 s layer on top showing source 1–2 s. + s.set_time(Tick::from_seconds_f64(0.6)); + let r = s.execute("footage.overlayEdit", json!({})).unwrap(); + let ov = LayerId(r["layer"].as_u64().unwrap()); + let comp = s.project.comp(cid).unwrap(); + assert_eq!(comp.layers[0].id, ov); + let l = comp.layer(ov).unwrap(); + assert_eq!((l.in_point.seconds(), l.out_point.seconds(), l.start_time.seconds()), (0.6, 1.6, -0.4)); + assert_eq!(comp.layer(clip).unwrap().out_point.seconds(), 8.0, "overlay leaves other layers alone"); + // Ripple Insert at 3 s: layers crossing 3 s are split, the rest moves back by 1 s. + s.set_time(Tick::from_seconds_f64(3.0)); + let r = s.execute("footage.rippleInsertEdit", json!({})).unwrap(); + let ins = LayerId(r["layer"].as_u64().unwrap()); + let comp = s.project.comp(cid).unwrap(); + assert_eq!(comp.layers.len(), 4, "overlay, insert, clip split in two"); + let l = comp.layer(ins).unwrap(); + assert_eq!((l.in_point.seconds(), l.out_point.seconds()), (3.0, 4.0)); + let head = comp.layer(clip).unwrap(); + assert_eq!((head.in_point.seconds(), head.out_point.seconds()), (0.0, 3.0)); + let tail = comp.layers.iter().find(|x| x.id != clip && x.id != ins && x.id != ov).unwrap(); + assert_eq!((tail.in_point.seconds(), tail.out_point.seconds(), tail.start_time.seconds()), (4.0, 9.0, 1.0)); + // Source time continuity: the tail shows source 3 s at comp 4 s. + assert_eq!(tail.layer_time(Tick::from_seconds_f64(4.0)).seconds(), 3.0); + // The overlay layer (in before 3 s, out before 3 s) didn't move. + assert_eq!(comp.layer(ov).unwrap().in_point.seconds(), 0.6); + // One undo step each. + assert!(s.undo()); + assert_eq!(s.project.comp(cid).unwrap().layers.len(), 2); + assert!(s.execute("footage.setOut", json!({"time": 0.5})).is_ok()); + assert_eq!(s.state.footage_panel.as_ref().unwrap().in_point, None, "an Out before the In clears the In"); +} + +// ---------------------------------------------------------------- Content-Aware Fill + +fn bg(x: f32, y: f32) -> [f32; 4] { + let s = 0.5 + 0.3 * (x * std::f32::consts::TAU / 10.0).sin(); + let n = effectcraft_raster::hash_noise(((x.floor() as i64).rem_euclid(20) / 4) as u32, ((y as u32) % 20) / 4, 3); + [s, 0.3 + 0.4 * n, 0.6 - 0.3 * s * n, 1.0] +} + +/// Panning textured background with a red "object" walking through the masked region. +fn caf_frame(f: u32, with_object: bool) -> Image { + let (w, h) = (96u32, 64u32); + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let mut p = bg(x as f32 + 2.0 * f as f32, y as f32); + let ox = 40 + (f % 4); + if with_object && (ox..ox + 10).contains(&x) && (26..36).contains(&y) { + p = [0.9, 0.1, 0.1, 1.0]; + } + img.set(x, y, p); + } + } + img +} + +#[test] +fn content_aware_fill_removes_an_object() { + let (mut s, cid, clip, _) = setup(96, 64, 0.48, |f| caf_frame(f, true)); + // Cut the object out with a Subtract mask; fill the work area (12 frames). + s.execute("layer.addMask", json!({"rect": [36.0, 22.0, 20.0, 18.0], "mode": "subtract"})).unwrap(); + let out = tmp("caf"); + s.execute("contentFill.set", json!({"method": "object", "range": "workArea", "alphaExpansion": 1})).unwrap(); + let r = s.execute("contentFill.generate", json!({"outputDir": out.to_string_lossy(), "wait": true})).unwrap(); + let fill = LayerId(r["layer"].as_u64().unwrap()); + let files: Vec = r["files"].as_array().unwrap().iter().map(|v| v.as_str().unwrap().to_string()).collect(); + assert_eq!(files.len(), 12); + // The Fill layer sits right above the source with the source's timing. + let comp = s.project.comp(cid).unwrap(); + let i = comp.layers.iter().position(|l| l.id == fill).unwrap(); + assert_eq!(comp.layers[i + 1].id, clip); + let fl = comp.layer(fill).unwrap(); + assert_eq!((fl.in_point, fl.out_point.seconds()), (Tick::ZERO, 0.48)); + let LayerSource::Footage { item } = fl.source else { panic!("footage") }; + let Some(ItemKind::Footage(f)) = s.project.item(item).map(|i| &i.kind) else { panic!() }; + assert_eq!((f.kind, f.sequence.len()), (FootageKind::Sequence, 12)); + assert!(s.project.item(item).unwrap().name.starts_with("Fill 1")); + // The filled frames match the clean background inside the mask, and stay steady over time. + let region: Vec = (0..96 * 64).map(|i| (36..56).contains(&(i % 96)) && (22..40).contains(&(i / 96))).collect(); + let mut prev: Option = None; + for (k, path) in files.iter().enumerate() { + let png = image::open(path).unwrap().to_rgba8(); + let img = Image::from_rgba8(96, 64, png.as_raw()); + let truth = caf_frame(k as u32, false); + let db = psnr(&img, &truth, Some(®ion)); + assert!(db > 25.0, "frame {k}: {db:.1} dB"); + // Outside the hole the frame is the source itself. + let outside: Vec = region.iter().map(|r| !r).collect(); + assert!(psnr(&img, &caf_frame(k as u32, true), Some(&outside)) > 40.0); + if let Some(p) = &prev { + // Temporal consistency: the change in the hole follows the 2 px/frame pan (compare + // against the previous fill shifted). + let mut shifted = p.clone(); + for y in 0..64 { + for x in 0..94 { + shifted.data[y * 96 + x] = p.data[y * 96 + x + 2]; + } + } + let inner: Vec = (0..96 * 64).map(|i| (38..54).contains(&(i % 96)) && (24..38).contains(&(i / 96))).collect(); + assert!(psnr(&img, &shifted, Some(&inner)) > 24.0, "frame {k} jumps"); + } + prev = Some(img); + } + // One undo step removes the fill layer. + assert!(s.undo()); + assert!(s.project.comp(cid).unwrap().layer(fill).is_none()); + std::fs::remove_dir_all(&out).unwrap(); + // Without a hole there is nothing to fill. + let (mut s2, _, _, _) = setup(32, 24, 0.2, |_| Image::filled(32, 24, [0.5, 0.5, 0.5, 1.0])); + assert!(s2.execute("contentFill.generate", json!({"wait": true, "outputDir": tmp("caf2").to_string_lossy()})).is_err()); +} + +#[test] +fn content_aware_fill_runs_in_the_background() { + let (mut s, _, _, _) = setup(48, 32, 0.2, |f| { + let mut img = Image::new(48, 32); + for y in 0..32 { + for x in 0..48 { + img.set(x, y, bg(x as f32 + f as f32, y as f32)); + } + } + img + }); + s.execute("layer.addMask", json!({"rect": [20.0, 10.0, 8.0, 8.0], "mode": "subtract"})).unwrap(); + let out = tmp("cafbg"); + let r = s.execute("contentFill.generate", json!({"method": "edgeBlend", "outputDir": out.to_string_lossy()})).unwrap(); + assert!(r["job"].as_str().unwrap().starts_with("task:")); + assert_eq!(s.jobs().len(), 1); + assert_eq!(s.jobs()[0].kind, "contentFill"); + s.wait_jobs(); + let rec = s.job_log.last().unwrap(); + assert_eq!(rec.status, "done", "{}", rec.message); + assert_eq!(rec.result["frames"], json!(5)); + std::fs::remove_dir_all(&out).unwrap(); +} + +// ---------------------------------------------------------------- Scene Edit Detection + +fn shot_frame(f: u32) -> Image { + // Shots: frames 0–9, 10–17, 18–24. + let (kind, k) = if f < 10 { + (0, f) + } else if f < 18 { + (1, f - 10) + } else { + (2, f - 18) + }; + let mut img = Image::new(80, 48); + for y in 0..48 { + for x in 0..80 { + let fx = x as f32 + 2.0 * k as f32; + let v = match kind { + 0 => 0.5 + 0.4 * (fx * 0.15).sin() * (y as f32 * 0.11).cos(), + 1 => { + if ((fx / 10.0).floor() as i32 + y as i32 / 10) % 2 == 0 { + 0.85 + } else { + 0.15 + } + } + _ => effectcraft_raster::hash_noise((fx.floor() as u32) / 4, y / 4, 5), + }; + let c = match kind { + 0 => [v, v * 0.6, 0.2], + 1 => [0.1, v * 0.8, v], + _ => [v * 0.9, v * 0.9, 0.5 + v * 0.4], + }; + img.set(x, y, [c[0], c[1], c[2], 1.0]); + } + } + img +} + +#[test] +fn scene_edit_detection_finds_cuts() { + let (mut s, cid, clip, _) = setup(80, 48, 1.0, shot_frame); + // Markers. + let r = s.execute("layer.sceneEditDetection", json!({"mode": "markers", "wait": true})).unwrap(); + assert_eq!(r["cuts"], json!([0.4, 0.72])); + let l = s.project.comp(cid).unwrap().layer(clip).unwrap(); + assert_eq!(l.markers.iter().map(|m| m.time.seconds()).collect::>(), vec![0.4, 0.72]); + s.undo(); + // Split Layers. + let r = s.execute("layer.sceneEditDetection", json!({"mode": "split", "wait": true})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect(); + assert_eq!(ids.len(), 3); + let comp = s.project.comp(cid).unwrap(); + let spans: Vec<(f64, f64)> = ids.iter().map(|i| comp.layer(LayerId(*i)).unwrap()).map(|l| (l.in_point.seconds(), l.out_point.seconds())).collect(); + assert_eq!(spans, vec![(0.0, 0.4), (0.4, 0.72), (0.72, 1.0)]); + s.undo(); + assert_eq!(s.project.comp(cid).unwrap().layers.len(), 1); + // Split and Precompose: three precomp layers, one undo step. + let r = s.execute("layer.sceneEditDetection", json!({"mode": "splitPrecompose", "wait": true})).unwrap(); + let comp = s.project.comp(cid).unwrap(); + assert_eq!(comp.layers.len(), 3); + for v in r["layers"].as_array().unwrap() { + let l = comp.layer(LayerId(v.as_u64().unwrap())).unwrap(); + assert!(matches!(l.source, LayerSource::Comp { .. })); + } + assert_eq!(s.history.undo.last().unwrap().0, "Scene Edit Detection"); + s.undo(); + assert_eq!(s.project.comp(cid).unwrap().layers.len(), 1); +} + +// ---------------------------------------------------------------- Auto-trace + +fn disc_frame(f: u32) -> Image { + let mut img = Image::new(80, 60); + let (cx, cy) = (30.0 + 2.0 * f as f64, 30.0); + for y in 0..60 { + for x in 0..80 { + let d = ((x as f64 + 0.5 - cx).powi(2) + (y as f64 + 0.5 - cy).powi(2)).sqrt(); + // Anti-aliased disc of radius 15 plus a 12 × 12 square, on transparency. + let a = (15.5 - d).clamp(0.0, 1.0) as f32; + let sq = (60..72).contains(&x) && (8..20).contains(&y); + let a = if sq { 1.0 } else { a }; + img.set(x, y, [a, a, a, a]); + } + } + img +} + +fn iou_with(path_set: &[effectcraft_keyframe::ShapePath], truth: &Image) -> f64 { + let paths: Vec<_> = path_set.iter().map(effectcraft_path::to_kurbo).collect(); + let cov = effectcraft_path::fill_coverage(&paths, &effectcraft_geom::Mat3::IDENTITY, truth.width, truth.height, effectcraft_path::FillRule::EvenOdd); + let (mut i, mut u) = (0.0, 0.0); + for (c, p) in cov.data.iter().zip(&truth.data) { + let (a, b) = (*c >= 0.5, p[3] >= 0.5); + i += (a && b) as u8 as f64; + u += (a || b) as u8 as f64; + } + i / u +} + +fn mask_paths(s: &Session, cid: ItemId, lid: LayerId, lt: Tick) -> Vec { + let l = s.project.comp(cid).unwrap().layer(lid).unwrap(); + l.masks() + .unwrap() + .groups() + .filter(|g| g.get("opacity").unwrap().value_at(lt).as_f64() > 0.0) + .map(|g| match g.get("path").unwrap().value_at(lt) { + effectcraft_project::Value::Path(p) => p, + _ => panic!("path"), + }) + .collect() +} + +#[test] +fn auto_trace_matches_shapes() { + let (mut s, cid, clip, _) = setup(80, 60, 0.4, disc_frame); + let r = s.execute("layer.autoTrace", json!({"layer": clip.0, "channel": "alpha", "tolerance": 0.5, "cornerRoundness": 30})).unwrap(); + assert_eq!(r["masks"], json!(2)); + let iou = iou_with(&mask_paths(&s, cid, clip, Tick::ZERO), &disc_frame(0)); + assert!(iou > 0.95, "IoU {iou}"); + s.undo(); + // Work area onto a new layer: one key per frame following the moving disc. + let r = s.execute("layer.autoTrace", json!({"layer": clip.0, "timeSpan": "workArea", "applyToNewLayer": true, "minimumArea": 20})).unwrap(); + let nl = LayerId(r["layer"].as_u64().unwrap()); + assert_ne!(nl, clip); + assert_eq!(r["frames"], json!(10)); + for f in [0u32, 5, 9] { + let t = FrameRate::new(FPS as i64, 1).tick_of(f as i64); + let iou = iou_with(&mask_paths(&s, cid, nl, t), &disc_frame(f)); + assert!(iou > 0.93, "frame {f}: IoU {iou}"); + } + // Luminance with Invert traces the background instead (one outline with holes). + s.undo(); + let r = s.execute("layer.autoTrace", json!({"layer": clip.0, "channel": "luminance", "invert": true, "threshold": 50})).unwrap(); + assert!(r["masks"].as_u64().unwrap() >= 1); +} + +// ---------------------------------------------------------------- Align Video to Data + +#[test] +fn align_video_to_data() { + let (mut s, cid, clip, _) = setup(16, 16, 20.0, |_| Image::filled(16, 16, [0.2, 0.2, 0.2, 1.0])); + let add_data = |s: &mut Session, name: &str, ext: &str, text: &str| -> u64 { + s.edit("data", None, |p, _| { + let f = + Footage { path: format!("{name}.{ext}"), kind: FootageKind::Data, codec: ext.to_uppercase(), data: Some(text.into()), ..Default::default() }; + Ok(p.add_item(name, Label::Sandstone, None, ItemKind::Footage(f)).0) + }) + .unwrap() + }; + let gps = add_data(&mut s, "gps", "csv", "lat,lon,timestamp\n1,2,2026-05-01T10:00:00Z\n1,2,2026-05-01T10:00:01Z\n"); + // The video started 5 s after the first sample: it moves 5 s later. + let r = s.execute("layer.alignVideoToData", json!({"data": gps, "videoStart": "2026-05-01T10:00:05Z"})).unwrap(); + assert_eq!(r["startTime"], json!(5.0)); + let l = s.project.comp(cid).unwrap().layer(clip).unwrap(); + assert_eq!((l.start_time.seconds(), l.in_point.seconds()), (5.0, 5.0)); + s.undo(); + // Data placed at 2 s, timestamps as times of day, video start as timecode. + let log = add_data(&mut s, "log", "json", r#"{"samples": [{"t": "09:59:58.0", "v": 1}, {"t": "09:59:59.0", "v": 2}]}"#); + let r = s.execute("layer.alignVideoToData", json!({"data": "log", "videoStart": "10:00:00:00", "dataStart": 2.0})).unwrap(); + assert_eq!(r["key"], json!("t")); + assert_eq!(r["startTime"], json!(4.0)); + // A video that started before the data moves earlier (negative start). + let r = s.execute("layer.alignVideoToData", json!({"data": log, "videoStart": "2026-05-01T09:59:57Z"})).unwrap(); + assert_eq!(r["startTime"], json!(-1.0)); + // Errors: unknown key, no data. + assert!(s.execute("layer.alignVideoToData", json!({"data": gps, "key": "nope", "videoStart": 0})).is_err()); + let _: Value = Value::Null; +} + +#[test] +fn panels_commands_are_wired() { + // The Window menu opens real panels now, and the former stubs are enabled. + let menus = crate::menus::entries(); + for panel in ["lumetriScopes", "footage", "mediaBrowser", "metadata", "progress", "contentAwareFill"] { + assert!(menus.iter().any(|(_, e)| e.command == "window.panel" && e.params["panel"] == panel), "{panel}"); + } + assert!(crate::find_command("window.unavailablePanel").is_none()); + let (s, _, _, _) = setup(8, 8, 1.0, |_| Image::filled(8, 8, [0.0; 4])); + for id in ["layer.autoTrace", "layer.sceneEditDetection", "layer.alignVideoToData", "layer.newContentAwareFill", "contentFill.generate"] { + assert!(s.is_enabled(id), "{id}"); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index e9a641e8..824bd929 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -118,6 +118,9 @@ pub struct ProjectSettings { /// when a GPU adapter exists) or Mercury Software Only (`false`, the CPU compositor). #[serde(default = "yes")] pub gpu_acceleration: bool, + /// The project's comment (Metadata panel). + #[serde(default, skip_serializing_if = "String::is_empty")] + pub comment: String, } impl Default for ProjectSettings { @@ -131,6 +134,7 @@ impl Default for ProjectSettings { frame_start: 0, audio_sample_rate: 48_000, gpu_acceleration: true, + comment: String::new(), } } } diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 91d7c335..298f6652 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -36,10 +36,16 @@ pub enum PanelKind { MaskInterpolation, ScriptConsole, EssentialGraphics, + LumetriScopes, + Footage, + MediaBrowser, + Metadata, + Progress, + ContentAwareFill, } impl PanelKind { - pub const ALL: [PanelKind; 26] = [ + pub const ALL: [PanelKind; 32] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -66,6 +72,12 @@ impl PanelKind { PanelKind::MaskInterpolation, PanelKind::ScriptConsole, PanelKind::EssentialGraphics, + PanelKind::LumetriScopes, + PanelKind::Footage, + PanelKind::MediaBrowser, + PanelKind::Metadata, + PanelKind::Progress, + PanelKind::ContentAwareFill, ]; pub fn title(self) -> &'static str { match self { @@ -95,6 +107,12 @@ impl PanelKind { PanelKind::MaskInterpolation => "Mask Interpolation", PanelKind::ScriptConsole => "Script Console", PanelKind::EssentialGraphics => "Essential Graphics", + PanelKind::LumetriScopes => "Lumetri Scopes", + PanelKind::Footage => "Footage", + PanelKind::MediaBrowser => "Media Browser", + PanelKind::Metadata => "Metadata", + PanelKind::Progress => "Progress", + PanelKind::ContentAwareFill => "Content-Aware Fill", } } pub fn id(self) -> String { @@ -102,7 +120,7 @@ impl PanelKind { } pub fn from_name(s: &str) -> Option { let n = s.to_ascii_lowercase().replace([' ', '_', '-', '&'], ""); - Self::ALL.iter().copied().find(|p| format!("{p:?}").to_ascii_lowercase() == n || p.title().to_ascii_lowercase().replace([' ', '&'], "") == n) + Self::ALL.iter().copied().find(|p| format!("{p:?}").to_ascii_lowercase() == n || p.title().to_ascii_lowercase().replace([' ', '&', '-'], "") == n) } /// Window-menu shortcut. pub fn window_shortcut(self) -> Option<&'static str> { @@ -295,7 +313,7 @@ pub fn workspace(name: &str) -> DockNode { hsplit( FixedA(360.0), tabs(&[EffectControls, Project], 0), - hsplit(FixedB(300.0), tabs(&[Composition, Layer], 0), right(&[Info, Preview], &[EffectsPresets])), + hsplit(FixedB(340.0), tabs(&[Composition, Layer], 0), right(&[LumetriScopes, Info, Preview], &[EffectsPresets])), ), tabs(&[Timeline], 0), ), @@ -315,17 +333,17 @@ pub fn workspace(name: &str) -> DockNode { Ratio(0.55), hsplit( FixedA(300.0), - tabs(&[Project, EffectControls, Flowchart, History], 0), + tabs(&[Project, EffectControls, Flowchart, History, MediaBrowser, Metadata], 0), hsplit( FixedB(300.0), - tabs(&[Composition, Layer], 0), + tabs(&[Composition, Layer, Footage], 0), right( - &[Info, Preview, Audio, Align, Character, Paragraph], - &[EffectsPresets, Properties, Tracker, Wiggler, Smoother, MotionSketch, Paint, Brushes, Markers], + &[Info, Preview, Audio, Align, Character, Paragraph, LumetriScopes], + &[EffectsPresets, Properties, Tracker, Wiggler, Smoother, MotionSketch, Paint, Brushes, Markers, ContentAwareFill], ), ), ), - tabs(&[Timeline, RenderQueue], 0), + tabs(&[Timeline, RenderQueue, Progress], 0), ), "Standard" => vsplit( Ratio(0.56), diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 912f2e68..9df1d6ee 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -787,6 +787,11 @@ impl EffectcraftApp { self.session.poll_mask_track(); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } + // Background tasks (Content-Aware Fill, Scene Edit Detection). + if !self.session.tasks.is_empty() { + self.session.poll_jobs(); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // Analyses running in a worker (the web app). if !self.session.offloaded.is_empty() { self.session.poll_offload(); diff --git a/crates/ui-egui/src/panels/content_fill_panel.rs b/crates/ui-egui/src/panels/content_fill_panel.rs new file mode 100644 index 00000000..5d534e59 --- /dev/null +++ b/crates/ui-egui/src/panels/content_fill_panel.rs @@ -0,0 +1,138 @@ +//! Content-Aware Fill panel (Window ▸ Content-Aware Fill): Fill Target, Alpha Expansion, Fill +//! Method (Object / Surface / Edge Blend), Lighting Correction, Range (Work Area / Entire +//! Duration), a reference frame from a layer of the comp, and Generate Fill Layer. The settings +//! live in `EditorState::content_fill` (`contentFill.set`); the button runs +//! `contentFill.generate` as a background job (Window ▸ Progress). + +use egui::{Align2, Rect, pos2, vec2}; +use serde_json::json; + +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +const ROW: f32 = 28.0; + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + let x0 = rect.min.x + 12.0; + let xv = x0 + 130.0; + let w = (rect.max.x - xv - 12.0).clamp(90.0, 220.0); + let mut y = rect.min.y + 10.0; + let st = app.session.state.content_fill.clone(); + let comp = app.session.active_comp().cloned(); + let layer = comp.as_ref().and_then(|c| app.session.state.selected_layers.first().and_then(|l| c.layer(*l)).cloned()); + // Fill Target. + p.text(pos2(x0, y + 9.0), Align2::LEFT_CENTER, "Fill Target", Tokens::semibold(12.0), t.text); + y += ROW - 6.0; + let target = match &layer { + Some(l) => { + let masks = l + .masks() + .map(|m| { + m.groups() + .filter(|g| { + matches!( + g.kind, + effectcraft_engine::project::GroupKind::Mask { mode: effectcraft_engine::project::MaskMode::Subtract, .. } + | effectcraft_engine::project::GroupKind::Mask { inverted: true, .. } + ) + }) + .count() + }) + .unwrap_or(0); + format!("{} — {masks} subtracting mask(s)", l.name) + } + None => "Select a layer with transparent areas (e.g. a Subtract mask)".into(), + }; + p.text(pos2(x0, y + 9.0), Align2::LEFT_CENTER, &target, Tokens::ui(11.5), if layer.is_some() { t.text_dim } else { t.text_faint }); + app.auto.add("contentFill.target", Rect::from_min_size(pos2(x0, y), vec2(rect.width() - 24.0, 18.0)), &target); + y += ROW; + let mut set = serde_json::Map::new(); + // Alpha Expansion. + kit::label(ui, pos2(x0, y), "Alpha Expansion:", &t); + let a = kit::number(app, ui, pos2(xv, y), "contentFill.alphaExpansion", st.alpha_expansion, 0.2, (0.0, 100.0), 0, " px"); + if (a - st.alpha_expansion).abs() > 1e-9 { + set.insert("alphaExpansion".into(), json!(a.round())); + } + y += ROW; + // Fill Method. + kit::label(ui, pos2(x0, y), "Fill Method:", &t); + let methods = ["object", "surface", "edgeBlend"]; + let cur = methods.iter().position(|m| *m == st.method).unwrap_or(0); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.method", &["Object", "Surface", "Edge Blend"], cur) { + set.insert("method".into(), json!(methods[i])); + } + y += ROW; + // Lighting Correction (Object / Surface). + kit::label(ui, pos2(x0, y), "Lighting Correction:", &t); + let lights = ["none", "subtle", "moderate", "strong"]; + let cur = lights.iter().position(|m| *m == st.lighting_correction).unwrap_or(0); + if st.method != "edgeBlend" + && let Some(i) = + kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.lighting", &["None", "Subtle", "Moderate", "Strong"], cur) + { + set.insert("lightingCorrection".into(), json!(lights[i])); + } + y += ROW; + // Range. + kit::label(ui, pos2(x0, y), "Range:", &t); + let cur = usize::from(st.range == "entire"); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.range", &["Work Area", "Entire Duration"], cur) { + set.insert("range".into(), json!(if i == 1 { "entire" } else { "workArea" })); + } + y += ROW; + // Reference frame: a layer of the comp at a time. + kit::label(ui, pos2(x0, y), "Reference Frame:", &t); + if let Some(c) = &comp { + let mut names = vec!["None".to_string()]; + names.extend(c.layers.iter().map(|l| l.name.clone())); + let cur = st.reference_layer.and_then(|r| c.layers.iter().position(|l| l.id == r)).map(|i| i + 1).unwrap_or(0); + let opts: Vec<&str> = names.iter().map(String::as_str).collect(); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.referenceLayer", &opts, cur) { + set.insert("referenceLayer".into(), if i == 0 { json!(null) } else { json!(c.layers[i - 1].id.0) }); + } + y += ROW; + if st.reference_layer.is_some() { + let at = st.reference_time.unwrap_or(0.0); + p.text(pos2(xv, y + 9.0), Align2::LEFT_CENTER, format!("at {}", kit::seconds(at)), Tokens::ui(11.5), t.text_dim); + if kit::button(app, ui, Rect::from_min_size(pos2(xv + 80.0, y), vec2(120.0, 20.0)), "contentFill.referenceHere", "Use Current Time", false) { + set.insert("referenceTime".into(), json!(app.session.time().seconds())); + } + y += ROW; + } + } else { + y += ROW; + } + if !set.is_empty() { + kit::exec(app, "contentFill.set", serde_json::Value::Object(set)); + } + // Generate, or the running job. + y += 6.0; + let running = app.session.jobs().into_iter().find(|j| j.kind == "contentFill"); + match running { + Some(j) => { + let frac = j.fraction.unwrap_or(0.0) as f32; + let bar = Rect::from_min_size(pos2(x0, y + 7.0), vec2((rect.width() - 110.0).max(60.0), 6.0)); + p.rect_filled(bar, 3.0, t.field_bg); + p.rect_filled(Rect::from_min_size(bar.min, vec2(bar.width() * frac, 6.0)), 3.0, t.accent); + p.text(pos2(x0, y + 24.0), Align2::LEFT_CENTER, format!("{} {:.0} %", j.message, frac * 100.0), Tokens::ui(11.0), t.text_dim); + if kit::button(app, ui, Rect::from_min_size(pos2(bar.max.x + 10.0, y), vec2(70.0, 20.0)), "contentFill.cancel", "Cancel", false) { + kit::exec(app, "jobs.cancel", json!({"job": j.id})); + } + ui.ctx().request_repaint_after(std::time::Duration::from_millis(150)); + } + None => { + if kit::button(app, ui, Rect::from_min_size(pos2(x0, y), vec2(160.0, 24.0)), "contentFill.generate", "Generate Fill Layer", true) { + match &layer { + Some(l) => { + kit::exec(app, "contentFill.generate", json!({"layer": l.id.0})); + } + None => app.ui.status = "Select the layer to fill first".into(), + } + } + } + } +} diff --git a/crates/ui-egui/src/panels/footage_panel.rs b/crates/ui-egui/src/panels/footage_panel.rs new file mode 100644 index 00000000..cfc69825 --- /dev/null +++ b/crates/ui-egui/src/panels/footage_panel.rs @@ -0,0 +1,150 @@ +//! The Footage panel (double-click footage in the Project panel): the source on its own time +//! ruler, play / pause, In and Out marks, and the Edit Target buttons — Overlay Edit and Ripple +//! Insert Edit — that cut the marked range into the active composition at its current time. +//! Everything runs `footage.*` commands (state in `EditorState::footage_panel`). + +use effectcraft_engine::project::{FootageKind, ItemKind}; +use effectcraft_engine::time::Tick; +use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.pasteboard); + let Some(view) = app.session.state.footage_panel.clone() else { + p.text(rect.center(), Align2::CENTER_CENTER, "Double-click footage in the Project panel to open it here", Tokens::ui(12.0), t.text_faint); + app.auto.add("footage.empty", rect, "Footage panel (empty)"); + return; + }; + let Some(item) = app.session.project.item(view.item).cloned() else { + app.session.state.footage_panel = None; + return; + }; + let footage = match &item.kind { + ItemKind::Footage(f) => Some(f.clone()), + _ => None, + }; + let (dur, fr) = match &footage { + Some(f) if f.kind != FootageKind::Still && f.duration > Tick::ZERO => (f.duration, f.frame_rate), + Some(f) => (Tick::from_seconds_f64(10.0), f.frame_rate), + None => (Tick::from_seconds_f64(10.0), effectcraft_engine::time::FrameRate::FPS_30), + }; + // Title strip. + let top = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); + p.rect_filled(top, 0.0, t.panel_bg); + p.text(pos2(top.min.x + 10.0, top.center().y), Align2::LEFT_CENTER, format!("Footage: {}", item.name), Tokens::semibold(12.0), t.text); + app.auto.add("footage.title", top, &item.name); + // Bottom: ruler and controls. + // Rows, bottom up: Edit Target buttons, transport, ruler. + let edit_row = Rect::from_min_max(pos2(rect.min.x, rect.max.y - 30.0), rect.max); + let bar = Rect::from_min_max(pos2(rect.min.x, edit_row.min.y - 28.0), pos2(rect.max.x, edit_row.min.y)); + let ruler = Rect::from_min_max(pos2(rect.min.x + 10.0, bar.min.y - 26.0), pos2(rect.max.x - 10.0, bar.min.y - 4.0)); + p.rect_filled(Rect::from_min_max(pos2(rect.min.x, ruler.min.y - 4.0), rect.max), 0.0, t.panel_bg); + // Canvas. + let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, ruler.min.y - 4.0)); + if let Some(f) = &footage + && f.has_video + { + let key = (view.item.0, view.time.0, app.session.revision); + let tid = egui::Id::new("footage-panel-tex"); + let cached: Option<((u64, i64, u64), egui::TextureHandle, [u32; 2])> = ctx.data(|d| d.get_temp(tid)); + let tex = match cached { + Some((k, tex, size)) if k == key => Some((tex, size)), + _ => app.session.footage.frame(view.item, f, view.time).map(|img| { + let ci = egui::ColorImage::from_rgba_unmultiplied([img.width as usize, img.height as usize], &img.to_rgba8()); + let tex = ctx.load_texture("footage-panel", ci, egui::TextureOptions::LINEAR); + ctx.data_mut(|d| d.insert_temp(tid, (key, tex.clone(), [img.width, img.height]))); + (tex, [img.width, img.height]) + }), + }; + if let Some((tex, [w, h])) = tex { + let s = ((area.width() - 20.0) / w as f32).min((area.height() - 20.0) / h as f32).max(0.01); + let r = Rect::from_center_size(area.center(), vec2(w as f32 * s, h as f32 * s)); + super::viewer::checker(&p.with_clip_rect(area), r); + p.image(tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } + } else if let ItemKind::Solid(so) = &item.kind { + let c = so.color; + p.rect_filled(area.shrink(20.0), 0.0, Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8)); + } else { + p.text(area.center(), Align2::CENTER_CENTER, "No video", Tokens::ui(12.0), t.text_faint); + } + // Ruler: ticks, In/Out span, CTI; click or drag to scrub. + let x_of = |tt: Tick| ruler.min.x + (tt.seconds() / dur.seconds().max(1e-9)) as f32 * ruler.width(); + p.rect_filled(ruler, 2.0, t.tl_ruler_bg); + let secs = dur.seconds().max(0.001); + let step = [0.04, 0.2, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0].into_iter().find(|s| (secs / s) as f32 * 50.0 < ruler.width()).unwrap_or(600.0); + let mut k = 0.0; + while k <= secs { + let x = ruler.min.x + (k / secs) as f32 * ruler.width(); + p.line_segment([pos2(x, ruler.max.y - 6.0), pos2(x, ruler.max.y)], Stroke::new(1.0, t.tl_ruler_tick)); + p.text(pos2(x + 2.0, ruler.min.y + 1.0), Align2::LEFT_TOP, kit::seconds(k), Tokens::ui(9.0), t.tl_ruler_text); + k += step; + } + let (a, b) = (view.in_point.unwrap_or(Tick::ZERO), view.out_point.unwrap_or(dur)); + p.rect_filled(Rect::from_min_max(pos2(x_of(a), ruler.max.y - 4.0), pos2(x_of(b), ruler.max.y)), 0.0, t.work_area); + let cx = x_of(view.time); + p.line_segment([pos2(cx, ruler.min.y), pos2(cx, ruler.max.y)], Stroke::new(1.5, t.cti)); + let resp = ui.interact(ruler, egui::Id::new("footage-ruler"), Sense::click_and_drag()); + app.auto.add("footage.ruler", ruler, "Time ruler"); + if (resp.clicked() || resp.dragged()) + && let Some(pt) = resp.interact_pointer_pos() + { + let s = ((pt.x - ruler.min.x) / ruler.width()).clamp(0.0, 1.0) as f64 * dur.seconds(); + kit::exec(app, "footage.setTime", json!({"time": s})); + } + // Playback. + let play_id = egui::Id::new("footage-playing"); + let playing: bool = ctx.data(|d| d.get_temp(play_id).unwrap_or(false)); + if playing { + let dt = ctx.input(|i| i.stable_dt).min(0.1) as f64; + let next = view.time.seconds() + dt; + let next = if next >= dur.seconds() - fr.frame_duration().seconds() { 0.0 } else { next }; + let _ = app.session.execute("footage.setTime", json!({"time": next})); + ctx.request_repaint(); + } + // Controls. + let y = bar.min.y + 5.0; + let mut x = bar.min.x + 10.0; + p.text( + pos2(x, bar.center().y), + Align2::LEFT_CENTER, + effectcraft_engine::time::format_timecode_ae(fr.frame_at(view.time), fr, false), + Tokens::mono(12.0), + t.timecode, + ); + x += 92.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(54.0, 20.0)), "footage.play", if playing { "Pause" } else { "Play" }, false) { + ctx.data_mut(|d| d.insert_temp(play_id, !playing)); + } + x += 60.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(28.0, 20.0)), "footage.setIn", "{", false) { + kit::exec(app, "footage.setIn", json!({})); + } + x += 32.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(28.0, 20.0)), "footage.setOut", "}", false) { + kit::exec(app, "footage.setOut", json!({})); + } + x += 36.0; + let span = format!("Δ {}", kit::seconds((b - a).seconds())); + p.text(pos2(x, bar.center().y), Align2::LEFT_CENTER, span, Tokens::ui(11.0), t.text_dim); + let comp_ok = app.session.active_comp_id().is_some(); + let target = app.session.active_comp_id().and_then(|c| app.session.project.item(c)).map(|i| i.name.clone()).unwrap_or_default(); + let ey = edit_row.min.y + 5.0; + let bx = (edit_row.max.x - 238.0).max(edit_row.min.x + 10.0); + if !target.is_empty() && bx > edit_row.min.x + 140.0 { + p.text(pos2(bx - 8.0, edit_row.center().y), Align2::RIGHT_CENTER, format!("Edit Target: {target}"), Tokens::ui(11.0), t.text_dim); + } + if kit::button(app, ui, Rect::from_min_size(pos2(bx, ey), vec2(110.0, 20.0)), "footage.rippleInsertEdit", "Ripple Insert Edit", false) && comp_ok { + kit::exec(app, "footage.rippleInsertEdit", json!({})); + } + if kit::button(app, ui, Rect::from_min_size(pos2(bx + 116.0, ey), vec2(110.0, 20.0)), "footage.overlayEdit", "Overlay Edit", true) && comp_ok { + kit::exec(app, "footage.overlayEdit", json!({})); + } +} diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 3e051948..8d63d170 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -343,6 +343,60 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ], ) } + "layer.autoTrace" if !has(p, &["channel", "timeSpan", "threshold"]) => ( + "Auto-trace".into(), + vec![ + Field::choice("timeSpan", "Time Span", &[("Current Frame", json!("currentFrame")), ("Work Area", json!("workArea"))], 0), + Field::choice( + "channel", + "Channel", + &[("Alpha", json!("alpha")), ("Red", json!("red")), ("Green", json!("green")), ("Blue", json!("blue")), ("Luminance", json!("luminance"))], + 0, + ), + Field::bool("invert", "Invert", false), + Field::num("blur", "Blur (pixels before auto-trace)", 1.0), + Field::num("tolerance", "Tolerance (pixels)", 1.0), + Field::num("threshold", "Threshold (%)", 50.0), + Field::num("minimumArea", "Minimum Area (pixels)", 10.0), + Field::num("cornerRoundness", "Corner Roundness (%)", 50.0), + Field::bool("applyToNewLayer", "Apply To New Layer", false), + ], + ), + "layer.sceneEditDetection" if !has(p, &["mode"]) => ( + "Scene Edit Detection".into(), + vec![ + Field::choice( + "mode", + "Action", + &[("Create Markers", json!("markers")), ("Split Layers", json!("split")), ("Split and Precompose", json!("splitPrecompose"))], + 0, + ), + Field::num("threshold", "Sensitivity threshold (0–1, lower finds more)", 0.25), + ], + ), + "layer.alignVideoToData" if !has(p, &["data", "videoStart"]) => { + let data: Vec<(String, Value)> = s + .project + .items + .values() + .filter(|i| matches!(&i.kind, effectcraft_engine::project::ItemKind::Footage(f) if f.kind == effectcraft_engine::project::FootageKind::Data)) + .map(|i| (i.name.clone(), json!(i.id.0))) + .collect(); + if data.is_empty() { + info(app, "Align Video to Data", "Import a data file (JSON, CSV or TSV with a time column) first: File ▸ Import."); + return true; + } + let opts: Vec<(&str, Value)> = data.iter().map(|(n, v)| (n.as_str(), v.clone())).collect(); + ( + "Align Video to Data".into(), + vec![ + Field::choice("data", "Data", &opts, 0), + Field::text("key", "Time field (empty: automatic)", ""), + Field::text("videoStart", "Video start (ISO date-time, hh:mm:ss, timecode; empty: file date)", ""), + Field::num("dataStart", "First sample at (comp seconds)", 0.0), + ], + ) + } "file.importPlaceholder" | "file.replaceWithPlaceholder" if p.as_object().is_none_or(|m| m.is_empty()) => ( "New Placeholder".into(), vec![ diff --git a/crates/ui-egui/src/panels/media_panels.rs b/crates/ui-egui/src/panels/media_panels.rs new file mode 100644 index 00000000..b1a3230e --- /dev/null +++ b/crates/ui-egui/src/panels/media_panels.rs @@ -0,0 +1,366 @@ +//! Media Browser, Metadata and Progress panels (Window menu). +//! +//! - **Media Browser**: favourites and the folder tree on the left, the folder's files on the +//! right with thumbnails (images and video, decoded through the media layer's probe), double- +//! click to open a folder or import a file, drag files into the Project panel or the timeline. +//! Desktop only (the web build has no file system to browse). +//! - **Metadata**: what we read from the selected item's file (codec, size, rate, duration, +//! colour profile, dates) and the project's, with editable item and project comments. +//! - **Progress**: every background job (render queue, track / mask track / Warp Stabilizer / +//! 3D Camera Tracker / Roto Brush analyses, Content-Aware Fill, Scene Edit Detection) with a +//! progress bar and a cancel button, and the recently finished ones. + +use effectcraft_engine::media_browser as mb; +use effectcraft_engine::project::ItemId; +use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::{Value, json}; + +use super::DragPayload; +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +// ---------------------------------------------------------------- Media Browser + +type Thumb = Option<(egui::TextureHandle, [u32; 2])>; + +/// Thumbnail of a file (cached per path; `None` while not decoded or not decodable). +fn thumbnail(app: &EffectcraftApp, ctx: &egui::Context, path: &str, budget: &mut u32) -> Thumb { + let id = egui::Id::new(("mb-thumb", path)); + if let Some(t) = ctx.data(|d| d.get_temp::(id)) { + return t; + } + if *budget == 0 { + return None; + } + *budget -= 1; + let thumb = (|| { + let imp = app.session.importer.clone()?; + let f = imp.probe(path).ok()?; + if !f.has_video { + return None; + } + // A synthetic item id keyed by the path (the frame cache keys on the path). + let hid = path.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ b as u64).wrapping_mul(0x100_0000_01b3)) | (1 << 62); + let img = app.session.footage.frame(ItemId(hid), &f, effectcraft_engine::time::Tick::ZERO)?; + let step = (img.width.max(img.height) as f32 / 128.0).ceil().max(1.0) as u32; + let (w, h) = (img.width.div_ceil(step), img.height.div_ceil(step)); + let full = img.to_rgba8(); + let mut px = Vec::with_capacity((w * h * 4) as usize); + for y in 0..h { + for x in 0..w { + let i = (((y * step) * img.width + x * step) * 4) as usize; + px.extend_from_slice(&full[i..i + 4]); + } + } + let tex = ctx.load_texture(format!("mb-{path}"), egui::ColorImage::from_rgba_unmultiplied([w as usize, h as usize], &px), egui::TextureOptions::LINEAR); + Some((tex, [w, h])) + })(); + ctx.data_mut(|d| d.insert_temp(id, thumb.clone())); + thumb +} + +pub fn media_browser(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + if !mb::available() { + p.text(rect.center(), Align2::CENTER_CENTER, "The Media Browser needs the desktop app (use File ▸ Import here)", Tokens::ui(12.0), t.text_faint); + app.auto.add("mediaBrowser.unavailable", rect, "Media Browser unavailable"); + return; + } + let st = app.session.state.media_browser.clone(); + let dir = st.folder.clone().unwrap_or_else(mb::home); + // Listing cached per folder for a second. + let lid = egui::Id::new(("mb-list", dir.clone(), st.importable_only)); + let now = ctx.input(|i| i.time); + let listing: Option<(f64, Result, String>)> = ctx.data(|d| d.get_temp(lid)); + let entries = match listing { + Some((at, l)) if now - at < 1.0 => l, + _ => { + let l = mb::list(&dir, st.importable_only); + ctx.data_mut(|d| d.insert_temp(lid, (now, l.clone()))); + l + } + }; + // Toolbar: up, home, path, favourite, importable only. + let y = rect.min.y + 6.0; + let mut x = rect.min.x + 8.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(30.0, 20.0)), "mediaBrowser.up", "↑", false) { + kit::exec(app, "mediaBrowser.go", json!({"path": ".."})); + } + x += 34.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(48.0, 20.0)), "mediaBrowser.home", "Home", false) { + kit::exec(app, "mediaBrowser.go", json!({})); + } + x += 54.0; + let fav = st.favorites.contains(&dir); + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(24.0, 20.0)), "mediaBrowser.favorite", if fav { "★" } else { "☆" }, false) { + kit::exec(app, if fav { "mediaBrowser.removeFavorite" } else { "mediaBrowser.addFavorite" }, json!({"path": dir})); + } + x += 30.0; + let only = kit::checkbox(app, ui, pos2(x, y + 2.0), "mediaBrowser.importableOnly", "Media only", st.importable_only); + if only != st.importable_only { + kit::exec(app, "mediaBrowser.go", json!({"path": dir, "importableOnly": only})); + } + x += 100.0; + p.text(pos2(x, y + 10.0), Align2::LEFT_CENTER, &dir, Tokens::ui(11.5), t.text_dim); + app.auto.add("mediaBrowser.path", Rect::from_min_max(pos2(x, y), pos2(rect.max.x - 8.0, y + 20.0)), &dir); + // Left column: favourites. + let side = Rect::from_min_max(pos2(rect.min.x, rect.min.y + 32.0), pos2(rect.min.x + 160.0_f32.min(rect.width() * 0.3), rect.max.y)); + p.line_segment([side.right_top(), side.right_bottom()], Stroke::new(1.0, t.separator)); + p.text(pos2(side.min.x + 10.0, side.min.y + 10.0), Align2::LEFT_CENTER, "Favorites", Tokens::semibold(11.5), t.text_dim); + let mut fy = side.min.y + 22.0; + let mut places: Vec<(String, String)> = vec![("Home".into(), mb::home())]; + places.extend( + st.favorites.iter().map(|f| (std::path::Path::new(f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()), f.clone())), + ); + for (i, (name, path)) in places.iter().enumerate() { + let r = Rect::from_min_size(pos2(side.min.x, fy), vec2(side.width(), 20.0)); + let resp = ui.interact(r, egui::Id::new(("mb-fav", i)), Sense::click()); + if *path == dir { + p.rect_filled(r, 0.0, t.row_selected); + } else if resp.hovered() { + p.rect_filled(r, 0.0, t.hover); + } + p.text(pos2(r.min.x + 14.0, r.center().y), Align2::LEFT_CENTER, name, Tokens::ui(12.0), t.text); + app.auto.add(&format!("mediaBrowser.place.{i}"), r, name); + if resp.clicked() { + kit::exec(app, "mediaBrowser.go", json!({"path": path})); + } + fy += 20.0; + } + // Files. + let list = Rect::from_min_max(pos2(side.max.x + 1.0, side.min.y), rect.max); + let entries = match entries { + Ok(e) => e, + Err(e) => { + p.text(list.center(), Align2::CENTER_CENTER, e, Tokens::ui(12.0), t.danger); + return; + } + }; + let tile = vec2(140.0, 112.0); + let cols = ((list.width() - 8.0) / tile.x).floor().max(1.0) as usize; + let mut budget = 2u32; + let scroll_id = egui::Id::new("mb-scroll"); + let mut scroll: f32 = ctx.data(|d| d.get_temp(scroll_id).unwrap_or(0.0)); + if ui.rect_contains_pointer(list) { + scroll -= ctx.input(|i| i.smooth_scroll_delta.y); + } + let rows = entries.len().div_ceil(cols); + scroll = scroll.clamp(0.0, (rows as f32 * tile.y - list.height() + 8.0).max(0.0)); + ctx.data_mut(|d| d.insert_temp(scroll_id, scroll)); + let lp = p.with_clip_rect(list); + let mut actions: Vec<(&str, Value)> = vec![]; + for (i, e) in entries.iter().enumerate() { + let r = Rect::from_min_size( + pos2(list.min.x + 8.0 + (i % cols) as f32 * tile.x, list.min.y + 8.0 + (i / cols) as f32 * tile.y - scroll), + tile - vec2(8.0, 8.0), + ); + if !r.intersects(list) { + continue; + } + let resp = ui.interact(r.intersect(list), egui::Id::new(("mb-entry", &e.path)), Sense::click_and_drag()); + app.auto.add(&format!("mediaBrowser.entry.{}", e.name), r, &e.name); + lp.rect_filled(r, 4.0, if resp.hovered() { t.hover } else { t.row }); + let thumb = Rect::from_min_max(r.min + vec2(6.0, 6.0), pos2(r.max.x - 6.0, r.max.y - 26.0)); + lp.rect_filled(thumb, 3.0, Color32::from_rgb(0x18, 0x18, 0x18)); + let kind = if e.is_dir { "Folder" } else { e.kind.unwrap_or("File") }; + match (e.is_dir, e.kind) { + (false, Some("image" | "video")) => match thumbnail(app, &ctx, &e.path, &mut budget) { + Some((tex, [w, h])) => { + let s = (thumb.width() / w as f32).min(thumb.height() / h as f32); + lp.image( + tex.id(), + Rect::from_center_size(thumb.center(), vec2(w as f32 * s, h as f32 * s)), + Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), + Color32::WHITE, + ); + } + None => { + lp.text(thumb.center(), Align2::CENTER_CENTER, kind, Tokens::ui(11.0), t.text_faint); + } + }, + _ => { + lp.text(thumb.center(), Align2::CENTER_CENTER, if e.is_dir { "▣" } else { kind }, Tokens::ui(if e.is_dir { 28.0 } else { 11.0 }), t.text_faint); + } + } + let name = if e.name.chars().count() > 20 { format!("{}…", e.name.chars().take(19).collect::()) } else { e.name.clone() }; + lp.text(pos2(r.min.x + 8.0, r.max.y - 12.0), Align2::LEFT_CENTER, name, Tokens::ui(11.5), t.text); + if resp.double_clicked() { + if e.is_dir { + actions.push(("mediaBrowser.go", json!({"path": e.path}))); + } else if e.kind.is_some() { + actions.push(("mediaBrowser.import", json!({"paths": [e.path]}))); + } + } + if resp.drag_started() && !e.is_dir && e.kind.is_some() { + egui::DragAndDrop::set_payload(&ctx, DragPayload::Files(vec![e.path.clone()])); + } + resp.context_menu(|ui| { + if !e.is_dir && ui.button("Import").clicked() { + actions.push(("mediaBrowser.import", json!({"paths": [e.path]}))); + ui.close(); + } + if !e.is_dir && ui.button("Import and Add to Composition").clicked() { + actions.push(("mediaBrowser.import", json!({"paths": [e.path], "addToComp": true}))); + ui.close(); + } + if e.is_dir && ui.button("Add to Favorites").clicked() { + actions.push(("mediaBrowser.addFavorite", json!({"path": e.path}))); + ui.close(); + } + }); + } + if entries.is_empty() { + lp.text(list.center(), Align2::CENTER_CENTER, "Empty folder", Tokens::ui(12.0), t.text_faint); + } + if budget == 0 { + ctx.request_repaint(); + } + for (id, p) in actions { + kit::exec(app, id, p); + } +} + +// ---------------------------------------------------------------- Metadata + +fn rows_of(v: &Value) -> Vec<(String, String)> { + let Some(o) = v.as_object() else { return vec![] }; + o.iter() + .filter(|(k, v)| !v.is_null() && !matches!(k.as_str(), "comment" | "id")) + .map(|(k, v)| { + let s = match v { + Value::String(s) => s.clone(), + Value::Number(n) => n.as_f64().map(|f| if f.fract() == 0.0 { format!("{f:.0}") } else { format!("{f:.3}") }).unwrap_or_default(), + other => other.to_string(), + }; + (k.clone(), s) + }) + .collect() +} + +/// An editable comment field; returns the edited text when it changes. +fn comment_field(ui: &mut egui::Ui, r: Rect, id: &str, text: &str) -> Option { + let key = egui::Id::new(("meta-comment", id)); + let mut buf: String = ui.data(|d| d.get_temp(key)).unwrap_or_else(|| text.to_string()); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(r)); + let resp = child.add(egui::TextEdit::multiline(&mut buf).desired_width(r.width()).desired_rows(2).hint_text("Add a comment")); + if resp.has_focus() || resp.changed() { + ui.data_mut(|d| d.insert_temp(key, buf.clone())); + } else { + ui.data_mut(|d| d.remove::(key)); + } + (resp.lost_focus() && buf != text).then_some(buf) +} + +pub fn metadata(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + let Ok(v) = app.session.execute("item.metadata", json!({})) else { return }; + let mut y = rect.min.y + 8.0; + let x0 = rect.min.x + 12.0; + let xv = x0 + 120.0; + let section = |p: &egui::Painter, y: &mut f32, title: &str| { + p.text(pos2(x0, *y + 8.0), Align2::LEFT_CENTER, title, Tokens::semibold(12.0), t.text); + *y += 22.0; + }; + if let Some(item) = v.get("item").filter(|i| !i.is_null()) { + let name = item["name"].as_str().unwrap_or_default().to_string(); + let id = item["id"].as_u64().unwrap_or(0); + section(&p, &mut y, &format!("{name} ({})", item["type"].as_str().unwrap_or(""))); + kit::label(ui, pos2(x0, y), "Comment", &t); + let r = Rect::from_min_size(pos2(xv, y), vec2((rect.max.x - xv - 12.0).max(80.0), 40.0)); + app.auto.add("metadata.itemComment", r, item["comment"].as_str().unwrap_or("")); + if let Some(c) = comment_field(ui, r, &format!("item{id}"), item["comment"].as_str().unwrap_or("")) { + kit::exec(app, "project.setComment", json!({"items": [id], "comment": c})); + } + y += 48.0; + for (k, val) in rows_of(item) { + if matches!(k.as_str(), "name" | "type") { + continue; + } + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &k, Tokens::ui(11.5), t.text_dim); + p.text(pos2(xv, y + 8.0), Align2::LEFT_CENTER, &val, Tokens::ui(11.5), t.text); + app.auto.add(&format!("metadata.item.{k}"), Rect::from_min_size(pos2(xv, y), vec2(200.0, 16.0)), &val); + y += 18.0; + } + y += 10.0; + } else { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Select an item in the Project panel", Tokens::ui(12.0), t.text_faint); + y += 28.0; + } + let proj = v["project"].clone(); + section(&p, &mut y, "Project"); + kit::label(ui, pos2(x0, y), "Comment", &t); + let r = Rect::from_min_size(pos2(xv, y), vec2((rect.max.x - xv - 12.0).max(80.0), 40.0)); + app.auto.add("metadata.projectComment", r, proj["comment"].as_str().unwrap_or("")); + if let Some(c) = comment_field(ui, r, "project", proj["comment"].as_str().unwrap_or("")) { + kit::exec(app, "project.setProjectComment", json!({"comment": c})); + } + y += 48.0; + for (k, val) in rows_of(&proj) { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &k, Tokens::ui(11.5), t.text_dim); + p.text(pos2(xv, y + 8.0), Align2::LEFT_CENTER, &val, Tokens::ui(11.5), t.text); + y += 18.0; + } +} + +// ---------------------------------------------------------------- Progress + +pub fn progress(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + if !app.session.tasks.is_empty() { + app.session.poll_jobs(); + } + let jobs = app.session.jobs(); + let mut y = rect.min.y + 8.0; + let x0 = rect.min.x + 12.0; + let w = (rect.width() - 24.0).max(60.0); + if jobs.is_empty() { + p.text(pos2(x0, y + 10.0), Align2::LEFT_CENTER, "No background jobs running", Tokens::ui(12.0), t.text_faint); + y += 28.0; + } else { + ui.ctx().request_repaint_after(std::time::Duration::from_millis(150)); + } + for j in &jobs { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &j.label, Tokens::medium(12.0), t.text); + let frac = j.fraction.unwrap_or(0.0) as f32; + let pct = j.fraction.map(|f| format!("{:.0} %", f * 100.0)).unwrap_or_else(|| "…".into()); + p.text(pos2(x0 + w - 70.0, y + 8.0), Align2::RIGHT_CENTER, format!("{pct} {}", j.message), Tokens::ui(11.0), t.text_dim); + let bar = Rect::from_min_size(pos2(x0, y + 20.0), vec2(w - 76.0, 6.0)); + p.rect_filled(bar, 3.0, t.field_bg); + p.rect_filled(Rect::from_min_size(bar.min, vec2(bar.width() * frac.clamp(0.0, 1.0), bar.height())), 3.0, t.accent); + app.auto.add(&format!("progress.job.{}", j.id), Rect::from_min_size(pos2(x0, y), vec2(w, 28.0)), &j.label); + if j.cancellable + && kit::button(app, ui, Rect::from_min_size(pos2(x0 + w - 64.0, y + 12.0), vec2(64.0, 18.0)), &format!("progress.cancel.{}", j.id), "Cancel", false) + { + kit::exec(app, "jobs.cancel", json!({"job": j.id})); + } + y += 38.0; + } + if !app.session.job_log.is_empty() { + p.line_segment([pos2(x0, y), pos2(x0 + w, y)], Stroke::new(1.0, t.separator)); + y += 6.0; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Finished", Tokens::semibold(11.5), t.text_dim); + y += 20.0; + for r in app.session.job_log.iter().rev().take(20) { + let col = match r.status.as_str() { + "done" => t.text, + "cancelled" => t.text_dim, + _ => t.danger, + }; + let msg = if r.message.is_empty() { format!("{} · {:.1} s", r.status, r.seconds) } else { format!("{} · {}", r.status, r.message) }; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &r.label, Tokens::ui(11.5), col); + p.text(pos2(x0 + w, y + 8.0), Align2::RIGHT_CENTER, msg, Tokens::ui(11.0), t.text_faint); + y += 18.0; + if y > rect.max.y { + break; + } + } + } +} diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 2e6d03ae..31daf811 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -3,6 +3,7 @@ pub mod anim_tools; pub mod camera_tracker_ui; pub mod comp_settings; +pub mod content_fill_panel; pub mod dialogs; pub mod dialogs_3d; pub mod effect_controls; @@ -10,6 +11,7 @@ pub mod effects_presets; pub mod essential; pub mod expr_bar; pub mod flowchart; +pub mod footage_panel; pub mod forms; pub mod fx_widgets; pub mod graph; @@ -20,14 +22,17 @@ pub mod key_dialogs; pub mod layer_panel; pub mod layer_styles_dialog; pub mod markers_ui; +pub mod media_panels; pub mod misc; pub mod paint_panels; +pub mod panel_kit; pub mod precomp; pub mod project; pub mod properties; pub mod puppet_tool; pub mod render_queue; pub mod roto_tool; +pub mod scopes_panel; pub mod script_console; pub mod settings; pub mod shortcut_editor; @@ -57,6 +62,8 @@ pub enum DragPayload { Effect(String), /// A property dragged from the timeline (to the Essential Graphics panel). Property { layer: u64, prop: u64 }, + /// Files from the Media Browser (import; into the timeline: import and add). + Files(Vec), } /// The current time formatted per project settings (timecode with `;` for drop-frame, or frames). @@ -103,6 +110,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Brushes => paint_panels::brushes(app, ui, rect), PanelKind::EssentialGraphics => essential::show(app, ui, rect), PanelKind::Flowchart => flowchart::show(app, ui, rect), + PanelKind::LumetriScopes => scopes_panel::show(app, ui, rect), + PanelKind::Footage => footage_panel::show(app, ui, rect), + PanelKind::MediaBrowser => media_panels::media_browser(app, ui, rect), + PanelKind::Metadata => media_panels::metadata(app, ui, rect), + PanelKind::Progress => media_panels::progress(app, ui, rect), + PanelKind::ContentAwareFill => content_fill_panel::show(app, ui, rect), } } diff --git a/crates/ui-egui/src/panels/panel_kit.rs b/crates/ui-egui/src/panels/panel_kit.rs new file mode 100644 index 00000000..cbf28c8a --- /dev/null +++ b/crates/ui-egui/src/panels/panel_kit.rs @@ -0,0 +1,70 @@ +//! Small immediate-mode controls shared by the Lumetri Scopes, Footage, Media Browser, Metadata, +//! Progress and Content-Aware Fill panels. Each registers its automation id. + +use egui::{Align2, Rect, pos2, vec2}; + +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// A dropdown; returns the newly chosen index. +pub fn dropdown(app: &mut EffectcraftApp, ui: &mut egui::Ui, r: Rect, auto: &str, options: &[&str], cur: usize) -> Option { + let t = app.tokens; + let id = egui::Id::new(("kit-dd", auto)); + let text = options.get(cur).copied().unwrap_or(""); + if widgets::dropdown(ui, r, text, &t, id).clicked() { + widgets::open_popup(ui, id); + } + app.auto.add(auto, r, text); + let opts: Vec = options.iter().map(|s| s.to_string()).collect(); + widgets::popup_menu(ui, id, r.left_bottom(), &opts, Some(cur)).filter(|i| *i != cur) +} + +/// A checkbox with a label; returns the new state. +pub fn checkbox(app: &mut EffectcraftApp, ui: &mut egui::Ui, at: egui::Pos2, auto: &str, text: &str, on: bool) -> bool { + let t = app.tokens; + let r = Rect::from_min_size(at, vec2(16.0, 16.0)); + let clicked = widgets::checkbox(ui, r, on, &t, egui::Id::new(("kit-cb", auto))).clicked(); + ui.painter().text(pos2(r.max.x + 6.0, r.center().y), Align2::LEFT_CENTER, text, Tokens::ui(12.0), t.text); + let full = Rect::from_min_size(at, vec2(22.0 + text.len() as f32 * 6.5, 16.0)); + app.auto.add(auto, full, text); + if clicked { !on } else { on } +} + +/// A text button; true when clicked. +pub fn button(app: &mut EffectcraftApp, ui: &mut egui::Ui, r: Rect, auto: &str, text: &str, primary: bool) -> bool { + let t = app.tokens; + let clicked = widgets::text_button(ui, r, text, primary, &t, egui::Id::new(("kit-btn", auto))).clicked(); + app.auto.add(auto, r, text); + clicked +} + +/// A dimmed label at (x, y-row). +pub fn label(ui: &egui::Ui, at: egui::Pos2, text: &str, t: &Tokens) { + ui.painter().text(pos2(at.x, at.y + 9.0), Align2::LEFT_CENTER, text, Tokens::ui(12.0), t.text_dim); +} + +/// A draggable number; returns the new value. +#[allow(clippy::too_many_arguments)] +pub fn number(app: &mut EffectcraftApp, ui: &mut egui::Ui, at: egui::Pos2, auto: &str, v: f64, speed: f64, range: (f64, f64), dec: usize, suffix: &str) -> f64 { + let t = app.tokens; + let (r, nv, _) = widgets::hot_number_at(ui, at, egui::Id::new(("kit-n", auto)), v, speed, range, dec, suffix, &t); + app.auto.add(auto, r, &format!("{v}")); + nv.unwrap_or(v) +} + +/// Run a command, reporting errors in the status bar. +pub fn exec(app: &mut EffectcraftApp, id: &str, p: serde_json::Value) -> Option { + match app.session.execute(id, p) { + Ok(v) => Some(v), + Err(e) => { + app.ui.status = e.to_string(); + None + } + } +} + +/// Format seconds as `m:ss.ff`. +pub fn seconds(s: f64) -> String { + let m = (s / 60.0).floor(); + format!("{}:{:05.2}", m as i64, s - m * 60.0) +} diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index ce750fac..f3898954 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -481,6 +481,11 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ItemKind::Folder if !app.ui.project_open_folders.remove(&id.0) => { app.ui.project_open_folders.insert(id.0); } + // Footage (not data) and solids open in the Footage panel. + ItemKind::Footage(f) if f.kind != effectcraft_engine::project::FootageKind::Data => { + actions.push(("footage.open".into(), json!({"item": id.0}))) + } + ItemKind::Solid(_) => actions.push(("footage.open".into(), json!({"item": id.0}))), _ => {} } } @@ -539,6 +544,16 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { egui::DragAndDrop::clear_payload(&ctx); } } + // Files dragged from the Media Browser: import. + if let Some(DragPayload::Files(paths)) = egui::DragAndDrop::payload::(&ctx).as_deref() + && hover_y.is_some() + { + lp.rect_stroke(list.shrink(1.0), 0.0, Stroke::new(1.0, t.accent.gamma_multiply(0.6)), egui::StrokeKind::Inside); + if ctx.input(|i| i.pointer.any_released()) { + actions.push(("mediaBrowser.import".into(), json!({"paths": paths}))); + egui::DragAndDrop::clear_payload(&ctx); + } + } // Drag ghost. if let Some(DragPayload::Item(iid)) = egui::DragAndDrop::payload::(&ctx).as_deref() && let Some(pos) = ctx.pointer_hover_pos() diff --git a/crates/ui-egui/src/panels/scopes_panel.rs b/crates/ui-egui/src/panels/scopes_panel.rs new file mode 100644 index 00000000..d220213b --- /dev/null +++ b/crates/ui-egui/src/panels/scopes_panel.rs @@ -0,0 +1,197 @@ +//! Lumetri Scopes panel (Window ▸ Lumetri Scopes): waveform (RGB / Luma / YC), vectorscope +//! (YUV / HLS), histogram and parade (RGB / YUV) of the active composition's frame at the current +//! time, with the colour standard (Rec. 601 / 709 / 2020), 8-bit or float scale and clamp. The +//! maths is `effectcraft_raster::scopes`; agents read the same numbers with `scopes.analyze`. + +use effectcraft_engine::commands::panels_cmds::scope_frame; +use effectcraft_engine::raster::scopes::{self, ColorStandard, Scope, ScopeKind, ScopeOpts}; +use egui::{Align2, Color32, Rect, Stroke, pos2, vec2}; +use serde::{Deserialize, Serialize}; + +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +/// Panel options (serde in the UI state). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ScopesState { + pub scope: String, + pub standard: String, + pub float: bool, + pub clamp: bool, +} + +impl Default for ScopesState { + fn default() -> Self { + ScopesState { scope: "waveformRgb".into(), standard: "rec709".into(), float: false, clamp: true } + } +} + +const STANDARDS: [&str; 3] = ["Rec. 601", "Rec. 709", "Rec. 2020"]; + +#[derive(Clone)] +struct Cached { + key: (u64, u64, i64, String, u32), + scope: Scope, + tex: Option, +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + let mut st = app.ui.scopes.clone(); + let kind = ScopeKind::from_name(&st.scope).unwrap_or(ScopeKind::WaveformRgb); + let standard = ColorStandard::from_name(&st.standard).unwrap_or_default(); + // Top bar: scope, standard, 8-bit / float, clamp. + let mut x = rect.min.x + 8.0; + let y = rect.min.y + 6.0; + let labels: Vec<&str> = ScopeKind::ALL.iter().map(|k| k.label()).collect(); + let cur = ScopeKind::ALL.iter().position(|k| *k == kind).unwrap_or(0); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(x, y), vec2(150.0, 20.0)), "scopes.kind", &labels, cur) { + st.scope = ScopeKind::ALL[i].name().into(); + } + x += 158.0; + let si = ColorStandard::ALL.iter().position(|c| *c == standard).unwrap_or(1); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(x, y), vec2(90.0, 20.0)), "scopes.standard", &STANDARDS, si) { + st.standard = ColorStandard::ALL[i].name().into(); + } + x += 98.0; + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(x, y), vec2(70.0, 20.0)), "scopes.scale", &["8 Bit", "Float"], usize::from(st.float)) { + st.float = i == 1; + } + x += 78.0; + st.clamp = kit::checkbox(app, ui, pos2(x, y + 2.0), "scopes.clamp", "Clamp Signal", st.clamp); + if st != app.ui.scopes { + app.ui.scopes = st.clone(); + } + let area = Rect::from_min_max(pos2(rect.min.x + 8.0, rect.min.y + 34.0), pos2(rect.max.x - 8.0, rect.max.y - 8.0)); + if area.width() < 40.0 || area.height() < 40.0 { + return; + } + p.rect_filled(area, 2.0, Color32::from_rgb(0x10, 0x10, 0x10)); + app.auto.add("scopes.plot", area, kind.label()); + let Some(cid) = app.session.active_comp_id() else { + p.text(area.center(), Align2::CENTER_CENTER, "Open a composition to see its scopes", Tokens::ui(12.0), t.text_faint); + return; + }; + // Plot rectangle: square for vectorscopes. + let plot = if kind.is_vectorscope() { + let s = area.width().min(area.height()) - 16.0; + Rect::from_center_size(area.center(), vec2(s, s)) + } else { + Rect::from_min_max(area.min + vec2(34.0, 8.0), area.max - vec2(8.0, 8.0)) + }; + let size = (plot.width().max(plot.height()) as u32).clamp(64, 512); + let o = ScopeOpts { + standard, + float: st.float, + clamp: st.clamp, + width: if kind.is_vectorscope() { size } else { (plot.width() as u32).clamp(64, 768) }, + height: if kind.is_vectorscope() { size } else { (plot.height() as u32).clamp(32, 512) }, + }; + let time = app.session.time(); + let key = (app.session.revision, cid.0, time.0, format!("{}{}{}{}{}x{}", st.scope, st.standard, st.float, st.clamp, o.width, o.height), 0); + let ctx = ui.ctx().clone(); + let id = egui::Id::new("scopes-cache"); + let cached: Option = ctx.data(|d| d.get_temp(id)); + let cached = match cached { + Some(c) if c.key == key => c, + _ => { + let Some(frame) = scope_frame(&app.session, cid, time, 480) else { return }; + let scope = scopes::compute(&frame, kind, &o); + let tex = (kind != ScopeKind::Histogram).then(|| { + let mono = matches!(kind, ScopeKind::WaveformLuma | ScopeKind::VectorscopeYuv | ScopeKind::VectorscopeHls); + let img = egui::ColorImage::from_rgba_unmultiplied([scope.width as usize, scope.height as usize], &tint(&scope, mono)); + ctx.load_texture("lumetri-scope", img, egui::TextureOptions::LINEAR) + }); + let c = Cached { key, scope, tex }; + ctx.data_mut(|d| d.insert_temp(id, c.clone())); + c + } + }; + graticule(&p, plot, kind, &o, &t); + if let Some(tex) = &cached.tex { + p.image(tex.id(), plot, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } else { + histogram(&p, plot, &cached.scope); + } +} + +/// Trace colours: green-tinted mono for luma/vectorscope, per-channel colours otherwise. +fn tint(s: &Scope, mono: bool) -> Vec { + let n = s.width as usize * s.height as usize; + let mut out = vec![0u8; n * 4]; + for i in 0..n { + let (r, g, b) = (s.planes[0][i], s.planes[1][i], s.planes[2][i]); + let (r, g, b) = if mono { + let m = r.max(g).max(b); + (m * 0.75, m, m * 0.75) + } else { + (r, g, b) + }; + let a = r.max(g).max(b).min(1.0); + out[i * 4] = (r.min(1.0) * 255.0) as u8; + out[i * 4 + 1] = (g.min(1.0) * 255.0) as u8; + out[i * 4 + 2] = (b.min(1.0) * 255.0) as u8; + out[i * 4 + 3] = (a * 255.0) as u8; + } + out +} + +fn graticule(p: &egui::Painter, plot: Rect, kind: ScopeKind, o: &ScopeOpts, t: &Tokens) { + let line = Stroke::new(1.0, Color32::from_rgba_unmultiplied(255, 255, 255, 28)); + if kind.is_vectorscope() { + let c = plot.center(); + for r in [0.25f32, 0.5, 0.75, 1.0] { + p.circle_stroke(c, plot.width() / 2.0 * r, line); + } + p.line_segment([pos2(plot.min.x, c.y), pos2(plot.max.x, c.y)], line); + p.line_segment([pos2(c.x, plot.min.y), pos2(c.x, plot.max.y)], line); + for (name, [u, v]) in scopes::vector_targets(o.standard, kind == ScopeKind::VectorscopeHls) { + let q = pos2(plot.min.x + (u + 0.5) * plot.width(), plot.min.y + (0.5 - v) * plot.height()); + p.rect_stroke(Rect::from_center_size(q, vec2(8.0, 8.0)), 0.0, Stroke::new(1.0, Color32::from_rgb(0xc8, 0x6a, 0x6a)), egui::StrokeKind::Middle); + p.text(q + vec2(7.0, -7.0), Align2::LEFT_BOTTOM, name, Tokens::ui(10.0), t.text_dim); + } + return; + } + if kind == ScopeKind::Histogram { + return; + } + let (lo, hi) = o.range(); + let marks: &[f32] = if o.float { &[-0.25, 0.0, 0.25, 0.5, 0.75, 1.0, 1.25] } else { &[0.0, 0.25, 0.5, 0.75, 1.0] }; + for v in marks { + let y = plot.max.y - (v - lo) / (hi - lo) * plot.height(); + p.line_segment([pos2(plot.min.x, y), pos2(plot.max.x, y)], line); + let label = if o.float { format!("{v:.2}") } else { format!("{}", (v * 255.0).round() as i32) }; + p.text(pos2(plot.min.x - 4.0, y), Align2::RIGHT_CENTER, label, Tokens::ui(9.5), t.text_faint); + } + if matches!(kind, ScopeKind::ParadeRgb | ScopeKind::ParadeYuv) { + for k in 1..3 { + let x = plot.min.x + plot.width() * k as f32 / 3.0; + p.line_segment([pos2(x, plot.min.y), pos2(x, plot.max.y)], line); + } + } +} + +fn histogram(p: &egui::Painter, plot: Rect, s: &Scope) { + let bars = scopes::histogram_bars(s); + let max = bars.iter().flat_map(|b| b.iter()).copied().max().unwrap_or(1).max(1) as f32; + let cols = [ + Color32::from_rgba_unmultiplied(230, 70, 70, 150), + Color32::from_rgba_unmultiplied(70, 220, 90, 150), + Color32::from_rgba_unmultiplied(80, 120, 255, 150), + ]; + let n = bars[0].len().max(1) as f32; + for (c, b) in bars.iter().enumerate() { + let pts: Vec = + b.iter().enumerate().map(|(i, v)| pos2(plot.min.x + i as f32 / n * plot.width(), plot.max.y - (*v as f32 / max).sqrt() * plot.height())).collect(); + for (i, q) in pts.iter().enumerate() { + p.line_segment([pos2(q.x, plot.max.y), *q], Stroke::new((plot.width() / n).max(1.0), cols[c].gamma_multiply(0.5))); + if i > 0 { + p.line_segment([pts[i - 1], *q], Stroke::new(1.0, cols[c])); + } + } + } +} diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 10ad62e6..72f39df8 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -1986,6 +1986,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("effect.apply".into(), json!({"effect": e, "layers": [l.0]}))); } } + crate::panels::DragPayload::Files(paths) => actions.push(("mediaBrowser.import".into(), json!({"paths": paths, "addToComp": true}))), crate::panels::DragPayload::Property { .. } => {} } egui::DragAndDrop::clear_payload(&ctx); diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index adffbbeb..07715d16 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -612,6 +612,9 @@ pub struct UiState { /// Flowchart panel options (layers, effects, solids, direction, root comp). #[serde(default)] pub flowchart: crate::panels::flowchart::FlowOptions, + /// Lumetri Scopes panel options. + #[serde(default)] + pub scopes: crate::panels::scopes_panel::ScopesState, } impl Default for UiState { @@ -665,6 +668,7 @@ impl Default for UiState { anim_tools: AnimToolsState::default(), mini_flowchart: None, flowchart: Default::default(), + scopes: Default::default(), } } } diff --git a/crates/ui-egui/tests/ui_panels.rs b/crates/ui-egui/tests/ui_panels.rs new file mode 100644 index 00000000..67d3db92 --- /dev/null +++ b/crates/ui-egui/tests/ui_panels.rs @@ -0,0 +1,104 @@ +//! Headless checks for the M13.6 panels: each opens from the Window menu as a real panel and +//! registers its automation ids; the Footage panel's buttons edit into the comp; the Progress +//! panel lists and cancels a job; Lumetri Scopes switch scope through their dropdown. + +use effectcraft_engine::Session; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui::{Event, Pos2, pos2}; +use egui_kittest::Harness; +use serde_json::json; + +fn harness() -> Harness<'static, EffectcraftApp> { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Panels", "width": 320, "height": 180, "duration": 4})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.run_steps(3); + h +} + +fn open(h: &mut Harness<'_, EffectcraftApp>, panel: &str) { + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "window.panel", json!({"panel": panel})).unwrap(); + h.run_steps(3); +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, id: &str) { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")).clone(); + let p: Pos2 = pos2(e.rect[0] + e.rect[2] / 2.0, e.rect[1] + e.rect[3] / 2.0); + h.input_mut().events.push(Event::PointerMoved(p)); + h.input_mut().events.push(Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + h.input_mut().events.push(Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(2); +} + +#[test] +fn window_menu_opens_the_new_panels() { + let mut h = harness(); + for (name, kind, auto) in [ + ("lumetriScopes", PanelKind::LumetriScopes, "scopes.kind"), + ("footage", PanelKind::Footage, "footage.empty"), + ("mediaBrowser", PanelKind::MediaBrowser, "mediaBrowser.path"), + ("metadata", PanelKind::Metadata, "metadata.projectComment"), + ("progress", PanelKind::Progress, ""), + ("contentAwareFill", PanelKind::ContentAwareFill, "contentFill.method"), + ] { + open(&mut h, name); + assert!(h.state().ui.dock.contains(kind) || h.state().ui.floating.iter().any(|f| f.panels.contains(&kind)), "{name} shown"); + if !auto.is_empty() { + assert!(h.state().auto.find(auto).is_some(), "{name}: {auto}"); + } + } +} + +#[test] +fn scopes_switch_kind() { + let mut h = harness(); + open(&mut h, "lumetriScopes"); + assert!(h.state().auto.find("scopes.plot").is_some()); + h.state_mut().ui.scopes.scope = "vectorscopeYuv".into(); + h.run_steps(2); + assert_eq!(h.state().auto.find("scopes.plot").unwrap().label, "Vectorscope YUV"); + h.state_mut().ui.scopes.scope = "histogram".into(); + h.run_steps(2); + assert_eq!(h.state().auto.find("scopes.kind").unwrap().label, "Histogram"); +} + +#[test] +fn footage_panel_buttons_edit_into_the_comp() { + let mut h = harness(); + let item = h.state().session.project.items.values().find(|i| matches!(i.kind, effectcraft_engine::project::ItemKind::Solid(_))).unwrap().id.0; + h.state_mut().session.execute("footage.open", json!({"item": item})).unwrap(); + h.run_steps(4); + assert!(h.state().ui.dock.contains(PanelKind::Footage) || h.state().ui.floating.iter().any(|f| f.panels.contains(&PanelKind::Footage))); + assert!(h.state().auto.find("footage.overlayEdit").is_some()); + let before = h.state().session.active_comp().unwrap().layers.len(); + click(&mut h, "footage.overlayEdit"); + assert_eq!(h.state().session.active_comp().unwrap().layers.len(), before + 1); + click(&mut h, "footage.rippleInsertEdit"); + assert!(h.state().session.active_comp().unwrap().layers.len() >= before + 2); +} + +#[test] +fn progress_panel_cancels_a_job() { + let mut h = harness(); + h.state_mut() + .session + .spawn_task("test", "Long analysis", false, |ctl| { + while ctl.progress(1, 2) { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + Err("cancelled".into()) + }) + .unwrap(); + open(&mut h, "progress"); + let id = h.state().session.jobs()[0].id.clone(); + assert!(h.state().auto.find(&format!("progress.job.{id}")).is_some()); + click(&mut h, &format!("progress.cancel.{id}")); + h.state_mut().session.wait_jobs(); + h.run_steps(2); + assert!(h.state().session.jobs().is_empty()); + assert_eq!(h.state().session.job_log.last().unwrap().status, "cancelled"); +} diff --git a/docs/architecture.md b/docs/architecture.md index 39acd396..f37f4e73 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -267,6 +267,42 @@ demand (`self_at`) and `roto.propagate` fills in the background; `Session::edit` Key in step with the layer's source, masks and upstream effects. Freeze stores final 8-bit mattes in the effect. +**Content-Aware Fill** (`raster::inpaint`, `engine::commands::content_fill`): the hole is +where the layer's masked source (`Renderer::layer_input` with no effects) is under half +opacity, grown by Alpha Expansion. Optical flow between neighbouring membrane-filled frames is +*completed* inside the holes (membrane interpolation of the surrounding tiles' motion); each +missing pixel follows it forwards and backwards to the nearest frame (or reference frame) where +it is visible, the two candidates blended by temporal distance, with Lighting Correction adding +the membrane of the border mismatch (Poisson-style). What no frame shows is synthesised by +PatchMatch (Barnes et al. 2009) inside Wexler et al.'s (2007) multi-scale EM: Object refines each +frame starting from the previous result warped along the flow; Surface carries one synthesised +fill through the range; Edge Blend is the membrane fill alone. The frames are written as a PNG +sequence and imported into a "Fill" layer above the source. It runs as a generic **background +task** (`engine::jobs`: a closure with progress and cancel on a thread, inline on wasm32 or +with `wait`, whose result is applied on the UI thread as one undo step); the Progress panel and +`jobs.list` / `jobs.cancel` show these tasks together with the render queue and the analyses. + +**Scene Edit Detection** (`raster::cuts`) compares consecutive frames by colour-histogram +distance and motion-compensated (block-flow warped) luma error, takes their geometric mean and +marks frames that exceed a threshold and several times their neighbours' median; the cuts +become layer markers, split layers, or split-and-precomposed layers. **Auto-trace** +(`path::trace`) runs marching squares on the chosen channel, Douglas–Peucker simplification and +cubic fitting with Corner Roundness blending chord and smooth tangents; holes become Subtract +masks, and a work-area trace keys Mask Path per frame. + +**Align Video to Data** reads data footage (File ▸ Import of `.json`, `.csv`, `.tsv`, kept as +text in the project). Supported data: JSON as an array of objects or an object holding one +(`{"samples": [...]}`), CSV/TSV with a header row (quoted fields allowed). The time key is the +`key` parameter or the first field named `time`, `timestamp`, `t`, `date`, `datetime`, `utc`, +`gps_time`, `seconds` or `time_s`; values may be ISO 8601 date-times (with `Z` or `±hh:mm`), +times of day (`hh:mm:ss.sss`), timecode (`hh:mm:ss:ff` at the footage rate), Unix seconds or +Unix milliseconds. The layer's Start Time becomes `dataStart + videoStart − firstSample` +(`videoStart` given, or the file's creation date on the desktop; reduced to a time of day when +the data has no dates). + +**Lumetri Scopes** (`raster::scopes`) are density plots of the viewer frame (rendered over the +comp background) with BT.601 / BT.709 / BT.2020 luma and colour-difference coefficients. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index fbe12a5f..a38ed840 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -24,9 +24,8 @@ renders); stroke taper/wave and multi-segment dashes; variable mask feather poin rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur and blend modes; a few viewer and colour-management menu items; OpenType features and variable -font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; -Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and -the "better than After Effects" items (a plug-in API, branching history, GPU particles). +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; face +tracking; and the "better than After Effects" items (a plug-in API, branching history, GPU particles). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for history. @@ -167,6 +166,25 @@ style runs; the Layer Style dialog (LYR-9); Graph Editor snapping to markers and Reference Axes; Animate Text ▸ Variable Font Axes. Remaining in these rows: Lottie can't carry taper/wave, Advanced 3D's DOF has no iris shapes, and variable axes don't change advances. +## Update: M13.6 panels and content tools + +The last placeholder panels became real panels with automation ids: **Lumetri Scopes** +(waveform RGB / Luma / YC, vectorscope YUV / HLS, histogram, parade RGB / YUV; Rec. 601 / 709 / +2020, 8-bit or float scale, clamp; `scopes.analyze`), **Footage** (double-click footage: its own +time ruler, play, In/Out, Overlay Edit and Ripple Insert Edit; `footage.*`), **Media Browser** +(desktop file system, favourites, thumbnails, import and drag to the Project panel or timeline; +`mediaBrowser.*`), **Metadata** (codec, size, rate, duration, colour profile, file dates; item +and project comments; `item.metadata`) and **Progress** (every render and analysis with +progress and cancel; `jobs.*`). **Content-Aware Fill** (panel and Layer ▸ New ▸ Content-Aware +Fill Layer): Object / Surface / Edge Blend, Work Area / Entire Duration, Alpha Expansion, +Lighting Correction and a reference frame from a layer, rendered to a PNG sequence in a Fill +layer above the source as a background job. **Scene Edit Detection** (markers, split, split and +precompose), **Auto-trace** (alpha / RGB / luminance, current frame or work area, to the layer +or a new layer) and **Align Video to Data** (JSON / CSV / TSV time keys) replace their disabled +menu entries. Not yet: creating a reference frame by editing a still (After Effects hands it to +Photoshop), Content-Aware Fill's learned model (ours is PatchMatch + flow propagation), and Media +Browser in the web app. + ## Highest-value gaps, in order 1. ~~On-canvas text editing and per-character styles~~ (landed: M9.9–M9.10).