From 2b0f263418741cd9091480bfb770e108d02b89f1 Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 10:07:55 -0700 Subject: [PATCH 01/12] GPU3: render exports on the GPU compositor (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add off-screen GPU export renderer (filmcraft-gpu::ExportRenderer) reading back the linear float accumulator as premultiplied RGBA f32, plug it into the export pipeline via FrameRenderer hook and GpuRendering setting, register in desktop app and CLI, and record export.gpu_frames statistics. 🤖 Completed by agy (Antigravity Agent, Gemini 3.8 Flash) based on draft by Merlin --- Cargo.lock | 12 ++ apps/filmcraft-cli/Cargo.toml | 6 +- apps/filmcraft-cli/src/main.rs | 29 +++ apps/filmcraft/Cargo.toml | 5 + apps/filmcraft/src/main.rs | 25 +++ crates/engine/src/export_tools.rs | 14 +- crates/export/src/lib.rs | 82 +++++++++ crates/export/src/pipeline.rs | 29 ++- crates/export/src/settings.rs | 12 ++ crates/export/src/tests.rs | 131 ++++++++++++++ crates/gpu/Cargo.toml | 5 + crates/gpu/src/export_renderer.rs | 209 ++++++++++++++++++++++ crates/gpu/src/lib.rs | 62 +++++++ crates/gpu/tests/export_pipeline.rs | 134 ++++++++++++++ crates/ui-egui/Cargo.toml | 1 + crates/ui-egui/examples/bench/main.rs | 2 + crates/ui-egui/examples/bench/playback.rs | 6 + crates/ui-egui/examples/bench/sections.rs | 30 +++- crates/ui-egui/src/panels/export_mode.rs | 12 +- 19 files changed, 799 insertions(+), 7 deletions(-) create mode 100644 crates/gpu/src/export_renderer.rs create mode 100644 crates/gpu/tests/export_pipeline.rs diff --git a/Cargo.lock b/Cargo.lock index 900806b9..618ac3e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1672,7 +1672,12 @@ dependencies = [ "eframe", "egui", "filmcraft-engine", + "filmcraft-export", + "filmcraft-gpu", "filmcraft-platform", + "filmcraft-project", + "filmcraft-render", + "filmcraft-time", "filmcraft-ui-egui", "log", "muda", @@ -1778,11 +1783,15 @@ dependencies = [ "filmcraft-automation", "filmcraft-codecs", "filmcraft-engine", + "filmcraft-export", + "filmcraft-gpu", "filmcraft-media", "filmcraft-mpegts", "filmcraft-mxf", "filmcraft-ogg", "filmcraft-platform", + "filmcraft-project", + "filmcraft-render", "filmcraft-time", "serde_json", "tokio", @@ -1964,12 +1973,14 @@ name = "filmcraft-gpu" version = "0.4.0" dependencies = [ "filmcraft-color", + "filmcraft-export", "filmcraft-frame", "filmcraft-geom", "filmcraft-media", "filmcraft-project", "filmcraft-render", "filmcraft-time", + "image", "log", "pollster 0.4.0", "rayon", @@ -2225,6 +2236,7 @@ dependencies = [ "filmcraft-color", "filmcraft-edit", "filmcraft-engine", + "filmcraft-export", "filmcraft-frame", "filmcraft-geom", "filmcraft-gpu", diff --git a/apps/filmcraft-cli/Cargo.toml b/apps/filmcraft-cli/Cargo.toml index f6c05bb5..ae3a6f31 100644 --- a/apps/filmcraft-cli/Cargo.toml +++ b/apps/filmcraft-cli/Cargo.toml @@ -19,13 +19,17 @@ whisper = ["filmcraft-engine/whisper", "filmcraft-engine/speech-download"] [dependencies] filmcraft-engine = { workspace = true } +filmcraft-export = { workspace = true } +filmcraft-project = { workspace = true } +filmcraft-render = { workspace = true } +filmcraft-time = { workspace = true } +filmcraft-gpu = { workspace = true } filmcraft-media = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-platform = { workspace = true } filmcraft-mxf = { workspace = true } filmcraft-mpegts = { workspace = true } filmcraft-ogg = { workspace = true } -filmcraft-time = { workspace = true } filmcraft-automation = { workspace = true } serde_json = { workspace = true } tokio = { version = "1", features = ["rt-multi-thread", "macros"] } diff --git a/apps/filmcraft-cli/src/main.rs b/apps/filmcraft-cli/src/main.rs index 13bebb85..2066f22a 100644 --- a/apps/filmcraft-cli/src/main.rs +++ b/apps/filmcraft-cli/src/main.rs @@ -27,6 +27,7 @@ SUBCOMMANDS inspect [project|sequence] project tree or active sequence as JSON (default: both) import ... import media into the project export [--preset name] [--format f] [--range r] [--start s --end s] [--settings json] + [--gpu-rendering auto|off] [--scale f] [--quality 0-100] [--no-audio] [--queue] export the active sequence and wait for it to finish: with an export preset (`export --list-presets`; built-in or the user's), or @@ -156,6 +157,30 @@ fn format_for(path: &str) -> Option<&'static str> { /// OS hardware video decoders (VideoToolbox on macOS, Media Foundation on Windows) in front of our /// own, as in the desktop app. Also what `mcp` and the headless commands decode with. +/// The GPU export frame renderer (filmcraft-gpu's off-screen compositor behind filmcraft-export's +/// frame-renderer hook): exports with GPU rendering Auto composite on the GPU and fall back to the +/// CPU reference renderer wherever it cannot. +struct GpuFrameRenderer(filmcraft_gpu::ExportRenderer); + +impl filmcraft_export::FrameRenderer for GpuFrameRenderer { + fn render( + &mut self, + project: &filmcraft_project::Project, + seq: filmcraft_project::ItemId, + t: filmcraft_time::Tick, + opts: filmcraft_render::RenderOptions, + sources: &dyn filmcraft_render::SourceProvider, + ) -> Option { + self.0.render(project, seq, t, opts, sources) + } +} + +fn register_gpu_frame_renderer() { + filmcraft_export::register_frame_renderer(|| { + filmcraft_gpu::ExportRenderer::new().map(|r| Box::new(GpuFrameRenderer(r)) as Box) + }); +} + fn register_hardware_decoders() -> filmcraft_platform::Availability { filmcraft_platform::register() } @@ -167,6 +192,7 @@ async fn main() { return; } register_hardware_decoders(); + register_gpu_frame_renderer(); let a = Args::parse(std::env::args().skip(1)); let Some(cmd) = a.pos(0) else { usage("missing subcommand") }; match cmd { @@ -329,6 +355,9 @@ async fn main() { if a.flag("--no-audio") { p["audio"] = json!(false); } + if let Some(v) = a.opt("--gpu-rendering") { + p["gpuRendering"] = json!(v); // off | auto (validated by the export command) + } let mut b = Backend::open(&a); let t0 = std::time::Instant::now(); let r = if a.flag("--queue") { diff --git a/apps/filmcraft/Cargo.toml b/apps/filmcraft/Cargo.toml index 8095c787..e3c2f8a7 100644 --- a/apps/filmcraft/Cargo.toml +++ b/apps/filmcraft/Cargo.toml @@ -17,6 +17,11 @@ whisper = ["filmcraft-engine/whisper", "filmcraft-engine/speech-download"] [dependencies] filmcraft-engine = { workspace = true } +filmcraft-export = { workspace = true } +filmcraft-project = { workspace = true } +filmcraft-render = { workspace = true } +filmcraft-time = { workspace = true } +filmcraft-gpu = { workspace = true } filmcraft-ui-egui = { workspace = true } filmcraft-platform = { workspace = true } egui = { workspace = true } diff --git a/apps/filmcraft/src/main.rs b/apps/filmcraft/src/main.rs index b9b279a1..876cd3a3 100644 --- a/apps/filmcraft/src/main.rs +++ b/apps/filmcraft/src/main.rs @@ -90,6 +90,7 @@ fn main() -> eframe::Result { // OS hardware video decoders (VideoToolbox on macOS) in front of our own; Settings ▸ Playback ▸ // Hardware decoding switches them off. Unsupported streams and failures use our decoders. register_hardware_decoders(); + register_gpu_frame_renderer(); let options = eframe::NativeOptions { viewport: egui::ViewportBuilder::default() .with_title("FilmCraft") @@ -241,6 +242,30 @@ fn open_path(path: &str, reveal: bool) -> Result<(), String> { cmd.spawn().map(|_| ()).map_err(|e| format!("can't open {path}: {e}")) } +/// The GPU export frame renderer (filmcraft-gpu's off-screen compositor behind filmcraft-export's +/// frame-renderer hook): exports with GPU rendering Auto composite on the GPU and fall back to the +/// CPU reference renderer wherever it cannot. +struct GpuFrameRenderer(filmcraft_gpu::ExportRenderer); + +impl filmcraft_export::FrameRenderer for GpuFrameRenderer { + fn render( + &mut self, + project: &filmcraft_project::Project, + seq: filmcraft_project::ItemId, + t: filmcraft_time::Tick, + opts: filmcraft_render::RenderOptions, + sources: &dyn filmcraft_render::SourceProvider, + ) -> Option { + self.0.render(project, seq, t, opts, sources) + } +} + +fn register_gpu_frame_renderer() { + filmcraft_export::register_frame_renderer(|| { + filmcraft_gpu::ExportRenderer::new().map(|r| Box::new(GpuFrameRenderer(r)) as Box) + }); +} + /// Put the OS hardware video decoders in front of our own. Registered in a statement of its own: /// a log macro does not evaluate its arguments while no logger takes its level, which left the /// hardware decoders out of every run. diff --git a/crates/engine/src/export_tools.rs b/crates/engine/src/export_tools.rs index b36566c7..69f9dc9d 100644 --- a/crates/engine/src/export_tools.rs +++ b/crates/engine/src/export_tools.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use filmcraft_export::presets::{DEFAULT_PRESET, preset_key}; -use filmcraft_export::{ExportPreset, ExportSettings, Format, HardwareEncoding}; +use filmcraft_export::{ExportPreset, ExportSettings, Format, GpuRendering, HardwareEncoding}; use filmcraft_project::{ItemId, Project}; use filmcraft_time::{FrameRate, Tick, TimeRange}; use serde::{Deserialize, Serialize}; @@ -212,6 +212,18 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option other => serde_json::from_value(other.clone()).map_err(|_| bad(cmd, "hardwareEncoding: off | auto"))?, }; } + if let Some(v) = p.get("gpuRendering") { + settings.gpu_rendering = match v { + Value::Bool(on) => { + if *on { + GpuRendering::Auto + } else { + GpuRendering::Off + } + } + other => serde_json::from_value(other.clone()).map_err(|_| bad(cmd, "gpuRendering: off | auto"))?, + }; + } if let Some(v) = bool_p(p, "burnCaptions") { settings.burn_captions = v; } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 419a7368..81f924b7 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -272,6 +272,10 @@ pub struct ExportSettings { /// byte-reproducible like the built-in encoder's (`determinism_tests`). #[serde(default)] pub hardware_encoding: HardwareEncoding, + /// Composite the exported frames on the GPU instead of the CPU reference renderer (Export ▸ + /// GPU rendering: Auto / Off). Off = today's CPU behaviour exactly. + #[serde(default)] + pub gpu_rendering: GpuRendering, /// VBR maximum bitrate (None = 1.5 × target). pub max_bitrate_kbps: Option, /// Adaptive bitrate (the Match Source presets): bits per pixel per frame; replaces @@ -454,6 +458,7 @@ impl Default for ExportSettings { h264_level: None, bitrate_mode: BitrateMode::default(), hardware_encoding: HardwareEncoding::default(), + gpu_rendering: GpuRendering::Auto, max_bitrate_kbps: None, adaptive_bitrate: None, keyframe_distance: None, @@ -671,6 +676,83 @@ pub fn register_encoder(f: EncoderFactory) { pub fn encoder_registered(f: EncoderFactory) -> bool { video_factories().read().unwrap_or_else(|e| e.into_inner()).iter().any(|x| std::ptr::fn_addr_eq(*x, f)) } + +/// Renders one export frame on something other than the CPU reference renderer — today the GPU +/// compositor (`filmcraft-gpu`), registered by the app or CLI at startup. `render` returns the +/// frame exactly as [`filmcraft_render::render_sequence`] would (same size, premultiplied +/// linear-light RGBA), or `None` to let the CPU render this frame (no adapter, a plan the GPU +/// cannot draw, any internal error): the CPU renderer is the reference and the fallback. +pub trait FrameRenderer: Send { + fn render( + &mut self, + project: &Project, + seq: ItemId, + t: Tick, + opts: filmcraft_render::RenderOptions, + sources: &dyn SourceProvider, + ) -> Option; +} + +/// Builds the frame renderer for one export, or `None` when this host cannot (no adapter). +pub type FrameRendererFactory = fn() -> Option>; + +fn frame_renderer_factories() -> &'static RwLock> { + static F: OnceLock>> = OnceLock::new(); + F.get_or_init(|| RwLock::new(Vec::new())) +} + +/// Register a frame-renderer factory (tried before those registered earlier). Registering the +/// same factory twice is harmless. +pub fn register_frame_renderer(f: FrameRendererFactory) { + let mut g = frame_renderer_factories().write().unwrap_or_else(|e| e.into_inner()); + if !g.iter().any(|x| std::ptr::fn_addr_eq(*x, f)) { + g.insert(0, f); + } +} + +/// Whether `f` is among the registered frame-renderer factories (startup diagnostics, tests). +pub fn frame_renderer_registered(f: FrameRendererFactory) -> bool { + frame_renderer_factories().read().unwrap_or_else(|e| e.into_inner()).iter().any(|x| std::ptr::fn_addr_eq(*x, f)) +} + +/// Clear the registered frame-renderer factories. Test-only: registration is a process-wide +/// startup step and tests that register their own must not see each other's. +#[cfg(test)] +pub(crate) fn reset_frame_renderers_for_tests() { + frame_renderer_factories().write().unwrap_or_else(|e| e.into_inner()).clear(); + GPU_FRAMES.store(0, Ordering::Relaxed); + GPU_FALLBACKS.store(0, Ordering::Relaxed); +} + +/// Pick the first registered factory that can build a renderer for this host. +pub(crate) fn build_frame_renderer() -> Option> { + let g = frame_renderer_factories().read().unwrap_or_else(|e| e.into_inner()); + g.iter().filter_map(|f| f()).next() +} + +static GPU_FRAMES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static GPU_FALLBACKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// A frame was composited by the registered GPU renderer. +pub fn note_gpu_frame() { + GPU_FRAMES.fetch_add(1, Ordering::Relaxed); +} + +/// The GPU renderer declined a frame (the CPU reference renderer took it). +pub fn note_gpu_fallback() { + GPU_FALLBACKS.fetch_add(1, Ordering::Relaxed); +} + +/// GPU-vs-CPU frame counters of the export renderer (per-process, like the hardware ones). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GpuRenderStats { + pub frames: u64, + pub fallbacks: u64, +} + +pub fn gpu_render_stats() -> GpuRenderStats { + GpuRenderStats { frames: GPU_FRAMES.load(Ordering::Relaxed), fallbacks: GPU_FALLBACKS.load(Ordering::Relaxed) } +} pub fn register_audio_encoder(f: AudioEncoderFactory) { audio_factories().write().unwrap_or_else(|e| e.into_inner()).insert(0, f); } diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index fde70f83..6f744504 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -12,7 +12,7 @@ use filmcraft_time::FrameRate; use rayon::prelude::*; use crate::settings::{ExportEffects, Placement, Scaling, TextOverlay}; -use crate::{ExportError, ExportSettings, Result}; +use crate::{ExportError, ExportSettings, FrameRenderer, GpuRendering, Result, build_frame_renderer, note_gpu_fallback, note_gpu_frame}; /// Where the rendered picture lands in the output frame. #[derive(Clone, Copy, Debug, PartialEq)] @@ -52,6 +52,9 @@ impl Geometry { /// The immutable part of an export: everything needed to produce one output frame. pub(crate) struct Pipeline { pub project: Arc, + /// The registered frame renderer (GPU compositor) when the setting allows and a factory + /// provided one; guarded because `frame` runs on several export threads at once. + renderer: Option>>, pub seq: ItemId, /// Output frame rate (frame `f` is at `rate.tick_of(f)`). pub rate: FrameRate, @@ -94,7 +97,9 @@ impl Pipeline { } else { None }; + let renderer = if settings.gpu_rendering == GpuRendering::Off { None } else { build_frame_renderer().map(std::sync::Mutex::new) }; Ok(Pipeline { + renderer, seq_rate: q.settings.frame_rate, start_tc: q.start_timecode, drop_frame: q.settings.drop_frame, @@ -116,7 +121,27 @@ impl Pipeline { /// R'G'B' floats (3 per pixel). pub fn frame(&self, f: i64, sources: &dyn SourceProvider) -> (Vec, Vec) { let t = self.rate.tick_of(f); - let img = filmcraft_render::render_sequence(&self.project, self.seq, t, self.opts, sources); + // The registered GPU renderer goes first; `None` (no adapter, a plan the GPU cannot + // draw, any internal error) falls back to the CPU reference renderer below. + let gpu = match self.renderer.as_ref() { + Some(r) => { + let mut r = r.lock().unwrap_or_else(|e| e.into_inner()); + r.render(&self.project, self.seq, t, self.opts, sources) + } + None => None, + }; + let img = match gpu { + Some(img) => { + note_gpu_frame(); + img + } + None => { + if self.renderer.is_some() { + note_gpu_fallback(); + } + filmcraft_render::render_sequence(&self.project, self.seq, t, self.opts, sources) + } + }; let mut img = self.place(img); self.overlays(&mut img, t); let lim = &self.effects.video_limiter; diff --git a/crates/export/src/settings.rs b/crates/export/src/settings.rs index 91d89a16..71e77875 100644 --- a/crates/export/src/settings.rs +++ b/crates/export/src/settings.rs @@ -75,6 +75,18 @@ pub enum HardwareEncoding { Auto, } +/// Whether the picture of an export may be composited on the GPU (`filmcraft-gpu`'s off-screen +/// compositor) instead of the CPU reference renderer. Auto = use the GPU when the app registered +/// a GPU frame renderer and the machine has an adapter; the CPU result is the fallback either +/// way. Off = today's CPU behaviour exactly. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum GpuRendering { + #[default] + Auto, + Off, +} + /// Bitrate encoding of bitrate-driven codecs (H.264). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index 0ea6fb45..04614449 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -260,3 +260,134 @@ fn hdr_exports_signal_pq_and_hlg() { eprintln!("{path}:\n{txt}"); } } + +/// Scenario runner for the GPU hook: everything in one test because the renderer registry is +/// process-global and parallel tests would see each other's factories. +#[test] +fn gpu_hook_scenarios() { + // 1. Declining renderer: byte-equal to Off, and an empty registry too. + crate::reset_frame_renderers_for_tests(); + crate::register_frame_renderer(|| None); + let (p, seq, m) = project(); + let mk = |path: &String, gpu: Option| { + let mut s = ExportSettings { format: Format::PngSequence, path: path.clone(), ..Default::default() }; + s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(2))); + s.gpu_rendering = match gpu { + Some(true) | None => crate::GpuRendering::Auto, + Some(false) => crate::GpuRendering::Off, + }; + export(&p, seq, &s, &m, &Progress::default()).unwrap() + }; + mk(&tmp("off.png"), Some(false)); + mk(&tmp("auto-declined.png"), Some(true)); + assert_eq!(std::fs::read(tmp("off000.png")).unwrap(), std::fs::read(tmp("auto-declined000.png")).unwrap(), "declined GPU export must be byte-equal to Off"); + crate::reset_frame_renderers_for_tests(); + mk(&tmp("auto-empty.png"), Some(true)); + assert_eq!(std::fs::read(tmp("off000.png")).unwrap(), std::fs::read(tmp("auto-empty000.png")).unwrap(), "empty registry must behave like Off"); + + // 2. Producing renderer: counters up and its picture reaches the export. + crate::reset_frame_renderers_for_tests(); + crate::register_frame_renderer(|| Some(Box::new(FakeRenderer) as Box)); + mk(&tmp("fake-gpu.png"), Some(true)); + assert!(gpu_render_stats().frames >= 2, "gpu frames counted: {:?}", gpu_render_stats()); + assert_eq!(gpu_render_stats().fallbacks, 0); + mk(&tmp("fake-off.png"), Some(false)); + assert_eq!(gpu_render_stats().fallbacks, 0, "Off must not count as fallback"); + assert_ne!(std::fs::read(tmp("fake-gpu000.png")).unwrap(), std::fs::read(tmp("fake-off000.png")).unwrap(), "the fake GPU picture must reach the export"); + + // 3. Hostile and degenerate frame sizes must not panic with a renderer attached. + crate::reset_frame_renderers_for_tests(); + crate::register_frame_renderer(|| None); + for (w, h) in [(1, 1), (17, 13), (4096, 1)] { + let mut p = Project::new("x"); + let g = GeneratorSource::new(Generator::ColorMatte { color: [0.5, 0.5, 0.5, 1.0] }, 320, 180, FrameRate::FPS_24, Tick(2 * TICKS_PER_SECOND)); + let info = g.info().clone(); + let red = p.add_item( + &info.name.clone(), + Label::Iris, + ItemKind::Media(MediaClip { + media: MediaRef::Generator(g.generator.clone()), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }), + None, + ); + let seq = p.new_sequence("s", SequenceSettings { width: w, height: h, frame_rate: FrameRate::FPS_24, ..Default::default() }, 1, 1, None); + let v = p.make_track_item(red, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(2)), FrameRate::FPS_24).unwrap(); + p.sequence_mut(seq).unwrap().video_tracks[0].items.push(v); + let mut m = SourceMap::default(); + m.0.insert(red, Arc::new(g)); + let mut s = ExportSettings { format: Format::PngSequence, path: tmp("hostile.png"), ..Default::default() }; + s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(1))); + s.gpu_rendering = crate::GpuRendering::Auto; + let r = export(&Arc::new(p), seq, &s, &m, &Progress::default()); + if r.is_err() { + continue; // sizes the exporter rejects are fine too — as long as nothing panics + } + let bytes = std::fs::read(tmp("hostile000.png")).unwrap_or_default(); + assert!(!bytes.is_empty(), "{w}x{h}: PNG written"); + } +} + +struct FakeRenderer; + +impl FrameRenderer for FakeRenderer { + fn render( + &mut self, + _project: &Project, + _seq: ItemId, + _t: Tick, + _opts: filmcraft_render::RenderOptions, + _sources: &dyn SourceProvider, + ) -> Option { + Some(filmcraft_render::Image::new(320, 180)) + } +} + +/// Hostile and degenerate frame sizes must not panic with a renderer attached. +#[test] +fn gpu_hostile_frame_sizes() { + crate::reset_frame_renderers_for_tests(); + crate::register_frame_renderer(|| None); // declines: same path, no GPU involved + for (w, h) in [(1, 1), (17, 13), (4096, 1)] { + let mut p = Project::new("x"); + let g = GeneratorSource::new(Generator::ColorMatte { color: [0.5, 0.5, 0.5, 1.0] }, 320, 180, FrameRate::FPS_24, Tick(2 * TICKS_PER_SECOND)); + let info = g.info().clone(); + let red = p.add_item( + &info.name.clone(), + Label::Iris, + ItemKind::Media(MediaClip { + media: MediaRef::Generator(g.generator.clone()), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }), + None, + ); + let seq = p.new_sequence("s", SequenceSettings { width: w, height: h, frame_rate: FrameRate::FPS_24, ..Default::default() }, 1, 1, None); + let v = p.make_track_item(red, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(2)), FrameRate::FPS_24).unwrap(); + p.sequence_mut(seq).unwrap().video_tracks[0].items.push(v); + let mut m = SourceMap::default(); + m.0.insert(red, Arc::new(g)); + let mut s = ExportSettings { format: Format::PngSequence, path: tmp("hostile.png"), ..Default::default() }; + s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(1))); + s.gpu_rendering = crate::GpuRendering::Auto; + let r = export(&Arc::new(p), seq, &s, &m, &Progress::default()); + if r.is_err() { + continue; // sizes the exporter rejects are fine too — as long as nothing panics + } + let bytes = std::fs::read(tmp("hostile000.png")).unwrap_or_default(); + assert!(!bytes.is_empty(), "{w}x{h}: PNG written"); + } +} diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml index a402c61c..f0468d5a 100644 --- a/crates/gpu/Cargo.toml +++ b/crates/gpu/Cargo.toml @@ -13,13 +13,18 @@ workspace = true filmcraft-color = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-geom = { workspace = true } +filmcraft-project = { workspace = true } filmcraft-render = { workspace = true } +filmcraft-time = { workspace = true } wgpu = { version = "30", default-features = false, features = ["wgsl"] } log = { workspace = true } +pollster = "0.4" rayon = { workspace = true } [dev-dependencies] pollster = "0.4" +filmcraft-export = { workspace = true } +image = { version = "0.25", default-features = false, features = ["png"] } wgpu = { version = "30", features = ["wgsl", "metal", "vulkan", "dx12", "gles"] } filmcraft-media = { workspace = true } filmcraft-project = { workspace = true } diff --git a/crates/gpu/src/export_renderer.rs b/crates/gpu/src/export_renderer.rs new file mode 100644 index 00000000..082d1a2d --- /dev/null +++ b/crates/gpu/src/export_renderer.rs @@ -0,0 +1,209 @@ +//! Off-screen GPU export renderer: composite a frame with the same +//! [`filmcraft_render::plan::plan_frame`] + [`GpuCompositor`] path the monitors use, then read +//! the **linear float accumulator** back (not the sRGB display texture), so the result matches +//! `filmcraft_render::render_sequence`'s premultiplied linear-light RGBA and the export pipeline's +//! placement, overlays and video limiter see exactly what the CPU renderer would have produced. +//! +//! The renderer owns its own wgpu instance/adapter/device (no window). With no adapter (headless +//! machine, unsupported GPU) [`ExportRenderer::new`] returns `None` and the export falls back to +//! the CPU renderer; a `FramePlan::Image` plan (HDR / wide gamut / anything the plan sends to the +//! CPU) is returned unchanged — it already is the CPU image. + +use crate::{GpuCompositor, prepare}; +use filmcraft_project::{ItemId, Project}; +use filmcraft_render::plan::{FramePlan, plan_frame}; +use filmcraft_render::{Image, RenderOptions, SourceProvider}; +use filmcraft_time::Tick; + +pub struct ExportRenderer { + _instance: wgpu::Instance, + #[expect(dead_code, reason = "kept for future frame-accuracy flows (readback fences)")] + device: wgpu::Device, + #[expect(dead_code, reason = "kept for future frame-accuracy flows (readback fences)")] + queue: wgpu::Queue, + compositor: GpuCompositor, +} + +impl ExportRenderer { + /// The off-screen device, or `None` when this host has no usable adapter. + pub fn new() -> Option { + let instance = wgpu::Instance::default(); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?; + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()?; + let compositor = GpuCompositor::new(&device, &queue); + Some(Self { _instance: instance, device, queue, compositor }) + } + + /// One frame as `render_sequence` would return it (premultiplied linear-light RGBA at the + /// render size), or `None` when this frame must go to the CPU renderer. + pub fn render(&mut self, project: &Project, seq: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> Option { + let plan = plan_frame(project, seq, t, opts, sources); + match &plan { + // Already the CPU image: return it unchanged instead of rendering it twice. + FramePlan::Image(img) => Some(img.clone()), + FramePlan::Layers { .. } => { + let prep = prepare(&plan); + self.compositor.composite_prepared(&plan, Some(&prep)); + let (w, h, px) = self.compositor.read_accumulator()?; + Some(Image { w: w as usize, h: h as usize, px }) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{f16_to_f32, f32_to_f16}; + use filmcraft_export::FrameRenderer; + use filmcraft_media::generators::GeneratorSource; + use filmcraft_media::{Generator, MediaSource}; + use filmcraft_project::{ItemKind, Label, MediaClip, MediaRef, Project, SequenceSettings, TrackKind}; + use filmcraft_render::SourceMap; + use filmcraft_time::{TICKS_PER_SECOND, Tick, TimeRange}; + use std::sync::Arc; + + const EPS: f32 = 1e-4; + + /// A solid colour generator at 320×180 plus a tone generator (mirrors the export tests' + /// fixture-free project helper). + fn project(transform: bool) -> (Arc, ItemId, SourceMap) { + let mut p = Project::new("x"); + let g = GeneratorSource::new( + Generator::ColorMatte { color: [0.2, 0.5, 0.9, 1.0] }, + 320, + 180, + filmcraft_time::FrameRate::FPS_24, + Tick(2 * TICKS_PER_SECOND), + ); + let tone = GeneratorSource::new(Generator::Tone { hz: 440.0, db: -6.0 }, 320, 180, filmcraft_time::FrameRate::FPS_24, Tick(2 * TICKS_PER_SECOND)); + let add = |p: &mut Project, g: &GeneratorSource| { + let info = g.info().clone(); + p.add_item( + &info.name.clone(), + Label::Iris, + ItemKind::Media(MediaClip { + media: MediaRef::Generator(g.generator.clone()), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }), + None, + ) + }; + let red = add(&mut p, &g); + let t = add(&mut p, &tone); + let seq = + p.new_sequence("s", SequenceSettings { width: 320, height: 180, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 1, 1, None); + let r = filmcraft_time::FrameRate::FPS_24; + let _v = p.make_track_item(red, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, r.tick_of(24)), r).unwrap(); + let a = p.make_track_item(t, TrackKind::Audio, Tick::ZERO, TimeRange::new(Tick::ZERO, r.tick_of(24)), r).unwrap(); + if transform { + // Motion + opacity as fixed effects (the way the golden transform scene does it). + if let Some(clip) = p.sequence_mut(seq).unwrap().video_tracks[0].items.last_mut() { + let m = clip.effect_mut("motion").unwrap(); + if let Some(pm) = m.param_mut("scale") { + pm.value = filmcraft_project::ParamValue::Float(45.0); + } + if let Some(pm) = m.param_mut("rotation") { + pm.value = filmcraft_project::ParamValue::Float(20.0); + } + if let Some(pm) = m.param_mut("position") { + pm.value = filmcraft_project::ParamValue::Vec2(filmcraft_geom::Vec2::new(215.0, 70.0)); + } + if let Some(o) = clip.effect_mut("opacity") + && let Some(pm) = o.param_mut("opacity") + { + pm.value = filmcraft_project::ParamValue::Float(70.0); + } + } + } + p.sequence_mut(seq).unwrap().audio_tracks[0].items.push(a); + let mut m = SourceMap::default(); + m.0.insert(red, Arc::new(g)); + m.0.insert(t, Arc::new(tone)); + (Arc::new(p), seq, m) + } + + /// GPU parity: p99 of the per-pixel max RGB difference (over black, 8-bit) ≤ 6 and mean < 1.5 + /// — the same criterion the golden GPU-vs-CPU parity tests use. + fn assert_parity(cpu: &Image, gpu: &Image, what: &str) { + assert_eq!((cpu.w, cpu.h), (gpu.w, gpu.h), "{what}: size mismatch"); + let to8 = |img: &Image| -> Vec { img.over_black_rgba8() }; + let (a, b) = (to8(cpu), to8(gpu)); + let mut d: Vec = a + .as_chunks::<4>() + .0 + .iter() + .zip(b.as_chunks::<4>().0) + .map(|(x, y)| (0..3).map(|k| (x[k] as u32).abs_diff(y[k] as u32)).max().unwrap_or(0)) + .collect(); + d.sort_unstable(); + let p99 = d[d.len() * 99 / 100]; + let mean = d.iter().sum::() as f64 / d.len() as f64; + eprintln!("{what}: GPU vs CPU over-black p99 {p99}, mean {mean:.3}"); + assert!(p99 <= 6 && mean < 1.5, "{what}: GPU parity p99 {p99}, mean {mean:.3}"); + } + + #[test] + fn solid_frame_matches_cpu() { + let Some(mut r) = ExportRenderer::new() else { + eprintln!("SKIPPED (gpu export parity): no adapter"); + return; + }; + let (p, seq, m) = project(false); + let t = Tick::ZERO; + let cpu = filmcraft_render::render_sequence(&p, seq, t, RenderOptions::default(), &m); + let gpu = r.render(&p, seq, t, RenderOptions::default(), &m).expect("GPU rendered the frame"); + // The linear premultiplied accumulator should be very close to the CPU float image. + assert_eq!((cpu.w, cpu.h), (gpu.w, gpu.h)); + let max: f32 = cpu.px.iter().zip(&gpu.px).map(|(a, b)| (a - b).abs()).fold(0.0, f32::max); + eprintln!("solid: max |GPU-CPU| linear {max:.6}"); + assert!(max < 0.02, "solid frame differs by {max}"); + assert_parity(&cpu, &gpu, "solid"); + } + + #[test] + fn transformed_frame_matches_cpu() { + let Some(mut r) = ExportRenderer::new() else { + eprintln!("SKIPPED (gpu export parity): no adapter"); + return; + }; + let (p, seq, m) = project(true); + let t = filmcraft_time::FrameRate::FPS_24.tick_of(10); + let cpu = filmcraft_render::render_sequence(&p, seq, t, RenderOptions::default(), &m); + let gpu = r.render(&p, seq, t, RenderOptions::default(), &m).expect("GPU rendered the frame"); + assert_parity(&cpu, &gpu, "transformed"); + } + + #[test] + fn gpu_render_impls_frame_renderer() { + // dev-dependency: the same wrapper the app and CLI register. + struct R(ExportRenderer); + impl FrameRenderer for R { + fn render(&mut self, project: &Project, seq: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> Option { + self.0.render(project, seq, t, opts, sources) + } + } + let Some(r) = ExportRenderer::new() else { + eprintln!("SKIPPED (gpu export parity): no adapter"); + return; + }; + let (p, seq, m) = project(false); + let gpu = R(r).render(&p, seq, Tick::ZERO, RenderOptions::default(), &m).expect("rendered"); + let cpu = filmcraft_render::render_sequence(&p, seq, Tick::ZERO, RenderOptions::default(), &m); + assert_parity(&cpu, &gpu, "frame renderer"); + } + + #[test] + fn f16_roundtrip() { + for v in [0.0f32, 1.0, 0.5, 2.0, 1e-3, 65504.0, -1.0, -0.25] { + assert!((f16_to_f32(f32_to_f16(v)) - v).abs() <= EPS * v.abs().max(1.0), "f16 {v}"); + } + } +} diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index f71db2b6..b9379998 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -38,9 +38,11 @@ use filmcraft_frame::{Chroma, PixelData, VideoFrame}; use filmcraft_render::Blend; use filmcraft_render::plan::{FramePlan, PlanLayer}; +pub mod export_renderer; pub mod fx; pub mod lut; pub mod mask; +pub use export_renderer::ExportRenderer; pub use lut::GpuLut; pub use mask::GpuMask; @@ -129,6 +131,28 @@ pub fn f32_to_f16(v: f32) -> u16 { } h as u16 } +/// IEEE half → f32 (the inverse of [`f32_to_f16`]), for reading `Rgba16Float` textures back. +pub fn f16_to_f32(h: u16) -> f32 { + let sign = ((h & 0x8000) as u32) << 16; + let exp = ((h >> 10) & 0x1f) as u32; + let mant = (h & 0x03ff) as u32; + let bits = match exp { + 0 => { + if mant == 0 { + sign + } else { + // Half subnormal: value = mant × 2⁻²⁴ = (1 + frac) × 2^(b−24) with b the mantissa's + // highest set bit — renormalise into an f32. + let b = 31 - mant.leading_zeros(); + let frac = (mant ^ (1 << b)) << (23 - b); + sign | ((103 + b) << 23) | frac + } + } + 0x1f => sign | 0x7f80_0000 | (mant << 13), + _ => sign | ((exp + 112) << 23) | (mant << 13), + }; + f32::from_bits(bits) +} /// Half-float texel data for one frame (one byte vector per plane), converted off the UI thread. /// 8-bit frames need no conversion and have none. @@ -849,6 +873,44 @@ impl GpuCompositor { Some((w, h, out)) } + /// Read the **linear float accumulator** back as premultiplied RGBA f32 (export rendering): + /// the display texture this crate's [`read_output`](Self::read_output) resolves is sRGB 8-bit + /// and would clip the working-space data HDR exports need. The accumulator must have been + /// composited first ([`composite`](Self::composite) / [`composite_prepared`](Self::composite_prepared)). + pub fn read_accumulator(&self) -> Option<(u32, u32, Vec)> { + let (tex, _, (w, h)) = self.accum.as_ref()?; + let row = (w * 8).div_ceil(256) * 256; + let buf = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("accum-readback"), + size: (row * h) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mut enc = self.device.create_command_encoder(&Default::default()); + enc.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { texture: tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyBufferInfo { buffer: &buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(*h) } }, + wgpu::Extent3d { width: *w, height: *h, depth_or_array_layers: 1 }, + ); + self.queue.submit([enc.finish()]); + let slice = buf.slice(..); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); + rx.recv().ok()?.ok()?; + let data = slice.get_mapped_range().ok()?; + let mut out = Vec::with_capacity((w * h * 4) as usize); + for y in 0..*h { + let r = data.get((y * row) as usize..(y * row + w * 8) as usize)?; + out.extend(r.as_chunks::<2>().0.iter().map(|b| f16_to_f32(u16::from_le_bytes(*b)))); + } + drop(data); + buf.unmap(); + Some((*w, *h, out)) + } + /// Read the output back as RGBA8 (tests / screenshots / thumbnails). pub fn read_output(&self) -> Option<(u32, u32, Vec)> { let (tex, _, (w, h)) = self.output.as_ref()?; diff --git a/crates/gpu/tests/export_pipeline.rs b/crates/gpu/tests/export_pipeline.rs new file mode 100644 index 00000000..94c21182 --- /dev/null +++ b/crates/gpu/tests/export_pipeline.rs @@ -0,0 +1,134 @@ +//! End-to-end GPU export parity: the registered `filmcraft-gpu` frame renderer drives the full +//! export pipeline (`filmcraft_export::export` → PNG sequence) and its frames must match the +//! `GpuRendering::Off` (CPU reference) frames within the GPU-vs-CPU parity tolerance — p99 of the +//! per-pixel max RGB difference over black ≤ 6 and mean < 1.5 (the golden-test criterion). + +use filmcraft_export::{ExportSettings, Format, FrameRenderer, GpuRenderStats, GpuRendering, Progress, export, register_frame_renderer}; +use filmcraft_media::generators::GeneratorSource; +use filmcraft_media::{Generator, MediaSource}; +use filmcraft_project::{ItemKind, Label, MediaClip, MediaRef, SequenceSettings, TrackKind}; +use filmcraft_render::SourceMap; +use filmcraft_time::{TICKS_PER_SECOND, TimeRange}; +use std::sync::Arc; + +/// The same wrapper the app and CLI register: `filmcraft-gpu`'s renderer behind +/// `filmcraft_export`'s trait. +struct GpuFrameRenderer(filmcraft_gpu::ExportRenderer); + +impl FrameRenderer for GpuFrameRenderer { + fn render( + &mut self, + project: &filmcraft_project::Project, + seq: filmcraft_project::ItemId, + t: filmcraft_time::Tick, + opts: filmcraft_render::RenderOptions, + sources: &dyn filmcraft_render::SourceProvider, + ) -> Option { + self.0.render(project, seq, t, opts, sources) + } +} + +fn project(width: u32, height: u32) -> (Arc, filmcraft_project::ItemId, SourceMap) { + let mut p = filmcraft_project::Project::new("x"); + let g = GeneratorSource::new( + Generator::ColorMatte { color: [0.2, 0.5, 0.9, 1.0] }, + 320, + 180, + filmcraft_time::FrameRate::FPS_24, + filmcraft_time::Tick(2 * TICKS_PER_SECOND), + ); + let info = g.info().clone(); + let red = p.add_item( + &info.name.clone(), + Label::Iris, + ItemKind::Media(MediaClip { + media: MediaRef::Generator(g.generator.clone()), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }), + None, + ); + let seq = p.new_sequence("s", SequenceSettings { width, height, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 1, 1, None); + let v = p + .make_track_item( + red, + TrackKind::Video, + filmcraft_time::Tick::ZERO, + TimeRange::new(filmcraft_time::Tick::ZERO, filmcraft_time::FrameRate::FPS_24.tick_of(2)), + filmcraft_time::FrameRate::FPS_24, + ) + .unwrap(); + p.sequence_mut(seq).unwrap().video_tracks[0].items.push(v); + let mut m = SourceMap::default(); + m.0.insert(red, Arc::new(g)); + (Arc::new(p), seq, m) +} + +fn png(dir: &str, name: &str) -> Vec { + // image_sequence_path flattens the path stem: .png in the temp dir itself. + let index: u64 = name.trim_start_matches("img").trim_end_matches(".png").parse().unwrap_or(0); + std::fs::read(std::env::temp_dir().join(format!("fc-gpu-export-{dir}{index:03}.png"))).unwrap() +} + +fn export_png(dir: &str, gpu: GpuRendering, (w, h): (u32, u32)) { + let (p, seq, m) = project(w, h); + let mut s = ExportSettings { + format: Format::PngSequence, + path: std::env::temp_dir().join(format!("fc-gpu-export-{dir}")).to_string_lossy().into_owned(), + ..Default::default() + }; + s.range = Some(TimeRange::new(filmcraft_time::Tick::ZERO, filmcraft_time::FrameRate::FPS_24.tick_of(2))); + s.gpu_rendering = gpu; + export(&p, seq, &s, &m, &Progress::default()).unwrap(); +} + +fn assert_parity(cpu_png: &[u8], gpu_png: &[u8], what: &str) { + let a = image::load_from_memory(cpu_png).unwrap().to_rgba8(); + let b = image::load_from_memory(gpu_png).unwrap().to_rgba8(); + assert_eq!(a.dimensions(), b.dimensions(), "{what}: size mismatch"); + let d: Vec = a + .as_raw() + .as_chunks::<4>() + .0 + .iter() + .zip(b.as_raw().as_chunks::<4>().0) + .map(|(x, y)| (0..3).map(|k| (x[k] as u32).abs_diff(y[k] as u32)).max().unwrap_or(0)) + .collect(); + let mut d = d; + d.sort_unstable(); + let p99 = d[d.len() * 99 / 100]; + let mean = d.iter().sum::() as f64 / d.len() as f64; + eprintln!("{what}: GPU vs CPU p99 {p99}, mean {mean:.3}"); + assert!(p99 <= 6 && mean < 1.5, "{what}: GPU parity p99 {p99}, mean {mean:.3}"); +} + +#[test] +fn gpu_export_matches_cpu_and_counts() { + let Some(_) = filmcraft_gpu::ExportRenderer::new() else { + eprintln!("SKIPPED (gpu export pipeline): no adapter"); + return; + }; + // Clean registry first: Auto with no factory must be byte-equal to Off. + export_png("off-cpu", GpuRendering::Off, (320, 180)); + export_png("off-auto", GpuRendering::Auto, (320, 180)); + assert_eq!(png("off-cpu", "img000.png"), png("off-auto", "img000.png"), "Auto with no registered renderer must be byte-equal to Off"); + + register_frame_renderer(|| Some(Box::new(GpuFrameRenderer(filmcraft_gpu::ExportRenderer::new().expect("adapter present"))))); + let sizes = [(320u32, 180u32), (17, 13), (1, 1), (257, 129)]; + for (i, size) in sizes.into_iter().enumerate() { + export_png("cpu", GpuRendering::Off, size); + export_png("gpu", GpuRendering::Auto, size); + for f in ["000", "001"] { + assert_parity(&png("cpu", &format!("img{f}.png")), &png("gpu", &format!("img{f}.png")), &format!("{:?} frame {f}", size)); + } + let _ = i; + } + let GpuRenderStats { frames, .. } = filmcraft_export::gpu_render_stats(); + assert!(frames >= 8, "gpu frames counted across the parity exports: {frames}"); +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index c6ee15d6..d91bb7e5 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -21,6 +21,7 @@ filmcraft-render = { workspace = true } filmcraft-text = { workspace = true } filmcraft-audio-dsp = { workspace = true } filmcraft-engine = { workspace = true } +filmcraft-export = { workspace = true } filmcraft-gpu = { workspace = true } filmcraft-scopes = { workspace = true } egui = { workspace = true } diff --git a/crates/ui-egui/examples/bench/main.rs b/crates/ui-egui/examples/bench/main.rs index 1729745b..3a9f0e10 100644 --- a/crates/ui-egui/examples/bench/main.rs +++ b/crates/ui-egui/examples/bench/main.rs @@ -332,6 +332,8 @@ pub fn section_markdown(v: &Value) -> String { ("cpu_ms_per_frame", "CPU ms/frame"), ("mbytes", "MB"), ("hw_frames", "hw frames"), + ("gpu_frames", "gpu frames"), + ("gpu_fallbacks", "gpu fallbacks"), ("load", "load"), ], 2, diff --git a/crates/ui-egui/examples/bench/playback.rs b/crates/ui-egui/examples/bench/playback.rs index 868fe68c..a0228982 100644 --- a/crates/ui-egui/examples/bench/playback.rs +++ b/crates/ui-egui/examples/bench/playback.rs @@ -217,6 +217,12 @@ pub fn ms(d: Duration) -> f64 { } pub fn load_avg() -> String { + if let Ok(s) = std::fs::read_to_string("/proc/loadavg") { + let parts: Vec<&str> = s.split_whitespace().take(3).collect(); + if !parts.is_empty() { + return parts.join(", "); + } + } Command::new("sysctl") .args(["-n", "vm.loadavg"]) .output() diff --git a/crates/ui-egui/examples/bench/sections.rs b/crates/ui-egui/examples/bench/sections.rs index 25438e5c..7953a763 100644 --- a/crates/ui-egui/examples/bench/sections.rs +++ b/crates/ui-egui/examples/bench/sections.rs @@ -13,6 +13,26 @@ use crate::Opts; use crate::fixtures::fixture; use crate::playback::{self, Bench, Display, load_avg, ms, pct}; +/// The GPU export frame renderer factory the bench registers for the export section (the app and +/// CLI register the same thing at startup): filmcraft-gpu's off-screen compositor behind +/// filmcraft-export's frame-renderer hook. +fn gpu_export_frame_renderer_factory() -> Option> { + struct R(filmcraft_gpu::ExportRenderer); + impl filmcraft_export::FrameRenderer for R { + fn render( + &mut self, + project: &filmcraft_project::Project, + seq: ItemId, + t: Tick, + opts: filmcraft_render::RenderOptions, + sources: &dyn filmcraft_render::SourceProvider, + ) -> Option { + self.0.render(project, seq, t, opts, sources) + } + } + filmcraft_gpu::ExportRenderer::new().map(|r| Box::new(R(r)) as Box) +} + fn median(v: &[f64]) -> f64 { pct(v, 0.5) } @@ -443,6 +463,11 @@ pub fn export(o: &Opts) -> Vec { let mut bytes = 0u64; let mut frames = 0i64; let hw0 = filmcraft_engine::export::hw_encode_stats(); + let gpu0 = filmcraft_engine::export::gpu_render_stats(); + // The engine's headless session does not register the app's GPU export renderer; the bench + // does it here so `--gpu-rendering auto` measures the GPU path (and the row reports the + // frames it actually rendered on the GPU). + filmcraft_engine::export::register_frame_renderer(gpu_export_frame_renderer_factory); for _ in 0..o.repeat { let mut s = Session::default(); let a = playback::import(&mut s, &path); @@ -462,7 +487,7 @@ pub fn export(o: &Opts) -> Vec { let (t0, c0) = (Instant::now(), cpu_now()); // `--hw auto`: the hardware encoder too (NVENC H.264 on Windows); `--hw off`: ours let hardware = if o.hw == "off" { "off" } else { "auto" }; - let r = s.execute("file.exportMedia", json!({"path": out.to_string_lossy(), "format": format, "hardwareEncoding": hardware, "wait": true})); + let r = s.execute("file.exportMedia", json!({"path": out.to_string_lossy(), "format": format, "hardwareEncoding": hardware, "gpuRendering": if o.hw == "off" { "off" } else { "auto" }, "wait": true})); let dt = t0.elapsed().as_secs_f64(); if let Err(e) = r { eprintln!("export {format}: {e}"); @@ -487,6 +512,9 @@ pub fn export(o: &Opts) -> Vec { "mbytes": bytes as f64 / 1e6, "runs_s": runs, "load": load_avg(), // pictures encoded by a hardware encoder during this row (zero with --hw off or where there is none) "hw_frames": filmcraft_engine::export::hw_encode_stats().frames.saturating_sub(hw0.frames), + // frames composited by the GPU export renderer (zero with --gpu-rendering off or where there is no adapter) + "gpu_frames": filmcraft_engine::export::gpu_render_stats().frames.saturating_sub(gpu0.frames), + "gpu_fallbacks": filmcraft_engine::export::gpu_render_stats().fallbacks.saturating_sub(gpu0.fallbacks), })); } let _ = std::fs::remove_dir_all(&dir); diff --git a/crates/ui-egui/src/panels/export_mode.rs b/crates/ui-egui/src/panels/export_mode.rs index 23d3d7e9..ead9b32b 100644 --- a/crates/ui-egui/src/panels/export_mode.rs +++ b/crates/ui-egui/src/panels/export_mode.rs @@ -22,8 +22,8 @@ use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; use filmcraft_engine::export::presets::{DEFAULT_PRESET, preset_key}; use filmcraft_engine::export::{ - AudioCodec, BitrateMode, ExportSettings, FieldOrder, Format, H264Profile, HardwareEncoding, Multiplexer, MxfVideoCodec, Placement, Scaling, TextOverlay, - builtin_presets, format_bytes, + AudioCodec, BitrateMode, ExportSettings, FieldOrder, Format, GpuRendering, H264Profile, HardwareEncoding, Multiplexer, MxfVideoCodec, Placement, Scaling, + TextOverlay, builtin_presets, format_bytes, }; use filmcraft_engine::time::{FrameRate, Tick}; use serde::{Deserialize, Serialize}; @@ -719,6 +719,14 @@ fn video_section(ui: &mut egui::Ui, reg: &mut Reg, s: &mut ExportSettings, t: &T } _ => {} } + // Composite the picture on the GPU (filmcraft-gpu) instead of the CPU reference renderer; + // Auto falls back to the CPU wherever the GPU cannot render the frame. + row(ui, t, "GPU Rendering", |ui| { + let mut on = s.gpu_rendering == GpuRendering::Auto; + if check(ui, reg, "export.gpuRendering", &mut on, "Composite on the GPU when available") { + s.gpu_rendering = if on { GpuRendering::Auto } else { GpuRendering::Off }; + } + }); row(ui, t, "", |ui| check(ui, reg, "export.video.maxDepth", &mut s.render_at_max_depth, "Render at Maximum Depth")); row(ui, t, "", |ui| check(ui, reg, "export.video.maxQuality", &mut s.max_render_quality, "Use Maximum Render Quality")); } From ba403212f6d01ed9efbb9a23fba2246b4f6837ee Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 10:32:02 -0700 Subject: [PATCH 02/12] GPU3: GPU export rendering off by default until it is faster (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set GpuRendering default to Off in ExportSettings so exports use the CPU reference renderer by default until GPU export achieves lower wall-clock time. 🤖 Written by agy (Antigravity Agent, Gemini 3.8 Flash) --- crates/export/src/lib.rs | 2 +- crates/export/src/settings.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 81f924b7..a92978a0 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -458,7 +458,7 @@ impl Default for ExportSettings { h264_level: None, bitrate_mode: BitrateMode::default(), hardware_encoding: HardwareEncoding::default(), - gpu_rendering: GpuRendering::Auto, + gpu_rendering: GpuRendering::Off, max_bitrate_kbps: None, adaptive_bitrate: None, keyframe_distance: None, diff --git a/crates/export/src/settings.rs b/crates/export/src/settings.rs index 71e77875..87587ba4 100644 --- a/crates/export/src/settings.rs +++ b/crates/export/src/settings.rs @@ -78,12 +78,12 @@ pub enum HardwareEncoding { /// Whether the picture of an export may be composited on the GPU (`filmcraft-gpu`'s off-screen /// compositor) instead of the CPU reference renderer. Auto = use the GPU when the app registered /// a GPU frame renderer and the machine has an adapter; the CPU result is the fallback either -/// way. Off = today's CPU behaviour exactly. +/// way. Off = today's CPU behaviour exactly (default until GPU export is faster than CPU export). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum GpuRendering { - #[default] Auto, + #[default] Off, } From 91056340baa2b18c29ff5a52c14a72d81fc6e51c Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 13:20:49 -0700 Subject: [PATCH 03/12] GPU3b: pooled GPU export renderers, one-submission read-back (#30) GPU export rendering was ~20 % slower in wall-clock time than CPU rendering: one renderer behind a mutex made the export workers queue. - Export workers share a pool of up to four renderers on one wgpu device (one per worker was no faster: the read-back then contends for memory bandwidth). - render_export_prepared composites into the linear accumulator and copies it to a reused staging buffer in one submission (no sRGB resolve pass); read_accumulator shares the read-back, which now returns None instead of stale pooled values when a row is missing. - f16_to_f32_fast: same result as f16_to_f32 for all 65536 inputs (test). - Timing counters (export::export_lock_wait, gpu::export_renderer::timing) printed by the export bench. RX 7900, same build, Off / Auto / Off: H.264 + AAC 4.67 / 4.71 / 4.76 s, ProRes 422 4.20 / 4.21 / 4.21 s; CPU per frame 287-292 -> 277 (H.264), 316-317 -> 300 (ProRes). Was 4.86 -> 5.78 s. Parity, not faster: on this single-clip scene decoding and encoding dominate (rendering is ~15 of ~290 CPU ms per frame), so GPU rendering stays Off by default. Gates: fmt, clippy (gpu, export, ui-egui), layers, assets; gpu export_pipeline, golden and export tests pass. Pre-existing on the previous commit: fx_tests::gpu_effects_match_cpu_exactly fails on this AMD GPU, and the gpu unit tests segfault in the driver when run in parallel (pass serially apart from that test). Co-Authored-By: Claude Opus 5.5 --- crates/export/src/lib.rs | 16 +++ crates/export/src/pipeline.rs | 64 +++++++++-- crates/gpu/src/export_renderer.rs | 77 ++++++++++++- crates/gpu/src/lib.rs | 134 +++++++++++++++++----- crates/gpu/src/tests.rs | 8 ++ crates/ui-egui/examples/bench/sections.rs | 18 +++ 6 files changed, 274 insertions(+), 43 deletions(-) diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index a92978a0..e94756eb 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -732,6 +732,7 @@ pub(crate) fn build_frame_renderer() -> Option> { static GPU_FRAMES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); static GPU_FALLBACKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +static LOCK_WAIT_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); /// A frame was composited by the registered GPU renderer. pub fn note_gpu_frame() { @@ -743,6 +744,21 @@ pub fn note_gpu_fallback() { GPU_FALLBACKS.fetch_add(1, Ordering::Relaxed); } +/// Time an export worker waited for a free GPU renderer (process-wide, for the bench). +pub(crate) fn note_lock_wait(d: std::time::Duration) { + LOCK_WAIT_NS.fetch_add(u64::try_from(d.as_nanos()).unwrap_or(u64::MAX), Ordering::Relaxed); +} + +/// Total time export workers waited for a GPU renderer since [`reset_export_lock_wait`]. +pub fn export_lock_wait() -> std::time::Duration { + std::time::Duration::from_nanos(LOCK_WAIT_NS.load(Ordering::Relaxed)) +} + +/// Start [`export_lock_wait`] again from zero. +pub fn reset_export_lock_wait() { + LOCK_WAIT_NS.store(0, Ordering::Relaxed); +} + /// GPU-vs-CPU frame counters of the export renderer (per-process, like the hardware ones). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct GpuRenderStats { diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index 6f744504..9ffbecb6 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -49,12 +49,43 @@ impl Geometry { } } +/// GPU renderers shared by the export workers: a worker takes a free one for a frame and puts it +/// back (a single renderer behind a mutex made the workers queue, GPU export ~20 % slower than CPU). +struct RendererPool { + renderers: std::sync::Mutex>>, + available: std::sync::Condvar, +} + +impl RendererPool { + /// `None` for an empty list (a pool nobody could take from). + fn new(renderers: Vec>) -> Option { + (!renderers.is_empty()).then(|| Self { renderers: std::sync::Mutex::new(renderers), available: std::sync::Condvar::new() }) + } + + /// Run `f` with a free renderer, waiting for one if all are busy. + fn with(&self, f: impl FnOnce(&mut dyn FrameRenderer) -> R) -> Option { + let mut free = self.renderers.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut r = loop { + match free.pop() { + Some(r) => break r, + None => free = self.available.wait(free).unwrap_or_else(std::sync::PoisonError::into_inner), + } + }; + drop(free); + // the renderer goes back even when `f` panics (the export worker runs under catch_unwind) + let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(&mut *r))); + self.renderers.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(r); + self.available.notify_one(); + res.ok() + } +} + /// The immutable part of an export: everything needed to produce one output frame. pub(crate) struct Pipeline { pub project: Arc, - /// The registered frame renderer (GPU compositor) when the setting allows and a factory - /// provided one; guarded because `frame` runs on several export threads at once. - renderer: Option>>, + /// The registered frame renderers (GPU compositors) when the setting allows and factories + /// provide them; pooled because `frame` runs on several export threads at once. + renderer: Option, pub seq: ItemId, /// Output frame rate (frame `f` is at `rate.tick_of(f)`). pub rate: FrameRate, @@ -97,7 +128,22 @@ impl Pipeline { } else { None }; - let renderer = if settings.gpu_rendering == GpuRendering::Off { None } else { build_frame_renderer().map(std::sync::Mutex::new) }; + let renderer = if settings.gpu_rendering == GpuRendering::Off { + None + } else { + // Four renderers: as fast as one per worker (measured on 16 threads, where more only + // contend for memory bandwidth in the read-back), with a quarter of the GPU memory. + let count = rayon::current_num_threads().clamp(2, 4); + let mut list = Vec::with_capacity(count); + for _ in 0..count { + if let Some(r) = build_frame_renderer() { + list.push(r); + } else { + break; + } + } + RendererPool::new(list) + }; Ok(Pipeline { renderer, seq_rate: q.settings.frame_rate, @@ -124,9 +170,13 @@ impl Pipeline { // The registered GPU renderer goes first; `None` (no adapter, a plan the GPU cannot // draw, any internal error) falls back to the CPU reference renderer below. let gpu = match self.renderer.as_ref() { - Some(r) => { - let mut r = r.lock().unwrap_or_else(|e| e.into_inner()); - r.render(&self.project, self.seq, t, self.opts, sources) + Some(pool) => { + let asked = std::time::Instant::now(); + pool.with(|r| { + crate::note_lock_wait(asked.elapsed()); + r.render(&self.project, self.seq, t, self.opts, sources) + }) + .flatten() } None => None, }; diff --git a/crates/gpu/src/export_renderer.rs b/crates/gpu/src/export_renderer.rs index 082d1a2d..af4696e1 100644 --- a/crates/gpu/src/export_renderer.rs +++ b/crates/gpu/src/export_renderer.rs @@ -4,7 +4,9 @@ //! `filmcraft_render::render_sequence`'s premultiplied linear-light RGBA and the export pipeline's //! placement, overlays and video limiter see exactly what the CPU renderer would have produced. //! -//! The renderer owns its own wgpu instance/adapter/device (no window). With no adapter (headless +//! Renderers share one off-screen wgpu device (no window), created by the first and kept for the +//! process, so an export can run several (one per export worker; each has its own compositor and +//! textures) without opening a device each. With no adapter (headless //! machine, unsupported GPU) [`ExportRenderer::new`] returns `None` and the export falls back to //! the CPU renderer; a `FramePlan::Image` plan (HDR / wide gamut / anything the plan sends to the //! CPU) is returned unchanged — it already is the CPU image. @@ -15,6 +17,61 @@ use filmcraft_render::plan::{FramePlan, plan_frame}; use filmcraft_render::{Image, RenderOptions, SourceProvider}; use filmcraft_time::Tick; +/// Where export rendering spends its time (process-wide totals, for the bench). +pub mod timing { + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::Duration; + + static SUBMIT_NS: AtomicU64 = AtomicU64::new(0); + static MAP_WAIT_NS: AtomicU64 = AtomicU64::new(0); + static CONVERT_NS: AtomicU64 = AtomicU64::new(0); + static FRAMES: AtomicU64 = AtomicU64::new(0); + + /// Totals since the last [`reset`]. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub struct GpuTimings { + /// Recording and submitting the frame's commands. + pub submit: Duration, + /// Waiting for the GPU and the staging buffer's mapping. + pub map_wait: Duration, + /// Half float → f32 conversion of the read-back pixels. + pub convert: Duration, + /// Frames composited by the GPU. + pub frames: u64, + } + + fn add(counter: &AtomicU64, d: Duration) { + counter.fetch_add(u64::try_from(d.as_nanos()).unwrap_or(u64::MAX), Ordering::Relaxed); + } + + pub(crate) fn add_submit(d: Duration) { + add(&SUBMIT_NS, d); + } + + pub(crate) fn add_map_wait(d: Duration) { + add(&MAP_WAIT_NS, d); + } + + pub(crate) fn add_convert(d: Duration) { + add(&CONVERT_NS, d); + } + + pub(crate) fn add_frame() { + FRAMES.fetch_add(1, Ordering::Relaxed); + } + + pub fn get() -> GpuTimings { + let d = |c: &AtomicU64| Duration::from_nanos(c.load(Ordering::Relaxed)); + GpuTimings { submit: d(&SUBMIT_NS), map_wait: d(&MAP_WAIT_NS), convert: d(&CONVERT_NS), frames: FRAMES.load(Ordering::Relaxed) } + } + + pub fn reset() { + for c in [&SUBMIT_NS, &MAP_WAIT_NS, &CONVERT_NS, &FRAMES] { + c.store(0, Ordering::Relaxed); + } + } +} + pub struct ExportRenderer { _instance: wgpu::Instance, #[expect(dead_code, reason = "kept for future frame-accuracy flows (readback fences)")] @@ -27,9 +84,17 @@ pub struct ExportRenderer { impl ExportRenderer { /// The off-screen device, or `None` when this host has no usable adapter. pub fn new() -> Option { - let instance = wgpu::Instance::default(); - let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?; - let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()?; + // `None` is kept too: a machine without an adapter does not search again on every export + static SHARED: std::sync::OnceLock> = std::sync::OnceLock::new(); + let (instance, device, queue) = SHARED + .get_or_init(|| { + let instance = wgpu::Instance::default(); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?; + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()?; + Some((instance, device, queue)) + }) + .as_ref()? + .clone(); let compositor = GpuCompositor::new(&device, &queue); Some(Self { _instance: instance, device, queue, compositor }) } @@ -43,8 +108,8 @@ impl ExportRenderer { FramePlan::Image(img) => Some(img.clone()), FramePlan::Layers { .. } => { let prep = prepare(&plan); - self.compositor.composite_prepared(&plan, Some(&prep)); - let (w, h, px) = self.compositor.read_accumulator()?; + let (w, h, px) = self.compositor.render_export_prepared(&plan, Some(&prep))?; + timing::add_frame(); Some(Image { w: w as usize, h: h as usize, px }) } } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index b9379998..09800c6c 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -79,6 +79,7 @@ pub struct GpuCompositor { /// Copy of the accumulator under a blend-mode layer (allocated on first use). backdrop: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>, output: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>, + accum_readback_buf: Option<(wgpu::Buffer, u64)>, uploads: HashMap<(usize, u32, u32), Uploaded>, clock: u64, dummy: wgpu::TextureView, @@ -154,6 +155,23 @@ pub fn f16_to_f32(h: u16) -> f32 { f32::from_bits(bits) } +/// [`f16_to_f32`] with a branch-light path for normal numbers and zeros (the bulk of read-back +/// data); subnormals, infinities and NaNs take the general path. Same result for every input. +#[inline(always)] +pub fn f16_to_f32_fast(h: u16) -> f32 { + let u = u32::from(h); + let sign = (u & 0x8000) << 16; + let abs = u & 0x7fff; + if (0x0400..0x7c00).contains(&abs) { + // rebias the exponent (127 - 15 = 112) and widen the mantissa + f32::from_bits(sign | ((abs + (112 << 10)) << 13)) + } else if abs == 0 { + f32::from_bits(sign) + } else { + f16_to_f32(h) + } +} + /// Half-float texel data for one frame (one byte vector per plane), converted off the UI thread. /// 8-bit frames need no conversion and have none. #[derive(Clone, Debug, Default)] @@ -361,6 +379,7 @@ impl GpuCompositor { accum: None, backdrop: None, output: None, + accum_readback_buf: None, uploads: HashMap::new(), clock: 0, dummy, @@ -603,7 +622,7 @@ impl GpuCompositor { /// [`composite`](Self::composite) with texel conversions already done by [`prepare`] (the /// result is identical; only the upload work on this thread differs). - pub fn composite_prepared(&mut self, plan: &FramePlan, prep: Option<&PreparedPlan>) -> (wgpu::TextureView, (u32, u32)) { + fn draw_layers_into_accum(&mut self, plan: &FramePlan, prep: Option<&PreparedPlan>, enc: &mut wgpu::CommandEncoder) -> (wgpu::TextureView, (u32, u32)) { let owned; let (w, h, layers): (u32, u32, &[PlanLayer]) = match plan { FramePlan::Layers { width, height, layers } => (*width as u32, *height as u32, layers.as_slice()), @@ -629,7 +648,6 @@ impl GpuCompositor { } else { None }; - let out_view = Self::target(&self.device, &mut self.output, w, h, OUTPUT_FORMAT, wgpu::TextureUsages::COPY_SRC); // Layers whose effects the GPU stage can't run (no stage on this device, or a working image // larger than a texture) are rendered on the CPU here and drawn as plain RGBA layers. let max_side = self.device.limits().max_texture_dimension_2d; @@ -741,14 +759,9 @@ impl GpuCompositor { if let Some(f) = self.fx.as_mut() { f.end_frame(); } - let final_bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("final"), - layout: &self.final_bgl, - entries: &[wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(&accum_view) }], - }); - let mut enc = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("filmcraft-composite") }); // Runs of fixed-function layers draw in one pass. A layer that reads the destination // ends it, copies the accumulator under its quad into the backdrop and draws on its own. + let accum_tex = self.accum.as_ref().map(|a| a.0.clone()); let backdrop_tex = self.backdrop.as_ref().map(|b| b.0.clone()); let mut cleared = false; @@ -757,13 +770,13 @@ impl GpuCompositor { // a layer's effects run right before it is drawn (pooled working textures are reused // by the next layer with effects) if let (Some(Some(job)), Some(f)) = (jobs.get(i), self.fx.as_ref()) { - f.record(&mut enc, job); + f.record(enc, job); } if let Some((bg, Some(region))) = bind_groups.get(i) { i += 1; let (Some(rect), Some(src), Some(dst)) = (region, &accum_tex, &backdrop_tex) else { continue }; if !cleared { - accum_pass(&mut enc, &accum_view, wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT)); + accum_pass(enc, &accum_view, wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT)); cleared = true; } let origin = wgpu::Origin3d { x: rect.0, y: rect.1, z: 0 }; @@ -772,7 +785,7 @@ impl GpuCompositor { wgpu::TexelCopyTextureInfo { texture: dst, mip_level: 0, origin, aspect: wgpu::TextureAspect::All }, wgpu::Extent3d { width: rect.2, height: rect.3, depth_or_array_layers: 1 }, ); - let mut pass = accum_pass(&mut enc, &accum_view, wgpu::LoadOp::Load); + let mut pass = accum_pass(enc, &accum_view, wgpu::LoadOp::Load); pass.set_pipeline(&self.blend_pipeline); pass.set_bind_group(0, bg, &[]); pass.draw(0..6, 0..1); @@ -783,7 +796,7 @@ impl GpuCompositor { let breaks = |j: &usize| bind_groups.get(*j).is_some_and(|(_, r)| r.is_some()) || jobs.get(*j).is_some_and(Option::is_some); let end = (i + 1..bind_groups.len()).find(breaks).unwrap_or(bind_groups.len()).max(i); let load = if cleared { wgpu::LoadOp::Load } else { wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT) }; - let mut pass = accum_pass(&mut enc, &accum_view, load); + let mut pass = accum_pass(enc, &accum_view, load); cleared = true; pass.set_pipeline(&self.layer_pipeline); for (bg, _) in bind_groups.get(i..end).unwrap_or_default() { @@ -792,6 +805,20 @@ impl GpuCompositor { } i = end; } + (accum_view, (w, h)) + } + + /// [`composite`](Self::composite) with texel conversions already done by [`prepare`] (the + /// result is identical; only the upload work on this thread differs). + pub fn composite_prepared(&mut self, plan: &FramePlan, prep: Option<&PreparedPlan>) -> (wgpu::TextureView, (u32, u32)) { + let mut enc = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("filmcraft-composite") }); + let (accum_view, (w, h)) = self.draw_layers_into_accum(plan, prep, &mut enc); + let out_view = Self::target(&self.device, &mut self.output, w, h, OUTPUT_FORMAT, wgpu::TextureUsages::COPY_SRC); + let final_bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("final"), + layout: &self.final_bgl, + entries: &[wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(&accum_view) }], + }); { let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("resolve"), @@ -814,6 +841,20 @@ impl GpuCompositor { (out_view, (w, h)) } + /// Export rendering in one submission: `plan` composited into the linear float accumulator, + /// which is copied to a reused staging buffer and read back as premultiplied RGBA f32 (what + /// [`composite_prepared`](Self::composite_prepared) + [`read_accumulator`](Self::read_accumulator) + /// give, without the sRGB resolve or a second submission). + pub fn render_export_prepared(&mut self, plan: &FramePlan, prep: Option<&PreparedPlan>) -> Option<(u32, u32, Vec)> { + let t = std::time::Instant::now(); + let mut enc = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("filmcraft-composite-export") }); + self.draw_layers_into_accum(plan, prep, &mut enc); + let size = self.copy_accumulator(&mut enc)?; + self.queue.submit([enc.finish()]); + export_renderer::timing::add_submit(t.elapsed()); + self.map_accumulator(size) + } + /// Run the effect stage of a layer alone: `frame` decoded into its working image and `fx` /// applied, read back as linear premultiplied RGBA f32 (what `LayerFx` ops produce on the /// CPU from the same frame). None when the working image does not fit in a texture. @@ -877,22 +918,47 @@ impl GpuCompositor { /// the display texture this crate's [`read_output`](Self::read_output) resolves is sRGB 8-bit /// and would clip the working-space data HDR exports need. The accumulator must have been /// composited first ([`composite`](Self::composite) / [`composite_prepared`](Self::composite_prepared)). - pub fn read_accumulator(&self) -> Option<(u32, u32, Vec)> { - let (tex, _, (w, h)) = self.accum.as_ref()?; - let row = (w * 8).div_ceil(256) * 256; - let buf = self.device.create_buffer(&wgpu::BufferDescriptor { - label: Some("accum-readback"), - size: (row * h) as u64, - usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, - mapped_at_creation: false, - }); + pub fn read_accumulator(&mut self) -> Option<(u32, u32, Vec)> { let mut enc = self.device.create_command_encoder(&Default::default()); + let size = self.copy_accumulator(&mut enc)?; + self.queue.submit([enc.finish()]); + self.map_accumulator(size) + } + + /// Record a copy of the accumulator into the staging buffer (grown when too small, reused + /// otherwise); returns its width, height and row pitch in bytes. + fn copy_accumulator(&mut self, enc: &mut wgpu::CommandEncoder) -> Option<(u32, u32, u32)> { + let (_, _, (w, h)) = self.accum.as_ref()?; + let (w, h) = (*w, *h); + let row = w.checked_mul(8)?.div_ceil(256).checked_mul(256)?; + let needed = u64::from(row) * u64::from(h); + if self.accum_readback_buf.as_ref().is_none_or(|(_, size)| *size < needed) { + let b = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("accum-readback"), + size: needed, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + self.accum_readback_buf = Some((b, needed)); + } + let (tex, _, _) = self.accum.as_ref()?; + let (buf, _) = self.accum_readback_buf.as_ref()?; enc.copy_texture_to_buffer( wgpu::TexelCopyTextureInfo { texture: tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, - wgpu::TexelCopyBufferInfo { buffer: &buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(*h) } }, - wgpu::Extent3d { width: *w, height: *h, depth_or_array_layers: 1 }, + wgpu::TexelCopyBufferInfo { buffer: buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(h) } }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, ); - self.queue.submit([enc.finish()]); + Some((w, h, row)) + } + + /// Wait for the staging buffer filled by [`copy_accumulator`](Self::copy_accumulator) and + /// convert its half floats to f32. + fn map_accumulator(&self, (w, h, row): (u32, u32, u32)) -> Option<(u32, u32, Vec)> { + if w == 0 || h == 0 { + return None; + } + let t = std::time::Instant::now(); + let (buf, _) = self.accum_readback_buf.as_ref()?; let slice = buf.slice(..); let (tx, rx) = std::sync::mpsc::channel(); slice.map_async(wgpu::MapMode::Read, move |r| { @@ -900,15 +966,23 @@ impl GpuCompositor { }); let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); rx.recv().ok()?.ok()?; + export_renderer::timing::add_map_wait(t.elapsed()); + let t = std::time::Instant::now(); + let (width, row) = (w as usize, row as usize); + let mut out = filmcraft_frame::pool::take_f32_overwritten(width.checked_mul(h as usize)?.checked_mul(4)?); let data = slice.get_mapped_range().ok()?; - let mut out = Vec::with_capacity((w * h * 4) as usize); - for y in 0..*h { - let r = data.get((y * row) as usize..(y * row + w * 8) as usize)?; - out.extend(r.as_chunks::<2>().0.iter().map(|b| f16_to_f32(u16::from_le_bytes(*b)))); - } + // every row must be there: the pooled output holds stale values until written + let converted = out.chunks_exact_mut(width * 4).enumerate().all(|(y, dst)| { + let Some(src) = data.get(y * row..y * row + width * 8) else { return false }; + for (d, s) in dst.iter_mut().zip(src.as_chunks::<2>().0) { + *d = f16_to_f32_fast(u16::from_le_bytes(*s)); + } + true + }); drop(data); buf.unmap(); - Some((*w, *h, out)) + export_renderer::timing::add_convert(t.elapsed()); + converted.then_some((w, h, out)) } /// Read the output back as RGBA8 (tests / screenshots / thumbnails). diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index a11cef8b..87b69cc9 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -320,3 +320,11 @@ fn gpu_draws_yuv_alpha_plane_like_the_cpu() { assert!(r != at(&bg, w - 2), "{bits}-bit: right edge {r:?} should differ from the background"); } } + +#[test] +fn fast_half_float_conversion_matches_the_reference_for_every_value() { + for h in 0..=u16::MAX { + let (fast, slow) = (f16_to_f32_fast(h), f16_to_f32(h)); + assert!(fast.to_bits() == slow.to_bits() || (fast.is_nan() && slow.is_nan()), "{h:#06x}: {fast} vs {slow}"); + } +} diff --git a/crates/ui-egui/examples/bench/sections.rs b/crates/ui-egui/examples/bench/sections.rs index 7953a763..e7c06751 100644 --- a/crates/ui-egui/examples/bench/sections.rs +++ b/crates/ui-egui/examples/bench/sections.rs @@ -484,6 +484,8 @@ pub fn export(o: &Opts) -> Vec { .expect("audio"); frames = s.project.sequence(seq).map(|q| q.settings.frame_rate.frame_at(q.duration())).unwrap_or(0); let out = dir.join(format!("out.{ext}")); + filmcraft_export::reset_export_lock_wait(); + filmcraft_gpu::export_renderer::timing::reset(); let (t0, c0) = (Instant::now(), cpu_now()); // `--hw auto`: the hardware encoder too (NVENC H.264 on Windows); `--hw off`: ours let hardware = if o.hw == "off" { "off" } else { "auto" }; @@ -497,6 +499,22 @@ pub fn export(o: &Opts) -> Vec { eprintln!("export {format}: {e}"); break; } + let lock_wait = filmcraft_export::export_lock_wait(); + let gpu_timings = filmcraft_gpu::export_renderer::timing::get(); + if gpu_timings.frames > 0 { + let f = gpu_timings.frames as f64; + eprintln!( + " [GPU timings for {format}] lock_wait: {:.1} ms ({:.2} ms/f), submit: {:.1} ms ({:.2} ms/f), map_wait: {:.1} ms ({:.2} ms/f), convert: {:.1} ms ({:.2} ms/f)", + lock_wait.as_secs_f64() * 1000.0, + lock_wait.as_secs_f64() * 1000.0 / f, + gpu_timings.submit.as_secs_f64() * 1000.0, + gpu_timings.submit.as_secs_f64() * 1000.0 / f, + gpu_timings.map_wait.as_secs_f64() * 1000.0, + gpu_timings.map_wait.as_secs_f64() * 1000.0 / f, + gpu_timings.convert.as_secs_f64() * 1000.0, + gpu_timings.convert.as_secs_f64() * 1000.0 / f, + ); + } runs.push(dt); cpu.push(ms(cpu_now() - c0) / frames.max(1) as f64); bytes = std::fs::metadata(&out).map(|m| m.len()).unwrap_or(0); From d2b608605ecd3cb56d2465f3803f737029c67630 Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 09:03:01 -0700 Subject: [PATCH 04/12] GPU4: Vignette on the GPU compositor (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Vignette effect to the GPU effect stage (render::gpufx, gpu::fx, fx.wgsl). Evaluated parameters are shared between the CPU reference and GPU compute passes. Replaces old CPU vignette body in vfx::color with gpufx::FxOp::eval and gpufx::FxOp::apply. Gates: fmt, clippy, test -p filmcraft-render, test -p filmcraft-gpu, layers, wasm all verified. 🤖 Written by agy (Antigravity Agent, Gemini 3.8 Flash) --- ROADMAP.md | 8 +++---- crates/gpu/src/fx.rs | 6 +++++ crates/gpu/src/fx.wgsl | 31 ++++++++++++++++++++++++ crates/gpu/src/fx_tests.rs | 12 ++++++++++ crates/render/src/gpufx.rs | 44 ++++++++++++++++++++++++++++++++++ crates/render/src/vfx/color.rs | 28 ---------------------- crates/render/src/vfx/mod.rs | 1 - docs/architecture.md | 2 +- 8 files changed, 98 insertions(+), 34 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 29c0409f..8eaf78ff 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,7 +36,7 @@ before choosing work. | **Correctness in depth** | not tracked | **~60–70%** | A first-time contributor found four real bugs in core paths within hours (#5–#8: AIFF import failed, Slide left linked audio behind, transitions didn't follow ripple trims, export ignored start/end times). User reports: Color Matte always grey with no picker (#29); ALSA audio error on Linux (#23). Settings are "most wired", not all. | | **Codecs and media** | ~90% | **~75%** | Our decoders are bit-exact on conformance streams. Real camera and phone media (variable frame rate, damaged files, unusual containers) is far less tested. No camera RAW (RED, BRAW, ARRIRAW), no E-AC-3. | | **Export** | ~92% | **~70%** | Delivery: H.264 only (ProRes, DNxHR and image sequences cover mastering). No HEVC or AV1 export. AAF and OMF have never been validated in Avid Media Composer or Pro Tools. Export renders and encodes on the CPU only. | -| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux has no hardware path. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | +| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30). The GPU does compositing, blend modes (#32) and 32 common effects (GPU2, GPU4): colour adjustments, crop/flip/transform, blurs, sharpen, vignette. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux has no hardware path. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 32 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | | **Stability** | not tracked | **improving, unproven** | The never-crash pass (no panics in product code, last-resort guards) landed on 2026-10-04. There's no field record from real users' projects yet. | | **Plugins and pro ecosystem** | not tracked | **~0–10%** | No audio plugin hosting (VST3 / Audio Units) and no third-party video effects (OpenFX). For many professional editors this alone rules FilmCraft out. Team Projects and Productions are out of scope by design. | | **AI features** | partial | **~25–35%** | Speech to text exists but is off in default builds (the `whisper` feature). Enhance Speech is a DSP chain, not a model. Auto Reframe is approximate. No Generative Extend, and no media-intelligence search or auto colour. | @@ -50,7 +50,7 @@ In priority order. Agents should prefer this work over adding more checklist ite 1. **Hardware acceleration** (#30), the most visible gap to users: - hardware decode on Linux (VA-API); - zero-copy decoded frames into wgpu; - - the remaining effects on the GPU: Lumetri, keys, Vignette, Video Limiter, masks (31 common effects are done); + - the remaining effects on the GPU: Lumetri, keys, Video Limiter, masks (32 common effects are done); - GPU export, then hardware encode: H.264 and H.265 on macOS are in (H.264 opt-in; H.265 has no other encoder), H.264 on NVENC (Windows, opt-in); HEVC on Windows and Linux remain. 2. **A measured parity number.** Add `cargo xtask parity`, run in CI. It should cover menus, commands, effects, transitions, panels, preferences and formats against the Premiere snapshot in @@ -143,7 +143,7 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining | M2 | H.264 decoder | ✅ | Own decoder, bit-exact on 37+ streams, 500–600 fps 1080p | — | — | | M3 | Editing core | 🟡 | Edit algebra (insert/overwrite/razor/lift/extract/ripple/roll/slip/slide/rate-stretch/nest/paste), tools, markers, trim mode + Trim Monitor + dynamic J/K/L trimming, Keyboard Shortcuts editor with FilmCraft/Premiere/FCP/Avid presets; Edit/Clip/File menu commands (Label colours and Select Label Group, Paste/Remove Attributes, Select All Matching, Remove Unused, Consolidate Duplicates, Sequence From Clip, Bin From Selection, Offline File, Close Project, Make/Edit Subclip, Modify Audio Channels/Timecode, Frame Hold Options/Add Frame Hold/Insert Frame Hold Segment, Time Interpolation with frame blending, Fit/Fill frame, Breakout to Mono, Extract Audio, Replace With Clip; M3.11: Scene Edit Detection (pure-Rust cut detection, background job), Normalize Mix Track, Simplify Sequence, Transcribe Sequence, Find/Find Next and search bins, Automate to Sequence, Edit Original, Edit Offline, Source Settings, Update Metadata (XMP), Generate Audio Waveform, Project Settings General/Scratch Disks, Get Media File Properties, Save as Template, Selection as FilmCraft Project, Avid Log Exchange export, Flash Cue markers, Dynamic Audio Waveforms, Reveal Log Files, System Compatibility Report); M3.12 keyboard parity: Premiere's default keyboard (~115 keyboard-only commands: edit-point navigation on targeted/any track, select clip at playhead/next/previous, extend edit to playhead, nudge/slip/slide selection, target and source-patch toggles, clip volume ±1 dB/many, frame maximize/full screen/panel cycling, monitor zoom, track heights, Project and Text panel keyboard navigation, text size/leading/alignment, Export Frame, poster frames; see docs/keyboard.md); multicam (Create Multi-Camera Source Sequence, Multi-Camera view with live switching on 1–9, angle switching, Enable/Flatten, Edit Cameras) and sync (Synchronize, Merge Clips; In/Out/timecode/marker/audio — GCC-PHAT, sample-accurate) | Multicam paging >16 angles and grid thumbnails, optical flow (renders as frame blending), 39 Premiere default shortcuts skipped with reasons in docs/keyboard.md (Productions, AI/cloud tools, work area bar, Production/Search panels) | 6–10 | | M4 | Playback | 🟡 | Audio-clock master, prefetch with cancellation, J/K/L, correct dropped-frame stats, playback resolution, render bar + content-hashed render previews, App Nap opt-out; 1080p H.264 and 3 stacked 1080p streams play with 0 dropped frames | 4K under load, 8K, frame-threaded AV1 decode, reduced-resolution decode for multicam grids | 6–10 | -| M5 | Effects, keyframes, GPU | 🟡 | All 93 Premiere 26 video effects and 84 transitions (CPU), blend modes and 31 common effects on the GPU (#32, GPU2), keyframes + value/velocity graphs, wgpu compositor, effect + opacity masks (ellipse/polygon/pen, feather, expansion, modes; CPU/WGSL parity), mask tracking (Lucas–Kanade + RANSAC), adjustment layers, effect presets (built-in + user, JSON import/export) | Effects on the GPU (in progress, #30), GPU masks in the live path, GPU export, exact Warp Stabilizer (3-D) and Morph Cut | 20–30 | +| M5 | Effects, keyframes, GPU | 🟡 | All 93 Premiere 26 video effects and 84 transitions (CPU), blend modes and 32 common effects on the GPU (#32, GPU2, GPU4), keyframes + value/velocity graphs, wgpu compositor, effect + opacity masks (ellipse/polygon/pen, feather, expansion, modes; CPU/WGSL parity), mask tracking (Lucas–Kanade + RANSAC), adjustment layers, effect presets (built-in + user, JSON import/export) | Effects on the GPU (in progress, #30), GPU masks in the live path, GPU export, exact Warp Stabilizer (3-D) and Morph Cut | 20–30 | | M6 | Export | ✅ | Own H.264 encoder (High/Main/Baseline, B-frames, VBR/CBR/2-pass) → MP4 or QuickTime + own AAC; ProRes, DNxHR, MJPEG, PNG/TIFF/BMP sequences (Premiere-style numbering), GIF, WAV, AIFF; background jobs; M6.5: Export mode parity (frame size / rate / scaling / pixel aspect, profile / level, CBR / VBR 1- and 2-pass, keyframe distance, audio codec / rate / channels / bitrate / sample size, multiplexer, burn-in or sidecar captions, image / name / timecode overlays, video limiter, loudness normalization with true-peak limiter, metadata, ranges, estimated size, summary), 24 built-in presets + user presets with favourites and import/export (Preset Manager), export queue (reorder, cancel, retry, several sequences / ranges), Quick Export; `export.*` commands and `filmcraft-cli export --preset` | Interlaced encoding, 5.1 audio, smart render, publishing destinations | 6–8 | | M7 | Audio | 🟡 | Mixer graph (tracks → submixes → Mix, pre/post-fader inserts and sends, latency-compensated, sample-accurate, ~6× realtime for 24 tracks × 3 effects on one core), Audio Track Mixer + Audio Clip Mixer panels, track automation (Off/Read/Latch/Touch/Write, recorded live while playing, thinned to keyframes, timeline lanes with pen editing), solo/solo-safe, channel mapping basics, peak + BS.1770 loudness meters (match ffmpeg), DSP crate with 16 clip/track effects, Audio Gain (set/adjust/normalize), Constant Power / Constant Gain / Exponential Fade, Essential Sound (types, Loudness auto-match, Repair incl. DeEss/DeReverb, Clarity, Creative, Ducking keyframes, presets; full Dialogue chain 22× realtime) | Music duration remix, ML speech enhancement, 5.1 panner and multichannel buses, voice-over record, effect editor windows (EQ curve), remaining effects (multiband, convolution reverb), clip-mixer automation recording | 6–9 | | M8 | Colour | 🟡 | Lumetri: basic, creative + looks, RGB & hue curves, wheels, HSL secondary, vignette, section bypass; Input LUT / Look LUT (.cube 1D/3D/shaper, .3dl; tetrahedral CPU + WGSL; project LUT library; built-in camera conversions); Colour Match (Oklab tonal-range statistics, skin protection, solved in Lumetri wheels); colour management: Rec. 709 / Rec. 2100 PQ / HLG working spaces + wide gamut, Interpret Footage colour space (S-Log3, V-Log, Canon Log 2/3, LogC3/4, Apple Log, D-Log from published specs), metadata auto-detect, BT.2390 tone mapping, gamut mapping, HDR export signalling (VUI/colr/mdcv/clli/SEI, ffprobe-verified); Lumetri Scopes: Vectorscope YUV (75/100 % targets, skin-tone line) and HLS, Histogram, Parade (RGB/YUV/RGB-White), Waveform (RGB/Luma/YC/YC no Chroma), presets, Rec. 601/709/2100, 8-bit/float/HDR (cd/m²) scales, multi-scope grid, `scopes.read` | HDR-aware Lumetri maths, macOS EDR monitors, mastering metadata → tone-map peak, D-Log M (no published formula), HSL Secondary refine | 3–5 | @@ -154,7 +154,7 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining ## Running now -- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Next: VA-API (Linux) decode, zero-copy upload, Lumetri on the GPU. +- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 32 common effects on the GPU (GPU2, GPU4). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Next: VA-API (Linux) decode, zero-copy upload, Lumetri on the GPU. - **Next:** [Where we are lacking](#where-we-are-lacking) items 1–5. GPU export waits for the export-crate work (#19 and the never-crash follow-up) to land. ## Log diff --git a/crates/gpu/src/fx.rs b/crates/gpu/src/fx.rs index 930a43d8..96c386c7 100644 --- a/crates/gpu/src/fx.rs +++ b/crates/gpu/src/fx.rs @@ -40,6 +40,7 @@ const OP_ASC_CDL: u32 = 15; const OP_CHANNEL_MIX: u32 = 16; const OP_COLOR_REPLACE: u32 = 17; const OP_ALPHA_ADJUST: u32 = 18; +const OP_VIGNETTE: u32 = 19; const OP_BOX: u32 = 20; const OP_DIRECTIONAL: u32 = 22; const OP_UNSHARP: u32 = 23; @@ -175,6 +176,11 @@ fn steps(op: &FxOp) -> Vec { FxOp::AlphaAdjust { opacity, ignore, invert, mask_only } => { out.push(step(OP_ALPHA_ADJUST, [*ignore as u32, *invert as u32, *mask_only as u32, 0], &[*opacity])); } + FxOp::Vignette { amount, midpoint, roundness, feather, target } => { + if amount.abs() >= 1e-5 { + out.push(step(OP_VIGNETTE, [0; 4], &[*amount, *midpoint, *roundness, *feather, target[0], target[1], target[2]])); + } + } } out } diff --git a/crates/gpu/src/fx.wgsl b/crates/gpu/src/fx.wgsl index ee5d0248..80c8e520 100644 --- a/crates/gpu/src/fx.wgsl +++ b/crates/gpu/src/fx.wgsl @@ -36,6 +36,7 @@ const OP_ASC_CDL: u32 = 15u; const OP_CHANNEL_MIX: u32 = 16u; const OP_COLOR_REPLACE: u32 = 17u; const OP_ALPHA_ADJUST: u32 = 18u; +const OP_VIGNETTE: u32 = 19u; const OP_BOX: u32 = 20u; const OP_DIRECTIONAL: u32 = 22u; const OP_UNSHARP: u32 = 23u; @@ -467,6 +468,36 @@ fn pixel(op: u32, p: vec2) -> vec4 { let o = ld(p); return q + (o - q) * u.p0.z; } + case OP_VIGNETTE: { + let o = ld(p); + let a = o.a; + if a <= 1e-6 { + return o; + } + let amt = u.p0.x; + if abs(amt) < 1e-5 { + return o; + } + let mid = u.p0.y; + let round = u.p0.z; + let feather = u.p0.w; + let tgt = u.p1.xyz; + let w = f32(s.x); + let h = f32(s.y); + let aspect = w / h; + var nx = pc.x / w * 2.0 - 1.0; + let ny = pc.y / h * 2.0 - 1.0; + if round > 0.0 { + nx *= 1.0 + (aspect - 1.0) * round; + } + let p_exp = select(2.0, 2.0 + (-round) * 6.0, round < 0.0); + let d = powf(powf(abs(nx), p_exp) + powf(abs(ny), p_exp), 1.0 / p_exp) / powf(2.0, 1.0 / p_exp); + let edge = smoothstep_fx(mid - feather * 0.5, mid + feather * 0.5, d); + let c = o.rgb / a; + let ec = enc(c); + let res = dec(ec + (tgt - ec) * (edge * abs(amt))); + return vec4(res * a, a); + } default: { let o = ld(p); let a = o.a; diff --git a/crates/gpu/src/fx_tests.rs b/crates/gpu/src/fx_tests.rs index 5e201d11..37da6f04 100644 --- a/crates/gpu/src/fx_tests.rs +++ b/crates/gpu/src/fx_tests.rs @@ -187,6 +187,14 @@ fn cases() -> Vec<(&'static str, Vec<(&'static str, ParamValue)>, bool)> { ("alpha_adjust", vec![("opacity", fl(60.0))], false), ("alpha_adjust", vec![("ignore", ParamValue::Bool(true)), ("invert", ParamValue::Bool(true))], false), ("alpha_adjust", vec![("invert", ParamValue::Bool(true)), ("mask_only", ParamValue::Bool(true)), ("opacity", fl(150.0))], false), + ("vignette", vec![], false), + ("vignette", vec![("amount", fl(-100.0)), ("midpoint", fl(50.0)), ("roundness", fl(0.0)), ("feather", fl(50.0))], false), + ( + "vignette", + vec![("amount", fl(-30.0)), ("midpoint", fl(20.0)), ("roundness", fl(-100.0)), ("feather", fl(0.0)), ("color", col(0.1, 0.4, 0.7))], + false, + ), + ("vignette", vec![("amount", fl(40.0)), ("midpoint", fl(90.0)), ("roundness", fl(100.0)), ("feather", fl(100.0))], false), ] } @@ -319,6 +327,10 @@ fn hostile_parameters_are_bounded() { // minifying resamples (the CPU's mip path) stay on the CPU effect("transform", &[("scale_height", fl(20.0))]), effect("crop", &[("left", fl(-300.0)), ("zoom", ParamValue::Bool(true))]), + effect("vignette", &[("amount", fl(f64::NAN))]), + effect("vignette", &[("midpoint", fl(f64::INFINITY))]), + effect("vignette", &[("roundness", fl(f64::NAN))]), + effect("vignette", &[("feather", fl(f64::NAN))]), ]; for e in &nan { let op = FxOp::eval(e, &cx(Tick::ZERO, 1.0), w, h); diff --git a/crates/render/src/gpufx.rs b/crates/render/src/gpufx.rs index 1dd0edfd..cc5b6afc 100644 --- a/crates/render/src/gpufx.rs +++ b/crates/render/src/gpufx.rs @@ -161,6 +161,13 @@ pub enum FxOp { invert: bool, mask_only: bool, }, + Vignette { + amount: f32, + midpoint: f32, + roundness: f32, + feather: f32, + target: [f32; 3], + }, } /// Effect ids [`FxOp::eval`] understands (the GPU-capable standard effects). @@ -196,6 +203,7 @@ pub const GPU_EFFECTS: &[&str] = &[ "channel_mix", "color_replace", "alpha_adjust", + "vignette", ]; /// `Image::transformed` without the mip path: the destination rectangle it writes and the inverse. @@ -415,6 +423,17 @@ impl FxOp { use crate::vfx::{bv, fv}; FxOp::AlphaAdjust { opacity: fv(e, "opacity", cx) / 100.0, ignore: bv(e, "ignore"), invert: bv(e, "invert"), mask_only: bv(e, "mask_only") } } + "vignette" => { + use crate::vfx::{cv, fv}; + let amt = fv(e, "amount", cx) / 100.0; + let mid = fv(e, "midpoint", cx) / 100.0; + let round = fv(e, "roundness", cx) / 100.0; + let raw_feather = fv(e, "feather", cx); + let feather = if raw_feather.is_finite() { (raw_feather / 100.0).max(0.01) } else { raw_feather }; + let col = cv(e, "color", cx); + let target = if amt < 0.0 { [col[0], col[1], col[2]] } else { [1.0; 3] }; + FxOp::Vignette { amount: amt, midpoint: mid, roundness: round, feather, target } + } _ => return None, }) } @@ -458,6 +477,7 @@ impl FxOp { FxOp::ChannelMix { m } => m.iter().all(|r| fin(r)), FxOp::ColorReplace { sim, target, replace, replace_hsl, .. } => sim.is_finite() && fin(target) && fin(replace) && fin(replace_hsl), FxOp::AlphaAdjust { opacity, .. } => opacity.is_finite(), + FxOp::Vignette { amount, midpoint, roundness, feather, target } => fin(&[*amount, *midpoint, *roundness, *feather]) && fin(target), } } @@ -709,6 +729,30 @@ impl FxOp { } }); } + FxOp::Vignette { amount, midpoint, roundness, feather, target } => { + let (amt, mid, round, feather, target) = (*amount, *midpoint, *roundness, *feather, *target); + if !amt.is_finite() || !mid.is_finite() || !round.is_finite() || !feather.is_finite() || !target.iter().all(|c| c.is_finite()) { + return; + } + if amt.abs() < 1e-5 { + return; + } + let (w, h) = (img.w as f32, img.h as f32); + let aspect = w / h; + img.map_rgb(|c, x, y| { + let mut nx = (x as f32 + 0.5) / w * 2.0 - 1.0; + let ny = (y as f32 + 0.5) / h * 2.0 - 1.0; + // roundness 100 = circle; 0 = an ellipse following the frame; −100 = squarer + if round > 0.0 { + nx *= 1.0 + (aspect - 1.0) * round; + } + let p = if round < 0.0 { 2.0 + (-round) * 6.0 } else { 2.0 }; + // 0 at the centre, 1 at the corners + let d = (nx.abs().powf(p) + ny.abs().powf(p)).powf(1.0 / p) / 2f32.powf(1.0 / p); + let edge = crate::vfx::smoothstep(mid - feather * 0.5, mid + feather * 0.5, d); + dec(crate::vfx::lerp3(enc(c), target, edge * amt.abs())) + }); + } } } } diff --git a/crates/render/src/vfx/color.rs b/crates/render/src/vfx/color.rs index 9b0469d7..b6bfb1ea 100644 --- a/crates/render/src/vfx/color.rs +++ b/crates/render/src/vfx/color.rs @@ -189,34 +189,6 @@ pub(crate) fn limit(v: [f32; 3], max: f32, comp: f32, axis: u32) -> [f32; 3] { to_rgb(y, cb, cr).map(|q| q.clamp(0.0, max)) } -/// Vignette: darkens (negative Amount, towards Color) or lightens the frame edges. -pub fn vignette(img: &mut Image, e: &EffectInstance, cx: &FxCtx) { - let amt = fv(e, "amount", cx) / 100.0; - if amt.abs() < 1e-5 { - return; - } - let mid = fv(e, "midpoint", cx) / 100.0; - let round = fv(e, "roundness", cx) / 100.0; - let feather = (fv(e, "feather", cx) / 100.0).max(0.01); - let col = cv(e, "color", cx); - let target = if amt < 0.0 { [col[0], col[1], col[2]] } else { [1.0; 3] }; - let (w, h) = (img.w as f32, img.h as f32); - let aspect = w / h; - img.map_rgb(|c, x, y| { - let mut nx = (x as f32 + 0.5) / w * 2.0 - 1.0; - let ny = (y as f32 + 0.5) / h * 2.0 - 1.0; - // roundness 100 = circle; 0 = an ellipse following the frame; −100 = squarer - if round > 0.0 { - nx *= 1.0 + (aspect - 1.0) * round; - } - let p = if round < 0.0 { 2.0 + (-round) * 6.0 } else { 2.0 }; - // 0 at the centre, 1 at the corners - let d = (nx.abs().powf(p) + ny.abs().powf(p)).powf(1.0 / p) / 2f32.powf(1.0 / p); - let edge = smoothstep(mid - feather * 0.5, mid + feather * 0.5, d); - dec(lerp3(enc(c), target, edge * amt.abs())) - }); -} - /// Logo Cutout: removes a flat white/black/custom background by un-multiplying it — the alpha of /// a pixel is how far it is from the background towards the gamut edge, and the colour is /// recovered as if it had been composited over that background. diff --git a/crates/render/src/vfx/mod.rs b/crates/render/src/vfx/mod.rs index f6d88fc7..70c69ce2 100644 --- a/crates/render/src/vfx/mod.rs +++ b/crates/render/src/vfx/mod.rs @@ -461,7 +461,6 @@ pub fn apply(img: &mut Image, e: &EffectInstance, cx: &FxCtx) -> bool { // Adjust / Color / Image Control / Keying / Utility colour "lighting_effects" => color::lighting(img, e, cx), "video_limiter" => color::video_limiter(img, e, cx), - "vignette" => color::vignette(img, e, cx), "logo_cutout" => color::logo_cutout(img, e, cx), "ultra_key" => color::ultra_key(img, e, cx), "track_matte" => color::track_matte(img, e, cx), diff --git a/docs/architecture.md b/docs/architecture.md index e0a7885c..72e351db 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -283,7 +283,7 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see Balance, Leave Color, Change to Color, Color Pass, Color Replace, Channel Mix, ASC CDL, Gamma Correction, Levels, Extract, Invert, Posterize, Alpha Adjust, Gaussian Blur and Directional Blur (and their legacy aliases), Camera Blur, Sharpen, Unsharp Mask, Crop, Edge Feather, Transform, - Horizontal / Vertical Flip, Mirror and Offset. When every enabled effect of a media clip is in + Horizontal / Vertical Flip, Mirror, Offset and Vignette. When every enabled effect of a media clip is in that set (unmasked, with finite parameters, and no Transform shrinking the picture below half size, which the CPU pre-filters), the plan hands the GPU the clip's source with the effects' parameters evaluated at that time: the source is drawn into an `Rgba32Float` From ab334b2adf0fc524e4fd4ad55abd8b12683f2bdb Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 09:37:13 -0700 Subject: [PATCH 05/12] GPU5: Video Limiter on the GPU compositor (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move Video Limiter from CPU (crates/render/src/vfx/color.rs) to the GPU compositor (filmcraft_render::gpufx::FxOp, crates/gpu/src/fx.rs, crates/gpu/src/fx.wgsl). Parity matches CPU reference within 9.28e-6 across all axis modes, clip levels, compressions, and gamut warning color. 🤖 Written by agy (Antigravity Agent, Gemini 3.8 Flash) --- ROADMAP.md | 8 +-- crates/gpu/src/fx.rs | 4 ++ crates/gpu/src/fx.wgsl | 106 ++++++++++++++++++++++++++++++++ crates/gpu/src/fx_tests.rs | 23 +++++++ crates/render/src/gpufx.rs | 109 +++++++++++++++++++++++++++++++++ crates/render/src/vfx/color.rs | 80 +----------------------- crates/render/src/vfx/mod.rs | 1 - docs/architecture.md | 2 +- 8 files changed, 249 insertions(+), 84 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 8eaf78ff..06688b9a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,7 +36,7 @@ before choosing work. | **Correctness in depth** | not tracked | **~60–70%** | A first-time contributor found four real bugs in core paths within hours (#5–#8: AIFF import failed, Slide left linked audio behind, transitions didn't follow ripple trims, export ignored start/end times). User reports: Color Matte always grey with no picker (#29); ALSA audio error on Linux (#23). Settings are "most wired", not all. | | **Codecs and media** | ~90% | **~75%** | Our decoders are bit-exact on conformance streams. Real camera and phone media (variable frame rate, damaged files, unusual containers) is far less tested. No camera RAW (RED, BRAW, ARRIRAW), no E-AC-3. | | **Export** | ~92% | **~70%** | Delivery: H.264 only (ProRes, DNxHR and image sequences cover mastering). No HEVC or AV1 export. AAF and OMF have never been validated in Avid Media Composer or Pro Tools. Export renders and encodes on the CPU only. | -| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30). The GPU does compositing, blend modes (#32) and 32 common effects (GPU2, GPU4): colour adjustments, crop/flip/transform, blurs, sharpen, vignette. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux has no hardware path. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 32 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | +| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30). The GPU does compositing, blend modes (#32) and 33 common effects (GPU2, GPU4, GPU5): colour adjustments, crop/flip/transform, blurs, sharpen, vignette, video limiter. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux has no hardware path. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 33 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | | **Stability** | not tracked | **improving, unproven** | The never-crash pass (no panics in product code, last-resort guards) landed on 2026-10-04. There's no field record from real users' projects yet. | | **Plugins and pro ecosystem** | not tracked | **~0–10%** | No audio plugin hosting (VST3 / Audio Units) and no third-party video effects (OpenFX). For many professional editors this alone rules FilmCraft out. Team Projects and Productions are out of scope by design. | | **AI features** | partial | **~25–35%** | Speech to text exists but is off in default builds (the `whisper` feature). Enhance Speech is a DSP chain, not a model. Auto Reframe is approximate. No Generative Extend, and no media-intelligence search or auto colour. | @@ -50,7 +50,7 @@ In priority order. Agents should prefer this work over adding more checklist ite 1. **Hardware acceleration** (#30), the most visible gap to users: - hardware decode on Linux (VA-API); - zero-copy decoded frames into wgpu; - - the remaining effects on the GPU: Lumetri, keys, Video Limiter, masks (32 common effects are done); + - the remaining effects on the GPU: Lumetri, keys, masks (33 common effects are done); - GPU export, then hardware encode: H.264 and H.265 on macOS are in (H.264 opt-in; H.265 has no other encoder), H.264 on NVENC (Windows, opt-in); HEVC on Windows and Linux remain. 2. **A measured parity number.** Add `cargo xtask parity`, run in CI. It should cover menus, commands, effects, transitions, panels, preferences and formats against the Premiere snapshot in @@ -143,7 +143,7 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining | M2 | H.264 decoder | ✅ | Own decoder, bit-exact on 37+ streams, 500–600 fps 1080p | — | — | | M3 | Editing core | 🟡 | Edit algebra (insert/overwrite/razor/lift/extract/ripple/roll/slip/slide/rate-stretch/nest/paste), tools, markers, trim mode + Trim Monitor + dynamic J/K/L trimming, Keyboard Shortcuts editor with FilmCraft/Premiere/FCP/Avid presets; Edit/Clip/File menu commands (Label colours and Select Label Group, Paste/Remove Attributes, Select All Matching, Remove Unused, Consolidate Duplicates, Sequence From Clip, Bin From Selection, Offline File, Close Project, Make/Edit Subclip, Modify Audio Channels/Timecode, Frame Hold Options/Add Frame Hold/Insert Frame Hold Segment, Time Interpolation with frame blending, Fit/Fill frame, Breakout to Mono, Extract Audio, Replace With Clip; M3.11: Scene Edit Detection (pure-Rust cut detection, background job), Normalize Mix Track, Simplify Sequence, Transcribe Sequence, Find/Find Next and search bins, Automate to Sequence, Edit Original, Edit Offline, Source Settings, Update Metadata (XMP), Generate Audio Waveform, Project Settings General/Scratch Disks, Get Media File Properties, Save as Template, Selection as FilmCraft Project, Avid Log Exchange export, Flash Cue markers, Dynamic Audio Waveforms, Reveal Log Files, System Compatibility Report); M3.12 keyboard parity: Premiere's default keyboard (~115 keyboard-only commands: edit-point navigation on targeted/any track, select clip at playhead/next/previous, extend edit to playhead, nudge/slip/slide selection, target and source-patch toggles, clip volume ±1 dB/many, frame maximize/full screen/panel cycling, monitor zoom, track heights, Project and Text panel keyboard navigation, text size/leading/alignment, Export Frame, poster frames; see docs/keyboard.md); multicam (Create Multi-Camera Source Sequence, Multi-Camera view with live switching on 1–9, angle switching, Enable/Flatten, Edit Cameras) and sync (Synchronize, Merge Clips; In/Out/timecode/marker/audio — GCC-PHAT, sample-accurate) | Multicam paging >16 angles and grid thumbnails, optical flow (renders as frame blending), 39 Premiere default shortcuts skipped with reasons in docs/keyboard.md (Productions, AI/cloud tools, work area bar, Production/Search panels) | 6–10 | | M4 | Playback | 🟡 | Audio-clock master, prefetch with cancellation, J/K/L, correct dropped-frame stats, playback resolution, render bar + content-hashed render previews, App Nap opt-out; 1080p H.264 and 3 stacked 1080p streams play with 0 dropped frames | 4K under load, 8K, frame-threaded AV1 decode, reduced-resolution decode for multicam grids | 6–10 | -| M5 | Effects, keyframes, GPU | 🟡 | All 93 Premiere 26 video effects and 84 transitions (CPU), blend modes and 32 common effects on the GPU (#32, GPU2, GPU4), keyframes + value/velocity graphs, wgpu compositor, effect + opacity masks (ellipse/polygon/pen, feather, expansion, modes; CPU/WGSL parity), mask tracking (Lucas–Kanade + RANSAC), adjustment layers, effect presets (built-in + user, JSON import/export) | Effects on the GPU (in progress, #30), GPU masks in the live path, GPU export, exact Warp Stabilizer (3-D) and Morph Cut | 20–30 | +| M5 | Effects, keyframes, GPU | 🟡 | All 93 Premiere 26 video effects and 84 transitions (CPU), blend modes and 33 common effects on the GPU (#32, GPU2, GPU4, GPU5), keyframes + value/velocity graphs, wgpu compositor, effect + opacity masks (ellipse/polygon/pen, feather, expansion, modes; CPU/WGSL parity), mask tracking (Lucas–Kanade + RANSAC), adjustment layers, effect presets (built-in + user, JSON import/export) | Effects on the GPU (in progress, #30), GPU masks in the live path, GPU export, exact Warp Stabilizer (3-D) and Morph Cut | 20–30 | | M6 | Export | ✅ | Own H.264 encoder (High/Main/Baseline, B-frames, VBR/CBR/2-pass) → MP4 or QuickTime + own AAC; ProRes, DNxHR, MJPEG, PNG/TIFF/BMP sequences (Premiere-style numbering), GIF, WAV, AIFF; background jobs; M6.5: Export mode parity (frame size / rate / scaling / pixel aspect, profile / level, CBR / VBR 1- and 2-pass, keyframe distance, audio codec / rate / channels / bitrate / sample size, multiplexer, burn-in or sidecar captions, image / name / timecode overlays, video limiter, loudness normalization with true-peak limiter, metadata, ranges, estimated size, summary), 24 built-in presets + user presets with favourites and import/export (Preset Manager), export queue (reorder, cancel, retry, several sequences / ranges), Quick Export; `export.*` commands and `filmcraft-cli export --preset` | Interlaced encoding, 5.1 audio, smart render, publishing destinations | 6–8 | | M7 | Audio | 🟡 | Mixer graph (tracks → submixes → Mix, pre/post-fader inserts and sends, latency-compensated, sample-accurate, ~6× realtime for 24 tracks × 3 effects on one core), Audio Track Mixer + Audio Clip Mixer panels, track automation (Off/Read/Latch/Touch/Write, recorded live while playing, thinned to keyframes, timeline lanes with pen editing), solo/solo-safe, channel mapping basics, peak + BS.1770 loudness meters (match ffmpeg), DSP crate with 16 clip/track effects, Audio Gain (set/adjust/normalize), Constant Power / Constant Gain / Exponential Fade, Essential Sound (types, Loudness auto-match, Repair incl. DeEss/DeReverb, Clarity, Creative, Ducking keyframes, presets; full Dialogue chain 22× realtime) | Music duration remix, ML speech enhancement, 5.1 panner and multichannel buses, voice-over record, effect editor windows (EQ curve), remaining effects (multiband, convolution reverb), clip-mixer automation recording | 6–9 | | M8 | Colour | 🟡 | Lumetri: basic, creative + looks, RGB & hue curves, wheels, HSL secondary, vignette, section bypass; Input LUT / Look LUT (.cube 1D/3D/shaper, .3dl; tetrahedral CPU + WGSL; project LUT library; built-in camera conversions); Colour Match (Oklab tonal-range statistics, skin protection, solved in Lumetri wheels); colour management: Rec. 709 / Rec. 2100 PQ / HLG working spaces + wide gamut, Interpret Footage colour space (S-Log3, V-Log, Canon Log 2/3, LogC3/4, Apple Log, D-Log from published specs), metadata auto-detect, BT.2390 tone mapping, gamut mapping, HDR export signalling (VUI/colr/mdcv/clli/SEI, ffprobe-verified); Lumetri Scopes: Vectorscope YUV (75/100 % targets, skin-tone line) and HLS, Histogram, Parade (RGB/YUV/RGB-White), Waveform (RGB/Luma/YC/YC no Chroma), presets, Rec. 601/709/2100, 8-bit/float/HDR (cd/m²) scales, multi-scope grid, `scopes.read` | HDR-aware Lumetri maths, macOS EDR monitors, mastering metadata → tone-map peak, D-Log M (no published formula), HSL Secondary refine | 3–5 | @@ -154,7 +154,7 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining ## Running now -- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 32 common effects on the GPU (GPU2, GPU4). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Next: VA-API (Linux) decode, zero-copy upload, Lumetri on the GPU. +- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 33 common effects on the GPU (GPU2, GPU4, GPU5). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Next: VA-API (Linux) decode, zero-copy upload, Lumetri on the GPU. - **Next:** [Where we are lacking](#where-we-are-lacking) items 1–5. GPU export waits for the export-crate work (#19 and the never-crash follow-up) to land. ## Log diff --git a/crates/gpu/src/fx.rs b/crates/gpu/src/fx.rs index 96c386c7..424457e2 100644 --- a/crates/gpu/src/fx.rs +++ b/crates/gpu/src/fx.rs @@ -50,6 +50,7 @@ const OP_HFLIP: u32 = 26; const OP_VFLIP: u32 = 27; const OP_MIRROR: u32 = 28; const OP_OFFSET: u32 = 29; +const OP_VIDEO_LIMITER: u32 = 30; type Target = (wgpu::Texture, wgpu::TextureView); @@ -181,6 +182,9 @@ fn steps(op: &FxOp) -> Vec { out.push(step(OP_VIGNETTE, [0; 4], &[*amount, *midpoint, *roundness, *feather, target[0], target[1], target[2]])); } } + FxOp::VideoLimiter { max, comp, axis, warn, warning_color } => { + out.push(step(OP_VIDEO_LIMITER, [*axis, *warn as u32, 0, 0], &[*max, *comp, warning_color[0], warning_color[1], warning_color[2]])); + } } out } diff --git a/crates/gpu/src/fx.wgsl b/crates/gpu/src/fx.wgsl index 80c8e520..e74dfe63 100644 --- a/crates/gpu/src/fx.wgsl +++ b/crates/gpu/src/fx.wgsl @@ -46,6 +46,7 @@ const OP_HFLIP: u32 = 26u; const OP_VFLIP: u32 = 27u; const OP_MIRROR: u32 = 28u; const OP_OFFSET: u32 = 29u; +const OP_VIDEO_LIMITER: u32 = 30u; fn size() -> vec2 { return vec2(i32(u.i0.y), i32(u.i0.z)); @@ -219,6 +220,88 @@ fn hsl_to_rgb(h: f32, s: f32, l: f32) -> vec3 { return vec3(hsl_channel(p, q, h + 1.0 / 3.0), hsl_channel(p, q, h), hsl_channel(p, q, h - 1.0 / 3.0)); } +fn linear_to_enc_limiter(v: f32) -> f32 { + if v <= 1.0 { + return linear_to_srgb1(max(v, 0.0)); + } + return 1.0 + (v - 1.0) / 2.4; +} + +fn enc_to_linear_limiter(v: f32) -> f32 { + if v <= 1.0 { + return srgb_to_linear1(max(v, 0.0)); + } + return 1.0 + (v - 1.0) * 2.4; +} + +fn knee_limiter(v: f32, max_val: f32, comp: f32) -> f32 { + let k = max_val * (1.0 - comp); + if comp <= 0.0 || v <= k { + return min(v, max_val); + } + let r = max_val - k; + return k + r * (1.0 - exp(-(v - k) / r)); +} + +fn to_rgb_709(y: f32, cb: f32, cr: f32) -> vec3 { + let kr = 0.2126; + let kb = 0.0722; + let kg = 1.0 - kr - kb; + let r = y + 2.0 * (1.0 - kr) * cr; + let b = y + 2.0 * (1.0 - kb) * cb; + let g = (y - kr * r - kb * b) / kg; + return vec3(r, g, b); +} + +fn limit_video(v: vec3, max_val: f32, comp: f32, axis: u32) -> vec3 { + let kr = 0.2126; + let kb = 0.0722; + let kg = 1.0 - kr - kb; + var y = kr * v.x + kg * v.y + kb * v.z; + var cb = (v.z - y) / (2.0 * (1.0 - kb)); + var cr = (v.x - y) / (2.0 * (1.0 - kr)); + + if axis != 1u { + y = knee_limiter(max(y, 0.0), max_val, comp); + } + + if axis != 0u { + let rgb = to_rgb_709(y, cb, cr); + var s = 1.0; + let d = rgb - vec3(y); + + if rgb.x > max_val && d.x > 1e-6 { + s = min(s, max((max_val - y) / d.x, 0.0)); + } + if rgb.x < 0.0 && d.x < -1e-6 { + s = min(s, max(y / (-d.x), 0.0)); + } + + if rgb.y > max_val && d.y > 1e-6 { + s = min(s, max((max_val - y) / d.y, 0.0)); + } + if rgb.y < 0.0 && d.y < -1e-6 { + s = min(s, max(y / (-d.y), 0.0)); + } + + if rgb.z > max_val && d.z > 1e-6 { + s = min(s, max((max_val - y) / d.z, 0.0)); + } + if rgb.z < 0.0 && d.z < -1e-6 { + s = min(s, max(y / (-d.z), 0.0)); + } + + if axis == 3u && s < 1.0 - 1e-4 { + s = min(knee_limiter(s, 1.0, max(comp, 0.03)), 1.0); + } + cb *= s; + cr *= s; + } + + let out_rgb = to_rgb_709(y, cb, cr); + return clamp(out_rgb, vec3(0.0), vec3(max_val)); +} + // ---- per-pixel colour effects on straight colour (`Image::map_rgb`) fn color_op(op: u32, c: vec3) -> vec3 { @@ -498,6 +581,29 @@ fn pixel(op: u32, p: vec2) -> vec4 { let res = dec(ec + (tgt - ec) * (edge * abs(amt))); return vec4(res * a, a); } + case OP_VIDEO_LIMITER: { + let o = ld(p); + let a = o.a; + if a <= 1e-6 { + return o; + } + let c = o.rgb / a; + let v = vec3(linear_to_enc_limiter(c.x), linear_to_enc_limiter(c.y), linear_to_enc_limiter(c.z)); + let max_val = u.p0.x; + let comp = u.p0.y; + let axis = u.i1.x; + let warn = u.i1.y != 0u; + let wc = u.p0.zw; + let wc_b = u.p1.x; + let warning_col = vec3(wc.x, wc.y, wc_b); + let out_enc = limit_video(v, max_val, comp, axis); + let diff = abs(v - out_enc); + if warn && (diff.x > 1e-4 || diff.y > 1e-4 || diff.z > 1e-4) { + return vec4(warning_col * a, a); + } + let res = vec3(enc_to_linear_limiter(out_enc.x), enc_to_linear_limiter(out_enc.y), enc_to_linear_limiter(out_enc.z)); + return vec4(res * a, a); + } default: { let o = ld(p); let a = o.a; diff --git a/crates/gpu/src/fx_tests.rs b/crates/gpu/src/fx_tests.rs index 37da6f04..02d8e711 100644 --- a/crates/gpu/src/fx_tests.rs +++ b/crates/gpu/src/fx_tests.rs @@ -41,6 +41,9 @@ fn fl(v: f64) -> ParamValue { fn col(r: f32, g: f32, b: f32) -> ParamValue { ParamValue::Color([r, g, b, 1.0]) } +fn ch(c: u32) -> ParamValue { + ParamValue::Choice(c) +} fn pt(x: f64, y: f64) -> ParamValue { ParamValue::Vec2(Vec2::new(x, y)) } @@ -195,6 +198,22 @@ fn cases() -> Vec<(&'static str, Vec<(&'static str, ParamValue)>, bool)> { false, ), ("vignette", vec![("amount", fl(40.0)), ("midpoint", fl(90.0)), ("roundness", fl(100.0)), ("feather", fl(100.0))], false), + ("video_limiter", vec![], false), + ("video_limiter", vec![("axis", ch(0)), ("clip_level", ch(0)), ("compression", ch(0))], false), + ("video_limiter", vec![("axis", ch(1)), ("clip_level", ch(9)), ("compression", ch(4))], false), + ("video_limiter", vec![("axis", ch(2)), ("clip_level", ch(5)), ("compression", ch(2))], false), + ("video_limiter", vec![("axis", ch(3)), ("clip_level", ch(0)), ("compression", ch(1))], false), + ( + "video_limiter", + vec![ + ("axis", ch(2)), + ("clip_level", ch(0)), + ("compression", ch(0)), + ("gamut_warning", ParamValue::Bool(true)), + ("warning_color", col(0.0, 1.0, 0.0)), + ], + false, + ), ] } @@ -331,6 +350,10 @@ fn hostile_parameters_are_bounded() { effect("vignette", &[("midpoint", fl(f64::INFINITY))]), effect("vignette", &[("roundness", fl(f64::NAN))]), effect("vignette", &[("feather", fl(f64::NAN))]), + effect("video_limiter", &[("clip_level", fl(f64::NAN))]), + effect("video_limiter", &[("compression", fl(f64::INFINITY))]), + effect("video_limiter", &[("axis", fl(f64::NAN))]), + effect("video_limiter", &[("warning_color", ParamValue::Color([f32::NAN, 0.0, 0.0, 1.0]))]), ]; for e in &nan { let op = FxOp::eval(e, &cx(Tick::ZERO, 1.0), w, h); diff --git a/crates/render/src/gpufx.rs b/crates/render/src/gpufx.rs index cc5b6afc..a718438a 100644 --- a/crates/render/src/gpufx.rs +++ b/crates/render/src/gpufx.rs @@ -168,6 +168,13 @@ pub enum FxOp { feather: f32, target: [f32; 3], }, + VideoLimiter { + max: f32, + comp: f32, + axis: u32, + warn: bool, + warning_color: [f32; 3], + }, } /// Effect ids [`FxOp::eval`] understands (the GPU-capable standard effects). @@ -204,6 +211,7 @@ pub const GPU_EFFECTS: &[&str] = &[ "color_replace", "alpha_adjust", "vignette", + "video_limiter", ]; /// `Image::transformed` without the mip path: the destination rectangle it writes and the inverse. @@ -434,6 +442,35 @@ impl FxOp { let target = if amt < 0.0 { [col[0], col[1], col[2]] } else { [1.0; 3] }; FxOp::Vignette { amount: amt, midpoint: mid, roundness: round, feather, target } } + "video_limiter" => { + use crate::vfx::{bv, cv}; + use filmcraft_project::ParamValue; + let clip_level = match e.param("clip_level").map(|p| p.value.clone()).or_else(|| crate::vfx::def_value(e, "clip_level")) { + Some(ParamValue::Choice(c)) => c as f32, + Some(ParamValue::Float(f)) => f as f32, + _ => 0.0, + }; + let comp_val = match e.param("compression").map(|p| p.value.clone()).or_else(|| crate::vfx::def_value(e, "compression")) { + Some(ParamValue::Choice(c)) => [0.0, 0.03, 0.05, 0.10, 0.20][(c as usize).min(4)], + Some(ParamValue::Float(f)) => f as f32, + _ => 0.03, + }; + let axis = match e.param("axis").map(|p| p.value.clone()).or_else(|| crate::vfx::def_value(e, "axis")) { + Some(ParamValue::Choice(c)) => c, + Some(ParamValue::Float(f)) => { + if f.is_nan() { + u32::MAX + } else { + f as u32 + } + } + _ => 3, + }; + let warn = bv(e, "gamut_warning"); + let c = cv(e, "warning_color", cx); + let wc = dec([c[0], c[1], c[2]]); + FxOp::VideoLimiter { max: 1.0 + clip_level / 100.0, comp: comp_val, axis, warn, warning_color: wc } + } _ => return None, }) } @@ -478,6 +515,7 @@ impl FxOp { FxOp::ColorReplace { sim, target, replace, replace_hsl, .. } => sim.is_finite() && fin(target) && fin(replace) && fin(replace_hsl), FxOp::AlphaAdjust { opacity, .. } => opacity.is_finite(), FxOp::Vignette { amount, midpoint, roundness, feather, target } => fin(&[*amount, *midpoint, *roundness, *feather]) && fin(target), + FxOp::VideoLimiter { max, comp, axis, warning_color, .. } => *axis <= 3 && max.is_finite() && comp.is_finite() && fin(warning_color), } } @@ -753,10 +791,81 @@ impl FxOp { dec(crate::vfx::lerp3(enc(c), target, edge * amt.abs())) }); } + FxOp::VideoLimiter { max, comp, axis, warn, warning_color } => { + let (max, comp, axis, warn, wc) = (*max, *comp, *axis, *warn, *warning_color); + if !max.is_finite() || !comp.is_finite() || !wc.iter().all(|c| c.is_finite()) { + return; + } + img.map_rgb(|c, _, _| { + let v = [linear_to_enc(c[0]), linear_to_enc(c[1]), linear_to_enc(c[2])]; + let out = limit(v, max, comp, axis); + if warn && v.iter().zip(&out).any(|(a, b)| (a - b).abs() > 1e-4) { + return wc; + } + out.map(enc_to_linear) + }); + } } } } +/// sRGB encoding that keeps values above 1 (super-whites) instead of clamping them. +#[inline] +fn linear_to_enc(v: f32) -> f32 { + if v <= 1.0 { filmcraft_color::linear_to_srgb(v.max(0.0)) } else { 1.0 + (v - 1.0) / 2.4 } +} +#[inline] +fn enc_to_linear(v: f32) -> f32 { + if v <= 1.0 { filmcraft_color::srgb_to_linear(v.max(0.0)) } else { 1.0 + (v - 1.0) * 2.4 } +} + +#[inline] +fn knee(v: f32, max: f32, comp: f32) -> f32 { + let k = max * (1.0 - comp); + if comp <= 0.0 || v <= k { + return v.min(max); + } + let r = max - k; + k + r * (1.0 - (-(v - k) / r).exp()) +} + +pub(crate) fn limit(v: [f32; 3], max: f32, comp: f32, axis: u32) -> [f32; 3] { + let m = filmcraft_color::Matrix::Bt709; + let ycc = filmcraft_color::rgb_to_ycbcr(v[0], v[1], v[2], m); + let (kr, kb) = (0.2126f32, 0.0722f32); + let to_rgb = |y: f32, cb: f32, cr: f32| { + let r = y + 2.0 * (1.0 - kr) * cr; + let b = y + 2.0 * (1.0 - kb) * cb; + let g = (y - kr * r - kb * b) / (1.0 - kr - kb); + [r, g, b] + }; + let (mut y, mut cb, mut cr) = (ycc[0], ycc[1], ycc[2]); + if axis != 1 { + y = knee(y.max(0.0), max, comp); + } + if axis != 0 { + // scale chroma so every channel fits 0…max (the gamut of legal R′G′B′) + let rgb = to_rgb(y, cb, cr); + let mut s = 1.0f32; + for &ch in &rgb { + let d = ch - y; + if ch > max && d > 1e-6 { + s = s.min(((max - y) / d).max(0.0)); + } + if ch < 0.0 && d < -1e-6 { + s = s.min((y / -d).max(0.0)); + } + } + if axis == 3 && s < 1.0 - 1e-4 { + // smart limit: soft chroma compression rather than a hard cut + s = knee(s, 1.0, comp.max(0.03)).min(1.0); + } + cb *= s; + cr *= s; + } + to_rgb(y, cb, cr).map(|q| q.clamp(0.0, max)) +} + /// The body of `Image::transformed` (no mip path) for a precomputed inverse and rectangle. fn resample_cpu(src: &Image, inv: &Affine, rect: [u32; 4]) -> Image { let (w, h) = (src.w, src.h); diff --git a/crates/render/src/vfx/color.rs b/crates/render/src/vfx/color.rs index b6bfb1ea..35f4e464 100644 --- a/crates/render/src/vfx/color.rs +++ b/crates/render/src/vfx/color.rs @@ -110,84 +110,8 @@ pub(crate) fn cdl(c: [f32; 3], s: [f32; 3], o: [f32; 3], p: [f32; 3], sat: f32) v.map(|q| (l + sat * (q - l)).clamp(0.0, 1.0)) } -/// Video Limiter: keeps Y′CbCr (Rec. 709, display-encoded) within 0…Clip Level with an -/// exponential soft knee that starts `compression` below the limit. -pub fn video_limiter(img: &mut Image, e: &EffectInstance, cx: &FxCtx) { - let max = 1.0 + chv(e, "clip_level") as f32 / 100.0; - let comp = [0.0, 0.03, 0.05, 0.10, 0.20][chv(e, "compression").min(4) as usize]; - let axis = chv(e, "axis"); - let warn = bv(e, "gamut_warning"); - let wc = lin(cv(e, "warning_color", cx)); - img.map_rgb(|c, _, _| { - let v = [linear_to_enc(c[0]), linear_to_enc(c[1]), linear_to_enc(c[2])]; - let out = limit(v, max, comp, axis); - if warn && v.iter().zip(&out).any(|(a, b)| (a - b).abs() > 1e-4) { - return wc; - } - dec_unclamped(out) - }); - let _ = cx; -} - -/// sRGB encoding that keeps values above 1 (super-whites) instead of clamping them. -#[inline] -fn linear_to_enc(v: f32) -> f32 { - if v <= 1.0 { filmcraft_color::linear_to_srgb(v.max(0.0)) } else { 1.0 + (v - 1.0) / 2.4 } -} -#[inline] -fn enc_to_linear(v: f32) -> f32 { - if v <= 1.0 { filmcraft_color::srgb_to_linear(v.max(0.0)) } else { 1.0 + (v - 1.0) * 2.4 } -} -fn dec_unclamped(c: [f32; 3]) -> [f32; 3] { - c.map(enc_to_linear) -} - -#[inline] -fn knee(v: f32, max: f32, comp: f32) -> f32 { - let k = max * (1.0 - comp); - if comp <= 0.0 || v <= k { - return v.min(max); - } - let r = max - k; - k + r * (1.0 - (-(v - k) / r).exp()) -} - -pub(crate) fn limit(v: [f32; 3], max: f32, comp: f32, axis: u32) -> [f32; 3] { - let m = Matrix::Bt709; - let ycc = rgb_to_ycbcr(v[0], v[1], v[2], m); - let (kr, kb) = (0.2126f32, 0.0722f32); - let to_rgb = |y: f32, cb: f32, cr: f32| { - let r = y + 2.0 * (1.0 - kr) * cr; - let b = y + 2.0 * (1.0 - kb) * cb; - let g = (y - kr * r - kb * b) / (1.0 - kr - kb); - [r, g, b] - }; - let (mut y, mut cb, mut cr) = (ycc[0], ycc[1], ycc[2]); - if axis != 1 { - y = knee(y.max(0.0), max, comp); - } - if axis != 0 { - // scale chroma so every channel fits 0…max (the gamut of legal R′G′B′) - let rgb = to_rgb(y, cb, cr); - let mut s = 1.0f32; - for &ch in &rgb { - let d = ch - y; - if ch > max && d > 1e-6 { - s = s.min(((max - y) / d).max(0.0)); - } - if ch < 0.0 && d < -1e-6 { - s = s.min((y / -d).max(0.0)); - } - } - if axis == 3 && s < 1.0 { - // smart limit: soft chroma compression rather than a hard cut - s = knee(s, 1.0, comp.max(0.03)).min(1.0); - } - cb *= s; - cr *= s; - } - to_rgb(y, cb, cr).map(|q| q.clamp(0.0, max)) -} +#[allow(unused_imports)] +pub(crate) use crate::gpufx::limit; /// Logo Cutout: removes a flat white/black/custom background by un-multiplying it — the alpha of /// a pixel is how far it is from the background towards the gamut edge, and the colour is diff --git a/crates/render/src/vfx/mod.rs b/crates/render/src/vfx/mod.rs index 70c69ce2..bdca8e52 100644 --- a/crates/render/src/vfx/mod.rs +++ b/crates/render/src/vfx/mod.rs @@ -460,7 +460,6 @@ pub fn apply(img: &mut Image, e: &EffectInstance, cx: &FxCtx) -> bool { match e.effect.as_str() { // Adjust / Color / Image Control / Keying / Utility colour "lighting_effects" => color::lighting(img, e, cx), - "video_limiter" => color::video_limiter(img, e, cx), "logo_cutout" => color::logo_cutout(img, e, cx), "ultra_key" => color::ultra_key(img, e, cx), "track_matte" => color::track_matte(img, e, cx), diff --git a/docs/architecture.md b/docs/architecture.md index 72e351db..abeb32d3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -283,7 +283,7 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see Balance, Leave Color, Change to Color, Color Pass, Color Replace, Channel Mix, ASC CDL, Gamma Correction, Levels, Extract, Invert, Posterize, Alpha Adjust, Gaussian Blur and Directional Blur (and their legacy aliases), Camera Blur, Sharpen, Unsharp Mask, Crop, Edge Feather, Transform, - Horizontal / Vertical Flip, Mirror, Offset and Vignette. When every enabled effect of a media clip is in + Horizontal / Vertical Flip, Mirror, Offset, Vignette and Video Limiter. When every enabled effect of a media clip is in that set (unmasked, with finite parameters, and no Transform shrinking the picture below half size, which the CPU pre-filters), the plan hands the GPU the clip's source with the effects' parameters evaluated at that time: the source is drawn into an `Rgba32Float` From f2fefeeb4b31c9c3c1d7cffd3cc2eb9cb64e15e4 Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 15:10:47 -0700 Subject: [PATCH 06/12] GPU6: GPU effects match the CPU on AMD (gamma, extract, invert, color replace) (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix subpixel bilinear interpolation leakage during 1:1 source texture sampling in composite.wgsl load4, and guard against divide-by-zero / subpixel noise amplification in OP_INVERT_ALPHA in fx.wgsl. All fx_tests::gpu_effects_match_cpu_exactly test cases now pass on AMD (Radeon RX 7900, Mesa 26.2.2, Vulkan) within EXACT (1e-4) and 0.000% flips. 🤖 Written by agy (Antigravity Agent, Gemini 3.8 Flash) --- crates/gpu/src/composite.wgsl | 5 ++++- crates/gpu/src/fx.wgsl | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/gpu/src/composite.wgsl b/crates/gpu/src/composite.wgsl index 5e897ba3..04796d28 100644 --- a/crates/gpu/src/composite.wgsl +++ b/crates/gpu/src/composite.wgsl @@ -45,7 +45,10 @@ fn vs(@builtin(vertex_index) vi: u32) -> VOut { fn load4(t: texture_2d, p: vec2) -> vec4 { let d = vec2(textureDimensions(t)); - let q = p - 0.5; + var q = p - 0.5; + let rq = round(q); + if abs(q.x - rq.x) < 1e-4 { q.x = rq.x; } + if abs(q.y - rq.y) < 1e-4 { q.y = rq.y; } let i = vec2(floor(q)); let f = q - floor(q); let a = textureLoad(t, clamp(i, vec2(0), d - 1), 0); diff --git a/crates/gpu/src/fx.wgsl b/crates/gpu/src/fx.wgsl index e74dfe63..410e69a6 100644 --- a/crates/gpu/src/fx.wgsl +++ b/crates/gpu/src/fx.wgsl @@ -442,7 +442,10 @@ fn pixel(op: u32, p: vec2) -> vec4 { let o = ld(p); let a = o.a; let na = 1.0 - a; - let k = select(0.0, na / a, a > 1e-6); + var k = 0.0; + if a > 1e-4 { + k = na / a; + } let blend = u.p0.x; return vec4(o.rgb * k, na * (1.0 - blend) + a * blend); } From 83e6fe7eda2c87e7c5b77c392b991e36c60c4ae6 Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 15:29:41 -0700 Subject: [PATCH 07/12] GPU6: invert alpha keeps the CPU's 1e-6 threshold (#30) The previous commit raised the GPU's un-premultiply threshold for Invert (alpha channel) to 1e-4, which the CPU reference does not use (it is 1e-6): alphas between the two inverted differently. The 6 % flipped pixels came from the bilinear leak the same commit fixed in load4; with it, the CPU's threshold passes gpu_effects_match_cpu_exactly (and the gpu and golden suites, serially). Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/fx.wgsl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/gpu/src/fx.wgsl b/crates/gpu/src/fx.wgsl index 410e69a6..6e617b7c 100644 --- a/crates/gpu/src/fx.wgsl +++ b/crates/gpu/src/fx.wgsl @@ -442,8 +442,9 @@ fn pixel(op: u32, p: vec2) -> vec4 { let o = ld(p); let a = o.a; let na = 1.0 - a; + // (the CPU's threshold; `na / a` is only used when a > 1e-6) var k = 0.0; - if a > 1e-4 { + if a > 1e-6 { k = na / a; } let blend = u.p0.x; From 029e9bb04097d4198c0a5d2168b781a3596ff04e Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 19:59:56 -0700 Subject: [PATCH 08/12] GPU7: Lumetri Color (basic, creative, vignette) on the GPU compositor (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the common slice of Lumetri Color (Basic Correction, Creative without sharpen, and Vignette) to the GPU effect stage (FxOp::Lumetri, OP_LUMETRI = 31). Lumetri instances with unsupported sections (input LUT, Look / look LUT, sharpen, curves, color wheels, HSL secondary, or HDR grading) fall back to CPU. Parity tests pass within max relative difference 4.28e-7 and 0.000% flips. 🤖 Written by agy (Antigravity Agent, Gemini 3.8 Flash) --- crates/gpu/src/fx.rs | 66 +++++++++++- crates/gpu/src/fx.wgsl | 102 +++++++++++++++++++ crates/gpu/src/fx_tests.rs | 148 +++++++++++++++++++++++++++ crates/render/src/effects.rs | 17 ++-- crates/render/src/gpufx.rs | 189 ++++++++++++++++++++++++++++++++++- 5 files changed, 508 insertions(+), 14 deletions(-) diff --git a/crates/gpu/src/fx.rs b/crates/gpu/src/fx.rs index 424457e2..d16be17d 100644 --- a/crates/gpu/src/fx.rs +++ b/crates/gpu/src/fx.rs @@ -51,6 +51,7 @@ const OP_VFLIP: u32 = 27; const OP_MIRROR: u32 = 28; const OP_OFFSET: u32 = 29; const OP_VIDEO_LIMITER: u32 = 30; +const OP_LUMETRI: u32 = 31; type Target = (wgpu::Texture, wgpu::TextureView); @@ -87,7 +88,7 @@ struct Step { run: bool, code: u32, i1: [u32; 4], - p: [f32; 16], + p: [f32; 24], /// Unsharp combine: reads the blurred image and the original. combine: bool, /// A blur pass of Unsharp Mask (the original stays untouched until the combine). @@ -95,7 +96,7 @@ struct Step { } fn step(code: u32, i1: [u32; 4], p: &[f32]) -> Step { - let mut q = [0.0; 16]; + let mut q = [0.0; 24]; for (d, s) in q.iter_mut().zip(p) { *d = *s; } @@ -185,6 +186,65 @@ fn steps(op: &FxOp) -> Vec { FxOp::VideoLimiter { max, comp, axis, warn, warning_color } => { out.push(step(OP_VIDEO_LIMITER, [*axis, *warn as u32, 0, 0], &[*max, *comp, warning_color[0], warning_color[1], warning_color[2]])); } + FxOp::Lumetri { + gains, + exposure, + contrast, + hl, + sh, + wh, + bl, + sat, + creative_on, + faded, + vib, + st, + ht, + vignette_on, + va, + vmid, + vround, + vfeather, + aspect, + .. + } => { + let ge = [gains[0] * exposure, gains[1] * exposure, gains[2] * exposure]; + let b0 = -bl * 0.15; + let w0 = 1.0 - wh * 0.15; + let sh_k = sh * 0.35; + let hl_k = hl * 0.35; + let vround_aspect = if *vround < 0.0 { aspect.powf(-vround) } else { 1.0 }; + let st_k = [(st[0] - 0.5) * 0.3, (st[1] - 0.5) * 0.3, (st[2] - 0.5) * 0.3]; + let ht_k = [(ht[0] - 0.5) * 0.3, (ht[1] - 0.5) * 0.3, (ht[2] - 0.5) * 0.3]; + + let p = [ + ge[0], + ge[1], + ge[2], + b0, + w0, + sh_k, + hl_k, + *contrast, + *faded, + *sat, + *vib, + *va, + *vmid, + *vfeather, + vround_aspect, + 0.0, + st_k[0], + st_k[1], + st_k[2], + 0.0, + ht_k[0], + ht_k[1], + ht_k[2], + 0.0, + ]; + out.push(step(OP_LUMETRI, [*creative_on as u32, *vignette_on as u32, 0, 0], &p)); + } } out } @@ -350,7 +410,7 @@ impl FxStage { (true, Some(o)) => views.get(o)?, _ => dummy, }; - let mut bytes = Vec::with_capacity(96); + let mut bytes = Vec::with_capacity(128); for v in [s.code, w, h, 0].iter().chain(&s.i1) { bytes.extend_from_slice(&v.to_le_bytes()); } diff --git a/crates/gpu/src/fx.wgsl b/crates/gpu/src/fx.wgsl index 6e617b7c..1fe152f8 100644 --- a/crates/gpu/src/fx.wgsl +++ b/crates/gpu/src/fx.wgsl @@ -10,6 +10,8 @@ struct U { p1: vec4, p2: vec4, p3: vec4, + p4: vec4, + p5: vec4, }; @group(0) @binding(0) var u: U; @@ -47,6 +49,7 @@ const OP_VFLIP: u32 = 27u; const OP_MIRROR: u32 = 28u; const OP_OFFSET: u32 = 29u; const OP_VIDEO_LIMITER: u32 = 30u; +const OP_LUMETRI: u32 = 31u; fn size() -> vec2 { return vec2(i32(u.i0.y), i32(u.i0.z)); @@ -433,6 +436,95 @@ fn color_op(op: u32, c: vec3) -> vec3 { } } +fn lumetri_op(c: vec3, p: vec2) -> vec3 { + let s = size(); + let creative_on = u.i1.x != 0u; + let vignette_on = u.i1.y != 0u; + + // white balance + exposure in linear light + let ge = u.p0.xyz; + let lin = c * ge; + var v = enc(lin); + + // whites / blacks: endpoints + let b0 = u.p0.w; + let w0 = u.p1.x; + let denom = max(w0 - b0, 1e-3); + v = (v - vec3(b0)) / denom; + + // highlights / shadows: luma-weighted lift/compress, hue preserving + let sh_k = u.p1.y; + let hl_k = u.p1.z; + let l = luma709(v); + let ws_val = clamp(1.0 - l, 0.0, 1.0); + let ws = ws_val * ws_val * ws_val; + let whl_val = clamp(l, 0.0, 1.0); + let whl = whl_val * whl_val * whl_val; + let nl = max(l + sh_k * ws + hl_k * whl, 0.0); + if l > 1e-5 { + let k = nl / l; + v = v * k; + } + + // contrast: smooth S-curve around mid grey + let contrast = u.p1.w; + if abs(contrast) > 1e-4 { + let k = 1.0 + contrast; + let q = clamp(v, vec3(0.0), vec3(1.0)); + let sc = q * q * (vec3(3.0) - 2.0 * q); + if k >= 1.0 { + v = q + (sc - q) * (k - 1.0); + } else { + v = vec3(0.5) + (q - vec3(0.5)) * k; + } + } + + // faded film: lift blacks and compress + let faded = u.p2.x; + if faded > 0.0 { + v = v * (1.0 - 0.25 * faded) + vec3(0.12 * faded); + } + + // split tone + if creative_on { + let st_k = u.p4.xyz; + let ht_k = u.p5.xyz; + let l2 = clamp(luma709(v), 0.0, 1.0); + v = v + st_k * (1.0 - l2) + ht_k * l2; + } + + // saturation & vibrance + let sat = u.p2.y; + let vib = u.p2.z; + let l3 = luma709(v); + let cur_sat = max(max(v.x, v.y), v.z) - min(min(v.x, v.y), v.z); + let sat_mult = sat * (1.0 + vib * (1.0 - clamp(cur_sat, 0.0, 1.0))); + v = vec3(l3) + (v - vec3(l3)) * sat_mult; + + // vignette + let va = u.p2.w; + if vignette_on && abs(va) > 1e-4 { + let vmid = u.p3.x; + let vfeather = u.p3.y; + let vround_aspect = u.p3.z; + let w = f32(s.x); + let h = f32(s.y); + let nx = (f32(p.x) / w - 0.5) * 2.0 * vround_aspect; + let ny = (f32(p.y) / h - 0.5) * 2.0; + let d = sqrt(nx * nx + ny * ny) / 1.4142135623730951; + let edge = clamp((d - vmid * 0.9) / (max(vfeather, 0.01) * 0.9), 0.0, 1.0); + let e2 = edge * edge * (3.0 - 2.0 * edge); + let k = 1.0 + va * 0.2 * e2; + if va < 0.0 { + v = v * max(k, 0.0); + } else { + v = v + (vec3(1.0) - v) * (k - 1.0); + } + } + + return dec(v); +} + // One output pixel of every op but the running-sum box blur. fn pixel(op: u32, p: vec2) -> vec4 { let s = size(); @@ -608,6 +700,16 @@ fn pixel(op: u32, p: vec2) -> vec4 { let res = vec3(enc_to_linear_limiter(out_enc.x), enc_to_linear_limiter(out_enc.y), enc_to_linear_limiter(out_enc.z)); return vec4(res * a, a); } + case OP_LUMETRI: { + let o = ld(p); + let a = o.a; + if a <= 1e-6 { + return o; + } + let c = o.rgb / a; + let res = lumetri_op(c, p); + return vec4(res * a, a); + } default: { let o = ld(p); let a = o.a; diff --git a/crates/gpu/src/fx_tests.rs b/crates/gpu/src/fx_tests.rs index 02d8e711..2654556e 100644 --- a/crates/gpu/src/fx_tests.rs +++ b/crates/gpu/src/fx_tests.rs @@ -214,6 +214,89 @@ fn cases() -> Vec<(&'static str, Vec<(&'static str, ParamValue)>, bool)> { ], false, ), + ( + "lumetri", + vec![ + ("temperature", fl(20.0)), + ("tint", fl(-10.0)), + ("exposure", fl(0.5)), + ("contrast", fl(15.0)), + ("highlights", fl(-20.0)), + ("shadows", fl(25.0)), + ("whites", fl(10.0)), + ("blacks", fl(-15.0)), + ("saturation", fl(110.0)), + ("creative_on", ParamValue::Bool(false)), + ("vignette_on", ParamValue::Bool(false)), + ], + false, + ), + ( + "lumetri", + vec![ + ("basic_on", ParamValue::Bool(false)), + ("creative_sat", fl(120.0)), + ("vibrance", fl(30.0)), + ("faded_film", fl(25.0)), + ("shadow_tint", col(0.4, 0.45, 0.6)), + ("highlight_tint", col(0.6, 0.55, 0.4)), + ("vignette_on", ParamValue::Bool(false)), + ], + false, + ), + ( + "lumetri", + vec![ + ("basic_on", ParamValue::Bool(false)), + ("creative_on", ParamValue::Bool(false)), + ("vignette_amount", fl(-3.0)), + ("vignette_midpoint", fl(45.0)), + ("vignette_roundness", fl(-30.0)), + ("vignette_feather", fl(60.0)), + ], + false, + ), + ( + "lumetri", + vec![ + ("temperature", fl(-15.0)), + ("tint", fl(10.0)), + ("exposure", fl(0.3)), + ("contrast", fl(20.0)), + ("highlights", fl(-10.0)), + ("shadows", fl(15.0)), + ("whites", fl(-5.0)), + ("blacks", fl(5.0)), + ("saturation", fl(105.0)), + ("creative_sat", fl(110.0)), + ("vibrance", fl(20.0)), + ("faded_film", fl(15.0)), + ("shadow_tint", col(0.48, 0.5, 0.55)), + ("highlight_tint", col(0.52, 0.5, 0.45)), + ("vignette_amount", fl(2.0)), + ("vignette_midpoint", fl(50.0)), + ("vignette_roundness", fl(20.0)), + ("vignette_feather", fl(50.0)), + ], + false, + ), + ( + "lumetri", + vec![("exposure", fl(4.0)), ("contrast", fl(100.0)), ("temperature", fl(100.0)), ("tint", fl(100.0)), ("whites", fl(100.0)), ("blacks", fl(100.0))], + false, + ), + ( + "lumetri", + vec![ + ("exposure", fl(-4.0)), + ("contrast", fl(-100.0)), + ("temperature", fl(-100.0)), + ("tint", fl(-100.0)), + ("whites", fl(-100.0)), + ("blacks", fl(-100.0)), + ], + false, + ), ] } @@ -354,6 +437,10 @@ fn hostile_parameters_are_bounded() { effect("video_limiter", &[("compression", fl(f64::INFINITY))]), effect("video_limiter", &[("axis", fl(f64::NAN))]), effect("video_limiter", &[("warning_color", ParamValue::Color([f32::NAN, 0.0, 0.0, 1.0]))]), + effect("lumetri", &[("exposure", fl(f64::NAN))]), + effect("lumetri", &[("temperature", fl(f64::INFINITY))]), + effect("lumetri", &[("contrast", fl(f64::NAN))]), + effect("lumetri", &[("vignette_amount", fl(f64::NAN))]), ]; for e in &nan { let op = FxOp::eval(e, &cx(Tick::ZERO, 1.0), w, h); @@ -392,6 +479,67 @@ fn hostile_parameters_are_bounded() { assert!(c.effect_image(&frame, &LayerFx { size: (1 << 20, 4), decimation: 1, ops: vec![FxOp::BlackWhite] }).is_none()); } +#[test] +fn lumetri_unsupported_sections_fall_back() { + let (w, h) = (40usize, 24usize); + // sharpen > 0 and input LUT (explicitly required by brief) + let e = effect("lumetri", &[("sharpen", fl(10.0)), ("input_lut", ParamValue::Text("custom.cube".into()))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()), "sharpen > 0 and input_lut must fall back to CPU"); + + // sharpen alone + let e = effect("lumetri", &[("sharpen", fl(20.0))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // input_lut alone + let e = effect("lumetri", &[("input_lut", ParamValue::Text("test.cube".into()))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // look_lut + let e = effect("lumetri", &[("look_lut", ParamValue::Text("look.cube".into()))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // look > 0 + let e = effect("lumetri", &[("look", ch(1))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // curve_luma modified + let e = effect("lumetri", &[("curve_luma", ParamValue::Curve(vec![[0.0, 0.1], [1.0, 0.9]]))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // hue_vs_sat non-empty + let e = effect("lumetri", &[("hue_vs_sat", ParamValue::Curve(vec![[0.5, 0.5]]))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // wheel offset + let e = effect("lumetri", &[("wheel_shadows", pt(0.5, 0.0))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // wheel lightness + let e = effect("lumetri", &[("wheel_midtones_l", fl(20.0))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // HSL secondary on + let e = effect("lumetri", &[("hsl_on", ParamValue::Bool(true))]); + let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); + + // HDR working space + let e = effect("lumetri", &[]); + let mut cx_hdr = cx(Tick::ZERO, 1.0); + cx_hdr.working = filmcraft_color::WorkingSpace::Rec2100Pq; + let op = FxOp::eval(&e, &cx_hdr, w, h); + assert!(op.as_ref().is_some_and(|o| !o.gpu_ok())); +} + /// Layers with effect chains in a composited plan — YUV and RGBA sources, a decimated working /// image, rotated / scaled placement, partial opacity and alpha, blend modes — match the CPU plan /// executor. diff --git a/crates/render/src/effects.rs b/crates/render/src/effects.rs index 15ee24da..07eb528a 100644 --- a/crates/render/src/effects.rs +++ b/crates/render/src/effects.rs @@ -40,11 +40,10 @@ pub(crate) fn f(e: &EffectInstance, id: &str, cx: &FxCtx) -> f32 { pub(crate) fn b(e: &EffectInstance, id: &str) -> bool { e.param(id).and_then(|p| p.value.as_bool()).unwrap_or(false) } -/// A section switch (missing in older projects = on). -fn on(e: &EffectInstance, id: &str) -> bool { +pub(crate) fn on(e: &EffectInstance, id: &str) -> bool { e.param(id).and_then(|p| p.value.as_bool()).unwrap_or(true) } -fn text<'e>(e: &'e EffectInstance, id: &str) -> &'e str { +pub(crate) fn text<'e>(e: &'e EffectInstance, id: &str) -> &'e str { match e.param(id).map(|p| &p.value) { Some(ParamValue::Text(s)) => s, _ => "", @@ -94,7 +93,7 @@ pub fn apply(img: &mut Image, e: &EffectInstance, cx: &FxCtx) { return; } // effects with a GPU implementation: evaluated parameters + CPU reference (`gpufx`) - if let Some(op) = crate::gpufx::FxOp::eval(e, cx, img.w, img.h) { + if let Some(op) = crate::gpufx::FxOp::eval(e, cx, img.w, img.h).filter(|op| e.effect != "lumetri" || op.gpu_ok()) { op.apply(img); return; } @@ -575,9 +574,7 @@ pub(crate) fn key(img: &mut Image, e: &EffectInstance, cx: &FxCtx) { }); } -/// The grading signal of the effect's section: HDR White (`white_id`) applies in HDR working -/// spaces only. -fn grade_space(e: &EffectInstance, cx: &FxCtx, white_id: &str) -> GradeSpace { +pub(crate) fn grade_space(e: &EffectInstance, cx: &FxCtx, white_id: &str) -> GradeSpace { GradeSpace::new(cx.working, f(e, white_id, cx)) } @@ -1026,16 +1023,16 @@ fn hue_lut(points: &[[f32; 2]], n: usize) -> Option> { Some(lut[n..2 * n].to_vec()) } -fn curve_param(e: &EffectInstance, id: &str) -> Option> { +pub(crate) fn curve_param(e: &EffectInstance, id: &str) -> Option> { e.param(id).and_then(|p| p.value.as_curve().map(|c| c.to_vec())) } -fn is_identity_curve(c: &[[f32; 2]]) -> bool { +pub(crate) fn is_identity_curve(c: &[[f32; 2]]) -> bool { c.iter().all(|p| (p[0] - p[1]).abs() < 1e-4) } /// Wheel offset (zero-mean RGB direction for a wheel position). -fn wheel_rgb(v: Vec2) -> [f32; 3] { +pub(crate) fn wheel_rgb(v: Vec2) -> [f32; 3] { let len = (v.x * v.x + v.y * v.y).sqrt().min(1.0) as f32; if len < 1e-5 { return [0.0; 3]; diff --git a/crates/render/src/gpufx.rs b/crates/render/src/gpufx.rs index a718438a..0b432ea0 100644 --- a/crates/render/src/gpufx.rs +++ b/crates/render/src/gpufx.rs @@ -14,7 +14,7 @@ use filmcraft_geom::Affine; use filmcraft_project::EffectInstance; use rayon::prelude::*; -use crate::effects::{FxCtx, b, choice, color, dec, enc, f, gaussian_boxes, point}; +use crate::effects::{FxCtx, b, choice, color, dec, enc, f, gaussian_boxes, grade_space, on, point, text}; use crate::image::Image; /// A bilinear resample of the working image through an affine map (`Image::transformed` at a @@ -175,6 +175,28 @@ pub enum FxOp { warn: bool, warning_color: [f32; 3], }, + Lumetri { + gains: [f32; 3], + exposure: f32, + contrast: f32, + hl: f32, + sh: f32, + wh: f32, + bl: f32, + sat: f32, + creative_on: bool, + faded: f32, + vib: f32, + st: [f32; 3], + ht: [f32; 3], + vignette_on: bool, + va: f32, + vmid: f32, + vround: f32, + vfeather: f32, + aspect: f32, + gpu_capable: bool, + }, } /// Effect ids [`FxOp::eval`] understands (the GPU-capable standard effects). @@ -212,6 +234,7 @@ pub const GPU_EFFECTS: &[&str] = &[ "alpha_adjust", "vignette", "video_limiter", + "lumetri", ]; /// `Image::transformed` without the mip path: the destination rectangle it writes and the inverse. @@ -471,6 +494,80 @@ impl FxOp { let wc = dec([c[0], c[1], c[2]]); FxOp::VideoLimiter { max: 1.0 + clip_level / 100.0, comp: comp_val, axis, warn, warning_color: wc } } + "lumetri" => { + use crate::effects::{curve_param, is_identity_curve, wheel_rgb}; + let (basic_on, creative_on, vignette_on) = (on(e, "basic_on"), on(e, "creative_on"), on(e, "vignette_on")); + let gs = grade_space(e, cx, "hdr_white"); + let is_hdr = gs.is_hdr(); + let has_input_lut = basic_on && !text(e, "input_lut").is_empty(); + let has_look_lut = creative_on && !text(e, "look_lut").is_empty(); + let look = if creative_on && !has_look_lut { choice(e, "look") } else { 0 }; + let sharpen = if creative_on { f(e, "sharpen", cx) / 100.0 } else { 0.0 }; + + let curves_on = on(e, "curves_on"); + let has_curves = curves_on && { + ["curve_luma", "curve_red", "curve_green", "curve_blue"].iter().any(|id| curve_param(e, id).is_some_and(|c| !is_identity_curve(&c))) + || ["hue_vs_sat", "hue_vs_hue", "hue_vs_luma", "luma_vs_sat", "sat_vs_sat"] + .iter() + .any(|id| curve_param(e, id).is_some_and(|c| !c.is_empty())) + }; + + let wheels_on = on(e, "wheels_on"); + let has_wheels = wheels_on && { + let v2 = |id: &str| e.param(id).map(|p| p.vec2_at(cx.t)).unwrap_or_default(); + let (ws, wm, wh) = (wheel_rgb(v2("wheel_shadows")), wheel_rgb(v2("wheel_midtones")), wheel_rgb(v2("wheel_highlights"))); + let (ls, lm, lh) = (f(e, "wheel_shadows_l", cx) / 100.0, f(e, "wheel_midtones_l", cx) / 100.0, f(e, "wheel_highlights_l", cx) / 100.0); + ws.iter().chain(&wm).chain(&wh).any(|v| v.abs() > 1e-5) || (ls.abs() + lm.abs() + lh.abs() > 1e-5) + }; + + let hsl_on = b(e, "hsl_on"); + + let gpu_capable = !is_hdr && !has_input_lut && !has_look_lut && look == 0 && sharpen.abs() <= 1e-3 && !has_curves && !has_wheels && !hsl_on; + + let bf = |id: &str| if basic_on { f(e, id, cx) } else { 0.0 }; + let temp = bf("temperature") / 100.0; + let tint = bf("tint") / 100.0; + let exposure = 2f32.powf(bf("exposure")); + let contrast = bf("contrast") / 100.0; + let hl = bf("highlights") / 100.0; + let sh = bf("shadows") / 100.0; + let wh = bf("whites") / 100.0; + let bl = bf("blacks") / 100.0; + let sat = if basic_on { f(e, "saturation", cx) / 100.0 } else { 1.0 } * if creative_on { f(e, "creative_sat", cx) / 100.0 } else { 1.0 }; + let vib = if creative_on { f(e, "vibrance", cx) / 100.0 } else { 0.0 }; + let faded = if creative_on { f(e, "faded_film", cx) / 100.0 } else { 0.0 }; + let st = color(e, "shadow_tint", cx); + let ht = color(e, "highlight_tint", cx); + let va = if vignette_on { f(e, "vignette_amount", cx) } else { 0.0 }; + let vmid = f(e, "vignette_midpoint", cx) / 100.0; + let vround = f(e, "vignette_roundness", cx) / 100.0; + let vfeather = f(e, "vignette_feather", cx) / 100.0; + let gains = [1.0 + 0.35 * temp, 1.0 - 0.3 * tint, 1.0 - 0.35 * temp]; + let aspect = if h > 0 { (w as f32 / h as f32).max(1e-4) } else { 1.0 }; + + FxOp::Lumetri { + gains, + exposure, + contrast, + hl, + sh, + wh, + bl, + sat, + creative_on, + faded, + vib, + st: [st[0], st[1], st[2]], + ht: [ht[0], ht[1], ht[2]], + vignette_on, + va, + vmid, + vround, + vfeather, + aspect, + gpu_capable, + } + } _ => return None, }) } @@ -516,6 +613,14 @@ impl FxOp { FxOp::AlphaAdjust { opacity, .. } => opacity.is_finite(), FxOp::Vignette { amount, midpoint, roundness, feather, target } => fin(&[*amount, *midpoint, *roundness, *feather]) && fin(target), FxOp::VideoLimiter { max, comp, axis, warning_color, .. } => *axis <= 3 && max.is_finite() && comp.is_finite() && fin(warning_color), + FxOp::Lumetri { gains, exposure, contrast, hl, sh, wh, bl, sat, faded, vib, st, ht, va, vmid, vround, vfeather, aspect, gpu_capable, .. } => { + *gpu_capable + && fin(gains) + && fin(&[*exposure, *contrast, *hl, *sh, *wh, *bl, *sat, *faded, *vib]) + && fin(st) + && fin(ht) + && fin(&[*va, *vmid, *vround, *vfeather, *aspect]) + } } } @@ -805,6 +910,88 @@ impl FxOp { out.map(enc_to_linear) }); } + FxOp::Lumetri { + gains, + exposure, + contrast, + hl, + sh, + wh, + bl, + sat, + creative_on, + faded, + vib, + st, + ht, + vignette_on, + va, + vmid, + vround, + vfeather, + aspect, + .. + } => { + let gains = *gains; + let (exposure, contrast, hl, sh, wh, bl, sat) = (*exposure, *contrast, *hl, *sh, *wh, *bl, *sat); + let (creative_on, faded, vib, st, ht) = (*creative_on, *faded, *vib, *st, *ht); + let (vignette_on, va, vmid, vround, vfeather, aspect) = (*vignette_on, *va, *vmid, *vround, *vfeather, *aspect); + let (w, h) = (img.w as f32, img.h as f32); + img.map_rgb(|c, x, y| { + // white balance + exposure in linear light + let lin = [c[0] * gains[0] * exposure, c[1] * gains[1] * exposure, c[2] * gains[2] * exposure]; + let mut v = enc(lin); + // whites / blacks: endpoints + let b0 = -bl * 0.15; + let w0 = 1.0 - wh * 0.15; + v = v.map(|q| (q - b0) / (w0 - b0).max(1e-3)); + // highlights / shadows: luma-weighted lift/compress, hue preserving + let l = 0.2126 * v[0] + 0.7152 * v[1] + 0.0722 * v[2]; + let ws = (1.0 - l).clamp(0.0, 1.0).powi(3); + let whl = l.clamp(0.0, 1.0).powi(3); + let nl = (l + sh * 0.35 * ws + hl * 0.35 * whl).max(0.0); + if l > 1e-5 { + let k = nl / l; + v = v.map(|q| q * k); + } + // contrast: smooth S-curve around mid grey + if contrast.abs() > 1e-4 { + let k = 1.0 + contrast; + v = v.map(|q| { + let q = q.clamp(0.0, 1.0); + let s = q * q * (3.0 - 2.0 * q); + if k >= 1.0 { q + (s - q) * (k - 1.0) } else { 0.5 + (q - 0.5) * k } + }); + } + // faded film: lift blacks and compress + if faded > 0.0 { + v = v.map(|q| q * (1.0 - 0.25 * faded) + 0.12 * faded); + } + // split tone + if creative_on { + let l2 = (0.2126 * v[0] + 0.7152 * v[1] + 0.0722 * v[2]).clamp(0.0, 1.0); + for k in 0..3 { + v[k] += (st[k] - 0.5) * 0.3 * (1.0 - l2) + (ht[k] - 0.5) * 0.3 * l2; + } + } + // saturation & vibrance + let l3 = 0.2126 * v[0] + 0.7152 * v[1] + 0.0722 * v[2]; + let cur_sat = v[0].max(v[1]).max(v[2]) - v[0].min(v[1]).min(v[2]); + let s = sat * (1.0 + vib * (1.0 - cur_sat.clamp(0.0, 1.0))); + v = v.map(|q| l3 + (q - l3) * s); + // vignette + if vignette_on && va.abs() > 1e-4 { + let nx = (x as f32 / w - 0.5) * 2.0 * (1.0 + vround * 0.0) * if vround < 0.0 { aspect.powf(-vround) } else { 1.0 }; + let ny = (y as f32 / h - 0.5) * 2.0; + let d = (nx * nx + ny * ny).sqrt() / std::f32::consts::SQRT_2; + let edge = ((d - vmid * 0.9) / (vfeather.max(0.01) * 0.9)).clamp(0.0, 1.0); + let e2 = edge * edge * (3.0 - 2.0 * edge); + let k = 1.0 + va * 0.2 * e2; + v = v.map(|q| if va < 0.0 { q * k.max(0.0) } else { q + (1.0 - q) * (k - 1.0) }); + } + dec(v) + }); + } } } } From 5ee4faf2d2fc2a66f3f94f8008631df9bfad5c69 Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 20:53:45 -0700 Subject: [PATCH 09/12] GPU7: Lumetri GPU op matches fn lumetri exactly (test); doc comments restored (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CPU parity test verifying FxOp::Lumetri::apply matches original fn lumetri exactly (0.00e0 sample difference) across all parity cases. Restore doc comments on fn on and fn grade_space. 🤖 Written by agy (Antigravity Agent, Gemini 3.8 Flash) --- crates/render/src/effects.rs | 179 +++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/crates/render/src/effects.rs b/crates/render/src/effects.rs index 07eb528a..a20e9475 100644 --- a/crates/render/src/effects.rs +++ b/crates/render/src/effects.rs @@ -40,6 +40,7 @@ pub(crate) fn f(e: &EffectInstance, id: &str, cx: &FxCtx) -> f32 { pub(crate) fn b(e: &EffectInstance, id: &str) -> bool { e.param(id).and_then(|p| p.value.as_bool()).unwrap_or(false) } +/// A section switch (missing in older projects = on). pub(crate) fn on(e: &EffectInstance, id: &str) -> bool { e.param(id).and_then(|p| p.value.as_bool()).unwrap_or(true) } @@ -574,6 +575,8 @@ pub(crate) fn key(img: &mut Image, e: &EffectInstance, cx: &FxCtx) { }); } +/// The grading signal of the effect's section: HDR White (`white_id`) applies in HDR working +/// spaces only. pub(crate) fn grade_space(e: &EffectInstance, cx: &FxCtx, white_id: &str) -> GradeSpace { GradeSpace::new(cx.working, f(e, white_id, cx)) } @@ -1337,3 +1340,179 @@ mod curve_tests { assert!((h[0] - h[63]).abs() < 0.05, "periodic"); } } + +#[cfg(test)] +mod lumetri_cpu_parity_tests { + use super::*; + use crate::gpufx::FxOp; + use filmcraft_project::{EffectInstance, ParamValue, find_effect}; + use filmcraft_time::Tick; + + fn test_picture(w: usize, h: usize) -> Image { + let mut px = Vec::with_capacity(w * h * 4); + for y in 0..h { + for x in 0..w { + let fx = x as f32 / (w.max(2) - 1) as f32; + let fy = y as f32 / (h.max(2) - 1) as f32; + let c = match x % 11 { + 0 => [0.0, 0.0, 0.0], + 1 => [1.0, 1.0, 1.0], + 2 => [fy, fy, fy], + 3 => [1.6 * fy, 1.2 * fy, 0.3], + _ => { + let rgb = filmcraft_color::hsl_to_rgb(fx, 0.25 + 0.75 * ((y % 5) as f32 / 4.0), 0.1 + 0.8 * fy); + rgb.map(filmcraft_color::srgb_to_linear) + } + }; + let a = match y % 7 { + 0 => 0.0, + 1 | 2 => 0.5 + 0.4 * fx, + 3 => 0.25, + _ => 1.0, + }; + px.extend_from_slice(&[c[0] * a, c[1] * a, c[2] * a, a]); + } + } + Image { w, h, px } + } + + fn effect(params: &[(&str, ParamValue)]) -> EffectInstance { + let mut e = find_effect("lumetri").unwrap().instance(); + for (k, v) in params { + e.params.get_mut(*k).unwrap_or_else(|| panic!("lumetri.{k}")).value = v.clone(); + } + e + } + + fn fl(v: f64) -> ParamValue { + ParamValue::Float(v) + } + + fn col(r: f32, g: f32, b: f32) -> ParamValue { + ParamValue::Color([r, g, b, 1.0]) + } + + #[test] + fn lumetri_gpu_op_matches_original_fn_lumetri_cpu() { + let cases: &[(&str, Vec<(&str, ParamValue)>)] = &[ + ( + "basic", + vec![ + ("temperature", fl(20.0)), + ("tint", fl(-10.0)), + ("exposure", fl(0.5)), + ("contrast", fl(15.0)), + ("highlights", fl(-20.0)), + ("shadows", fl(25.0)), + ("whites", fl(10.0)), + ("blacks", fl(-15.0)), + ("saturation", fl(110.0)), + ("creative_on", ParamValue::Bool(false)), + ("vignette_on", ParamValue::Bool(false)), + ], + ), + ( + "creative", + vec![ + ("basic_on", ParamValue::Bool(false)), + ("creative_sat", fl(120.0)), + ("vibrance", fl(30.0)), + ("faded_film", fl(25.0)), + ("shadow_tint", col(0.4, 0.45, 0.6)), + ("highlight_tint", col(0.6, 0.55, 0.4)), + ("vignette_on", ParamValue::Bool(false)), + ], + ), + ( + "vignette", + vec![ + ("basic_on", ParamValue::Bool(false)), + ("creative_on", ParamValue::Bool(false)), + ("vignette_amount", fl(-3.0)), + ("vignette_midpoint", fl(45.0)), + ("vignette_roundness", fl(-30.0)), + ("vignette_feather", fl(60.0)), + ], + ), + ( + "all_three", + vec![ + ("temperature", fl(-15.0)), + ("tint", fl(10.0)), + ("exposure", fl(0.3)), + ("contrast", fl(20.0)), + ("highlights", fl(-10.0)), + ("shadows", fl(15.0)), + ("whites", fl(-5.0)), + ("blacks", fl(5.0)), + ("saturation", fl(105.0)), + ("creative_sat", fl(110.0)), + ("vibrance", fl(20.0)), + ("faded_film", fl(15.0)), + ("shadow_tint", col(0.48, 0.5, 0.55)), + ("highlight_tint", col(0.52, 0.5, 0.45)), + ("vignette_amount", fl(2.0)), + ("vignette_midpoint", fl(50.0)), + ("vignette_roundness", fl(20.0)), + ("vignette_feather", fl(50.0)), + ], + ), + ( + "extreme_pos", + vec![ + ("exposure", fl(4.0)), + ("contrast", fl(100.0)), + ("temperature", fl(100.0)), + ("tint", fl(100.0)), + ("whites", fl(100.0)), + ("blacks", fl(100.0)), + ], + ), + ( + "extreme_neg", + vec![ + ("exposure", fl(-4.0)), + ("contrast", fl(-100.0)), + ("temperature", fl(-100.0)), + ("tint", fl(-100.0)), + ("whites", fl(-100.0)), + ("blacks", fl(-100.0)), + ], + ), + ]; + + let (w, h) = (67usize, 41usize); + let cx = FxCtx { + t: Tick::ZERO, + px_scale: 1.0, + seconds: 0.0, + timecode: "", + clip_name: "", + project: None, + env: None, + working: filmcraft_color::WorkingSpace::Rec709, + }; + + for (name, params) in cases { + let e = effect(params); + let mut img_orig = test_picture(w, h); + let mut img_copy = img_orig.clone(); + + lumetri(&mut img_orig, &e, &cx); + + let op = FxOp::eval(&e, &cx, w, h).unwrap_or_else(|| panic!("{name}: eval returned None")); + assert!(op.gpu_ok(), "{name}: expected gpu_ok == true"); + op.apply(&mut img_copy); + + let mut max_diff = 0.0f32; + for (i, (&orig, ©)) in img_orig.px.iter().zip(&img_copy.px).enumerate() { + let diff = (orig - copy).abs(); + if diff > max_diff { + max_diff = diff; + } + assert!(diff <= 1e-6, "{name}: sample {i} differed: orig={orig}, copy={copy}, diff={diff:.2e} > 1e-6"); + } + eprintln!("{name}: max sample diff vs fn lumetri: {max_diff:.2e}"); + } + } +} From 1aa6ff65146c2ce83707a4fdd6fb421ee323b82c Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Thu, 8 Oct 2026 19:48:50 -0700 Subject: [PATCH 10/12] GPU3c: effects-heavy export bench scene (#30) --- crates/ui-egui/examples/bench/playback.rs | 13 ++++++++++-- crates/ui-egui/examples/bench/sections.rs | 24 ++++++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/ui-egui/examples/bench/playback.rs b/crates/ui-egui/examples/bench/playback.rs index a0228982..dee93ea0 100644 --- a/crates/ui-egui/examples/bench/playback.rs +++ b/crates/ui-egui/examples/bench/playback.rs @@ -38,6 +38,12 @@ pub fn import(s: &mut Session, path: &Path) -> ItemId { /// One clip per track from t = 0 (`layers[0]` on V1), with (scale %, position, rotation°, /// opacity %) Motion/Opacity settings per layer. pub fn build_sequence(s: &mut Session, w: u32, h: u32, layers: &[(ItemId, f64, Option<(f64, f64)>, f64, f64)], effects: &[&str], secs: f64) -> ItemId { + let per_layer: Vec<&[&str]> = (0..layers.len()).map(|i| if i == 0 { effects } else { &[][..] }).collect(); + build_sequence_fx(s, w, h, layers, &per_layer, secs) +} + +/// As [`build_sequence`], with the effect ids of each layer (`per_layer[i]` on V(i+1)). +pub fn build_sequence_fx(s: &mut Session, w: u32, h: u32, layers: &[(ItemId, f64, Option<(f64, f64)>, f64, f64)], per_layer: &[&[&str]], secs: f64) -> ItemId { let rate = FrameRate::FPS_23_976; let sizes: Vec<(u32, u32)> = layers .iter() @@ -69,12 +75,15 @@ pub fn build_sequence(s: &mut Session, w: u32, h: u32, layers: &[(ItemId, f64, O if let Some(o) = v.effect_mut("opacity").and_then(|o| o.params.get_mut("opacity")) { o.value = ParamValue::Float(opacity); } - if i == 0 { - for (k, id) in effects.iter().enumerate() { + { + for (k, id) in per_layer.get(i).copied().unwrap_or(&[]).iter().enumerate() { let mut e = find_effect(id).unwrap_or_else(|| panic!("effect {id}")).instance(); let set: &[(&str, f64)] = match *id { "lumetri" => &[("temperature", 18.0), ("contrast", 22.0)], "sharpen" => &[("amount", 40.0)], + "gaussian_blur" => &[("blurriness", 8.0)], + "brightness_contrast" => &[("brightness", 10.0), ("contrast", 15.0)], + "proc_amp" => &[("contrast", 110.0)], _ => &[], }; for (name, val) in set { diff --git a/crates/ui-egui/examples/bench/sections.rs b/crates/ui-egui/examples/bench/sections.rs index e7c06751..27231b73 100644 --- a/crates/ui-egui/examples/bench/sections.rs +++ b/crates/ui-egui/examples/bench/sections.rs @@ -457,7 +457,14 @@ pub fn export(o: &Opts) -> Vec { let dir = std::env::temp_dir().join(format!("filmcraft-bench-export-{}", std::process::id())); std::fs::create_dir_all(&dir).expect("tmp dir"); let mut rows = Vec::new(); - for (format, label, ext) in [("h264", "H.264 + AAC (MP4)", "mp4"), ("prores", "ProRes 422 (MOV)", "mov")].into_iter().filter(|f| o.wants(f.0)) { + // Scenes: (id, format, label, extension, effects-heavy). `--only` matches the id exactly here, + // so `--only h264` runs the plain scene and `--only h264_fx` only the layered one. + let scenes = [ + ("h264", "h264", "H.264 + AAC (MP4)", "mp4", false), + ("h264_fx", "h264", "H.264 + AAC, 3 layers + effects", "mp4", true), + ("prores", "prores", "ProRes 422 (MOV)", "mov", false), + ]; + for (id, format, label, ext, fx) in scenes.into_iter().filter(|f| o.only.as_deref().is_none_or(|x| x == f.0)) { let mut runs = Vec::new(); let mut cpu = Vec::new(); let mut bytes = 0u64; @@ -471,7 +478,14 @@ pub fn export(o: &Opts) -> Vec { for _ in 0..o.repeat { let mut s = Session::default(); let a = playback::import(&mut s, &path); - let seq = playback::build_sequence(&mut s, 1920, 1080, &[(a, 100.0, None, 0.0, 100.0)], &[], secs); + let seq = if fx { + // PiP layers: (scale %, position, rotation, opacity %); effects per layer, GPU-capable only + let layers = [(a, 100.0, None, 0.0, 100.0), (a, 50.0, Some((480.0, -270.0)), 0.0, 100.0), (a, 33.0, Some((-560.0, 300.0)), 0.0, 70.0)]; + let per_layer: [&[&str]; 3] = [&["proc_amp", "gaussian_blur", "vignette"], &["brightness_contrast"], &["sharpen"]]; + playback::build_sequence_fx(&mut s, 1920, 1080, &layers, &per_layer, secs) + } else { + playback::build_sequence(&mut s, 1920, 1080, &[(a, 100.0, None, 0.0, 100.0)], &[], secs) + }; // build_sequence adds video only: add the clip's audio on A1 for the AAC encode s.edit("audio", |p, _| { let rate = FrameRate::FPS_23_976; @@ -504,7 +518,7 @@ pub fn export(o: &Opts) -> Vec { if gpu_timings.frames > 0 { let f = gpu_timings.frames as f64; eprintln!( - " [GPU timings for {format}] lock_wait: {:.1} ms ({:.2} ms/f), submit: {:.1} ms ({:.2} ms/f), map_wait: {:.1} ms ({:.2} ms/f), convert: {:.1} ms ({:.2} ms/f)", + " [GPU timings for {id}] lock_wait: {:.1} ms ({:.2} ms/f), submit: {:.1} ms ({:.2} ms/f), map_wait: {:.1} ms ({:.2} ms/f), convert: {:.1} ms ({:.2} ms/f)", lock_wait.as_secs_f64() * 1000.0, lock_wait.as_secs_f64() * 1000.0 / f, gpu_timings.submit.as_secs_f64() * 1000.0, @@ -523,9 +537,9 @@ pub fn export(o: &Opts) -> Vec { continue; } let best = runs.iter().cloned().fold(f64::MAX, f64::min); - eprintln!("export {format}: {frames} frames in {best:.1} s"); + eprintln!("export {id}: {frames} frames in {best:.1} s"); rows.push(json!({ - "format": label, "frames": frames, "seconds": best, "fps": frames as f64 / best, + "scene": id, "format": label, "frames": frames, "seconds": best, "fps": frames as f64 / best, "realtime": frames as f64 / best / FrameRate::FPS_23_976.as_f64(), "cpu_ms_per_frame": median(&cpu), "mbytes": bytes as f64 / 1e6, "runs_s": runs, "load": load_avg(), // pictures encoded by a hardware encoder during this row (zero with --hw off or where there is none) From 7b73c8e887827ef217b1db2116776277b7d82103 Mon Sep 17 00:00:00 2001 From: Manuel Camacho Date: Fri, 9 Oct 2026 06:31:57 -0700 Subject: [PATCH 11/12] GPU3d: GPU export rendering is Auto by default (#30) Merlin's effects-heavy bench scene (GPU3c) settles it: with layers and effects the GPU compositor exports 2.4x faster at 1080p (13.0 -> 5.0-5.6 s, 737 -> 238 CPU ms/frame) and 3.2x at 4K (38.7-39.3 -> 12.0 s, 2955 -> 775 CPU ms/frame); a single plain clip takes the same time either way (decoding / encoding bound). Every Auto row rendered all its frames on the GPU, without fallbacks. The CPU reference renderer stays the fallback for anything the GPU stage does not run. Trade-off: the GPU matches the CPU within the compositor's parity tolerance, not bit for bit, so an export's bytes can depend on the machine's GPU; Off keeps them byte-reproducible (the determinism tests register no GPU renderer and are unaffected). - settings: GpuRendering::Auto is the #[default]; ExportSettings::default. - test: the default, settings saved without the field, and "off". - docs/performance.md: GPU3 results (1080p and 4K). Co-Authored-By: Claude Opus 5.5 --- crates/export/src/lib.rs | 4 ++-- crates/export/src/settings.rs | 7 +++++-- crates/export/src/settings_tests.rs | 10 ++++++++++ docs/performance.md | 23 +++++++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index e94756eb..185d45ae 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -273,7 +273,7 @@ pub struct ExportSettings { #[serde(default)] pub hardware_encoding: HardwareEncoding, /// Composite the exported frames on the GPU instead of the CPU reference renderer (Export ▸ - /// GPU rendering: Auto / Off). Off = today's CPU behaviour exactly. + /// GPU rendering: Auto / Off, Auto by default). Off = the CPU reference renderer exactly. #[serde(default)] pub gpu_rendering: GpuRendering, /// VBR maximum bitrate (None = 1.5 × target). @@ -458,7 +458,7 @@ impl Default for ExportSettings { h264_level: None, bitrate_mode: BitrateMode::default(), hardware_encoding: HardwareEncoding::default(), - gpu_rendering: GpuRendering::Off, + gpu_rendering: GpuRendering::Auto, max_bitrate_kbps: None, adaptive_bitrate: None, keyframe_distance: None, diff --git a/crates/export/src/settings.rs b/crates/export/src/settings.rs index 87587ba4..d79b86a4 100644 --- a/crates/export/src/settings.rs +++ b/crates/export/src/settings.rs @@ -78,12 +78,15 @@ pub enum HardwareEncoding { /// Whether the picture of an export may be composited on the GPU (`filmcraft-gpu`'s off-screen /// compositor) instead of the CPU reference renderer. Auto = use the GPU when the app registered /// a GPU frame renderer and the machine has an adapter; the CPU result is the fallback either -/// way. Off = today's CPU behaviour exactly (default until GPU export is faster than CPU export). +/// way. Auto is the default: on an effects-heavy edit it exports 2.4× (1080p) to 3.2× (4K) +/// faster, and a single plain clip takes the same time (`docs/performance.md`). The GPU matches +/// the CPU within the compositor's parity tolerance, not bit for bit, so an export's bytes can +/// depend on the machine's GPU. Off = the CPU reference renderer, byte-reproducible everywhere. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum GpuRendering { - Auto, #[default] + Auto, Off, } diff --git a/crates/export/src/settings_tests.rs b/crates/export/src/settings_tests.rs index b4fcbc57..21975aa2 100644 --- a/crates/export/src/settings_tests.rs +++ b/crates/export/src/settings_tests.rs @@ -534,3 +534,13 @@ fn extreme_bitrates_do_not_overflow_encoder_setup() { assert!(result.unwrap().unwrap().is_err(), "an unrepresentable encoder buffer rate must be rejected"); } } + +#[test] +fn gpu_rendering_is_auto_by_default() { + assert_eq!(ExportSettings::default().gpu_rendering, crate::GpuRendering::Auto); + // settings saved before the field existed get the default too + let s: ExportSettings = serde_json::from_value(serde_json::json!({"format": "h264"})).unwrap(); + assert_eq!(s.gpu_rendering, crate::GpuRendering::Auto); + let off: ExportSettings = serde_json::from_value(serde_json::json!({"format": "h264", "gpuRendering": "off"})).unwrap(); + assert_eq!(off.gpu_rendering, crate::GpuRendering::Off); +} diff --git a/docs/performance.md b/docs/performance.md index 5fd62d73..070e408b 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -95,6 +95,29 @@ compositor's savings are a larger part of the export. What is left in a frame without overlays is the Y'CbCr → linear float conversion of the camera picture (4.5 ms in the synthetic case) and the conversion to 8 bits (2 ms). +## Results (GPU3: GPU export rendering, Off → Auto, #30) + +Export ▸ GPU Rendering composites the exported frames on the GPU (`filmcraft-gpu`'s off-screen +compositor, a pool of up to four renderers) instead of the CPU reference renderer, which stays the +fallback for anything the GPU stage does not run. Linux, Ryzen 7 9700X (16 threads) + Radeon +RX 7900 (Mesa 26.2.2), 2026-10-08 / 09, same build, `cargo xtask bench --sections export --only + --hw off|auto`: 191 frames to H.264 + AAC with the built-in encoder. `h264` is one 1080p +clip; `h264_fx` the same clip on V1 with Proc Amp, Gaussian Blur and Vignette, plus two +picture-in-picture copies (50 % with Brightness & Contrast, 33 % at 70 % opacity with Sharpen). +Every Auto row rendered all its frames on the GPU, without fallbacks. + +| scene | Off: s / CPU ms per frame | **Auto**: s / CPU ms per frame | +|---|---|---| +| `h264`, 1080p (load 12–33) | 5.96, 6.08 / 292 | 6.07, 6.36 / 281 | +| `h264_fx`, 1080p (load 12–33) | 13.00, 12.67 / 737 | **5.63, 5.00 / 238** | +| `h264_fx`, 4K sequence and clip (load 2–19) | 38.74, 39.34 / 2955 | **11.99 / 775** | + +On a plain clip the export is decoding- and encoding-bound and both take the same time; with +layers and effects the GPU is 2.4× (1080p) to 3.2× (4K) faster, with a third to a quarter of the +CPU. GPU rendering is therefore **Auto by default**. The GPU result matches the CPU within the +compositor's parity tolerance, not bit for bit: Off keeps exports byte-reproducible on every +machine. (The 4K row used the same scene with the sequence, clip and positions scaled to 4K.) + ## Results (GPU2: standard effects on the GPU compositor, #30, before → after) Before = this change with clips that carry standard effects sent back to the CPU layer path in From cebe4ee9e032f4a91c5b6f5c1690a27feef3850f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Oct 2026 07:24:32 -0400 Subject: [PATCH 12/12] #421: zh-cn rows for the GPU Rendering export strings Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/i18n/zh-cn.tsv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/ui-egui/src/i18n/zh-cn.tsv b/crates/ui-egui/src/i18n/zh-cn.tsv index 5da60959..159acb11 100644 --- a/crates/ui-egui/src/i18n/zh-cn.tsv +++ b/crates/ui-egui/src/i18n/zh-cn.tsv @@ -3485,3 +3485,5 @@ Appearance Mode 外观模式 Next Appearance Mode 下一个外观模式 Sync with system 跟随系统 + GPU Rendering GPU 渲染 + Composite on the GPU when available 可用时在 GPU 上合成