diff --git a/Cargo.lock b/Cargo.lock index 124738ce..aa3eb572 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1351,6 +1351,21 @@ dependencies = [ "serde", ] +[[package]] +name = "filmcraft-gpu" +version = "0.1.0" +dependencies = [ + "filmcraft-color", + "filmcraft-frame", + "filmcraft-geom", + "filmcraft-media", + "filmcraft-render", + "filmcraft-time", + "log", + "pollster 0.4.0", + "wgpu", +] + [[package]] name = "filmcraft-h264" version = "0.1.0" @@ -1439,6 +1454,7 @@ dependencies = [ "filmcraft-engine", "filmcraft-frame", "filmcraft-geom", + "filmcraft-gpu", "filmcraft-media", "filmcraft-project", "filmcraft-render", diff --git a/Cargo.toml b/Cargo.toml index 597b8f10..026d6bf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ filmcraft-codecs = { path = "crates/codecs" } filmcraft-project = { path = "crates/project" } filmcraft-edit = { path = "crates/edit" } filmcraft-render = { path = "crates/render" } +filmcraft-gpu = { path = "crates/gpu" } filmcraft-export = { path = "crates/export" } filmcraft-engine = { path = "crates/engine" } filmcraft-ui-egui = { path = "crates/ui-egui" } diff --git a/apps/filmcraft/src/main.rs b/apps/filmcraft/src/main.rs index 7b287b0c..8680a3c0 100644 --- a/apps/filmcraft/src/main.rs +++ b/apps/filmcraft/src/main.rs @@ -59,6 +59,11 @@ fn main() -> eframe::Result { } let mut app = FilmcraftApp::new(session); app.integrated_titlebar = cfg!(target_os = "macos"); + if let Some(rs) = cc.wgpu_render_state.clone() + && std::env::var_os("FILMCRAFT_CPU_COMPOSITE").is_none() + { + app.set_wgpu(rs); + } if let Some(out) = audio::CpalOut::new() { app.audio = Some(Box::new(out)); } diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml new file mode 100644 index 00000000..b8bba22f --- /dev/null +++ b/crates/gpu/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "filmcraft-gpu" +description = "wgpu compositor: YUV/RGB layer sampling, linear-light premultiplied compositing in WGSL" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +filmcraft-color = { workspace = true } +filmcraft-frame = { workspace = true } +filmcraft-geom = { workspace = true } +filmcraft-render = { workspace = true } +wgpu = { version = "30", default-features = false, features = ["wgsl"] } +log = { workspace = true } + +[dev-dependencies] +pollster = "0.4" +wgpu = { version = "30", features = ["wgsl", "metal", "vulkan", "dx12", "gles"] } +filmcraft-media = { workspace = true } +filmcraft-time = { workspace = true } diff --git a/crates/gpu/src/composite.wgsl b/crates/gpu/src/composite.wgsl new file mode 100644 index 00000000..02621a50 --- /dev/null +++ b/crates/gpu/src/composite.wgsl @@ -0,0 +1,134 @@ +// FilmCraft GPU compositor. +// Layers are drawn as transformed quads; the fragment shader samples the source (YUV planes or +// RGBA) with manual bilinear filtering (textureLoad, so any float format works) and N×N +// supersampling over the pixel footprint when minifying, converts to linear light, premultiplies +// and scales by opacity. Blending into the Rgba16Float accumulator is premultiplied "over". + +struct U { + m0: vec4, // a b c d + m1: vec4, // e f out_w out_h + src: vec4, // src_w src_h chroma_w chroma_h + p0: vec4, // opacity, kind (0 rgba8 srgb straight, 1 rgba16f premul linear, 2 yuv), taps, transfer (0 srgb, 1 linear, 2 pq, 3 hlg) + p1: vec4, // y_off y_scale c_off c_scale (code units) + p2: vec4, // kr kb code_scale footprint +}; + +@group(0) @binding(0) var u: U; +@group(0) @binding(1) var tex0: texture_2d; +@group(0) @binding(2) var tex1: texture_2d; +@group(0) @binding(3) var tex2: texture_2d; + +struct VOut { + @builtin(position) pos: vec4, + @location(0) sp: vec2, +}; + +@vertex +fn vs(@builtin(vertex_index) vi: u32) -> VOut { + var corners = array, 6>(vec2(0.0, 0.0), vec2(1.0, 0.0), vec2(0.0, 1.0), vec2(0.0, 1.0), vec2(1.0, 0.0), vec2(1.0, 1.0)); + let c = corners[vi]; + let s = c * u.src.xy; + let o = vec2(u.m0.x * s.x + u.m0.z * s.y + u.m1.x, u.m0.y * s.x + u.m0.w * s.y + u.m1.y); + var out: VOut; + out.pos = vec4(o.x / u.m1.z * 2.0 - 1.0, 1.0 - o.y / u.m1.w * 2.0, 0.0, 1.0); + out.sp = s; + return out; +} + +fn load4(t: texture_2d, p: vec2) -> vec4 { + let d = vec2(textureDimensions(t)); + let q = p - 0.5; + let i = vec2(floor(q)); + let f = q - floor(q); + let a = textureLoad(t, clamp(i, vec2(0), d - 1), 0); + let b = textureLoad(t, clamp(i + vec2(1, 0), vec2(0), d - 1), 0); + let cc = textureLoad(t, clamp(i + vec2(0, 1), vec2(0), d - 1), 0); + let dd = textureLoad(t, clamp(i + vec2(1, 1), vec2(0), d - 1), 0); + return mix(mix(a, b, f.x), mix(cc, dd, f.x), f.y); +} + +fn srgb_to_linear(v: vec3) -> vec3 { + let lo = v / 12.92; + let hi = pow((max(v, vec3(0.0)) + 0.055) / 1.055, vec3(2.4)); + return select(hi, lo, v <= vec3(0.04045)); +} + +fn pq_eotf(e: vec3) -> vec3 { + let m1 = 0.1593017578125; let m2 = 78.84375; let c1 = 0.8359375; let c2 = 18.8515625; let c3 = 18.6875; + let p = pow(max(e, vec3(0.0)), vec3(1.0 / m2)); + return pow(max(p - c1, vec3(0.0)) / (c2 - c3 * p), vec3(1.0 / m1)) * 100.0; +} + +fn hlg_inv(e: vec3) -> vec3 { + let a = 0.17883277; let b = 0.28466892; let c = 0.55991073; + let lo = e * e / 3.0; + let hi = (exp((e - c) / a) + b) / 12.0; + return select(hi, lo, e <= vec3(0.5)); +} + +fn to_linear(v: vec3) -> vec3 { + let t = u32(u.p0.w); + if t == 1u { return v; } + if t == 2u { return pq_eotf(v); } + if t == 3u { return hlg_inv(v); } + return srgb_to_linear(clamp(v, vec3(0.0), vec3(1.0))); +} + +// One linear premultiplied sample at source position p. +fn sample(p: vec2) -> vec4 { + let kind = u32(u.p0.y); + if kind == 0u { + // rgba8 sRGB texture: loads are already linear, straight alpha + let c = load4(tex0, p); + return vec4(c.rgb * c.a, c.a); + } + if kind == 1u { + return load4(tex0, p); + } + let cs = u.p2.z; + let yc = load4(tex0, p).r * cs; + let cp = p * u.src.zw / u.src.xy; + let cb = load4(tex1, cp).r * cs; + let cr = load4(tex2, cp).r * cs; + let y = (yc - u.p1.x) / u.p1.y; + let b = (cb - u.p1.z) / u.p1.w; + let r = (cr - u.p1.z) / u.p1.w; + let kr = u.p2.x; let kb = u.p2.y; let kg = 1.0 - kr - kb; + let R = y + 2.0 * (1.0 - kr) * r; + let B = y + 2.0 * (1.0 - kb) * b; + let G = (y - kr * R - kb * B) / kg; + return vec4(to_linear(vec3(R, G, B)), 1.0); +} + +@fragment +fn fs(in: VOut) -> @location(0) vec4 { + let n = max(u32(u.p0.z), 1u); + let fp = u.p2.w; + var acc = vec4(0.0); + for (var j = 0u; j < n; j++) { + for (var i = 0u; i < n; i++) { + let off = (vec2(f32(i), f32(j)) + 0.5) / f32(n) - 0.5; + acc += sample(in.sp + off * fp); + } + } + return acc / f32(n * n) * u.p0.x; +} + +// ---- final pass: accumulator (linear premul) over black → sRGB target + +@group(0) @binding(0) var accum: texture_2d; + +@vertex +fn vs_full(@builtin(vertex_index) vi: u32) -> @builtin(position) vec4 { + let x = f32((vi << 1u) & 2u) * 2.0 - 1.0; + let y = f32(vi & 2u) * 2.0 - 1.0; + return vec4(x, y, 0.0, 1.0); +} + +@fragment +fn fs_full(@builtin(position) pos: vec4) -> @location(0) vec4 { + let c = clamp(textureLoad(accum, vec2(pos.xy), 0).rgb, vec3(0.0), vec3(1.0)); + let lo = c * 12.92; + let hi = 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055; + return vec4(select(hi, lo, c <= vec3(0.0031308)), 1.0); +} diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs new file mode 100644 index 00000000..83e46f79 --- /dev/null +++ b/crates/gpu/src/lib.rs @@ -0,0 +1,467 @@ +//! The wgpu compositor. +//! +//! Executes a [`FramePlan`](filmcraft_render::plan::FramePlan) on the GPU: every layer's source is +//! uploaded as textures (Y/Cb/Cr planes stay YUV — conversion to linear RGB happens per pixel in +//! the shader), drawn as a transformed quad with premultiplied "over" blending into a Rgba16Float +//! accumulator, then resolved over black into an sRGB `Rgba8UnormSrgb` texture that the UI +//! registers as a native texture. Uploads are cached by pixel-buffer identity, so a paused frame or +//! a still costs nothing. +//! +//! The CPU plan executor (`filmcraft_render::plan::execute_cpu`) is the oracle; tests compare. + +use std::collections::HashMap; +use std::sync::Arc; + +use filmcraft_color::{Matrix, Range, Transfer}; +use filmcraft_frame::{Chroma, PixelData, VideoFrame}; +use filmcraft_render::plan::{FramePlan, PlanLayer}; + +/// Output texture format: gamma-encoded RGBA8 (what egui expects of native textures); the resolve +/// shader applies the sRGB encoding. +pub const OUTPUT_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; +const ACCUM_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba16Float; + +struct Uploaded { + views: [wgpu::TextureView; 3], + kind: u32, + code_scale: f32, + chroma: (u32, u32), + last_used: u64, +} + +pub struct GpuCompositor { + device: wgpu::Device, + queue: wgpu::Queue, + layer_pipeline: wgpu::RenderPipeline, + final_pipeline: wgpu::RenderPipeline, + layer_bgl: wgpu::BindGroupLayout, + final_bgl: wgpu::BindGroupLayout, + accum: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>, + output: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>, + uploads: HashMap<(usize, u32, u32), Uploaded>, + clock: u64, + dummy: wgpu::TextureView, + /// Total bytes uploaded (stats). + pub uploaded_bytes: u64, +} + +fn tex_entry(binding: u32) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + } +} + +/// f32 → IEEE half (round to nearest even), no dependency needed. +pub fn f32_to_f16(v: f32) -> u16 { + let x = v.to_bits(); + let sign = ((x >> 16) & 0x8000) as u16; + let exp = ((x >> 23) & 0xff) as i32; + let mant = x & 0x7f_ffff; + if exp == 0xff { + return sign | 0x7c00 | if mant != 0 { 0x200 } else { 0 }; + } + let e = exp - 127 + 15; + if e >= 0x1f { + return sign | 0x7c00; + } + if e <= 0 { + if e < -10 { + return sign; + } + let m = (mant | 0x80_0000) >> (1 - e); + let round = (m >> 12) & 1; + return sign | (((m >> 13) + round) as u16); + } + let m = mant >> 13; + let round_bits = mant & 0x1fff; + let mut h = (sign as u32) | ((e as u32) << 10) | m; + if round_bits > 0x1000 || (round_bits == 0x1000 && (m & 1) == 1) { + h += 1; + } + h as u16 +} + +impl GpuCompositor { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("filmcraft-composite"), + source: wgpu::ShaderSource::Wgsl(include_str!("composite.wgsl").into()), + }); + let layer_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("layer"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }, + tex_entry(1), + tex_entry(2), + tex_entry(3), + ], + }); + let final_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("final"), + entries: &[wgpu::BindGroupLayoutEntry { binding: 0, ..tex_entry(0) }], + }); + let pl = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("layer"), bind_group_layouts: &[Some(&layer_bgl)], immediate_size: 0 }); + let layer_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("layer"), + layout: Some(&pl), + vertex: wgpu::VertexState { module: &shader, entry_point: Some("vs"), compilation_options: Default::default(), buffers: &[] }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: ACCUM_FORMAT, + blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + let fpl = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("final"), bind_group_layouts: &[Some(&final_bgl)], immediate_size: 0 }); + let final_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("final"), + layout: Some(&fpl), + vertex: wgpu::VertexState { module: &shader, entry_point: Some("vs_full"), compilation_options: Default::default(), buffers: &[] }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_full"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { format: OUTPUT_FORMAT, blend: None, write_mask: wgpu::ColorWrites::ALL })], + }), + multiview_mask: None, + cache: None, + }); + let dummy_tex = device.create_texture(&wgpu::TextureDescriptor { + label: Some("dummy"), + size: wgpu::Extent3d { width: 1, height: 1, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + let dummy = dummy_tex.create_view(&Default::default()); + Self { + device: device.clone(), + queue: queue.clone(), + layer_pipeline, + final_pipeline, + layer_bgl, + final_bgl, + accum: None, + output: None, + uploads: HashMap::new(), + clock: 0, + dummy, + uploaded_bytes: 0, + } + } + + fn target( + device: &wgpu::Device, + slot: &mut Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>, + w: u32, + h: u32, + format: wgpu::TextureFormat, + extra: wgpu::TextureUsages, + ) { + if slot.as_ref().is_some_and(|s| s.2 == (w, h)) { + return; + } + let t = device.create_texture(&wgpu::TextureDescriptor { + label: Some("filmcraft-target"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING | extra, + view_formats: &[], + }); + let v = t.create_view(&Default::default()); + *slot = Some((t, v, (w, h))); + } + + fn plane_texture(&mut self, w: u32, h: u32, format: wgpu::TextureFormat, bytes: &[u8], bpp: u32) -> wgpu::TextureView { + let t = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("filmcraft-plane"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { texture: &t, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + bytes, + wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w * bpp), rows_per_image: Some(h) }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + self.uploaded_bytes += bytes.len() as u64; + t.create_view(&Default::default()) + } + + /// Upload (or reuse) the textures of a frame. + fn upload(&mut self, f: &VideoFrame) -> (usize, u32, u32) { + let id = match &f.data { + PixelData::Rgba8(d) => Arc::as_ptr(d) as *const u8 as usize, + PixelData::RgbaF32(d) => Arc::as_ptr(d) as *const u8 as usize, + PixelData::Yuv8 { planes, .. } => Arc::as_ptr(&planes[0]) as *const u8 as usize, + PixelData::Yuv16 { planes, .. } => Arc::as_ptr(&planes[0]) as *const u8 as usize, + }; + let key = (id, f.width, f.height); + self.clock += 1; + if let Some(u) = self.uploads.get_mut(&key) { + u.last_used = self.clock; + return key; + } + let (w, h) = (f.width, f.height); + let d0 = self.dummy.clone(); + let dummy = || d0.clone(); + let up = match &f.data { + PixelData::Rgba8(d) => { + let v = self.plane_texture(w, h, wgpu::TextureFormat::Rgba8UnormSrgb, d, 4); + Uploaded { views: [v, dummy(), dummy()], kind: 0, code_scale: 1.0, chroma: (w, h), last_used: self.clock } + } + PixelData::RgbaF32(d) => { + let half: Vec = d.iter().flat_map(|v| f32_to_f16(*v).to_le_bytes()).collect(); + let v = self.plane_texture(w, h, wgpu::TextureFormat::Rgba16Float, &half, 8); + Uploaded { views: [v, dummy(), dummy()], kind: 1, code_scale: 1.0, chroma: (w, h), last_used: self.clock } + } + PixelData::Yuv8 { planes, chroma, .. } => { + let (sx, sy) = chroma.shifts(); + let (cw, ch) = (w.div_ceil(1 << sx), h.div_ceil(1 << sy)); + let y = self.plane_texture(w, h, wgpu::TextureFormat::R8Unorm, &planes[0], 1); + let u = self.plane_texture(cw, ch, wgpu::TextureFormat::R8Unorm, &planes[1], 1); + let v = self.plane_texture(cw, ch, wgpu::TextureFormat::R8Unorm, &planes[2], 1); + Uploaded { views: [y, u, v], kind: 2, code_scale: 255.0, chroma: (cw, ch), last_used: self.clock } + } + PixelData::Yuv16 { planes, chroma, bits, .. } => { + let (sx, sy) = chroma.shifts(); + let (cw, ch) = (w.div_ceil(1 << sx), h.div_ceil(1 << sy)); + let scale = (1u32 << bits) as f32; + let conv = |p: &[u16]| -> Vec { p.iter().flat_map(|v| f32_to_f16(*v as f32 / scale).to_le_bytes()).collect() }; + let (py, pu, pv) = (conv(&planes[0]), conv(&planes[1]), conv(&planes[2])); + let y = self.plane_texture(w, h, wgpu::TextureFormat::R16Float, &py, 2); + let u = self.plane_texture(cw, ch, wgpu::TextureFormat::R16Float, &pu, 2); + let v = self.plane_texture(cw, ch, wgpu::TextureFormat::R16Float, &pv, 2); + let _ = Chroma::C420; + Uploaded { views: [y, u, v], kind: 2, code_scale: scale, chroma: (cw, ch), last_used: self.clock } + } + }; + self.uploads.insert(key, up); + // keep the most recent uploads (enough for several stacked layers + playback lookahead) + if self.uploads.len() > 24 { + let mut v: Vec<(u64, (usize, u32, u32))> = self.uploads.iter().map(|(k, u)| (u.last_used, *k)).collect(); + v.sort_unstable(); + for (_, k) in v.into_iter().take(self.uploads.len() - 24) { + self.uploads.remove(&k); + } + } + key + } + + fn uniforms(&self, l: &PlanLayer, key: (usize, u32, u32), out: (u32, u32)) -> [f32; 24] { + let up = &self.uploads[&key]; + let f = &l.frame; + let m = &l.matrix; + let (kr, kb) = f.color.matrix.kr_kb(); + let bits_scale = up.code_scale / 255.0; // code units relative to 8-bit + let (yo, ys, co, cs) = match (f.color.range, up.kind) { + (Range::Limited, 2) => (16.0 * bits_scale, 219.0 * bits_scale, 128.0 * bits_scale, 224.0 * bits_scale), + (Range::Full, 2) => (0.0, up.code_scale - 1.0, up.code_scale / 2.0, up.code_scale - 1.0), + _ => (0.0, 1.0, 0.0, 1.0), + }; + let transfer = match f.color.transfer { + Transfer::Linear => 1.0, + Transfer::Pq => 2.0, + Transfer::Hlg => 3.0, + _ => 0.0, + }; + // source pixels per output pixel → supersampling taps + let sx = (m.a * m.a + m.b * m.b).sqrt(); + let sy = (m.c * m.c + m.d * m.d).sqrt(); + let footprint = (1.0 / sx.min(sy).max(1e-6)) as f32; + let taps = if footprint > 1.25 { footprint.ceil().min(4.0) } else { 1.0 }; + let _ = Matrix::Bt709; + [ + m.a as f32, + m.b as f32, + m.c as f32, + m.d as f32, + m.e as f32, + m.f as f32, + out.0 as f32, + out.1 as f32, + f.width as f32, + f.height as f32, + up.chroma.0 as f32, + up.chroma.1 as f32, + l.opacity, + up.kind as f32, + taps, + transfer, + yo, + ys, + co, + cs, + kr, + kb, + up.code_scale, + footprint.max(1.0), + ] + } + + /// Composite a plan; returns the output view (sRGB, over black) and its size. + pub fn composite(&mut self, plan: &FramePlan) -> (&wgpu::TextureView, (u32, u32)) { + let owned; + let (w, h, layers): (u32, u32, &[PlanLayer]) = match plan { + FramePlan::Layers { width, height, layers } => (*width as u32, *height as u32, layers.as_slice()), + FramePlan::Image(img) => { + owned = [PlanLayer { + frame: Arc::new(VideoFrame::rgba_f32(img.w as u32, img.h as u32, img.px.clone())), + matrix: filmcraft_geom::Affine::IDENTITY, + opacity: 1.0, + }]; + (img.w as u32, img.h as u32, &owned[..]) + } + }; + let (w, h) = (w.max(1), h.max(1)); + Self::target(&self.device, &mut self.accum, w, h, ACCUM_FORMAT, wgpu::TextureUsages::empty()); + Self::target(&self.device, &mut self.output, w, h, OUTPUT_FORMAT, wgpu::TextureUsages::COPY_SRC); + let keys: Vec<(usize, u32, u32)> = layers.iter().map(|l| self.upload(&l.frame)).collect(); + let mut bind_groups = Vec::with_capacity(layers.len()); + for (l, k) in layers.iter().zip(&keys) { + let u = self.uniforms(l, *k, (w, h)); + let bytes: Vec = u.iter().flat_map(|v| v.to_le_bytes()).collect(); + let buf = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("layer-u"), + size: bytes.len() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + self.queue.write_buffer(&buf, 0, &bytes); + let up = &self.uploads[k]; + let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("layer"), + layout: &self.layer_bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&up.views[0]) }, + wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&up.views[1]) }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&up.views[2]) }, + ], + }); + bind_groups.push(bg); + } + let accum_view = &self.accum.as_ref().expect("accum").1; + let out_view = &self.output.as_ref().expect("output").1; + let final_bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("final"), + layout: &self.final_bgl, + entries: &[wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(accum_view) }], + }); + let mut enc = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("filmcraft-composite") }); + { + let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("layers"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: accum_view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), store: wgpu::StoreOp::Store }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(&self.layer_pipeline); + for bg in &bind_groups { + pass.set_bind_group(0, bg, &[]); + pass.draw(0..6, 0..1); + } + } + { + let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("resolve"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: out_view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), store: wgpu::StoreOp::Store }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(&self.final_pipeline); + pass.set_bind_group(0, &final_bg, &[]); + pass.draw(0..3, 0..1); + } + self.queue.submit([enc.finish()]); + (&self.output.as_ref().expect("output").1, (w, h)) + } + + /// Read the output back as RGBA8 (tests / screenshots / thumbnails). + pub fn read_output(&self) -> Option<(u32, u32, Vec)> { + let (tex, _, (w, h)) = self.output.as_ref()?; + let row = (w * 4).div_ceil(256) * 256; + let buf = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("readback"), + size: (row * h) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mut enc = self.device.create_command_encoder(&Default::default()); + enc.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { texture: tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyBufferInfo { buffer: &buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(*h) } }, + wgpu::Extent3d { width: *w, height: *h, depth_or_array_layers: 1 }, + ); + self.queue.submit([enc.finish()]); + let slice = buf.slice(..); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); + rx.recv().ok()?.ok()?; + let data = slice.get_mapped_range().ok()?; + let mut out = Vec::with_capacity((w * h * 4) as usize); + for y in 0..*h { + out.extend_from_slice(&data[(y * row) as usize..(y * row + w * 4) as usize]); + } + drop(data); + buf.unmap(); + Some((*w, *h, out)) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs new file mode 100644 index 00000000..9aacbde9 --- /dev/null +++ b/crates/gpu/src/tests.rs @@ -0,0 +1,79 @@ +use super::*; +use filmcraft_geom::{Affine, Vec2}; +use filmcraft_render::plan::execute_cpu; + +fn device() -> Option<(wgpu::Device, wgpu::Queue)> { + let instance = wgpu::Instance::default(); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?; + pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok() +} + +fn yuv_frame(w: u32, h: u32) -> Arc { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let y: Vec = (0..w * h).map(|i| (16 + ((i % w) * 219 / w)) as u8).collect(); + let u: Vec = (0..cw * ch).map(|i| (64 + (i / cw) * 128 / ch) as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (200 - (i % cw) * 100 / cw) as u8).collect(); + Arc::new(VideoFrame { + width: w, + height: h, + data: PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma: Chroma::C420, alpha: None }, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + pts: Default::default(), + }) +} + +#[test] +fn gpu_matches_cpu_plan() { + let Some((dev, q)) = device() else { + eprintln!("no GPU adapter; skipping"); + return; + }; + let mut c = GpuCompositor::new(&dev, &q); + let pixels = filmcraft_media::generators::render( + &filmcraft_media::Generator::Demo(filmcraft_media::DemoScene::OceanSunset), + 320, + 180, + 1.0, + 24, + filmcraft_time::FrameRate::FPS_24, + ); + let rgba = Arc::new(VideoFrame::rgba8(320, 180, pixels)); + let (w, h) = (320usize, 180usize); + let plan = FramePlan::Layers { + width: w, + height: h, + layers: vec![ + PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.5, 0.5), opacity: 1.0 }, + PlanLayer { frame: rgba, matrix: Affine::motion(Vec2::new(200.0, 100.0), Vec2::new(0.4, 0.4), 12.0, Vec2::new(160.0, 90.0)), opacity: 0.7 }, + ], + }; + let cpu = execute_cpu(&plan).over_black_rgba8(); + c.composite(&plan); + let (gw, gh, gpu) = c.read_output().expect("readback"); + assert_eq!((gw as usize, gh as usize), (w, h)); + // compare away from antialiased edges: mean abs error and 99th percentile + let mut diffs: Vec = + cpu.chunks(4).zip(gpu.chunks(4)).map(|(a, b)| (0..3).map(|k| (a[k] as i32 - b[k] as i32).unsigned_abs()).max().unwrap_or(0)).collect(); + diffs.sort_unstable(); + let p99 = diffs[diffs.len() * 99 / 100]; + let mean = diffs.iter().sum::() as f64 / diffs.len() as f64; + assert!(p99 <= 6 && mean < 1.5, "p99 {p99}, mean {mean}"); + // cache: compositing the same plan again uploads nothing + let before = c.uploaded_bytes; + c.composite(&plan); + assert_eq!(c.uploaded_bytes, before); +} + +#[test] +fn half_float_conversion() { + for v in [0.0f32, 1.0, 0.5, 0.123, 65504.0, -2.0, 1e-5] { + let h = f32_to_f16(v); + // decode + let s = if h & 0x8000 != 0 { -1.0 } else { 1.0 }; + let e = ((h >> 10) & 0x1f) as i32; + let m = (h & 0x3ff) as f32; + let d = if e == 0 { s * m * 2f32.powi(-24) } else { s * (1.0 + m / 1024.0) * 2f32.powi(e - 15) }; + assert!((d - v).abs() <= v.abs() * 1e-3 + 1e-6, "{v} → {d}"); + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 531678c0..4f6de63e 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -11,6 +11,7 @@ pub mod audio; pub mod blend; pub mod effects; pub mod image; +pub mod plan; pub mod transitions; use std::sync::Arc; @@ -130,7 +131,7 @@ fn with_opacity(mut img: Image, op: f32) -> Image { img } -fn opacity_blend(item: &TrackItem, mt: Tick) -> (f32, Blend) { +pub(crate) fn opacity_blend(item: &TrackItem, mt: Tick) -> (f32, Blend) { match item.effect("opacity") { Some(e) if e.enabled => { let op = (e.f64_at("opacity", mt) / 100.0).clamp(0.0, 1.0) as f32; @@ -145,7 +146,7 @@ fn opacity_blend(item: &TrackItem, mt: Tick) -> (f32, Blend) { } /// Size of an item's source at full resolution. -fn source_size(project: &Project, item: ItemId) -> Option<(u32, u32)> { +pub(crate) fn source_size(project: &Project, item: ItemId) -> Option<(u32, u32)> { match &project.item(item)?.kind { ItemKind::Media(m) => m.info.video.as_ref().map(|v| (v.width, v.height)), ItemKind::Sequence(s) => Some((s.settings.width, s.settings.height)), @@ -186,7 +187,7 @@ pub fn motion_matrix(seq: &Sequence, item: &TrackItem, src: (u32, u32), mt: Tick /// Render one track item's layer at timeline time `t` into a canvas-sized image. /// Returns (layer, opacity, blend). -fn item_layer( +pub(crate) fn item_layer( project: &Project, seq: &Sequence, item: &TrackItem, diff --git a/crates/render/src/plan.rs b/crates/render/src/plan.rs new file mode 100644 index 00000000..993a24d2 --- /dev/null +++ b/crates/render/src/plan.rs @@ -0,0 +1,172 @@ +//! Frame plans for the GPU compositor. +//! +//! [`plan_frame`] resolves what is visible at a time into a list of layers the GPU can draw +//! directly: a decoded source frame (YUV planes or RGBA), the matrix from source pixels to output +//! pixels, and an opacity. Anything the shaders don't cover yet — non-Normal blend modes, +//! standard effects, adjustment layers, nested sequences, non-dissolve transitions — is rendered +//! on the CPU for that layer (or the whole frame) and handed over as a pre-composited image, so +//! the GPU path is always exact with respect to the CPU reference. + +use std::sync::Arc; + +use filmcraft_frame::VideoFrame; +use filmcraft_geom::Affine; +use filmcraft_media::FrameRequest; +use filmcraft_project::{ItemId, ItemKind, Project, Sequence, TrackItem}; +use filmcraft_time::Tick; + +use crate::{Blend, RenderOptions, SourceProvider, motion_matrix, output_size}; + +/// One layer for the GPU, bottom to top. +#[derive(Clone)] +pub struct PlanLayer { + pub frame: Arc, + /// Maps frame pixels (0..w, 0..h) to output pixels. + pub matrix: Affine, + pub opacity: f32, +} + +#[derive(Clone)] +pub enum FramePlan { + /// Draw these layers over black. + Layers { width: usize, height: usize, layers: Vec }, + /// The CPU produced the final image (fallback). + Image(crate::Image), +} + +fn cpu_frame(img: crate::Image) -> Arc { + Arc::new(VideoFrame::rgba_f32(img.w as u32, img.h as u32, img.px)) +} + +fn simple_transition(id: &str) -> bool { + matches!(id, "cross_dissolve" | "dip_to_black" | "dip_to_white" | "non_additive_dissolve" | "morph_cut") +} + +/// Whether a track item can be drawn by the GPU as-is (no standard effects, Normal blend). +fn gpu_simple(project: &Project, item: &TrackItem, mt: Tick) -> bool { + let is_media = project.item(item.item).is_some_and(|p| matches!(p.kind, ItemKind::Media(_) | ItemKind::Subclip { .. })); + let no_fx = !item.has_standard_effects(); + let normal = + item.effect("opacity").is_none_or(|e| !e.enabled || e.param("blend").is_none_or(|p| matches!(p.value, filmcraft_project::ParamValue::Choice(0)))); + let _ = mt; + is_media && no_fx && normal +} + +/// Plan the frame at timeline `t`. +pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> FramePlan { + let Some(seq) = project.sequence(seq_id) else { return FramePlan::Image(crate::Image::new(1, 1)) }; + let (w, h) = output_size(seq, opts.scale); + // Whole-frame fallback: adjustment layers or complex transitions anywhere at t. + for tr in &seq.video_tracks { + if !tr.enabled { + continue; + } + if let Some(trn) = tr.transitions.iter().find(|x| x.range().contains(t)) + && !simple_transition(&trn.effect.effect) + { + return FramePlan::Image(crate::render_sequence(project, seq_id, t, opts, sources)); + } + if let Some(it) = tr.item_at(t) + && (project.item(it.item).is_some_and(|p| matches!(p.kind, ItemKind::AdjustmentLayer { .. })) + || crate::opacity_blend(it, it.source_time_at(t)).1 != Blend::Normal) + { + return FramePlan::Image(crate::render_sequence(project, seq_id, t, opts, sources)); + } + } + let mut layers = Vec::new(); + for tr in &seq.video_tracks { + if !tr.enabled { + continue; + } + if let Some(trn) = tr.transitions.iter().find(|x| x.range().contains(t)) { + let mut p = trn.progress(t) as f32; + if trn.reverse { + p = 1.0 - p; + } + let a = trn.from.and_then(|id| tr.item(id)).filter(|i| i.enabled); + let b = trn.to.and_then(|id| tr.item(id)).filter(|i| i.enabled); + match trn.effect.effect.as_str() { + "dip_to_black" | "dip_to_white" => { + let col = if trn.effect.effect == "dip_to_black" { [0.0, 0.0, 0.0, 1.0] } else { [1.0, 1.0, 1.0, 1.0] }; + layers.push(PlanLayer { + frame: Arc::new(VideoFrame::rgba_f32(1, 1, col.to_vec())), + matrix: Affine::scale(w as f64, h as f64), + opacity: 1.0, + }); + let (it, k) = if p < 0.5 { (a, 1.0 - p * 2.0) } else { (b, (p - 0.5) * 2.0) }; + if let Some(it) = it { + push_item(project, seq, it, t, opts, sources, k, &mut layers); + } + } + _ => { + // cross dissolve: A at full, B over it at p (premultiplied over == linear mix when A is opaque) + if let Some(it) = a { + push_item(project, seq, it, t, opts, sources, 1.0 - if b.is_none() { p } else { 0.0 }, &mut layers); + } + if let Some(it) = b { + push_item(project, seq, it, t, opts, sources, p, &mut layers); + } + } + } + continue; + } + let Some(item) = tr.item_at(t) else { continue }; + if !item.enabled { + continue; + } + push_item(project, seq, item, t, opts, sources, 1.0, &mut layers); + } + FramePlan::Layers { width: w, height: h, layers } +} + +#[allow(clippy::too_many_arguments)] +fn push_item( + project: &Project, + seq: &Sequence, + item: &TrackItem, + t: Tick, + opts: RenderOptions, + sources: &dyn SourceProvider, + extra_opacity: f32, + out: &mut Vec, +) { + let mt = item.source_time_at(t); + let (op, bl) = crate::opacity_blend(item, mt); + if gpu_simple(project, item, mt) && bl == Blend::Normal { + let Some(src) = sources.source(item.item) else { return }; + let Some(size) = crate::source_size(project, item.item) else { return }; + let motion = motion_matrix(seq, item, size, mt); + let lin = ((motion.a * motion.a + motion.b * motion.b).sqrt()).max((motion.c * motion.c + motion.d * motion.d).sqrt()); + let want = (lin * opts.scale as f64).clamp(1.0 / 64.0, 1.0) as f32; + let Ok(frame) = src.video_frame(FrameRequest { time: mt, scale: want }) else { return }; + let px_scale = frame.width as f64 / size.0.max(1) as f64; + let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion).then_apply(&Affine::scale(1.0 / px_scale, 1.0 / px_scale)); + out.push(PlanLayer { frame, matrix: m, opacity: op * extra_opacity }); + return; + } + // CPU-rendered layer (standard effects): drawn by the GPU as a pre-rendered canvas image. + let tc = filmcraft_time::format_time(t, seq.settings.frame_rate, seq.settings.drop_frame, filmcraft_time::TimeDisplay::Timecode, 48_000); + if let Some((img, op2, _)) = crate::item_layer(project, seq, item, t, opts, sources, &tc) { + out.push(PlanLayer { frame: cpu_frame(img), matrix: Affine::IDENTITY, opacity: op2 * extra_opacity }); + } +} + +/// Execute a plan on the CPU (reference for the GPU compositor). +pub fn execute_cpu(plan: &FramePlan) -> crate::Image { + match plan { + FramePlan::Image(img) => img.clone(), + FramePlan::Layers { width, height, layers } => { + let mut canvas = crate::Image::new(*width, *height); + for l in layers { + let src = crate::Image { w: l.frame.width as usize, h: l.frame.height as usize, px: l.frame.to_linear_f32() }; + let placed = if l.matrix == Affine::scale(*width as f64, *height as f64) && src.w == 1 && src.h == 1 { + crate::Image::filled(*width, *height, src.get(0, 0)) + } else { + src.transformed(*width, *height, &l.matrix) + }; + crate::blend::composite(&mut canvas, &placed, l.opacity, Blend::Normal); + } + canvas + } + } +} diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index b4b75210..e0e71880 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -167,3 +167,29 @@ fn audio_mix_bars_tone() { let expect = 10f32.powf(-26.0 / 20.0); assert!((peak - expect).abs() < 0.01, "{peak} vs {expect}"); } + +#[test] +fn plan_matches_reference_renderer() { + let (mut p, red, ocean, seq, map) = setup(); + place(&mut p, seq, 0, ocean, 0, 48); + let top = place(&mut p, seq, 1, red, 10, 20); + { + let s = p.sequence_mut(seq).unwrap(); + let (_, it) = s.find_item_mut(top).unwrap(); + let m = it.effect_mut("motion").unwrap(); + m.params.get_mut("scale").unwrap().value = ParamValue::Float(40.0); + m.params.get_mut("rotation").unwrap().value = ParamValue::Float(15.0); + it.effect_mut("opacity").unwrap().params.get_mut("opacity").unwrap().value = ParamValue::Float(70.0); + } + let r = FrameRate::FPS_24; + for f in [0, 12, 20, 40] { + let opts = RenderOptions { scale: 0.5, ..Default::default() }; + let reference = render_sequence(&p, seq, r.tick_of(f), opts, &map); + let plan = plan::plan_frame(&p, seq, r.tick_of(f), opts, &map); + assert!(matches!(plan, plan::FramePlan::Layers { .. }), "simple frame should be GPU-drawable"); + let got = plan::execute_cpu(&plan); + assert_eq!((got.w, got.h), (reference.w, reference.h)); + let max = got.px.iter().zip(&reference.px).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); + assert!(max < 0.02, "frame {f}: max diff {max}"); + } +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 88f70b7f..4a45e40a 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -19,6 +19,7 @@ filmcraft-project = { workspace = true } filmcraft-edit = { workspace = true } filmcraft-render = { workspace = true } filmcraft-engine = { workspace = true } +filmcraft-gpu = { workspace = true } egui = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit"] } serde = { workspace = true } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index b56c7afa..20925140 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -18,6 +18,8 @@ pub enum Target { Sequence(ItemId), /// A single project item at a media time (Source monitor, thumbnails). Item(ItemId), + /// A GPU frame plan of a sequence (decoded layers + transforms), composited on the GPU. + SequencePlan(ItemId), } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -48,6 +50,7 @@ struct Shared { queue: Mutex>, cv: Condvar, done: Mutex, + plans: Mutex, u64)>>, in_flight: Mutex>, } @@ -92,6 +95,7 @@ impl FrameServer { queue: Mutex::new(VecDeque::new()), cv: Condvar::new(), done: Mutex::new(Cache { map: HashMap::new(), bytes: 0, budget: 768 << 20, clock: 0 }), + plans: Mutex::new(HashMap::new()), in_flight: Mutex::new(Vec::new()), }); let repaint: Arc>> = Arc::new(Mutex::new(None)); @@ -125,9 +129,24 @@ impl FrameServer { }) } + pub fn get_plan(&self, k: &FrameKey) -> Option> { + self.shared.plans.lock().unwrap_or_else(|e| e.into_inner()).get(k).map(|(p, _)| p.clone()) + } + + /// Nearest cached plan at or before `frame`. + pub fn nearest_plan(&self, key: FrameKey, max_back: i64) -> Option<(FrameKey, Arc)> { + let g = self.shared.plans.lock().unwrap_or_else(|e| e.into_inner()); + (0..=max_back).find_map(|d| { + let k = FrameKey { frame: key.frame - d, ..key }; + g.get(&k).map(|(p, _)| (k, p.clone())) + }) + } + /// Queue a job unless it is cached, queued or in flight. pub fn request(&self, key: FrameKey, time: Tick, scale: f32, project: &Arc, prio: u32) { - if self.shared.done.lock().unwrap_or_else(|e| e.into_inner()).map.contains_key(&key) { + if self.shared.done.lock().unwrap_or_else(|e| e.into_inner()).map.contains_key(&key) + || self.shared.plans.lock().unwrap_or_else(|e| e.into_inner()).contains_key(&key) + { return; } if self.shared.in_flight.lock().unwrap_or_else(|e| e.into_inner()).contains(&key) { @@ -168,8 +187,20 @@ impl FrameServer { fn run_one_sync(&self) { let job = { self.shared.queue.lock().unwrap_or_else(|e| e.into_inner()).pop_front() }; if let Some(job) = job { - let img = render_job(&job, &self.pool, &self.services); - self.shared.done.lock().unwrap_or_else(|e| e.into_inner()).insert(job.key, Arc::new(img)); + if let Target::SequencePlan(seq) = job.key.target { + let provider = self.pool.provider(job.project.clone(), self.services.clone()); + let plan = filmcraft_render::plan::plan_frame( + &job.project, + seq, + job.time, + filmcraft_render::RenderOptions { scale: job.scale, ..Default::default() }, + &provider, + ); + self.shared.plans.lock().unwrap_or_else(|e| e.into_inner()).insert(job.key, (Arc::new(plan), 0)); + } else { + let img = render_job(&job, &self.pool, &self.services); + self.shared.done.lock().unwrap_or_else(|e| e.into_inner()).insert(job.key, Arc::new(img)); + } } } } @@ -189,8 +220,29 @@ fn worker(sh: Arc, pool: Arc, services: Arc, re } }; sh.in_flight.lock().unwrap_or_else(|e| e.into_inner()).push(job.key); - let img = render_job(&job, &pool, &services); - sh.done.lock().unwrap_or_else(|e| e.into_inner()).insert(job.key, Arc::new(img)); + if let Target::SequencePlan(seq) = job.key.target { + let provider = pool.provider(job.project.clone(), services.clone()); + let plan = filmcraft_render::plan::plan_frame( + &job.project, + seq, + job.time, + filmcraft_render::RenderOptions { scale: job.scale, ..Default::default() }, + &provider, + ); + let mut g = sh.plans.lock().unwrap_or_else(|e| e.into_inner()); + let clock = g.values().map(|v| v.1).max().unwrap_or(0) + 1; + g.insert(job.key, (Arc::new(plan), clock)); + if g.len() > 96 { + let mut v: Vec<(u64, FrameKey)> = g.iter().map(|(k, v)| (v.1, *k)).collect(); + v.sort_unstable_by_key(|x| x.0); + for (_, k) in v.into_iter().take(g.len() - 96) { + g.remove(&k); + } + } + } else { + let img = render_job(&job, &pool, &services); + sh.done.lock().unwrap_or_else(|e| e.into_inner()).insert(job.key, Arc::new(img)); + } sh.in_flight.lock().unwrap_or_else(|e| e.into_inner()).retain(|k| *k != job.key); if let Some(ctx) = repaint.lock().unwrap_or_else(|e| e.into_inner()).as_ref() { ctx.request_repaint(); @@ -209,6 +261,13 @@ fn render_job(job: &Job, pool: &Arc, services: &Arc) -> &provider, )), Target::Item(item) => filmcraft_render::render_item(&job.project, item, job.time, job.scale, &provider), + Target::SequencePlan(seq) => Some(filmcraft_render::render_sequence( + &job.project, + seq, + job.time, + filmcraft_render::RenderOptions { scale: job.scale, ..Default::default() }, + &provider, + )), }; match img { Some(img) => Rgba { w: img.w, h: img.h, px: img.over_black_rgba8() }, diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e3459345..791ca7d0 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -102,6 +102,52 @@ pub struct FilmcraftApp { bindings: Vec<(egui::Modifiers, egui::Key, String)>, pub toast: Option<(String, f64)>, pub tl: panels::timeline::TlState, + /// GPU compositor (when running on wgpu): device state + compositor + the egui texture it feeds. + pub gpu: Option, +} + +pub struct GpuState { + pub render_state: eframe::egui_wgpu::RenderState, + pub compositor: filmcraft_gpu::GpuCompositor, + pub texture: Option, + pub last_key: Option, + pub size: (u32, u32), + /// Composite time of the last frame (ms). + pub last_ms: f32, +} + +impl FilmcraftApp { + /// Enable the GPU compositor on the eframe wgpu device. + pub fn set_wgpu(&mut self, rs: eframe::egui_wgpu::RenderState) { + let compositor = filmcraft_gpu::GpuCompositor::new(&rs.device, &rs.queue); + self.gpu = Some(GpuState { render_state: rs, compositor, texture: None, last_key: None, size: (0, 0), last_ms: 0.0 }); + } + + /// Composite a plan on the GPU and return the egui texture showing it. + pub fn gpu_present(&mut self, key: FrameKey, plan: &filmcraft_render::plan::FramePlan) -> Option<(egui::TextureId, (u32, u32))> { + let g = self.gpu.as_mut()?; + if g.last_key == Some(key) + && let Some(t) = g.texture + { + return Some((t, g.size)); + } + let t0 = std::time::Instant::now(); + let (view, size) = g.compositor.composite(plan); + let view = view.clone(); + g.last_ms = t0.elapsed().as_secs_f32() * 1000.0; + let mut renderer = g.render_state.renderer.write(); + let id = match g.texture { + Some(id) => { + renderer.update_egui_texture_from_wgpu_texture(&g.render_state.device, &view, eframe::wgpu::FilterMode::Linear, id); + id + } + None => renderer.register_native_texture(&g.render_state.device, &view, eframe::wgpu::FilterMode::Linear), + }; + g.texture = Some(id); + g.last_key = Some(key); + g.size = size; + Some((id, size)) + } } impl FilmcraftApp { @@ -139,6 +185,7 @@ impl FilmcraftApp { bindings: menus::bindings(), toast: None, tl: Default::default(), + gpu: None, } } diff --git a/crates/ui-egui/src/panels/monitor.rs b/crates/ui-egui/src/panels/monitor.rs index 96d5c779..f7b4a1f7 100644 --- a/crates/ui-egui/src/panels/monitor.rs +++ b/crates/ui-egui/src/panels/monitor.rs @@ -83,7 +83,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) .project .item(match target { Target::Item(i) => i, - Target::Sequence(s) => s, + Target::Sequence(s) | Target::SequencePlan(s) => s, }) .is_some_and(|i| !matches!(i.kind, ItemKind::Sequence(_))) { @@ -92,6 +92,11 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) app.session.revision }; let size_key = (scale * 1000.0) as u32; + let use_gpu = which == Which::Program && app.gpu.is_some(); + let target = match (use_gpu, target) { + (true, Target::Sequence(s)) => Target::SequencePlan(s), + (_, t) => t, + }; let key = FrameKey { target, frame, size: size_key, revision: rev }; let project = app.session.project.clone(); let playing = which == Which::Program && app.playback.playing; @@ -110,7 +115,20 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) app.frames.retain_queue(|k| k.target != target || (k.revision == rev && (k.frame - cur) * dir >= 0 && (k.frame - cur).abs() < 40)); } let tex_name = format!("monitor-{prefix}"); - let shown = if let Some(img) = app.frames.get(&key) { + let shown = if use_gpu { + let exact = app.frames.get_plan(&key).map(|p| (key, p)); + if exact.is_some() { + if playing { + app.playback.shown += 1; + } + } else if playing && app.playback.last_frame_shown() != frame { + app.playback.dropped += 1; + } + match exact.or_else(|| app.frames.nearest_plan(key, 6)) { + Some((k, plan)) => app.gpu_present(k, &plan).map(|(id, _)| id), + None => app.gpu.as_ref().and_then(|g| g.texture), + } + } else if let Some(img) = app.frames.get(&key) { if playing { app.playback.shown += 1; }