From 93856382e732a7fd2508f4b550c29be22c1622ba Mon Sep 17 00:00:00 2001 From: bflatastic Date: Wed, 7 Oct 2026 01:46:35 +0700 Subject: [PATCH] Preview: Audio, Lock and Shy keep the cache; playback with audio waits for frames (#103) Toggling a layer's Audio, Lock or Shy switch (or Hide Shy Layers) threw away every cached frame: the RAM preview identity hashes each comp's address, which any edit replaces, and the layer and disk cache keys hashed every switch. None of these change pixels. - Switches::pixels, Layer::same_pixels and Comp::same_pixels (exhaustive field lists, so a new field must be considered) and Comp::pixel_form. - Layer cache keys hash Switches::pixels; the disk cache's comp content key hashes Comp::pixel_form. - Frames::identity: when a new revision's comps draw the same pixels as those of the comp's last new identity, that identity (and the project holding its comps' addresses) is kept, so the frames stay, also when the switch is toggled back. With audio on, playback skipped frames that weren't rendered yet. Now the sound waits: until the frames ahead (to the end of the preview, or what fits in the RAM preview) are cached, every frame shows as it renders, silently, like a preview without audio; then the sound starts in sync. Reaching an uncached frame stops the sound instead of skipping frames. Tests: audio_lock_and_shy_switches_keep_the_frames (frames.rs, nested comps and toggling back), audio_lock_and_shy_keep_cached_layers (engine), the disk key in tests_disk_cache, and preview_with_audio_shows_every_frame_and_sounds_once_cached. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/tests.rs | 21 ++++ crates/engine/src/tests_disk_cache.rs | 6 ++ crates/project/src/lib.rs | 133 +++++++++++++++++++++++++ crates/render/src/cache.rs | 3 +- crates/render/src/disk_cache.rs | 7 +- crates/ui-egui/src/frames.rs | 98 ++++++++++++++++-- crates/ui-egui/src/lib.rs | 70 +++++++++---- crates/ui-egui/tests/ui_audio_scrub.rs | 40 +++++++- docs/architecture.md | 6 +- 9 files changed, 348 insertions(+), 36 deletions(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a0acbd1a..6ae18dd4 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -47,6 +47,27 @@ fn demo_layer_cache_is_transparent() { } } +/// Audio, Lock and Shy don't change pixels: toggling them keeps the cached layers (#103). +#[test] +fn audio_lock_and_shy_keep_cached_layers() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "C", "width": 160, "height": 90, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#3080ff", "width": 80, "height": 40})).unwrap(); + s.execute("effect.apply", json!({"layer": "#1", "effect": "Gaussian Blur"})).unwrap(); + s.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 12})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let t = s.time(); + s.render(cid, t, Default::default()); + let before = s.layer_cache.stats(); + for sw in ["audio", "lock", "shy"] { + s.execute("layer.setSwitch", json!({"layers": ["#1"], "switch": sw, "value": sw != "audio"})).unwrap(); + } + s.render(cid, t, Default::default()); + let after = s.layer_cache.stats(); + assert_eq!(after.misses, before.misses, "no layer rendered again"); + assert!(after.hits > before.hits); +} + #[test] fn every_command_has_unique_id_and_runs_or_reports() { let mut ids = std::collections::HashSet::new(); diff --git a/crates/engine/src/tests_disk_cache.rs b/crates/engine/src/tests_disk_cache.rs index cc63cb43..3091964a 100644 --- a/crates/engine/src/tests_disk_cache.rs +++ b/crates/engine/src/tests_disk_cache.rs @@ -117,6 +117,12 @@ fn footage_changes_change_the_salt() { s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap(); let k2 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); assert_ne!(k1, k2); + // Switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers) don't change it (#103). + for sw in ["audio", "lock", "shy"] { + s.execute("layer.setSwitch", json!({"layers": ["#1"], "switch": sw, "value": sw != "audio"})).unwrap(); + } + s.execute("comp.setSwitch", json!({"switch": "hideShy", "value": true})).unwrap(); + assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k2); // Unrelated comps do not change it. s.execute("comp.new", json!({"name": "Other", "width": 10, "height": 10, "frameRate": 30, "duration": 1})).unwrap(); assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k2); diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 36cbe7ed..ebd1bdef 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -613,6 +613,139 @@ impl Default for Switches { } } +// What a frame's pixels depend on. Caches key on it: the Audio, Lock and Shy switches and Hide +// Shy Layers don't change pixels, so toggling them keeps cached frames (#103). + +impl Switches { + /// These switches with Audio, Lock and Shy at their defaults. + pub fn pixels(&self) -> Switches { + Switches { audio: true, locked: false, shy: false, ..self.clone() } + } +} + +impl Layer { + /// Draws the same pixels as `o`: equal but for the Audio, Lock and Shy switches. + pub fn same_pixels(&self, o: &Layer) -> bool { + // Every field (no `..`: a new field must be considered here); the property tree last. + let Layer { + id, + name, + source, + label, + comment, + start_time, + in_point, + out_point, + stretch, + switches, + blend_mode, + preserve_transparency, + track_matte, + parent, + markers, + markers_locked, + auto_orient, + environment, + environment_background, + props, + } = self; + *id == o.id + && *name == o.name + && *source == o.source + && *label == o.label + && *comment == o.comment + && *start_time == o.start_time + && *in_point == o.in_point + && *out_point == o.out_point + && *stretch == o.stretch + && switches.pixels() == o.switches.pixels() + && *blend_mode == o.blend_mode + && *preserve_transparency == o.preserve_transparency + && *track_matte == o.track_matte + && *parent == o.parent + && *markers == o.markers + && *markers_locked == o.markers_locked + && *auto_orient == o.auto_orient + && *environment == o.environment + && *environment_background == o.environment_background + && *props == o.props + } +} + +impl Comp { + /// Draws the same pixels as `o`: equal but for its layers' Audio, Lock and Shy switches and + /// Hide Shy Layers. + pub fn same_pixels(&self, o: &Comp) -> bool { + // Every field (no `..`: a new field must be considered here); the layers last. + let Comp { + width, + height, + pixel_aspect, + frame_rate, + duration, + display_start, + background, + work_area, + layers, + markers, + shutter_angle, + shutter_phase, + motion_blur_samples, + motion_blur_adaptive_limit, + renderer, + hide_shy: _, + enable_motion_blur, + enable_frame_blending, + draft_3d, + preserve_frame_rate, + preserve_resolution, + poster_time, + global_light, + guides, + essential, + } = self; + *width == o.width + && *height == o.height + && *pixel_aspect == o.pixel_aspect + && *frame_rate == o.frame_rate + && *duration == o.duration + && *display_start == o.display_start + && *background == o.background + && *work_area == o.work_area + && *markers == o.markers + && *shutter_angle == o.shutter_angle + && *shutter_phase == o.shutter_phase + && *motion_blur_samples == o.motion_blur_samples + && *motion_blur_adaptive_limit == o.motion_blur_adaptive_limit + && *renderer == o.renderer + && *enable_motion_blur == o.enable_motion_blur + && *enable_frame_blending == o.enable_frame_blending + && *draft_3d == o.draft_3d + && *preserve_frame_rate == o.preserve_frame_rate + && *preserve_resolution == o.preserve_resolution + && *poster_time == o.poster_time + && *global_light == o.global_light + && *guides == o.guides + && *essential == o.essential + && layers.len() == o.layers.len() + && layers.iter().zip(&o.layers).all(|(a, b)| a.same_pixels(b)) + } + + /// The comp with those switches at their defaults (borrowed when they already are), for + /// keys that hash the whole comp. + pub fn pixel_form(&self) -> std::borrow::Cow<'_, Comp> { + if !self.hide_shy && self.layers.iter().all(|l| l.switches == l.switches.pixels()) { + return std::borrow::Cow::Borrowed(self); + } + let mut c = self.clone(); + c.hide_shy = false; + for l in &mut c.layers { + l.switches = l.switches.pixels(); + } + std::borrow::Cow::Owned(c) + } +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum MatteKind { #[default] diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index e487e6ce..b30f67cb 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -558,7 +558,8 @@ fn key_any(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, blur: bool, fo draft.hash(&mut h); layer.id.hash(&mut h); hash_debug(&mut h, &layer.source); - hash_debug(&mut h, &layer.switches); + // (Audio, Lock and Shy don't change pixels.) + hash_debug(&mut h, &layer.switches.pixels()); // Mask motion blur and effects that read the shutter depend on whether the layer is motion // blurred in this render and on the comp's shutter. if layer.masks().is_some_and(|m| !m.children.is_empty()) || layer.effects().is_some_and(|fx| !fx.children.is_empty()) { diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index b35a3aa7..c1884fb3 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -780,7 +780,7 @@ pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { /// Content key of a composition's pixels: project render settings, the comp and everything it /// uses (nested comps, footage, solids, their proxies and Use Proxy switches, footage and proxy /// files' size and time). With expressions in play (which can read any comp) the whole project -/// counts. +/// counts. Switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers) don't count. pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 { use effectcraft_project::{ItemKind, LayerSource}; let mut h = Hash128::default(); @@ -810,7 +810,10 @@ pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraf let Some(it) = project.item(id) else { continue }; h.write_u64(id.0); h.write(it.name.as_bytes()); - hash_debug(&mut h, &it.kind); + match &it.kind { + ItemKind::Comp(c) => hash_debug(&mut h, &c.pixel_form()), + k => hash_debug(&mut h, k), + } if let ItemKind::Footage(f) = &it.kind { footage_stamp(&mut h, f); } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index a01a9612..3e3bd06e 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -5,7 +5,8 @@ //! *content* ([`comp_content`]: the comp and everything it uses), frame, scale, view and render //! options; the timeline draws the cached range as the green cache bar. An edit keeps the frames //! of every comp it doesn't touch, and undo finds the frames of the state it returns to, as in -//! After Effects. When the budget is full the least recently shown frames go first. +//! After Effects. Switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers) keep +//! the frames too. When the budget is full the least recently shown frames go first. //! //! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] hands them //! to a [`RemoteFrames`] (the browser's frame worker, a second engine instance fed with project @@ -25,7 +26,7 @@ use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; -use effectcraft_engine::project::{ItemId, ItemKind, LayerSource, Node, Project, PropGroup}; +use effectcraft_engine::project::{Comp, ItemId, ItemKind, LayerSource, Node, Project, PropGroup}; use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; @@ -191,6 +192,10 @@ pub struct RenderSource { pub disk: Option>, } +/// A [`comp_content`] identity and, when it was found in another project, that project +/// ([`Frames::identity`]). +type Identity = (u64, Option>); + /// Content keys of (project revision, comp), computed once per revision. type ContentKeys = Arc>>; @@ -214,6 +219,11 @@ fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) /// hands out), and by value the project settings and the footage, solids and proxies it uses. /// With expressions in its comps (which can read anything) every item counts. pub fn comp_content(project: &Project, comp: ItemId) -> u64 { + comp_content_with(project, comp, |_, c| Arc::as_ptr(c)) +} + +/// [`comp_content`] with each comp's address given by `addr`. +fn comp_content_with(project: &Project, comp: ItemId, addr: impl Fn(ItemId, &Arc) -> *const Comp) -> u64 { use std::hash::{Hash, Hasher}; fn has_expression(g: &PropGroup) -> bool { g.children.iter().any(|c| match c { @@ -247,7 +257,7 @@ pub fn comp_content(project: &Project, comp: ItemId) -> u64 { it.name.hash(&mut h); } match &it.kind { - ItemKind::Comp(c) => (Arc::as_ptr(c) as usize).hash(&mut h), + ItemKind::Comp(c) => (addr(id, c) as usize).hash(&mut h), k => format!("{k:?}").hash(&mut h), } format!("{:?}", it.proxy).hash(&mut h); @@ -342,8 +352,11 @@ pub struct Frames { /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, content_keys: ContentKeys, - /// [`comp_content`] identities by (revision, comp). - identities: Mutex>, + /// [`comp_content`] identities by (revision, comp), with the project that keeps an identity + /// found by [`Frames::identity`] in another project. + identities: Mutex>, + /// The last new identity of each comp and its project (see [`Frames::identity`]). + bases: Mutex)>>, /// A project snapshot for each identity handed out, with when it was last asked for (see /// [`Frames::content_of`]); the counter. keepers: Mutex<(u64, HashMap)>)>, @@ -373,6 +386,7 @@ impl Default for Frames { last_ms: Arc::new(Mutex::new(0.0)), content_keys: Arc::default(), identities: Mutex::default(), + bases: Mutex::default(), keepers: Mutex::default(), remote: None, remote_busy: Arc::default(), @@ -428,22 +442,22 @@ impl Frames { /// revision). The project is kept while frames of that identity are cached or rendering, so /// the comp addresses it hashes can't be reused by other comps meanwhile. pub fn content_of(&self, project: &Arc, revision: u64, comp: ItemId) -> u64 { - let known = self.identities.lock().ok().and_then(|m| m.get(&(revision, comp.0)).copied()); - let id = known.unwrap_or_else(|| { - let id = comp_content(project, comp); + let known = self.identities.lock().ok().and_then(|m| m.get(&(revision, comp.0)).cloned()); + let (id, base) = known.unwrap_or_else(|| { + let found = self.identity(project, comp); if let Ok(mut m) = self.identities.lock() { if m.len() > 256 { m.clear(); } - m.insert((revision, comp.0), id); + m.insert((revision, comp.0), found.clone()); } - id + found }); if let Ok(mut guard) = self.keepers.lock() { let (clock, k) = &mut *guard; *clock += 1; let now = *clock; - k.entry(id).or_insert_with(|| (now, project.clone())).0 = now; + k.entry(id).or_insert_with(|| (now, base.unwrap_or_else(|| project.clone()))).0 = now; if k.len() > MAX_KEEPERS { let inflight: HashSet = self.inflight.lock().map(|s| s.iter().map(|f| f.content).collect()).unwrap_or_default(); if let Ok(mut c) = self.cache.lock() { @@ -464,6 +478,30 @@ impl Frames { id } + /// The [`comp_content`] identity of `comp` in `project`, or the comp's last new identity when + /// every comp it draws is the same or draws the same pixels there ([`Comp::same_pixels`]: + /// only Audio, Lock or Shy switches changed), so toggling those keeps the cached frames + /// (#103). That identity hashes the addresses of the comps in its project, which is + /// returned to keep them. + fn identity(&self, project: &Arc, comp: ItemId) -> Identity { + let plain = comp_content(project, comp); + let Ok(mut bases) = self.bases.lock() else { return (plain, None) }; + if let Some((base_id, base)) = bases.get(&comp.0) + && *base_id != plain + && comp_content_with(project, comp, |id, c| match base.item(id).map(|i| &i.kind) { + Some(ItemKind::Comp(b)) if Arc::ptr_eq(b, c) || b.same_pixels(c) => Arc::as_ptr(b), + _ => Arc::as_ptr(c), + }) == *base_id + { + return (*base_id, Some(base.clone())); + } + if bases.len() > MAX_KEEPERS { + bases.clear(); + } + bases.insert(comp.0, (plain, project.clone())); + (plain, None) + } + pub fn is_cached(&self, k: &FrameKey) -> bool { self.cache.lock().map(|c| c.map.contains_key(k)).unwrap_or(false) } @@ -970,4 +1008,42 @@ mod tests { comp.width = 16; assert_ne!(f.content_of(&p, 2, a), before); } + + /// Audio, Lock and Shy (and Hide Shy Layers) don't change pixels: toggling them, in the comp + /// or a comp it nests, keeps its frames, also when toggled back (#103). + #[test] + fn audio_lock_and_shy_switches_keep_the_frames() { + use effectcraft_engine::Session; + use serde_json::json; + let mut s = Session::default(); + let id = |v: serde_json::Value, k: &str| v[k].as_u64().unwrap(); + let inner = ItemId(id(s.execute("comp.new", json!({"name": "Inner", "width": 8, "height": 8, "duration": 1})).unwrap(), "comp")); + let solid = id(s.execute("layer.newSolid", json!({"comp": inner.0, "color": "#ff0000"})).unwrap(), "layer"); + let outer = ItemId(id(s.execute("comp.new", json!({"name": "Outer", "width": 8, "height": 8, "duration": 1})).unwrap(), "comp")); + let nested = id(s.execute("layer.addItem", json!({"comp": outer.0, "item": inner.0})).unwrap(), "layer"); + let f = Frames::default(); + let at = |s: &Session| f.content_of(&s.project, s.revision, outer); + let o0 = at(&s); + let switch = |s: &mut Session, comp: ItemId, layer: u64, sw: &str, v: bool| { + s.execute("layer.setSwitch", json!({"comp": comp.0, "layers": [layer], "switch": sw, "value": v})).unwrap(); + }; + for (comp, layer, sw) in [(outer, nested, "audio"), (outer, nested, "lock"), (outer, nested, "shy"), (inner, solid, "audio"), (inner, solid, "shy")] { + // Audio off; Lock and Shy on. + switch(&mut s, comp, layer, sw, sw != "audio"); + assert_eq!(at(&s), o0, "{sw} on in {comp:?}"); + } + s.execute("comp.setSwitch", json!({"comp": outer.0, "switch": "hideShy", "value": true})).unwrap(); + assert_eq!(at(&s), o0, "Hide Shy Layers"); + switch(&mut s, outer, nested, "audio", true); + switch(&mut s, outer, nested, "lock", false); + assert_eq!(at(&s), o0, "toggled back"); + // Switches that change pixels still do. + switch(&mut s, inner, solid, "video", false); + let o1 = at(&s); + assert_ne!(o1, o0, "Video"); + switch(&mut s, outer, nested, "audio", false); + assert_eq!(at(&s), o1, "a mute after a real edit keeps that edit's frames"); + switch(&mut s, inner, solid, "video", true); + assert_ne!(at(&s), o1); + } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 764c6b69..2f25561f 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -135,6 +135,10 @@ pub struct Playback { pub restore_max: Option>, /// Audio-only previews: the frame under the audible sample (the current time stays put). pub audio_frame: Option, + /// The preview's sound waits for frames: while the frames ahead aren't cached, every frame + /// shows as it renders, silently; the sound starts once the rest of the preview (or what + /// fits in the RAM preview) is cached, so it never plays over skipped frames (#103). + pub audio_held: bool, /// When the last frames were shown (seconds, the last second's worth): the achieved rate. pub shown_at: std::collections::VecDeque, } @@ -684,31 +688,33 @@ impl EffectcraftApp { layer_controls: preset.include_layer_controls, restore_max, audio_frame: (!plan.video).then_some(plan.first), + audio_held: false, shown_at: Default::default(), }; if !self.playback.caching { - self.start_audio(fr.tick_of(plan.first)); + if plan.video { + self.playback.audio_held = self.audio_plays(); + } else { + self.start_audio(fr.tick_of(plan.first)); + } } } - /// Start audio preview from comp time `t` when the preview includes audio at the comp's - /// frame rate, the comp has something audible and an output device opens. + /// The preview includes audio at the comp's frame rate and the comp has something audible. + fn audio_plays(&self) -> bool { + let (Some(pl), Some(cid)) = (self.playback.plan, self.session.active_comp_id()) else { return false }; + let Some(c) = self.session.project.comp(cid) else { return false }; + // A preview at another frame rate plays slower or faster than real time: silent. + pl.audio && (pl.fps - c.frame_rate.as_f64()).abs() <= 0.01 && effectcraft_engine::render::audio::comp_has_audio(&self.session.project, cid) + } + + /// Start audio preview from comp time `t` when [`Self::audio_plays`] and an output device + /// opens. fn start_audio(&mut self, t: Tick) { self.audio = None; self.meter.clipped = [false; 2]; - let Some(pl) = self.playback.plan else { return }; - if !pl.audio { - return; - } - let Some(cid) = self.session.active_comp_id() else { return }; - let Some(c) = self.session.project.comp_arc(cid) else { return }; - // A preview at another frame rate plays slower or faster than real time: silent. - if (pl.fps - c.frame_rate.as_f64()).abs() > 0.01 { - return; - } - if !effectcraft_engine::render::audio::comp_has_audio(&self.session.project, cid) { - return; - } + let (Some(pl), Some(cid)) = (self.playback.plan, self.session.active_comp_id()) else { return }; + let Some(c) = self.session.project.comp_arc(cid).filter(|_| self.audio_plays()) else { return }; let out = audio::AudioOutput::from_prefs(&self.session.prefs); let Some(dev) = self.hooks.audio_device.as_ref().and_then(|f| f(&out)) else { return }; let mix = self.session.prefs.audio.preview_sample_rate; @@ -935,8 +941,17 @@ impl EffectcraftApp { return; } let snap = |f: i64| wa + (f - wa).div_euclid(step) * step; - // Audio clock drives playback: show the frame under the audible sample (frames that - // are not rendered yet are dropped, never delayed). + // Held sound starts once the frames ahead are cached (see `Playback::audio_held`). + if self.playback.audio_held + && let Some(p) = plan + && self.cached_ahead(&p, &series, cur, self.frames_that_fit(&c, scale)) + { + self.playback.audio_held = false; + self.playback.start_wall = now; + self.playback.start_frame = cur; + self.start_audio(fr.tick_of(cur)); + } + // Audio clock drives playback: show the frame under the audible sample. if let Some(a) = &self.audio { if a.finished() { if video { @@ -950,12 +965,20 @@ impl EffectcraftApp { let target = snap(audio::frame_of_sample(a.clock(), a.rate, fr).clamp(wa, wb)); if !video { self.playback.audio_frame = Some(target); - } else { - self.playback.waiting = !self.frames.is_cached(&FrameKey { frame: target, ..series }); + } else if self.frames.is_cached(&FrameKey { frame: target, ..series }) { + self.playback.waiting = false; if target != cur { self.session.set_time(fr.tick_of(target)); self.playback.frame_shown(now); } + } else { + // Rendering can't keep up: rather than skip frames, the sound stops and every + // frame shows as it renders until the frames ahead are cached again. + self.audio = None; + self.playback.audio_held = true; + self.playback.waiting = true; + self.playback.start_wall = now; + self.playback.start_frame = cur; } ctx.request_repaint(); return; @@ -1065,6 +1088,13 @@ impl EffectcraftApp { /// How many viewer frames of `comp` at `scale` the RAM preview budget holds (90 % of it, /// at least one). + /// The frames of `plan` from `cur` to its end, or as many as fit in the RAM preview, are + /// cached: sound can play from `cur` in real time. + fn cached_ahead(&self, plan: &effectcraft_engine::preview::PreviewPlan, series: &FrameKey, cur: i64, fit: i64) -> bool { + let cached = self.frames.cached_frames(series); + (0..fit).map_while(|k| plan.frame_after(cur, k)).take_while(|f| *f >= cur).all(|f| cached.binary_search(&f).is_ok()) + } + fn frames_that_fit(&self, comp: &effectcraft_engine::project::Comp, scale: f64) -> i64 { let side = |n: u32| (n as f64 * scale).ceil().max(1.0); let frame = side(comp.width) * side(comp.height) * 4.0; diff --git a/crates/ui-egui/tests/ui_audio_scrub.rs b/crates/ui-egui/tests/ui_audio_scrub.rs index caba66d7..cf17fb18 100644 --- a/crates/ui-egui/tests/ui_audio_scrub.rs +++ b/crates/ui-egui/tests/ui_audio_scrub.rs @@ -1,6 +1,6 @@ //! Audio scrubbing: Ctrl/Cmd-dragging the current time plays one frame of the comp's mix at each //! new frame (each snippet replacing what is still queued); holding still repeats nothing; silent -//! comps open no output; an idle scrub closes it. +//! comps open no output; an idle scrub closes it. Previews with audio wait for frames. use std::sync::{Arc, Mutex}; @@ -107,3 +107,41 @@ fn cmd_dragging_the_current_time_scrubs_and_an_idle_scrub_closes() { } assert!(h.state().scrub.is_none() && fake.0.lock().unwrap().1, "closed and stopped"); } + +/// A preview with audio waits for frames like a silent one (#103): while the frames ahead aren't +/// cached, every frame shows as it renders, silently; the sound starts once the rest of the +/// preview is cached, and a frame that isn't cached stops it instead of being skipped. +#[test] +fn preview_with_audio_shows_every_frame_and_sounds_once_cached() { + let fake = Fake::default(); + let (app, cid) = setup(true, &fake); + // A little under one frame of input time per step. + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 31.0).build_eframe(|_| app); + h.run_steps(2); + let frame = |h: &Harness<'_, EffectcraftApp>| { + let app = h.state(); + app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time()) + }; + let now = h.ctx.input(|i| i.time); + h.state_mut().play(now); + assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet"); + let mut last = frame(&h); + let mut shown = 0; + for _ in 0..2000 { + h.step(); + if h.state().audio.is_some() { + break; + } + let f = frame(&h); + assert!(f == last || f == last + 1 || f < last, "{last} → {f}: a frame was skipped"); + shown += usize::from(f != last); + last = f; + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started once the frames ahead were cached"); + assert!(shown > 0, "frames played silently first"); + // A frame that isn't cached stops the sound rather than being skipped. + h.state().frames.clear(); + h.step(); + assert!(h.state().audio.is_none() && h.state().playback.audio_held && h.state().playback.playing); +} diff --git a/docs/architecture.md b/docs/architecture.md index a82d44b6..ee09059a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -163,7 +163,11 @@ keys, with a nesting-depth guard. **Audio** is mixed by `render::audio::mix_comp` in blocks: Audio switch, solo, Audio Levels and the Effect > Audio effects (`effects::audio_fx`, applied per layer with a pre-roll so blocks are independent). Export and preview playback share it; the desktop app plays it through cpal and the -audio clock drives preview playback (`ui-egui::audio`). +audio clock drives preview playback (`ui-egui::audio`). The sound starts once the frames ahead +are cached; until then, and whenever playback reaches a frame that isn't cached, every frame +shows as it renders, silently, rather than being skipped. Layer and frame caches ignore the +switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers: `Switches::pixels`, +`Comp::same_pixels`), so toggling them keeps the cached frames. **Layer styles** (Layer ▸ Layer Styles; `crates/render/src/styles.rs`) render in layer space and may grow the layer's bounds. Drop Shadow and Outer Glow become separate passes composited below the