diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index f8651cd6..3df9ec45 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Condvar, Mutex}; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemId}; use effectcraft_raster::{Image, Px}; @@ -53,8 +53,25 @@ struct Entry { /// version 1 files of sources longer than about 12:36 at 48 kHz were silent from there on (#172). const CONFORM_VERSION: u32 = 2; +/// Footage longer than this (24 hours) is read from its decoder rather than conformed: its +/// duration comes from the file's header, and a damaged one claiming days of audio had the +/// conform thread allocate all of it at once (#408). +const MAX_CONFORM_SECONDS: i64 = 24 * 3600; + +/// The sample frames of `footage`'s audio at `rate` Hz that a conformed file holds, `None` past +/// [`MAX_CONFORM_SECONDS`]. +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +fn conform_frames(footage: &Footage, rate: u32) -> Option { + let frames = footage.duration.to_units_floor(i64::from(rate)).max(0); + if frames > MAX_CONFORM_SECONDS.saturating_mul(i64::from(rate)) { + return None; + } + usize::try_from(frames).ok() +} + /// The source time of sample frame `at` at `rate` Hz. (`at` × ticks per second overflows `i64` /// past about 12:36 at 48 kHz.) +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] fn sample_time(at: usize, rate: u32) -> Tick { Tick::from_units(i64::try_from(at).unwrap_or(i64::MAX), i64::from(rate)) } @@ -135,6 +152,8 @@ pub struct PoolStats { /// Frames currently cached and their total size in bytes. pub frames: usize, pub bytes: usize, + /// Movie and audio files opened (each path once, however many callers ask for it at once). + pub opened: u64, } #[derive(Default)] @@ -146,9 +165,15 @@ struct CacheState { last: HashMap, i64>, } +/// A movie/audio source by path, opened by the first caller while the others wait for it +/// (`None`: failed to open; not retried until `forget`). +type SourceSlot = Arc>>; + struct Inner { - /// Opened movie/audio sources by path (`None`: failed to open; not retried until `forget`). - sources: Mutex, Option>>, + /// Opened movie/audio sources by path. + sources: Mutex, SourceSlot>>, + /// Files opened into `sources` (see [`PoolStats::opened`]). + opened: AtomicU64, /// In-memory files registered with `add_bytes` (web builds, tests). files: Mutex>>, /// Parsed 3D models by path (`None`: failed to load; not retried until `forget`). @@ -228,6 +253,7 @@ impl MediaPool { hits: AtomicU64::new(0), misses: AtomicU64::new(0), prefetched: AtomicU64::new(0), + opened: AtomicU64::new(0), conform: Mutex::default(), conforming: Mutex::default(), }), @@ -259,6 +285,7 @@ impl MediaPool { prefetched: i.prefetched.load(Ordering::Relaxed), frames: c.lru.map.len(), bytes: c.lru.bytes, + opened: i.opened.load(Ordering::Relaxed), } } @@ -336,30 +363,21 @@ impl MediaPool { Some(out) } - /// Conform `footage`'s audio at `rate` into `file` on a worker thread (once). + /// Conform `frames` sample frames of `footage`'s audio at `rate` into `file` on a worker + /// thread (once). #[cfg(not(target_arch = "wasm32"))] - fn conform(&self, footage: &Footage, file: std::path::PathBuf, rate: u32) { + fn conform(&self, footage: &Footage, file: std::path::PathBuf, frames: usize, rate: u32) { if !lock(&self.inner.conforming).insert(file.clone()) { return; } let (pool, f) = (self.clone(), footage.clone()); let spawned = std::thread::Builder::new().name("ec-conform-audio".into()).spawn(move || { - let frames = f.duration.to_units_floor(rate as i64).max(0) as usize; - let mut bytes = Vec::with_capacity(frames * 8); - let mut at = 0usize; - while at < frames { - let n = (frames - at).min(1 << 16); - let t = sample_time(at, rate); - for v in pool.decode_audio(&f, t, n, rate) { - bytes.extend_from_slice(&v.to_le_bytes()); - } - at += n; - } - let ok = file.parent().is_some_and(|d| std::fs::create_dir_all(d).is_ok()) && { - let tmp = file.with_extension("tmp"); - std::fs::write(&tmp, &bytes).is_ok() && std::fs::rename(&tmp, &file).is_ok() - }; + let tmp = file.with_extension("tmp"); + let ok = file.parent().is_some_and(|d| std::fs::create_dir_all(d).is_ok()) + && pool.write_conformed(&f, &tmp, frames, rate).is_ok() + && std::fs::rename(&tmp, &file).is_ok(); if !ok { + let _ = std::fs::remove_file(&tmp); log::warn!("media: could not write conformed audio {}", file.display()); } lock(&pool.inner.conforming).remove(&file); @@ -369,6 +387,23 @@ impl MediaPool { } } + /// Write `frames` sample frames of `footage`'s audio at `rate` to `path` (raw interleaved + /// stereo `f32`) a chunk at a time: the whole track was held in memory before it was written. + #[cfg(not(target_arch = "wasm32"))] + fn write_conformed(&self, footage: &Footage, path: &std::path::Path, frames: usize, rate: u32) -> std::io::Result<()> { + use std::io::Write; + let mut out = std::io::BufWriter::new(std::fs::File::create(path)?); + let mut at = 0usize; + while at < frames { + let n = (frames - at).min(1 << 16); + for v in self.decode_audio(footage, sample_time(at, rate), n, rate) { + out.write_all(&v.to_le_bytes())?; + } + at += n; + } + out.flush() + } + /// `frames` stereo sample frames of `footage`'s audio starting at source time `start`, at /// `rate` Hz, interleaved (L R L R …). Mono is duplicated to both channels; channels beyond /// the first two are dropped. Silence where there is no audio (or before the media starts). @@ -378,7 +413,9 @@ impl MediaPool { return vec![0.0; frames * 2]; } #[cfg(not(target_arch = "wasm32"))] - if let Some(file) = self.conformed_path(&footage.path, rate) { + if let Some(conform) = conform_frames(footage, rate) + && let Some(file) = self.conformed_path(&footage.path, rate) + { let s0 = start.to_units_floor(rate as i64); if file.exists() && !lock(&self.inner.conforming).contains(&file) { let skip = (-s0).max(0) as usize; @@ -391,7 +428,7 @@ impl MediaPool { return out; } } else { - self.conform(footage, file, rate); + self.conform(footage, file, conform, rate); } } self.decode_audio(footage, start, frames, rate) @@ -478,24 +515,28 @@ impl Inner { std::fs::read(path).map(Into::into).map_err(|e| MediaError::Io(format!("{path}: {e}"))) } - /// The opened movie/audio source for `path`. + /// The opened movie/audio source for `path`. The first caller opens it, outside the map's + /// lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF files are + /// decoded whole); callers asking for the same file meanwhile wait for that instead of opening + /// it again: a preview's frame workers, audio feeder, conform and waveform threads each + /// decoded a long FLAC in full at once (#408). fn source(&self, path: &Arc) -> Option { - if let Some(s) = lock(&self.sources).get(path) { - return s.clone(); - } - // open outside the lock (reading and parsing a large file takes a moment) - let opened = self.read(path).and_then(|bytes| { - let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from) - }); - let opened = match opened { - Ok(s) => Some(s), - Err(e) => { - log::warn!("media: cannot open {path}: {e}"); - None + let slot = lock(&self.sources).entry(path.clone()).or_default().clone(); + slot.get_or_init(|| { + self.opened.fetch_add(1, Ordering::Relaxed); + let opened = self.read(path).and_then(|bytes| { + let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from) + }); + match opened { + Ok(s) => Some(s), + Err(e) => { + log::warn!("media: cannot open {path}: {e}"); + None + } } - }; - lock(&self.sources).entry(path.clone()).or_insert(opened).clone() + }) + .clone() } /// The frame index (in the footage's interpreted rate) shown at source time `t`, after @@ -747,6 +788,37 @@ mod tests { assert!((sample_time(36_372_480, 48_000).seconds() - 757.76).abs() < 1e-9); } + /// #408: footage whose header claims a month of audio isn't conformed: the conform thread + /// allocated the whole conformed track up front (a terabyte here), and the failed allocation + /// aborted the app. Its audio is read from the decoder instead (silence: this file doesn't + /// decode). + #[test] + fn a_header_claiming_a_month_of_audio_is_not_conformed() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-conform-month"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("month.flac"); + std::fs::write(&path, b"fLaC, not really").unwrap(); + let footage = Footage { + path: path.to_string_lossy().into(), + kind: FootageKind::Audio, + duration: Tick::from_seconds_f64(30.0 * 86_400.0), + has_audio: true, + ..Default::default() + }; + assert_eq!(conform_frames(&footage, 48_000), None); + assert_eq!(conform_frames(&Footage { duration: Tick::from_seconds_f64(60.0), ..footage.clone() }, 48_000), Some(2_880_000)); + let pool = MediaPool::new(); + let conformed = dir.join("conformed"); + pool.set_conform_folder(Some(conformed.clone())); + let s = pool.audio_samples(&footage, Tick::from_seconds_f64(3600.0), 1024, 48_000); + assert_eq!(s.len(), 2048); + assert!(s.iter().all(|v| *v == 0.0)); + assert!(lock(&pool.inner.conforming).is_empty(), "no conform thread started"); + assert!(!conformed.exists(), "nothing was written"); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn frame_index_loops_and_clamps() { let r = FrameRate::FPS_30; diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rs index b473337c..ed113c35 100644 --- a/crates/media/tests/footage.rs +++ b/crates/media/tests/footage.rs @@ -545,3 +545,109 @@ fn exr_still_is_linear_float() { let b = img.data[1]; assert!(b[3] == 0.5 && b[0] > 0.85 && b[0] < 1.0, "{b:?}"); } + +/// A FLAC of `secs` seconds at `rate` Hz with `channels` channels of a 440 Hz sine (first two: +/// −24 dBFS RMS, the rest quieter), samples of `fmt` (`s16`, or `s32` with 24 significant bits). +fn sine_flac(secs: u32, rate: u32, channels: u32, fmt: &str) -> Option { + let name = format!("sine_{secs}s_{rate}_{channels}ch_{fmt}.flac"); + let src = format!("sine=frequency=440:sample_rate={rate}:duration={secs}"); + let pan = format!("pan={channels}c|{}", (0..channels).map(|c| format!("c{c}={}*c0", if c < 2 { "1" } else { "0.5" })).collect::>().join("|")); + let mut args = vec!["-f", "lavfi", "-i", &src, "-af", &pan, "-sample_fmt", fmt]; + if fmt == "s32" { + args.extend(["-bits_per_raw_sample", "24"]); + } + args.extend(["-c:a", "flac"]); + ffmpeg_fixture(&name, &args) +} + +/// #408 robustness: FLAC footage of any rate, depth and channel count reads correctly as the +/// preview feeder reads it, one device block at a time at the mix rate, through its end and past +/// it: two minutes of 24-bit, 6-channel, 96 kHz audio, and mono 8 kHz, 11.025 kHz, 192 kHz and +/// 8-channel files. +#[test] +fn flac_audio_of_any_shape_reads_in_device_blocks() { + for (secs, rate, channels, fmt) in [(120, 96_000, 6, "s32"), (5, 8_000, 1, "s16"), (5, 11_025, 2, "s16"), (5, 192_000, 2, "s32"), (5, 48_000, 8, "s16")] { + let Some(p) = sine_flac(secs, rate, channels, fmt) else { return }; + let f = probe(&p).expect("probe"); + assert!(f.has_audio && f.kind == FootageKind::Audio, "{}", p.display()); + assert!((f.duration.seconds() - f64::from(secs)).abs() < 0.01, "{}: {}", p.display(), f.duration.seconds()); + let pool = MediaPool::new(); + let mix = 48_000i64; + let total = f.duration.to_units_floor(mix); + let (mut at, mut sum, mut n, mut worst) = (0i64, 0.0f64, 0usize, std::time::Duration::ZERO); + let t0 = Instant::now(); + while at < total + 4096 { + let t = Instant::now(); + let s = pool.audio_samples(&f, Tick::from_units(at, mix), 1024, 48_000); + if at > 0 { + worst = worst.max(t.elapsed()); + } + assert_eq!(s.len(), 2048); + assert!(s.iter().all(|v| v.is_finite() && v.abs() <= 1.0), "{} at {at}", p.display()); + if at + 1024 <= total { + sum += s.iter().map(|v| f64::from(*v) * f64::from(*v)).sum::(); + n += s.len(); + } else if at >= total { + assert!(s.iter().all(|v| *v == 0.0), "{}: silence past the end", p.display()); + } + at += 1024; + } + // ffmpeg's sine source has an amplitude of 1/8, on both channels (mono is duplicated). + let rms = (sum / n.max(1) as f64).sqrt(); + assert!((rms - 0.125 / 2f64.sqrt()).abs() < 0.003, "{}: rms {rms}", p.display()); + eprintln!("{}: read {secs} s in {:?}, slowest block after the first {worst:?}", p.display(), t0.elapsed()); + assert_eq!(pool.stats().opened, 1, "opened once, not per block"); + } +} + +/// #408: the first reads of a FLAC from many threads at once (a preview's frame workers, audio +/// feeder, conform and waveform threads) decode it once: each decoded the whole file, so memory +/// and time grew with the number of threads. +#[test] +fn concurrent_first_reads_open_a_file_once() { + let Some(p) = sine_flac(60, 44_100, 2, "s16") else { return }; + let f = probe(&p).expect("probe"); + let pool = MediaPool::new(); + let start = Arc::new(std::sync::Barrier::new(8)); + let reads: Vec<_> = (0..8) + .map(|_| { + let (pool, f, start) = (pool.clone(), f.clone(), start.clone()); + std::thread::spawn(move || { + start.wait(); + pool.audio_samples(&f, Tick(TICKS_PER_SECOND), 1024, 48_000) + }) + }) + .collect(); + let reads: Vec> = reads.into_iter().map(|t| t.join().expect("read")).collect(); + assert_eq!(pool.stats().opened, 1, "the file was opened once"); + assert!(reads.iter().all(|r| *r == reads[0]) && reads[0].iter().any(|v| *v != 0.0)); +} + +/// Settings ▸ Disk ▸ Conformed Audio Folder: a FLAC's conformed file, written a chunk at a time, +/// reads back exactly as the decoder reads. +#[test] +fn conformed_flac_audio_reads_back_like_the_decoder() { + let Some(p) = sine_flac(5, 44_100, 2, "s16") else { return }; + let f = probe(&p).expect("probe"); + let folder = fixtures().join(format!("conformed-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&folder); + let pool = MediaPool::new(); + let at = Tick::from_seconds_f64(2.5); + let decoded = pool.audio_samples(&f, at, 4800, 48_000); + pool.set_conform_folder(Some(folder.clone())); + let file = pool.conformed_path(&p.to_string_lossy(), 48_000).expect("a conformed path"); + pool.audio_samples(&f, at, 4800, 48_000); + let t0 = Instant::now(); + while !file.exists() { + assert!(t0.elapsed().as_secs() < 60, "the conformed file was never written"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + // (Renamed into place once complete.) Raw interleaved stereo f32 at 48 kHz. + let bytes = std::fs::read(&file).expect("conformed file"); + assert_eq!(bytes.len(), 5 * 48_000 * 8); + let s0 = at.to_units_floor(48_000) as usize * 8; + let conformed: Vec = bytes[s0..s0 + 4800 * 8].as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect(); + assert_eq!(conformed, decoded); + assert_eq!(pool.audio_samples(&f, at, 4800, 48_000), decoded); + let _ = std::fs::remove_dir_all(&folder); +} diff --git a/crates/text/src/fonts.rs b/crates/text/src/fonts.rs index c5ad8b9b..2a2c08e8 100644 --- a/crates/text/src/fonts.rs +++ b/crates/text/src/fonts.rs @@ -504,7 +504,8 @@ fn resolve_by_name(name: &str) -> Option { /// The writing systems that need a dedicated fallback family: the bundled fonts cover none of /// them, and picking "any face with the glyph" gives Chinese glyph shapes to Japanese text (or the -/// other way round), since Han ideographs are shared. +/// other way round), since Han ideographs are shared, and a calligraphic display face to Arabic, +/// Hebrew, Indic or Thai text when that is the first installed face that covers it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] enum FallbackScript { /// Kana: unambiguously Japanese. @@ -514,9 +515,23 @@ enum FallbackScript { /// Han ideographs, CJK punctuation and full-width forms: shared by Japanese, Chinese and /// Korean; the locale decides which family to try first. Han, + Arabic, + Hebrew, + /// Devanagari, Bengali, Gurmukhi, Gujarati, Oriya, Tamil, Telugu, Kannada, Malayalam and + /// Sinhala (one family covers them all on Windows; elsewhere each has its own). + Indic, + Thai, Other, } +impl FallbackScript { + /// Kana, Hangul or Han: these share the Han ideographs, so each tries the others' families + /// next. + fn is_cjk(self) -> bool { + matches!(self, FallbackScript::Japanese | FallbackScript::Korean | FallbackScript::Han) + } +} + fn script_of(c: char) -> FallbackScript { match c as u32 { 0x3040..=0x30FF | 0x31F0..=0x31FF | 0xFF66..=0xFF9F => FallbackScript::Japanese, @@ -530,6 +545,10 @@ fn script_of(c: char) -> FallbackScript { | 0xFE30..=0xFE4F | 0xFF00..=0xFFEF | 0x20000..=0x3FFFF => FallbackScript::Han, + 0x0600..=0x06FF | 0x0750..=0x077F | 0x0870..=0x08FF | 0xFB50..=0xFDFF | 0xFE70..=0xFEFE => FallbackScript::Arabic, + 0x0590..=0x05FF | 0xFB1D..=0xFB4F => FallbackScript::Hebrew, + 0x0900..=0x0DFF | 0xA8E0..=0xA8FF => FallbackScript::Indic, + 0x0E00..=0x0E7F => FallbackScript::Thai, _ => FallbackScript::Other, } } @@ -568,6 +587,38 @@ const CHINESE_FAMILIES: &[&str] = &[ "WenQuanYi Micro Hei", ]; const KOREAN_FAMILIES: &[&str] = &["Malgun Gothic", "Gulim", "Apple SD Gothic Neo", "Noto Sans CJK KR", "Noto Sans KR", "Source Han Sans KR", "NanumGothic"]; +/// The platforms' plain text families for the other scripts the bundled fonts lack, in the same +/// order (Windows, macOS, then the Linux/Noto names). +const ARABIC_FAMILIES: &[&str] = &["Segoe UI", "Tahoma", "Geeza Pro", "Noto Sans Arabic", "Noto Naskh Arabic", "DejaVu Sans", "Arial"]; +const HEBREW_FAMILIES: &[&str] = &["Segoe UI", "Tahoma", "Arial Hebrew", "Noto Sans Hebrew", "DejaVu Sans", "Arial"]; +const INDIC_FAMILIES: &[&str] = &[ + "Nirmala UI", + "Kohinoor Devanagari", + "Kohinoor Bangla", + "Kohinoor Gujarati", + "Kohinoor Telugu", + "Devanagari Sangam MN", + "Bangla Sangam MN", + "Gurmukhi Sangam MN", + "Gujarati Sangam MN", + "Oriya Sangam MN", + "Tamil Sangam MN", + "Telugu Sangam MN", + "Kannada Sangam MN", + "Malayalam Sangam MN", + "Sinhala Sangam MN", + "Noto Sans Devanagari", + "Noto Sans Bengali", + "Noto Sans Gurmukhi", + "Noto Sans Gujarati", + "Noto Sans Oriya", + "Noto Sans Tamil", + "Noto Sans Telugu", + "Noto Sans Kannada", + "Noto Sans Malayalam", + "Noto Sans Sinhala", +]; +const THAI_FAMILIES: &[&str] = &["Leelawadee UI", "Tahoma", "Thonburi", "Noto Sans Thai", "Noto Sans Thai UI", "Loma", "Garuda"]; /// The CJK locale the host asked fallbacks to prefer, when it has one (see [`set_cjk_locale`]). static REQUESTED_CJK_LOCALE: RwLock> = RwLock::new(None); @@ -628,6 +679,10 @@ fn families_for(s: FallbackScript) -> &'static [&'static str] { FallbackScript::Japanese => JAPANESE_FAMILIES, FallbackScript::Han => CHINESE_FAMILIES, FallbackScript::Korean => KOREAN_FAMILIES, + FallbackScript::Arabic => ARABIC_FAMILIES, + FallbackScript::Hebrew => HEBREW_FAMILIES, + FallbackScript::Indic => INDIC_FAMILIES, + FallbackScript::Thai => THAI_FAMILIES, FallbackScript::Other => &[], } } @@ -741,11 +796,12 @@ fn system_fallback(c: char, style: &str) -> Option { let order: Vec = match script { FallbackScript::Other => vec![], FallbackScript::Han => han_order().to_vec(), - s => { + s if s.is_cjk() => { let mut v = vec![s]; v.extend(han_order().iter().copied().filter(|o| *o != s)); v } + s => vec![s], }; for s in order { for fam in families_for(s) { @@ -851,7 +907,33 @@ mod tests { assert_eq!(script_of('A'), FallbackScript::Han); assert_eq!(script_of('\u{20000}'), FallbackScript::Han); assert_eq!(script_of('A'), FallbackScript::Other); - assert_eq!(script_of('\u{5d0}'), FallbackScript::Other); + assert_eq!(script_of('م'), FallbackScript::Arabic); + assert_eq!(script_of('\u{fefb}'), FallbackScript::Arabic); + assert_eq!(script_of('\u{5d0}'), FallbackScript::Hebrew); + assert_eq!(script_of('क'), FallbackScript::Indic); + assert_eq!(script_of('த'), FallbackScript::Indic); + assert_eq!(script_of('ก'), FallbackScript::Thai); + assert_eq!(script_of('\u{feff}'), FallbackScript::Other, "the byte order mark is no Arabic letter"); + } + + /// Native: Arabic, Hebrew, Indic and Thai text in a Latin-only family falls back to the + /// platform's plain text family for its script (#394), not to the first installed face that + /// covers it (on Windows a calligraphic Arabic face sorts first). Skipped per script where no + /// family from its list is installed. + #[test] + fn other_scripts_fall_back_to_a_plain_system_family() { + scan_system(); + let inter = resolve("Inter", "Regular").face; + for (c, families) in [('م', ARABIC_FAMILIES), ('א', HEBREW_FAMILIES), ('क', INDIC_FAMILIES), ('த', INDIC_FAMILIES), ('ก', THAI_FAMILIES)] { + let covering = families.iter().any(|n| resolve_in(n, "Regular").is_some_and(|r| face(r.face).has_char(c))); + if !covering { + eprintln!("{c}: no family from the list is installed; skipping"); + continue; + } + let f = face(fallback_for(c, inter)); + assert!(f.has_char(c), "{c}: {} lacks it", f.info.family); + assert!(families.iter().any(|n| f.info.family.eq_ignore_ascii_case(n)), "{c} → {}", f.info.family); + } } /// Native: Japanese text in a Latin-only family falls back to an installed Japanese family diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs index dbc37b5d..7972e694 100644 --- a/crates/ui-egui/src/audio.rs +++ b/crates/ui-egui/src/audio.rs @@ -211,23 +211,23 @@ impl AudioPlayback { let span = PlaySpan::new(start, wa, wb, looping, rate); let feed = Arc::new(AudioFeed::default()); let stop = Arc::new(AtomicBool::new(false)); - // Prime ~50 ms so the device does not start on an empty queue. - let mut cursor = span.start; - let prime = (rate as usize / 20).div_ceil(BLOCK); - for _ in 0..prime { - cursor = feed_block(&src, comp, &feed, &span, cursor, rate, mix); - } if cfg!(target_arch = "wasm32") { - let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, feeder: Some((src, comp, cursor, mix)), rate, span }; + // The device takes samples only in `pump`, which fills the queue first. + let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, feeder: Some((src, comp, span.start, mix)), rate, span }; p.device.start(feed)?; p.pump(); return Ok(p); } + // All mixing happens on the feeder thread, the first block too: reading footage for the + // first time opens it (a FLAC or MP3 file decodes whole), which froze the UI for as long + // as that took (#408). The device plays silence until the first block arrives, and the + // clock doesn't count it. let thread = { let (feed, stop) = (feed.clone(), stop.clone()); std::thread::Builder::new() .name("ec-audio-feed".into()) .spawn(move || { + let mut cursor = span.start; let target = queue_target(rate); while !stop.load(Ordering::Relaxed) { if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { @@ -485,6 +485,80 @@ mod tests { assert_eq!(f.take_peaks(), [0.0, 0.0]); } + /// #408: starting a preview with sound returns at once even when reading the footage takes + /// long (the first read of a long FLAC decodes all of it): the first block was mixed on the + /// calling (UI) thread. The sound arrives from the feeder thread, and the clock waits for it. + #[test] + fn starting_audio_does_not_wait_for_slow_footage() { + use effectcraft_engine::color::Label; + use effectcraft_engine::project::{Comp, Footage, FootageKind, ItemKind, LayerSource, Project, build}; + use effectcraft_engine::render::FootageSource; + use effectcraft_engine::render::cache::LayerCache; + use std::time::{Duration, Instant}; + + /// Footage audio that takes `delay` to read the first time (then a constant 0.5). + struct Slow { + delay: Duration, + opened: Mutex, + } + impl FootageSource for Slow { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, _: Tick, frames: usize, _: u32) -> Option> { + let mut opened = self.opened.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !*opened { + std::thread::sleep(self.delay); + *opened = true; + } + Some(vec![0.5; frames * 2]) + } + } + /// An output that only takes the feed. + struct Device; + impl AudioDevice for Device { + fn sample_rate(&self) -> u32 { + 48000 + } + fn start(&mut self, _: Arc) -> Result<(), String> { + Ok(()) + } + fn stop(&mut self) {} + } + + let mut p = Project::default(); + let footage = + Footage { path: "long.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(600.0), has_audio: true, ..Default::default() }; + let item = p.add_item("long.flac", Label::SeaFoam, None, ItemKind::Footage(footage)); + let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0)); + let layer = build::layer(&mut p, &comp, "long.flac", LayerSource::Footage { item }, (0, 0), None); + comp.layers = vec![layer]; + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let src = RenderSource { + project: Arc::new(p), + footage: Arc::new(Slow { delay: Duration::from_secs(2), opened: Mutex::new(false) }), + expr: None, + layer_cache: Arc::new(LayerCache::default()), + gpu: None, + gpu_display: false, + disk: None, + }; + let t = Instant::now(); + let mut playback = + AudioPlayback::start(Box::new(Device), src, cid, Tick::ZERO, Tick::ZERO, Tick::from_seconds_f64(10.0), true, 0).expect("the preview starts"); + assert!(t.elapsed() < Duration::from_secs(1), "start waited {:?} for the footage", t.elapsed()); + assert_eq!(playback.clock(), 0, "the clock waits for the sound"); + // The feeder thread delivers it once the footage is open. + while playback.feed.queued_frames() == 0 { + assert!(t.elapsed() < Duration::from_secs(30), "the feeder never delivered"); + std::thread::sleep(Duration::from_millis(10)); + } + let mut out = [0.0f32; 8]; + playback.feed.pull(&mut out); + assert_eq!(out, [0.5; 8]); + playback.stop(); + } + #[test] fn meter_decays_and_holds_peaks() { let mut m = Meter::default(); diff --git a/crates/ui-egui/src/menu_keys.rs b/crates/ui-egui/src/menu_keys.rs index 897f01f8..39738530 100644 --- a/crates/ui-egui/src/menu_keys.rs +++ b/crates/ui-egui/src/menu_keys.rs @@ -114,27 +114,31 @@ pub fn active(ctx: &Context) -> bool { ctx.data(|d| d.get_temp(active_id())).unwrap_or(false) } -/// Alt pressed and released with nothing else in between (no key, click or wheel), off macOS. +/// Alt pressed and released with nothing else in between (no other modifier, key, click or wheel), +/// off macOS. Alt only counts when it went down with no other modifier held: Windows switches the +/// keyboard layout with Alt+Shift, and releasing Shift first leaves Alt held alone, which must not +/// take the keyboard from the field being typed in (#394). fn alt_tapped(ctx: &Context) -> bool { if cfg!(target_os = "macos") { return false; } - let mut armed: bool = ctx.data(|d| d.get_temp(alt_id())).unwrap_or(false); + // (armed, the modifiers held after the last change) + let (mut armed, mut held): (bool, Modifiers) = ctx.data(|d| d.get_temp(alt_id())).unwrap_or_default(); let mut tapped = false; ctx.input(|i| { for e in &i.events { match e { - Event::ModifiersChanged(m) if *m == Modifiers::ALT => armed = true, Event::ModifiersChanged(m) => { tapped |= armed && *m == Modifiers::NONE; - armed = false; + armed = *m == Modifiers::ALT && held == Modifiers::NONE; + held = *m; } Event::PointerMoved(_) | Event::MouseMoved(_) | Event::Zoom(_) => {} _ => armed = false, } } }); - ctx.data_mut(|d| d.insert_temp(alt_id(), armed)); + ctx.data_mut(|d| d.insert_temp(alt_id(), (armed, held))); tapped } diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 5876a396..1d61ae4c 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -1747,6 +1747,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if let Some((rl, mut buf)) = renaming.filter(|(rl, _)| *rl == layer.id.0) { let mut child = ui.new_child(egui::UiBuilder::new().max_rect(name_rect.shrink2(vec2(0.0, 2.0)))); let er = child.add(egui::TextEdit::singleline(&mut buf).font(Tokens::ui(12.0)).desired_width(name_rect.width())); + app.auto.add(&format!("timeline.layer.{rl}.nameEdit"), er.rect, &buf); if ctx.data_mut(|d| d.remove_temp::(rename_focus_id())).unwrap_or(false) { // Just started: focus the field with the whole name selected. er.request_focus(); diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index 7b13112f..74ab0b7e 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -290,22 +290,44 @@ pub fn install(ctx: &egui::Context, t: &Tokens, language: &str) { fonts.families.entry(FontFamily::Monospace).or_default().insert(0, "jbmono".into()); fonts.families.insert(FontFamily::Name("semibold".into()), vec!["inter-semibold".into(), "inter".into()]); fonts.families.insert(FontFamily::Name("medium".into()), vec!["inter-medium".into(), "inter".into()]); - install_cjk_fallbacks(&mut fonts, language); + install_script_fallbacks(&mut fonts, language); ctx.set_fonts(fonts); apply_visuals(ctx, t); } -/// Register the CJK system fallbacks for `language` in every font family, and return their names in -/// registration order (empty when nothing installed covers a probe, so a machine without CJK fonts -/// simply keeps the bundled ones). +/// Scripts the bundled fonts lack beyond CJK, each probed with one of its letters: names typed in +/// them (layers, items, comments) are drawn by an installed face instead of missing-glyph boxes +/// (#394). egui shapes each run of one face, so Arabic letters join and a word reads right to +/// left; it has no bidirectional layout, so the words of a name still follow each other left to +/// right. +#[cfg(not(target_arch = "wasm32"))] +const SCRIPT_PROBES: &[(char, &str)] = &[ + ('م', "arabic-system"), + ('א', "hebrew-system"), + ('क', "devanagari-system"), + ('ক', "bengali-system"), + ('ਕ', "gurmukhi-system"), + ('ક', "gujarati-system"), + ('க', "tamil-system"), + ('క', "telugu-system"), + ('ಕ', "kannada-system"), + ('ക', "malayalam-system"), + ('ก', "thai-system"), +]; + +/// Register the system fallbacks for scripts the bundled fonts lack in every font family, and +/// return their names in registration order (empty when nothing installed covers a probe, so a +/// machine without those fonts simply keeps the bundled ones). A face that covers several probes +/// (Segoe UI draws Arabic and Hebrew, Nirmala UI every Indic script) is registered once. /// -/// The text engine has its own script-aware fallback (#84), but which face it picks for Han -/// ideographs depends on the locale environment: a Chinese interface on a Japanese-locale machine -/// would be drawn with the Japanese face. The interface language is the better signal, so the probe -/// for its script comes first — kana for Japanese, a Han character for the Chinese catalogs. Both -/// scripts are registered, so file, layer and template names in the other one still render. The web -/// build has no system fonts and installs nothing. -fn install_cjk_fallbacks(family_fonts: &mut FontDefinitions, language: &str) -> Vec { +/// CJK comes first. The text engine has its own script-aware fallback (#84), but which face it +/// picks for Han ideographs depends on the locale environment: a Chinese interface on a +/// Japanese-locale machine would be drawn with the Japanese face. The interface language is the +/// better signal, so the probe for its script comes first — kana for Japanese, a Han character for +/// the Chinese catalogs. Both scripts are registered, so file, layer and template names in the +/// other one still render; then [`SCRIPT_PROBES`]. The web build has no system fonts and installs +/// nothing. +fn install_script_fallbacks(family_fonts: &mut FontDefinitions, language: &str) -> Vec { #[cfg(target_arch = "wasm32")] { let _ = (family_fonts, language); @@ -321,10 +343,10 @@ fn install_cjk_fallbacks(family_fonts: &mut FontDefinitions, language: &str) -> // Han text follows the interface language, not the operating system's locale. fonts::set_cjk_locale(cjk_locale(language)); let base = fonts::resolve("Inter", "Regular").face; - let probes = if is_chinese(language) { [HAN, KANA] } else { [KANA, HAN] }; + let cjk = if is_chinese(language) { [HAN, KANA] } else { [KANA, HAN] }; let mut names = Vec::new(); let mut seen = Vec::new(); - for (probe, name) in probes { + for &(probe, name) in cjk.iter().chain(SCRIPT_PROBES) { let id = fonts::fallback_for(probe, base); if seen.contains(&id) { continue; @@ -418,17 +440,20 @@ pub fn apply_visuals(ctx: &egui::Context, t: &Tokens) { } #[cfg(all(test, not(target_arch = "wasm32")))] -mod japanese_font_tests { - #[test] - fn installed_japanese_fallback_is_available_in_all_ui_families() { - use effectcraft_text::fonts; +mod script_font_tests { + use effectcraft_text::fonts; + + /// Whether an installed face covers `probe` (else the glyph check is skipped). + fn installed(probe: char) -> bool { let base = fonts::resolve("Inter", "Regular").face; - if !fonts::face(fonts::fallback_for('あ', base)).has_char('あ') { - eprintln!("no Japanese system font installed; skipping glyph coverage"); - return; - } + fonts::face(fonts::fallback_for(probe, base)).has_char(probe) + } + + /// Every character of `text` draws a real glyph in every UI font family with the `language` + /// interface's fonts installed. + fn assert_drawn(language: &str, text: &str) { let ctx = egui::Context::default(); - super::install(&ctx, &super::Tokens::for_kind(super::ThemeKind::Dark), "ja"); + super::install(&ctx, &super::Tokens::for_kind(super::ThemeKind::Dark), language); let mut out = ctx.run_ui(egui::RawInput::default(), |_| {}); // No renderer here: drop the frame's texture uploads (egui asserts on unhandled ones in debug). out.textures_delta.clear(); @@ -441,11 +466,11 @@ mod japanese_font_tests { ] { let font = egui::FontId::new(13.0, family); // epaint 0.36's has_glyph compares face keys, so it reports a false - // negative when a real Japanese glyph shares the replacement face. + // negative when a real glyph shares the replacement face. // Check the rendered atlas glyph instead of that face-level predicate. let missing = fonts.layout_no_wrap("\u{10ffff}".into(), font.clone(), egui::Color32::WHITE); let missing_uv = missing.rows[0].glyphs[0].uv_rect; - for ch in "日本語コンポジションレイヤーエフェクト設定".chars() { + for ch in text.chars() { let rendered = fonts.layout_no_wrap(ch.to_string(), font.clone(), egui::Color32::WHITE); let uv = rendered.rows[0].glyphs[0].uv_rect; assert!(uv.size.x > 0.0 && uv.size.y > 0.0, "empty {ch} in {font:?}"); @@ -454,6 +479,61 @@ mod japanese_font_tests { } }); } + + #[test] + fn installed_japanese_fallback_is_available_in_all_ui_families() { + if !installed('あ') { + eprintln!("no Japanese system font installed; skipping glyph coverage"); + return; + } + assert_drawn("ja", "日本語コンポジションレイヤーエフェクト設定"); + } + + /// #394: Arabic, Hebrew, Indic and Thai names are drawn by an installed face in an English + /// interface: the UI fonts were Inter and the CJK fallbacks only, so a layer named in Arabic + /// showed as missing-glyph boxes. + #[test] + fn installed_script_fallbacks_draw_arabic_hebrew_indic_and_thai_names() { + // Letters only: a vowel sign or virama on its own is drawn on a dotted circle. + for text in ["طبقةمرحبا", "שכבה", "परत", "কলম", "ਕਲਮ", "કલમ", "கடல", "కలమ", "ಕಲಮ", "കലമ", "กขค"] + { + let Some(probe) = text.chars().next() else { continue }; + if !installed(probe) { + eprintln!("no system font covers {text}; skipping its glyph coverage"); + continue; + } + assert_drawn("en", text); + } + } + + /// egui shapes each run of one face: Arabic letters take their joined forms, and an Arabic + /// word reads right to left (its first letter drawn rightmost). + #[test] + fn arabic_words_are_joined_and_read_right_to_left() { + if !installed('م') { + eprintln!("no Arabic system font installed; skipping"); + return; + } + let ctx = egui::Context::default(); + super::install(&ctx, &super::Tokens::for_kind(super::ThemeKind::Dark), "en"); + let mut out = ctx.run_ui(egui::RawInput::default(), |_| {}); + out.textures_delta.clear(); + ctx.fonts_mut(|fonts| { + let font = egui::FontId::new(13.0, egui::FontFamily::Proportional); + let missing = fonts.layout_no_wrap("\u{10ffff}".into(), font.clone(), egui::Color32::WHITE).rows[0].glyphs[0].uv_rect; + let alone = fonts.layout_no_wrap("م".into(), font.clone(), egui::Color32::WHITE); + let word = fonts.layout_no_wrap("مرحبا".into(), font, egui::Color32::WHITE); + let glyphs = &word.rows[0].glyphs; + let meem = glyphs.iter().find(|g| g.chr == 'م').expect("the meem is laid out"); + let alif = glyphs.iter().find(|g| g.chr == 'ا').expect("the alif is laid out"); + assert!(meem.pos.x > alif.pos.x, "the first letter is drawn rightmost: {} vs {}", meem.pos.x, alif.pos.x); + let isolated = alone.rows[0].glyphs[0].uv_rect; + for uv in [meem.uv_rect, isolated] { + assert_ne!((uv.min, uv.max), (missing.min, missing.max), "a real glyph, not the replacement"); + } + assert_ne!((meem.uv_rect.min, meem.uv_rect.max), (isolated.min, isolated.max), "the meem takes its initial form"); + }); + } } #[cfg(test)] @@ -475,40 +555,49 @@ mod tests { assert_eq!(cjk_locale("en"), ""); assert_eq!(cjk_locale("system"), ""); let mut chinese = FontDefinitions::default(); - let installed = install_cjk_fallbacks(&mut chinese, "zh-hans"); - if installed.is_empty() { + let installed = install_script_fallbacks(&mut chinese, "zh-hans"); + let cjk = |names: &[String]| names.iter().filter(|n| matches!(n.as_str(), "chinese-system" | "japanese-system")).cloned().collect::>(); + if cjk(&installed).is_empty() { eprintln!("no CJK font is installed: only the language plumbing can be checked here"); } else { assert_eq!(installed.first().map(String::as_str), Some("chinese-system"), "the Han probe comes first"); assert!(chinese.font_data.contains_key("chinese-system")); - let expected: Vec<&str> = installed.iter().map(String::as_str).collect(); - for (family, stack) in &chinese.families { - assert_eq!(tail(stack, installed.len()), expected, "{family:?}"); - } + } + let expected: Vec<&str> = installed.iter().map(String::as_str).collect(); + for (family, stack) in &chinese.families { + assert_eq!(tail(stack, installed.len()), expected, "{family:?}"); } // A Japanese interface keeps the kana probe first, as it was before the language existed. let mut japanese = FontDefinitions::default(); - let installed = install_cjk_fallbacks(&mut japanese, "ja"); - if let Some(first) = installed.first() { + let installed = install_script_fallbacks(&mut japanese, "ja"); + if let Some(first) = cjk(&installed).first() { + assert_eq!(installed.first(), Some(first)); assert_eq!(first, "japanese-system", "the kana probe comes first"); } // And an English interface leaves the operating system's locale in charge. let mut english = FontDefinitions::default(); - install_cjk_fallbacks(&mut english, "en"); + install_script_fallbacks(&mut english, "en"); } #[test] - fn a_cjk_face_is_registered_at_most_once() { + fn a_fallback_face_is_registered_at_most_once() { let mut fonts = FontDefinitions::default(); - let installed = install_cjk_fallbacks(&mut fonts, "zh-hant"); + let installed = install_script_fallbacks(&mut fonts, "zh-hant"); let mut unique = installed.clone(); unique.sort(); unique.dedup(); - assert_eq!(unique, installed, "no face may be registered twice"); - for name in &installed { - assert!(fonts.font_data.contains_key(name), "{name} is missing its font data"); + assert_eq!(unique.len(), installed.len(), "no name may be registered twice: {installed:?}"); + let faces: Vec<_> = installed + .iter() + .map(|name| { + let data = fonts.font_data.get(name).unwrap_or_else(|| panic!("{name} is missing its font data")); + (data.font.as_ref(), data.index) + }) + .collect(); + for (i, a) in faces.iter().enumerate() { + assert!(!faces[i + 1..].contains(a), "{} shares its face with a later fallback: {installed:?}", installed[i]); } // Leave the operating system's locale behind for whatever runs next. - install_cjk_fallbacks(&mut fonts, "system"); + install_script_fallbacks(&mut fonts, "system"); } } diff --git a/crates/ui-egui/tests/ui_inline_edit.rs b/crates/ui-egui/tests/ui_inline_edit.rs index 5f0e10ad..fc5c522e 100644 --- a/crates/ui-egui/tests/ui_inline_edit.rs +++ b/crates/ui-egui/tests/ui_inline_edit.rs @@ -5,6 +5,7 @@ use effectcraft_engine::Session; use effectcraft_ui_egui::EffectcraftApp; use egui::{Event, Key, Modifiers, PointerButton, Pos2, pos2}; use egui_kittest::Harness; +use egui_kittest::kittest::{NodeT, Queryable}; use serde_json::json; fn harness() -> (Harness<'static, EffectcraftApp>, u64) { @@ -422,3 +423,54 @@ fn ctrl_click_on_the_time_display_toggles_timecode_and_frames() { click_with(&mut h, at, Modifiers::COMMAND); assert_eq!(style(&h), TimeDisplayStyle::Timecode); } + +/// Hold Alt and Shift and let go of them, as Windows' keyboard-layout hotkey does; `shift_first` +/// releases Shift before Alt. +fn switch_layout(h: &mut Harness<'_, EffectcraftApp>, shift_first: bool) { + let held = if shift_first { Modifiers::ALT } else { Modifiers::SHIFT }; + for m in [Modifiers::ALT, Modifiers::ALT | Modifiers::SHIFT, held, Modifiers::NONE] { + h.input_mut().events.push(Event::ModifiersChanged(m)); + h.step(); + } + h.run_steps(2); +} + +/// #394: switching the keyboard layout to Arabic with Alt+Shift while a name is being typed +/// leaves the field with the keyboard: releasing Shift before Alt left Alt held alone, which +/// counted as an Alt tap, so the menu bar took the keyboard, the rename ended and the Arabic +/// letters went nowhere. Layer and Project names keep what is typed in any script. +#[test] +fn a_layout_switch_keeps_the_name_field_and_arabic_names_are_kept() { + let (mut h, comp) = harness(); + let layer = h.state_mut().session.execute("layer.newNull", json!({})).unwrap()["layer"].as_u64().unwrap(); + h.run_steps(3); + let layer_name = + |h: &Harness<'_, EffectcraftApp>| h.state().session.active_comp().unwrap().layer(effectcraft_engine::project::LayerId(layer)).unwrap().name.clone(); + let p = center(&h, &format!("timeline.layer.{layer}.row")); + click(&mut h, p, 2); + assert!(h.state().auto.find(&format!("timeline.layer.{layer}.nameEdit")).is_some(), "double-click opened the rename field"); + for shift_first in [true, false] { + switch_layout(&mut h, shift_first); + assert!(h.state().auto.find(&format!("timeline.layer.{layer}.nameEdit")).is_some(), "the rename field stays open"); + type_text(&mut h, if shift_first { "طبقة " } else { "مرحبا" }); + } + key(&mut h, Key::Enter); + assert_eq!(layer_name(&h), "طبقة مرحبا"); + // The Project panel's rename field, with Hebrew, Devanagari and Thai. + let p = center(&h, &format!("project.item.{comp}.name")); + click(&mut h, p, 1); + key(&mut h, Key::Enter); + switch_layout(&mut h, true); + type_text(&mut h, "שכבה नाम ชื่อ"); + key(&mut h, Key::Enter); + assert_eq!(h.state().session.project.items.values().find(|i| i.id.0 == comp).unwrap().name, "שכבה नाम ชื่อ"); + // A plain Alt tap still takes the keyboard to the menu bar (#279). + if !cfg!(target_os = "macos") { + h.input_mut().events.push(Event::ModifiersChanged(Modifiers::ALT)); + h.step(); + h.input_mut().events.push(Event::ModifiersChanged(Modifiers::NONE)); + h.run_steps(2); + let focused = h.query_by(|n| n.is_focused()).and_then(|n| n.accesskit_node().label()).unwrap_or_default(); + assert_eq!(focused.trim(), "File", "Alt focused the menu bar"); + } +}