shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: support integer CPAL devices and recover voice-over/audio failures 

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
cirocueto2013-wq 2026-10-07 20:33:33 -03:00
parent a4536bcfed
commit 7151d329d6
8 changed files with 529 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
apps/filmcraft/src/audio.rs
View File

@ -2,7 +2,7 @@
//! ("Device Class"), the output device, the I/O buffer size and the sample rate.
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use filmcraft_engine::settings::AudioHardwarePrefs;
@ -11,6 +11,7 @@ use filmcraft_ui_egui::{AudioDevices, AudioOut};
pub struct CpalOut {
stream: Option<cpal::Stream>,
played: Arc<AtomicU64>,
failed: Arc<AtomicBool>,
rate: u32,
channels: u16,
hw: AudioHardwarePrefs,
@ -41,34 +42,82 @@ fn output_device(h: &cpal::Host, name: &str) -> Option<cpal::Device> {
}
impl CpalOut {
pub fn new() -> Option<Self> {
pub fn new() -> Self {
let host = cpal::default_host();
let dev = host.default_output_device()?;
let cfg = dev.default_output_config().ok()?;
Some(Self {
let cfg = host.default_output_device().and_then(|dev| dev.default_output_config().ok());
Self {
stream: None,
played: Arc::new(AtomicU64::new(0)),
rate: cfg.sample_rate().0,
channels: cfg.channels(),
failed: Arc::new(AtomicBool::new(false)),
rate: cfg.as_ref().map_or(48_000, |c| c.sample_rate().0),
channels: cfg.as_ref().map_or(2, |c| c.channels()),
hw: AudioHardwarePrefs::default(),
document_rate: None,
})
}
}
/// The stream configuration the settings ask for, falling back to the device default.
fn config(&self, dev: &cpal::Device) -> Result<cpal::SupportedStreamConfig, String> {
let default = dev.default_output_config().map_err(|e| e.to_string())?;
let want = if self.hw.force_document_rate { self.document_rate.unwrap_or(self.hw.sample_rate) } else { self.hw.sample_rate };
let exact = dev.supported_output_configs().ok().and_then(|mut it| {
it.find(|c| {
c.sample_format() == cpal::SampleFormat::F32
&& c.channels() == default.channels()
&& (c.min_sample_rate().0..=c.max_sample_rate().0).contains(&want)
})
.and_then(|c| c.try_with_sample_rate(cpal::SampleRate(want)))
let exact = dev.supported_output_configs().ok().and_then(|it| {
it.filter(|c| c.channels() > 0 && (c.min_sample_rate().0..=c.max_sample_rate().0).contains(&want))
.max_by_key(|c| {
(c.channels() == default.channels(), c.sample_format() == cpal::SampleFormat::F32, c.sample_format() == default.sample_format())
})
.and_then(|c| c.try_with_sample_rate(cpal::SampleRate(want)))
});
Ok(exact.unwrap_or(default))
}
fn converted_stream<T>(
&self,
dev: &cpal::Device,
config: &cpal::StreamConfig,
mut fill: Box<dyn FnMut(&mut [f32], usize) + Send>,
) -> Result<cpal::Stream, cpal::BuildStreamError>
where
T: cpal::SizedSample + cpal::FromSample<f32>,
{
let channels = usize::from(config.channels);
let played = self.played.clone();
let failed = self.failed.clone();
let error_flag = failed.clone();
let mut scratch = Vec::new();
dev.build_output_stream(
config,
move |buf: &mut [T], _| {
// Reuse the conversion buffer. A pathological driver buffer must not panic the audio thread.
if buf.len() > 1_048_576 || scratch.try_reserve(buf.len().saturating_sub(scratch.len())).is_err() {
buf.fill(T::EQUILIBRIUM);
if !failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: unable to allocate the audio conversion buffer");
}
return;
}
scratch.resize(buf.len(), 0.0);
fill(&mut scratch, channels);
convert_samples(&scratch, buf);
played.fetch_add((buf.len() / channels) as u64, Ordering::SeqCst);
},
move |e| stream_error(&error_flag, e),
None,
)
}
}
fn stream_error(failed: &AtomicBool, e: cpal::StreamError) {
if !failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: audio stream error: {e}");
}
}
fn convert_samples<T: cpal::Sample + cpal::FromSample<f32>>(input: &[f32], output: &mut [T]) {
// Integer conversion represents [-1, 1). In particular I24 cannot represent +1 exactly.
let upper = f32::from_bits(1.0_f32.to_bits() - 1);
for (sample, converted) in input.iter().zip(output) {
*converted = T::from_sample(if sample.is_finite() { sample.clamp(-1.0, upper) } else { 0.0 });
}
}
impl AudioOut for CpalOut {
@ -78,6 +127,9 @@ impl AudioOut for CpalOut {
let dev = output_device(&host, &self.hw.default_output).ok_or("no output device")?;
let cfg = self.config(&dev)?;
let channels = cfg.channels() as usize;
if channels == 0 || cfg.sample_rate().0 == 0 {
return Err("the output device returned an invalid channel count or sample rate".into());
}
let mut config: cpal::StreamConfig = cfg.clone().into();
if let cpal::SupportedBufferSize::Range { min, max } = cfg.buffer_size()
&& (*min..=*max).contains(&self.hw.buffer_size)
@ -87,8 +139,9 @@ impl AudioOut for CpalOut {
self.rate = cfg.sample_rate().0;
self.channels = cfg.channels();
self.played.store(0, Ordering::SeqCst);
self.failed.store(false, Ordering::SeqCst);
let played = self.played.clone();
let err = |e| eprintln!("filmcraft: audio stream error: {e}");
let failed = self.failed.clone();
let stream = match cfg.sample_format() {
cpal::SampleFormat::F32 => dev.build_output_stream(
&config,
@ -96,9 +149,19 @@ impl AudioOut for CpalOut {
fill(buf, channels);
played.fetch_add((buf.len() / channels) as u64, Ordering::SeqCst);
},
err,
move |e| stream_error(&failed, e),
None,
),
cpal::SampleFormat::I8 => self.converted_stream::<i8>(&dev, &config, fill),
cpal::SampleFormat::I16 => self.converted_stream::<i16>(&dev, &config, fill),
cpal::SampleFormat::I24 => self.converted_stream::<cpal::I24>(&dev, &config, fill),
cpal::SampleFormat::I32 => self.converted_stream::<i32>(&dev, &config, fill),
cpal::SampleFormat::I64 => self.converted_stream::<i64>(&dev, &config, fill),
cpal::SampleFormat::U8 => self.converted_stream::<u8>(&dev, &config, fill),
cpal::SampleFormat::U16 => self.converted_stream::<u16>(&dev, &config, fill),
cpal::SampleFormat::U32 => self.converted_stream::<u32>(&dev, &config, fill),
cpal::SampleFormat::U64 => self.converted_stream::<u64>(&dev, &config, fill),
cpal::SampleFormat::F64 => self.converted_stream::<f64>(&dev, &config, fill),
other => return Err(format!("unsupported sample format {other:?}")),
}
.map_err(|e| e.to_string())?;
@ -116,7 +179,7 @@ impl AudioOut for CpalOut {
self.channels as usize
}
fn played_frames(&self) -> Option<u64> {
self.stream.as_ref().map(|_| self.played.load(Ordering::SeqCst))
self.stream.as_ref().filter(|_| !self.failed.load(Ordering::SeqCst)).map(|_| self.played.load(Ordering::SeqCst))
}
fn devices(&self) -> AudioDevices {
let h = host(&self.hw.device_class);
@ -138,3 +201,35 @@ impl AudioOut for CpalOut {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use cpal::Sample;
#[test]
fn integer_conversion_clips_and_preserves_interleaved_channels() {
let mut signed = [0_i16; 8];
convert_samples(&[-2.0, 2.0, -0.5, 0.5, 0.0, f32::NAN, f32::INFINITY, f32::NEG_INFINITY], &mut signed);
assert_eq!(signed, [i16::MIN, i16::MAX, -16_384, 16_384, 0, 0, 0, 0]);
let mut unsigned = [0_u16; 3];
convert_samples(&[-1.0, 0.0, 1.0], &mut unsigned);
assert_eq!(unsigned, [0, 32_768, u16::MAX]);
}
#[test]
fn full_scale_24_bit_samples_remain_representable() {
let mut samples = [cpal::I24::EQUILIBRIUM; 3];
convert_samples(&[-1.0, 0.0, 1.0], &mut samples);
assert_eq!(samples.map(|s| s.inner()), [-8_388_608, 0, 8_388_607]);
}
#[test]
fn failed_stream_relinquishes_the_playback_clock() {
let failed = AtomicBool::new(false);
stream_error(&failed, cpal::StreamError::DeviceNotAvailable);
assert!(failed.load(Ordering::SeqCst));
let output = CpalOut::new();
assert_eq!(output.played_frames(), None);
}
}

226
apps/filmcraft/src/audio_in.rs
View File

@ -14,11 +14,13 @@ pub struct CpalIn {
host: String,
buf: Arc<Mutex<Vec<Vec<f32>>>>,
stop: Option<mpsc::Sender<()>>,
worker: Option<std::thread::JoinHandle<()>>,
error: Arc<Mutex<Option<String>>>,
}
impl CpalIn {
pub fn new(host: &str) -> Self {
Self { host: host.to_string(), buf: Arc::default(), stop: None }
Self { host: host.to_string(), buf: Arc::default(), stop: None, worker: None, error: Arc::default() }
}
}
@ -46,7 +48,63 @@ fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
fn report_error(error: &Mutex<Option<String>>, message: String) {
let mut error = lock(error);
if error.is_none() {
eprintln!("filmcraft: audio input error: {message}");
*error = Some(message);
}
}
fn append<T: cpal::Sample>(data: &[T], channels: usize, buf: &Mutex<Vec<Vec<f32>>>, error: &Mutex<Option<String>>)
where
f32: cpal::FromSample<T>,
{
if lock(error).is_some() {
return;
}
let mut b = lock(buf);
if channels == 0 || channels > 256 || data.len() > 1_048_576 {
report_error(error, "invalid input device buffer".into());
return;
}
if b.len() != channels {
*b = vec![Vec::new(); channels];
}
// The recorder drains at stop. Bound accumulated samples and report exhaustion instead of
// allowing a long take or a broken/unpaced driver to exhaust the application heap.
let samples = b.first().map_or(0, Vec::len).saturating_mul(channels).saturating_add(data.len());
let frames = data.len() / channels;
if samples > 268_435_456 || b.iter_mut().any(|c| c.try_reserve(frames).is_err()) {
report_error(error, "voice-over input buffer is full; recording failed".into());
return;
}
for frame in data.chunks_exact(channels) {
for (channel, sample) in b.iter_mut().zip(frame) {
let sample = sample.to_sample::<f32>();
channel.push(if sample.is_finite() { sample.clamp(-1.0, 1.0) } else { 0.0 });
}
}
}
fn input_stream<T: cpal::SizedSample>(
dev: &cpal::Device,
config: &cpal::StreamConfig,
buf: Arc<Mutex<Vec<Vec<f32>>>>,
error: Arc<Mutex<Option<String>>>,
) -> Result<cpal::Stream, cpal::BuildStreamError>
where
f32: cpal::FromSample<T>,
{
let channels = usize::from(config.channels);
let stream_error = error.clone();
dev.build_input_stream(config, move |data: &[T], _| append(data, channels, &buf, &error), move |e| report_error(&stream_error, e.to_string()), None)
}
impl AudioInput for CpalIn {
fn configure_host(&mut self, name: &str) {
self.host = name.to_string();
}
fn devices(&self) -> Vec<String> {
host(&self.host).input_devices().map(|d| d.filter_map(|x| x.name().ok()).collect()).unwrap_or_default()
}
@ -56,68 +114,77 @@ impl AudioInput for CpalIn {
fn start(&mut self, name: &str, sample_rate: u32) -> Result<InputFormat, String> {
self.stop();
lock(&self.buf).clear();
*lock(&self.error) = None;
let (host_name, name) = (self.host.clone(), name.to_string());
let buf = self.buf.clone();
let error = self.error.clone();
let (fmt_tx, fmt_rx) = mpsc::channel::<Result<InputFormat, String>>();
let (stop_tx, stop_rx) = mpsc::channel::<()>();
std::thread::Builder::new()
.name("voice-over input".into())
.spawn(move || {
let open = || -> Result<(cpal::Stream, InputFormat), String> {
let h = host(&host_name);
let dev = device(&h, &name).ok_or("no input device")?;
let default = dev.default_input_config().map_err(|e| e.to_string())?;
// the sequence rate when the device offers it in f32, else the device default
let cfg = dev
.supported_input_configs()
.ok()
.and_then(|mut it| {
it.find(|c| c.sample_format() == cpal::SampleFormat::F32 && (c.min_sample_rate().0..=c.max_sample_rate().0).contains(&sample_rate))
.and_then(|c| c.try_with_sample_rate(cpal::SampleRate(sample_rate)))
})
.unwrap_or(default);
if cfg.sample_format() != cpal::SampleFormat::F32 {
return Err(format!("unsupported input sample format {:?}", cfg.sample_format()));
}
let ch = cfg.channels() as usize;
let fmt = InputFormat { sample_rate: cfg.sample_rate().0, channels: cfg.channels() };
let stream = dev
.build_input_stream(
&cfg.into(),
move |data: &[f32], _| {
let mut b = lock(&buf);
if b.len() != ch {
*b = vec![Vec::new(); ch];
}
for frame in data.chunks_exact(ch.max(1)) {
for (c, x) in frame.iter().enumerate() {
b[c].push(*x);
}
}
},
|e| eprintln!("filmcraft: audio input error: {e}"),
None,
)
self.worker = Some(
std::thread::Builder::new()
.name("voice-over input".into())
.spawn(move || {
let open = || -> Result<(cpal::Stream, InputFormat), String> {
let h = host(&host_name);
let dev = device(&h, &name).ok_or("no input device")?;
let default = dev.default_input_config().map_err(|e| e.to_string())?;
// Prefer f32 at the sequence rate, retaining integer-only input devices.
let cfg = dev
.supported_input_configs()
.ok()
.and_then(|it| {
it.filter(|c| c.channels() > 0 && (c.min_sample_rate().0..=c.max_sample_rate().0).contains(&sample_rate))
.max_by_key(|c| (c.channels() == default.channels(), c.sample_format() == cpal::SampleFormat::F32))
.and_then(|c| c.try_with_sample_rate(cpal::SampleRate(sample_rate)))
})
.unwrap_or(default);
if cfg.channels() == 0 || cfg.channels() > 256 || cfg.sample_rate().0 == 0 {
return Err("the input device returned an invalid format".into());
}
let fmt = InputFormat { sample_rate: cfg.sample_rate().0, channels: cfg.channels() };
let format = cfg.sample_format();
let config = cfg.into();
let stream = match format {
cpal::SampleFormat::I8 => input_stream::<i8>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::I16 => input_stream::<i16>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::I24 => input_stream::<cpal::I24>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::I32 => input_stream::<i32>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::I64 => input_stream::<i64>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::U8 => input_stream::<u8>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::U16 => input_stream::<u16>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::U32 => input_stream::<u32>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::U64 => input_stream::<u64>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::F32 => input_stream::<f32>(&dev, &config, buf.clone(), error.clone()),
cpal::SampleFormat::F64 => input_stream::<f64>(&dev, &config, buf.clone(), error.clone()),
other => return Err(format!("unsupported input sample format {other:?}")),
}
.map_err(|e| e.to_string())?;
stream.play().map_err(|e| e.to_string())?;
Ok((stream, fmt))
};
match open() {
Ok((stream, fmt)) => {
let _ = fmt_tx.send(Ok(fmt));
// keep the stream alive until stop (or the sender is dropped)
let _ = stop_rx.recv();
drop(stream);
stream.play().map_err(|e| e.to_string())?;
Ok((stream, fmt))
};
let opened = std::panic::catch_unwind(std::panic::AssertUnwindSafe(open)).unwrap_or_else(|_| Err("audio input worker panicked".into()));
match opened {
Ok((stream, fmt)) => {
let _ = fmt_tx.send(Ok(fmt));
// keep the stream alive until stop (or the sender is dropped)
let _ = stop_rx.recv();
drop(stream);
}
Err(e) => {
let _ = fmt_tx.send(Err(e));
}
}
Err(e) => {
let _ = fmt_tx.send(Err(e));
}
}
})
.map_err(|e| e.to_string())?;
let fmt = fmt_rx.recv().map_err(|e| e.to_string())??;
})
.map_err(|e| e.to_string())?,
);
self.stop = Some(stop_tx);
Ok(fmt)
match fmt_rx.recv().map_err(|e| e.to_string()).and_then(|f| f) {
Ok(fmt) => Ok(fmt),
Err(e) => {
self.stop();
Err(e)
}
}
}
fn read(&mut self, frames: usize) -> Vec<Vec<f32>> {
let mut b = lock(&self.buf);
@ -131,9 +198,56 @@ impl AudioInput for CpalIn {
fn discard(&mut self) {
lock(&self.buf).iter_mut().for_each(Vec::clear);
}
fn error(&self) -> Option<String> {
lock(&self.error).clone()
}
fn stop(&mut self) {
if let Some(tx) = self.stop.take() {
let _ = tx.send(());
}
if let Some(worker) = self.worker.take()
&& worker.join().is_err()
{
report_error(&self.error, "audio input worker panicked".into());
}
}
}
impl Drop for CpalIn {
fn drop(&mut self) {
self.stop();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn signed_and_unsigned_input_is_deinterleaved_to_float() {
let buf = Mutex::new(Vec::new());
let error = Mutex::new(None);
append(&[i16::MIN, i16::MAX, 0, 16384], 2, &buf, &error);
let data = lock(&buf);
assert_eq!(data[0], [-1.0, 0.0]);
assert!((data[1][0] - 1.0).abs() < 0.0001);
assert_eq!(data[1][1], 0.5);
drop(data);
lock(&buf).clear();
append(&[0_u16, 32768, 65535, 32768], 2, &buf, &error);
let data = lock(&buf);
assert_eq!(data[1], [0.0, 0.0]);
assert_eq!(data[0][0], -1.0);
assert!(lock(&error).is_none());
}
#[test]
fn invalid_input_is_reported_without_panicking_or_appending() {
let buf = Mutex::new(Vec::new());
let error = Mutex::new(None);
append(&[0.0_f32], 0, &buf, &error);
assert!(lock(&error).is_some());
append(&[1.0_f32], 1, &buf, &error);
assert!(lock(&buf).is_empty());
}
}

7
apps/filmcraft/src/main.rs
View File

@ -164,10 +164,9 @@ fn main() -> eframe::Result {
{
app.set_wgpu(rs);
}
if let Some(out) = audio::CpalOut::new() {
// Settings ▸ Audio Hardware is applied on the first frame (`apply_prefs`)
app.audio = Some(Box::new(out));
}
// Keep device selection available even when the default device is unavailable.
// Settings ▸ Audio Hardware is applied on the first frame (`apply_prefs`).
app.audio = Some(Box::new(audio::CpalOut::new()));
app.hooks.pick_files = Some(Box::new(|exts: &[&str]| {
rfd::FileDialog::new().add_filter("Media", exts).pick_files().unwrap_or_default().into_iter().map(|p| p.to_string_lossy().to_string()).collect()
}));

90
crates/engine/src/voiceover.rs
View File

@ -43,6 +43,8 @@ pub struct InputFormat {
/// An audio input device (cpal on desktop, [`SyntheticInput`] headless).
pub trait AudioInput: Send {
/// Select the host for device discovery and the next take, preserving any active capture.
fn configure_host(&mut self, _host: &str) {}
/// Names of the devices that can be chosen as Source.
fn devices(&self) -> Vec<String>;
/// Input channels of `device` ("" = the default device).
@ -52,6 +54,10 @@ pub trait AudioInput: Send {
/// Up to `frames` captured frames (planar, every device channel), oldest first. A live device
/// returns what it has captured; a synthetic one generates exactly `frames`.
fn read(&mut self, frames: usize) -> Vec<Vec<f32>>;
/// A device error must not be saved as a successful silent take.
fn error(&self) -> Option<String> {
None
}
/// Drop everything captured so far: the next sample read is "now".
fn discard(&mut self);
fn stop(&mut self);
@ -172,6 +178,8 @@ pub struct VoiceOver {
pub rec: Option<Recording>,
/// Frames already read from the input in this recording (captured before stop).
captured: Vec<f32>,
/// A stopped take awaiting a successful save; retries keep its original end point.
pending_stop: Option<Tick>,
}
impl VoiceOver {
@ -333,13 +341,17 @@ fn start(s: &mut Session, p: &Value) -> Result<Value> {
let vo = s.prefs.voice_over.clone();
let punch = mark_in.is_some() && mark_out.is_some_and(|o| Some(o) > mark_in);
let record_start = if punch { mark_in.unwrap_or_default() } else { time_p(s, p, "").unwrap_or(s.playhead()) };
let preroll = Tick::from_seconds_f64(f64_p(p, "preroll").unwrap_or(vo.preroll_seconds).max(0.0));
if record_start < Tick::ZERO {
return Err(bad("audio.voiceover.start", "the record point cannot be negative"));
}
let preroll = Tick::from_seconds_f64(f64_p(p, "preroll").unwrap_or(vo.preroll_seconds).clamp(0.0, 60.0));
let capture_start = (record_start - preroll).max(Tick::ZERO);
let punch_out = if punch { mark_out } else { None };
let device = if vo.source.is_empty() { s.prefs.audio_hardware.default_input.clone() } else { vo.source.clone() };
let format = s.voiceover.input().start(&device, sr).map_err(|e| EngineError::Other(format!("voice-over input: {e}")))?;
let channel = vo.input_channel.min(format.channels.saturating_sub(1) as u32);
s.voiceover.captured.clear();
s.voiceover.pending_stop = None;
s.voiceover.rec = Some(Recording { seq: seq_id, track, record_start, capture_start, punch_out, format, channel });
let cues: Vec<i64> = if vo.countdown_sound_cues { cue_times(capture_start, record_start).into_iter().map(|t| t.0).collect() } else { Vec::new() };
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
@ -357,11 +369,18 @@ fn start(s: &mut Session, p: &Value) -> Result<Value> {
/// Playback really started at `time`: drop what was captured so far and count from there.
fn sync(s: &mut Session, p: &Value) -> Result<Value> {
if s.voiceover.pending_stop.is_some() {
return Err(bad("audio.voiceover.sync", "save or discard the stopped take before restarting capture"));
}
if s.voiceover.rec.is_none() {
return Err(bad("audio.voiceover.sync", "no voice-over is recording"));
}
let t = time_p(s, p, "").unwrap_or(s.playhead());
s.voiceover.input().discard();
s.voiceover.captured.clear();
s.voiceover.pending_stop = None;
let Some(rec) = s.voiceover.rec.as_mut() else { return Err(bad("audio.voiceover.sync", "no voice-over is recording")) };
rec.capture_start = t.min(rec.record_start);
rec.capture_start = t.clamp(Tick::ZERO, rec.record_start);
Ok(json!({"captureStart": rec.capture_start.0}))
}
@ -381,29 +400,52 @@ fn record_dir(s: &Session, p: &Value) -> String {
}
fn stop(s: &mut Session, p: &Value) -> Result<Value> {
let rec = s.voiceover.rec.take().ok_or_else(|| bad("audio.voiceover.stop", "no voice-over is recording"))?;
let t_stop = time_p(s, p, "").unwrap_or(s.playhead());
let rec = s.voiceover.rec.clone().ok_or_else(|| bad("audio.voiceover.stop", "no voice-over is recording"))?;
if bool_p(p, "discard").unwrap_or(false) {
s.voiceover.input().stop();
s.voiceover.rec = None;
s.voiceover.pending_stop = None;
s.voiceover.captured.clear();
return Ok(json!({"recording": false, "placed": false}));
}
let t_stop = s.voiceover.pending_stop.unwrap_or_else(|| time_p(s, p, "").unwrap_or(s.playhead()));
let sr = rec.format.sample_rate.max(1) as i64;
let c0 = rec.capture_start.to_units_floor(sr);
let r0 = rec.record_start.to_units_floor(sr);
let end = rec.punch_out.map_or(t_stop, |o| o.min(t_stop)).to_units_floor(sr);
// read the input up to the end of the take
let want = (end - c0).max(0) as usize;
let have = s.voiceover.captured.len();
let input = s.voiceover.input();
let got = input.read(want.saturating_sub(have));
input.stop();
let ch = rec.channel as usize;
let mut mono = std::mem::take(&mut s.voiceover.captured);
if let Some(c) = got.get(ch).or(got.first()) {
mono.extend_from_slice(c);
let want = end.saturating_sub(c0).max(0) as usize;
if want.saturating_mul(usize::from(rec.format.channels).max(1)) > 268_435_456 {
return Err(bad("audio.voiceover.stop", "recording exceeds the input buffer limit"));
}
mono.resize(want, 0.0);
let skip = (r0 - c0).clamp(0, want as i64) as usize;
let samples = &mono[skip..];
if samples.is_empty() || bool_p(p, "discard").unwrap_or(false) {
let have = s.voiceover.captured.len();
s.voiceover.captured.try_reserve(want.saturating_sub(have)).map_err(|e| bad("audio.voiceover.stop", format!("unable to allocate the recording: {e}")))?;
s.voiceover.pending_stop = Some(t_stop);
let input = s.voiceover.input();
input.stop();
let got = input.read(want.saturating_sub(have));
let input_error = input.error();
let ch = rec.channel as usize;
if let Some(c) = got.get(ch).or(got.first()) {
s.voiceover.captured.extend_from_slice(c);
}
if let Some(error) = input_error {
return Err(bad("audio.voiceover.stop", format!("input failed: {error}")));
}
s.voiceover.captured.resize(want, 0.0);
let skip = r0.saturating_sub(c0).clamp(0, want as i64) as usize;
let samples = s.voiceover.captured.get(skip..).ok_or_else(|| bad("audio.voiceover.stop", "invalid recording trim range"))?;
if samples.is_empty() {
s.voiceover.rec = None;
s.voiceover.pending_stop = None;
s.voiceover.captured.clear();
return Ok(json!({"recording": false, "placed": false}));
}
let sample_count = samples.len();
let sequence = s.project.sequence(rec.seq).ok_or(EngineError::NoSequence)?;
if !sequence.audio_tracks.iter().any(|track| track.id == rec.track && !track.locked) {
return Err(bad("audio.voiceover.stop", "the record track was deleted or locked; restore it before saving the take"));
}
// write the file
let dir = record_dir(s, p);
let base = s.prefs.voice_over.name.clone();
@ -418,14 +460,15 @@ fn stop(s: &mut Session, p: &Value) -> Result<Value> {
k += 1;
};
if !cfg!(target_arch = "wasm32") {
let _ = std::fs::create_dir_all(&dir);
std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?;
}
let samples = s.voiceover.captured.get(skip..).ok_or_else(|| bad("audio.voiceover.stop", "invalid recording trim range"))?;
let bytes = write_wav_f32(samples, sr as u32);
s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
// import + place as one undo step
let n0 = s.history.undo.len();
let item = crate::commands::import_bytes(s, &path, bytes.into(), None)?;
let dur = Tick::from_units(samples.len() as i64, sr);
let dur = Tick::from_units(sample_count as i64, sr);
let at = rec.record_start;
let track = rec.track;
let rate = s.project.sequence(rec.seq).map(|q| q.settings.frame_rate).unwrap_or_default();
@ -450,17 +493,20 @@ fn stop(s: &mut Session, p: &Value) -> Result<Value> {
});
crate::clip_ops::collapse_history(s, n0, "Record Voice-over");
let clip = clip?;
let seq = s.project.sequence(rec.seq).ok_or(EngineError::NoSequence)?;
let track_label = crate::mixer::strip_label(s.project.sequence(rec.seq).ok_or(EngineError::NoSequence)?, track);
s.voiceover.rec = None;
s.voiceover.pending_stop = None;
s.voiceover.captured.clear();
Ok(json!({
"recording": false,
"placed": true,
"path": path,
"item": item.0,
"clip": clip.0,
"track": crate::mixer::strip_label(seq, track),
"track": track_label,
"start": at.0,
"duration": dur.0,
"samples": samples.len(),
"samples": sample_count,
"sampleRate": sr,
}))
}

32
crates/engine/src/voiceover_tests.rs
View File

@ -37,6 +37,38 @@ fn track_clips(s: &Session, i: usize) -> Vec<filmcraft_project::TrackItem> {
s.active_sequence().unwrap().audio_tracks[i].items.clone()
}
#[test]
fn a_failed_recording_save_keeps_the_take_for_retry() {
let (mut s, track, index) = demo();
s.execute("playhead.set", json!({"seconds":0})).unwrap();
s.execute("audio.voiceover.start", json!({"track":track, "preroll":0})).unwrap();
let dir = tmp("save-retry");
std::fs::write(&dir, b"not a directory").unwrap();
assert!(s.execute("audio.voiceover.stop", json!({"time":sec(1.0),"dir":dir})).is_err());
assert!(s.voiceover.recording(), "failed save must retain the recording for retry");
assert!(s.execute("audio.voiceover.sync", json!({"time":sec(1.5)})).is_err(), "playback must not discard an unsaved take");
std::fs::remove_file(&dir).unwrap();
let result = s.execute("audio.voiceover.stop", json!({"time":sec(2.0),"dir":dir})).unwrap();
assert_eq!(result["samples"], 48000, "retry preserves the stopped take's end point");
assert_eq!(result["placed"], true);
assert!(!s.voiceover.recording());
assert_eq!(track_clips(&s, index).len(), 1);
let audio = mix(&s, 0, 48000);
assert!(audio[0][0] > 0.49, "retry retained the first captured sample");
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn hostile_record_points_and_preroll_are_bounded() {
let (mut s, track, _) = demo();
assert!(s.execute("audio.voiceover.start", json!({"track":track,"time":-1,"preroll":1e300})).is_err());
assert!(!s.voiceover.recording());
let result = s.execute("audio.voiceover.start", json!({"track":track,"time":sec(120.0),"preroll":1e300})).unwrap();
assert_eq!(result["captureStart"], sec(60.0));
assert!(result["cues"].as_array().unwrap().len() <= 61);
s.execute("audio.voiceover.stop", json!({"discard":true})).unwrap();
}
#[test]
fn commands_are_registered_and_disabled_without_a_recording() {
for id in ["audio.voiceover.settings", "audio.voiceover.start", "audio.voiceover.sync", "audio.voiceover.stop"] {

97
crates/ui-egui/src/lib.rs
View File

@ -463,13 +463,18 @@ impl FilmcraftApp {
self.frames.set_cache_budget(p.memory.frame_cache_mb as usize * (1 << 20));
self.ui.play_after_render = p.timeline.play_after_rendering;
if prev.as_ref().is_none_or(|q| q.audio_hardware != p.audio_hardware) {
if let Some(input) = self.session.voiceover.input.as_mut() {
input.configure_host(&p.audio_hardware.device_class);
}
let rate = self.session.active_sequence().map(|q| q.settings.sample_rate);
let playing = self.playback.playing && self.playback.audio_clock;
let playing = self.playback.playing && self.playback.preroll.is_none();
if let Some(a) = self.audio.as_mut() {
a.stop();
a.configure(&p.audio_hardware, rate);
}
if playing {
self.playback.anchor_tick = self.session.playhead();
self.playback.anchor_time = ctx.input(|i| i.time);
self.start_audio();
}
}
@ -765,6 +770,18 @@ impl FilmcraftApp {
}
let rate = self.session.sequence_rate();
let reading = if self.playback.audio_clock { self.audio.as_ref().and_then(|a| a.played_frames().map(|f| (f, a.sample_rate()))) } else { None };
if self.playback.audio_clock && reading.is_none() {
// A device error relinquishes its clock immediately. Resume from the displayed
// frame, rather than jumping to the old wall-clock anchor after buffered playback.
self.playback.anchor_tick = self.session.playhead();
self.playback.anchor_time = now;
self.playback.audio_clock = false;
if let Some(audio) = self.audio.as_mut() {
audio.stop();
}
log::warn!("audio output lost its playback clock; playing without sound");
self.ui.status = "Audio output failed: playing without sound (check Settings ▸ Audio Hardware)".into();
}
if let Some((f, sr)) = reading {
if f != self.playback.audio_seen.0 {
self.playback.audio_seen = (f, now);
@ -1582,3 +1599,81 @@ mod gpu_fallback_tests {
assert_eq!(super::plan_side(&p), 4000);
}
}
#[cfg(test)]
mod audio_recovery_tests {
use super::*;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
struct Device {
ready: bool,
starts: Arc<AtomicUsize>,
reading: Option<u64>,
}
impl AudioOut for Device {
fn sample_rate(&self) -> u32 {
48000
}
fn channels(&self) -> usize {
2
}
fn configure(&mut self, hardware: &filmcraft_engine::settings::AudioHardwarePrefs, _: Option<u32>) {
self.ready = hardware.default_output == "available";
}
fn start(&mut self, _: Box<dyn FnMut(&mut [f32], usize) + Send>) -> Result<u32, String> {
self.starts.fetch_add(1, Ordering::Relaxed);
if self.ready { Ok(48000) } else { Err("device unavailable".into()) }
}
fn stop(&mut self) {}
fn played_frames(&self) -> Option<u64> {
self.reading
}
}
#[test]
fn changing_hardware_recovers_wall_clock_playback_without_rewinding() {
let mut session = Session::default();
session.execute("file.newSequence", json!({"width":16,"height":16})).unwrap();
let ctx = egui::Context::default();
let mut app = FilmcraftApp::new(session);
let starts = Arc::new(AtomicUsize::new(0));
app.audio = Some(Box::new(Device { ready: false, starts: starts.clone(), reading: Some(0) }));
app.apply_prefs(&ctx);
app.play(1.0);
app.end_preroll(0.0);
assert!(!app.playback.audio_clock);
app.session.set_playhead(Tick::from_seconds_f64(2.0));
app.session.prefs.audio_hardware.default_output = "available".into();
app.apply_prefs(&ctx);
assert_eq!(starts.load(Ordering::Relaxed), 2);
assert!(app.playback.audio_clock);
assert_eq!(app.playback.anchor_tick, app.session.playhead());
app.stop();
}
#[test]
fn a_failed_device_clock_resumes_from_the_displayed_frame() {
let mut session = Session::default();
session.execute("file.openDemoProject", json!({})).unwrap();
let ctx = egui::Context::default();
let mut app = FilmcraftApp::new(session);
app.audio = Some(Box::new(Device { ready: true, starts: Arc::new(AtomicUsize::new(0)), reading: None }));
let displayed = app.session.sequence_rate().tick_of(48);
app.session.set_playhead(displayed);
app.playback.playing = true;
app.playback.audio_clock = true;
app.playback.anchor_tick = Tick::ZERO;
app.playback.anchor_time = 0.0;
let mut output = ctx.run_ui(egui::RawInput { time: Some(10.0), ..Default::default() }, |ui| app.advance_playback(ui.ctx()));
output.textures_delta.clear();
assert!(!app.playback.audio_clock);
assert_eq!(app.playback.anchor_tick, displayed);
assert_eq!(app.session.playhead(), displayed);
assert!(app.ui.status.contains("Audio output failed"));
app.stop();
}
}

31
crates/ui-egui/src/panels/voiceover.rs
View File

@ -234,6 +234,7 @@ fn settings_dialog(app: &mut FilmcraftApp, ctx: &egui::Context) {
let mut elems: Vec<(String, Rect, String)> = Vec::new();
let mut close = false;
let mut apply = false;
let mut discard = false;
let accent = app.tokens.accent;
egui::Window::new("Voice-Over Record Settings").collapsible(false).resizable(false).anchor(Align2::CENTER_CENTER, [0.0, 0.0]).show(ctx, |ui| {
egui::Grid::new("voiceover-grid").num_columns(2).spacing([12.0, 8.0]).show(ui, |ui| {
@ -243,6 +244,7 @@ fn settings_dialog(app: &mut FilmcraftApp, ctx: &egui::Context) {
ui.end_row();
ui.label("Source:");
let shown = if d.prefs.source.is_empty() { "Default Input".to_string() } else { d.prefs.source.clone() };
let previous_source = d.prefs.source.clone();
let cb = egui::ComboBox::from_id_salt("voiceover-source").selected_text(&shown).width(220.0).show_ui(ui, |ui| {
let mut opts = vec![(String::new(), "Default Input".to_string())];
opts.extend(d.devices.iter().map(|x| (x.clone(), x.clone())));
@ -255,9 +257,17 @@ fn settings_dialog(app: &mut FilmcraftApp, ctx: &egui::Context) {
}
});
elems.push(("voiceover.source".into(), cb.response.rect, shown));
if d.prefs.source != previous_source {
let device = if d.prefs.source.is_empty() { app.session.prefs.audio_hardware.default_input.as_str() } else { d.prefs.source.as_str() };
if let Some(input) = app.session.voiceover.input.as_ref() {
d.channels = input.channels(device);
d.prefs.input_channel = d.prefs.input_channel.min(u32::from(d.channels.saturating_sub(1)));
}
}
ui.end_row();
ui.label("Input:");
let n = d.channels.max(1) as u32;
let n = u32::from(d.channels.clamp(1, 64));
d.prefs.input_channel = d.prefs.input_channel.min(63);
let cur = format!("Channel {}", d.prefs.input_channel + 1);
let cb = egui::ComboBox::from_id_salt("voiceover-input").selected_text(&cur).width(220.0).show_ui(ui, |ui| {
for c in 0..n.max(d.prefs.input_channel + 1) {
@ -291,6 +301,13 @@ fn settings_dialog(app: &mut FilmcraftApp, ctx: &egui::Context) {
);
ui.add_space(8.0);
ui.horizontal(|ui| {
if app.session.voiceover.recording() {
let r = ui.button("Discard take");
elems.push(("voiceover.discard".into(), r.rect, "Discard the active take".into()));
if r.clicked() {
discard = true;
}
}
let c = ui.button("Cancel");
elems.push(("voiceover.cancel".into(), c.rect, "Cancel".into()));
if c.clicked() {
@ -306,6 +323,18 @@ fn settings_dialog(app: &mut FilmcraftApp, ctx: &egui::Context) {
for (id, r, l) in elems {
app.auto.add(&id, r, &l);
}
if discard {
match app.session.execute("audio.voiceover.stop", json!({"discard":true})) {
Ok(_) => {
if app.playback.playing {
app.stop();
}
app.ui.status = "Voice-over take discarded".into();
close = true;
}
Err(error) => app.ui.status = error.to_string(),
}
}
if ctx.input(|i| i.key_pressed(egui::Key::Escape)) {
close = true;
}

17
crates/ui-egui/tests/voiceover_ui.rs
View File

@ -74,6 +74,23 @@ impl Driver {
}
}
#[test]
fn a_failed_take_can_be_discarded_from_the_settings_dialog() {
let mut d = Driver::demo();
d.exec("audio.voiceover.start", json!({"track":"A1","time":0,"preroll":0}));
let blocker = std::env::temp_dir().join(format!("filmcraft-voiceover-ui-blocker-{}", std::process::id()));
std::fs::write(&blocker, b"not a directory").unwrap();
let failure = d.call("engine.execute", json!({"command":"audio.voiceover.stop","params":{"seconds":1,"dir":blocker}}));
assert_eq!(failure["ok"], false);
assert!(d.harness.state().session.voiceover.recording());
d.ok("ui.menu.invoke", json!({"id":"voiceover.settingsDialog"}));
d.frames(4); // Allow the newly anchored window to settle before using its button rectangle.
d.click("voiceover.discard");
assert!(!d.harness.state().session.voiceover.recording(), "{}", d.harness.state().ui.status);
d.exec("file.newProject", json!({}));
std::fs::remove_file(blocker).unwrap();
}
#[test]
fn settings_dialog_from_the_track_header_persists_on_ok() {
let mut d = Driver::demo();