diff --git a/Cargo.lock b/Cargo.lock index 3588e0e0..e555131b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,6 +1817,7 @@ dependencies = [ "filmcraft-vp9", "image", "log", + "serde_json", "symphonia", "thiserror 2.0.21", "web-time", diff --git a/apps/filmcraft-web/src/webcodecs.rs b/apps/filmcraft-web/src/webcodecs.rs index 612a782b..99b3df3b 100644 --- a/apps/filmcraft-web/src/webcodecs.rs +++ b/apps/filmcraft-web/src/webcodecs.rs @@ -12,8 +12,7 @@ //! the job on a later frame, when the decoder's output callback has delivered the picture. //! - A session that has not delivered the frame it was started for is not restarted by other //! requests (thumbnails and the monitor would otherwise keep resetting each other). -//! - Outputs are converted to RGBA by drawing the `VideoFrame` on an `OffscreenCanvas` (the -//! browser applies the stream's colour description). +//! - Native YUV planes feed the desktop frame/compositor contract. Unsupported browser formats or geometry use Canvas RGBA. //! - Any decoder error or unsupported configuration switches the source to our decoder. //! //! `?nowebcodecs` disables it. Support is probed at start-up with `VideoDecoder.isConfigSupported`. @@ -34,6 +33,20 @@ use wasm_bindgen::JsCast; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; +mod pixels; + +#[derive(Default)] +struct Transfers { + native: u64, + canvas: u64, + closed: u64, + stale: u64, + bytes: u64, + native_ms: f64, + canvas_ms: f64, + reasons: BTreeMap<&'static str, u64>, +} + /// Samples fed past the wanted one (decoders hold pictures for reordering). const LOOKAHEAD: usize = 8; /// Decoded frames kept per source. @@ -52,6 +65,7 @@ thread_local! { static DECODED: Cell = const { Cell::new(0) }; static FALLBACKS: Cell = const { Cell::new(0) }; static SOURCES: Cell = const { Cell::new(0) }; + static TRANSFERS: RefCell = RefCell::new(Transfers::default()); } fn now_ms() -> f64 { @@ -160,11 +174,15 @@ pub fn stats() -> serde_json::Value { .iter() .map(|(id, s)| { json!({"id": id, "start": s.start, "next": s.next, "target": s.target, "needsKey": s.needs_key, "flushing": s.flushing.get(), - "errored": s.errored.get(), "queue": s.queue(), "idleMs": now_ms() - s.last_output.get()}) + "errored": s.errored.get(), "queue": s.queue(), "copies": s.copying.get(), "idleMs": now_ms() - s.last_output.get()}) }) .collect() }); - json!({"sources": SOURCES.with(Cell::get), "framesDecoded": DECODED.with(Cell::get), "fallbacks": FALLBACKS.with(Cell::get), "sessions": sessions}) + let transfers = TRANSFERS.with(|s| { + let s = s.borrow(); + json!({"nativeFrames":s.native,"canvasFrames":s.canvas,"closedFrames":s.closed,"staleFrames":s.stale,"nativeBytes":s.bytes,"nativeMs":s.native_ms,"canvasMs":s.canvas_ms,"canvasReasons":s.reasons}) + }); + json!({"sources": SOURCES.with(Cell::get), "framesDecoded": DECODED.with(Cell::get), "fallbacks": FALLBACKS.with(Cell::get), "sessions": sessions,"transfers":transfers}) } /// A [`SharedReader`] as the demuxer's byte source. @@ -210,6 +228,8 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &SharedReader) -> Option Option>, @@ -229,6 +250,8 @@ struct Config { height: u32, /// Clockwise quarter turns from the track matrix, applied to decoded frames. rotation: u8, + color: filmcraft_color::ColorInfo, + par: (u32, u32), } #[derive(Default)] @@ -253,6 +276,9 @@ struct Session { /// Highest presentation time output since the session (re)started. out_max: Rc>, out: Rc>>, + copying: Rc>, + generation: Rc>, + alive: Rc>, _closures: (Closure, Closure), } @@ -286,17 +312,65 @@ impl Session { let errored = Rc::new(Cell::new(false)); let last_output = Rc::new(Cell::new(now_ms())); let out_max = Rc::new(Cell::new(i64::MIN)); + let copying = Rc::new(Cell::new(0u32)); + let generation = Rc::new(Cell::new(0u64)); + let alive = Rc::new(Cell::new(true)); let (o, lo, om) = (out.clone(), last_output.clone(), out_max.clone()); + let (copies, epoch, active, config, errors) = (copying.clone(), generation.clone(), alive.clone(), cfg.clone(), errored.clone()); let on_output = Closure::::new(move |frame: JsValue| { let ts = get(&frame, "timestamp").as_f64().unwrap_or(0.0) as i64; - om.set(om.get().max(ts)); - if let Some(f) = to_rgba(&frame) { - o.borrow_mut().push((ts, f)); - DECODED.with(|c| c.set(c.get() + 1)); - } - let _ = call(&frame, "close", &[]); - lo.set(now_ms()); - crate::repaint(); + let expected = epoch.get(); + let (o, lo, om, copies, epoch, active, config, errors) = + (o.clone(), lo.clone(), om.clone(), copies.clone(), epoch.clone(), active.clone(), config.clone(), errors.clone()); + copies.set(copies.get().saturating_add(1)); + wasm_bindgen_futures::spawn_local(async move { + let begin = now_ms(); + let converted = match pixels::read(&frame, &config).await { + Ok((f, bytes)) => { + TRANSFERS.with(|s| { + let mut s = s.borrow_mut(); + s.native = s.native.saturating_add(1); + s.bytes = s.bytes.saturating_add(bytes as u64); + s.native_ms += now_ms() - begin; + }); + Some(f) + } + Err(reason) => { + let begin = now_ms(); + let f = to_rgba(&frame); + TRANSFERS.with(|s| { + let mut s = s.borrow_mut(); + s.canvas = s.canvas.saturating_add(1); + s.canvas_ms += now_ms() - begin; + let count = s.reasons.entry(reason).or_default(); + *count = count.saturating_add(1); + }); + f + } + }; + let _ = call(&frame, "close", &[]); + copies.set(copies.get().saturating_sub(1)); + TRANSFERS.with(|s| { + let mut s = s.borrow_mut(); + s.closed = s.closed.saturating_add(1); + }); + if !active.get() || epoch.get() != expected { + TRANSFERS.with(|s| { + let mut s = s.borrow_mut(); + s.stale = s.stale.saturating_add(1); + }); + return; + } + if let Some(f) = converted { + o.borrow_mut().push((ts, f)); + om.set(om.get().max(ts)); + DECODED.with(|c| c.set(c.get().saturating_add(1))); + } else { + errors.set(true); + } + lo.set(now_ms()); + crate::repaint(); + }); }); let e = errored.clone(); let on_error = Closure::::new(move |err: JsValue| { @@ -317,6 +391,9 @@ impl Session { last_output, out_max, out, + copying, + generation, + alive, _closures: (on_output, on_error), }; s.configure(cfg)?; @@ -341,6 +418,14 @@ impl Session { } } +impl Drop for Session { + fn drop(&mut self) { + self.alive.set(false); + self.generation.set(self.generation.get().wrapping_add(1)); + let _ = call(&self.decoder, "close", &[]); + } +} + /// An MP4/MOV source whose video is decoded by the browser. pub struct WcSource { inner: SharedSource, @@ -388,12 +473,15 @@ impl WcSource { // already output but the cache dropped needs a restart). let covers = !s.needs_key && s.start <= i && key <= s.next && (i >= s.next || want > s.out_max.get()); if !covers { - // Don't interrupt a session that is still working towards its own frame. - let busy = !s.needs_key && s.target != i64::MIN && s.out_max.get() < s.target && now_ms() - s.last_output.get() < STALE_MS; + // Don't interrupt a session that is still working towards its own frame, unless it + // has gone quiet for STALE_MS (a copyTo that never settles must not wedge it). + let fresh = now_ms() - s.last_output.get() < STALE_MS; + let busy = fresh && (s.copying.get() > 0 || (!s.needs_key && s.target != i64::MIN && s.out_max.get() < s.target)); if busy { return Drive::Waiting; } if !s.needs_key || s.next > 0 { + s.generation.set(s.generation.get().wrapping_add(1)); let _ = call(&s.decoder, "reset", &[]); if s.configure(&self.config).is_err() { return Drive::Failed; @@ -410,7 +498,7 @@ impl WcSource { s.target = want; } let upto = (i + LOOKAHEAD).min(n - 1); - while s.next <= upto && s.queue() < MAX_QUEUE { + while s.next <= upto && s.queue().saturating_add(s.copying.get()) < MAX_QUEUE { let smp = &t.samples[s.next]; let mut data = vec![0u8; smp.size as usize]; match self.reader.read_at(smp.offset, &mut data) { @@ -480,10 +568,8 @@ impl WcSource { impl Drop for WcSource { fn drop(&mut self) { SESSIONS.with(|ss| { - if let Ok(mut ss) = ss.try_borrow_mut() - && let Some(s) = ss.remove(&self.id) - { - let _ = call(&s.decoder, "close", &[]); + if let Ok(mut ss) = ss.try_borrow_mut() { + ss.remove(&self.id); } }); } diff --git a/apps/filmcraft-web/src/webcodecs/pixels.rs b/apps/filmcraft-web/src/webcodecs/pixels.rs new file mode 100644 index 00000000..dc99e771 --- /dev/null +++ b/apps/filmcraft-web/src/webcodecs/pixels.rs @@ -0,0 +1,148 @@ +//! Browser planes -> core frames; no color conversion or chroma reconstruction here. + +use std::sync::Arc; + +use filmcraft_color::{ColorInfo, Matrix, Primaries, Range, Transfer}; +use filmcraft_frame::{Chroma, PixelData, VideoFrame}; +use filmcraft_time::Tick; +use wasm_bindgen::{JsCast, JsValue}; +use wasm_bindgen_futures::JsFuture; + +use super::{Config, call, get, obj}; + +// One 8K 4:4:4 16-bit picture + alpha fits within 256 MiB; cap browser-driven allocations. +const MAX_COPY_BYTES: u32 = 256 * 1024 * 1024; + +fn integer(value: JsValue) -> Option { + let n = value.as_f64()?; + if n.is_finite() && n >= 0.0 && n <= u32::MAX as f64 && n.fract() == 0.0 { Some(n as u32) } else { None } +} + +fn color(frame: &JsValue, fallback: ColorInfo) -> Option { + let c = get(frame, "colorSpace"); + let matrix = match get(&c, "matrix").as_string().as_deref() { + Some("bt709") => Matrix::Bt709, + Some("bt470bg" | "smpte170m") => Matrix::Bt601, + Some("bt2020-ncl") => Matrix::Bt2020Ncl, + None => fallback.matrix, + _ => return None, + }; + let transfer = match get(&c, "transfer").as_string().as_deref() { + Some("bt709") => Transfer::Bt709, + Some("iec61966-2-1") => Transfer::Srgb, + Some("smpte2084") => Transfer::Pq, + Some("arib-std-b67") => Transfer::Hlg, + None => fallback.transfer, + _ => return None, + }; + let primaries = match get(&c, "primaries").as_string().as_deref() { + Some("bt709") => Primaries::Bt709, + Some("bt470bg") => Primaries::Bt601_625, + Some("smpte170m") => Primaries::Bt601_525, + Some("bt2020") => Primaries::Bt2020, + None => fallback.primaries, + _ => return None, + }; + let range = match get(&c, "fullRange").as_bool() { + Some(true) => Range::Full, + Some(false) => Range::Limited, + None => fallback.range, + }; + Some(ColorInfo { matrix, transfer, primaries, range }) +} + +fn plane(raw: &js_sys::Uint8Array, layout: &JsValue, width: u32, height: u32) -> Result, &'static str> { + let offset = integer(get(layout, "offset")).ok_or("layout")?; + let stride = integer(get(layout, "stride")).filter(|s| *s >= width).ok_or("layout")?; + let end = + height.checked_sub(1).and_then(|h| h.checked_mul(stride)).and_then(|n| offset.checked_add(n)).and_then(|n| n.checked_add(width)).ok_or("layout")?; + if end > raw.length() { + return Err("layout"); + } + let len = width.checked_mul(height).filter(|n| *n <= MAX_COPY_BYTES).ok_or("allocation")? as usize; + let mut out = Vec::new(); + out.try_reserve_exact(len).map_err(|_| "memory")?; + out.resize(len, 0); + if stride == width { + raw.subarray(offset, end).copy_to(&mut out); + } else { + for (y, row) in out.chunks_exact_mut(width as usize).enumerate() { + // Checked for the last row above; earlier rows are smaller, but stay checked anyway. + let start = u32::try_from(y).ok().and_then(|y| y.checked_mul(stride)).and_then(|n| offset.checked_add(n)).ok_or("layout")?; + let row_end = start.checked_add(width).filter(|e| *e <= end).ok_or("layout")?; + raw.subarray(start, row_end).copy_to(row); + } + } + Ok(out) +} + +/// Copy native 8-bit YUV planes; unsupported formats/geometry return a counted Canvas fallback reason. +pub(super) async fn read(frame: &JsValue, cfg: &Config) -> Result<(VideoFrame, u32), &'static str> { + let format = get(frame, "format").as_string().ok_or("format")?; + let (chroma, alpha, interleaved) = match format.as_str() { + "NV12" => (Chroma::C420, false, true), + "I420" => (Chroma::C420, false, false), + "I420A" => (Chroma::C420, true, false), + "I422" => (Chroma::C422, false, false), + "I422A" => (Chroma::C422, true, false), + "I444" => (Chroma::C444, false, false), + "I444A" => (Chroma::C444, true, false), + _ => return Err("format"), + }; + let rect = get(frame, "visibleRect"); + let w = integer(get(&rect, "width")).filter(|n| *n > 0).ok_or("geometry")?; + let h = integer(get(&rect, "height")).filter(|n| *n > 0).ok_or("geometry")?; + if integer(get(frame, "displayWidth")) != Some(w) + || integer(get(frame, "displayHeight")) != Some(h) + || get(frame, "rotation").as_f64().is_some_and(|r| r != 0.0) + || get(frame, "flip").as_bool() == Some(true) + { + return Err("geometry"); + } + let color = color(frame, cfg.color).ok_or("color")?; + let options = obj(&[("rect", rect)]); + let bytes = + integer(call(frame, "allocationSize", &[&options]).map_err(|_| "allocation")?).filter(|n| *n > 0 && *n <= MAX_COPY_BYTES).ok_or("allocation")?; + let array: js_sys::Function = get(&js_sys::global(), "Uint8Array").dyn_into().map_err(|_| "allocation")?; + let raw: js_sys::Uint8Array = + js_sys::Reflect::construct(&array, &js_sys::Array::of1(&bytes.into())).and_then(|v| v.dyn_into()).map_err(|_| "allocation")?; + let promise = call(frame, "copyTo", &[raw.as_ref(), &options]).map_err(|_| "copy")?.dyn_into::().map_err(|_| "copy")?; + let layout = js_sys::Array::from(&JsFuture::from(promise).await.map_err(|_| "copy")?); + let count = if interleaved { + 2 + } else if alpha { + 4 + } else { + 3 + }; + if layout.length() != count { + return Err("layout"); + } + let (sx, sy) = chroma.shifts(); + let (cw, ch) = (w.div_ceil(1 << sx), h.div_ceil(1 << sy)); + let y = plane(&raw, &layout.get(0), w, h)?; + let (u, v) = if interleaved { + let uv = plane(&raw, &layout.get(1), cw.checked_mul(2).ok_or("allocation")?, ch)?; + let pairs = uv.as_chunks::<2>().0; + let (mut u, mut v) = (Vec::new(), Vec::new()); + u.try_reserve_exact(pairs.len()).map_err(|_| "memory")?; + v.try_reserve_exact(pairs.len()).map_err(|_| "memory")?; + u.extend(pairs.iter().map(|p| p[0])); + v.extend(pairs.iter().map(|p| p[1])); + (u, v) + } else { + (plane(&raw, &layout.get(1), cw, ch)?, plane(&raw, &layout.get(2), cw, ch)?) + }; + let alpha = if alpha { Some(Arc::new(plane(&raw, &layout.get(3), w, h)?)) } else { None }; + Ok(( + VideoFrame { + width: w, + height: h, + data: PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma, alpha }, + color, + par: cfg.par, + pts: Tick::ZERO, + }, + bytes, + )) +} diff --git a/contributors/people.toml b/contributors/people.toml index e54b15f3..25cf020e 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -22,6 +22,10 @@ real_name = "Brandon Thomas" display_name = "Brandon Thomas" sync_github_name = false +[people.sanyamsingh001] +display_name = "sanyamsingh001" +sync_github_name = false + [people.switch87] real_name = "Gert Pellin" display_name = "Gert Pellin" diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml index 8765f3eb..3a490e2c 100644 --- a/crates/codecs/Cargo.toml +++ b/crates/codecs/Cargo.toml @@ -41,3 +41,4 @@ symphonia = { version = "0.5.5", default-features = false, features = ["aac", "m [dev-dependencies] filmcraft-testkit = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/codecs/tests/mpeg_oracle.rs b/crates/codecs/tests/mpeg_oracle.rs index 5e302493..c2685d23 100644 --- a/crates/codecs/tests/mpeg_oracle.rs +++ b/crates/codecs/tests/mpeg_oracle.rs @@ -177,16 +177,64 @@ fn open(p: &Path) -> MpegSource { MpegSource::open(p.file_name().unwrap().to_str().unwrap(), bytes(p)).unwrap() } -/// ffprobe's video frames in display order: presentation timestamps (90 kHz), including -/// inferred timestamps where an MPEG program stream omits a picture's PTS. +/// ffprobe's video frames in display order: PTS (90 kHz). A program-stream picture +/// need not carry its own PTS; use the decoder's presentation timestamp for that picture. fn ffprobe_frame_pts(file: &Path) -> Option> { - let fp = filmcraft_testkit::ffprobe()?; + let fp = filmcraft_testkit::ffprobe_or_skip("MPEG video frame presentation timestamps")?; let o = std::process::Command::new(fp) - .args(["-v", "error", "-select_streams", "v:0", "-show_entries", "frame=best_effort_timestamp", "-of", "csv=p=0"]) + .args(["-v", "error", "-select_streams", "v:0", "-show_entries", "frame=pts,best_effort_timestamp", "-of", "json"]) .arg(file) .output() - .ok()?; - Some(String::from_utf8_lossy(&o.stdout).lines().filter_map(|l| l.trim().trim_end_matches(',').parse().ok()).collect()) + .unwrap_or_else(|error| panic!("{}: cannot run ffprobe: {error}", file.display())); + assert!(o.status.success(), "{}: ffprobe failed: {}", file.display(), String::from_utf8_lossy(&o.stderr)); + Some(parse_ffprobe_frame_pts(&o.stdout).unwrap_or_else(|error| panic!("{}: {error}", file.display()))) +} + +fn parse_ffprobe_frame_pts(bytes: &[u8]) -> Result, String> { + let data: serde_json::Value = serde_json::from_slice(bytes).map_err(|error| format!("invalid ffprobe JSON: {error}"))?; + let frames = data.get("frames").and_then(serde_json::Value::as_array).ok_or("ffprobe output is missing its frames array")?; + if frames.is_empty() { + return Err("ffprobe returned no video frames".into()); + } + frames + .iter() + .enumerate() + .map(|(index, frame)| { + let field = if frame.get("pts").is_some() { "pts" } else { "best_effort_timestamp" }; + frame.get(field).and_then(serde_json::Value::as_i64).ok_or_else(|| format!("ffprobe video frame {index} is missing an integer {field}")) + }) + .collect() +} + +#[test] +fn ffprobe_pts_preserve_missing_raw_timestamps_and_side_data() { + let output = br#"{"frames":[ + {"pts":131400,"best_effort_timestamp":131401,"side_data_list":[{},{}]}, + {"best_effort_timestamp":135000,"side_data_list":[{}]}, + {"pts":138600,"best_effort_timestamp":138600} + ]}"#; + assert_eq!(parse_ffprobe_frame_pts(output).unwrap(), [131400, 135000, 138600], "keep every display frame, with raw PTS preferred when present"); + assert_eq!(parse_ffprobe_frame_pts(br#"{"frames":[{"pts":-3600},{"pts":0},{"pts":9223372036854775807}]}"#).unwrap(), [-3600, 0, i64::MAX]); +} + +#[test] +fn malformed_ffprobe_pts_fail_instead_of_dropping_frames() { + for output in [ + "not JSON", + "{}", + r#"{"frames":{}}"#, + r#"{"frames":[]}"#, + r#"{"frames":[{}]}"#, + r#"{"frames":[null]}"#, + r#"{"frames":[{"pts":"N/A","best_effort_timestamp":135000}]}"#, + r#"{"frames":[{"pts":null,"best_effort_timestamp":135000}]}"#, + r#"{"frames":[{"pts":1.5}]}"#, + r#"{"frames":[{"pts":9223372036854775808}]}"#, + r#"{"frames":[{"best_effort_timestamp":"135000"}]}"#, + r#"{"frames":[{"pts":131400},{"best_effort_timestamp":null},{"pts":138600}]}"#, + ] { + assert!(parse_ffprobe_frame_pts(output.as_bytes()).is_err(), "malformed oracle output must fail: {output}"); + } } /// Decode every frame in order and `seeks` random frames; compare with ffmpeg within `tol`. diff --git a/crates/frame/src/pool.rs b/crates/frame/src/pool.rs index d5c0ba99..b9952398 100644 --- a/crates/frame/src/pool.rs +++ b/crates/frame/src/pool.rs @@ -241,21 +241,37 @@ mod tests { pts: filmcraft_time::Tick::ZERO, }) }; - let before = stats().reused; - // a frame someone else still holds is left alone + let plane_ptrs = |frame: &VideoFrame| match &frame.data { + PixelData::Yuv8 { planes, .. } => planes.each_ref().map(|plane| plane.as_ptr() as usize), + _ => panic!("fixture has three byte planes"), + }; + // A frame someone else still holds is left alone. The reuse statistic is global: + // concurrent float-buffer tests can change it without touching these planes. let shared = yuv(1); let held = shared.clone(); + let held_ptrs = plane_ptrs(&held); recycle(shared); - assert!(take_u8(n).capacity() >= n); - assert_eq!(stats().reused, before); + let fresh = take_u8(n); + assert!(fresh.capacity() >= n); + assert!(!held_ptrs.contains(&(fresh.as_ptr() as usize)), "a shared plane cannot be handed out"); assert_eq!(held.byte_size(), n + 2 * c); - // an unshared frame's planes come back, empty - recycle(yuv(2)); + // An unshared frame's exact allocations come back, empty. Equal-size chroma + // planes may be taken in either order. + let unshared = yuv(2); + let returned_ptrs = plane_ptrs(&unshared); + recycle(unshared); let (y, u, v) = (take_u8(n), take_u8(c), take_u8(c)); - assert!(y.is_empty() && y.capacity() >= n && u.capacity() >= c && v.capacity() >= c); - assert_eq!(stats().reused, before + 3); - // and are not handed out twice - take_u8(n); - assert_eq!(stats().reused, before + 3); + assert!(y.is_empty() && u.is_empty() && v.is_empty()); + assert!(y.capacity() >= n && u.capacity() >= c && v.capacity() >= c); + assert_eq!(y.as_ptr() as usize, returned_ptrs[0]); + let mut expected_chroma = [returned_ptrs[1], returned_ptrs[2]]; + let mut actual_chroma = [u.as_ptr() as usize, v.as_ptr() as usize]; + expected_chroma.sort_unstable(); + actual_chroma.sort_unstable(); + assert_eq!(actual_chroma, expected_chroma); + // The returned allocations remain owned by y/u/v and cannot be handed out twice. + let another = take_u8(n); + assert!(another.capacity() >= n); + assert!(!returned_ptrs.contains(&(another.as_ptr() as usize))); } }