Merge origin/main into i18n-spanish-panels

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

# Conflicts:
#	crates/codecs/tests/mpeg_oracle.rs
This commit is contained in:
Brandon Thomas 2026-10-09 09:59:28 -04:00
commit 156535a6e4
7 changed files with 341 additions and 37 deletions

1
Cargo.lock generated
View File

@ -1817,6 +1817,7 @@ dependencies = [
"filmcraft-vp9",
"image",
"log",
"serde_json",
"symphonia",
"thiserror 2.0.21",
"web-time",

View File

@ -12,8 +12,7 @@
//! the job on a later frame, when the decoder's output callback has delivered the picture.
//! - A session that has not delivered the frame it was started for is not restarted by other
//! requests (thumbnails and the monitor would otherwise keep resetting each other).
//! - Outputs are converted to RGBA by drawing the `VideoFrame` on an `OffscreenCanvas` (the
//! browser applies the stream's colour description).
//! - Native YUV planes feed the desktop frame/compositor contract. Unsupported browser formats or geometry use Canvas RGBA.
//! - Any decoder error or unsupported configuration switches the source to our decoder.
//!
//! `?nowebcodecs` disables it. Support is probed at start-up with `VideoDecoder.isConfigSupported`.
@ -34,6 +33,20 @@ use wasm_bindgen::JsCast;
use wasm_bindgen::prelude::*;
use wasm_bindgen_futures::JsFuture;
mod pixels;
#[derive(Default)]
struct Transfers {
native: u64,
canvas: u64,
closed: u64,
stale: u64,
bytes: u64,
native_ms: f64,
canvas_ms: f64,
reasons: BTreeMap<&'static str, u64>,
}
/// Samples fed past the wanted one (decoders hold pictures for reordering).
const LOOKAHEAD: usize = 8;
/// Decoded frames kept per source.
@ -52,6 +65,7 @@ thread_local! {
static DECODED: Cell<u64> = const { Cell::new(0) };
static FALLBACKS: Cell<u64> = const { Cell::new(0) };
static SOURCES: Cell<u64> = const { Cell::new(0) };
static TRANSFERS: RefCell<Transfers> = RefCell::new(Transfers::default());
}
fn now_ms() -> f64 {
@ -160,11 +174,15 @@ pub fn stats() -> serde_json::Value {
.iter()
.map(|(id, s)| {
json!({"id": id, "start": s.start, "next": s.next, "target": s.target, "needsKey": s.needs_key, "flushing": s.flushing.get(),
"errored": s.errored.get(), "queue": s.queue(), "idleMs": now_ms() - s.last_output.get()})
"errored": s.errored.get(), "queue": s.queue(), "copies": s.copying.get(), "idleMs": now_ms() - s.last_output.get()})
})
.collect()
});
json!({"sources": SOURCES.with(Cell::get), "framesDecoded": DECODED.with(Cell::get), "fallbacks": FALLBACKS.with(Cell::get), "sessions": sessions})
let transfers = TRANSFERS.with(|s| {
let s = s.borrow();
json!({"nativeFrames":s.native,"canvasFrames":s.canvas,"closedFrames":s.closed,"staleFrames":s.stale,"nativeBytes":s.bytes,"nativeMs":s.native_ms,"canvasMs":s.canvas_ms,"canvasReasons":s.reasons})
});
json!({"sources": SOURCES.with(Cell::get), "framesDecoded": DECODED.with(Cell::get), "fallbacks": FALLBACKS.with(Cell::get), "sessions": sessions,"transfers":transfers})
}
/// A [`SharedReader`] as the demuxer's byte source.
@ -210,6 +228,8 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &SharedReader) -> Option<R
});
SOURCES.with(|c| c.set(c.get() + 1));
log::info!("{name}: video decoded with WebCodecs ({codec})");
let color = info.video.as_ref().map(|v| v.color).unwrap_or_default();
let par = info.video.as_ref().map(|v| v.par).unwrap_or((1, 1));
Some(Ok(Arc::new(WcSource {
inner,
info,
@ -217,11 +237,12 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &SharedReader) -> Option<R
file,
track,
id,
config: Config { codec, description, width: w, height: h, rotation },
config: Config { codec, description, width: w, height: h, rotation, color, par },
state: Mutex::new(Cache::default()),
})))
}
#[derive(Clone)]
struct Config {
codec: String,
description: Option<Vec<u8>>,
@ -229,6 +250,8 @@ struct Config {
height: u32,
/// Clockwise quarter turns from the track matrix, applied to decoded frames.
rotation: u8,
color: filmcraft_color::ColorInfo,
par: (u32, u32),
}
#[derive(Default)]
@ -253,6 +276,9 @@ struct Session {
/// Highest presentation time output since the session (re)started.
out_max: Rc<Cell<i64>>,
out: Rc<RefCell<Vec<(i64, VideoFrame)>>>,
copying: Rc<Cell<u32>>,
generation: Rc<Cell<u64>>,
alive: Rc<Cell<bool>>,
_closures: (Closure<dyn FnMut(JsValue)>, Closure<dyn FnMut(JsValue)>),
}
@ -286,17 +312,65 @@ impl Session {
let errored = Rc::new(Cell::new(false));
let last_output = Rc::new(Cell::new(now_ms()));
let out_max = Rc::new(Cell::new(i64::MIN));
let copying = Rc::new(Cell::new(0u32));
let generation = Rc::new(Cell::new(0u64));
let alive = Rc::new(Cell::new(true));
let (o, lo, om) = (out.clone(), last_output.clone(), out_max.clone());
let (copies, epoch, active, config, errors) = (copying.clone(), generation.clone(), alive.clone(), cfg.clone(), errored.clone());
let on_output = Closure::<dyn FnMut(JsValue)>::new(move |frame: JsValue| {
let ts = get(&frame, "timestamp").as_f64().unwrap_or(0.0) as i64;
om.set(om.get().max(ts));
if let Some(f) = to_rgba(&frame) {
o.borrow_mut().push((ts, f));
DECODED.with(|c| c.set(c.get() + 1));
}
let _ = call(&frame, "close", &[]);
lo.set(now_ms());
crate::repaint();
let expected = epoch.get();
let (o, lo, om, copies, epoch, active, config, errors) =
(o.clone(), lo.clone(), om.clone(), copies.clone(), epoch.clone(), active.clone(), config.clone(), errors.clone());
copies.set(copies.get().saturating_add(1));
wasm_bindgen_futures::spawn_local(async move {
let begin = now_ms();
let converted = match pixels::read(&frame, &config).await {
Ok((f, bytes)) => {
TRANSFERS.with(|s| {
let mut s = s.borrow_mut();
s.native = s.native.saturating_add(1);
s.bytes = s.bytes.saturating_add(bytes as u64);
s.native_ms += now_ms() - begin;
});
Some(f)
}
Err(reason) => {
let begin = now_ms();
let f = to_rgba(&frame);
TRANSFERS.with(|s| {
let mut s = s.borrow_mut();
s.canvas = s.canvas.saturating_add(1);
s.canvas_ms += now_ms() - begin;
let count = s.reasons.entry(reason).or_default();
*count = count.saturating_add(1);
});
f
}
};
let _ = call(&frame, "close", &[]);
copies.set(copies.get().saturating_sub(1));
TRANSFERS.with(|s| {
let mut s = s.borrow_mut();
s.closed = s.closed.saturating_add(1);
});
if !active.get() || epoch.get() != expected {
TRANSFERS.with(|s| {
let mut s = s.borrow_mut();
s.stale = s.stale.saturating_add(1);
});
return;
}
if let Some(f) = converted {
o.borrow_mut().push((ts, f));
om.set(om.get().max(ts));
DECODED.with(|c| c.set(c.get().saturating_add(1)));
} else {
errors.set(true);
}
lo.set(now_ms());
crate::repaint();
});
});
let e = errored.clone();
let on_error = Closure::<dyn FnMut(JsValue)>::new(move |err: JsValue| {
@ -317,6 +391,9 @@ impl Session {
last_output,
out_max,
out,
copying,
generation,
alive,
_closures: (on_output, on_error),
};
s.configure(cfg)?;
@ -341,6 +418,14 @@ impl Session {
}
}
impl Drop for Session {
fn drop(&mut self) {
self.alive.set(false);
self.generation.set(self.generation.get().wrapping_add(1));
let _ = call(&self.decoder, "close", &[]);
}
}
/// An MP4/MOV source whose video is decoded by the browser.
pub struct WcSource {
inner: SharedSource,
@ -388,12 +473,15 @@ impl WcSource {
// already output but the cache dropped needs a restart).
let covers = !s.needs_key && s.start <= i && key <= s.next && (i >= s.next || want > s.out_max.get());
if !covers {
// Don't interrupt a session that is still working towards its own frame.
let busy = !s.needs_key && s.target != i64::MIN && s.out_max.get() < s.target && now_ms() - s.last_output.get() < STALE_MS;
// Don't interrupt a session that is still working towards its own frame, unless it
// has gone quiet for STALE_MS (a copyTo that never settles must not wedge it).
let fresh = now_ms() - s.last_output.get() < STALE_MS;
let busy = fresh && (s.copying.get() > 0 || (!s.needs_key && s.target != i64::MIN && s.out_max.get() < s.target));
if busy {
return Drive::Waiting;
}
if !s.needs_key || s.next > 0 {
s.generation.set(s.generation.get().wrapping_add(1));
let _ = call(&s.decoder, "reset", &[]);
if s.configure(&self.config).is_err() {
return Drive::Failed;
@ -410,7 +498,7 @@ impl WcSource {
s.target = want;
}
let upto = (i + LOOKAHEAD).min(n - 1);
while s.next <= upto && s.queue() < MAX_QUEUE {
while s.next <= upto && s.queue().saturating_add(s.copying.get()) < MAX_QUEUE {
let smp = &t.samples[s.next];
let mut data = vec![0u8; smp.size as usize];
match self.reader.read_at(smp.offset, &mut data) {
@ -480,10 +568,8 @@ impl WcSource {
impl Drop for WcSource {
fn drop(&mut self) {
SESSIONS.with(|ss| {
if let Ok(mut ss) = ss.try_borrow_mut()
&& let Some(s) = ss.remove(&self.id)
{
let _ = call(&s.decoder, "close", &[]);
if let Ok(mut ss) = ss.try_borrow_mut() {
ss.remove(&self.id);
}
});
}

View File

@ -0,0 +1,148 @@
//! Browser planes -> core frames; no color conversion or chroma reconstruction here.
use std::sync::Arc;
use filmcraft_color::{ColorInfo, Matrix, Primaries, Range, Transfer};
use filmcraft_frame::{Chroma, PixelData, VideoFrame};
use filmcraft_time::Tick;
use wasm_bindgen::{JsCast, JsValue};
use wasm_bindgen_futures::JsFuture;
use super::{Config, call, get, obj};
// One 8K 4:4:4 16-bit picture + alpha fits within 256 MiB; cap browser-driven allocations.
const MAX_COPY_BYTES: u32 = 256 * 1024 * 1024;
fn integer(value: JsValue) -> Option<u32> {
let n = value.as_f64()?;
if n.is_finite() && n >= 0.0 && n <= u32::MAX as f64 && n.fract() == 0.0 { Some(n as u32) } else { None }
}
fn color(frame: &JsValue, fallback: ColorInfo) -> Option<ColorInfo> {
let c = get(frame, "colorSpace");
let matrix = match get(&c, "matrix").as_string().as_deref() {
Some("bt709") => Matrix::Bt709,
Some("bt470bg" | "smpte170m") => Matrix::Bt601,
Some("bt2020-ncl") => Matrix::Bt2020Ncl,
None => fallback.matrix,
_ => return None,
};
let transfer = match get(&c, "transfer").as_string().as_deref() {
Some("bt709") => Transfer::Bt709,
Some("iec61966-2-1") => Transfer::Srgb,
Some("smpte2084") => Transfer::Pq,
Some("arib-std-b67") => Transfer::Hlg,
None => fallback.transfer,
_ => return None,
};
let primaries = match get(&c, "primaries").as_string().as_deref() {
Some("bt709") => Primaries::Bt709,
Some("bt470bg") => Primaries::Bt601_625,
Some("smpte170m") => Primaries::Bt601_525,
Some("bt2020") => Primaries::Bt2020,
None => fallback.primaries,
_ => return None,
};
let range = match get(&c, "fullRange").as_bool() {
Some(true) => Range::Full,
Some(false) => Range::Limited,
None => fallback.range,
};
Some(ColorInfo { matrix, transfer, primaries, range })
}
fn plane(raw: &js_sys::Uint8Array, layout: &JsValue, width: u32, height: u32) -> Result<Vec<u8>, &'static str> {
let offset = integer(get(layout, "offset")).ok_or("layout")?;
let stride = integer(get(layout, "stride")).filter(|s| *s >= width).ok_or("layout")?;
let end =
height.checked_sub(1).and_then(|h| h.checked_mul(stride)).and_then(|n| offset.checked_add(n)).and_then(|n| n.checked_add(width)).ok_or("layout")?;
if end > raw.length() {
return Err("layout");
}
let len = width.checked_mul(height).filter(|n| *n <= MAX_COPY_BYTES).ok_or("allocation")? as usize;
let mut out = Vec::new();
out.try_reserve_exact(len).map_err(|_| "memory")?;
out.resize(len, 0);
if stride == width {
raw.subarray(offset, end).copy_to(&mut out);
} else {
for (y, row) in out.chunks_exact_mut(width as usize).enumerate() {
// Checked for the last row above; earlier rows are smaller, but stay checked anyway.
let start = u32::try_from(y).ok().and_then(|y| y.checked_mul(stride)).and_then(|n| offset.checked_add(n)).ok_or("layout")?;
let row_end = start.checked_add(width).filter(|e| *e <= end).ok_or("layout")?;
raw.subarray(start, row_end).copy_to(row);
}
}
Ok(out)
}
/// Copy native 8-bit YUV planes; unsupported formats/geometry return a counted Canvas fallback reason.
pub(super) async fn read(frame: &JsValue, cfg: &Config) -> Result<(VideoFrame, u32), &'static str> {
let format = get(frame, "format").as_string().ok_or("format")?;
let (chroma, alpha, interleaved) = match format.as_str() {
"NV12" => (Chroma::C420, false, true),
"I420" => (Chroma::C420, false, false),
"I420A" => (Chroma::C420, true, false),
"I422" => (Chroma::C422, false, false),
"I422A" => (Chroma::C422, true, false),
"I444" => (Chroma::C444, false, false),
"I444A" => (Chroma::C444, true, false),
_ => return Err("format"),
};
let rect = get(frame, "visibleRect");
let w = integer(get(&rect, "width")).filter(|n| *n > 0).ok_or("geometry")?;
let h = integer(get(&rect, "height")).filter(|n| *n > 0).ok_or("geometry")?;
if integer(get(frame, "displayWidth")) != Some(w)
|| integer(get(frame, "displayHeight")) != Some(h)
|| get(frame, "rotation").as_f64().is_some_and(|r| r != 0.0)
|| get(frame, "flip").as_bool() == Some(true)
{
return Err("geometry");
}
let color = color(frame, cfg.color).ok_or("color")?;
let options = obj(&[("rect", rect)]);
let bytes =
integer(call(frame, "allocationSize", &[&options]).map_err(|_| "allocation")?).filter(|n| *n > 0 && *n <= MAX_COPY_BYTES).ok_or("allocation")?;
let array: js_sys::Function = get(&js_sys::global(), "Uint8Array").dyn_into().map_err(|_| "allocation")?;
let raw: js_sys::Uint8Array =
js_sys::Reflect::construct(&array, &js_sys::Array::of1(&bytes.into())).and_then(|v| v.dyn_into()).map_err(|_| "allocation")?;
let promise = call(frame, "copyTo", &[raw.as_ref(), &options]).map_err(|_| "copy")?.dyn_into::<js_sys::Promise>().map_err(|_| "copy")?;
let layout = js_sys::Array::from(&JsFuture::from(promise).await.map_err(|_| "copy")?);
let count = if interleaved {
2
} else if alpha {
4
} else {
3
};
if layout.length() != count {
return Err("layout");
}
let (sx, sy) = chroma.shifts();
let (cw, ch) = (w.div_ceil(1 << sx), h.div_ceil(1 << sy));
let y = plane(&raw, &layout.get(0), w, h)?;
let (u, v) = if interleaved {
let uv = plane(&raw, &layout.get(1), cw.checked_mul(2).ok_or("allocation")?, ch)?;
let pairs = uv.as_chunks::<2>().0;
let (mut u, mut v) = (Vec::new(), Vec::new());
u.try_reserve_exact(pairs.len()).map_err(|_| "memory")?;
v.try_reserve_exact(pairs.len()).map_err(|_| "memory")?;
u.extend(pairs.iter().map(|p| p[0]));
v.extend(pairs.iter().map(|p| p[1]));
(u, v)
} else {
(plane(&raw, &layout.get(1), cw, ch)?, plane(&raw, &layout.get(2), cw, ch)?)
};
let alpha = if alpha { Some(Arc::new(plane(&raw, &layout.get(3), w, h)?)) } else { None };
Ok((
VideoFrame {
width: w,
height: h,
data: PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma, alpha },
color,
par: cfg.par,
pts: Tick::ZERO,
},
bytes,
))
}

View File

@ -22,6 +22,10 @@ real_name = "Brandon Thomas"
display_name = "Brandon Thomas"
sync_github_name = false
[people.sanyamsingh001]
display_name = "sanyamsingh001"
sync_github_name = false
[people.switch87]
real_name = "Gert Pellin"
display_name = "Gert Pellin"

View File

@ -41,3 +41,4 @@ symphonia = { version = "0.5.5", default-features = false, features = ["aac", "m
[dev-dependencies]
filmcraft-testkit = { workspace = true }
serde_json = { workspace = true }

View File

@ -177,16 +177,64 @@ fn open(p: &Path) -> MpegSource {
MpegSource::open(p.file_name().unwrap().to_str().unwrap(), bytes(p)).unwrap()
}
/// ffprobe's video frames in display order: presentation timestamps (90 kHz), including
/// inferred timestamps where an MPEG program stream omits a picture's PTS.
/// ffprobe's video frames in display order: PTS (90 kHz). A program-stream picture
/// need not carry its own PTS; use the decoder's presentation timestamp for that picture.
fn ffprobe_frame_pts(file: &Path) -> Option<Vec<i64>> {
let fp = filmcraft_testkit::ffprobe()?;
let fp = filmcraft_testkit::ffprobe_or_skip("MPEG video frame presentation timestamps")?;
let o = std::process::Command::new(fp)
.args(["-v", "error", "-select_streams", "v:0", "-show_entries", "frame=best_effort_timestamp", "-of", "csv=p=0"])
.args(["-v", "error", "-select_streams", "v:0", "-show_entries", "frame=pts,best_effort_timestamp", "-of", "json"])
.arg(file)
.output()
.ok()?;
Some(String::from_utf8_lossy(&o.stdout).lines().filter_map(|l| l.trim().trim_end_matches(',').parse().ok()).collect())
.unwrap_or_else(|error| panic!("{}: cannot run ffprobe: {error}", file.display()));
assert!(o.status.success(), "{}: ffprobe failed: {}", file.display(), String::from_utf8_lossy(&o.stderr));
Some(parse_ffprobe_frame_pts(&o.stdout).unwrap_or_else(|error| panic!("{}: {error}", file.display())))
}
fn parse_ffprobe_frame_pts(bytes: &[u8]) -> Result<Vec<i64>, String> {
let data: serde_json::Value = serde_json::from_slice(bytes).map_err(|error| format!("invalid ffprobe JSON: {error}"))?;
let frames = data.get("frames").and_then(serde_json::Value::as_array).ok_or("ffprobe output is missing its frames array")?;
if frames.is_empty() {
return Err("ffprobe returned no video frames".into());
}
frames
.iter()
.enumerate()
.map(|(index, frame)| {
let field = if frame.get("pts").is_some() { "pts" } else { "best_effort_timestamp" };
frame.get(field).and_then(serde_json::Value::as_i64).ok_or_else(|| format!("ffprobe video frame {index} is missing an integer {field}"))
})
.collect()
}
#[test]
fn ffprobe_pts_preserve_missing_raw_timestamps_and_side_data() {
let output = br#"{"frames":[
{"pts":131400,"best_effort_timestamp":131401,"side_data_list":[{},{}]},
{"best_effort_timestamp":135000,"side_data_list":[{}]},
{"pts":138600,"best_effort_timestamp":138600}
]}"#;
assert_eq!(parse_ffprobe_frame_pts(output).unwrap(), [131400, 135000, 138600], "keep every display frame, with raw PTS preferred when present");
assert_eq!(parse_ffprobe_frame_pts(br#"{"frames":[{"pts":-3600},{"pts":0},{"pts":9223372036854775807}]}"#).unwrap(), [-3600, 0, i64::MAX]);
}
#[test]
fn malformed_ffprobe_pts_fail_instead_of_dropping_frames() {
for output in [
"not JSON",
"{}",
r#"{"frames":{}}"#,
r#"{"frames":[]}"#,
r#"{"frames":[{}]}"#,
r#"{"frames":[null]}"#,
r#"{"frames":[{"pts":"N/A","best_effort_timestamp":135000}]}"#,
r#"{"frames":[{"pts":null,"best_effort_timestamp":135000}]}"#,
r#"{"frames":[{"pts":1.5}]}"#,
r#"{"frames":[{"pts":9223372036854775808}]}"#,
r#"{"frames":[{"best_effort_timestamp":"135000"}]}"#,
r#"{"frames":[{"pts":131400},{"best_effort_timestamp":null},{"pts":138600}]}"#,
] {
assert!(parse_ffprobe_frame_pts(output.as_bytes()).is_err(), "malformed oracle output must fail: {output}");
}
}
/// Decode every frame in order and `seeks` random frames; compare with ffmpeg within `tol`.

View File

@ -241,21 +241,37 @@ mod tests {
pts: filmcraft_time::Tick::ZERO,
})
};
let before = stats().reused;
// a frame someone else still holds is left alone
let plane_ptrs = |frame: &VideoFrame| match &frame.data {
PixelData::Yuv8 { planes, .. } => planes.each_ref().map(|plane| plane.as_ptr() as usize),
_ => panic!("fixture has three byte planes"),
};
// A frame someone else still holds is left alone. The reuse statistic is global:
// concurrent float-buffer tests can change it without touching these planes.
let shared = yuv(1);
let held = shared.clone();
let held_ptrs = plane_ptrs(&held);
recycle(shared);
assert!(take_u8(n).capacity() >= n);
assert_eq!(stats().reused, before);
let fresh = take_u8(n);
assert!(fresh.capacity() >= n);
assert!(!held_ptrs.contains(&(fresh.as_ptr() as usize)), "a shared plane cannot be handed out");
assert_eq!(held.byte_size(), n + 2 * c);
// an unshared frame's planes come back, empty
recycle(yuv(2));
// An unshared frame's exact allocations come back, empty. Equal-size chroma
// planes may be taken in either order.
let unshared = yuv(2);
let returned_ptrs = plane_ptrs(&unshared);
recycle(unshared);
let (y, u, v) = (take_u8(n), take_u8(c), take_u8(c));
assert!(y.is_empty() && y.capacity() >= n && u.capacity() >= c && v.capacity() >= c);
assert_eq!(stats().reused, before + 3);
// and are not handed out twice
take_u8(n);
assert_eq!(stats().reused, before + 3);
assert!(y.is_empty() && u.is_empty() && v.is_empty());
assert!(y.capacity() >= n && u.capacity() >= c && v.capacity() >= c);
assert_eq!(y.as_ptr() as usize, returned_ptrs[0]);
let mut expected_chroma = [returned_ptrs[1], returned_ptrs[2]];
let mut actual_chroma = [u.as_ptr() as usize, v.as_ptr() as usize];
expected_chroma.sort_unstable();
actual_chroma.sort_unstable();
assert_eq!(actual_chroma, expected_chroma);
// The returned allocations remain owned by y/u/v and cannot be handed out twice.
let another = take_u8(n);
assert!(another.capacity() >= n);
assert!(!returned_ptrs.contains(&(another.as_ptr() as usize)));
}
}