diff --git a/Cargo.lock b/Cargo.lock index 866ae15f..9b28b8ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1226,11 +1226,13 @@ dependencies = [ "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "effectcraft-vp9enc", "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", "filmcraft-opus", "filmcraft-prores", + "filmcraft-vp9", "gif", "image", "log", @@ -1485,6 +1487,10 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-vp9enc" +version = "0.1.0" + [[package]] name = "effectcraft-web" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 08a7d19d..dc2ce073 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } effectcraft-psd = { path = "crates/psd" } +effectcraft-vp9enc = { path = "crates/vp9enc" } effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } @@ -71,6 +72,7 @@ filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf2 filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -140,6 +142,9 @@ opt-level = 3 [profile.dev.package.effectcraft-export] opt-level = 3 +[profile.dev.package.effectcraft-vp9enc] +opt-level = 3 + # FilmCraft codec hot loops (same settings as FilmCraft's own workspace). [profile.dev.package.filmcraft-h264] opt-level = 3 diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 90d7ef66..3a04e15c 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -9,7 +9,7 @@ //! effectcraft-cli get [--time S] read a property //! effectcraft-cli set [--time S] [--expression E] //! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png] -//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|png|jpeg|tiff|exr|gif] [--start S] [--end S] +//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index c905ce6c..be6cebfb 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -131,7 +131,7 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { let mut any = false; if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); any = true; } @@ -256,7 +256,7 @@ fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; it.output = OutputModule::for_format(f); } apply_settings(&mut it.settings, p, "renderQueue.add")?; @@ -434,7 +434,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { let n = it.extra_outputs.len() + 2; let mut om = it.output.clone(); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om = OutputModule::for_format(f); } if str_p(p, "output").is_none() && str_p(p, "path").is_none() { @@ -509,7 +509,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", can_add, add ), diff --git a/crates/engine/src/rq_tests.rs b/crates/engine/src/rq_tests.rs index c4aa08b8..f172b5e9 100644 --- a/crates/engine/src/rq_tests.rs +++ b/crates/engine/src/rq_tests.rs @@ -159,6 +159,6 @@ fn without_exporter() { s.execute("renderQueue.add", json!({})).unwrap(); assert!(!s.is_enabled("renderQueue.render")); let f = s.execute("renderQueue.formats", json!({})).unwrap(); - assert_eq!(f["formats"].as_array().unwrap().len(), 7); + assert_eq!(f["formats"].as_array().unwrap().len(), 10); assert_eq!(f["formats"][0]["available"], false); } diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 5bdd0fee..15f9b17d 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-export" -description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, PNG/JPEG/TIFF/EXR sequences, animated GIF" +description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, WebM (VP9 + Opus), PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF" version.workspace = true edition.workspace = true license.workspace = true @@ -19,6 +19,7 @@ filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } effectcraft-opusenc = { workspace = true } +effectcraft-vp9enc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -30,3 +31,4 @@ log = { workspace = true } effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } filmcraft-opus = { workspace = true } +filmcraft-vp9 = { workspace = true } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index dc4c35c8..a2af6a4f 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,6 +7,8 @@ //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | +//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -18,6 +20,7 @@ mod encode; mod out; +mod webm; use web_time::Instant; @@ -131,6 +134,9 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result< let mut report = match job.output.format { OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs new file mode 100644 index 00000000..eb99cd22 --- /dev/null +++ b/crates/export/src/webm.rs @@ -0,0 +1,368 @@ +//! WebM output (VP9 video, optional VP9 alpha, Opus audio) and audio-only WAV / AIFF. +//! +//! The WebM container is written from the Matroska / WebM specifications (IETF RFC 8794 EBML, +//! the Matroska element registry and the WebM container guidelines): EBML header, Segment +//! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are +//! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose +//! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every +//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit +//! extended sample rate) follow their published file layouts. + +use std::io::{Seek, SeekFrom, Write}; + +use effectcraft_project::Comp; +use effectcraft_project::render_queue::Channels; +use effectcraft_time::{TICKS_PER_SECOND, Tick}; +use rayon::prelude::*; + +use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; + +const OPUS_RATE: u32 = 48_000; +const OPUS_BITRATE: u32 = 192_000; + +// Element ids (with their length markers). +const EBML: u32 = 0x1A45_DFA3; +const SEGMENT: u32 = 0x1853_8067; +const INFO: u32 = 0x1549_A966; +const TRACKS: u32 = 0x1654_AE6B; +const TRACK_ENTRY: u32 = 0xAE; +const CLUSTER: u32 = 0x1F43_B675; + +fn id_bytes(id: u32) -> Vec { + let b = id.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(3); + b[skip..].to_vec() +} + +/// EBML variable-size integer for an element data size (shortest form). +fn vint(n: u64) -> Vec { + for len in 1..=8u32 { + let max = (1u64 << (7 * len)) - 2; + if n <= max { + let marked = n | (1u64 << (7 * len)); + return marked.to_be_bytes()[8 - len as usize..].to_vec(); + } + } + let mut v = vec![0x01]; + v.extend_from_slice(&n.to_be_bytes()[1..]); + v +} + +fn el(out: &mut Vec, id: u32, data: &[u8]) { + out.extend(id_bytes(id)); + out.extend(vint(data.len() as u64)); + out.extend_from_slice(data); +} + +fn el_uint(out: &mut Vec, id: u32, v: u64) { + let b = v.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(7); + el(out, id, &b[skip..]); +} + +fn el_float(out: &mut Vec, id: u32, v: f64) { + el(out, id, &v.to_be_bytes()); +} + +fn el_str(out: &mut Vec, id: u32, s: &str) { + el(out, id, s.as_bytes()); +} + +/// A block payload: track number, timestamp relative to the cluster, flags, frame. +fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { + let mut b = vint(track); + b.extend_from_slice(&rel.to_be_bytes()); + b.push(flags); + b.extend_from_slice(data); + b +} + +struct Frame { + ms: i64, + track: u64, + data: Vec, + alpha: Option>, +} + +fn cluster(frames: &mut [Frame]) -> Vec { + frames.sort_by_key(|f| (f.ms, f.track)); + let base = frames.first().map_or(0, |f| f.ms); + let mut body = vec![]; + el_uint(&mut body, 0xE7, base.max(0) as u64); + for f in frames.iter() { + let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; + match &f.alpha { + None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + Some(a) => { + let mut g = vec![]; + el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + let mut more = vec![]; + el_uint(&mut more, 0xEE, 1); + el(&mut more, 0xA5, a); + let mut adds = vec![]; + el(&mut adds, 0xA6, &more); + el(&mut g, 0x75A1, &adds); + el(&mut body, 0xA0, &g); + } + } + } + let mut out = vec![]; + el(&mut out, CLUSTER, &body); + out +} + +pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let rate = job.settings.rate(comp); + let alpha = job.output.channels == Channels::Rgba; + let quality = job.output.quality.clamp(1, 100); + let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let with_audio = wants_audio(job); + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let total = st.total; + let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; + let duration_ms = frame_ms(total) as f64; + + let mut file = crate::out::create(job.sink, job.path)?; + // EBML header. + let mut head = vec![]; + let mut eb = vec![]; + el_uint(&mut eb, 0x4286, 1); + el_uint(&mut eb, 0x42F7, 1); + el_uint(&mut eb, 0x42F2, 4); + el_uint(&mut eb, 0x42F3, 8); + el_str(&mut eb, 0x4282, "webm"); + el_uint(&mut eb, 0x4287, 4); + el_uint(&mut eb, 0x4285, 2); + el(&mut head, EBML, &eb); + // Segment with an 8-byte size patched at the end. + head.extend(id_bytes(SEGMENT)); + let size_pos = head.len() as u64; + head.extend_from_slice(&[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + let seg_start = head.len() as u64; + let mut info = vec![]; + el_uint(&mut info, 0x2AD7B1, 1_000_000); + el_float(&mut info, 0x4489, duration_ms); + el_str(&mut info, 0x4D80, "EffectCraft"); + el_str(&mut info, 0x5741, "EffectCraft"); + el(&mut head, INFO, &info); + let mut tracks = vec![]; + let mut v = vec![]; + el_uint(&mut v, 0xD7, 1); + el_uint(&mut v, 0x73C5, 1); + el_uint(&mut v, 0x83, 1); + el_uint(&mut v, 0x9C, 0); + el_str(&mut v, 0x86, "V_VP9"); + el_uint(&mut v, 0x23E383, (1_000_000_000u128 * rate.den as u128 / rate.num.max(1) as u128) as u64); + if alpha { + el_uint(&mut v, 0x55EE, 1); + } + let mut video = vec![]; + el_uint(&mut video, 0xB0, w as u64); + el_uint(&mut video, 0xBA, h as u64); + if alpha { + el_uint(&mut video, 0x53C0, 1); + } + el(&mut v, 0xE0, &video); + el(&mut tracks, TRACK_ENTRY, &v); + if let Some(o) = &opus { + let mut a = vec![]; + el_uint(&mut a, 0xD7, 2); + el_uint(&mut a, 0x73C5, 2); + el_uint(&mut a, 0x83, 2); + el_uint(&mut a, 0x9C, 0); + el_str(&mut a, 0x86, "A_OPUS"); + el(&mut a, 0x63A2, &o.opus_head()); + el_uint(&mut a, 0x56AA, o.pre_skip() as u64 * 1_000_000_000 / OPUS_RATE as u64); + el_uint(&mut a, 0x56BB, 80_000_000); + let mut au = vec![]; + el_float(&mut au, 0xB5, OPUS_RATE as f64); + el_uint(&mut au, 0x9F, 2); + el(&mut a, 0xE1, &au); + el(&mut tracks, TRACK_ENTRY, &a); + } + el(&mut head, TRACKS, &tracks); + file.write_all(&head).map_err(io)?; + let mut seg_len = head.len() as u64 - seg_start; + + // Audio: mixed in blocks, encoded in 20 ms packets. + let (span_start, span_end) = job.settings.span(comp); + let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); + let mut pcm_left: Vec = vec![]; + let mut packets = 0u64; + let mut audio_until = |until: Tick, frames: &mut Vec, finish: bool| { + let Some(enc) = opus.as_mut() else { return }; + let end = until.to_units_floor(OPUS_RATE as i64); + if end > cursor { + let n = (end - cursor) as usize; + let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); + pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + cursor = end; + } + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + if finish && !pcm_left.is_empty() { + // Pad the last packet (pre-skip + padding cover the encoder delay). + let pad = (fs - pcm_left.len() % fs) % fs + fs; + pcm_left.extend(std::iter::repeat_n(0.0, pad)); + } + while pcm_left.len() >= fs { + let pkt = enc.encode_float(&pcm_left[..fs]); + pcm_left.drain(..fs); + frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + packets += 1; + } + }; + + let batch = batch_size(); + let mut i = 0; + while i < total { + let end = (i + batch).min(total); + let encoded: Vec<(Vec, Option>)> = (i..end) + .into_par_iter() + .map(|k| { + let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); + let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); + let color = enc.encode_yuv420(&y, &u, &vv); + let a = alpha.then(|| { + let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); + ea.encode_yuv420(&y, &u, &vv) + }); + (color, a) + }) + .collect(); + let mut frames: Vec = + encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; + audio_until(t_end, &mut frames, end >= total); + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + st.advance(end - i)?; + i = end; + } + if total == 0 { + let mut frames = vec![]; + audio_until(span_end, &mut frames, true); + if !frames.is_empty() { + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + } + } + // Patch the Segment size. + let mut size = [0u8; 8]; + size[0] = 0x01; + size[1..].copy_from_slice(&seg_len.to_be_bytes()[1..]); + file.seek(SeekFrom::Start(size_pos)).map_err(io)?; + file.write_all(&size).map_err(io)?; + file.seek(SeekFrom::End(0)).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) +} + +// ---------------------------------------------------------------- audio-only outputs + +fn ieee_extended(v: f64) -> [u8; 10] { + // 80-bit IEEE 754 extended precision (AIFF COMM sample rate). + let mut out = [0u8; 10]; + if v <= 0.0 { + return out; + } + let e = v.log2().floor() as i32; + let mant = (v / 2f64.powi(e) * (1u64 << 63) as f64) as u64; + out[..2].copy_from_slice(&((e + 16383) as u16).to_be_bytes()); + out[2..].copy_from_slice(&mant.to_be_bytes()); + out +} + +/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. +pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { + let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let (span_start, span_end) = job.settings.span(comp); + let start = span_start.to_units_floor(sr as i64); + let end = span_end.to_units_floor(sr as i64); + let n = (end - start).max(0) as usize; + let mut pcm = Vec::with_capacity(n * 4); + // Mix in one-second blocks, reporting progress in frames. + let block = sr as usize; + let mut done = 0usize; + let total_frames = st.total.max(1); + let mut reported = 0u64; + while done < n { + let k = block.min(n - done); + let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); + for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { + let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; + pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); + } + done += k; + let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); + if now > reported && st.total > 0 { + st.advance(now - reported)?; + reported = now; + } + } + if reported < st.total { + st.advance(st.total - reported)?; + } + let mut out = vec![]; + let frames = (pcm.len() / 4) as u32; + if aiff { + let mut comm = vec![]; + comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&frames.to_be_bytes()); + comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&ieee_extended(sr as f64)); + let ssnd_len = 8 + pcm.len(); + out.extend_from_slice(b"FORM"); + out.extend_from_slice(&((4 + 8 + comm.len() + 8 + ssnd_len + ssnd_len % 2) as u32).to_be_bytes()); + out.extend_from_slice(b"AIFF"); + out.extend_from_slice(b"COMM"); + out.extend_from_slice(&(comm.len() as u32).to_be_bytes()); + out.extend_from_slice(&comm); + out.extend_from_slice(b"SSND"); + out.extend_from_slice(&(ssnd_len as u32).to_be_bytes()); + out.extend_from_slice(&[0; 8]); + out.extend_from_slice(&pcm); + if ssnd_len % 2 == 1 { + out.push(0); + } + } else { + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&sr.to_le_bytes()); + out.extend_from_slice(&(sr * 4).to_le_bytes()); + out.extend_from_slice(&4u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); + out.extend_from_slice(&pcm); + } + let mut file = crate::out::create(job.sink, job.path)?; + file.write_all(&out).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vints_and_ids() { + assert_eq!(vint(0), vec![0x80]); + assert_eq!(vint(126), vec![0xFE]); + assert_eq!(vint(127), vec![0x40, 0x7F]); + assert_eq!(id_bytes(0xA3), vec![0xA3]); + assert_eq!(id_bytes(EBML), vec![0x1A, 0x45, 0xDF, 0xA3]); + assert_eq!(ieee_extended(44100.0), [0x40, 0x0E, 0xAC, 0x44, 0, 0, 0, 0, 0, 0]); + assert_eq!(ieee_extended(48000.0)[..4], [0x40, 0x0E, 0xBB, 0x80]); + } +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 186aa1a9..1ade594f 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -363,3 +363,95 @@ fn movies_carry_audio() { assert!(!effectcraft_media::probe(&path).expect("probe").has_audio); } } + +#[test] +fn webm_vp9_opus_roundtrip() { + let d = out_dir("webm"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + let om = OutputModule { quality: 90, audio: AudioOutput::Auto, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("comp.webm"); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio && r.bytes > 200, "{r:?}"); + // Decoded back through FilmCraft (Matroska demuxer, VP9 and Opus decoders). + let f = decode_movie(&path, FRAMES, 0.08); + assert!(f.has_audio); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + let l = rms(s.iter().step_by(2).copied()); + assert!((l - 0.3536).abs() < 0.06, "left RMS {l}"); + ffmpeg_check(&path, 0.08); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("vp9|64|48|yuv420p"), "{s}"); + assert!(s.contains("opus"), "{s}"); + } +} + +#[test] +fn webm_with_alpha() { + let d = out_dir("webm-alpha"); + let (p, cid) = project(None); + // Transparent outside the red solid. + let om = OutputModule { channels: Channels::Rgba, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("alpha.webm"); + run(&p, cid, &NoFootage, &om, &path); + let f = effectcraft_media::probe(&path).expect("probe our output"); + assert_eq!((f.width, f.height), (W, H)); + let bytes = std::fs::read(&path).unwrap(); + // AlphaMode element (0x53C0) and BlockAdditions (0x75A1). + assert!(bytes.windows(3).any(|w| w == [0x53, 0xC0, 0x81]), "AlphaMode 1"); + assert!(bytes.windows(2).any(|w| w == [0x75, 0xA1]), "BlockAdditions"); + // ffmpeg's libvpx decoder reads the alpha channel. + let png = path.with_extension("alpha.png"); + let ok = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y", "-c:v", "libvpx-vp9", "-i"]) + .arg(&path) + .args(["-vf", "select=eq(n\\,8)", "-frames:v", "1", "-pix_fmt", "rgba"]) + .arg(&png) + .status() + .map(|s| s.success()) + .unwrap_or(false); + if ok { + let img = image::open(&png).unwrap().to_rgba8(); + assert!(img.get_pixel(2, 2)[3] < 20, "corner transparent"); + assert!(img.get_pixel(W / 2, H / 2)[3] > 235, "centre opaque"); + } else { + eprintln!("ffmpeg with libvpx not available: skipping alpha oracle"); + } +} + +#[test] +fn wav_and_aiff_audio_only() { + let d = out_dir("audio-only"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + for (fmt, name, magic) in [(OutputFormat::Wav, "out.wav", &b"RIFF"[..]), (OutputFormat::Aiff, "out.aif", &b"FORM"[..])] { + let om = OutputModule::for_format(fmt); + let path = d.join(name); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio); + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(&bytes[..4], magic); + // 1.2 s of 48 kHz 16-bit stereo. + assert!((bytes.len() as i64 - 57_600 * 4).abs() < 200, "{name}: {}", bytes.len()); + let l = if fmt == OutputFormat::Wav { + // Decoded by FilmCraft. + let f = effectcraft_media::probe(&path).expect("probe audio"); + assert!(f.has_audio && !f.has_video, "{name}"); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + rms(s.iter().step_by(2).copied()) + } else { + // AIFF: big-endian samples after the SSND chunk's offset/block size. + let at = bytes.windows(4).position(|w| w == b"SSND").expect("SSND") + 16; + let left = bytes[at..].chunks_exact(4).skip(14_400).take(24_000).map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32767.0); + rms(left) + }; + assert!((l - 0.3536).abs() < 0.01, "{name}: left RMS {l}"); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("pcm_s16"), "{s}"); + } + } +} diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs new file mode 100644 index 00000000..8156d850 --- /dev/null +++ b/crates/export/tests/vp9_roundtrip.rs @@ -0,0 +1,149 @@ +//! The VP9 intra encoder (`effectcraft-vp9enc`) round-trips through FilmCraft's VP9 decoder (and +//! ffmpeg as an external oracle when installed): sizes, lossless exactness, PSNR. + +use effectcraft_vp9enc::{EncoderConfig, Vp9Encoder}; +use filmcraft_vp9::{Decoder, Plane}; + +fn plane_u8(p: &Plane) -> Vec { + match p { + Plane::U8(v) => v.clone(), + Plane::U16(v) => v.iter().map(|x| *x as u8).collect(), + } +} + +/// Synthetic planes: gradients, noise and sharp edges. +fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut s = 99u32 + kind; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 24) as u8 + }; + let y: Vec = (0..w * h) + .map(|i| { + let (x, yy) = (i % w, i / w); + match kind { + 0 => ((x * 255) / w.max(1)) as u8 / 2 + ((yy * 255) / h.max(1)) as u8 / 2, + 1 => rnd(), + _ => { + if (x / 7 + yy / 5) % 2 == 0 { + 30 + } else { + 220 + } + } + } + }) + .collect(); + let u: Vec = (0..cw * ch).map(|i| (64 + (i % cw) * 128 / cw.max(1)) as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (200 - (i / cw) * 100 / ch.max(1)) as u8).collect(); + (y, u, v) +} + +fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { + let mut d = Decoder::new(); + let mut pics = d.decode(frame, 0).expect("decode"); + assert_eq!(pics.len(), 1); + pics.remove(0) +} + +fn crop(p: &[u8], stride: usize, w: usize, h: usize) -> Vec { + (0..h).flat_map(|y| p[y * stride..y * stride + w].to_vec()).collect() +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse: f64 = a.iter().zip(b).map(|(x, y)| (*x as f64 - *y as f64).powi(2)).sum::() / a.len() as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } +} + +#[test] +fn lossless_is_exact_for_odd_sizes() { + for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { + for kind in 0..3 { + let (y, u, v) = source(w, h, kind); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert_eq!((p.width, p.height), (w as u32, h as u32)); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + assert_eq!(crop(&plane_u8(&p.y), p.y_stride, w, h), y, "{w}x{h} kind {kind}: luma"); + assert_eq!(crop(&plane_u8(&p.u), p.uv_stride, cw, ch), u, "{w}x{h} kind {kind}: u"); + assert_eq!(crop(&plane_u8(&p.v), p.uv_stride, cw, ch), v, "{w}x{h} kind {kind}: v"); + } + } +} + +#[test] +fn lossy_quality_and_size() { + let (w, h) = (200, 120); + for kind in [0, 2] { + let (y, u, v) = source(w, h, kind); + let mut last = 0usize; + for q in [30u8, 60, 80, 95] { + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + let got = crop(&plane_u8(&p.y), p.y_stride, w, h); + let db = psnr(&got, &y); + let min = match q { + 30 => 20.0, + 60 => 28.0, + 80 => 32.0, + _ => 38.0, + }; + assert!(db > min, "kind {kind} q{q}: PSNR {db:.1} dB, {} bytes", f.len()); + assert!(f.len() >= last, "higher quality, more bytes"); + last = f.len(); + } + } +} + +#[test] +fn wide_frames_use_tile_columns() { + // 4160 px > 64 superblocks: two tile columns are required. + let (w, h) = (4160, 16); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); +} + +#[test] +fn ffmpeg_decodes_our_stream() { + let (w, h) = (96, 64); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let frame = e.encode_yuv420(&y, &u, &v); + // IVF container (one frame). + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + ivf.extend_from_slice(&(frame.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&0u64.to_le_bytes()); + ivf.extend_from_slice(&frame); + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); + assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); +} diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs index eb5be076..590112c9 100644 --- a/crates/host/tests/vector_import.rs +++ b/crates/host/tests/vector_import.rs @@ -161,3 +161,26 @@ fn svg_footage_and_shapes_from_vector_layer() { assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); } } + +#[test] +fn svg_footage_continuously_rasterizes() { + let path = tmp("cr.svg"); + std::fs::write(&path, SVG).unwrap(); + let doc = effectcraft_svg::parse(SVG).unwrap(); + let (w, h) = doc.pixel_size(); + let mut s = effectcraft_host::session(); + let item = s.execute_checked("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "C", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + for (k, v) in [("transform/anchor", json!([0, 0])), ("transform/position", json!([0, 0])), ("transform/scale", json!([300, 300]))] { + s.execute("prop.set", json!({"layer": lid, "path": k, "value": v})).unwrap(); + } + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let want = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let soft = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + s.execute("layer.setSwitch", json!({"layers": [lid], "switch": "collapse", "value": true})).unwrap(); + let sharp = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + assert!(sharp < 0.004, "continuously rasterised diff {sharp}"); + assert!(sharp < soft, "sharper than the upscaled pixels ({sharp} vs {soft})"); +} diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index dce33d8a..1edcf5fd 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -392,4 +392,3 @@ mod tests { } } } - diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index c41462b5..d1e6683c 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -136,17 +136,26 @@ pub enum OutputFormat { ExrSequence, /// Animated GIF. Gif, + /// WebM: VP9 video (alpha with RGB + Alpha) + Opus audio. + WebM, + /// Audio only: WAV (16-bit PCM). + Wav, + /// Audio only: AIFF (16-bit PCM). + Aiff, } impl OutputFormat { - pub const ALL: [OutputFormat; 7] = [ + pub const ALL: [OutputFormat; 10] = [ OutputFormat::H264, OutputFormat::ProRes, + OutputFormat::WebM, OutputFormat::PngSequence, OutputFormat::JpegSequence, OutputFormat::TiffSequence, OutputFormat::ExrSequence, OutputFormat::Gif, + OutputFormat::Wav, + OutputFormat::Aiff, ]; pub fn label(self) -> &'static str { @@ -158,6 +167,9 @@ impl OutputFormat { OutputFormat::TiffSequence => "TIFF Sequence", OutputFormat::ExrSequence => "OpenEXR Sequence", OutputFormat::Gif => "Animated GIF", + OutputFormat::WebM => "WebM (VP9 + Opus)", + OutputFormat::Wav => "WAV", + OutputFormat::Aiff => "AIFF", } } pub fn extension(self) -> &'static str { @@ -169,19 +181,26 @@ impl OutputFormat { OutputFormat::TiffSequence => "tif", OutputFormat::ExrSequence => "exr", OutputFormat::Gif => "gif", + OutputFormat::WebM => "webm", + OutputFormat::Wav => "wav", + OutputFormat::Aiff => "aif", } } pub fn is_sequence(self) -> bool { matches!(self, OutputFormat::PngSequence | OutputFormat::JpegSequence | OutputFormat::TiffSequence | OutputFormat::ExrSequence) } pub fn is_movie(self) -> bool { - matches!(self, OutputFormat::H264 | OutputFormat::ProRes) + matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM) + } + /// Audio-only outputs (no video is rendered). + pub fn is_audio_only(self) -> bool { + matches!(self, OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_alpha(self) -> bool { - !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence) + !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_audio(self) -> bool { - self.is_movie() + self.is_movie() || self.is_audio_only() } /// The written frame size for a rendered size: H.264 needs even dimensions (rounded down, /// at least 2). @@ -201,6 +220,9 @@ impl OutputFormat { "tif" | "tiff" | "tiffsequence" => OutputFormat::TiffSequence, "exr" | "openexr" | "openexrsequence" => OutputFormat::ExrSequence, "gif" | "animatedgif" => OutputFormat::Gif, + "webm" | "vp9" | "webmvp9opus" => OutputFormat::WebM, + "wav" | "wave" => OutputFormat::Wav, + "aif" | "aiff" | "aifc" => OutputFormat::Aiff, _ => return None, }) } diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index 154dd4c2..06403181 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -108,7 +108,7 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } - OutputFormat::JpegSequence => { + OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml new file mode 100644 index 00000000..62144b25 --- /dev/null +++ b/crates/vp9enc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-vp9enc" +description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md new file mode 100644 index 00000000..c8f781b8 --- /dev/null +++ b/crates/vp9enc/README.md @@ -0,0 +1,15 @@ +# effectcraft-vp9enc + +A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, +BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 +DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 +(mathematically lossless). + +Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March +2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode +info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) +and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The +boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec +tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR +Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an +external test oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs new file mode 100644 index 00000000..45db92dd --- /dev/null +++ b/crates/vp9enc/src/bool.rs @@ -0,0 +1,193 @@ +//! Boolean (arithmetic) encoder: the inverse of the VP9 boolean decoder (spec §9.2), written as +//! the classic carry-propagating range coder described in RFC 6386 §7.3 (VP8 and VP9 share the +//! bool coder). + +pub struct BoolEncoder { + out: Vec, + range: u32, + bottom: u32, + bit_count: i32, +} + +impl Default for BoolEncoder { + fn default() -> Self { + BoolEncoder::new() + } +} + +impl BoolEncoder { + pub fn new() -> Self { + BoolEncoder { out: Vec::new(), range: 255, bottom: 0, bit_count: 24 } + } + + fn add_one(&mut self) { + let mut i = self.out.len(); + while i > 0 { + i -= 1; + if self.out[i] == 255 { + self.out[i] = 0; + } else { + self.out[i] += 1; + return; + } + } + } + + /// Write `bit` with probability `prob`/256 of it being 0. + #[inline] + pub fn write(&mut self, bit: bool, prob: u8) { + let split = 1 + (((self.range - 1) * prob as u32) >> 8); + if bit { + self.bottom = self.bottom.wrapping_add(split); + self.range -= split; + } else { + self.range = split; + } + while self.range < 128 { + self.range <<= 1; + if self.bottom & (1 << 31) != 0 { + self.add_one(); + } + self.bottom <<= 1; + self.bit_count -= 1; + if self.bit_count == 0 { + self.out.push((self.bottom >> 24) as u8); + self.bottom &= (1 << 24) - 1; + self.bit_count = 8; + } + } + } + + pub fn literal(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.write((v >> i) & 1 != 0, 128); + } + } + + /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). + pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { + // Find the path to the leaf. + fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.push((node >> 1, b == 1)); + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if path(tree, t as usize, value, out) { + return true; + } + out.pop(); + } + false + } + let mut p = Vec::with_capacity(8); + path(tree, 0, value, &mut p); + for (i, bit) in p { + self.write(bit, probs[i]); + } + } + + /// Flush (RFC 6386 §7.3) and return the bytes. + pub fn finish(mut self) -> Vec { + let mut c = self.bit_count; + let mut v = self.bottom; + if v & (1u32 << (32 - c)) != 0 { + self.add_one(); + } + v <<= c & 7; + c >>= 3; + while { + c -= 1; + c >= 0 + } { + v <<= 8; + } + for _ in 0..4 { + self.out.push((v >> 24) as u8); + v <<= 8; + } + self.out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A straightforward decoder per VP9 spec §9.2. + struct Dec<'a> { + d: &'a [u8], + pos: usize, + value: u64, + bits: i32, + range: u32, + } + impl<'a> Dec<'a> { + fn new(d: &'a [u8]) -> Self { + let mut x = Dec { d, pos: 0, value: 0, bits: 0, range: 255 }; + x.fill(); + x + } + fn fill(&mut self) { + while self.bits <= 48 { + let b = if self.pos < self.d.len() { self.d[self.pos] } else { 0 }; + self.pos += 1; + self.value |= (b as u64) << (56 - self.bits); + self.bits += 8; + } + } + fn read(&mut self, p: u8) -> bool { + let split = 1 + (((self.range - 1) * p as u32) >> 8); + let big = (split as u64) << 56; + let bit = if self.value >= big { + self.range -= split; + self.value -= big; + true + } else { + self.range = split; + false + }; + while self.range < 128 { + self.range <<= 1; + self.value <<= 1; + self.bits -= 1; + } + self.fill(); + bit + } + } + + #[test] + fn round_trip() { + let mut e = BoolEncoder::new(); + let mut seq = vec![]; + let mut s = 12345u32; + for i in 0..20000 { + s = s.wrapping_mul(1103515245).wrapping_add(12345); + let p = ((s >> 8) % 255 + 1) as u8; + let bit = (s >> 20) % 256 >= p as u32 || i % 97 == 0; + seq.push((bit, p)); + e.write(bit, p); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for (i, (b, p)) in seq.into_iter().enumerate() { + assert_eq!(dec.read(p), b, "bit {i}"); + } + } + + #[test] + fn all_ones_carry() { + let mut e = BoolEncoder::new(); + for _ in 0..5000 { + e.write(true, 1); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for _ in 0..5000 { + assert!(dec.read(1)); + } + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs new file mode 100644 index 00000000..02cbb0e2 --- /dev/null +++ b/crates/vp9enc/src/lib.rs @@ -0,0 +1,857 @@ +//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & +//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see +//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width +//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard +//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by +//! prediction error, default probabilities (no probability updates). The encoder reconstructs +//! exactly what a decoder reconstructs (the specification's integer inverse transforms and +//! intra edge rules), so prediction never drifts. + +mod bool; +mod tables; + +use bool::BoolEncoder; +use tables::*; + +const DC_PRED: u8 = 0; +const V_PRED: u8 = 1; +const H_PRED: u8 = 2; +const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; + +const DCT_DCT: u8 = 0; +const ADST_DCT: u8 = 1; +const DCT_ADST: u8 = 2; +const ADST_ADST: u8 = 3; + +/// mode2txfm_map for the modes used here. +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// Encoder settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + /// 1..=100; 100 is lossless. + pub quality: u8, + /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. + pub full_range: bool, +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +// ---------------------------------------------------------------- transforms (8.7) + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms, so `inverse4(forward(res)) ≈ res`. +fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +// ---------------------------------------------------------------- planes and prediction + +struct Plane { + w: usize, + /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). + max_x: usize, + max_y: usize, + src: Vec, + rec: Vec, +} + +impl Plane { + fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { + let mut src = vec![0u8; w * h]; + for y in 0..h { + let sy = y.min(vh.saturating_sub(1)); + for x in 0..w { + let sx = x.min(vw.saturating_sub(1)); + src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); + } + } + Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + } +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- the encoder + +pub struct Vp9Encoder { + cfg: EncoderConfig, + qidx: u8, + lossless: bool, +} + +/// Per-tile coding state. +struct Tile<'a> { + enc: &'a Vp9Encoder, + bc: BoolEncoder, + planes: &'a mut [Plane; 3], + mi_cols: usize, + mi_rows: usize, + col_start: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// y mode of every 8×8 block (for mode contexts). + modes: &'a mut [u8], +} + +impl Vp9Encoder { + pub fn new(cfg: EncoderConfig) -> Self { + let q = cfg.quality.clamp(1, 100); + let lossless = q >= 100; + // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; + Vp9Encoder { cfg, qidx, lossless } + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// ((w+1)/2)×((h+1)/2), tightly packed). + pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let mi_cols = w.div_ceil(8); + let mi_rows = h.div_ceil(8); + let sb_cols = mi_cols.div_ceil(8); + let sb_rows = mi_rows.div_ceil(8); + let (pw, ph) = (sb_cols * 64, sb_rows * 64); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut planes = [ + Plane::new(y, w, h, pw, ph, mi_cols * 8 - 1, mi_rows * 8 - 1), + Plane::new(u, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + Plane::new(v, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + ]; + // Tile columns: as few as the width allows (tile_info, 6.2.13). + let mut min_log2 = 0; + while (64usize << min_log2) < sb_cols { + min_log2 += 1; + } + let mut max_log2 = 1; + while (sb_cols >> max_log2) >= 4 { + max_log2 += 1; + } + max_log2 -= 1; + let tile_cols = 1usize << min_log2; + let mut modes = vec![0u8; mi_cols * mi_rows]; + let mut tiles = vec![]; + for t in 0..tile_cols { + let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); + let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); + let mut tile = Tile { + enc: self, + bc: BoolEncoder::new(), + planes: &mut planes, + mi_cols, + mi_rows, + col_start: start, + above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; mi_cols + 8], + left_part: vec![0; 8], + modes: &mut modes, + }; + tile.bc.write(false, 128); + for r in (0..mi_rows).step_by(8) { + for p in 0..3 { + tile.left_nz[p].fill(0); + } + tile.left_part.fill(0); + for c in (start..end).step_by(8) { + tile.partition(r, c, 12); + } + } + tiles.push(tile.bc.finish()); + } + // Compressed header. + let mut ch_enc = BoolEncoder::new(); + ch_enc.write(false, 128); + if !self.lossless { + ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + } + ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) + for _ in 0..3 { + ch_enc.write(false, 252); // skip probabilities unchanged + } + let compressed = ch_enc.finish(); + // Uncompressed header. + let mut bw = BitWriter::default(); + bw.put(2, 2); // frame_marker + bw.put(0, 1); // profile low bit + bw.put(0, 1); // profile high bit + bw.put(0, 1); // show_existing_frame + bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(1, 1); // show_frame + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + bw.put(0, 2); // frame_context_idx + bw.put(0, 6); // loop_filter_level + bw.put(0, 3); // sharpness + bw.put(0, 1); // mode_ref_delta_enabled + bw.put(self.qidx as u32, 8); + bw.put(0, 1); // delta_q_y_dc + bw.put(0, 1); // delta_q_uv_dc + bw.put(0, 1); // delta_q_uv_ac + bw.put(0, 1); // segmentation_enabled + if min_log2 < max_log2 { + bw.put(0, 1); // no extra tile columns + } + bw.put(0, 1); // tile_rows_log2 = 0 + bw.put(compressed.len() as u32, 16); + let mut out = bw.finish(); + out.extend_from_slice(&compressed); + let n = tiles.len(); + for (i, t) in tiles.into_iter().enumerate() { + if i + 1 < n { + out.extend_from_slice(&(t.len() as u32).to_be_bytes()); + } + out.extend_from_slice(&t); + } + out + } +} + +#[derive(Default)] +struct BitWriter { + out: Vec, + acc: u32, + n: u32, +} + +impl BitWriter { + fn put(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1); + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out + } +} + +const BLOCK_8X8: u8 = 3; +const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; +const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; + +impl Tile<'_> { + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + if r >= self.mi_rows || c >= self.mi_cols { + return; + } + let num8 = NUM_8X8[bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < self.mi_rows; + let has_cols = (c + half) < self.mi_cols; + let bsl = MI_WIDTH_LOG2[bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let above = (above >> boffset) & 1; + let left = (left >> boffset) & 1; + let ctx = bsl * 4 + left as usize * 2 + above as usize; + let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; + if bsize == BLOCK_8X8 { + // PARTITION_NONE. + self.bc.write(false, probs[0]); + self.block(r, c); + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.bc.tree(&PARTITION_TREE, probs, 3); + } else if has_cols { + self.bc.write(true, probs[1]); + } else if has_rows { + self.bc.write(true, probs[2]); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + fn block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. + let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). + self.bc.write(false, DEFAULT_SKIP_PROB[0]); + let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; + let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); + let o = y_mode as usize * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); + self.modes[r * self.mi_cols + c] = y_mode; + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.mi_cols * 8) >> ss; + let max_y = (self.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); + // Quantised levels (raster order). + let levels: [i32; 16] = if self.enc.lossless { + forward_wht(&res) + } else { + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let qs = if k == 0 { dcq } else { acq } as f64; + let a = coef.abs() / qs; + // Rounding with a small dead zone for AC coefficients. + let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; + lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; + } + lv + }; + let sc = scan(tx_type); + let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); + // Tokens. + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, tx_type, &levels, eob, ctx0); + // Reconstruct exactly as the decoder does. + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { dcq } else { acq }; + } + let resid = if eob == 0 { + [0i32; 16] + } else if self.enc.lossless { + inverse_wht(&deq) + } else { + inverse4(&deq, tx_type) + }; + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. + let base = (ptype * 2) * 6 * 6 * 3; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = base + (band * 6 + ctx) * 3; + let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; + if check_eob { + let more = c < eob; + self.bc.write(more, p[0]); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.bc.write(false, p[1]); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.bc.write(true, p[1]); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.bc.write(false, p[2]); + } else { + self.bc.write(true, p[2]); + let pp = pareto(p[2]); + let path: &[(usize, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.bc.write(b, pp[n]); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.bc.write(levels[pos] < 0, 128); + c += 1; + } + } +} + +// ---------------------------------------------------------------- colour conversion + +/// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). +pub fn rgba_to_yuv420(rgba: &[u8], w: u32, h: u32, full_range: bool) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut y = vec![0u8; w * h]; + let mut u = vec![0u8; cw * ch]; + let mut v = vec![0u8; cw * ch]; + let (ys, yo, cs) = if full_range { (255.0, 0.0, 255.0) } else { (219.0, 16.0, 224.0) }; + let px = |x: usize, yy: usize| -> [f32; 3] { + let i = (yy.min(h - 1) * w + x.min(w - 1)) * 4; + [rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0] + }; + let luma = |c: [f32; 3]| 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; + for yy in 0..h { + for x in 0..w { + y[yy * w + x] = (luma(px(x, yy)) * ys + yo).round().clamp(0.0, 255.0) as u8; + } + } + for cy in 0..ch { + for cx in 0..cw { + let mut acc = [0.0f32; 3]; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let c = px(cx * 2 + dx, cy * 2 + dy); + for k in 0..3 { + acc[k] += c[k] / 4.0; + } + } + let l = luma(acc); + let cb = (acc[2] - l) / 1.8556; + let cr = (acc[0] - l) / 1.5748; + u[cy * cw + cx] = (cb * cs + 128.0).round().clamp(0.0, 255.0) as u8; + v[cy * cw + cx] = (cr * cs + 128.0).round().clamp(0.0, 255.0) as u8; + } + } + (y, u, v) +} + +/// The alpha channel as a luma plane (full range) with neutral chroma — the second VP9 stream of +/// WebM alpha. +pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let y: Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); + (y, vec![128; cw * ch], vec![128; cw * ch]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs new file mode 100644 index 00000000..dc8027a6 --- /dev/null +++ b/crates/vp9enc/src/tables.rs @@ -0,0 +1,130 @@ +//! Tables from the VP9 Bitstream & Decoding Process Specification v0.6 (section 10), as +//! transcribed in FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0, the same +//! licence as this crate). + +#![allow(clippy::unreadable_literal)] + +pub const DC_QLOOKUP: [i32; 256] = [ + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, + 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 70, 70, 71, 72, + 73, 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, 111, 113, 114, + 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, 177, 180, + 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, + 284, 288, 292, 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, + 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, + 775, 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, +]; + +pub const AC_QLOOKUP: [i32; 256] = [ + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, + 136, 138, 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, 373, 380, 387, + 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, + 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, + 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, +]; + +pub const DEFAULT_SCAN_4X4: [u16; 16] = [0, 4, 1, 5, 8, 2, 12, 9, 3, 6, 13, 10, 7, 14, 11, 15]; + +pub const COL_SCAN_4X4: [u16; 16] = [0, 4, 8, 1, 12, 5, 9, 2, 13, 6, 10, 3, 7, 14, 11, 15]; + +pub const ROW_SCAN_4X4: [u16; 16] = [0, 1, 4, 2, 5, 3, 6, 8, 9, 7, 12, 10, 13, 11, 14, 15]; + +pub const COEFBAND_4X4: [u8; 16] = [0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5]; + +pub const ENERGY_CLASS: [u8; 12] = [0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5]; + +pub const PARETO_TABLE: [u8; 1024] = [ + 3, 86, 128, 6, 86, 23, 88, 29, 9, 86, 129, 17, 88, 61, 94, 76, 15, 87, 129, 28, 89, 93, 100, 110, 20, 88, 130, 38, 91, 118, 106, 136, 26, 89, 131, 48, 92, + 139, 111, 156, 31, 90, 131, 58, 94, 156, 117, 171, 37, 90, 132, 66, 95, 171, 122, 184, 42, 91, 132, 75, 97, 183, 127, 194, 47, 92, 133, 83, 98, 193, 132, + 202, 52, 93, 133, 90, 100, 201, 137, 208, 57, 94, 134, 98, 101, 208, 142, 214, 62, 94, 135, 105, 103, 214, 146, 218, 66, 95, 135, 111, 104, 219, 151, 222, + 71, 96, 136, 117, 106, 224, 155, 225, 76, 97, 136, 123, 107, 227, 159, 228, 80, 98, 137, 129, 109, 231, 162, 231, 84, 98, 138, 134, 110, 234, 166, 233, 89, + 99, 138, 140, 112, 236, 170, 235, 93, 100, 139, 145, 113, 238, 173, 236, 97, 101, 140, 149, 115, 240, 176, 238, 101, 102, 140, 154, 116, 242, 179, 239, + 105, 103, 141, 158, 118, 243, 182, 240, 109, 104, 141, 162, 119, 244, 185, 241, 113, 104, 142, 166, 120, 245, 187, 242, 116, 105, 143, 170, 122, 246, 190, + 243, 120, 106, 143, 173, 123, 247, 192, 244, 123, 107, 144, 177, 125, 248, 195, 244, 127, 108, 145, 180, 126, 249, 197, 245, 130, 109, 145, 183, 128, 249, + 199, 245, 134, 110, 146, 186, 129, 250, 201, 246, 137, 111, 147, 189, 131, 251, 203, 246, 140, 112, 147, 192, 132, 251, 205, 247, 143, 113, 148, 194, 133, + 251, 207, 247, 146, 114, 149, 197, 135, 252, 208, 248, 149, 115, 149, 199, 136, 252, 210, 248, 152, 115, 150, 201, 138, 252, 211, 248, 155, 116, 151, 204, + 139, 253, 213, 249, 158, 117, 151, 206, 140, 253, 214, 249, 161, 118, 152, 208, 142, 253, 216, 249, 163, 119, 153, 210, 143, 253, 217, 249, 166, 120, 153, + 212, 144, 254, 218, 250, 168, 121, 154, 213, 146, 254, 220, 250, 171, 122, 155, 215, 147, 254, 221, 250, 173, 123, 155, 217, 148, 254, 222, 250, 176, 124, + 156, 218, 150, 254, 223, 250, 178, 125, 157, 220, 151, 254, 224, 251, 180, 126, 157, 221, 152, 254, 225, 251, 183, 127, 158, 222, 153, 254, 226, 251, 185, + 128, 159, 224, 155, 255, 227, 251, 187, 129, 160, 225, 156, 255, 228, 251, 189, 131, 160, 226, 157, 255, 228, 251, 191, 132, 161, 227, 159, 255, 229, 251, + 193, 133, 162, 228, 160, 255, 230, 252, 195, 134, 163, 230, 161, 255, 231, 252, 197, 135, 163, 231, 162, 255, 231, 252, 199, 136, 164, 232, 163, 255, 232, + 252, 201, 137, 165, 233, 165, 255, 233, 252, 202, 138, 166, 233, 166, 255, 233, 252, 204, 139, 166, 234, 167, 255, 234, 252, 206, 140, 167, 235, 168, 255, + 235, 252, 207, 141, 168, 236, 169, 255, 235, 252, 209, 142, 169, 237, 171, 255, 236, 252, 210, 144, 169, 237, 172, 255, 236, 252, 212, 145, 170, 238, 173, + 255, 237, 252, 214, 146, 171, 239, 174, 255, 237, 253, 215, 147, 172, 240, 175, 255, 238, 253, 216, 148, 173, 240, 176, 255, 238, 253, 218, 149, 173, 241, + 177, 255, 239, 253, 219, 150, 174, 241, 179, 255, 239, 253, 220, 152, 175, 242, 180, 255, 240, 253, 222, 153, 176, 242, 181, 255, 240, 253, 223, 154, 177, + 243, 182, 255, 240, 253, 224, 155, 178, 244, 183, 255, 241, 253, 225, 156, 178, 244, 184, 255, 241, 253, 226, 158, 179, 244, 185, 255, 242, 253, 228, 159, + 180, 245, 186, 255, 242, 253, 229, 160, 181, 245, 187, 255, 242, 253, 230, 161, 182, 246, 188, 255, 243, 253, 231, 163, 183, 246, 189, 255, 243, 253, 232, + 164, 184, 247, 190, 255, 243, 253, 233, 165, 185, 247, 191, 255, 244, 253, 234, 166, 185, 247, 192, 255, 244, 253, 235, 168, 186, 248, 193, 255, 244, 253, + 236, 169, 187, 248, 194, 255, 244, 253, 236, 170, 188, 248, 195, 255, 245, 253, 237, 171, 189, 249, 196, 255, 245, 254, 238, 173, 190, 249, 197, 255, 245, + 254, 239, 174, 191, 249, 198, 255, 245, 254, 240, 175, 192, 249, 199, 255, 246, 254, 240, 177, 193, 250, 200, 255, 246, 254, 241, 178, 194, 250, 201, 255, + 246, 254, 242, 179, 195, 250, 202, 255, 246, 254, 242, 181, 196, 250, 203, 255, 247, 254, 243, 182, 197, 251, 204, 255, 247, 254, 244, 184, 198, 251, 205, + 255, 247, 254, 244, 185, 199, 251, 206, 255, 247, 254, 245, 186, 200, 251, 207, 255, 247, 254, 246, 188, 201, 252, 207, 255, 248, 254, 246, 189, 202, 252, + 208, 255, 248, 254, 247, 191, 203, 252, 209, 255, 248, 254, 247, 192, 204, 252, 210, 255, 248, 254, 248, 194, 205, 252, 211, 255, 248, 254, 248, 195, 206, + 252, 212, 255, 249, 254, 249, 197, 207, 253, 213, 255, 249, 254, 249, 198, 208, 253, 214, 255, 249, 254, 250, 200, 210, 253, 215, 255, 249, 254, 250, 201, + 211, 253, 215, 255, 249, 254, 250, 203, 212, 253, 216, 255, 249, 254, 251, 204, 213, 253, 217, 255, 250, 254, 251, 206, 214, 254, 218, 255, 250, 254, 252, + 207, 216, 254, 219, 255, 250, 254, 252, 209, 217, 254, 220, 255, 250, 254, 252, 211, 218, 254, 221, 255, 250, 254, 253, 213, 219, 254, 222, 255, 250, 254, + 253, 214, 221, 254, 223, 255, 250, 254, 253, 216, 222, 254, 224, 255, 251, 254, 253, 218, 224, 254, 225, 255, 251, 254, 254, 220, 225, 254, 225, 255, 251, + 254, 254, 222, 227, 255, 226, 255, 251, 254, 254, 224, 228, 255, 227, 255, 251, 254, 254, 226, 230, 255, 228, 255, 251, 254, 255, 228, 231, 255, 230, 255, + 251, 254, 255, 230, 233, 255, 231, 255, 252, 254, 255, 232, 235, 255, 232, 255, 252, 254, 255, 235, 237, 255, 233, 255, 252, 254, 255, 238, 240, 255, 235, + 255, 252, 255, 255, 241, 243, 255, 236, 255, 252, 254, 255, 246, 247, 255, 239, 255, 253, 255, +]; + +pub const KF_PARTITION_PROBS: [u8; 48] = [ + 158, 97, 94, 93, 24, 99, 85, 119, 44, 62, 59, 67, 149, 53, 53, 94, 20, 48, 83, 53, 24, 52, 18, 18, 150, 40, 39, 78, 12, 26, 67, 33, 11, 24, 7, 5, 174, 35, + 49, 68, 11, 27, 57, 15, 9, 12, 3, 3, +]; + +pub const KF_Y_MODE_PROBS: [u8; 900] = [ + 137, 30, 42, 148, 151, 207, 70, 52, 91, 92, 45, 102, 136, 116, 180, 74, 90, 100, 73, 32, 19, 187, 222, 215, 46, 34, 100, 91, 30, 32, 116, 121, 186, 93, 86, + 94, 72, 35, 36, 149, 68, 206, 68, 63, 105, 73, 31, 28, 138, 57, 124, 55, 122, 151, 67, 23, 21, 140, 126, 197, 40, 37, 171, 86, 27, 28, 128, 154, 212, 45, + 43, 53, 74, 32, 27, 107, 86, 160, 63, 134, 102, 59, 67, 44, 140, 161, 202, 78, 67, 119, 63, 36, 126, 146, 123, 158, 60, 90, 96, 43, 46, 168, 134, 107, 128, + 69, 142, 92, 44, 29, 68, 159, 201, 177, 50, 57, 77, 58, 38, 76, 114, 97, 172, 78, 133, 92, 46, 41, 76, 140, 63, 184, 69, 112, 57, 38, 32, 85, 140, 46, 112, + 54, 151, 133, 39, 27, 61, 131, 110, 175, 44, 75, 136, 52, 30, 74, 113, 130, 175, 51, 64, 58, 47, 35, 80, 100, 74, 143, 64, 163, 74, 36, 61, 116, 114, 128, + 162, 80, 125, 82, 82, 26, 26, 171, 208, 204, 44, 32, 105, 55, 44, 68, 166, 179, 192, 57, 57, 108, 42, 26, 11, 199, 241, 228, 23, 15, 85, 68, 42, 19, 131, + 160, 199, 55, 52, 83, 58, 50, 25, 139, 115, 232, 39, 52, 118, 50, 35, 33, 153, 104, 162, 64, 59, 131, 44, 24, 16, 150, 177, 202, 33, 19, 156, 55, 27, 12, + 153, 203, 218, 26, 27, 49, 53, 49, 21, 110, 116, 168, 59, 80, 76, 38, 72, 19, 168, 203, 212, 50, 50, 107, 103, 26, 36, 129, 132, 201, 83, 80, 93, 59, 38, + 83, 112, 103, 162, 98, 136, 90, 62, 30, 23, 158, 200, 207, 59, 57, 50, 67, 30, 29, 84, 86, 191, 102, 91, 59, 60, 32, 33, 112, 71, 220, 64, 89, 104, 53, 26, + 34, 130, 56, 149, 84, 120, 103, 53, 21, 23, 133, 109, 210, 56, 77, 172, 77, 19, 29, 112, 142, 228, 55, 66, 36, 61, 29, 29, 93, 97, 165, 83, 175, 162, 47, + 47, 43, 114, 137, 181, 100, 99, 95, 69, 23, 29, 128, 83, 199, 46, 44, 101, 53, 40, 55, 139, 69, 183, 61, 80, 110, 40, 29, 19, 161, 180, 207, 43, 24, 91, + 60, 34, 19, 105, 61, 198, 53, 64, 89, 52, 31, 22, 158, 40, 209, 58, 62, 89, 44, 31, 29, 147, 46, 158, 56, 102, 198, 35, 19, 12, 135, 87, 209, 41, 45, 167, + 55, 25, 21, 118, 95, 215, 38, 39, 66, 51, 38, 25, 113, 58, 164, 70, 93, 97, 47, 54, 34, 146, 108, 203, 72, 103, 151, 64, 19, 37, 156, 66, 138, 49, 95, 133, + 46, 27, 80, 150, 55, 124, 55, 121, 135, 36, 23, 27, 165, 149, 166, 54, 64, 118, 53, 21, 36, 131, 63, 163, 60, 109, 81, 40, 26, 35, 154, 40, 185, 51, 97, + 123, 35, 19, 34, 179, 19, 97, 48, 129, 124, 36, 20, 26, 136, 62, 164, 33, 77, 154, 45, 18, 32, 130, 90, 157, 40, 79, 91, 45, 26, 28, 129, 45, 129, 49, 147, + 123, 38, 44, 51, 136, 74, 162, 57, 97, 121, 75, 17, 22, 136, 138, 185, 32, 34, 166, 56, 39, 58, 133, 117, 173, 48, 53, 187, 35, 21, 12, 161, 212, 207, 20, + 23, 145, 56, 29, 19, 117, 109, 181, 55, 68, 112, 47, 29, 17, 153, 64, 220, 59, 51, 114, 46, 16, 24, 136, 76, 147, 41, 64, 172, 34, 17, 11, 108, 152, 187, + 13, 15, 209, 51, 24, 14, 115, 133, 209, 32, 26, 104, 55, 30, 18, 122, 79, 179, 44, 88, 116, 37, 49, 25, 129, 168, 164, 41, 54, 148, 82, 22, 32, 127, 143, + 213, 39, 41, 70, 62, 44, 61, 123, 105, 189, 48, 57, 64, 47, 25, 17, 175, 222, 220, 24, 30, 86, 68, 36, 17, 106, 102, 206, 59, 74, 74, 57, 39, 23, 151, 68, + 216, 55, 63, 58, 49, 30, 35, 141, 70, 168, 82, 40, 115, 51, 25, 15, 136, 129, 202, 38, 35, 139, 68, 26, 16, 111, 141, 215, 29, 28, 28, 59, 39, 19, 114, 75, + 180, 77, 104, 42, 40, 61, 26, 126, 152, 206, 61, 59, 93, 78, 23, 39, 111, 117, 170, 74, 124, 94, 48, 34, 86, 101, 92, 146, 78, 179, 134, 47, 22, 24, 138, + 187, 178, 68, 69, 59, 56, 25, 33, 105, 112, 187, 95, 177, 129, 48, 31, 27, 114, 63, 183, 82, 116, 56, 43, 28, 37, 121, 63, 123, 61, 192, 169, 42, 17, 24, + 109, 97, 177, 56, 76, 122, 58, 18, 28, 105, 139, 182, 70, 92, 63, 46, 23, 32, 74, 86, 150, 67, 183, 88, 36, 38, 48, 92, 122, 165, 88, 137, 91, 65, 70, 60, + 155, 159, 199, 61, 60, 81, 44, 78, 115, 132, 119, 173, 71, 112, 93, 39, 38, 21, 184, 227, 206, 42, 32, 64, 58, 47, 36, 124, 137, 193, 80, 82, 78, 49, 50, + 35, 144, 95, 205, 63, 78, 59, 41, 53, 52, 148, 71, 142, 65, 128, 51, 40, 36, 28, 143, 143, 202, 40, 55, 137, 52, 34, 29, 129, 183, 227, 42, 35, 43, 42, 44, + 44, 104, 105, 164, 64, 130, 80, 43, 81, 53, 140, 169, 204, 68, 84, 72, +]; + +pub const KF_UV_MODE_PROBS: [u8; 90] = [ + 144, 11, 54, 157, 195, 130, 46, 58, 108, 118, 15, 123, 148, 131, 101, 44, 93, 131, 113, 12, 23, 188, 226, 142, 26, 32, 125, 120, 11, 50, 123, 163, 135, 64, + 77, 103, 113, 9, 36, 155, 111, 157, 32, 44, 161, 116, 9, 55, 176, 76, 96, 37, 61, 149, 115, 9, 28, 141, 161, 167, 21, 25, 193, 120, 12, 32, 145, 195, 142, + 32, 38, 86, 116, 12, 64, 120, 140, 125, 49, 115, 121, 102, 19, 66, 162, 182, 122, 35, 59, 128, +]; + +pub const DEFAULT_SKIP_PROB: [u8; 3] = [192, 128, 64]; + +pub const DEFAULT_COEF_PROBS: [u8; 432] = [ + 195, 29, 183, 84, 49, 136, 8, 42, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 107, 169, 35, 99, 159, 17, 82, 140, 8, 66, 114, 2, 44, 76, 1, 19, 32, 40, 132, 201, + 29, 114, 187, 13, 91, 157, 7, 75, 127, 3, 58, 95, 1, 28, 47, 69, 142, 221, 42, 122, 201, 15, 91, 159, 6, 67, 121, 1, 42, 77, 1, 17, 31, 102, 148, 228, 67, + 117, 204, 17, 82, 154, 6, 59, 114, 2, 39, 75, 1, 15, 29, 156, 57, 233, 119, 57, 212, 58, 48, 163, 29, 40, 124, 12, 30, 81, 3, 12, 31, 191, 107, 226, 124, + 117, 204, 25, 99, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 148, 210, 37, 126, 194, 8, 93, 157, 2, 68, 118, 1, 39, 69, 1, 17, 33, 41, 151, 213, 27, 123, 193, 3, + 82, 144, 1, 58, 105, 1, 32, 60, 1, 13, 26, 59, 159, 220, 23, 126, 198, 4, 88, 151, 1, 66, 114, 1, 38, 71, 1, 18, 34, 114, 136, 232, 51, 114, 207, 11, 83, + 155, 3, 56, 105, 1, 33, 65, 1, 17, 34, 149, 65, 234, 121, 57, 215, 61, 49, 166, 28, 36, 114, 12, 25, 76, 3, 16, 42, 214, 49, 220, 132, 63, 188, 42, 65, + 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 137, 221, 104, 131, 216, 49, 111, 192, 21, 87, 155, 2, 49, 87, 1, 16, 28, 89, 163, 230, 90, 137, 220, 29, 100, 183, 10, + 70, 135, 2, 42, 81, 1, 17, 33, 108, 167, 237, 55, 133, 222, 15, 97, 179, 4, 72, 135, 1, 45, 85, 1, 19, 38, 124, 146, 240, 66, 124, 224, 17, 88, 175, 4, 58, + 122, 1, 36, 75, 1, 18, 37, 141, 79, 241, 126, 70, 227, 66, 58, 182, 30, 44, 136, 12, 34, 96, 2, 20, 47, 229, 99, 249, 143, 111, 235, 46, 109, 192, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 82, 158, 236, 94, 146, 224, 25, 117, 191, 9, 87, 149, 3, 56, 99, 1, 33, 57, 83, 167, 237, 68, 145, 222, 10, 103, 177, 2, 72, 131, 1, 41, + 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, + 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, +]; diff --git a/docs/architecture.md b/docs/architecture.md index c3d869c0..096c75bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,18 +16,21 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `vp9enc` | VP9 intra-frame encoder (profile 0, 8-bit 4:2:0; lossless at quality 100) for WebM export, from the VP9 bitstream specification | | L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | +| L1 | `psd` | Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit; RGB/CMYK/Gray/Lab) and a minimal writer, from Adobe's published format specification | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `svg` | SVG import (W3C SVG 1.1/2 static subset: shapes, paths, transforms, `use`, CSS, gradients) and rasterisation at any scale | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | -| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | +| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, Photoshop and SVG stills, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | -| L3 | `export` | Render queue encoding: H.264, ProRes, PNG/JPEG/TIFF/EXR sequences, GIF, audio | +| L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | @@ -76,6 +79,14 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +A persistent **disk cache** (`render::disk_cache`, Settings ▸ Media & Disk Cache) backs both +the layer cache and the viewer's RAM preview: layer buffers that were slow to render and every +preview frame are written (LZ4, atomic rename, checksummed) under 128-bit content keys — the +comp's content hash (the comp, what it uses, footage file size and time) plus frame, scale and +view for frames; the layer key salted with the footage fingerprint for layers — so entries +survive restarts and never go stale. LRU eviction keeps it under the size limit; Edit ▸ Purge +empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. + **Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through `EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* diff --git a/docs/parity.md b/docs/parity.md index d259fec2..5d291ca7 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -75,7 +75,7 @@ library groundwork for mask tracking and Warp Stabilizer. What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of -the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, +the new tracking library, a JavaScript scripting object model, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and wasm threads. @@ -84,9 +84,9 @@ wasm threads. | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 85% | 4.2 | multiple output modules, pre-render, WebM, audio-only | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | -| Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | +| Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | 76% | 3.3 | a JavaScript scripting object model | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | | Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | @@ -98,7 +98,7 @@ wasm threads. | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | -| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | +| Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | @@ -125,7 +125,7 @@ Mocha and Cineware are third-party and not counted. 10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. 11. Auto-save, crash recovery, recent projects (in progress). 12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). -13. Lottie (in progress), WebM, SVG and PSD import. +13. ~~Lottie, WebM, SVG and PSD import~~ (landed; a disk cache too). 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7e7616a5..d822a1b9 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,6 +23,7 @@ const LAYERS: &[(&str, u8)] = &[ ("time", 0), ("geom", 0), ("color", 0), + ("vp9enc", 0), ("testkit", 0), ("opusenc", 0), ("raster", 1),