From 0797ab4ed9d076b56a212a809076ef6d9ce659d9 Mon Sep 17 00:00:00 2001 From: JordyUSA <75485565+JordyUSA@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:21:30 +0700 Subject: [PATCH] #598: import AVI files (Motion JPEG, H.264, HEVC, uncompressed; PCM, MP3, MP2, AC-3) AviSource maps AVI streams onto the decoders, opened from bytes or a streaming reader (Media Browser probe): - Video: Motion JPEG, H.264 and HEVC (Annex B, our decoders) and uncompressed BGR24 / BGRX32 / YUY2 / UYVY / I420 / YV12 / NV12 / grey (new RawVideoDecoder). Empty "drop frame" chunks hold the picture before. AVI has no timestamps: H.264 with B-frames is stored in decode order, so the presentation order comes from the pictures' order counts (mxf::avc_presentation_order), keeping the slots the pictures occupy. A writer that kept a fine time base (ffmpeg copying an MP4's 1/600) fills the slots between pictures with empty chunks; the reported frame rate is the one the pictures play at. DivX / Xvid and DV are named and report that they have no decoder; their sound plays. - Audio: PCM (integer and float) from the chunks; MPEG audio and AC-3 frames, which AVI chunks need not follow, found again across chunk boundaries with the new filmcraft_mpegts::frame_bytes. - Motion JPEG frames without Huffman tables (the AVI1 convention) decode with the standard ITU-T T.81 Annex K tables (also for MOV). Tests against ffmpeg's decode of the same files: H.264 with B-frames bit-exact under random access, also copied from a 1/600 MP4; uncompressed video and PCM bit-exact; JPEG and MP3 / MP2 / AC-3 within decoder tolerance; drop frames; DivX; JPEG without tables equal to the same frame with them. Engine: an AVI imports, makes a sequence, plays picture and sound, and probes without reading it whole. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- crates/codecs/Cargo.toml | 1 + crates/codecs/src/avi.rs | 585 ++++++++++++++++++++++++++ crates/codecs/src/avi_tests.rs | 232 ++++++++++ crates/codecs/src/lib.rs | 29 +- crates/codecs/src/tests.rs | 8 +- crates/codecs/src/video.rs | 220 +++++++++- crates/engine/src/avi_import_tests.rs | 58 +++ crates/engine/src/lib.rs | 2 + crates/engine/src/media_pool.rs | 2 +- crates/mpegts/src/es.rs | 14 + crates/mpegts/src/lib.rs | 2 +- 12 files changed, 1146 insertions(+), 10 deletions(-) create mode 100644 crates/codecs/src/avi.rs create mode 100644 crates/codecs/src/avi_tests.rs create mode 100644 crates/engine/src/avi_import_tests.rs diff --git a/README.md b/README.md index fc1acfeb..80b79b0b 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus | **MP3, FLAC, ALAC, Vorbis** | ✓ | | Via the [symphonia](https://github.com/pdeljanov/Symphonia) crate (MPL-2.0) for now, to be replaced by our own | | **MP4 / MOV** | ✓ | ✓ | Fragmented MP4, edit lists, timecode tracks | | **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata | +| **AVI** | ✓ | | AVI 1.0 and OpenDML (files past 1 GB); Motion JPEG, H.264, HEVC and uncompressed RGB / YUV with PCM, MP3, MP2 or AC-3. DivX / Xvid and DV not yet | | **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF | Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows and Linux (opt-in), and H.265 (HEVC Main, 8-bit) export works through it (NVENC on Windows, VideoToolbox on macOS) ([#30](https://github.com/storytold/filmcraft/issues/30)). @@ -301,7 +302,7 @@ A layered Cargo workspace: | Layer | Crates | |---|---| -| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `bitstream` | +| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `avi`, `bitstream` | | Foundations | `time`, `geom`, `color`, `frame`, `media`, `text`, `project`, `audio-dsp` | | Editing and interchange | `edit`, `codecs`, `captions`, `format` (project files), `interchange` | | Rendering and output | `render`, `gpu`, `export` | diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml index 3a490e2c..3faf97da 100644 --- a/crates/codecs/Cargo.toml +++ b/crates/codecs/Cargo.toml @@ -14,6 +14,7 @@ filmcraft-time = { workspace = true } filmcraft-color = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-media = { workspace = true } +filmcraft-avi = { workspace = true } filmcraft-isobmff = { workspace = true } filmcraft-matroska = { workspace = true } filmcraft-mxf = { workspace = true } diff --git a/crates/codecs/src/avi.rs b/crates/codecs/src/avi.rs new file mode 100644 index 00000000..65952442 --- /dev/null +++ b/crates/codecs/src/avi.rs @@ -0,0 +1,585 @@ +//! AVI media source (`filmcraft-avi` demux, GOP-aware video via [`crate::gop`]). +//! +//! Video: Motion JPEG (frames without their own Huffman tables get the standard ones), H.264 and +//! HEVC (Annex B byte streams, our decoders), and uncompressed RGB / YUV ([`RawFormat`]). A frame +//! slot with an empty chunk (a "drop frame") repeats the picture before it. MPEG-4 Part 2 +//! (DivX / Xvid) and DV have no decoder yet: their streams are listed and report the codec. +//! +//! Audio: PCM (integer and float) straight from the chunks; MPEG audio (MP3, MP2) and AC-3 frames, +//! which AVI chunks need not follow, are found again across chunk boundaries +//! ([`filmcraft_mpegts::frame_bytes`]) and decoded as packets. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use filmcraft_avi::{AviFile, BitmapInfo, StreamKind, WaveFormat}; +use filmcraft_frame::{AudioBuffer, VideoFrame}; +use filmcraft_isobmff::{PcmConfig, SampleEntry}; +use filmcraft_media::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, SharedSource, VideoStreamInfo}; +use filmcraft_time::{FrameRate, Tick}; + +use crate::audio::{PacketDecoder, decode_pcm}; +use crate::gop::{GopCache, VideoSamples}; +use crate::video::{RawFormat, RawVideoDecoder, VideoDecoder}; +use crate::{CodecError, make_video_decoder}; + +pub fn sniff(b: &[u8]) -> bool { + filmcraft_avi::sniff(b) +} + +#[derive(Clone, Debug, PartialEq)] +enum VCodec { + Jpeg, + H264, + Hevc, + Raw(RawFormat), + /// No decoder: the codec's name. + Unsupported(String), +} + +fn fourcc_text(f: [u8; 4]) -> String { + let t: String = f.iter().map(|&c| if c.is_ascii_graphic() { c as char } else { '?' }).collect(); + t.trim_end_matches('?').to_string() +} + +/// The decoder for a video stream and its label. +fn video_codec(b: &BitmapInfo) -> (VCodec, String) { + if let Some(raw) = RawFormat::from_avi(b.compression, b.bit_count) { + return (VCodec::Raw(raw), raw.label().into()); + } + let mut f = b.compression; + f.make_ascii_uppercase(); + match &f { + b"MJPG" | b"AVRN" | b"LJPG" | b"JPGL" | b"DMB1" | b"JPEG" | b"MJPA" => (VCodec::Jpeg, "Motion JPEG".into()), + b"H264" | b"X264" | b"AVC1" | b"DAVC" | b"VSSH" => (VCodec::H264, "H.264".into()), + b"HEVC" | b"H265" | b"HEV1" | b"HVC1" | b"X265" => (VCodec::Hevc, "HEVC".into()), + b"DIVX" | b"DX50" | b"XVID" | b"FMP4" | b"MP4V" | b"M4S2" | b"3IV2" | b"DIV3" | b"MP43" | b"MP42" => { + (VCodec::Unsupported("MPEG-4 Part 2 (DivX / Xvid)".into()), "MPEG-4 Part 2 (DivX / Xvid)".into()) + } + b"DVSD" | b"DV25" | b"DV50" | b"DVHD" | b"DVSL" | b"CDVC" | b"DVCP" | b"DVPP" => (VCodec::Unsupported("DV".into()), "DV".into()), + _ => { + let name = fourcc_text(b.compression); + (VCodec::Unsupported(format!("\"{name}\"")), name) + } + } +} + +/// One picture of the video stream: its frame slot (in `scale / rate` units) and chunk. +#[derive(Clone, Copy, Debug)] +struct VFrame { + pts: i64, + chunk: usize, + key: bool, +} + +#[derive(Clone, Debug)] +enum ACodec { + Pcm(PcmConfig), + /// MPEG audio (layer 1-3) or AC-3 frames, found again across chunks. + Framed { + ts: filmcraft_mpegts::Codec, + layer: u8, + }, + Unsupported(String), +} + +fn audio_codec(w: &WaveFormat) -> (ACodec, String) { + let pcm = |float: bool| { + let bits = w.bits_per_sample; + let cfg = PcmConfig { bits, float, big_endian: false, signed: bits > 8, channels: u32::from(w.channels.max(1)), sample_rate: f64::from(w.sample_rate) }; + (ACodec::Pcm(cfg), format!("PCM {bits}-bit{}", if float { " float" } else { "" })) + }; + match w.format_tag { + 1 if matches!(w.bits_per_sample, 8 | 16 | 24 | 32) => pcm(false), + 3 if matches!(w.bits_per_sample, 32 | 64) => pcm(true), + 0x55 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 3 }, "MP3".into()), + 0x50 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 2 }, "MPEG Audio".into()), + 0x2000 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::Ac3, layer: 0 }, "AC-3".into()), + tag => { + let name = format!("audio format 0x{tag:04X}"); + (ACodec::Unsupported(name.clone()), name) + } + } +} + +/// A slice of a chunk: (chunk, offset in it, length). +type Piece = (usize, usize, usize); + +struct AudioState { + decoder: Option, + packets: HashMap>>>, + order: Vec, + last_decoded: Option, +} + +struct AviAudio { + stream: usize, + codec: ACodec, + /// The pieces of packet `k` are `pieces[first[k]..first[k + 1]]`. + pieces: Vec, + first: Vec, + /// Packet start positions in sample frames. + starts: Vec, + state: Mutex, +} + +pub struct AviSource { + info: MediaInfo, + bytes: crate::Src, + file: AviFile, + vstream: Option, + vcodec: VCodec, + /// Pictures in stored (decode) order. + vframes: Vec, + /// `vframes` indices in presentation order (H.264 with B-frames is stored in decode order), + /// and their presentation times. + vorder: Vec, + vorder_pts: Vec, + /// Frame slots of the video stream (where the last picture ends). + vslots: i64, + rate: FrameRate, + bottom_up: bool, + video: GopCache, + audios: Vec, +} + +/// Whether an Annex B access unit holds an H.264 IDR or HEVC IRAP picture. +fn random_access(data: &[u8], hevc: bool) -> bool { + filmcraft_bitstream::annexb_nals(data).iter().any(|n| match n.first() { + Some(h) if hevc => (16..=23).contains(&((h >> 1) & 0x3F)), + Some(h) => h & 0x1F == 5, + None => false, + }) +} + +/// The packets of a framed audio stream: every frame header found in the stream of chunks, and +/// the samples each frame decodes to. Bytes that start no frame are skipped (resynchronisation). +fn split_frames( + file: &AviFile, + bytes: &crate::Src, + s: usize, + ts: &filmcraft_mpegts::Codec, +) -> (Vec, Vec, Vec, Option) { + let Some(st) = file.streams.get(s) else { return Default::default() }; + // chunk start positions in the concatenated stream + let mut at = Vec::with_capacity(st.chunks.len()); + let mut total = 0usize; + for c in &st.chunks { + at.push(total); + total = total.saturating_add(c.size as usize); + } + let locate = |pos: usize| -> Option<(usize, usize)> { + let k = at.partition_point(|&a| a <= pos).checked_sub(1)?; + Some((k, pos - at.get(k)?)) + }; + let (mut pieces, mut first, mut samples) = (Vec::new(), Vec::new(), Vec::new()); + let mut info = None; + // a sliding window over the chunks: `win` holds the stream from `base` + let (mut win, mut base, mut next) = (Vec::::new(), 0usize, 0usize); + let mut pos = 0usize; + while pos < total { + // keep at least a header's worth (and the frame, when we know its length) in the window + while win.len() < (pos - base) + 4096 && next < st.chunks.len() { + win.extend(file.read_chunk(&bytes, s, next).unwrap_or_default()); + next += 1; + } + let Some(here) = win.get(pos - base..) else { break }; + match filmcraft_mpegts::frame_bytes(ts, here) { + Some(n) if pos + n <= total => { + if info.is_none() { + info = filmcraft_mpegts::frame_info(ts, here); + } + let k = filmcraft_mpegts::frame_info(ts, here).map_or(0, |f| f.samples); + first.push(pieces.len()); + let mut p = pos; + while p < pos + n { + let Some((c, off)) = locate(p) else { break }; + let len = (st.chunks.get(c).map_or(0, |ch| ch.size as usize) - off).min(pos + n - p); + if len == 0 { + break; + } + pieces.push((c, off, len)); + p += len; + } + samples.push(k); + pos += n; + } + _ if here.len() < 8 && next >= st.chunks.len() => break, + _ => pos += 1, + } + // drop what is behind + if pos - base > 1 << 16 { + let cut = pos - base; + win.drain(..cut.min(win.len())); + base += cut; + } + } + (pieces, first, samples, info) +} + +impl AviSource { + pub fn open(name: &str, bytes: Arc<[u8]>) -> crate::Result { + Self::open_reader(name, Arc::new(filmcraft_media::reader::MemReader(bytes))) + } + + /// Open from a random-access reader: the index now, frames and audio on demand. + pub fn open_reader(name: &str, reader: filmcraft_media::SharedReader) -> crate::Result { + let bytes = crate::Src(reader); + let file = filmcraft_avi::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?; + let vstream = file.streams.iter().position(|s| s.kind == StreamKind::Video && s.video.is_some() && !s.chunks.is_empty()); + let astreams: Vec = file + .streams + .iter() + .enumerate() + .filter(|(_, s)| s.kind == StreamKind::Audio && s.audio.is_some() && !s.chunks.is_empty()) + .map(|(i, _)| i) + .take(filmcraft_media::MAX_AUDIO_STREAMS) + .collect(); + if vstream.is_none() && astreams.is_empty() { + return Err(CodecError::Unsupported("no playable streams".into())); + } + // video: frame slots, pictures and key frames + let (mut vcodec, mut vframes, mut vslots, mut rate, mut bottom_up, mut video) = + (VCodec::Unsupported(String::new()), Vec::new(), 0i64, FrameRate::new(25, 1), false, None); + if let Some(vs) = vstream.and_then(|i| file.streams.get(i)) { + let b = vs.video.clone().unwrap_or_default(); + let (codec, label) = video_codec(&b); + rate = FrameRate::new(i64::from(vs.rate), i64::from(vs.scale)); + bottom_up = b.height > 0; + let (w, h) = (b.width.unsigned_abs(), b.height.unsigned_abs()); + for (k, c) in vs.chunks.iter().enumerate() { + if c.size == 0 { + continue; + } + let key = match &codec { + VCodec::H264 | VCodec::Hevc if !vs.keyframes_known => { + file.read_chunk(&bytes, vstream.unwrap_or(0), k).is_ok_and(|d| random_access(&d, codec == VCodec::Hevc)) + } + VCodec::H264 | VCodec::Hevc => c.key, + _ => true, + }; + vframes.push(VFrame { pts: i64::from(vs.start) + k as i64, chunk: k, key }); + } + vslots = i64::from(vs.start) + vs.chunks.len() as i64; + // AVI has no timestamps: H.264 with B-frames is stored in decode order, and the + // presentation order comes from the pictures' order counts. A first look at the start + // of the stream decides whether it reorders at all (reading every frame's header is the + // cost of a reordering stream only). + if codec == VCodec::H264 + && let Some(si) = vstream + { + let heads = |n: usize| -> Vec> { + vframes + .iter() + .take(n) + .map(|f| { + file.streams + .get(si) + .and_then(|s| s.chunks.get(f.chunk)) + .and_then(|c| filmcraft_media::reader::read_range(&*bytes.0, c.offset, (c.size as usize).min(8 << 10)).ok()) + .unwrap_or_default() + }) + .collect() + }; + let reorders = |p: &[i64]| p.windows(2).any(|w| w[1] < w[0]); + if crate::mxf::avc_presentation_order(&heads(300)).is_some_and(|p| reorders(&p)) + && let Some(ranks) = crate::mxf::avc_presentation_order(&heads(vframes.len())) + { + // the pictures keep the slots they occupy (empty drop-frame chunks between them + // stay), shown in their presentation order + let mut slots: Vec = vframes.iter().map(|f| f.pts).collect(); + slots.sort_unstable(); + for (f, r) in vframes.iter_mut().zip(ranks) { + if let Some(&p) = usize::try_from(r).ok().and_then(|r| slots.get(r)) { + f.pts = p; + } + } + } + } + let secs = vslots as f64 / rate.as_f64().max(1e-9); + // The rate the pictures play at: a writer that kept a fine time base (ffmpeg copying an + // MP4's 1/600) fills the slots between pictures with empty chunks, every 20th slot + // holding a picture at 30 fps. A regular spacing makes the rate; slots stay the units. + let shown = if vframes.len() > 1 { + let mut gaps: Vec = vframes.windows(2).map(|w| w[1].chunk as i64 - w[0].chunk as i64).filter(|g| *g > 0).take(1000).collect(); + gaps.sort_unstable(); + let gap = gaps.get(gaps.len() / 2).copied().unwrap_or(1).max(1); + let regular = gaps.iter().filter(|g| **g == gap).count() * 10 >= gaps.len() * 9; + if gap > 1 && regular { FrameRate::new(i64::from(vs.rate), i64::from(vs.scale).saturating_mul(gap)) } else { rate } + } else { + rate + }; + let bitrate = (secs > 0.0).then(|| (vs.chunks.iter().map(|c| u64::from(c.size)).sum::() as f64 * 8.0 / secs) as u64); + video = Some(VideoStreamInfo { + width: w, + height: h, + frame_rate: shown, + par: (1, 1), + codec: label, + pixel_format: String::new(), + color: crate::stream_color::resolve(w, h, &[]), + has_alpha: false, + bitrate, + hdr: None, + }); + vcodec = codec; + } + // audio: packets and their start positions + let mut audios = Vec::new(); + let mut audio_streams = Vec::new(); + for &s in &astreams { + let Some(st) = file.streams.get(s) else { continue }; + let w = st.audio.clone().unwrap_or_default(); + let (codec, label) = audio_codec(&w); + let mut sample_rate = w.sample_rate.max(1); + let mut channels = u32::from(w.channels.max(1)); + let (pieces, first, starts) = match &codec { + ACodec::Pcm(cfg) => { + let block = usize::from(w.block_align.max(1)).max((usize::from(cfg.bits).div_ceil(8)) * channels as usize); + let mut t = 0i64; + let mut starts = Vec::with_capacity(st.chunks.len()); + for c in &st.chunks { + starts.push(t); + t = t.saturating_add((c.size as usize / block) as i64); + } + let pieces: Vec = st.chunks.iter().enumerate().map(|(k, c)| (k, 0, c.size as usize)).collect(); + let first = (0..pieces.len()).collect(); + (pieces, first, starts) + } + ACodec::Framed { ts, .. } => { + let (pieces, first, samples, info) = split_frames(&file, &bytes, s, ts); + if let Some(i) = info { + sample_rate = i.sample_rate.max(1); + channels = i.channels.max(1); + } + let mut t = 0i64; + let starts = samples + .iter() + .map(|&k| { + let s = t; + t = t.saturating_add(i64::from(k)); + s + }) + .collect(); + (pieces, first, starts) + } + ACodec::Unsupported(_) => (Vec::new(), Vec::new(), Vec::new()), + }; + // a stream that starts late (dwStart, in scale / rate units) + let delay = (i128::from(st.start) * i128::from(st.scale) * i128::from(sample_rate) / i128::from(st.rate.max(1))) as i64; + let starts: Vec = starts.into_iter().map(|s: i64| s.saturating_add(delay)).collect(); + audio_streams.push(AudioStreamInfo { + sample_rate, + channels, + codec: label, + bits_per_sample: (w.bits_per_sample > 0).then_some(u32::from(w.bits_per_sample)), + }); + let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }); + audios.push(AviAudio { stream: s, codec, pieces, first, starts, state }); + } + // the longer of the picture and the sound + let vdur = if vslots > 0 { rate.tick_of(vslots) } else { Tick::ZERO }; + let adur = audios + .iter() + .zip(&audio_streams) + .filter_map(|(a, i)| { + let last = a.starts.last()?; + let tail = a.starts.len().checked_sub(2).and_then(|k| a.starts.get(k)).map_or(0, |p| last - p); + Some(Tick::from_rational(last + tail, 1, i64::from(i.sample_rate))) + }) + .max() + .unwrap_or_default(); + let info = MediaInfo { + name: name.to_string(), + kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly }, + duration: vdur.max(adur), + video, + audio_streams, + container: "AVI".into(), + start_timecode: None, + file_size: Some(bytes.0.len()), + }; + let mut vorder: Vec = (0..vframes.len()).collect(); + vorder.sort_by_key(|&i| vframes.get(i).map_or(0, |f| f.pts)); + let vorder_pts = vorder.iter().map(|&i| vframes.get(i).map_or(0, |f| f.pts)).collect(); + Ok(Self { info, bytes, file, vstream, vcodec, vframes, vorder, vorder_pts, vslots, rate, bottom_up, video: GopCache::new(None), audios }) + } + + fn read(&self, stream: usize, chunk: usize) -> crate::Result> { + self.file.read_chunk(&self.bytes, stream, chunk).map_err(|e| CodecError::Container(e.to_string())) + } + + /// The bytes of audio packet `k`. + fn packet(&self, a: &AviAudio, k: usize) -> crate::Result> { + let (from, to) = ( + a.first.get(k).copied().ok_or_else(|| CodecError::Container("audio packet out of range".into()))?, + a.first.get(k + 1).copied().unwrap_or(a.pieces.len()), + ); + let mut out = Vec::new(); + for &(c, off, len) in a.pieces.get(from..to).unwrap_or_default() { + let chunk = self.read(a.stream, c)?; + out.extend_from_slice(chunk.get(off..off + len).ok_or_else(|| CodecError::Container("audio chunk shorter than its index".into()))?); + } + Ok(out) + } + + fn audio_packet(&self, a: &AviAudio, st: &mut AudioState, ainfo: &AudioStreamInfo, i: usize) -> crate::Result>>> { + if let Some(p) = st.packets.get(&i) { + return Ok(p.clone()); + } + let data = self.packet(a, i)?; + let decoded = match &a.codec { + ACodec::Pcm(cfg) => decode_pcm(&data, cfg), + ACodec::Unsupported(name) => return Err(CodecError::Unsupported(name.clone())), + ACodec::Framed { ts, layer } => { + if st.decoder.is_none() { + st.decoder = Some(match ts { + filmcraft_mpegts::Codec::Ac3 => PacketDecoder::ac3()?, + _ => PacketDecoder::mpeg_audio(*layer, ainfo.sample_rate)?, + }); + } + let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?; + // non-sequential access: reset and prime with the packet before (the bit reservoir) + if st.last_decoded.is_none_or(|l| l + 1 != i) { + d.reset(); + if let Some(prev) = i.checked_sub(1).and_then(|j| self.packet(a, j).ok()) { + let _ = d.decode(&prev, 0); + } + } + let start = a.starts.get(i).copied().unwrap_or(0).max(0) as u64; + let r = d.decode(&data, start); + st.last_decoded = Some(i); + r.unwrap_or_default() + } + }; + let p = Arc::new(decoded); + st.packets.insert(i, p.clone()); + st.order.push(i); + if st.order.len() > 4096 { + let old = st.order.remove(0); + st.packets.remove(&old); + } + Ok(p) + } +} + +/// The video stream as a [`VideoSamples`] table. +struct AviVideo<'a> { + src: &'a AviSource, + stream: usize, +} + +impl VideoSamples for AviVideo<'_> { + fn count(&self) -> usize { + self.src.vframes.len() + } + fn pts(&self, i: usize) -> i64 { + self.src.vframes.get(i).map_or(0, |f| f.pts) + } + fn sync_before(&self, i: usize) -> usize { + let i = i.min(self.src.vframes.len().saturating_sub(1)); + self.src.vframes.get(..=i).and_then(|f| f.iter().rposition(|f| f.key)).unwrap_or(0) + } + fn sample_at(&self, t: i64) -> Option { + // the picture shown in slot `t`: the last one at or before it (empty chunks repeat it) + if t >= self.src.vslots { + return None; + } + let k = self.src.vorder_pts.partition_point(|&p| p <= t).checked_sub(1)?; + self.src.vorder.get(k).copied() + } + fn read(&self, i: usize) -> crate::Result> { + let f = self.src.vframes.get(i).ok_or_else(|| CodecError::Container("frame out of range".into()))?; + self.src.read(self.stream, f.chunk) + } + fn make_decoder(&self) -> crate::Result> { + let v = self.src.info.video.as_ref().ok_or_else(|| CodecError::Unsupported("no video stream".into()))?; + let (w, h) = (v.width.min(u32::from(u16::MAX)) as u16, v.height.min(u32::from(u16::MAX)) as u16); + match &self.src.vcodec { + VCodec::Jpeg => make_video_decoder(&SampleEntry::jpeg(w, h)), + VCodec::H264 => Ok(Box::new(crate::video::H264Decoder::annexb())), + VCodec::Hevc => Ok(Box::new(crate::video::HevcDecoder::annexb())), + VCodec::Raw(f) => Ok(Box::new(RawVideoDecoder::new(*f, v.width, v.height, self.src.bottom_up)?)), + VCodec::Unsupported(name) => Err(CodecError::Unsupported(format!("no decoder for {name} video"))), + } + } +} + +impl MediaSource for AviSource { + fn info(&self) -> &MediaInfo { + &self.info + } + + fn video_frame(&self, req: FrameRequest) -> Result, MediaError> { + let s = self.vstream.ok_or(MediaError::NoStream("video"))?; + let t = req.time.max(Tick::ZERO); + let slot = |t: Tick| self.rate.frame_at(t); + let late = filmcraft_media::cancel::catch_up().map(|m| slot(t - m)); + Ok(self.video.frame_late(&AviVideo { src: self, stream: s }, slot(t), late)?) + } + + fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result { + self.audio_stream(0, start, frames, sample_rate) + } + + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result { + let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?; + let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?; + if let ACodec::Unsupported(name) = &a.codec { + return Err(MediaError::Unsupported(format!("{name} audio"))); + } + let ch = ainfo.channels.max(1) as usize; + let ratio = f64::from(ainfo.sample_rate) / f64::from(sample_rate.max(1)); + let s0 = (start as f64 * ratio).floor() as i64; + let need = (frames as f64 * ratio).ceil() as i64 + 2; + let mut src: Vec> = vec![vec![0.0; need.max(0) as usize]; ch]; + let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner()); + let mut i = a.starts.partition_point(|&x| x <= s0).saturating_sub(1); + while let Some(&pk_start) = a.starts.get(i) { + if pk_start >= s0 + need { + break; + } + let pk = self.audio_packet(a, &mut st, ainfo, i)?; + for (c, dst) in src.iter_mut().enumerate() { + let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue }; + for (k, v) in chan.iter().enumerate() { + let pos = pk_start + k as i64 - s0; + if let Some(d) = usize::try_from(pos).ok().and_then(|p| dst.get_mut(p)) { + *d = *v; + } + } + } + i += 1; + } + drop(st); + let frac = s0 as f64 - start as f64 * ratio; + let mut out = AudioBuffer::silence(sample_rate, ch, frames); + for (c, o) in out.channels.iter_mut().enumerate() { + let Some(s) = src.get(c) else { continue }; + for (k, d) in o.iter_mut().enumerate() { + let pos = k as f64 * ratio - frac; + let i0 = pos.floor().max(0.0) as usize; + let f = (pos - i0 as f64) as f32; + let a = s.get(i0).copied().unwrap_or(0.0); + let b = s.get(i0 + 1).copied().unwrap_or(a); + *d = a + (b - a) * f; + } + } + Ok(out) + } +} + +pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option> { + if !sniff(&bytes) { + return None; + } + Some(AviSource::open(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into)) +} + +/// [`filmcraft_media::ReaderOpener`] for AVI. +pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedReader) -> Option> { + if !sniff(head) { + return None; + } + Some(AviSource::open_reader(name, reader.clone()).map(|s| Arc::new(s) as SharedSource).map_err(Into::into)) +} diff --git a/crates/codecs/src/avi_tests.rs b/crates/codecs/src/avi_tests.rs new file mode 100644 index 00000000..714eda19 --- /dev/null +++ b/crates/codecs/src/avi_tests.rs @@ -0,0 +1,232 @@ +//! AVI files made by ffmpeg (skipped without it), checked against ffmpeg's decode of the same file: +//! bit-exact where the decoder is (H.264, uncompressed video, PCM), within decoder tolerance where +//! it differs (JPEG, MP3 / MP2 / AC-3). + +use std::path::Path; +use std::process::Command; +use std::sync::Arc; + +use filmcraft_media::FrameRequest; + +use crate::tests::{ffmpeg, fixture_path, reference_yuv, yuv_bytes}; + +const PIC: [&str; 4] = ["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2"]; +const TONE: [&str; 4] = ["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=2"]; + +fn avi(name: &str, video: &[&str], audio: &[&str]) -> Option<(std::path::PathBuf, filmcraft_media::SharedSource)> { + let mut args: Vec<&str> = PIC.to_vec(); + args.extend_from_slice(&TONE); + args.extend_from_slice(video); + args.extend_from_slice(audio); + let path = fixture_path(name, &args)?; + let bytes: Arc<[u8]> = std::fs::read(&path).ok()?.into(); + Some((path.clone(), crate::open_bytes(name, bytes).unwrap_or_else(|e| panic!("{name}: {e}")))) +} + +/// ffmpeg's decode of the first audio stream as interleaved f32 at its own rate. +fn reference_audio(path: &Path) -> Vec { + let out = Command::new(ffmpeg().unwrap()) + .args(["-v", "error", "-i"]) + .arg(path) + .args(["-map", "0:a:0", "-f", "f32le", "-c:a", "pcm_f32le", "-"]) + .output() + .unwrap(); + out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect() +} + +/// The first `n` mono samples of stream 0 as FilmCraft decodes them. +fn ours(src: &filmcraft_media::SharedSource, n: usize) -> Vec { + src.audio(0, n, 48_000).unwrap().channels[0].clone() +} + +fn check_info(name: &str, src: &filmcraft_media::SharedSource, vcodec: &str, acodec: &str) { + let info = src.info(); + assert_eq!(info.container, "AVI", "{name}"); + let v = info.video.as_ref().unwrap(); + assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, vcodec), "{name}"); + assert_eq!(v.frame_rate.as_f64(), 25.0, "{name}"); + let a = &info.audio_streams[0]; + assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (48_000, 1, acodec), "{name}"); + let secs = info.duration.seconds(); + assert!((secs - 2.0).abs() < 0.1, "{name}: duration {secs}"); +} + +#[test] +fn h264_frames_are_bit_exact_with_random_access_and_mp3_plays() { + let Some((path, src)) = + avi("avi_h264_mp3.avi", &["-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p"], &["-c:a", "libmp3lame", "-b:a", "128k"]) + else { + return; + }; + check_info("h264", &src, "H.264", "MP3"); + let reference = reference_yuv(&path, "yuv420p").unwrap(); + let frame = reference.len() / 50; + let rate = src.info().frame_rate(); + for k in [37, 3, 24, 49, 0, 13, 12, 11, 30, 31, 32] { + let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap(); + assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "H.264 frame {k} differs from ffmpeg"); + } + // MP3 (a different decoder): the tone, in time with ffmpeg's decode + let r = reference_audio(&path); + let a = ours(&src, 48_000); + let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200); + // ffmpeg's test tone is at 1/8 of full scale + assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "MP3: tone {rms_ref}, difference {rms_diff}"); + // random access into the middle matches sequential decoding + let mid = src.audio(24_000, 4_800, 48_000).unwrap().channels[0].clone(); + let worst = mid.iter().zip(&a[24_000..28_800]).map(|(x, y)| (x - y).abs()).fold(0.0, f32::max); + assert!(worst < 0.01, "MP3 seek: {worst}"); +} + +fn rms_and_diff(reference: &[f32], ours: &[f32], range: std::ops::Range) -> (f32, f32) { + let n = range.len() as f32; + let rms = (range.clone().map(|i| reference[i].powi(2)).sum::() / n).sqrt(); + let diff = (range.map(|i| (reference[i] - ours[i]).powi(2)).sum::() / n).sqrt(); + (rms, diff) +} + +#[test] +fn mjpeg_and_pcm() { + let Some((path, src)) = avi("avi_mjpeg_pcm.avi", &["-c:v", "mjpeg", "-q:v", "3", "-pix_fmt", "yuvj420p"], &["-c:a", "pcm_s16le"]) else { return }; + check_info("mjpeg", &src, "Motion JPEG", "PCM 16-bit"); + let reference = reference_yuv(&path, "rgba").unwrap(); + let frame = reference.len() / 50; + let rate = src.info().frame_rate(); + for k in [10, 0, 49] { + let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap().to_rgba8(); + let r = &reference[k * frame..(k + 1) * frame]; + let mean = f.iter().zip(r).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::() as f64 / f.len() as f64; + // another JPEG decoder and chroma upsampling than ffmpeg's: close, not identical + assert!(mean < 6.0, "JPEG frame {k}: mean difference {mean}"); + } + // PCM: exactly ffmpeg's samples + let r = reference_audio(&path); + assert_eq!(ours(&src, 48_000), r[..48_000].to_vec()); +} + +#[test] +fn uncompressed_video_is_bit_exact() { + for (name, pix, codec, reference_fmt) in [ + ("avi_bgr24.avi", "bgr24", "Uncompressed RGB 24-bit", "rgba"), + ("avi_yuyv.avi", "yuyv422", "Uncompressed YUV 4:2:2", "yuv422p"), + ("avi_i420.avi", "yuv420p", "Uncompressed YUV 4:2:0", "yuv420p"), + ("avi_nv12.avi", "nv12", "Uncompressed YUV 4:2:0", "yuv420p"), + ] { + let Some((path, src)) = avi(name, &["-c:v", "rawvideo", "-pix_fmt", pix], &["-c:a", "pcm_s16le"]) else { return }; + check_info(name, &src, codec, "PCM 16-bit"); + let reference = reference_yuv(&path, reference_fmt).unwrap(); + let frame = reference.len() / 50; + let rate = src.info().frame_rate(); + for k in [7, 0, 49] { + let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap(); + let got = if reference_fmt == "rgba" { f.to_rgba8() } else { yuv_bytes(&f) }; + assert!(got == reference[k * frame..(k + 1) * frame], "{name}: frame {k} differs from ffmpeg"); + } + } +} + +#[test] +fn mp2_and_ac3_audio() { + for (name, codec, label) in [("avi_mp2.avi", "mp2", "MPEG Audio"), ("avi_ac3.avi", "ac3", "AC-3")] { + let Some((path, src)) = avi(name, &["-c:v", "mjpeg"], &["-c:a", codec, "-b:a", "192k"]) else { return }; + assert_eq!(src.info().audio_streams[0].codec, label, "{name}"); + let r = reference_audio(&path); + let a = ours(&src, 48_000); + let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200); + assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "{name}: tone {rms_ref}, difference {rms_diff}"); + } +} + +/// Dropped frames are stored as empty chunks: their slot shows the picture before. +#[test] +fn drop_frames_hold_the_previous_picture() { + let args = [ + "-f", + "lavfi", + "-i", + "testsrc2=s=160x120:r=25:d=2", + "-vf", + "select='not(eq(n\\,10))'", + "-fps_mode", + "passthrough", + "-c:v", + "rawvideo", + "-pix_fmt", + "bgr24", + ]; + let Some(path) = fixture_path("avi_drop.avi", &args) else { return }; + let src = crate::open_bytes("avi_drop.avi", std::fs::read(&path).unwrap().into()).unwrap(); + let rate = src.info().frame_rate(); + let at = |k: i64| src.video_frame(FrameRequest::full(rate.tick_of(k))).unwrap().to_rgba8(); + assert!(at(10) == at(9), "the dropped slot holds frame 9"); + assert!(at(11) != at(9)); +} + +/// DivX / Xvid has no decoder yet: the file opens, the codec is named, the picture reports it, and +/// the sound plays. +#[test] +fn mpeg4_part2_is_named_and_its_audio_plays() { + let Some((_, src)) = avi("avi_xvid.avi", &["-c:v", "mpeg4", "-vtag", "XVID"], &["-c:a", "pcm_s16le"]) else { return }; + assert_eq!(src.info().video.as_ref().unwrap().codec, "MPEG-4 Part 2 (DivX / Xvid)"); + let e = src.video_frame(FrameRequest::full(filmcraft_time::Tick::ZERO)).unwrap_err().to_string(); + assert!(e.contains("MPEG-4 Part 2"), "{e}"); + assert!(ours(&src, 4_800).iter().any(|s| s.abs() > 0.1)); +} + +/// A Motion JPEG frame without Huffman tables (the AVI1 convention) decodes with the standard ones: +/// exactly as the same frame with them. +#[test] +fn jpeg_without_huffman_tables_uses_the_standard_ones() { + let Some(path) = fixture_path( + "default_huffman.jpg", + &["-f", "lavfi", "-i", "testsrc2=s=160x120", "-frames:v", "1", "-c:v", "mjpeg", "-huffman", "default", "-q:v", "3"], + ) else { + return; + }; + let jpeg = std::fs::read(&path).unwrap(); + // remove every DHT segment + let mut stripped = jpeg[..2].to_vec(); + let mut at = 2; + while at + 4 <= jpeg.len() && jpeg[at] == 0xFF && jpeg[at + 1] != 0xDA { + let len = u16::from_be_bytes([jpeg[at + 2], jpeg[at + 3]]) as usize; + if jpeg[at + 1] != 0xC4 { + stripped.extend_from_slice(&jpeg[at..at + 2 + len]); + } + at += 2 + len; + } + stripped.extend_from_slice(&jpeg[at..]); + assert!(stripped.len() < jpeg.len() && !stripped.windows(2).any(|w| w == [0xFF, 0xC4])); + use crate::video::VideoDecoder; + let mut d = crate::video::MjpegDecoder; + let a = d.decode(&jpeg, 0).unwrap().remove(0).frame.to_rgba8(); + let b = d.decode(&stripped, 0).unwrap().remove(0).frame.to_rgba8(); + assert!(a == b); + // the inserted segment is 418 bytes and well-formed: 4 tables of 12, 12, 162, 162 codes + let with = crate::video::with_default_huffman(&stripped); + assert_eq!(with.len(), stripped.len() + 420); +} + +/// H.264 with B-frames copied from an MP4 whose time base is 1/600 (ffmpeg keeps it): 600 frame +/// slots a second, a picture in every 24th at 25 fps, empty drop chunks between. The pictures stay +/// at their times in presentation order, and the stream reports the rate they play at. +#[test] +fn copied_h264_keeps_its_timing_with_a_fine_time_base() { + let mp4 = + ["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2", "-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p", "-video_track_timescale", "600"]; + let Some(src_mp4) = fixture_path("avi_copy_source.mp4", &mp4) else { return }; + let src_path = src_mp4.to_string_lossy().into_owned(); + let Some(path) = fixture_path("avi_copy_600.avi", &["-i", &src_path, "-c:v", "copy", "-bsf:v", "h264_mp4toannexb", "-f", "avi"]) else { return }; + let src = crate::open_bytes("avi_copy_600.avi", std::fs::read(&path).unwrap().into()).unwrap(); + let v = src.info().video.clone().unwrap(); + assert_eq!(v.frame_rate.as_f64(), 25.0, "the rate the pictures play at, not the 600 slots"); + let reference = reference_yuv(&path, "yuv420p").unwrap(); + let frame = reference.len() / 50; + for k in [37usize, 3, 24, 49, 0, 13, 12, 11, 30] { + let at = filmcraft_time::Tick::from_rational(k as i64, 1, 25); + let f = src.video_frame(FrameRequest::full(at)).unwrap(); + assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "picture {k} at {:.2} s differs from ffmpeg", k as f64 / 25.0); + } + // between pictures, the one before holds + let mid = src.video_frame(FrameRequest::full(filmcraft_time::Tick::from_rational(1, 1, 50))).unwrap(); + assert!(yuv_bytes(&mid) == reference[0..frame]); +} diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs index a31da3a7..b728820f 100644 --- a/crates/codecs/src/lib.rs +++ b/crates/codecs/src/lib.rs @@ -13,6 +13,8 @@ //! - [`MpegSource`]: MPEG-2 transport streams (`.ts`, `.m2ts`, `.mts`), program streams (`.mpg`, //! `.vob`, `.mod`) and MPEG-1/2 video elementary streams: MPEG-1/2, H.264 and HEVC video; MPEG //! audio, AAC (ADTS / LATM), AC-3 and LPCM. +//! - [`AviSource`]: AVI (OpenDML included): Motion JPEG, H.264, HEVC and uncompressed video; PCM, +//! MP3 / MP2 and AC-3 audio. //! - [`AudioFileSource`]: standalone compressed audio files (MP3, FLAC, AIFF, …). //! - [`openers`]: the openers to register with the engine's media pool. @@ -20,6 +22,7 @@ pub mod apv; pub mod audio; +pub mod avi; pub mod gop; pub mod hw; mod hw_frame; @@ -35,6 +38,7 @@ use std::sync::{Arc, RwLock}; pub use apv::ApvSource; pub use audio::AudioFileSource; +pub use avi::AviSource; pub use gop::{FRAME_BUDGET, GopStats, cached_bytes, gop_stats, live_decoders}; pub use mkv::MkvSource; pub use mp4::Mp4Source; @@ -124,12 +128,22 @@ pub fn software_video_decoder(entry: &filmcraft_isobmff::SampleEntry) -> Result< /// Openers for the engine's media pool (MP4/MOV, Matroska/WebM, MXF, Ogg Opus/Vorbis, MPEG TS/PS and /// MPEG-1/2 video elementary streams, APV raw bitstreams, standalone audio). pub fn openers() -> Vec { - vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, audio::opener] + vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, avi::opener, audio::opener] } fn reader_registry() -> &'static RwLock> { static R: std::sync::OnceLock>> = std::sync::OnceLock::new(); - R.get_or_init(|| RwLock::new(vec![mp4::reader_opener, mkv::reader_opener, mxf::reader_opener, ogg::reader_opener, mpeg::reader_opener, apv::reader_opener])) + R.get_or_init(|| { + RwLock::new(vec![ + mp4::reader_opener, + mkv::reader_opener, + mxf::reader_opener, + ogg::reader_opener, + mpeg::reader_opener, + apv::reader_opener, + avi::reader_opener, + ]) + }) } /// Openers that read containers through a [`filmcraft_media::ByteReader`] (index now, samples on @@ -160,6 +174,15 @@ impl filmcraft_isobmff::ByteSource for Src { } } +impl filmcraft_avi::ByteSource for Src { + fn len(&self) -> u64 { + self.0.len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> { + self.0.read_at(offset, buf) + } +} + impl filmcraft_matroska::ByteSource for Src { fn len(&self) -> u64 { self.0.len() @@ -177,6 +200,8 @@ pub fn open_bytes(name: &str, bytes: Arc<[u8]>) -> std::result::Result Option<&'static str> { +pub(crate) fn ffmpeg() -> Option<&'static str> { ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].into_iter().find(|p| std::path::Path::new(p).exists()) } @@ -231,13 +231,13 @@ fn matroska_prores_pcm() { /// Path of a fixture under `target/fixtures/codecs/`, generated with ffmpeg when missing /// (`None` when ffmpeg or the encoder is unavailable). -fn fixture_path(name: &str, args: &[&str]) -> Option { +pub(crate) fn fixture_path(name: &str, args: &[&str]) -> Option { fixture(name, args)?; Some(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/codecs").join(name)) } /// ffmpeg's decode of `src` as raw planar video (the oracle). -fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option> { +pub(crate) fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option> { let name = format!("{}.{pix_fmt}.yuv", src.file_name()?.to_str()?); let path = src.to_str()?; let b = fixture(&name, &["-i", path, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", pix_fmt])?; @@ -245,7 +245,7 @@ fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option> { } /// Planes of a decoded YUV frame as little-endian bytes (ffmpeg rawvideo layout). -fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec { +pub(crate) fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec { match &f.data { filmcraft_frame::PixelData::Yuv8 { planes, .. } => planes.iter().flat_map(|p| p.iter().copied()).collect(), filmcraft_frame::PixelData::Yuv16 { planes, .. } => planes.iter().flat_map(|p| p.iter().flat_map(|s| s.to_le_bytes())).collect(), diff --git a/crates/codecs/src/video.rs b/crates/codecs/src/video.rs index 86b21b7d..8b9c3196 100644 --- a/crates/codecs/src/video.rs +++ b/crates/codecs/src/video.rs @@ -178,13 +178,69 @@ pub fn hvcc_length_size_and_tid(hvcc: &[u8]) -> (usize, Option) { /// A factory returns `None` when it does not handle the entry. pub type VideoDecoderFactory = fn(&SampleEntry) -> Option>>; +/// The typical Huffman tables of ITU-T T.81 Annex K.3 as one DHT segment (luminance and +/// chrominance, DC and AC): what a Motion JPEG frame without its own tables is coded with (AVI's +/// "MJPG", the AVI1 convention of capture cards and cameras). +const DEFAULT_DHT: &[u8] = &[ + 0xFF, 0xC4, 0x01, 0xA2, // + 0x00, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, // + 0x01, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, // + 0x10, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7D, // + 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1, 0x08, // + 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, // + 0x29, 0x2A, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, // + 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, // + 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, // + 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xE1, 0xE2, // + 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, // + 0x11, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77, // + 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, // + 0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62, 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25, 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, // + 0x27, 0x28, 0x29, 0x2A, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, // + 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, // + 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, // + 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, // + 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, +]; + +/// A JPEG with the [`DEFAULT_DHT`] inserted before its first scan when it defines no Huffman +/// table of its own (else unchanged, borrowed). +pub fn with_default_huffman(jpeg: &[u8]) -> std::borrow::Cow<'_, [u8]> { + use std::borrow::Cow; + if jpeg.get(..2) != Some(&[0xFF, 0xD8]) { + return Cow::Borrowed(jpeg); + } + // walk the marker segments up to the start of scan + let mut at = 2usize; + while let (Some(&0xFF), Some(&m)) = (jpeg.get(at), jpeg.get(at + 1)) { + match m { + 0xFF => at += 1, // fill byte + 0xC4 => return Cow::Borrowed(jpeg), + 0xDA => { + let mut out = Vec::with_capacity(jpeg.len() + DEFAULT_DHT.len()); + out.extend_from_slice(jpeg.get(..at).unwrap_or_default()); + out.extend_from_slice(DEFAULT_DHT); + out.extend_from_slice(jpeg.get(at..).unwrap_or_default()); + return Cow::Owned(out); + } + 0xD0..=0xD9 | 0x01 => at += 2, + _ => { + let Some(len) = jpeg.get(at + 2..at + 4).map(|b| usize::from(u16::from_be_bytes([b[0], b[1]]))) else { break }; + at += 2 + len; + } + } + } + Cow::Borrowed(jpeg) +} + /// Motion-JPEG / Photo-JPEG (each sample is a complete JPEG). pub struct MjpegDecoder; impl VideoDecoder for MjpegDecoder { fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + let sample = with_default_huffman(sample); // mjpa samples may contain two fields; decode the first image (field-merging lands with interlace support). - let img = image::load_from_memory_with_format(sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?; + let img = image::load_from_memory_with_format(&sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?; let rgba = img.to_rgba8(); let (w, h) = rgba.dimensions(); Ok(vec![DecodedFrame { pts, frame: VideoFrame::rgba8(w, h, rgba.into_raw()), draft: false }]) @@ -1017,3 +1073,165 @@ mod disposable_tests { assert!(!hevc_disposable(&sample(&[&nal(0, 1)]), 4, None)); } } + +/// Uncompressed video layouts (AVI `BI_RGB` and YUV FourCCs). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RawFormat { + /// 24-bit B, G, R; rows padded to 4 bytes. + Bgr24, + /// 32-bit B, G, R, X. + Bgrx32, + /// Packed 4:2:2: Y0 U Y1 V (`YUY2`, `YUYV`). + Yuy2, + /// Packed 4:2:2: U Y0 V Y1 (`UYVY`, `2vuy`). + Uyvy, + /// Planar 4:2:0: Y, U, V (`I420`, `IYUV`). + I420, + /// Planar 4:2:0: Y, V, U (`YV12`). + Yv12, + /// Y plane, interleaved U V at half resolution (`NV12`). + Nv12, + /// Luma only (`Y800`, `GREY`, `Y8 `). + Gray8, +} + +impl RawFormat { + /// The layout of an AVI video stream: `BI_RGB` with its bit count, or a YUV FourCC. + pub fn from_avi(compression: [u8; 4], bit_count: u16) -> Option { + if matches!(u32::from_le_bytes(compression), 0 | 3) { + return match bit_count { + 24 => Some(RawFormat::Bgr24), + 32 => Some(RawFormat::Bgrx32), + _ => None, + }; + } + let mut upper = compression; + upper.make_ascii_uppercase(); + Some(match &upper { + b"YUY2" | b"YUYV" | b"YUNV" | b"V422" => RawFormat::Yuy2, + b"UYVY" | b"2VUY" | b"UYNV" | b"HDYC" => RawFormat::Uyvy, + b"I420" | b"IYUV" => RawFormat::I420, + b"YV12" => RawFormat::Yv12, + b"NV12" => RawFormat::Nv12, + b"Y800" | b"GREY" | b"GRAY" | b"Y8 " => RawFormat::Gray8, + _ => return None, + }) + } + + pub fn label(self) -> &'static str { + match self { + RawFormat::Bgr24 => "Uncompressed RGB 24-bit", + RawFormat::Bgrx32 => "Uncompressed RGB 32-bit", + RawFormat::Yuy2 | RawFormat::Uyvy => "Uncompressed YUV 4:2:2", + RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 => "Uncompressed YUV 4:2:0", + RawFormat::Gray8 => "Uncompressed greyscale", + } + } +} + +/// Uncompressed frames (each sample one picture of a [`RawFormat`]). +pub struct RawVideoDecoder { + format: RawFormat, + width: u32, + height: u32, + /// RGB rows are stored bottom-up (a positive `BITMAPINFOHEADER` height). + bottom_up: bool, +} + +impl RawVideoDecoder { + pub fn new(format: RawFormat, width: u32, height: u32, bottom_up: bool) -> Result { + if width == 0 || height == 0 || width > 16_384 || height > 16_384 { + return Err(CodecError::Unsupported(format!("{width}x{height} uncompressed video"))); + } + Ok(Self { format, width, height, bottom_up }) + } + + fn yuv(&self, planes: [Vec; 3], chroma: filmcraft_frame::Chroma) -> VideoFrame { + use std::sync::Arc; + let [y, u, v] = planes; + VideoFrame { + width: self.width, + height: self.height, + data: filmcraft_frame::PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma, alpha: None }, + color: vui_color(self.width, self.height, 2, 2, false), + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + } + } + + fn picture(&self, s: &[u8]) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let short = || CodecError::Decode(format!("{} frame of {} bytes is too short for {w}x{h}", self.format.label(), s.len())); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + Ok(match self.format { + RawFormat::Bgr24 | RawFormat::Bgrx32 => { + let bpp = if self.format == RawFormat::Bgr24 { 3 } else { 4 }; + let stride = (w * bpp).div_ceil(4) * 4; + if s.len() < stride * h { + return Err(short()); + } + let mut rgba = Vec::with_capacity(w * h * 4); + for row in 0..h { + let src = if self.bottom_up { h - 1 - row } else { row }; + for px in s.get(src * stride..src * stride + w * bpp).unwrap_or_default().chunks_exact(bpp) { + rgba.extend_from_slice(&[px[2], px[1], px[0], 255]); + } + } + VideoFrame::rgba8(self.width, self.height, rgba) + } + RawFormat::Yuy2 | RawFormat::Uyvy => { + let stride = cw * 4; + if s.len() < stride * h { + return Err(short()); + } + let (yo, uo, vo, y2) = if self.format == RawFormat::Yuy2 { (0, 1, 3, 2) } else { (1, 0, 2, 3) }; + let (mut y, mut u, mut v) = (Vec::with_capacity(w * h), Vec::with_capacity(cw * h), Vec::with_capacity(cw * h)); + for row in s.chunks_exact(stride).take(h) { + for (k, q) in row.as_chunks::<4>().0.iter().enumerate() { + y.push(q[yo]); + if 2 * k + 1 < w { + y.push(q[y2]); + } + u.push(q[uo]); + v.push(q[vo]); + } + } + self.yuv([y, u, v], filmcraft_frame::Chroma::C422) + } + RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 | RawFormat::Gray8 => { + let luma = w * h; + let need = if self.format == RawFormat::Gray8 { luma } else { luma + 2 * cw * ch }; + if s.len() < need { + return Err(short()); + } + let y = s.get(..luma).unwrap_or_default().to_vec(); + let (u, v) = match self.format { + RawFormat::Gray8 => (vec![128; cw * ch], vec![128; cw * ch]), + RawFormat::Nv12 => s.get(luma..luma + 2 * cw * ch).unwrap_or_default().as_chunks::<2>().0.iter().map(|p| (p[0], p[1])).unzip(), + _ => { + let a = s.get(luma..luma + cw * ch).unwrap_or_default().to_vec(); + let b = s.get(luma + cw * ch..luma + 2 * cw * ch).unwrap_or_default().to_vec(); + if self.format == RawFormat::Yv12 { (b, a) } else { (a, b) } + } + }; + self.yuv([y, u, v], filmcraft_frame::Chroma::C420) + } + }) + } +} + +impl VideoDecoder for RawVideoDecoder { + fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + Ok(vec![DecodedFrame { pts, frame: self.picture(sample)?, draft: false }]) + } + fn flush(&mut self) -> Vec { + Vec::new() + } + fn reset(&mut self) {} + fn name(&self) -> &str { + self.format.label() + } + fn intra_only(&self) -> bool { + true + } +} diff --git a/crates/engine/src/avi_import_tests.rs b/crates/engine/src/avi_import_tests.rs new file mode 100644 index 00000000..43d8a301 --- /dev/null +++ b/crates/engine/src/avi_import_tests.rs @@ -0,0 +1,58 @@ +//! #598: AVI files import (ffmpeg-made, skipped without it): the Project item describes the +//! streams, a sequence made from it plays picture and sound, and the Media Browser's streaming +//! probe opens it without reading it whole. + +use serde_json::json; + +use crate::Session; +use filmcraft_project::ItemId; + +fn fixture() -> Option { + let ffmpeg = filmcraft_testkit::oracle::ffmpeg_or_skip("avi import")?; + let path = filmcraft_testkit::fixtures_dir("engine/avi").join("h264_mp3.avi"); + filmcraft_testkit::fixtures::generate(&path, |tmp| { + std::process::Command::new(&ffmpeg) + .args([ + "-y", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x120:rate=25:duration=2", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=2", + ]) + .args(["-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p", "-c:a", "libmp3lame", "-f", "avi"]) + .arg(tmp) + .status() + .is_ok_and(|s| s.success()) + }) +} + +#[test] +fn avi_imports_plays_and_probes() { + let Some(path) = fixture() else { return }; + let mut s = Session::default(); + let r = s.execute("file.import", json!({"paths": [path.to_string_lossy()]})).unwrap(); + assert!(r["errors"].as_array().is_none_or(|e| e.is_empty()), "{r}"); + let item = ItemId(r["items"][0].as_u64().unwrap()); + let info = s.project.item(item).unwrap().as_media().unwrap().info.clone(); + assert_eq!(info.container, "AVI"); + let v = info.video.as_ref().unwrap(); + assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, "H.264")); + assert_eq!(info.audio_streams.first().map(|a| a.codec.as_str()), Some("MP3")); + assert!((info.duration.seconds() - 2.0).abs() < 0.1, "{}", info.duration.seconds()); + // a sequence from the clip has its picture and sound + s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap(); + let q = s.active_sequence().unwrap().clone(); + assert!(!q.video_tracks[0].items.is_empty() && !q.audio_tracks[0].items.is_empty()); + let src = s.source(item).unwrap(); + assert!(src.video_frame(filmcraft_media::FrameRequest::full(filmcraft_time::Tick(filmcraft_time::TICKS_PER_SECOND))).is_ok()); + assert!(src.audio(48_000, 4_800, 48_000).unwrap().channels[0].iter().any(|v| v.abs() > 0.05)); + // the Media Browser's probe streams it (index now, frames on demand) + let probed = s.media.probe_file(&path.to_string_lossy(), &*s.services).unwrap(); + assert_eq!(probed.info().container, "AVI"); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index eea55b32..96ae9761 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1046,6 +1046,8 @@ mod audio_placement_tests; #[cfg(test)] mod autosave_tests; #[cfg(test)] +mod avi_import_tests; +#[cfg(test)] mod clip_ops_tests; #[cfg(test)] mod color_tests; diff --git a/crates/engine/src/media_pool.rs b/crates/engine/src/media_pool.rs index f82aa78b..06147b64 100644 --- a/crates/engine/src/media_pool.rs +++ b/crates/engine/src/media_pool.rs @@ -54,7 +54,7 @@ impl Default for MediaPool { } /// Largest file [`MediaPool::probe_file`] reads whole when no streaming reader takes it (WAV, -/// MP3 and other byte-opened formats, or an unsupported container such as AVI). +/// MP3 and other byte-opened formats, or an unsupported container). pub const PROBE_WHOLE_FILE_MAX: u64 = 64 * 1024 * 1024; /// Cache key of a media clip's full-resolution reference ("" matches anything: sources inserted diff --git a/crates/mpegts/src/es.rs b/crates/mpegts/src/es.rs index 7d567969..97b42d96 100644 --- a/crates/mpegts/src/es.rs +++ b/crates/mpegts/src/es.rs @@ -175,6 +175,20 @@ pub struct FrameInfo { const ADTS_RATES: [u32; 13] = [96_000, 88_200, 64_000, 48_000, 44_100, 32_000, 24_000, 22_050, 16_000, 12_000, 11_025, 8_000, 7_350]; /// Parse the frame header at the start of `b` (MPEG audio, ADTS, AC-3, E-AC-3). +/// Bytes of the audio frame whose header starts `b` (MPEG audio, ADTS, LATM, AC-3, E-AC-3), or +/// `None` without a valid header there. For containers that store frames in chunks that need not +/// follow them (AVI): the frames are found again with it. +pub fn frame_bytes(codec: &Codec, b: &[u8]) -> Option { + let sync = match codec { + Codec::MpegAudio => AudioSync::Mpa, + Codec::AacAdts => AudioSync::Adts, + Codec::AacLatm => AudioSync::Loas, + Codec::Ac3 | Codec::Eac3 => AudioSync::Ac3, + _ => return None, + }; + frame_len(sync, b) +} + pub fn frame_info(codec: &Codec, b: &[u8]) -> Option { match codec { Codec::MpegAudio => { diff --git a/crates/mpegts/src/lib.rs b/crates/mpegts/src/lib.rs index 074d3951..4330ebcb 100644 --- a/crates/mpegts/src/lib.rs +++ b/crates/mpegts/src/lib.rs @@ -21,7 +21,7 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] mod es; -pub use es::{FrameInfo, frame_info}; +pub use es::{FrameInfo, frame_bytes, frame_info}; mod pes; mod ps; mod ts;