diff --git a/Cargo.lock b/Cargo.lock index 3ae70732..d663c320 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1767,6 +1767,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-avi" +version = "0.6.0" +dependencies = [ + "filmcraft-testkit", +] + [[package]] name = "filmcraft-bitstream" version = "0.6.0" @@ -1821,6 +1828,7 @@ dependencies = [ "filmcraft-ac3", "filmcraft-apv", "filmcraft-av1", + "filmcraft-avi", "filmcraft-bitstream", "filmcraft-color", "filmcraft-dnx", diff --git a/Cargo.toml b/Cargo.toml index a0daa93e..b027c40c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ filmcraft-geom = { path = "crates/geom" } filmcraft-color = { path = "crates/color" } filmcraft-bitstream = { path = "crates/bitstream" } filmcraft-isobmff = { path = "crates/isobmff" } +filmcraft-avi = { path = "crates/avi" } filmcraft-matroska = { path = "crates/matroska" } filmcraft-mxf = { path = "crates/mxf" } filmcraft-cfb = { path = "crates/cfb" } diff --git a/README.md b/README.md index 0ce67cb3..6d1c35e7 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus | **MP3, FLAC, ALAC, Vorbis** | ✓ | | Via the [symphonia](https://github.com/pdeljanov/Symphonia) crate (MPL-2.0) for now, to be replaced by our own | | **MP4 / MOV** | ✓ | ✓ | Fragmented MP4, edit lists, timecode tracks | | **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata | +| **AVI** | ✓ | | AVI 1.0 and OpenDML (files past 1 GB); Motion JPEG, H.264, HEVC and uncompressed RGB / YUV with PCM, MP3, MP2 or AC-3. DivX / Xvid and DV not yet | | **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF | Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows and Linux (opt-in), and H.265 (HEVC Main, 8-bit) export works through it (NVENC on Windows, VideoToolbox on macOS) ([#30](https://github.com/storytold/filmcraft/issues/30)). @@ -304,7 +305,7 @@ A layered Cargo workspace: | Layer | Crates | |---|---| -| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `bitstream` | +| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `avi`, `bitstream` | | Foundations | `time`, `geom`, `color`, `frame`, `media`, `text`, `project`, `audio-dsp` | | Editing and interchange | `edit`, `codecs`, `captions`, `format` (project files), `interchange` | | Rendering and output | `render`, `gpu`, `export` | diff --git a/crates/avi/Cargo.toml b/crates/avi/Cargo.toml new file mode 100644 index 00000000..519c098d --- /dev/null +++ b/crates/avi/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "filmcraft-avi" +description = "Clean-room AVI (RIFF) and OpenDML demuxer" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] + +[dev-dependencies] +filmcraft-testkit = { workspace = true } diff --git a/crates/avi/README.md b/crates/avi/README.md new file mode 100644 index 00000000..d10ba02f --- /dev/null +++ b/crates/avi/README.md @@ -0,0 +1,43 @@ +# filmcraft-avi + +A clean-room AVI demuxer, written from Microsoft's published AVI RIFF file reference and the +OpenDML AVI File Format Extensions 1.02. Layer L0: no dependencies beyond `std`, no `unsafe`, builds +for `wasm32-unknown-unknown`. No FFmpeg, libavformat or other AVI implementation was read; ffmpeg +and ffprobe are used only as external **test oracles**. + +## API + +```rust +use filmcraft_avi::{open, StreamKind}; + +let file = open(&bytes)?; // any ByteSource: a slice, a file, a web Blob… +for (i, s) in file.streams.iter().enumerate() { + match s.kind { + StreamKind::Video => println!("{:?} {} fps", s.video.as_ref().map(|b| b.compression), s.rate as f64 / s.scale as f64), + StreamKind::Audio => println!("format 0x{:04x}", s.audio.as_ref().map_or(0, |w| w.format_tag)), + _ => {} + } + let first = file.read_chunk(&bytes, i, 0)?; // frame data, on demand +} +``` + +## What it reads + +| Part | | +|---|---| +| Headers | `avih`; per stream `strh`, `strf` (`BITMAPINFOHEADER`, `WAVEFORMATEX` / `WAVEFORMATEXTENSIBLE`), `strn` | +| Index | OpenDML `indx` super index → `ix##` standard indexes (field indexes too), else `idx1` (offsets relative to `movi` or absolute), else a scan of the `movi` lists | +| Large files | `RIFF AVIX` extensions: through the OpenDML index, or scanned after `idx1`, which only covers the first RIFF | +| Key frames | from the index; a scan marks every chunk and says so (`keyframes_known`) | +| Damage | sizes clamped to the file (a cut-off recording opens with what is there), `rec ` lists nested at most 3 deep, tables bounded by what the file could hold | + +`filmcraft-codecs` maps the streams onto decoders (`AviSource`): Motion JPEG, H.264, HEVC and +uncompressed RGB / YUV video; PCM, MP3, MP2 and AC-3 audio. + +## Tests + +`cargo test -p filmcraft-avi`: hand-built files for every index path (`idx1` relative and absolute, +`rec ` lists, OpenDML standard indexes, a scan), truncation, and every single-byte mutation and +truncation of them (never a panic). `tests/ffmpeg_oracle.rs` checks ffmpeg-made files (Motion JPEG, +H.264 with B-frames, BGR24, YUY2; PCM and MP3) against ffprobe; the ignored `opendml_file_past_1_gb` +checks a 1.16 GB OpenDML file with `AVIX` extensions. diff --git a/crates/avi/src/lib.rs b/crates/avi/src/lib.rs new file mode 100644 index 00000000..0294fdfe --- /dev/null +++ b/crates/avi/src/lib.rs @@ -0,0 +1,542 @@ +//! Clean-room AVI demuxer, written from Microsoft's published AVI RIFF file reference and the +//! OpenDML AVI File Format Extensions 1.02. +//! +//! [`open`] reads the `hdrl` header list (`avih`, and per stream `strh` / `strf` / `strn` / +//! `indx`) of the first `RIFF AVI ` and builds a per-stream table of [`Chunk`]s from the best index +//! the file has: the OpenDML super index (`indx` → `ix##` standard indexes, which also cover the +//! `RIFF AVIX` extensions of files past 1 GB), else the AVI 1.0 `idx1` (plus a scan of any `AVIX` +//! extension it cannot see), else a scan of the `movi` lists. Frame data is read on demand with +//! [`AviFile::read_chunk`]. Truncated files (a recording cut off) open with what is there. +//! +//! Every size and count comes from the file and is treated as hostile: reads are bounded by the +//! file length, tables by the number of chunks the file could hold, and nesting by a fixed depth. +//! +//! Layer L0: no dependencies beyond `std`, no `unsafe`, builds for `wasm32-unknown-unknown`. + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +mod source; +#[cfg(test)] +mod tests; + +pub use source::ByteSource; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Error { + /// Not an AVI file (no `RIFF` … `AVI ` header). + NotAvi, + /// A read failed. + Io(String), + /// Structurally invalid data. + Invalid(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::NotAvi => f.write_str("not an AVI file"), + Error::Io(e) => write!(f, "I/O error: {e}"), + Error::Invalid(e) => write!(f, "invalid AVI: {e}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Whether `head` starts like an AVI file: `RIFF` `AVI `. +pub fn sniff(head: &[u8]) -> bool { + head.get(..4) == Some(b"RIFF") && head.get(8..12) == Some(b"AVI ") +} + +/// `avih`: the main header (frame period, flags, frame count, size). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MainHeader { + pub micro_sec_per_frame: u32, + pub flags: u32, + pub total_frames: u32, + pub streams: u32, + pub width: u32, + pub height: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StreamKind { + Video, + Audio, + /// Text, MIDI or anything else (`fccType`). + Other([u8; 4]), +} + +/// `strf` of a video stream: a `BITMAPINFOHEADER`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BitmapInfo { + pub width: i32, + /// Positive: rows stored bottom-up (uncompressed RGB); negative: top-down. + pub height: i32, + pub bit_count: u16, + /// `biCompression` as four bytes: a FourCC (`MJPG`, `H264`, `YUY2`…) or 0 / 3 (`BI_RGB`, + /// `BI_BITFIELDS`) in little-endian. + pub compression: [u8; 4], + pub size_image: u32, + /// Bytes after the 40-byte header (codec setup, palette). + pub extra: Vec, +} + +impl BitmapInfo { + /// Uncompressed RGB (`BI_RGB` / `BI_BITFIELDS`). + pub fn is_rgb(&self) -> bool { + matches!(u32::from_le_bytes(self.compression), 0 | 3) + } +} + +/// `strf` of an audio stream: a `WAVEFORMATEX` (`WAVEFORMATEXTENSIBLE` is resolved to its sub-format). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct WaveFormat { + /// 1 PCM, 3 IEEE float, 0x50 MPEG audio, 0x55 MP3, 0x2000 AC-3, 0xFF AAC… + pub format_tag: u16, + pub channels: u16, + pub sample_rate: u32, + pub avg_bytes_per_sec: u32, + pub block_align: u16, + pub bits_per_sample: u16, + /// `cbSize` bytes after the 18-byte header. + pub extra: Vec, +} + +/// One stored frame or audio chunk. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Chunk { + /// Offset of the data (after the 8-byte chunk header). + pub offset: u64, + pub size: u32, + /// A key frame (always true for audio and when the index does not say). + pub key: bool, +} + +/// Where the chunk tables came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IndexSource { + /// OpenDML `indx` super index and `ix##` standard indexes. + OpenDml, + /// The AVI 1.0 `idx1` (plus a scan of the `AVIX` extensions). + Idx1, + /// No usable index: the `movi` lists were scanned (key frames unknown). + Scan, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Stream { + pub kind: StreamKind, + /// `fccHandler`. + pub handler: [u8; 4], + /// Time base: `rate / scale` units per second (frames per second for video). + pub scale: u32, + pub rate: u32, + /// Start time in `scale / rate` units. + pub start: u32, + /// Length in `scale / rate` units. + pub length: u32, + /// Bytes per sample for fixed-size samples (PCM block align), 0 when samples vary. + pub sample_size: u32, + pub video: Option, + pub audio: Option, + /// `strn`. + pub name: Option, + pub chunks: Vec, + /// Whether [`Chunk::key`] comes from an index (a scan cannot tell). + pub keyframes_known: bool, + /// The OpenDML super index entries (offsets of the `ix##` chunks). + super_index: Vec, +} + +impl Stream { + /// Stream index of a `movi` chunk id (`00dc`, `01wb`…): its first two characters. + fn number(id: &[u8; 4]) -> Option { + let d = |b: u8| (b as char).to_digit(16).map(|v| v as usize); + Some(d(id[0])? * 16 + d(id[1])?) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AviFile { + pub header: MainHeader, + pub streams: Vec, + pub index: IndexSource, +} + +/// Options for [`open_with`] (tests reach the fallback indexes with them). +#[derive(Clone, Copy, Debug, Default)] +pub struct OpenOptions { + pub ignore_odml: bool, + pub ignore_idx1: bool, +} + +/// Headers larger than this are refused (a real `hdrl` is a few kilobytes). +const MAX_HEADER: u64 = 16 << 20; +/// Streams beyond this are ignored. +const MAX_STREAMS: usize = 64; +/// `LIST rec ` inside `movi` nests once; deeper is refused. +const MAX_DEPTH: u32 = 3; + +fn io(e: std::io::Error) -> Error { + Error::Io(e.to_string()) +} + +fn read(src: &S, at: u64, n: u64) -> Result> { + let end = at.checked_add(n).filter(|e| *e <= src.len()).ok_or_else(|| Error::Invalid(format!("{n} bytes at {at} run past the end")))?; + let len = usize::try_from(end - at).map_err(|_| Error::Invalid("chunk too large".into()))?; + let mut buf = vec![0; len]; + src.read_at(at, &mut buf).map_err(io)?; + Ok(buf) +} + +fn fourcc(b: &[u8], at: usize) -> [u8; 4] { + let mut f = [0; 4]; + if let Some(s) = b.get(at..at + 4) { + f.copy_from_slice(s); + } + f +} + +fn le16(b: &[u8], at: usize) -> u16 { + b.get(at..at + 2).map_or(0, |s| u16::from_le_bytes([s[0], s[1]])) +} + +fn le32(b: &[u8], at: usize) -> u32 { + b.get(at..at + 4).map_or(0, |s| u32::from_le_bytes([s[0], s[1], s[2], s[3]])) +} + +fn le64(b: &[u8], at: usize) -> u64 { + u64::from(le32(b, at)) | (u64::from(le32(b, at + 4)) << 32) +} + +/// A chunk header at `at`: (id, size, data offset). The size is clamped to the file (a truncated +/// recording keeps what is there). +fn header_at(src: &S, at: u64) -> Result<([u8; 4], u64, u64)> { + let h = read(src, at, 8)?; + let data = at + 8; + let size = u64::from(le32(&h, 4)).min(src.len().saturating_sub(data)); + Ok((fourcc(&h, 0), size, data)) +} + +/// Iterate the chunks of `[from, to)`: (id, size, data offset). Stops at the first unreadable +/// header; every step moves forward, so it ends. +fn children(src: &S, from: u64, to: u64) -> Vec<([u8; 4], u64, u64)> { + let mut out = Vec::new(); + let mut at = from; + while at.checked_add(8).is_some_and(|e| e <= to) { + let Ok((id, size, data)) = header_at(src, at) else { break }; + out.push((id, size, data)); + // chunks are padded to an even size + let Some(next) = data.checked_add(size).and_then(|e| e.checked_add(size & 1)) else { break }; + if next <= at { + break; + } + at = next; + } + out +} + +/// Open an AVI file. +pub fn open(src: &S) -> Result { + open_with(src, OpenOptions::default()) +} + +pub fn open_with(src: &S, opts: OpenOptions) -> Result { + let head = read(src, 0, 12.min(src.len())).map_err(|_| Error::NotAvi)?; + if !sniff(&head) { + return Err(Error::NotAvi); + } + // the RIFF forms: the first `AVI `, then any `AVIX` extensions (OpenDML) + let mut hdrl = None; + let mut movis: Vec<(u64, u64, u64)> = Vec::new(); // (position of the `movi` id, data start, data end) + let mut idx1 = None; + for (id, size, data) in children(src, 0, src.len()) { + if id != *b"RIFF" || size < 4 { + continue; + } + let end = data + size; + let form = fourcc(&read(src, data, 4)?, 0); + if form != *b"AVI " && form != *b"AVIX" { + continue; + } + for (cid, csize, cdata) in children(src, data + 4, end) { + match &cid { + b"LIST" if csize >= 4 => { + let ltype = fourcc(&read(src, cdata, 4)?, 0); + if ltype == *b"hdrl" && hdrl.is_none() && form == *b"AVI " { + hdrl = Some((cdata + 4, cdata + csize)); + } else if ltype == *b"movi" { + movis.push((cdata, cdata + 4, cdata + csize)); + } + } + b"idx1" if form == *b"AVI " => idx1 = Some((cdata, csize)), + _ => {} + } + } + } + let (hfrom, hto) = hdrl.ok_or_else(|| Error::Invalid("no hdrl header list".into()))?; + if hto - hfrom > MAX_HEADER { + return Err(Error::Invalid("header list too large".into())); + } + let (header, mut streams) = parse_hdrl(src, hfrom, hto)?; + if streams.is_empty() { + return Err(Error::Invalid("no streams".into())); + } + // chunk tables from the best index + let odml = !opts.ignore_odml && streams.iter().any(|s| !s.super_index.is_empty()); + let index = if odml && read_odml(src, &mut streams) { + IndexSource::OpenDml + } else if let (false, Some((at, size)), Some(first)) = (opts.ignore_idx1, idx1, movis.first()) { + clear(&mut streams); + read_idx1(src, &mut streams, at, size, first.0); + // idx1 only covers the first RIFF: the AVIX extensions are scanned + for &(_, from, to) in movis.iter().skip(1) { + scan(src, &mut streams, from, to, 0); + } + if streams.iter().all(|s| s.chunks.is_empty()) { scan_all(src, &mut streams, &movis) } else { IndexSource::Idx1 } + } else { + scan_all(src, &mut streams, &movis) + }; + Ok(AviFile { header, streams, index }) +} + +fn clear(streams: &mut [Stream]) { + for s in streams.iter_mut() { + s.chunks.clear(); + s.keyframes_known = true; + } +} + +fn scan_all(src: &S, streams: &mut [Stream], movis: &[(u64, u64, u64)]) -> IndexSource { + clear(streams); + for &(_, from, to) in movis { + scan(src, streams, from, to, 0); + } + for s in streams.iter_mut() { + s.keyframes_known = false; + } + IndexSource::Scan +} + +fn parse_hdrl(src: &S, from: u64, to: u64) -> Result<(MainHeader, Vec)> { + let mut header = MainHeader::default(); + let mut streams = Vec::new(); + for (id, size, data) in children(src, from, to) { + match &id { + b"avih" => { + let b = read(src, data, size.min(64))?; + header = MainHeader { + micro_sec_per_frame: le32(&b, 0), + flags: le32(&b, 12), + total_frames: le32(&b, 16), + streams: le32(&b, 24), + width: le32(&b, 32), + height: le32(&b, 36), + }; + } + b"LIST" if size >= 4 && streams.len() < MAX_STREAMS && fourcc(&read(src, data, 4)?, 0) == *b"strl" => { + streams.push(parse_strl(src, data + 4, data + size)?); + } + _ => {} + } + } + Ok((header, streams)) +} + +fn parse_strl(src: &S, from: u64, to: u64) -> Result { + let mut s = Stream { + kind: StreamKind::Other([0; 4]), + handler: [0; 4], + scale: 1, + rate: 1, + start: 0, + length: 0, + sample_size: 0, + video: None, + audio: None, + name: None, + chunks: Vec::new(), + keyframes_known: true, + super_index: Vec::new(), + }; + let mut strf = None; + for (id, size, data) in children(src, from, to) { + match &id { + b"strh" => { + let b = read(src, data, size.min(64))?; + s.kind = match &fourcc(&b, 0) { + b"vids" => StreamKind::Video, + b"auds" => StreamKind::Audio, + other => StreamKind::Other(*other), + }; + s.handler = fourcc(&b, 4); + s.scale = le32(&b, 20); + s.rate = le32(&b, 24); + s.start = le32(&b, 28); + s.length = le32(&b, 32); + s.sample_size = le32(&b, 44); + } + b"strf" => strf = Some(read(src, data, size.min(1 << 20))?), + b"strn" => { + let b = read(src, data, size.min(4096))?; + let name = String::from_utf8_lossy(b.split(|&c| c == 0).next().unwrap_or_default()).trim().to_string(); + s.name = (!name.is_empty()).then_some(name); + } + b"indx" => s.super_index = super_index(&read(src, data, size.min(MAX_HEADER))?), + _ => {} + } + } + if let Some(b) = strf { + match s.kind { + StreamKind::Video => { + s.video = Some(BitmapInfo { + width: le32(&b, 4) as i32, + height: le32(&b, 8) as i32, + bit_count: le16(&b, 14), + compression: fourcc(&b, 16), + size_image: le32(&b, 20), + extra: b.get(40.min(b.len())..).unwrap_or_default().to_vec(), + }); + } + StreamKind::Audio => { + let cb = usize::from(le16(&b, 16)); + let extra = b.get(18.min(b.len())..(18 + cb).min(b.len())).unwrap_or_default().to_vec(); + let mut tag = le16(&b, 0); + // WAVE_FORMAT_EXTENSIBLE: the sub-format GUID starts with the real tag + if tag == 0xFFFE && extra.len() >= 22 { + tag = le16(&extra, 6); + } + s.audio = Some(WaveFormat { + format_tag: tag, + channels: le16(&b, 2), + sample_rate: le32(&b, 4), + avg_bytes_per_sec: le32(&b, 8), + block_align: le16(&b, 12), + bits_per_sample: le16(&b, 14), + extra, + }); + } + StreamKind::Other(_) => {} + } + } + if s.scale == 0 || s.rate == 0 { + (s.scale, s.rate) = (1, 1); + } + Ok(s) +} + +/// The `ix##` chunk offsets of an `indx` super index (`AVI_INDEX_OF_INDEXES`, 4 longs per entry). +fn super_index(b: &[u8]) -> Vec { + let longs = usize::from(le16(b, 0)); + let (subtype, itype, n) = (b.get(2).copied().unwrap_or(0), b.get(3).copied().unwrap_or(0xFF), le32(b, 4) as usize); + if itype != 0 || subtype != 0 || longs != 4 { + return Vec::new(); + } + // header 24 bytes, entries of 16: qwOffset, dwSize, dwDuration + let n = n.min(b.len().saturating_sub(24) / 16); + (0..n).map(|k| le64(b, 24 + k * 16)).filter(|&o| o > 0).collect() +} + +/// Fill the chunk tables from the OpenDML standard indexes; false (tables cleared) when a stream +/// that has data lacks one or none could be read. +fn read_odml(src: &S, streams: &mut [Stream]) -> bool { + let limit = src.len() / 8; + let mut total = 0u64; + for s in streams.iter_mut() { + s.chunks.clear(); + s.keyframes_known = true; + for &at in &s.super_index { + let Ok((id, size, data)) = header_at(src, at) else { continue }; + if &id[..2] != b"ix" { + continue; + } + let Ok(b) = read(src, data, size.min(MAX_HEADER * 8)) else { continue }; + let (longs, subtype, itype) = (usize::from(le16(&b, 0)), b.get(2).copied().unwrap_or(0), b.get(3).copied().unwrap_or(0)); + // AVI_INDEX_OF_CHUNKS; 2 longs per entry, or 3 for the field index (AVI_INDEX_2FIELD) + if itype != 1 || !(longs == 2 && subtype == 0 || longs == 3 && subtype == 1) { + continue; + } + let base = le64(&b, 12); + let step = longs * 4; + let n = (le32(&b, 4) as usize).min(b.len().saturating_sub(24) / step); + for k in 0..n { + let e = 24 + k * step; + let (off, raw) = (u64::from(le32(&b, e)), le32(&b, e + 4)); + let size = raw & 0x7FFF_FFFF; + let Some(offset) = base.checked_add(off).filter(|o| o.checked_add(u64::from(size)).is_some_and(|end| end <= src.len())) else { continue }; + total += 1; + if total > limit { + return false; + } + s.chunks.push(Chunk { offset, size, key: raw & 0x8000_0000 == 0 }); + } + } + } + let ok = streams.iter().any(|s| !s.chunks.is_empty()) && streams.iter().all(|s| !s.super_index.is_empty() || matches!(s.kind, StreamKind::Other(_))); + if !ok { + clear(streams); + } + ok +} + +/// Fill the chunk tables from `idx1` (16-byte entries: id, flags, offset, size). Offsets are +/// relative to the `movi` list's type id, or (some writers) absolute: the first entry decides. +fn read_idx1(src: &S, streams: &mut [Stream], at: u64, size: u64, movi: u64) { + let Ok(b) = read(src, at, size.min(src.len() / 2)) else { return }; + let entries: Vec<&[u8]> = b.as_chunks::<16>().0.iter().map(|e| e.as_slice()).collect(); + let id_at = |pos: u64| read(src, pos, 4).ok().map(|v| fourcc(&v, 0)); + let first = entries.iter().find(|e| le32(e, 4) & 1 == 0); + let relative = match first { + Some(e) => { + let (id, off) = (fourcc(e, 0), u64::from(le32(e, 8))); + id_at(movi + off) == Some(id) || id_at(off) != Some(id) + } + None => true, + }; + let base = if relative { movi } else { 0 }; + for e in entries { + let (id, flags, off, len) = (fourcc(e, 0), le32(e, 4), u64::from(le32(e, 8)), le32(e, 12)); + // AVIIF_LIST entries (`rec ` lists) and palette changes carry no samples + if flags & 1 != 0 || &id[2..] == b"pc" { + continue; + } + let Some(s) = Stream::number(&id).and_then(|n| streams.get_mut(n)) else { continue }; + let Some(offset) = base.checked_add(off).and_then(|o| o.checked_add(8)).filter(|o| o.checked_add(u64::from(len)).is_some_and(|end| end <= src.len())) + else { + continue; + }; + s.chunks.push(Chunk { offset, size: len, key: flags & 0x10 != 0 || s.kind != StreamKind::Video }); + } +} + +/// Append the data chunks of a `movi` list (`[from, to)`) to the streams' tables, in file order. +fn scan(src: &S, streams: &mut [Stream], from: u64, to: u64, depth: u32) { + if depth >= MAX_DEPTH { + return; + } + for (id, size, data) in children(src, from, to) { + if id == *b"LIST" { + if size >= 4 { + scan(src, streams, data + 4, data + size, depth + 1); + } + continue; + } + if &id[..2] == b"ix" || &id[2..] == b"pc" { + continue; + } + if let Some(s) = Stream::number(&id).and_then(|n| streams.get_mut(n)) { + s.chunks.push(Chunk { offset: data, size: u32::try_from(size).unwrap_or(u32::MAX), key: true }); + } + } +} + +impl AviFile { + /// The bytes of chunk `i` of stream `stream`. + pub fn read_chunk(&self, src: &S, stream: usize, i: usize) -> Result> { + let c = self.streams.get(stream).and_then(|s| s.chunks.get(i)).ok_or_else(|| Error::Invalid(format!("no chunk {i} in stream {stream}")))?; + read(src, c.offset, u64::from(c.size)) + } +} diff --git a/crates/avi/src/source.rs b/crates/avi/src/source.rs new file mode 100644 index 00000000..783cc190 --- /dev/null +++ b/crates/avi/src/source.rs @@ -0,0 +1,59 @@ +//! Random-access byte sources for the demuxer (same shape as the other container crates', so one +//! adapter serves them all). + +use std::io; + +/// Random-access, read-only byte source (file, in-memory buffer, web Blob…). +pub trait ByteSource { + /// Total length in bytes. + fn len(&self) -> u64; + /// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end. + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>; + /// True if the source holds no bytes. + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +fn eof() -> io::Error { + io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source") +} + +impl ByteSource for [u8] { + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + let start = usize::try_from(offset).map_err(|_| eof())?; + let end = start.checked_add(buf.len()).ok_or_else(eof)?; + buf.copy_from_slice(self.get(start..end).ok_or_else(eof)?); + Ok(()) + } +} + +impl ByteSource for Vec { + fn len(&self) -> u64 { + self.as_slice().len() as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.as_slice().read_at(offset, buf) + } +} + +impl ByteSource for &T { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} + +impl ByteSource for std::sync::Arc { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} diff --git a/crates/avi/src/tests.rs b/crates/avi/src/tests.rs new file mode 100644 index 00000000..248364d6 --- /dev/null +++ b/crates/avi/src/tests.rs @@ -0,0 +1,269 @@ +use super::*; + +fn chunk(id: &[u8; 4], data: &[u8]) -> Vec { + let mut v = id.to_vec(); + v.extend_from_slice(&(data.len() as u32).to_le_bytes()); + v.extend_from_slice(data); + if data.len() % 2 == 1 { + v.push(0); + } + v +} + +fn list(ltype: &[u8; 4], body: &[u8]) -> Vec { + let mut d = ltype.to_vec(); + d.extend_from_slice(body); + chunk(b"LIST", &d) +} + +fn strl(video: bool) -> Vec { + let mut strh = vec![0u8; 56]; + strh[..4].copy_from_slice(if video { b"vids" } else { b"auds" }); + strh[4..8].copy_from_slice(if video { b"MJPG" } else { &[0; 4] }); + let (scale, rate, ss) = if video { (1u32, 25u32, 0u32) } else { (4, 192_000, 4) }; + strh[20..24].copy_from_slice(&scale.to_le_bytes()); + strh[24..28].copy_from_slice(&rate.to_le_bytes()); + strh[44..48].copy_from_slice(&ss.to_le_bytes()); + let strf = if video { + let mut b = vec![0u8; 40]; + b[..4].copy_from_slice(&40u32.to_le_bytes()); + b[4..8].copy_from_slice(&64i32.to_le_bytes()); + b[8..12].copy_from_slice(&48i32.to_le_bytes()); + b[14..16].copy_from_slice(&24u16.to_le_bytes()); + b[16..20].copy_from_slice(b"MJPG"); + b + } else { + let mut b = vec![0u8; 18]; + b[..2].copy_from_slice(&1u16.to_le_bytes()); + b[2..4].copy_from_slice(&2u16.to_le_bytes()); + b[4..8].copy_from_slice(&48_000u32.to_le_bytes()); + b[8..12].copy_from_slice(&192_000u32.to_le_bytes()); + b[12..14].copy_from_slice(&4u16.to_le_bytes()); + b[14..16].copy_from_slice(&16u16.to_le_bytes()); + b + }; + let mut body = chunk(b"strh", &strh); + body.extend(chunk(b"strf", &strf)); + body.extend(chunk(b"strn", if video { b"Camera\0" } else { b"Mic\0" })); + list(b"strl", &body) +} + +/// A small AVI: 3 video frames (the second a delta, the third odd-sized) and 2 audio chunks, +/// optionally inside a `rec ` list, with an `idx1` whose offsets are relative to `movi` or absolute. +fn avi(idx1: bool, absolute: bool, rec: bool) -> Vec { + let mut avih = vec![0u8; 56]; + avih[0..4].copy_from_slice(&40_000u32.to_le_bytes()); + avih[12..16].copy_from_slice(&0x10u32.to_le_bytes()); + avih[16..20].copy_from_slice(&3u32.to_le_bytes()); + avih[24..28].copy_from_slice(&2u32.to_le_bytes()); + avih[32..36].copy_from_slice(&64u32.to_le_bytes()); + avih[36..40].copy_from_slice(&48u32.to_le_bytes()); + let mut hdrl = chunk(b"avih", &avih); + hdrl.extend(strl(true)); + hdrl.extend(strl(false)); + let hdrl = list(b"hdrl", &hdrl); + let samples: [(&[u8; 4], Vec, bool); 5] = + [(b"00dc", vec![1; 10], true), (b"01wb", vec![2; 16], true), (b"00dc", vec![3; 6], false), (b"01wb", vec![4; 8], true), (b"00dc", vec![5; 7], true)]; + let mut body = Vec::new(); + let mut entries = Vec::new(); + for (id, data, key) in &samples { + entries.push((**id, body.len() as u32 + 4, data.len() as u32, *key)); + body.extend(chunk(id, data)); + } + let movi_body = if rec { + let r = list(b"rec ", &body); + // the samples moved 12 bytes in (the rec list's header and type) + for e in &mut entries { + e.1 += 12; + } + r + } else { + body + }; + let movi = list(b"movi", &movi_body); + let mut riff = b"AVI ".to_vec(); + riff.extend(hdrl); + // position of the `movi` type id: after `RIFF` , what `riff` holds so far, `LIST` + let movi_at = (8 + riff.len() + 8) as u32; + riff.extend(movi); + if idx1 { + let mut ix = Vec::new(); + for (id, off, len, key) in entries { + ix.extend_from_slice(&id); + ix.extend_from_slice(&(if key { 0x10u32 } else { 0 }).to_le_bytes()); + ix.extend_from_slice(&(if absolute { off + movi_at } else { off }).to_le_bytes()); + ix.extend_from_slice(&len.to_le_bytes()); + } + riff.extend(chunk(b"idx1", &ix)); + } + chunk(b"RIFF", &riff) +} + +fn table(f: &AviFile, s: usize) -> Vec<(u32, bool)> { + f.streams[s].chunks.iter().map(|c| (c.size, c.key)).collect() +} + +#[test] +fn headers_and_every_index_path_agree() { + for (idx1, absolute, rec, want) in [ + (true, false, false, IndexSource::Idx1), + (true, true, false, IndexSource::Idx1), + (true, false, true, IndexSource::Idx1), + (false, false, false, IndexSource::Scan), + (false, false, true, IndexSource::Scan), + ] { + let file = avi(idx1, absolute, rec); + let f = open(&file).unwrap(); + let what = format!("idx1 {idx1} absolute {absolute} rec {rec}"); + assert_eq!(f.index, want, "{what}"); + assert_eq!((f.header.width, f.header.height, f.header.total_frames, f.header.micro_sec_per_frame), (64, 48, 3, 40_000)); + let (v, a) = (&f.streams[0], &f.streams[1]); + assert_eq!((v.kind, v.handler, v.scale, v.rate, v.name.as_deref()), (StreamKind::Video, *b"MJPG", 1, 25, Some("Camera"))); + assert_eq!(v.video.as_ref().map(|b| (b.width, b.height, b.bit_count, b.compression)), Some((64, 48, 24, *b"MJPG"))); + let w = a.audio.as_ref().unwrap(); + assert_eq!((w.format_tag, w.channels, w.sample_rate, w.block_align, w.bits_per_sample, a.sample_size), (1, 2, 48_000, 4, 16, 4)); + // the index knows the delta frame; a scan cannot + let keys = if idx1 { [true, false, true] } else { [true; 3] }; + assert_eq!(table(&f, 0), vec![(10, keys[0]), (6, keys[1]), (7, keys[2])], "{what}"); + assert_eq!(f.streams[0].keyframes_known, idx1); + assert_eq!(table(&f, 1), vec![(16, true), (8, true)], "{what}"); + assert_eq!(f.read_chunk(&file, 0, 1).unwrap(), vec![3; 6], "{what}"); + assert_eq!(f.read_chunk(&file, 0, 2).unwrap(), vec![5; 7], "{what}"); + assert_eq!(f.read_chunk(&file, 1, 1).unwrap(), vec![4; 8], "{what}"); + assert!(f.read_chunk(&file, 0, 3).is_err() && f.read_chunk(&file, 5, 0).is_err()); + } + // the idx1 path can be skipped + let f = open_with(&avi(true, false, false), OpenOptions { ignore_idx1: true, ..Default::default() }).unwrap(); + assert_eq!(f.index, IndexSource::Scan); +} + +#[test] +fn not_avi_and_truncated_recordings() { + assert_eq!(open(&b"RIFF\0\0\0\0WAVEfmt ".to_vec()), Err(Error::NotAvi)); + assert_eq!(open(&Vec::new()), Err(Error::NotAvi)); + assert!(sniff(&avi(true, false, false)) && !sniff(b"RIFF")); + // a recording cut off in its last frame: the headers and the whole frames are there + let full = avi(false, false, false); + let cut = full[..full.len() - 4].to_vec(); + let f = open(&cut).unwrap(); + assert_eq!(f.streams[0].chunks.len(), 3); + assert!(f.read_chunk(&cut, 0, 0).is_ok()); +} + +/// Every single-byte change and every truncation of a valid file opens or fails, never panics, and +/// whatever opens reads its chunks without panicking. +#[test] +fn mutations_never_panic() { + for base in [avi(true, false, true), avi(false, false, false), avi(true, true, false)] { + for cut in 0..base.len() { + let b = base[..cut].to_vec(); + if let Ok(f) = open(&b) { + for (s, st) in f.streams.iter().enumerate() { + for i in 0..st.chunks.len() { + let _ = f.read_chunk(&b, s, i); + } + } + } + } + for at in 0..base.len() { + for v in [0x00u8, 0xFF, 0x7F, 0x80] { + let mut b = base.clone(); + b[at] = v; + if let Ok(f) = open(&b) { + for (s, st) in f.streams.iter().enumerate() { + for i in 0..st.chunks.len() { + let _ = f.read_chunk(&b, s, i); + } + } + } + } + } + } +} + +/// An OpenDML file: each stream's `indx` super index points at an `ix##` standard index (base +/// offset + offsets to the data, bit 31 of the size marking delta frames). Built twice: the second +/// pass knows where the `ix##` chunks landed. +fn avi_odml(ix_at: [u64; 2]) -> (Vec, [u64; 2]) { + let indx = |stream: u8, at: u64| { + let mut b = Vec::new(); + b.extend_from_slice(&4u16.to_le_bytes()); // wLongsPerEntry + b.extend_from_slice(&[0, 0]); // subtype, AVI_INDEX_OF_INDEXES + b.extend_from_slice(&1u32.to_le_bytes()); + b.extend_from_slice(&[b'0', b'0' + stream, if stream == 0 { b'd' } else { b'w' }, if stream == 0 { b'c' } else { b'b' }]); + b.extend_from_slice(&[0; 12]); + b.extend_from_slice(&at.to_le_bytes()); + b.extend_from_slice(&0u32.to_le_bytes()); + b.extend_from_slice(&0u32.to_le_bytes()); + chunk(b"indx", &b) + }; + let mut hdrl = chunk(b"avih", &[0; 56]); + for (s, at) in ix_at.iter().enumerate() { + let l = strl(s == 0); + // strl = LIST strl …: append indx inside it + let mut body = l[12..].to_vec(); + body.extend(indx(s as u8, *at)); + hdrl.extend(list(b"strl", &body)); + } + let mut riff = b"AVI ".to_vec(); + riff.extend(list(b"hdrl", &hdrl)); + // movi: video 10 (key) 6 (delta) 7 (key), audio 16 8; then ix00, ix01 + let movi_data_at = 8 + riff.len() as u64 + 12; // RIFF hdr, so far, LIST hdr + `movi` + let mut body = Vec::new(); + let mut at: [Vec<(u64, u32, bool)>; 2] = [Vec::new(), Vec::new()]; + for (s, data, key) in [(0usize, vec![1u8; 10], true), (1, vec![2; 16], true), (0, vec![3; 6], false), (1, vec![4; 8], true), (0, vec![5; 7], true)] { + let id = if s == 0 { b"00dc" } else { b"01wb" }; + at[s].push((movi_data_at + body.len() as u64 + 8, data.len() as u32, key)); + body.extend(chunk(id, &data)); + } + let mut where_ix = [0u64; 2]; + for (s, entries) in at.iter().enumerate() { + let base = 100u64; // offsets are relative to qwBaseOffset + let mut b = Vec::new(); + b.extend_from_slice(&2u16.to_le_bytes()); + b.extend_from_slice(&[0, 1]); // subtype 0, AVI_INDEX_OF_CHUNKS + b.extend_from_slice(&(entries.len() as u32).to_le_bytes()); + b.extend_from_slice(if s == 0 { b"00dc" } else { b"01wb" }); + b.extend_from_slice(&base.to_le_bytes()); + b.extend_from_slice(&0u32.to_le_bytes()); + for &(off, size, key) in entries { + b.extend_from_slice(&((off - base) as u32).to_le_bytes()); + b.extend_from_slice(&(size | if key { 0 } else { 0x8000_0000 }).to_le_bytes()); + } + where_ix[s] = movi_data_at + body.len() as u64; + body.extend(chunk(if s == 0 { b"ix00" } else { b"ix01" }, &b)); + } + riff.extend(list(b"movi", &body)); + (chunk(b"RIFF", &riff), where_ix) +} + +#[test] +fn opendml_standard_indexes() { + let (_, at) = avi_odml([0, 0]); + let (file, again) = avi_odml(at); + assert_eq!(at, again, "the layout does not depend on the offsets"); + let f = open(&file).unwrap(); + assert_eq!(f.index, IndexSource::OpenDml); + assert_eq!(table(&f, 0), vec![(10, true), (6, false), (7, true)]); + assert_eq!(table(&f, 1), vec![(16, true), (8, true)]); + assert_eq!(f.read_chunk(&file, 0, 2).unwrap(), vec![5; 7]); + // ignoring the OpenDML index, the scan finds the same chunks (and skips the ix## chunks) + let s = open_with(&file, OpenOptions { ignore_odml: true, ..Default::default() }).unwrap(); + assert_eq!(s.index, IndexSource::Scan); + assert_eq!(s.streams[0].chunks.iter().map(|c| c.offset).collect::>(), f.streams[0].chunks.iter().map(|c| c.offset).collect::>()); + // a broken super index (pointing nowhere) falls back to a scan + let (bad, _) = avi_odml([3, 5]); + assert_eq!(open(&bad).unwrap().index, IndexSource::Scan); + // and mutations of the OpenDML file never panic + for i in 0..file.len() { + let mut b = file.clone(); + b[i] ^= 0xFF; + if let Ok(f) = open(&b) { + for (s, st) in f.streams.iter().enumerate() { + for k in 0..st.chunks.len() { + let _ = f.read_chunk(&b, s, k); + } + } + } + } +} diff --git a/crates/avi/tests/ffmpeg_oracle.rs b/crates/avi/tests/ffmpeg_oracle.rs new file mode 100644 index 00000000..7f1b1662 --- /dev/null +++ b/crates/avi/tests/ffmpeg_oracle.rs @@ -0,0 +1,151 @@ +//! ffmpeg-made AVI files (skipped without ffmpeg): the chunk tables match ffprobe's packet counts, +//! and every index the file has (`idx1`; OpenDML once a file passes 1 GB) and a `movi` scan describe +//! the same chunks. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use filmcraft_avi::{IndexSource, OpenOptions, StreamKind, open, open_with}; + +fn fixture(name: &str, args: &[&str]) -> Option { + let ffmpeg = filmcraft_testkit::ffmpeg_or_skip("avi oracle")?; + let out = filmcraft_testkit::fixtures_dir("avi").join(name); + filmcraft_testkit::fixtures::generate(&out, |tmp: &Path| { + let mut c = Command::new(&ffmpeg); + c.args(["-v", "error", "-y"]).args(args).args(["-f", "avi"]).arg(tmp); + c.status().is_ok_and(|s| s.success()) + }) +} + +fn packets(path: &Path, stream: usize) -> usize { + let ffprobe = filmcraft_testkit::ffprobe().expect("ffprobe next to ffmpeg"); + let out = Command::new(ffprobe) + .args(["-v", "error", "-select_streams", &stream.to_string(), "-count_packets", "-show_entries", "stream=nb_read_packets", "-of", "csv=p=0"]) + .arg(path) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().parse().unwrap_or(0) +} + +fn packet_bytes(path: &Path, stream: usize) -> u64 { + let ffprobe = filmcraft_testkit::ffprobe().expect("ffprobe next to ffmpeg"); + let out = Command::new(ffprobe) + .args(["-v", "error", "-select_streams", &stream.to_string(), "-show_entries", "packet=size", "-of", "csv=p=0"]) + .arg(path) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).lines().filter_map(|l| l.trim().parse::().ok()).sum() +} + +const TONE: [&str; 4] = ["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=2"]; + +fn cases() -> Vec<(&'static str, Vec<&'static str>, [u8; 4])> { + let pic = vec!["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2"]; + let with = |v: &[&'static str], a: &[&'static str]| -> Vec<&'static str> { + let mut x = pic.clone(); + x.extend_from_slice(&TONE); + x.extend_from_slice(v); + x.extend_from_slice(a); + x + }; + vec![ + ("mjpeg_pcm.avi", with(&["-c:v", "mjpeg", "-q:v", "4", "-pix_fmt", "yuvj420p"], &["-c:a", "pcm_s16le"]), *b"MJPG"), + ("h264_mp3.avi", with(&["-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p"], &["-c:a", "libmp3lame", "-b:a", "128k"]), *b"H264"), + ("bgr24_pcm.avi", with(&["-c:v", "rawvideo", "-pix_fmt", "bgr24"], &["-c:a", "pcm_s16le"]), [0; 4]), + ("yuyv_pcm.avi", with(&["-c:v", "rawvideo", "-pix_fmt", "yuyv422"], &["-c:a", "pcm_s16le"]), *b"YUY2"), + ] +} + +#[test] +fn ffmpeg_files_match_ffprobe_on_every_index_path() { + for (name, args, fourcc) in cases() { + let Some(path) = fixture(name, &args) else { return }; + let bytes = std::fs::read(&path).unwrap(); + let f = open(&bytes).unwrap(); + // ffmpeg reserves room for the OpenDML index but writes only idx1 under 1 GB + assert_eq!(f.index, IndexSource::Idx1, "{name}"); + assert_eq!(f.streams.len(), 2, "{name}"); + let (v, a) = (&f.streams[0], &f.streams[1]); + assert_eq!((v.kind, a.kind), (StreamKind::Video, StreamKind::Audio), "{name}"); + let bmp = v.video.as_ref().unwrap(); + assert_eq!((bmp.width, bmp.height.abs(), bmp.compression), (160, 120, fourcc), "{name}"); + assert_eq!((v.rate as f64 / v.scale as f64).round(), 25.0, "{name}"); + assert_eq!(a.audio.as_ref().map(|w| (w.sample_rate, w.channels)), Some((48_000, 1)), "{name}"); + // the tables hold what ffprobe reads + assert_eq!(v.chunks.len(), packets(&path, 0), "{name}: video"); + // PCM chunks are packets; ffprobe re-splits MP3 into its frames, which AVI chunks need not + // follow, so compressed audio compares by bytes + if a.audio.as_ref().is_some_and(|w| w.format_tag == 1) { + assert_eq!(a.chunks.len(), packets(&path, 1), "{name}: audio"); + } else { + let ours: u64 = a.chunks.iter().map(|c| u64::from(c.size)).sum(); + assert_eq!(ours, packet_bytes(&path, 1), "{name}: audio bytes"); + } + assert!(v.chunks[0].key, "{name}: the first frame is a key frame"); + if fourcc == *b"H264" { + assert!(v.chunks.iter().any(|c| !c.key), "{name}: the index marks the delta frames"); + } + // a scan finds the same chunks (it cannot know key frames) + let scanned = open_with(&bytes, OpenOptions { ignore_odml: true, ignore_idx1: true }).unwrap(); + assert_eq!(scanned.index, IndexSource::Scan, "{name}"); + for s in 0..2 { + assert_eq!(sizes(&scanned, s), sizes(&f, s), "{name}: scan stream {s}"); + } + } +} + +fn sizes(f: &filmcraft_avi::AviFile, s: usize) -> Vec<(u64, u32)> { + f.streams[s].chunks.iter().map(|c| (c.offset, c.size)).collect() +} + +/// A file past 1 GB, as long recordings are: ffmpeg writes `RIFF AVIX` extensions and the +/// OpenDML index. Needs ffmpeg and 1.2 GB of disk: `cargo test -p filmcraft-avi --test +/// ffmpeg_oracle -- --ignored`. +#[test] +#[ignore] +fn opendml_file_past_1_gb() { + let args = [ + "-f", + "lavfi", + "-i", + "testsrc2=s=640x480:r=25:d=50", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:d=50", + "-c:v", + "rawvideo", + "-pix_fmt", + "bgr24", + "-c:a", + "pcm_s16le", + ]; + let Some(path) = fixture("large_opendml.avi", &args) else { return }; + let file = std::fs::File::open(&path).unwrap(); + let src = FileSource(file); + let f = open(&src).unwrap(); + assert_eq!(f.index, IndexSource::OpenDml); + assert_eq!(f.streams[0].chunks.len(), packets(&path, 0)); + assert_eq!(f.streams[1].chunks.len(), packets(&path, 1)); + assert_eq!(f.streams[0].chunks.len(), 1250); + assert!(f.streams[0].chunks.last().unwrap().offset > 1 << 30, "the last frames are in an AVIX extension"); + // idx1 covers the first RIFF only; the AVIX extensions are scanned: the same chunks + let idx1 = open_with(&src, OpenOptions { ignore_odml: true, ..Default::default() }).unwrap(); + assert_eq!(idx1.index, IndexSource::Idx1); + for s in 0..2 { + assert_eq!(sizes(&idx1, s), sizes(&f, s), "stream {s}"); + } + assert_eq!(f.read_chunk(&src, 0, 1249).unwrap().len(), 640 * 480 * 3); +} + +/// A file on disk as a byte source, read where asked. +struct FileSource(std::fs::File); + +impl filmcraft_avi::ByteSource for FileSource { + fn len(&self) -> u64 { + self.0.metadata().map(|m| m.len()).unwrap_or(0) + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> { + std::os::unix::fs::FileExt::read_exact_at(&self.0, buf, offset) + } +} diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml index a3037e2c..39f00bd0 100644 --- a/crates/codecs/Cargo.toml +++ b/crates/codecs/Cargo.toml @@ -14,6 +14,7 @@ filmcraft-time = { workspace = true } filmcraft-color = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-media = { workspace = true } +filmcraft-avi = { workspace = true } filmcraft-isobmff = { workspace = true } filmcraft-matroska = { workspace = true } filmcraft-mxf = { workspace = true } diff --git a/crates/codecs/src/avi.rs b/crates/codecs/src/avi.rs new file mode 100644 index 00000000..65952442 --- /dev/null +++ b/crates/codecs/src/avi.rs @@ -0,0 +1,585 @@ +//! AVI media source (`filmcraft-avi` demux, GOP-aware video via [`crate::gop`]). +//! +//! Video: Motion JPEG (frames without their own Huffman tables get the standard ones), H.264 and +//! HEVC (Annex B byte streams, our decoders), and uncompressed RGB / YUV ([`RawFormat`]). A frame +//! slot with an empty chunk (a "drop frame") repeats the picture before it. MPEG-4 Part 2 +//! (DivX / Xvid) and DV have no decoder yet: their streams are listed and report the codec. +//! +//! Audio: PCM (integer and float) straight from the chunks; MPEG audio (MP3, MP2) and AC-3 frames, +//! which AVI chunks need not follow, are found again across chunk boundaries +//! ([`filmcraft_mpegts::frame_bytes`]) and decoded as packets. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use filmcraft_avi::{AviFile, BitmapInfo, StreamKind, WaveFormat}; +use filmcraft_frame::{AudioBuffer, VideoFrame}; +use filmcraft_isobmff::{PcmConfig, SampleEntry}; +use filmcraft_media::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, SharedSource, VideoStreamInfo}; +use filmcraft_time::{FrameRate, Tick}; + +use crate::audio::{PacketDecoder, decode_pcm}; +use crate::gop::{GopCache, VideoSamples}; +use crate::video::{RawFormat, RawVideoDecoder, VideoDecoder}; +use crate::{CodecError, make_video_decoder}; + +pub fn sniff(b: &[u8]) -> bool { + filmcraft_avi::sniff(b) +} + +#[derive(Clone, Debug, PartialEq)] +enum VCodec { + Jpeg, + H264, + Hevc, + Raw(RawFormat), + /// No decoder: the codec's name. + Unsupported(String), +} + +fn fourcc_text(f: [u8; 4]) -> String { + let t: String = f.iter().map(|&c| if c.is_ascii_graphic() { c as char } else { '?' }).collect(); + t.trim_end_matches('?').to_string() +} + +/// The decoder for a video stream and its label. +fn video_codec(b: &BitmapInfo) -> (VCodec, String) { + if let Some(raw) = RawFormat::from_avi(b.compression, b.bit_count) { + return (VCodec::Raw(raw), raw.label().into()); + } + let mut f = b.compression; + f.make_ascii_uppercase(); + match &f { + b"MJPG" | b"AVRN" | b"LJPG" | b"JPGL" | b"DMB1" | b"JPEG" | b"MJPA" => (VCodec::Jpeg, "Motion JPEG".into()), + b"H264" | b"X264" | b"AVC1" | b"DAVC" | b"VSSH" => (VCodec::H264, "H.264".into()), + b"HEVC" | b"H265" | b"HEV1" | b"HVC1" | b"X265" => (VCodec::Hevc, "HEVC".into()), + b"DIVX" | b"DX50" | b"XVID" | b"FMP4" | b"MP4V" | b"M4S2" | b"3IV2" | b"DIV3" | b"MP43" | b"MP42" => { + (VCodec::Unsupported("MPEG-4 Part 2 (DivX / Xvid)".into()), "MPEG-4 Part 2 (DivX / Xvid)".into()) + } + b"DVSD" | b"DV25" | b"DV50" | b"DVHD" | b"DVSL" | b"CDVC" | b"DVCP" | b"DVPP" => (VCodec::Unsupported("DV".into()), "DV".into()), + _ => { + let name = fourcc_text(b.compression); + (VCodec::Unsupported(format!("\"{name}\"")), name) + } + } +} + +/// One picture of the video stream: its frame slot (in `scale / rate` units) and chunk. +#[derive(Clone, Copy, Debug)] +struct VFrame { + pts: i64, + chunk: usize, + key: bool, +} + +#[derive(Clone, Debug)] +enum ACodec { + Pcm(PcmConfig), + /// MPEG audio (layer 1-3) or AC-3 frames, found again across chunks. + Framed { + ts: filmcraft_mpegts::Codec, + layer: u8, + }, + Unsupported(String), +} + +fn audio_codec(w: &WaveFormat) -> (ACodec, String) { + let pcm = |float: bool| { + let bits = w.bits_per_sample; + let cfg = PcmConfig { bits, float, big_endian: false, signed: bits > 8, channels: u32::from(w.channels.max(1)), sample_rate: f64::from(w.sample_rate) }; + (ACodec::Pcm(cfg), format!("PCM {bits}-bit{}", if float { " float" } else { "" })) + }; + match w.format_tag { + 1 if matches!(w.bits_per_sample, 8 | 16 | 24 | 32) => pcm(false), + 3 if matches!(w.bits_per_sample, 32 | 64) => pcm(true), + 0x55 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 3 }, "MP3".into()), + 0x50 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 2 }, "MPEG Audio".into()), + 0x2000 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::Ac3, layer: 0 }, "AC-3".into()), + tag => { + let name = format!("audio format 0x{tag:04X}"); + (ACodec::Unsupported(name.clone()), name) + } + } +} + +/// A slice of a chunk: (chunk, offset in it, length). +type Piece = (usize, usize, usize); + +struct AudioState { + decoder: Option, + packets: HashMap>>>, + order: Vec, + last_decoded: Option, +} + +struct AviAudio { + stream: usize, + codec: ACodec, + /// The pieces of packet `k` are `pieces[first[k]..first[k + 1]]`. + pieces: Vec, + first: Vec, + /// Packet start positions in sample frames. + starts: Vec, + state: Mutex, +} + +pub struct AviSource { + info: MediaInfo, + bytes: crate::Src, + file: AviFile, + vstream: Option, + vcodec: VCodec, + /// Pictures in stored (decode) order. + vframes: Vec, + /// `vframes` indices in presentation order (H.264 with B-frames is stored in decode order), + /// and their presentation times. + vorder: Vec, + vorder_pts: Vec, + /// Frame slots of the video stream (where the last picture ends). + vslots: i64, + rate: FrameRate, + bottom_up: bool, + video: GopCache, + audios: Vec, +} + +/// Whether an Annex B access unit holds an H.264 IDR or HEVC IRAP picture. +fn random_access(data: &[u8], hevc: bool) -> bool { + filmcraft_bitstream::annexb_nals(data).iter().any(|n| match n.first() { + Some(h) if hevc => (16..=23).contains(&((h >> 1) & 0x3F)), + Some(h) => h & 0x1F == 5, + None => false, + }) +} + +/// The packets of a framed audio stream: every frame header found in the stream of chunks, and +/// the samples each frame decodes to. Bytes that start no frame are skipped (resynchronisation). +fn split_frames( + file: &AviFile, + bytes: &crate::Src, + s: usize, + ts: &filmcraft_mpegts::Codec, +) -> (Vec, Vec, Vec, Option) { + let Some(st) = file.streams.get(s) else { return Default::default() }; + // chunk start positions in the concatenated stream + let mut at = Vec::with_capacity(st.chunks.len()); + let mut total = 0usize; + for c in &st.chunks { + at.push(total); + total = total.saturating_add(c.size as usize); + } + let locate = |pos: usize| -> Option<(usize, usize)> { + let k = at.partition_point(|&a| a <= pos).checked_sub(1)?; + Some((k, pos - at.get(k)?)) + }; + let (mut pieces, mut first, mut samples) = (Vec::new(), Vec::new(), Vec::new()); + let mut info = None; + // a sliding window over the chunks: `win` holds the stream from `base` + let (mut win, mut base, mut next) = (Vec::::new(), 0usize, 0usize); + let mut pos = 0usize; + while pos < total { + // keep at least a header's worth (and the frame, when we know its length) in the window + while win.len() < (pos - base) + 4096 && next < st.chunks.len() { + win.extend(file.read_chunk(&bytes, s, next).unwrap_or_default()); + next += 1; + } + let Some(here) = win.get(pos - base..) else { break }; + match filmcraft_mpegts::frame_bytes(ts, here) { + Some(n) if pos + n <= total => { + if info.is_none() { + info = filmcraft_mpegts::frame_info(ts, here); + } + let k = filmcraft_mpegts::frame_info(ts, here).map_or(0, |f| f.samples); + first.push(pieces.len()); + let mut p = pos; + while p < pos + n { + let Some((c, off)) = locate(p) else { break }; + let len = (st.chunks.get(c).map_or(0, |ch| ch.size as usize) - off).min(pos + n - p); + if len == 0 { + break; + } + pieces.push((c, off, len)); + p += len; + } + samples.push(k); + pos += n; + } + _ if here.len() < 8 && next >= st.chunks.len() => break, + _ => pos += 1, + } + // drop what is behind + if pos - base > 1 << 16 { + let cut = pos - base; + win.drain(..cut.min(win.len())); + base += cut; + } + } + (pieces, first, samples, info) +} + +impl AviSource { + pub fn open(name: &str, bytes: Arc<[u8]>) -> crate::Result { + Self::open_reader(name, Arc::new(filmcraft_media::reader::MemReader(bytes))) + } + + /// Open from a random-access reader: the index now, frames and audio on demand. + pub fn open_reader(name: &str, reader: filmcraft_media::SharedReader) -> crate::Result { + let bytes = crate::Src(reader); + let file = filmcraft_avi::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?; + let vstream = file.streams.iter().position(|s| s.kind == StreamKind::Video && s.video.is_some() && !s.chunks.is_empty()); + let astreams: Vec = file + .streams + .iter() + .enumerate() + .filter(|(_, s)| s.kind == StreamKind::Audio && s.audio.is_some() && !s.chunks.is_empty()) + .map(|(i, _)| i) + .take(filmcraft_media::MAX_AUDIO_STREAMS) + .collect(); + if vstream.is_none() && astreams.is_empty() { + return Err(CodecError::Unsupported("no playable streams".into())); + } + // video: frame slots, pictures and key frames + let (mut vcodec, mut vframes, mut vslots, mut rate, mut bottom_up, mut video) = + (VCodec::Unsupported(String::new()), Vec::new(), 0i64, FrameRate::new(25, 1), false, None); + if let Some(vs) = vstream.and_then(|i| file.streams.get(i)) { + let b = vs.video.clone().unwrap_or_default(); + let (codec, label) = video_codec(&b); + rate = FrameRate::new(i64::from(vs.rate), i64::from(vs.scale)); + bottom_up = b.height > 0; + let (w, h) = (b.width.unsigned_abs(), b.height.unsigned_abs()); + for (k, c) in vs.chunks.iter().enumerate() { + if c.size == 0 { + continue; + } + let key = match &codec { + VCodec::H264 | VCodec::Hevc if !vs.keyframes_known => { + file.read_chunk(&bytes, vstream.unwrap_or(0), k).is_ok_and(|d| random_access(&d, codec == VCodec::Hevc)) + } + VCodec::H264 | VCodec::Hevc => c.key, + _ => true, + }; + vframes.push(VFrame { pts: i64::from(vs.start) + k as i64, chunk: k, key }); + } + vslots = i64::from(vs.start) + vs.chunks.len() as i64; + // AVI has no timestamps: H.264 with B-frames is stored in decode order, and the + // presentation order comes from the pictures' order counts. A first look at the start + // of the stream decides whether it reorders at all (reading every frame's header is the + // cost of a reordering stream only). + if codec == VCodec::H264 + && let Some(si) = vstream + { + let heads = |n: usize| -> Vec> { + vframes + .iter() + .take(n) + .map(|f| { + file.streams + .get(si) + .and_then(|s| s.chunks.get(f.chunk)) + .and_then(|c| filmcraft_media::reader::read_range(&*bytes.0, c.offset, (c.size as usize).min(8 << 10)).ok()) + .unwrap_or_default() + }) + .collect() + }; + let reorders = |p: &[i64]| p.windows(2).any(|w| w[1] < w[0]); + if crate::mxf::avc_presentation_order(&heads(300)).is_some_and(|p| reorders(&p)) + && let Some(ranks) = crate::mxf::avc_presentation_order(&heads(vframes.len())) + { + // the pictures keep the slots they occupy (empty drop-frame chunks between them + // stay), shown in their presentation order + let mut slots: Vec = vframes.iter().map(|f| f.pts).collect(); + slots.sort_unstable(); + for (f, r) in vframes.iter_mut().zip(ranks) { + if let Some(&p) = usize::try_from(r).ok().and_then(|r| slots.get(r)) { + f.pts = p; + } + } + } + } + let secs = vslots as f64 / rate.as_f64().max(1e-9); + // The rate the pictures play at: a writer that kept a fine time base (ffmpeg copying an + // MP4's 1/600) fills the slots between pictures with empty chunks, every 20th slot + // holding a picture at 30 fps. A regular spacing makes the rate; slots stay the units. + let shown = if vframes.len() > 1 { + let mut gaps: Vec = vframes.windows(2).map(|w| w[1].chunk as i64 - w[0].chunk as i64).filter(|g| *g > 0).take(1000).collect(); + gaps.sort_unstable(); + let gap = gaps.get(gaps.len() / 2).copied().unwrap_or(1).max(1); + let regular = gaps.iter().filter(|g| **g == gap).count() * 10 >= gaps.len() * 9; + if gap > 1 && regular { FrameRate::new(i64::from(vs.rate), i64::from(vs.scale).saturating_mul(gap)) } else { rate } + } else { + rate + }; + let bitrate = (secs > 0.0).then(|| (vs.chunks.iter().map(|c| u64::from(c.size)).sum::() as f64 * 8.0 / secs) as u64); + video = Some(VideoStreamInfo { + width: w, + height: h, + frame_rate: shown, + par: (1, 1), + codec: label, + pixel_format: String::new(), + color: crate::stream_color::resolve(w, h, &[]), + has_alpha: false, + bitrate, + hdr: None, + }); + vcodec = codec; + } + // audio: packets and their start positions + let mut audios = Vec::new(); + let mut audio_streams = Vec::new(); + for &s in &astreams { + let Some(st) = file.streams.get(s) else { continue }; + let w = st.audio.clone().unwrap_or_default(); + let (codec, label) = audio_codec(&w); + let mut sample_rate = w.sample_rate.max(1); + let mut channels = u32::from(w.channels.max(1)); + let (pieces, first, starts) = match &codec { + ACodec::Pcm(cfg) => { + let block = usize::from(w.block_align.max(1)).max((usize::from(cfg.bits).div_ceil(8)) * channels as usize); + let mut t = 0i64; + let mut starts = Vec::with_capacity(st.chunks.len()); + for c in &st.chunks { + starts.push(t); + t = t.saturating_add((c.size as usize / block) as i64); + } + let pieces: Vec = st.chunks.iter().enumerate().map(|(k, c)| (k, 0, c.size as usize)).collect(); + let first = (0..pieces.len()).collect(); + (pieces, first, starts) + } + ACodec::Framed { ts, .. } => { + let (pieces, first, samples, info) = split_frames(&file, &bytes, s, ts); + if let Some(i) = info { + sample_rate = i.sample_rate.max(1); + channels = i.channels.max(1); + } + let mut t = 0i64; + let starts = samples + .iter() + .map(|&k| { + let s = t; + t = t.saturating_add(i64::from(k)); + s + }) + .collect(); + (pieces, first, starts) + } + ACodec::Unsupported(_) => (Vec::new(), Vec::new(), Vec::new()), + }; + // a stream that starts late (dwStart, in scale / rate units) + let delay = (i128::from(st.start) * i128::from(st.scale) * i128::from(sample_rate) / i128::from(st.rate.max(1))) as i64; + let starts: Vec = starts.into_iter().map(|s: i64| s.saturating_add(delay)).collect(); + audio_streams.push(AudioStreamInfo { + sample_rate, + channels, + codec: label, + bits_per_sample: (w.bits_per_sample > 0).then_some(u32::from(w.bits_per_sample)), + }); + let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }); + audios.push(AviAudio { stream: s, codec, pieces, first, starts, state }); + } + // the longer of the picture and the sound + let vdur = if vslots > 0 { rate.tick_of(vslots) } else { Tick::ZERO }; + let adur = audios + .iter() + .zip(&audio_streams) + .filter_map(|(a, i)| { + let last = a.starts.last()?; + let tail = a.starts.len().checked_sub(2).and_then(|k| a.starts.get(k)).map_or(0, |p| last - p); + Some(Tick::from_rational(last + tail, 1, i64::from(i.sample_rate))) + }) + .max() + .unwrap_or_default(); + let info = MediaInfo { + name: name.to_string(), + kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly }, + duration: vdur.max(adur), + video, + audio_streams, + container: "AVI".into(), + start_timecode: None, + file_size: Some(bytes.0.len()), + }; + let mut vorder: Vec = (0..vframes.len()).collect(); + vorder.sort_by_key(|&i| vframes.get(i).map_or(0, |f| f.pts)); + let vorder_pts = vorder.iter().map(|&i| vframes.get(i).map_or(0, |f| f.pts)).collect(); + Ok(Self { info, bytes, file, vstream, vcodec, vframes, vorder, vorder_pts, vslots, rate, bottom_up, video: GopCache::new(None), audios }) + } + + fn read(&self, stream: usize, chunk: usize) -> crate::Result> { + self.file.read_chunk(&self.bytes, stream, chunk).map_err(|e| CodecError::Container(e.to_string())) + } + + /// The bytes of audio packet `k`. + fn packet(&self, a: &AviAudio, k: usize) -> crate::Result> { + let (from, to) = ( + a.first.get(k).copied().ok_or_else(|| CodecError::Container("audio packet out of range".into()))?, + a.first.get(k + 1).copied().unwrap_or(a.pieces.len()), + ); + let mut out = Vec::new(); + for &(c, off, len) in a.pieces.get(from..to).unwrap_or_default() { + let chunk = self.read(a.stream, c)?; + out.extend_from_slice(chunk.get(off..off + len).ok_or_else(|| CodecError::Container("audio chunk shorter than its index".into()))?); + } + Ok(out) + } + + fn audio_packet(&self, a: &AviAudio, st: &mut AudioState, ainfo: &AudioStreamInfo, i: usize) -> crate::Result>>> { + if let Some(p) = st.packets.get(&i) { + return Ok(p.clone()); + } + let data = self.packet(a, i)?; + let decoded = match &a.codec { + ACodec::Pcm(cfg) => decode_pcm(&data, cfg), + ACodec::Unsupported(name) => return Err(CodecError::Unsupported(name.clone())), + ACodec::Framed { ts, layer } => { + if st.decoder.is_none() { + st.decoder = Some(match ts { + filmcraft_mpegts::Codec::Ac3 => PacketDecoder::ac3()?, + _ => PacketDecoder::mpeg_audio(*layer, ainfo.sample_rate)?, + }); + } + let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?; + // non-sequential access: reset and prime with the packet before (the bit reservoir) + if st.last_decoded.is_none_or(|l| l + 1 != i) { + d.reset(); + if let Some(prev) = i.checked_sub(1).and_then(|j| self.packet(a, j).ok()) { + let _ = d.decode(&prev, 0); + } + } + let start = a.starts.get(i).copied().unwrap_or(0).max(0) as u64; + let r = d.decode(&data, start); + st.last_decoded = Some(i); + r.unwrap_or_default() + } + }; + let p = Arc::new(decoded); + st.packets.insert(i, p.clone()); + st.order.push(i); + if st.order.len() > 4096 { + let old = st.order.remove(0); + st.packets.remove(&old); + } + Ok(p) + } +} + +/// The video stream as a [`VideoSamples`] table. +struct AviVideo<'a> { + src: &'a AviSource, + stream: usize, +} + +impl VideoSamples for AviVideo<'_> { + fn count(&self) -> usize { + self.src.vframes.len() + } + fn pts(&self, i: usize) -> i64 { + self.src.vframes.get(i).map_or(0, |f| f.pts) + } + fn sync_before(&self, i: usize) -> usize { + let i = i.min(self.src.vframes.len().saturating_sub(1)); + self.src.vframes.get(..=i).and_then(|f| f.iter().rposition(|f| f.key)).unwrap_or(0) + } + fn sample_at(&self, t: i64) -> Option { + // the picture shown in slot `t`: the last one at or before it (empty chunks repeat it) + if t >= self.src.vslots { + return None; + } + let k = self.src.vorder_pts.partition_point(|&p| p <= t).checked_sub(1)?; + self.src.vorder.get(k).copied() + } + fn read(&self, i: usize) -> crate::Result> { + let f = self.src.vframes.get(i).ok_or_else(|| CodecError::Container("frame out of range".into()))?; + self.src.read(self.stream, f.chunk) + } + fn make_decoder(&self) -> crate::Result> { + let v = self.src.info.video.as_ref().ok_or_else(|| CodecError::Unsupported("no video stream".into()))?; + let (w, h) = (v.width.min(u32::from(u16::MAX)) as u16, v.height.min(u32::from(u16::MAX)) as u16); + match &self.src.vcodec { + VCodec::Jpeg => make_video_decoder(&SampleEntry::jpeg(w, h)), + VCodec::H264 => Ok(Box::new(crate::video::H264Decoder::annexb())), + VCodec::Hevc => Ok(Box::new(crate::video::HevcDecoder::annexb())), + VCodec::Raw(f) => Ok(Box::new(RawVideoDecoder::new(*f, v.width, v.height, self.src.bottom_up)?)), + VCodec::Unsupported(name) => Err(CodecError::Unsupported(format!("no decoder for {name} video"))), + } + } +} + +impl MediaSource for AviSource { + fn info(&self) -> &MediaInfo { + &self.info + } + + fn video_frame(&self, req: FrameRequest) -> Result, MediaError> { + let s = self.vstream.ok_or(MediaError::NoStream("video"))?; + let t = req.time.max(Tick::ZERO); + let slot = |t: Tick| self.rate.frame_at(t); + let late = filmcraft_media::cancel::catch_up().map(|m| slot(t - m)); + Ok(self.video.frame_late(&AviVideo { src: self, stream: s }, slot(t), late)?) + } + + fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result { + self.audio_stream(0, start, frames, sample_rate) + } + + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result { + let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?; + let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?; + if let ACodec::Unsupported(name) = &a.codec { + return Err(MediaError::Unsupported(format!("{name} audio"))); + } + let ch = ainfo.channels.max(1) as usize; + let ratio = f64::from(ainfo.sample_rate) / f64::from(sample_rate.max(1)); + let s0 = (start as f64 * ratio).floor() as i64; + let need = (frames as f64 * ratio).ceil() as i64 + 2; + let mut src: Vec> = vec![vec![0.0; need.max(0) as usize]; ch]; + let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner()); + let mut i = a.starts.partition_point(|&x| x <= s0).saturating_sub(1); + while let Some(&pk_start) = a.starts.get(i) { + if pk_start >= s0 + need { + break; + } + let pk = self.audio_packet(a, &mut st, ainfo, i)?; + for (c, dst) in src.iter_mut().enumerate() { + let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue }; + for (k, v) in chan.iter().enumerate() { + let pos = pk_start + k as i64 - s0; + if let Some(d) = usize::try_from(pos).ok().and_then(|p| dst.get_mut(p)) { + *d = *v; + } + } + } + i += 1; + } + drop(st); + let frac = s0 as f64 - start as f64 * ratio; + let mut out = AudioBuffer::silence(sample_rate, ch, frames); + for (c, o) in out.channels.iter_mut().enumerate() { + let Some(s) = src.get(c) else { continue }; + for (k, d) in o.iter_mut().enumerate() { + let pos = k as f64 * ratio - frac; + let i0 = pos.floor().max(0.0) as usize; + let f = (pos - i0 as f64) as f32; + let a = s.get(i0).copied().unwrap_or(0.0); + let b = s.get(i0 + 1).copied().unwrap_or(a); + *d = a + (b - a) * f; + } + } + Ok(out) + } +} + +pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option> { + if !sniff(&bytes) { + return None; + } + Some(AviSource::open(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into)) +} + +/// [`filmcraft_media::ReaderOpener`] for AVI. +pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedReader) -> Option> { + if !sniff(head) { + return None; + } + Some(AviSource::open_reader(name, reader.clone()).map(|s| Arc::new(s) as SharedSource).map_err(Into::into)) +} diff --git a/crates/codecs/src/avi_tests.rs b/crates/codecs/src/avi_tests.rs new file mode 100644 index 00000000..714eda19 --- /dev/null +++ b/crates/codecs/src/avi_tests.rs @@ -0,0 +1,232 @@ +//! AVI files made by ffmpeg (skipped without it), checked against ffmpeg's decode of the same file: +//! bit-exact where the decoder is (H.264, uncompressed video, PCM), within decoder tolerance where +//! it differs (JPEG, MP3 / MP2 / AC-3). + +use std::path::Path; +use std::process::Command; +use std::sync::Arc; + +use filmcraft_media::FrameRequest; + +use crate::tests::{ffmpeg, fixture_path, reference_yuv, yuv_bytes}; + +const PIC: [&str; 4] = ["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2"]; +const TONE: [&str; 4] = ["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=2"]; + +fn avi(name: &str, video: &[&str], audio: &[&str]) -> Option<(std::path::PathBuf, filmcraft_media::SharedSource)> { + let mut args: Vec<&str> = PIC.to_vec(); + args.extend_from_slice(&TONE); + args.extend_from_slice(video); + args.extend_from_slice(audio); + let path = fixture_path(name, &args)?; + let bytes: Arc<[u8]> = std::fs::read(&path).ok()?.into(); + Some((path.clone(), crate::open_bytes(name, bytes).unwrap_or_else(|e| panic!("{name}: {e}")))) +} + +/// ffmpeg's decode of the first audio stream as interleaved f32 at its own rate. +fn reference_audio(path: &Path) -> Vec { + let out = Command::new(ffmpeg().unwrap()) + .args(["-v", "error", "-i"]) + .arg(path) + .args(["-map", "0:a:0", "-f", "f32le", "-c:a", "pcm_f32le", "-"]) + .output() + .unwrap(); + out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect() +} + +/// The first `n` mono samples of stream 0 as FilmCraft decodes them. +fn ours(src: &filmcraft_media::SharedSource, n: usize) -> Vec { + src.audio(0, n, 48_000).unwrap().channels[0].clone() +} + +fn check_info(name: &str, src: &filmcraft_media::SharedSource, vcodec: &str, acodec: &str) { + let info = src.info(); + assert_eq!(info.container, "AVI", "{name}"); + let v = info.video.as_ref().unwrap(); + assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, vcodec), "{name}"); + assert_eq!(v.frame_rate.as_f64(), 25.0, "{name}"); + let a = &info.audio_streams[0]; + assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (48_000, 1, acodec), "{name}"); + let secs = info.duration.seconds(); + assert!((secs - 2.0).abs() < 0.1, "{name}: duration {secs}"); +} + +#[test] +fn h264_frames_are_bit_exact_with_random_access_and_mp3_plays() { + let Some((path, src)) = + avi("avi_h264_mp3.avi", &["-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p"], &["-c:a", "libmp3lame", "-b:a", "128k"]) + else { + return; + }; + check_info("h264", &src, "H.264", "MP3"); + let reference = reference_yuv(&path, "yuv420p").unwrap(); + let frame = reference.len() / 50; + let rate = src.info().frame_rate(); + for k in [37, 3, 24, 49, 0, 13, 12, 11, 30, 31, 32] { + let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap(); + assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "H.264 frame {k} differs from ffmpeg"); + } + // MP3 (a different decoder): the tone, in time with ffmpeg's decode + let r = reference_audio(&path); + let a = ours(&src, 48_000); + let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200); + // ffmpeg's test tone is at 1/8 of full scale + assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "MP3: tone {rms_ref}, difference {rms_diff}"); + // random access into the middle matches sequential decoding + let mid = src.audio(24_000, 4_800, 48_000).unwrap().channels[0].clone(); + let worst = mid.iter().zip(&a[24_000..28_800]).map(|(x, y)| (x - y).abs()).fold(0.0, f32::max); + assert!(worst < 0.01, "MP3 seek: {worst}"); +} + +fn rms_and_diff(reference: &[f32], ours: &[f32], range: std::ops::Range) -> (f32, f32) { + let n = range.len() as f32; + let rms = (range.clone().map(|i| reference[i].powi(2)).sum::() / n).sqrt(); + let diff = (range.map(|i| (reference[i] - ours[i]).powi(2)).sum::() / n).sqrt(); + (rms, diff) +} + +#[test] +fn mjpeg_and_pcm() { + let Some((path, src)) = avi("avi_mjpeg_pcm.avi", &["-c:v", "mjpeg", "-q:v", "3", "-pix_fmt", "yuvj420p"], &["-c:a", "pcm_s16le"]) else { return }; + check_info("mjpeg", &src, "Motion JPEG", "PCM 16-bit"); + let reference = reference_yuv(&path, "rgba").unwrap(); + let frame = reference.len() / 50; + let rate = src.info().frame_rate(); + for k in [10, 0, 49] { + let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap().to_rgba8(); + let r = &reference[k * frame..(k + 1) * frame]; + let mean = f.iter().zip(r).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::() as f64 / f.len() as f64; + // another JPEG decoder and chroma upsampling than ffmpeg's: close, not identical + assert!(mean < 6.0, "JPEG frame {k}: mean difference {mean}"); + } + // PCM: exactly ffmpeg's samples + let r = reference_audio(&path); + assert_eq!(ours(&src, 48_000), r[..48_000].to_vec()); +} + +#[test] +fn uncompressed_video_is_bit_exact() { + for (name, pix, codec, reference_fmt) in [ + ("avi_bgr24.avi", "bgr24", "Uncompressed RGB 24-bit", "rgba"), + ("avi_yuyv.avi", "yuyv422", "Uncompressed YUV 4:2:2", "yuv422p"), + ("avi_i420.avi", "yuv420p", "Uncompressed YUV 4:2:0", "yuv420p"), + ("avi_nv12.avi", "nv12", "Uncompressed YUV 4:2:0", "yuv420p"), + ] { + let Some((path, src)) = avi(name, &["-c:v", "rawvideo", "-pix_fmt", pix], &["-c:a", "pcm_s16le"]) else { return }; + check_info(name, &src, codec, "PCM 16-bit"); + let reference = reference_yuv(&path, reference_fmt).unwrap(); + let frame = reference.len() / 50; + let rate = src.info().frame_rate(); + for k in [7, 0, 49] { + let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap(); + let got = if reference_fmt == "rgba" { f.to_rgba8() } else { yuv_bytes(&f) }; + assert!(got == reference[k * frame..(k + 1) * frame], "{name}: frame {k} differs from ffmpeg"); + } + } +} + +#[test] +fn mp2_and_ac3_audio() { + for (name, codec, label) in [("avi_mp2.avi", "mp2", "MPEG Audio"), ("avi_ac3.avi", "ac3", "AC-3")] { + let Some((path, src)) = avi(name, &["-c:v", "mjpeg"], &["-c:a", codec, "-b:a", "192k"]) else { return }; + assert_eq!(src.info().audio_streams[0].codec, label, "{name}"); + let r = reference_audio(&path); + let a = ours(&src, 48_000); + let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200); + assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "{name}: tone {rms_ref}, difference {rms_diff}"); + } +} + +/// Dropped frames are stored as empty chunks: their slot shows the picture before. +#[test] +fn drop_frames_hold_the_previous_picture() { + let args = [ + "-f", + "lavfi", + "-i", + "testsrc2=s=160x120:r=25:d=2", + "-vf", + "select='not(eq(n\\,10))'", + "-fps_mode", + "passthrough", + "-c:v", + "rawvideo", + "-pix_fmt", + "bgr24", + ]; + let Some(path) = fixture_path("avi_drop.avi", &args) else { return }; + let src = crate::open_bytes("avi_drop.avi", std::fs::read(&path).unwrap().into()).unwrap(); + let rate = src.info().frame_rate(); + let at = |k: i64| src.video_frame(FrameRequest::full(rate.tick_of(k))).unwrap().to_rgba8(); + assert!(at(10) == at(9), "the dropped slot holds frame 9"); + assert!(at(11) != at(9)); +} + +/// DivX / Xvid has no decoder yet: the file opens, the codec is named, the picture reports it, and +/// the sound plays. +#[test] +fn mpeg4_part2_is_named_and_its_audio_plays() { + let Some((_, src)) = avi("avi_xvid.avi", &["-c:v", "mpeg4", "-vtag", "XVID"], &["-c:a", "pcm_s16le"]) else { return }; + assert_eq!(src.info().video.as_ref().unwrap().codec, "MPEG-4 Part 2 (DivX / Xvid)"); + let e = src.video_frame(FrameRequest::full(filmcraft_time::Tick::ZERO)).unwrap_err().to_string(); + assert!(e.contains("MPEG-4 Part 2"), "{e}"); + assert!(ours(&src, 4_800).iter().any(|s| s.abs() > 0.1)); +} + +/// A Motion JPEG frame without Huffman tables (the AVI1 convention) decodes with the standard ones: +/// exactly as the same frame with them. +#[test] +fn jpeg_without_huffman_tables_uses_the_standard_ones() { + let Some(path) = fixture_path( + "default_huffman.jpg", + &["-f", "lavfi", "-i", "testsrc2=s=160x120", "-frames:v", "1", "-c:v", "mjpeg", "-huffman", "default", "-q:v", "3"], + ) else { + return; + }; + let jpeg = std::fs::read(&path).unwrap(); + // remove every DHT segment + let mut stripped = jpeg[..2].to_vec(); + let mut at = 2; + while at + 4 <= jpeg.len() && jpeg[at] == 0xFF && jpeg[at + 1] != 0xDA { + let len = u16::from_be_bytes([jpeg[at + 2], jpeg[at + 3]]) as usize; + if jpeg[at + 1] != 0xC4 { + stripped.extend_from_slice(&jpeg[at..at + 2 + len]); + } + at += 2 + len; + } + stripped.extend_from_slice(&jpeg[at..]); + assert!(stripped.len() < jpeg.len() && !stripped.windows(2).any(|w| w == [0xFF, 0xC4])); + use crate::video::VideoDecoder; + let mut d = crate::video::MjpegDecoder; + let a = d.decode(&jpeg, 0).unwrap().remove(0).frame.to_rgba8(); + let b = d.decode(&stripped, 0).unwrap().remove(0).frame.to_rgba8(); + assert!(a == b); + // the inserted segment is 418 bytes and well-formed: 4 tables of 12, 12, 162, 162 codes + let with = crate::video::with_default_huffman(&stripped); + assert_eq!(with.len(), stripped.len() + 420); +} + +/// H.264 with B-frames copied from an MP4 whose time base is 1/600 (ffmpeg keeps it): 600 frame +/// slots a second, a picture in every 24th at 25 fps, empty drop chunks between. The pictures stay +/// at their times in presentation order, and the stream reports the rate they play at. +#[test] +fn copied_h264_keeps_its_timing_with_a_fine_time_base() { + let mp4 = + ["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2", "-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p", "-video_track_timescale", "600"]; + let Some(src_mp4) = fixture_path("avi_copy_source.mp4", &mp4) else { return }; + let src_path = src_mp4.to_string_lossy().into_owned(); + let Some(path) = fixture_path("avi_copy_600.avi", &["-i", &src_path, "-c:v", "copy", "-bsf:v", "h264_mp4toannexb", "-f", "avi"]) else { return }; + let src = crate::open_bytes("avi_copy_600.avi", std::fs::read(&path).unwrap().into()).unwrap(); + let v = src.info().video.clone().unwrap(); + assert_eq!(v.frame_rate.as_f64(), 25.0, "the rate the pictures play at, not the 600 slots"); + let reference = reference_yuv(&path, "yuv420p").unwrap(); + let frame = reference.len() / 50; + for k in [37usize, 3, 24, 49, 0, 13, 12, 11, 30] { + let at = filmcraft_time::Tick::from_rational(k as i64, 1, 25); + let f = src.video_frame(FrameRequest::full(at)).unwrap(); + assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "picture {k} at {:.2} s differs from ffmpeg", k as f64 / 25.0); + } + // between pictures, the one before holds + let mid = src.video_frame(FrameRequest::full(filmcraft_time::Tick::from_rational(1, 1, 50))).unwrap(); + assert!(yuv_bytes(&mid) == reference[0..frame]); +} diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs index f7de9e3d..05f79e91 100644 --- a/crates/codecs/src/lib.rs +++ b/crates/codecs/src/lib.rs @@ -13,6 +13,8 @@ //! - [`MpegSource`]: MPEG-2 transport streams (`.ts`, `.m2ts`, `.mts`), program streams (`.mpg`, //! `.vob`, `.mod`) and MPEG-1/2 video elementary streams: MPEG-1/2, H.264 and HEVC video; MPEG //! audio, AAC (ADTS / LATM), AC-3 and LPCM. +//! - [`AviSource`]: AVI (OpenDML included): Motion JPEG, H.264, HEVC and uncompressed video; PCM, +//! MP3 / MP2 and AC-3 audio. //! - [`AudioFileSource`]: standalone compressed audio files (MP3, FLAC, AIFF, …). //! - [`openers`]: the openers to register with the engine's media pool. @@ -20,6 +22,7 @@ pub mod apv; pub mod audio; +pub mod avi; pub mod gop; pub mod hw; mod hw_frame; @@ -36,6 +39,7 @@ use std::sync::{Arc, RwLock}; pub use apv::ApvSource; pub use audio::AudioFileSource; +pub use avi::AviSource; pub use gop::{FRAME_BUDGET, GopStats, cached_bytes, gop_stats, live_decoders}; pub use mkv::MkvSource; pub use mp4::Mp4Source; @@ -125,12 +129,22 @@ pub fn software_video_decoder(entry: &filmcraft_isobmff::SampleEntry) -> Result< /// Openers for the engine's media pool (MP4/MOV, Matroska/WebM, MXF, Ogg Opus/Vorbis, MPEG TS/PS and /// MPEG-1/2 video elementary streams, APV raw bitstreams, standalone audio). pub fn openers() -> Vec { - vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, audio::opener] + vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, avi::opener, audio::opener] } fn reader_registry() -> &'static RwLock> { static R: std::sync::OnceLock>> = std::sync::OnceLock::new(); - R.get_or_init(|| RwLock::new(vec![mp4::reader_opener, mkv::reader_opener, mxf::reader_opener, ogg::reader_opener, mpeg::reader_opener, apv::reader_opener])) + R.get_or_init(|| { + RwLock::new(vec![ + mp4::reader_opener, + mkv::reader_opener, + mxf::reader_opener, + ogg::reader_opener, + mpeg::reader_opener, + apv::reader_opener, + avi::reader_opener, + ]) + }) } /// Openers that read containers through a [`filmcraft_media::ByteReader`] (index now, samples on @@ -161,6 +175,15 @@ impl filmcraft_isobmff::ByteSource for Src { } } +impl filmcraft_avi::ByteSource for Src { + fn len(&self) -> u64 { + self.0.len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> { + self.0.read_at(offset, buf) + } +} + impl filmcraft_matroska::ByteSource for Src { fn len(&self) -> u64 { self.0.len() @@ -178,6 +201,8 @@ pub fn open_bytes(name: &str, bytes: Arc<[u8]>) -> std::result::Result Option<&'static str> { +pub(crate) fn ffmpeg() -> Option<&'static str> { ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].into_iter().find(|p| std::path::Path::new(p).exists()) } @@ -274,13 +274,13 @@ fn matroska_reordered_video_without_default_duration_keeps_its_rate() { /// Path of a fixture under `target/fixtures/codecs/`, generated with ffmpeg when missing /// (`None` when ffmpeg or the encoder is unavailable). -fn fixture_path(name: &str, args: &[&str]) -> Option { +pub(crate) fn fixture_path(name: &str, args: &[&str]) -> Option { fixture(name, args)?; Some(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/codecs").join(name)) } /// ffmpeg's decode of `src` as raw planar video (the oracle). -fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option> { +pub(crate) fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option> { let name = format!("{}.{pix_fmt}.yuv", src.file_name()?.to_str()?); let path = src.to_str()?; let b = fixture(&name, &["-i", path, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", pix_fmt])?; @@ -288,7 +288,7 @@ fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option> { } /// Planes of a decoded YUV frame as little-endian bytes (ffmpeg rawvideo layout). -fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec { +pub(crate) fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec { match &f.data { filmcraft_frame::PixelData::Yuv8 { planes, .. } => planes.iter().flat_map(|p| p.iter().copied()).collect(), filmcraft_frame::PixelData::Yuv16 { planes, .. } => planes.iter().flat_map(|p| p.iter().flat_map(|s| s.to_le_bytes())).collect(), diff --git a/crates/codecs/src/video.rs b/crates/codecs/src/video.rs index e646c474..5f371f43 100644 --- a/crates/codecs/src/video.rs +++ b/crates/codecs/src/video.rs @@ -181,13 +181,69 @@ pub fn hvcc_length_size_and_tid(hvcc: &[u8]) -> (usize, Option) { /// A factory returns `None` when it does not handle the entry. pub type VideoDecoderFactory = fn(&SampleEntry) -> Option>>; +/// The typical Huffman tables of ITU-T T.81 Annex K.3 as one DHT segment (luminance and +/// chrominance, DC and AC): what a Motion JPEG frame without its own tables is coded with (AVI's +/// "MJPG", the AVI1 convention of capture cards and cameras). +const DEFAULT_DHT: &[u8] = &[ + 0xFF, 0xC4, 0x01, 0xA2, // + 0x00, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, // + 0x01, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, // + 0x10, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7D, // + 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1, 0x08, // + 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, // + 0x29, 0x2A, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, // + 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, // + 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, // + 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xE1, 0xE2, // + 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, // + 0x11, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77, // + 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, // + 0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62, 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25, 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, // + 0x27, 0x28, 0x29, 0x2A, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, // + 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, // + 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, // + 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, // + 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, +]; + +/// A JPEG with the [`DEFAULT_DHT`] inserted before its first scan when it defines no Huffman +/// table of its own (else unchanged, borrowed). +pub fn with_default_huffman(jpeg: &[u8]) -> std::borrow::Cow<'_, [u8]> { + use std::borrow::Cow; + if jpeg.get(..2) != Some(&[0xFF, 0xD8]) { + return Cow::Borrowed(jpeg); + } + // walk the marker segments up to the start of scan + let mut at = 2usize; + while let (Some(&0xFF), Some(&m)) = (jpeg.get(at), jpeg.get(at + 1)) { + match m { + 0xFF => at += 1, // fill byte + 0xC4 => return Cow::Borrowed(jpeg), + 0xDA => { + let mut out = Vec::with_capacity(jpeg.len() + DEFAULT_DHT.len()); + out.extend_from_slice(jpeg.get(..at).unwrap_or_default()); + out.extend_from_slice(DEFAULT_DHT); + out.extend_from_slice(jpeg.get(at..).unwrap_or_default()); + return Cow::Owned(out); + } + 0xD0..=0xD9 | 0x01 => at += 2, + _ => { + let Some(len) = jpeg.get(at + 2..at + 4).map(|b| usize::from(u16::from_be_bytes([b[0], b[1]]))) else { break }; + at += 2 + len; + } + } + } + Cow::Borrowed(jpeg) +} + /// Motion-JPEG / Photo-JPEG (each sample is a complete JPEG). pub struct MjpegDecoder; impl VideoDecoder for MjpegDecoder { fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + let sample = with_default_huffman(sample); // mjpa samples may contain two fields; decode the first image (field-merging lands with interlace support). - let img = image::load_from_memory_with_format(sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?; + let img = image::load_from_memory_with_format(&sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?; let rgba = img.to_rgba8(); let (w, h) = rgba.dimensions(); Ok(vec![DecodedFrame { pts, frame: VideoFrame::rgba8(w, h, rgba.into_raw()), draft: false }]) @@ -1038,3 +1094,165 @@ mod disposable_tests { assert!(!hevc_disposable(&sample(&[&nal(0, 1)]), 4, None)); } } + +/// Uncompressed video layouts (AVI `BI_RGB` and YUV FourCCs). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RawFormat { + /// 24-bit B, G, R; rows padded to 4 bytes. + Bgr24, + /// 32-bit B, G, R, X. + Bgrx32, + /// Packed 4:2:2: Y0 U Y1 V (`YUY2`, `YUYV`). + Yuy2, + /// Packed 4:2:2: U Y0 V Y1 (`UYVY`, `2vuy`). + Uyvy, + /// Planar 4:2:0: Y, U, V (`I420`, `IYUV`). + I420, + /// Planar 4:2:0: Y, V, U (`YV12`). + Yv12, + /// Y plane, interleaved U V at half resolution (`NV12`). + Nv12, + /// Luma only (`Y800`, `GREY`, `Y8 `). + Gray8, +} + +impl RawFormat { + /// The layout of an AVI video stream: `BI_RGB` with its bit count, or a YUV FourCC. + pub fn from_avi(compression: [u8; 4], bit_count: u16) -> Option { + if matches!(u32::from_le_bytes(compression), 0 | 3) { + return match bit_count { + 24 => Some(RawFormat::Bgr24), + 32 => Some(RawFormat::Bgrx32), + _ => None, + }; + } + let mut upper = compression; + upper.make_ascii_uppercase(); + Some(match &upper { + b"YUY2" | b"YUYV" | b"YUNV" | b"V422" => RawFormat::Yuy2, + b"UYVY" | b"2VUY" | b"UYNV" | b"HDYC" => RawFormat::Uyvy, + b"I420" | b"IYUV" => RawFormat::I420, + b"YV12" => RawFormat::Yv12, + b"NV12" => RawFormat::Nv12, + b"Y800" | b"GREY" | b"GRAY" | b"Y8 " => RawFormat::Gray8, + _ => return None, + }) + } + + pub fn label(self) -> &'static str { + match self { + RawFormat::Bgr24 => "Uncompressed RGB 24-bit", + RawFormat::Bgrx32 => "Uncompressed RGB 32-bit", + RawFormat::Yuy2 | RawFormat::Uyvy => "Uncompressed YUV 4:2:2", + RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 => "Uncompressed YUV 4:2:0", + RawFormat::Gray8 => "Uncompressed greyscale", + } + } +} + +/// Uncompressed frames (each sample one picture of a [`RawFormat`]). +pub struct RawVideoDecoder { + format: RawFormat, + width: u32, + height: u32, + /// RGB rows are stored bottom-up (a positive `BITMAPINFOHEADER` height). + bottom_up: bool, +} + +impl RawVideoDecoder { + pub fn new(format: RawFormat, width: u32, height: u32, bottom_up: bool) -> Result { + if width == 0 || height == 0 || width > 16_384 || height > 16_384 { + return Err(CodecError::Unsupported(format!("{width}x{height} uncompressed video"))); + } + Ok(Self { format, width, height, bottom_up }) + } + + fn yuv(&self, planes: [Vec; 3], chroma: filmcraft_frame::Chroma) -> VideoFrame { + use std::sync::Arc; + let [y, u, v] = planes; + VideoFrame { + width: self.width, + height: self.height, + data: filmcraft_frame::PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma, alpha: None }, + color: vui_color(self.width, self.height, 2, 2, 2, false), + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + } + } + + fn picture(&self, s: &[u8]) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let short = || CodecError::Decode(format!("{} frame of {} bytes is too short for {w}x{h}", self.format.label(), s.len())); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + Ok(match self.format { + RawFormat::Bgr24 | RawFormat::Bgrx32 => { + let bpp = if self.format == RawFormat::Bgr24 { 3 } else { 4 }; + let stride = (w * bpp).div_ceil(4) * 4; + if s.len() < stride * h { + return Err(short()); + } + let mut rgba = Vec::with_capacity(w * h * 4); + for row in 0..h { + let src = if self.bottom_up { h - 1 - row } else { row }; + for px in s.get(src * stride..src * stride + w * bpp).unwrap_or_default().chunks_exact(bpp) { + rgba.extend_from_slice(&[px[2], px[1], px[0], 255]); + } + } + VideoFrame::rgba8(self.width, self.height, rgba) + } + RawFormat::Yuy2 | RawFormat::Uyvy => { + let stride = cw * 4; + if s.len() < stride * h { + return Err(short()); + } + let (yo, uo, vo, y2) = if self.format == RawFormat::Yuy2 { (0, 1, 3, 2) } else { (1, 0, 2, 3) }; + let (mut y, mut u, mut v) = (Vec::with_capacity(w * h), Vec::with_capacity(cw * h), Vec::with_capacity(cw * h)); + for row in s.chunks_exact(stride).take(h) { + for (k, q) in row.as_chunks::<4>().0.iter().enumerate() { + y.push(q[yo]); + if 2 * k + 1 < w { + y.push(q[y2]); + } + u.push(q[uo]); + v.push(q[vo]); + } + } + self.yuv([y, u, v], filmcraft_frame::Chroma::C422) + } + RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 | RawFormat::Gray8 => { + let luma = w * h; + let need = if self.format == RawFormat::Gray8 { luma } else { luma + 2 * cw * ch }; + if s.len() < need { + return Err(short()); + } + let y = s.get(..luma).unwrap_or_default().to_vec(); + let (u, v) = match self.format { + RawFormat::Gray8 => (vec![128; cw * ch], vec![128; cw * ch]), + RawFormat::Nv12 => s.get(luma..luma + 2 * cw * ch).unwrap_or_default().as_chunks::<2>().0.iter().map(|p| (p[0], p[1])).unzip(), + _ => { + let a = s.get(luma..luma + cw * ch).unwrap_or_default().to_vec(); + let b = s.get(luma + cw * ch..luma + 2 * cw * ch).unwrap_or_default().to_vec(); + if self.format == RawFormat::Yv12 { (b, a) } else { (a, b) } + } + }; + self.yuv([y, u, v], filmcraft_frame::Chroma::C420) + } + }) + } +} + +impl VideoDecoder for RawVideoDecoder { + fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + Ok(vec![DecodedFrame { pts, frame: self.picture(sample)?, draft: false }]) + } + fn flush(&mut self) -> Vec { + Vec::new() + } + fn reset(&mut self) {} + fn name(&self) -> &str { + self.format.label() + } + fn intra_only(&self) -> bool { + true + } +} diff --git a/crates/engine/src/avi_import_tests.rs b/crates/engine/src/avi_import_tests.rs new file mode 100644 index 00000000..43d8a301 --- /dev/null +++ b/crates/engine/src/avi_import_tests.rs @@ -0,0 +1,58 @@ +//! #598: AVI files import (ffmpeg-made, skipped without it): the Project item describes the +//! streams, a sequence made from it plays picture and sound, and the Media Browser's streaming +//! probe opens it without reading it whole. + +use serde_json::json; + +use crate::Session; +use filmcraft_project::ItemId; + +fn fixture() -> Option { + let ffmpeg = filmcraft_testkit::oracle::ffmpeg_or_skip("avi import")?; + let path = filmcraft_testkit::fixtures_dir("engine/avi").join("h264_mp3.avi"); + filmcraft_testkit::fixtures::generate(&path, |tmp| { + std::process::Command::new(&ffmpeg) + .args([ + "-y", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x120:rate=25:duration=2", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=2", + ]) + .args(["-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p", "-c:a", "libmp3lame", "-f", "avi"]) + .arg(tmp) + .status() + .is_ok_and(|s| s.success()) + }) +} + +#[test] +fn avi_imports_plays_and_probes() { + let Some(path) = fixture() else { return }; + let mut s = Session::default(); + let r = s.execute("file.import", json!({"paths": [path.to_string_lossy()]})).unwrap(); + assert!(r["errors"].as_array().is_none_or(|e| e.is_empty()), "{r}"); + let item = ItemId(r["items"][0].as_u64().unwrap()); + let info = s.project.item(item).unwrap().as_media().unwrap().info.clone(); + assert_eq!(info.container, "AVI"); + let v = info.video.as_ref().unwrap(); + assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, "H.264")); + assert_eq!(info.audio_streams.first().map(|a| a.codec.as_str()), Some("MP3")); + assert!((info.duration.seconds() - 2.0).abs() < 0.1, "{}", info.duration.seconds()); + // a sequence from the clip has its picture and sound + s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap(); + let q = s.active_sequence().unwrap().clone(); + assert!(!q.video_tracks[0].items.is_empty() && !q.audio_tracks[0].items.is_empty()); + let src = s.source(item).unwrap(); + assert!(src.video_frame(filmcraft_media::FrameRequest::full(filmcraft_time::Tick(filmcraft_time::TICKS_PER_SECOND))).is_ok()); + assert!(src.audio(48_000, 4_800, 48_000).unwrap().channels[0].iter().any(|v| v.abs() > 0.05)); + // the Media Browser's probe streams it (index now, frames on demand) + let probed = s.media.probe_file(&path.to_string_lossy(), &*s.services).unwrap(); + assert_eq!(probed.info().container, "AVI"); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index b6b85243..f9829d4e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1174,6 +1174,8 @@ mod audio_placement_tests; #[cfg(test)] mod autosave_tests; #[cfg(test)] +mod avi_import_tests; +#[cfg(test)] mod clip_ops_tests; #[cfg(test)] mod color_tests; diff --git a/crates/engine/src/media_pool.rs b/crates/engine/src/media_pool.rs index 1126e893..8b4c4d95 100644 --- a/crates/engine/src/media_pool.rs +++ b/crates/engine/src/media_pool.rs @@ -70,7 +70,7 @@ impl Default for MediaPool { } /// Largest file [`MediaPool::probe_file`] reads whole when no streaming reader takes it (WAV, -/// MP3 and other byte-opened formats, or an unsupported container such as AVI). +/// MP3 and other byte-opened formats, or an unsupported container). pub const PROBE_WHOLE_FILE_MAX: u64 = 64 * 1024 * 1024; /// Cache key of a media clip's full-resolution reference ("" matches anything: sources inserted diff --git a/crates/mpegts/src/es.rs b/crates/mpegts/src/es.rs index eba1ffa0..88f03796 100644 --- a/crates/mpegts/src/es.rs +++ b/crates/mpegts/src/es.rs @@ -175,6 +175,20 @@ pub struct FrameInfo { const ADTS_RATES: [u32; 13] = [96_000, 88_200, 64_000, 48_000, 44_100, 32_000, 24_000, 22_050, 16_000, 12_000, 11_025, 8_000, 7_350]; /// Parse the frame header at the start of `b` (MPEG audio, ADTS, AC-3, E-AC-3). +/// Bytes of the audio frame whose header starts `b` (MPEG audio, ADTS, LATM, AC-3, E-AC-3), or +/// `None` without a valid header there. For containers that store frames in chunks that need not +/// follow them (AVI): the frames are found again with it. +pub fn frame_bytes(codec: &Codec, b: &[u8]) -> Option { + let sync = match codec { + Codec::MpegAudio => AudioSync::Mpa, + Codec::AacAdts => AudioSync::Adts, + Codec::AacLatm => AudioSync::Loas, + Codec::Ac3 | Codec::Eac3 => AudioSync::Ac3, + _ => return None, + }; + frame_len(sync, b) +} + pub fn frame_info(codec: &Codec, b: &[u8]) -> Option { match codec { Codec::MpegAudio => { diff --git a/crates/mpegts/src/lib.rs b/crates/mpegts/src/lib.rs index 074d3951..4330ebcb 100644 --- a/crates/mpegts/src/lib.rs +++ b/crates/mpegts/src/lib.rs @@ -21,7 +21,7 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] mod es; -pub use es::{FrameInfo, frame_info}; +pub use es::{FrameInfo, frame_bytes, frame_info}; mod pes; mod ps; mod ts; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 925c0b57..463fd86c 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -34,6 +34,7 @@ const LAYERS: &[(&str, u8)] = &[ ("color", 0), ("bitstream", 0), ("isobmff", 0), + ("avi", 0), ("matroska", 0), ("mxf", 0), ("cfb", 0),