shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1454 from vangel76/refactor-mpegts-simplify-optimize 

mpegts: simplify and avoid redundant work
This commit is contained in:
Brandon Thomas 2026-10-13 00:03:42 -04:00
parent eb27d5efff d034c58edd
commit e35a828c73
6 changed files with 251 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
crates/mpegts/src/es.rs
View File

@ -174,7 +174,6 @@ pub struct FrameInfo {
const ADTS_RATES: [u32; 13] = [96_000, 88_200, 64_000, 48_000, 44_100, 32_000, 24_000, 22_050, 16_000, 12_000, 11_025, 8_000, 7_350];
/// Parse the frame header at the start of `b` (MPEG audio, ADTS, AC-3, E-AC-3).
/// Bytes of the audio frame whose header starts `b` (MPEG audio, ADTS, LATM, AC-3, E-AC-3), or
/// `None` without a valid header there. For containers that store frames in chunks that need not
/// follow them (AVI): the frames are found again with it.

4
crates/mpegts/src/lib.rs
View File

@ -218,7 +218,7 @@ pub struct Unit {
/// No other unit depends on it (B pictures of MPEG-1/2 video, non-reference H.264/HEVC).
pub disposable: bool,
pub size: u32,
/// TS: index of the packet holding the first byte; PS: byte offset of its PES packet.
/// TS: byte offset of the packet holding the first byte; PS: byte offset of its PES packet.
pub(crate) pos: u64,
/// Offset of the first byte within that packet's elementary-stream payload.
pub(crate) offset: u32,
@ -345,3 +345,5 @@ impl Unwrap {
#[cfg(test)]
mod tests;
#[cfg(test)]
mod work_tests;

15
crates/mpegts/src/pes.rs
View File

@ -3,9 +3,6 @@
/// A parsed PES header.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct PesHeader {
pub stream_id: u8,
/// PES_packet_length (bytes after the length field; 0 = unbounded, video in TS).
pub packet_length: usize,
/// Bytes from the packet start code to the first payload byte.
pub header_len: usize,
/// Raw 33-bit timestamps.
@ -28,10 +25,8 @@ pub(crate) fn parse(b: &[u8]) -> Option<PesHeader> {
if b.len() < 6 || b[0] != 0 || b[1] != 0 || b[2] != 1 {
return None;
}
let stream_id = b[3];
let packet_length = u16::from_be_bytes([b[4], b[5]]) as usize;
if plain_stream(stream_id) {
return Some(PesHeader { stream_id, packet_length, header_len: 6, pts: None, dts: None });
if plain_stream(b[3]) {
return Some(PesHeader { header_len: 6, pts: None, dts: None });
}
if b.len() >= 9 && b[6] & 0xC0 == 0x80 {
// ISO/IEC 13818-1 syntax
@ -47,7 +42,7 @@ pub(crate) fn parse(b: &[u8]) -> Option<PesHeader> {
dts = Some(timestamp(&b[14..19]));
}
}
return Some(PesHeader { stream_id, packet_length, header_len, pts, dts });
return Some(PesHeader { header_len, pts, dts });
}
// ISO/IEC 11172-1 syntax: stuffing, STD buffer, timestamps
let mut p = 6;
@ -78,7 +73,7 @@ pub(crate) fn parse(b: &[u8]) -> Option<PesHeader> {
0 if c == 0x0F => p += 1,
_ => return None,
}
Some(PesHeader { stream_id, packet_length, header_len: p, pts, dts })
Some(PesHeader { header_len: p, pts, dts })
}
#[cfg(test)]
@ -98,7 +93,7 @@ mod tests {
b.extend(ts_bytes(1, dts));
b.push(0xAA);
let h = parse(&b).unwrap();
assert_eq!((h.stream_id, h.header_len, h.pts, h.dts), (0xE0, 19, Some(pts), Some(dts)));
assert_eq!((h.header_len, h.pts, h.dts), (19, Some(pts), Some(dts)));
// MPEG-1: stuffing, STD buffer, PTS only
let mut b = vec![0, 0, 1, 0xC0, 0, 20, 0xFF, 0xFF, 0x40, 0x20];
b.extend(ts_bytes(2, 90_000));

123
crates/mpegts/src/ps.rs
View File

@ -12,17 +12,33 @@ struct Win<'a, S: ByteSource + ?Sized> {
src: &'a S,
buf: Vec<u8>,
start: u64,
/// Bytes zero-filled by window refills (tests count them).
#[cfg(test)]
zero_filled: usize,
}
impl<'a, S: ByteSource + ?Sized> Win<'a, S> {
fn new(src: &'a S) -> Self {
Self { src, buf: Vec::new(), start: 0 }
Self {
src,
buf: Vec::new(),
start: 0,
#[cfg(test)]
zero_filled: 0,
}
}
/// The bytes [pos, pos + n) (fewer at the end of the file).
fn get(&mut self, pos: u64, n: usize) -> Result<&[u8]> {
let have = pos >= self.start && pos + n as u64 <= self.start + self.buf.len() as u64;
if !have {
let want = n.max(1 << 20);
// never more than the file still holds past `pos`: `read_some` would not fill it, and
// zeroing a megabyte for every packet near the end of the file is wasted work
let remaining = self.src.len().saturating_sub(pos);
let want = (n.max(1 << 20) as u64).min(remaining) as usize;
#[cfg(test)]
{
self.zero_filled += want.saturating_sub(self.buf.len());
}
self.buf.resize(want, 0);
let got = read_some(self.src, pos, &mut self.buf)?;
self.buf.truncate(got);
@ -119,13 +135,13 @@ pub(crate) fn scan<S: ByteSource + ?Sized>(src: &S, mpeg1: bool) -> Result<File>
let mut scr: Option<(i64, i64)> = None;
let mut resyncs = 0u64;
while pos + 4 <= len {
let head = win.get(pos, 32)?.to_vec();
let Some(plen) = packet_len(&head) else {
let head = win.get(pos, 32)?;
let Some(plen) = packet_len(head) else {
// resynchronise on the next pack header or PES start code
resyncs += 1;
let mut p = pos + 1;
let found = loop {
let w = win.get(p, 1 << 16)?.to_vec();
let w = win.get(p, 1 << 16)?;
if w.len() < 4 {
break None;
}
@ -162,7 +178,7 @@ pub(crate) fn scan<S: ByteSource + ?Sized>(src: &S, mpeg1: bool) -> Result<File>
scr = Some(scr.map_or((v, v), |(a, b): (i64, i64)| (a.min(v), b.max(v))));
} else if id == 0xBC {
// program stream map: elementary stream id → stream_type
let body = win.get(pos, plen)?.to_vec();
let body = win.get(pos, plen)?;
if body.len() >= 12 {
let info = u16::from_be_bytes([body[8], body[9]]) as usize;
let mut i = 10 + info;
@ -177,11 +193,11 @@ pub(crate) fn scan<S: ByteSource + ?Sized>(src: &S, mpeg1: bool) -> Result<File>
}
}
} else if id == 0xBD || (0xC0..=0xEF).contains(&id) || id == 0xFD {
let pkt = win.get(pos, plen)?.to_vec();
let pkt = win.get(pos, plen)?;
if pkt.len() < plen {
break; // truncated
}
if let Some(h) = pes::parse(&pkt) {
if let Some(h) = pes::parse(pkt) {
let payload = &pkt[h.header_len.min(plen)..];
let (sub, skip) = if id == 0xBD {
match payload.first() {
@ -242,14 +258,14 @@ pub(crate) fn read<S: ByteSource + ?Sized>(src: &S, st: &Stream, u: &Unit, out:
let len = src.len();
let mut win = Win::new(src);
while out.len() < need && pos + 4 <= len {
let head = win.get(pos, 32)?.to_vec();
let Some(plen) = packet_len(&head) else {
let head = win.get(pos, 32)?;
let Some(plen) = packet_len(head) else {
// corrupt: stop at what we have
return Ok(());
};
if head[3] == stream_id {
let pkt = win.get(pos, plen)?.to_vec();
if let Some(h) = pes::parse(&pkt) {
let pkt = win.get(pos, plen)?;
if let Some(h) = pes::parse(pkt) {
let payload = &pkt[h.header_len.min(pkt.len())..];
let matches = stream_id != 0xBD || payload.first().copied() == sub_id;
if matches {
@ -269,3 +285,86 @@ pub(crate) fn read<S: ByteSource + ?Sized>(src: &S, st: &Stream, u: &Unit, out:
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
/// The window refill before the zero-fill cap: always `max(n, 1 MiB)` bytes, counted when the
/// buffer has to grow.
struct OldWin<'a> {
src: &'a [u8],
buf: Vec<u8>,
start: u64,
zero_filled: usize,
}
impl OldWin<'_> {
fn get(&mut self, pos: u64, n: usize) -> Vec<u8> {
let have = pos >= self.start && pos + n as u64 <= self.start + self.buf.len() as u64;
if !have {
let want = n.max(1 << 20);
self.zero_filled += want.saturating_sub(self.buf.len());
self.buf.resize(want, 0);
let got = read_some(self.src, pos, &mut self.buf).unwrap();
self.buf.truncate(got);
self.start = pos;
}
let i = (pos - self.start) as usize;
let end = (i + n).min(self.buf.len());
self.buf[i.min(end)..end].to_vec()
}
}
fn data(len: usize) -> Vec<u8> {
(0..len).map(|i| (i * 31 + i / 7) as u8).collect()
}
#[test]
fn capped_window_returns_what_the_uncapped_window_returned() {
for len in [0usize, 1, 31, 32, 33, 1000, (1 << 20) - 1, 1 << 20, (1 << 20) + 5, (3 << 20) + 17] {
let d = data(len);
let mut new = Win::new(d.as_slice());
let mut old = OldWin { src: &d, buf: Vec::new(), start: 0, zero_filled: 0 };
let len64 = len as u64;
let mut positions = vec![
0,
1,
31,
32,
1 << 19,
1 << 20,
(1 << 20) - 1,
(1 << 20) + 1,
2 << 20,
len64.saturating_sub(40),
len64.saturating_sub(1),
len64,
len64 + 1,
len64 + 100,
];
positions.extend((0..40).map(|k| len64.saturating_sub(100) + k * 3));
for &pos in &positions {
for n in [0usize, 1, 4, 32, 33, 1000, 65_541, 1 << 20, (1 << 20) + 1, 2 << 20] {
assert_eq!(new.get(pos, n).unwrap(), old.get(pos, n).as_slice(), "len {len} pos {pos} n {n}");
}
}
}
}
#[test]
fn misses_near_the_end_zero_at_most_the_bytes_that_are_left() {
let len = (2 << 20) + 1000;
let d = data(len);
let mut new = Win::new(d.as_slice());
let mut old = OldWin { src: &d, buf: Vec::new(), start: 0, zero_filled: 0 };
// a PES packet whose length runs past the end of the file, asked for once per packet
let first = (2u64 << 20) as usize;
for k in 0..500 {
let pos = (first + k) as u64;
assert_eq!(new.get(pos, 2000).unwrap(), old.get(pos, 2000).as_slice());
}
assert!(new.zero_filled <= 1000, "zeroed {} bytes, only 1000 are left in the file", new.zero_filled);
assert!(old.zero_filled > 499 * ((1 << 20) - 1000), "the reference zeroes about a megabyte per miss: {}", old.zero_filled);
}
}

23
crates/mpegts/src/ts.rs
View File

@ -6,14 +6,29 @@ use crate::es::Splitter;
use crate::pes::{self, PesHeader};
use crate::{ByteSource, Codec, Error, File, Format, Result, Stream, StreamId, Unit, Unwrap, read_some};
/// Byte-wise table for the MSB-first CRC-32 (polynomial 0x04C11DB7): entry `i` is the CRC register
/// after shifting the byte `i` through eight bit steps.
const CRC_TABLE: [u32; 256] = {
let mut t = [0u32; 256];
let mut i = 0;
while i < 256 {
let mut crc = (i as u32) << 24;
let mut bit = 0;
while bit < 8 {
crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
bit += 1;
}
t[i] = crc;
i += 1;
}
t
};
/// CRC-32 of PSI sections (MSB-first, polynomial 0x04C11DB7, initial value all ones; Annex A).
pub(crate) fn crc32_mpeg(data: &[u8]) -> u32 {
let mut crc = 0xFFFF_FFFFu32;
for &b in data {
crc ^= (b as u32) << 24;
for _ in 0..8 {
crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
}
crc = (crc << 8) ^ CRC_TABLE[((crc >> 24) as u8 ^ b) as usize];
}
crc
}

113
crates/mpegts/src/work_tests.rs Normal file
View File

@ -0,0 +1,113 @@
//! Equality and counting tests for the redundant-work removals: the old implementation is kept
//! here as the reference, the new one must give the same results.
use super::*;
/// The bitwise CRC-32 that `ts::crc32_mpeg` replaced.
fn crc32_bitwise(data: &[u8]) -> u32 {
let mut crc = 0xFFFF_FFFFu32;
for &b in data {
crc ^= (b as u32) << 24;
for _ in 0..8 {
crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
}
}
crc
}
/// Deterministic xorshift bytes.
fn noise(seed: u32, n: usize) -> Vec<u8> {
let mut x = seed | 1;
(0..n)
.map(|_| {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
(x >> 8) as u8
})
.collect()
}
#[test]
fn table_crc_equals_the_bitwise_crc() {
// the standard check value of CRC-32/MPEG-2
assert_eq!(ts::crc32_mpeg(b"123456789"), 0x0376_E6E7);
assert_eq!(ts::crc32_mpeg(&[]), 0xFFFF_FFFF);
for b in 0..=255u8 {
assert_eq!(ts::crc32_mpeg(&[b]), crc32_bitwise(&[b]));
}
for n in [0usize, 1, 2, 3, 4, 5, 15, 16, 17, 183, 184, 1023, 1024, 4096] {
for seed in 1..6u32 {
let d = noise(seed * 7919 + n as u32, n);
assert_eq!(ts::crc32_mpeg(&d), crc32_bitwise(&d), "len {n} seed {seed}");
}
}
// a section followed by its own CRC checks to zero
let mut sec = noise(5, 40);
let crc = ts::crc32_mpeg(&sec);
sec.extend(crc.to_be_bytes());
assert_eq!(ts::crc32_mpeg(&sec), 0);
}
/// MPEG-1 layer II frame, 48 kHz 192 kb/s (576 bytes), distinct per `k` and `salt`.
fn mp2_frame(k: usize, salt: u8) -> Vec<u8> {
let mut v = vec![0xFF, 0xFD, 0xA4, 0x00];
v.extend((0..572).map(|i| (i as u8) ^ (k as u8) ^ salt));
v
}
fn pes_packet(stream_id: u8, payload: &[u8]) -> Vec<u8> {
let len = 3 + payload.len();
let mut v = vec![0, 0, 1, stream_id, (len >> 8) as u8, len as u8, 0x80, 0, 0];
v.extend_from_slice(payload);
v
}
/// Two audio streams interleaved in maximal PES packets over more than three window refills,
/// with a pack header per PES packet; MPEG-2 or MPEG-1 pack headers.
fn big_program_stream(mpeg1: bool, frames: usize) -> (Vec<u8>, Vec<Vec<u8>>, Vec<Vec<u8>>) {
let a: Vec<Vec<u8>> = (0..frames).map(|k| mp2_frame(k, 0)).collect();
let b: Vec<Vec<u8>> = (0..frames).map(|k| mp2_frame(k, 0x55)).collect();
let (ea, eb) = (a.concat(), b.concat());
let mut out = Vec::new();
for (ca, cb) in ea.chunks(60_000).zip(eb.chunks(60_000)) {
for (id, chunk) in [(0xC0, ca), (0xC1, cb)] {
if mpeg1 {
out.extend([0, 0, 1, 0xBA, 0x21, 0, 1, 0, 1, 0x80, 0, 1]);
} else {
out.extend([0, 0, 1, 0xBA, 0x44, 0, 4, 0, 4, 1, 1, 0x89, 0xC3, 0xF8]);
}
out.extend(pes_packet(id, chunk));
}
}
out.extend([0, 0, 1, 0xB9]);
(out, a, b)
}
#[test]
fn program_stream_with_large_pes_packets_gives_back_every_frame() {
for mpeg1 in [false, true] {
let (data, a, b) = big_program_stream(mpeg1, 2000); // about 1.2 MB per stream
assert!(data.len() > 2 << 20, "more than the 1 MiB window twice");
let f = open(&data).unwrap();
assert_eq!(f.format, Format::Ps { mpeg1 });
assert_eq!(f.streams.len(), 2);
for (s, frames) in f.streams.iter().zip([&a, &b]) {
assert_eq!(s.units.len(), frames.len());
for (i, fr) in frames.iter().enumerate() {
assert_eq!(&f.read_unit(&data, f.streams.iter().position(|x| std::ptr::eq(x, s)).unwrap(), i).unwrap(), fr, "mpeg1 {mpeg1} unit {i}");
}
}
// cut inside the last packets: never panics, and the units before the cut are intact
for cut in [data.len() - 1, data.len() - 4, data.len() - 30_000, data.len() - 61_000, 100_000, 1 << 20] {
let d = &data[..cut];
let Ok(f) = open(d) else { continue };
for (si, frames) in [&a, &b].into_iter().enumerate() {
let Some(s) = f.streams.get(si) else { continue };
for i in 0..s.units.len().saturating_sub(1) {
assert_eq!(f.read_unit(d, si, i).unwrap(), frames[i], "cut {cut} stream {si} unit {i}");
}
}
}
}
}