fix(codecs): large PNG exports no longer lose bytes at a band's sync flush (#827)

The parallel PNG encoder deflates bands of rows separately and joins them with
sync flushes. Each band started with an output buffer half its size and resumed
the flush when the buffer filled. miniz_oxide (flate2's Rust backend) can
return from a sync flush that filled the buffer before compressing its last
lookahead bytes; the next call only drains the pending output and still
reports the flush done. The band then lost up to 258 bytes, and every band
after it decoded as garbage ("invalid literal/length code"). Detailed images of
4 MB and up, such as a 3000x3000 illustration, exported unreadable PNGs.

deflate_band now gives each band an output buffer past deflate's worst case, so
one call consumes the band and completes the flush. If that single call does
not finish, the parallel path returns None and the serial encoder is used.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Brandon Thomas <echelon@gmail.com>
This commit is contained in:
ajsongpilot 2026-10-07 19:09:04 -06:00 committed by GitHub
parent 65ea2c973b
commit 5902c878e5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 42 additions and 16 deletions

View File

@ -264,22 +264,7 @@ fn parallel_idat(data: &[u8], w: usize, h: usize, bpp: usize, level: PngCompress
filter_row(row, prev, bpp, &mut scratch, &mut filtered);
}
let adler = adler32(1, &filtered);
let mut c = flate2::Compress::new(lvl, false);
let mut z = Vec::with_capacity(filtered.len() / 2 + 64);
let last = y1 == h;
let flush = if last { flate2::FlushCompress::Finish } else { flate2::FlushCompress::Sync };
loop {
let consumed = c.total_in() as usize;
if z.capacity() - z.len() < 64 * 1024 {
z.reserve(z.capacity().max(64 * 1024));
}
match c.compress_vec(&filtered[consumed..], &mut z, flush) {
Ok(flate2::Status::StreamEnd) => break,
Ok(_) if !last && c.total_in() as usize == filtered.len() && z.capacity() > z.len() => break,
Ok(_) => {}
Err(_) => return Vec::new(),
}
}
let Some(z) = deflate_band(&filtered, lvl, y1 == h) else { return Vec::new() };
out.push((i, z, adler, filtered.len()));
}
out
@ -308,6 +293,22 @@ fn parallel_idat(data: &[u8], w: usize, h: usize, bpp: usize, level: PngCompress
Some(out)
}
/// Raw deflate of one band: ends on a sync flush, or finishes the stream for the `last` band.
/// The output buffer is sized past deflate's worst case so that one call takes the whole band
/// and completes the flush: when a sync flush fills the buffer, miniz_oxide (flate2's Rust
/// backend) can return before compressing the last lookahead bytes, and the next call only
/// drains the pending output, so a resumed flush silently drops data. `None` (the caller then
/// uses the serial encoder) if the single call did not finish.
fn deflate_band(filtered: &[u8], lvl: flate2::Compression, last: bool) -> Option<Vec<u8>> {
let mut c = flate2::Compress::new(lvl, false);
// Stored blocks cost 5 bytes per 64 KiB, the flush an empty stored block: 1/8 is ample.
let mut z = Vec::with_capacity(filtered.len() + filtered.len() / 8 + 1024);
let flush = if last { flate2::FlushCompress::Finish } else { flate2::FlushCompress::Sync };
let status = c.compress_vec(filtered, &mut z, flush).ok()?;
let done = if last { status == flate2::Status::StreamEnd } else { c.total_in() as usize == filtered.len() && z.len() < z.capacity() };
done.then_some(z)
}
/// One scanline with the adaptive filter choice, appended to `out` (type byte + data).
fn filter_row(row: &[u8], prev: Option<&[u8]>, bpp: usize, scratch: &mut [u8], out: &mut Vec<u8>) {
let n = row.len();
@ -444,6 +445,31 @@ mod parallel_tests {
assert_eq!(adler32_combine(adler32(1, &a), adler32(1, &b), b.len()), adler32(1, &whole));
}
#[test]
fn bands_that_end_just_past_a_block_keep_all_their_data() {
// Deflate output ~0.6 of the input, cut just past one of miniz_oxide's block
// boundaries: a sync flush that fills the output buffer there used to lose the
// last lookahead bytes, corrupting every band after it.
for (k, len) in [(20u32, 136_916usize), (20, 345_044), (24, 195_796), (28, 189_142)] {
let mut x = 0x9E37_79B9u32 ^ k;
let data: Vec<u8> = (0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
((x >> 8) % k) as u8
})
.collect();
for last in [false, true] {
let z = deflate_band(&data, flate2::Compression::default(), last).unwrap();
let mut d = flate2::Decompress::new(false);
let mut back = Vec::with_capacity(len + 1);
d.decompress_vec(&z, &mut back, flate2::FlushDecompress::Sync).unwrap();
assert!(back == data, "k {k} len {len} last {last}: {} of {len} bytes", back.len());
}
}
}
#[test]
fn large_images_roundtrip_through_the_parallel_encoder() {
for (layout, sample, ch, bytes) in [(ChannelLayout::Rgb, SampleType::U8, 3usize, 1usize), (ChannelLayout::Rgba, SampleType::U16, 4, 2)] {