From be94112c8e3cc9d7af96c3be2eee40696d637bdc Mon Sep 17 00:00:00 2001 From: cyfung1031 <44498510+cyfung1031@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:21:20 +0900 Subject: [PATCH] M11: strip redundant ASCII stream wrappers during optimization (#989) * cos: decode only the first filter of a stream, strictly Stream::decoded_first_filter_strict decodes the outer filter of a chain and leaves the rest of the chain encoded, within a caller bound. The optimizer uses it to remove an ASCII85 or ASCIIHex wrapper without inflating the inner data. Co-Authored-By: Claude Haiku 5.5 * optimize: strip redundant ASCII wrappers from filtered streams A stream whose filter array starts with ASCII85Decode or ASCIIHexDecode loses that outer layer when the serialized replacement is smaller. The inner encoded bytes are kept exactly, and the decoded stream is unchanged. Streams with DecodeParms, image, object stream, or Crypt filters are left alone, as are streams whose wrapper fails strict decoding. Co-Authored-By: Claude Haiku 5.5 * optimize: document redundant ASCII-wrapper cleanup Co-Authored-By: Claude Haiku 5.5 --------- Co-authored-by: Claude Haiku 5.5 --- crates/cos/src/object.rs | 27 ++++++++ crates/optimize/README.md | 4 +- crates/optimize/src/lib.rs | 56 +++++++++++++--- crates/optimize/tests/optimize.rs | 104 ++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 10 deletions(-) diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 7053fd2f..dc7648d4 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -230,6 +230,17 @@ impl Stream { pdfcraft_filters::decode(&chain, &self.raw, MAX_DECODED).map_err(|e| CosError::Filter(e.to_string())) } + /// Data after the first filter alone, bounded by `max`: the rest of the chain stays encoded. + /// Any corruption in that first filter is an error. + pub fn decoded_first_filter_strict(&self, max: usize) -> Result, CosError> { + let mut chain = self.filters(); + chain.truncate(1); + if chain.is_empty() { + return self.raw_within(max); + } + pdfcraft_filters::decode(&chain, &self.raw, max.min(MAX_DECODED)).map_err(|e| CosError::Filter(e.to_string())) + } + fn raw_within(&self, max: usize) -> Result, CosError> { let max = max.min(MAX_DECODED); // An empty decoder chain still produces an owned output buffer: check its limit @@ -470,4 +481,20 @@ mod tests { assert_eq!(stream.decoded_within(3).unwrap(), b"abc"); assert_eq!(stream.decoded_strict().unwrap(), b"abc"); } + + #[test] + fn first_filter_decodes_only_the_outer_wrapper() { + let data = "0 0 m 100 100 l S\n".repeat(64).into_bytes(); + let inner = Stream::flate(Dict::new(), &data).raw.to_vec(); + for (outer, name, corrupt) in [(Filter::Ascii85, "ASCII85Decode", &b"{~>"[..]), (Filter::AsciiHex, "ASCIIHexDecode", &b"G0>"[..])] { + let encoded = pdfcraft_filters::encode(&outer, &Params::default(), &inner).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name(name), Object::name("FlateDecode")])); + let stream = Stream::from_raw(dict.clone(), encoded); + assert_eq!(stream.decoded_first_filter_strict(usize::MAX).unwrap(), inner); + assert_eq!(stream.decoded_strict().unwrap(), data); + assert!(matches!(stream.decoded_first_filter_strict(inner.len() - 1), Err(CosError::Filter(_)))); + assert!(matches!(Stream::from_raw(dict, corrupt.to_vec()).decoded_first_filter_strict(usize::MAX), Err(CosError::Filter(_)))); + } + } } diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 87d06128..7c1c8187 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -16,7 +16,9 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going An image is replaced only if the result is smaller. Left alone: CMYK and other colour spaces, image masks, decode arrays, more than 8 bits, JPEG 2000, JBIG2 and CCITT. - **Discard objects:** thumbnails, alternate images, document tags, print settings. -- **Clean up:** Flate for streams with no filter. +- **Clean up:** Flate for streams with no filter; for streams with an outer ASCII85 or ASCIIHex + layer, that layer is removed when the result is smaller (the inner filters keep their encoded + bytes). Streams with `/DecodeParms`, images and object streams are left alone. The engine runs Remove Hidden Information (`pdfcraft-redact`) for the user-data categories, then this, then merges identical objects and writes a full, compressed save diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index 5ebd006c..7cfcb09c 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -10,7 +10,8 @@ //! - **Discard objects:** page thumbnails, alternate images, document tags (structure tree), //! print settings. //! - **Clean up:** drop images and forms that pages list but never draw (such as a deleted -//! image); Flate-compress streams that have no filter. +//! image); Flate-compress streams that have no filter; remove a redundant outer ASCII85 or +//! ASCIIHex layer from streams that have one. //! //! Metadata, attachments, comments, scripts, private data, hidden layers, bookmarks and form //! fields are discarded by `pdfcraft-redact`'s Remove Hidden Information, which the engine @@ -25,7 +26,7 @@ mod unused; pub use audit::{SpaceCategory, SpaceUse, audit_space}; -use pdfcraft_cos::{Document, ObjRef, Object, Stream}; +use pdfcraft_cos::{Document, ObjRef, Object, Stream, serialize}; pub use images::effective_resolutions; @@ -84,7 +85,8 @@ pub struct Settings { pub discard_alternate_images: bool, pub discard_tags: bool, pub discard_print_settings: bool, - /// Flate-compress streams stored without a filter. + /// Flate-compress streams stored without a filter, and strip a redundant outer ASCII85 or + /// ASCIIHex layer from streams that have one. pub flate_unencoded: bool, /// Clean Up: remove links (and bookmark destinations) that point nowhere. pub remove_invalid_links: bool, @@ -227,17 +229,53 @@ pub fn optimize_with_progress(doc: &mut Document, settings: &Settings, progress: let obj = doc.get(r); let Object::Stream(s) = &*obj else { continue }; // XMP stays readable as plain text (PDF/A requires it); tiny streams don't gain. - if s.dict.contains(b"Filter") || s.dict.name(b"Type") == Some(b"Metadata") || s.dict.name(b"Type") == Some(b"XRef") || s.raw.len() < 64 { + if s.dict.name(b"Type") == Some(b"Metadata") || s.dict.name(b"Type") == Some(b"XRef") || s.raw.len() < 64 { continue; } - let mut d = s.dict.clone(); - d.remove(b"Length"); - let packed = Stream::flate(d, &s.raw); - if packed.raw.len() < s.raw.len() { - doc.set(r, Object::Stream(packed)); + let replacement = if s.dict.contains(b"Filter") { + strip_ascii_wrapper(s) + } else { + let mut d = s.dict.clone(); + d.remove(b"Length"); + let packed = Stream::flate(d, &s.raw); + (packed.raw.len() < s.raw.len()).then_some(packed) + }; + if let Some(replacement) = replacement { + doc.set(r, Object::Stream(replacement)); report.streams_compressed += 1; } } } Ok(report) } + +/// `s` without its outer ASCII85 or ASCIIHex layer (`[/ASCII85Decode /FlateDecode]` becomes +/// `[/FlateDecode]`), when that is smaller. Only the outer layer is decoded: strictly, and within +/// the stored size, which ASCII text never falls below, so the inner encoded bytes stay exact. +fn strip_ascii_wrapper(s: &Stream) -> Option { + if s.dict.contains(b"DecodeParms") || s.dict.name(b"Type") == Some(b"ObjStm") || s.dict.name(b"Subtype") == Some(b"Image") { + return None; + } + let Some(Object::Array(filters)) = s.dict.get(b"Filter") else { return None }; + let names = filters.iter().map(Object::as_name).collect::>>()?; + let (outer, inner) = names.split_first()?; + if !matches!(*outer, b"ASCII85Decode" | b"ASCIIHexDecode") || inner.is_empty() || inner.contains(&&b"Crypt"[..]) { + return None; + } + let data = s.decoded_first_filter_strict(s.raw.len()).ok()?; + let mut dict = s.dict.clone(); + dict.remove(b"Length"); + dict.set(b"Filter".to_vec(), Object::Array(inner.iter().map(|n| Object::Name(n.to_vec())).collect())); + let candidate = Stream::from_raw(dict, data); + (stored_len(&candidate)? < stored_len(s)?).then_some(candidate) +} + +/// Bytes the stream object takes in the file: its dictionary with `/Length` set as the writer +/// sets it, then its data. +fn stored_len(s: &Stream) -> Option { + let mut dict = s.dict.clone(); + dict.set(b"Length".to_vec(), Object::Int(i64::try_from(s.raw.len()).ok()?)); + let mut out = Vec::new(); + serialize(&Object::Dict(dict), &mut out); + out.len().checked_add(s.raw.len()) +} diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index ff4dc801..ac675a39 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -583,3 +583,107 @@ fn nothing_is_dropped_when_a_stream_cannot_be_checked() { assert_eq!(build(s), kept, "{dict:?}"); } } + +/// `data` as ASCII85 text with the `~>` end marker, as ASCII85Decode reads it. +fn ascii85(data: &[u8]) -> Vec { + let mut out = Vec::new(); + for chunk in data.chunks(4) { + let mut word = [0u8; 4]; + word[..chunk.len()].copy_from_slice(chunk); + let mut v = u32::from_be_bytes(word); + let mut digits = [0u8; 5]; + for d in digits.iter_mut().rev() { + *d = (v % 85) as u8 + b'!'; + v /= 85; + } + out.extend_from_slice(&digits[..chunk.len() + 1]); + } + out.extend_from_slice(b"~>"); + out +} + +/// `data` as hex digits with the `>` end marker, as ASCIIHexDecode reads it. +fn ascii_hex(data: &[u8]) -> Vec { + let mut out: Vec = data.iter().flat_map(|b| format!("{b:02X}").into_bytes()).collect(); + out.push(b'>'); + out +} + +fn filtered(names: &[&str], raw: Vec) -> Stream { + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::Array(names.iter().map(|n| Object::name(n)).collect())); + Stream::from_raw(d, raw) +} + +/// A one-page document whose page content is `content`, with the content's reference. +fn with_content(content: Stream) -> (Document, ObjRef) { + let mut doc = Document::new_empty(); + let pg = page(&mut doc, &[], ""); + let c = doc.add(Object::Stream(content)); + doc.update_dict(pg, |d| d.set(b"Contents".to_vec(), Object::Ref(c))).unwrap(); + (doc, c) +} + +fn vector_ops() -> Vec { + "0 0 m 612 792 l S\n".repeat(400).into_bytes() +} + +#[test] +fn ascii_wrappers_come_off_filtered_streams_and_the_inner_data_is_unchanged() { + let ops = vector_ops(); + let inner = Stream::flate(Dict::new(), &ops).raw.to_vec(); + for (outer, encoded) in [("ASCII85Decode", ascii85(&inner)), ("ASCIIHexDecode", ascii_hex(&inner))] { + let source = filtered(&[outer, "FlateDecode"], encoded); + let (mut doc, c) = with_content(source.clone()); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.streams_compressed, 1, "{outer}"); + let after = stream(&doc, c); + assert_eq!(after.dict.get(b"Filter"), Some(&Object::Array(vec![Object::name("FlateDecode")])), "{outer}"); + assert_eq!(after.raw.to_vec(), inner, "{outer}: the inner Flate bytes are kept exactly"); + assert_eq!(after.decoded_strict().unwrap(), ops, "{outer}"); + assert!(after.raw.len() < source.raw.len(), "{outer}"); + } +} + +#[test] +fn streams_the_wrapper_stripping_does_not_touch_stay_unchanged() { + let inner = Stream::flate(Dict::new(), &vector_ops()).raw.to_vec(); + let wrapped = ascii85(&inner); + let mut with_parms = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + with_parms.dict.set(b"DecodeParms".to_vec(), Object::Dict(Dict::new())); + let mut image = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + image.dict.set(b"Subtype".to_vec(), Object::name("Image")); + let mut objstm = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + objstm.dict.set(b"Type".to_vec(), Object::name("ObjStm")); + let mut metadata = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + metadata.dict.set(b"Type".to_vec(), Object::name("Metadata")); + let cases = [ + ("malformed ASCII85", filtered(&["ASCII85Decode", "FlateDecode"], b"{{ not ASCII85 ~>".to_vec())), + ("malformed ASCIIHex", filtered(&["ASCIIHexDecode", "FlateDecode"], b"G0>".to_vec())), + ("DecodeParms", with_parms), + ("single ASCII filter", filtered(&["ASCII85Decode"], wrapped.clone())), + ("image", image), + ("object stream", objstm), + ("metadata", metadata), + ("crypt after the wrapper", filtered(&["ASCII85Decode", "Crypt", "FlateDecode"], wrapped.clone())), + ("tiny", filtered(&["ASCII85Decode", "FlateDecode"], ascii85(b"ab"))), + ("single Flate", Stream::flate(Dict::new(), &vector_ops())), + ]; + for (label, s) in cases { + let (mut doc, c) = with_content(s.clone()); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.streams_compressed, 0, "{label}"); + assert_eq!(stream(&doc, c), s, "{label}"); + } +} + +#[test] +fn a_second_run_leaves_the_stripped_stream_alone() { + let inner = Stream::flate(Dict::new(), &vector_ops()).raw.to_vec(); + let (mut doc, c) = with_content(filtered(&["ASCII85Decode", "FlateDecode"], ascii85(&inner))); + optimize(&mut doc, &Settings::default()).unwrap(); + let once = stream(&doc, c); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.streams_compressed, 0); + assert_eq!(stream(&doc, c), once); +}