diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 7053fd2f..dc7648d4 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -230,6 +230,17 @@ impl Stream { pdfcraft_filters::decode(&chain, &self.raw, MAX_DECODED).map_err(|e| CosError::Filter(e.to_string())) } + /// Data after the first filter alone, bounded by `max`: the rest of the chain stays encoded. + /// Any corruption in that first filter is an error. + pub fn decoded_first_filter_strict(&self, max: usize) -> Result, CosError> { + let mut chain = self.filters(); + chain.truncate(1); + if chain.is_empty() { + return self.raw_within(max); + } + pdfcraft_filters::decode(&chain, &self.raw, max.min(MAX_DECODED)).map_err(|e| CosError::Filter(e.to_string())) + } + fn raw_within(&self, max: usize) -> Result, CosError> { let max = max.min(MAX_DECODED); // An empty decoder chain still produces an owned output buffer: check its limit @@ -470,4 +481,20 @@ mod tests { assert_eq!(stream.decoded_within(3).unwrap(), b"abc"); assert_eq!(stream.decoded_strict().unwrap(), b"abc"); } + + #[test] + fn first_filter_decodes_only_the_outer_wrapper() { + let data = "0 0 m 100 100 l S\n".repeat(64).into_bytes(); + let inner = Stream::flate(Dict::new(), &data).raw.to_vec(); + for (outer, name, corrupt) in [(Filter::Ascii85, "ASCII85Decode", &b"{~>"[..]), (Filter::AsciiHex, "ASCIIHexDecode", &b"G0>"[..])] { + let encoded = pdfcraft_filters::encode(&outer, &Params::default(), &inner).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name(name), Object::name("FlateDecode")])); + let stream = Stream::from_raw(dict.clone(), encoded); + assert_eq!(stream.decoded_first_filter_strict(usize::MAX).unwrap(), inner); + assert_eq!(stream.decoded_strict().unwrap(), data); + assert!(matches!(stream.decoded_first_filter_strict(inner.len() - 1), Err(CosError::Filter(_)))); + assert!(matches!(Stream::from_raw(dict, corrupt.to_vec()).decoded_first_filter_strict(usize::MAX), Err(CosError::Filter(_)))); + } + } } diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 87d06128..7c1c8187 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -16,7 +16,9 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going An image is replaced only if the result is smaller. Left alone: CMYK and other colour spaces, image masks, decode arrays, more than 8 bits, JPEG 2000, JBIG2 and CCITT. - **Discard objects:** thumbnails, alternate images, document tags, print settings. -- **Clean up:** Flate for streams with no filter. +- **Clean up:** Flate for streams with no filter; for streams with an outer ASCII85 or ASCIIHex + layer, that layer is removed when the result is smaller (the inner filters keep their encoded + bytes). Streams with `/DecodeParms`, images and object streams are left alone. The engine runs Remove Hidden Information (`pdfcraft-redact`) for the user-data categories, then this, then merges identical objects and writes a full, compressed save diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index 5ebd006c..7cfcb09c 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -10,7 +10,8 @@ //! - **Discard objects:** page thumbnails, alternate images, document tags (structure tree), //! print settings. //! - **Clean up:** drop images and forms that pages list but never draw (such as a deleted -//! image); Flate-compress streams that have no filter. +//! image); Flate-compress streams that have no filter; remove a redundant outer ASCII85 or +//! ASCIIHex layer from streams that have one. //! //! Metadata, attachments, comments, scripts, private data, hidden layers, bookmarks and form //! fields are discarded by `pdfcraft-redact`'s Remove Hidden Information, which the engine @@ -25,7 +26,7 @@ mod unused; pub use audit::{SpaceCategory, SpaceUse, audit_space}; -use pdfcraft_cos::{Document, ObjRef, Object, Stream}; +use pdfcraft_cos::{Document, ObjRef, Object, Stream, serialize}; pub use images::effective_resolutions; @@ -84,7 +85,8 @@ pub struct Settings { pub discard_alternate_images: bool, pub discard_tags: bool, pub discard_print_settings: bool, - /// Flate-compress streams stored without a filter. + /// Flate-compress streams stored without a filter, and strip a redundant outer ASCII85 or + /// ASCIIHex layer from streams that have one. pub flate_unencoded: bool, /// Clean Up: remove links (and bookmark destinations) that point nowhere. pub remove_invalid_links: bool, @@ -227,17 +229,53 @@ pub fn optimize_with_progress(doc: &mut Document, settings: &Settings, progress: let obj = doc.get(r); let Object::Stream(s) = &*obj else { continue }; // XMP stays readable as plain text (PDF/A requires it); tiny streams don't gain. - if s.dict.contains(b"Filter") || s.dict.name(b"Type") == Some(b"Metadata") || s.dict.name(b"Type") == Some(b"XRef") || s.raw.len() < 64 { + if s.dict.name(b"Type") == Some(b"Metadata") || s.dict.name(b"Type") == Some(b"XRef") || s.raw.len() < 64 { continue; } - let mut d = s.dict.clone(); - d.remove(b"Length"); - let packed = Stream::flate(d, &s.raw); - if packed.raw.len() < s.raw.len() { - doc.set(r, Object::Stream(packed)); + let replacement = if s.dict.contains(b"Filter") { + strip_ascii_wrapper(s) + } else { + let mut d = s.dict.clone(); + d.remove(b"Length"); + let packed = Stream::flate(d, &s.raw); + (packed.raw.len() < s.raw.len()).then_some(packed) + }; + if let Some(replacement) = replacement { + doc.set(r, Object::Stream(replacement)); report.streams_compressed += 1; } } } Ok(report) } + +/// `s` without its outer ASCII85 or ASCIIHex layer (`[/ASCII85Decode /FlateDecode]` becomes +/// `[/FlateDecode]`), when that is smaller. Only the outer layer is decoded: strictly, and within +/// the stored size, which ASCII text never falls below, so the inner encoded bytes stay exact. +fn strip_ascii_wrapper(s: &Stream) -> Option { + if s.dict.contains(b"DecodeParms") || s.dict.name(b"Type") == Some(b"ObjStm") || s.dict.name(b"Subtype") == Some(b"Image") { + return None; + } + let Some(Object::Array(filters)) = s.dict.get(b"Filter") else { return None }; + let names = filters.iter().map(Object::as_name).collect::>>()?; + let (outer, inner) = names.split_first()?; + if !matches!(*outer, b"ASCII85Decode" | b"ASCIIHexDecode") || inner.is_empty() || inner.contains(&&b"Crypt"[..]) { + return None; + } + let data = s.decoded_first_filter_strict(s.raw.len()).ok()?; + let mut dict = s.dict.clone(); + dict.remove(b"Length"); + dict.set(b"Filter".to_vec(), Object::Array(inner.iter().map(|n| Object::Name(n.to_vec())).collect())); + let candidate = Stream::from_raw(dict, data); + (stored_len(&candidate)? < stored_len(s)?).then_some(candidate) +} + +/// Bytes the stream object takes in the file: its dictionary with `/Length` set as the writer +/// sets it, then its data. +fn stored_len(s: &Stream) -> Option { + let mut dict = s.dict.clone(); + dict.set(b"Length".to_vec(), Object::Int(i64::try_from(s.raw.len()).ok()?)); + let mut out = Vec::new(); + serialize(&Object::Dict(dict), &mut out); + out.len().checked_add(s.raw.len()) +} diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index ff4dc801..ac675a39 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -583,3 +583,107 @@ fn nothing_is_dropped_when_a_stream_cannot_be_checked() { assert_eq!(build(s), kept, "{dict:?}"); } } + +/// `data` as ASCII85 text with the `~>` end marker, as ASCII85Decode reads it. +fn ascii85(data: &[u8]) -> Vec { + let mut out = Vec::new(); + for chunk in data.chunks(4) { + let mut word = [0u8; 4]; + word[..chunk.len()].copy_from_slice(chunk); + let mut v = u32::from_be_bytes(word); + let mut digits = [0u8; 5]; + for d in digits.iter_mut().rev() { + *d = (v % 85) as u8 + b'!'; + v /= 85; + } + out.extend_from_slice(&digits[..chunk.len() + 1]); + } + out.extend_from_slice(b"~>"); + out +} + +/// `data` as hex digits with the `>` end marker, as ASCIIHexDecode reads it. +fn ascii_hex(data: &[u8]) -> Vec { + let mut out: Vec = data.iter().flat_map(|b| format!("{b:02X}").into_bytes()).collect(); + out.push(b'>'); + out +} + +fn filtered(names: &[&str], raw: Vec) -> Stream { + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::Array(names.iter().map(|n| Object::name(n)).collect())); + Stream::from_raw(d, raw) +} + +/// A one-page document whose page content is `content`, with the content's reference. +fn with_content(content: Stream) -> (Document, ObjRef) { + let mut doc = Document::new_empty(); + let pg = page(&mut doc, &[], ""); + let c = doc.add(Object::Stream(content)); + doc.update_dict(pg, |d| d.set(b"Contents".to_vec(), Object::Ref(c))).unwrap(); + (doc, c) +} + +fn vector_ops() -> Vec { + "0 0 m 612 792 l S\n".repeat(400).into_bytes() +} + +#[test] +fn ascii_wrappers_come_off_filtered_streams_and_the_inner_data_is_unchanged() { + let ops = vector_ops(); + let inner = Stream::flate(Dict::new(), &ops).raw.to_vec(); + for (outer, encoded) in [("ASCII85Decode", ascii85(&inner)), ("ASCIIHexDecode", ascii_hex(&inner))] { + let source = filtered(&[outer, "FlateDecode"], encoded); + let (mut doc, c) = with_content(source.clone()); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.streams_compressed, 1, "{outer}"); + let after = stream(&doc, c); + assert_eq!(after.dict.get(b"Filter"), Some(&Object::Array(vec![Object::name("FlateDecode")])), "{outer}"); + assert_eq!(after.raw.to_vec(), inner, "{outer}: the inner Flate bytes are kept exactly"); + assert_eq!(after.decoded_strict().unwrap(), ops, "{outer}"); + assert!(after.raw.len() < source.raw.len(), "{outer}"); + } +} + +#[test] +fn streams_the_wrapper_stripping_does_not_touch_stay_unchanged() { + let inner = Stream::flate(Dict::new(), &vector_ops()).raw.to_vec(); + let wrapped = ascii85(&inner); + let mut with_parms = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + with_parms.dict.set(b"DecodeParms".to_vec(), Object::Dict(Dict::new())); + let mut image = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + image.dict.set(b"Subtype".to_vec(), Object::name("Image")); + let mut objstm = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + objstm.dict.set(b"Type".to_vec(), Object::name("ObjStm")); + let mut metadata = filtered(&["ASCII85Decode", "FlateDecode"], wrapped.clone()); + metadata.dict.set(b"Type".to_vec(), Object::name("Metadata")); + let cases = [ + ("malformed ASCII85", filtered(&["ASCII85Decode", "FlateDecode"], b"{{ not ASCII85 ~>".to_vec())), + ("malformed ASCIIHex", filtered(&["ASCIIHexDecode", "FlateDecode"], b"G0>".to_vec())), + ("DecodeParms", with_parms), + ("single ASCII filter", filtered(&["ASCII85Decode"], wrapped.clone())), + ("image", image), + ("object stream", objstm), + ("metadata", metadata), + ("crypt after the wrapper", filtered(&["ASCII85Decode", "Crypt", "FlateDecode"], wrapped.clone())), + ("tiny", filtered(&["ASCII85Decode", "FlateDecode"], ascii85(b"ab"))), + ("single Flate", Stream::flate(Dict::new(), &vector_ops())), + ]; + for (label, s) in cases { + let (mut doc, c) = with_content(s.clone()); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.streams_compressed, 0, "{label}"); + assert_eq!(stream(&doc, c), s, "{label}"); + } +} + +#[test] +fn a_second_run_leaves_the_stripped_stream_alone() { + let inner = Stream::flate(Dict::new(), &vector_ops()).raw.to_vec(); + let (mut doc, c) = with_content(filtered(&["ASCII85Decode", "FlateDecode"], ascii85(&inner))); + optimize(&mut doc, &Settings::default()).unwrap(); + let once = stream(&doc, c); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.streams_compressed, 0); + assert_eq!(stream(&doc, c), once); +}