Fix #1704: an Affinity document's pixel layers share one pixel allowance

Each pixel layer was held to 64 megapixels, and the archive's limits count
the bytes extracted from it; a layer whose tiles are flat colour needs none,
so a small file with many of them allocated 256 MB each without end. The
layers of one document now decode 512 megapixels (2 GB) at most together;
those past it are left out with a note, as an oversized one is.

crates/affinity/src/model.rs

@@ -296,12 +296,23 @@ const MAX_TREE_DEPTH: usize = 128;
 /// Total layer nodes imported.
 const MAX_NODES: usize = 500_000;
 
+/// Pixels decoded from pixel layers over a whole document (2 GB as RGBA). The archive's limits
+/// count the bytes extracted from it, and a layer of one flat colour needs none (#1704).
+const MAX_LAYER_PIXELS: u64 = 512 << 20;
+
 /// Read a document: archive, `doc.dat` and the model.
 pub fn read(bytes: &[u8], limits: Limits) -> Result<Document, Error> {
+    read_within(bytes, limits, MAX_LAYER_PIXELS)
+}
+
+/// [`read`], its pixel layers holding at most `pixels` pixels together.
+pub(crate) fn read_within(bytes: &[u8], limits: Limits, pixels: u64) -> Result<Document, Error> {
     let mut archive = Archive::open(bytes, limits)?;
     let doc = archive.read("doc.dat")?;
     let s = stream::parse(&doc)?;
-    Reader::new(&s, &mut archive).document()
+    let mut reader = Reader::new(&s, &mut archive);
+    reader.pixels_left = pixels;
+    reader.document()
 }
 
 pub(crate) struct Reader<'s, 'a, 'b> {
@@ -311,11 +322,13 @@ pub(crate) struct Reader<'s, 'a, 'b> {
     warnings: BTreeMap<String, usize>,
     nodes: usize,
     active: HashSet<ObjId>,
+    /// Pixels the pixel layers still to come may decode ([`MAX_LAYER_PIXELS`] in all).
+    pub(crate) pixels_left: u64,
 }
 
 impl<'s, 'a, 'b> Reader<'s, 'a, 'b> {
     fn new(s: &'s Stream, archive: &'b mut Archive<'a>) -> Self {
-        Self { s, archive, dpi: 72.0, warnings: BTreeMap::new(), nodes: 0, active: HashSet::new() }
+        Self { s, archive, dpi: 72.0, warnings: BTreeMap::new(), nodes: 0, active: HashSet::new(), pixels_left: MAX_LAYER_PIXELS }
     }
 
     pub(crate) fn warn(&mut self, what: impl Into<String>) {

crates/affinity/src/raster.rs

@@ -105,10 +105,17 @@ fn decode(r: &mut Reader, bitmap: ObjId, crop: (u32, u32, u32, u32), world: Affi
         return Ok(None);
     }
     let (w, h) = (x1 - x0, y1 - y0);
-    if u64::from(w) * u64::from(h) > MAX_PIXELS {
+    let pixels = u64::from(w) * u64::from(h);
+    if pixels > MAX_PIXELS {
         r.warn("pixel layers larger than 64 megapixels");
         return Ok(None);
     }
+    // Before anything is allocated: a layer's tiles can be flat colour, with nothing to extract.
+    let Some(left) = r.pixels_left.checked_sub(pixels) else {
+        r.warn("pixel layers beyond the pixels one document may hold");
+        return Ok(None);
+    };
+    r.pixels_left = left;
     let format = s.enumeration(bitmap, b"Frmt").map(|(f, _)| f);
     let (channels, bps) = match format {
         Some(0) => (4, 1),
@@ -404,6 +411,41 @@ mod tests {
         }
     }
 
+    /// Pixel layers of one flat colour: nothing in the archive but `doc.dat` (state 2 is a tile
+    /// of 0xFF). Written by hand for the reader, like [`fixture`].
+    fn flat_layers(count: u32, side: i32) -> Vec<u8> {
+        let layer = |i: u32| {
+            let mut bitmap = vec![(tag(b"Frmt"), F::Enum(0, 0)), (tag(b"BmpW"), F::I32(side)), (tag(b"BmpH"), F::I32(side))];
+            for c in 1..=4 {
+                bitmap.push((tag(&super::tag(b"TWi", c)), F::I32(1)));
+                bitmap.push((tag(&super::tag(b"THi", c)), F::I32(1)));
+                bitmap.push((tag(&super::tag(b"Sta", c)), F::U8s(vec![2])));
+            }
+            F::Def(10 + i, vec![tag(b"Rstr")], vec![(tag(b"Bitm"), F::Obj(tag(b"DyBm"), bitmap))])
+        };
+        let spread = F::Def(1, vec![tag(b"Sprd")], vec![(tag(b"Chld"), F::Shared((0..count).map(layer).collect()))]);
+        let doc = synth::stream(&[(tag(b"DocR"), F::Obj(tag(b"DocN"), vec![(tag(b"Chld"), F::Shared(vec![spread]))]))]);
+        synth::container(&[("doc.dat", &doc, Method::Zlib)], None)
+    }
+
+    /// #1704: the pixels of all pixel layers together are bounded, also when the archive holds
+    /// no data for them: the layers past the allowance are left out with a note.
+    #[test]
+    fn pixel_layers_share_one_allowance_for_the_whole_document() {
+        let file = flat_layers(5, 16);
+        let layers = |pixels: u64| {
+            let doc = crate::model::read_within(&file, crate::Limits::default(), pixels).unwrap();
+            let n = doc.spreads[0].nodes.iter().filter(|n| matches!(n.kind, Kind::Image(_))).count();
+            (n, doc.warnings)
+        };
+        assert_eq!(layers(5 * 256), (5, vec![]));
+        let (kept, warnings) = layers(3 * 256 + 255);
+        assert_eq!(kept, 3);
+        assert_eq!(warnings.len(), 1);
+        assert!(warnings[0].contains("pixel layers beyond the pixels one document may hold"), "{warnings:?}");
+        assert_eq!(crate::read(&file, crate::Limits::default()).unwrap().spreads[0].nodes.len(), 5);
+    }
+
     #[test]
     fn embedded_source_dimensions_and_pixel_budget_are_checked_before_decode() {
         let bytes = png();