Fix vector pixel bounds overflow (#747)

* Fix vector pixel bounds overflow

Saturate exclusive pixel bounds at i32::MAX and cover extreme and ordinary paths with regression tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: e77cd34a-ce3e-4981-996f-4d37d49df90d

* Cover extreme vector path fill commands

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: e85c024b-ace5-4415-9208-b8d4e4684505

---------

Copilot-Session: e77cd34a-ce3e-4981-996f-4d37d49df90d
Copilot-Session: e85c024b-ace5-4415-9208-b8d4e4684505

crates/engine/src/vector_cmds.rs

@@ -948,7 +948,8 @@ fn path_fill(s: &mut Session, p: &Value) -> Result<Value> {
     s.edit("Fill Path", |doc, _| {
         let area = doc.bounds();
         let r = vector::fill_rasterizer(&path, vector::DEFAULT_TOLERANCE);
-        let area = r.pixel_bounds().map_or(area, |b| b.inflate(feather.ceil() as i32 * 2).intersect(&area));
+        let feather_pad = (feather.ceil() as i32).saturating_mul(2);
+        let area = r.pixel_bounds().map_or(area, |b| b.inflate(feather_pad).intersect(&area));
         if area.is_empty() {
             return Ok(());
         }

crates/engine/src/vector_cmds/tests.rs

@@ -346,6 +346,28 @@ fn copy_and_paste_shape_fill_and_stroke() {
     assert!(shape(&s, target).stroke.is_none());
 }
 
+#[test]
+fn path_fill_handles_extreme_coordinates_and_feather() {
+    let mut s = session(48, 32, 8);
+    s.execute("layer.new.layer", json!({})).unwrap();
+    s.execute(
+        "path.set",
+        json!({
+            "name": "P",
+            "path": {"subpaths": [{
+                "closed": true,
+                "knots": [[0.0, 0.0], [3.0e9, 0.0], [3.0e9, 1.0e9]]
+            }]}
+        }),
+    )
+    .unwrap();
+    s.execute("path.fill", json!({"name": "P", "color": "#ff0000"})).unwrap();
+    s.execute("path.fill", json!({"name": "P", "color": "#ff0000", "feather": 1_073_741_824.0})).unwrap();
+
+    let layer = doc(&s).layer(s.active().unwrap().active_layer.unwrap()).unwrap();
+    assert_eq!(layer.surface().unwrap().tile_count(), 1);
+}
+
 #[test]
 fn vector_mask_commands_and_compositing() {
     let mut s = session(100, 100, 8);

crates/vector/src/raster.rs

@@ -93,13 +93,14 @@ impl Rasterizer {
     }
 
     /// Pixel bounds of everything that may be covered (`None` = nothing; the whole plane
-    /// when inverted is reported as `None` too — callers clip to their own area).
+    /// when inverted is reported as `None` too — callers clip to their own area). The upper
+    /// edge is clipped to `i32::MAX`, the largest representable exclusive rectangle edge.
     pub fn pixel_bounds(&self) -> Option<Rect> {
         if self.inverted {
             return None;
         }
         let (x0, y0, x1, y1) = self.bounds?;
-        let r = Rect::new(x0.floor() as i32, y0.floor() as i32, x1.ceil() as i32 + 1, y1.ceil() as i32 + 1);
+        let r = Rect::new(x0.floor() as i32, y0.floor() as i32, (x1.ceil() as i32).saturating_add(1), (y1.ceil() as i32).saturating_add(1));
         (!r.is_empty()).then_some(r)
     }
 
@@ -374,6 +375,8 @@ fn draw(acc: &mut [f32], x0: i32, xa: f64, ya: f64, xb: f64, yb: f64, w: f64) {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::coverage_surface;
+    use photocraft_color::PixelFormat;
 
     fn square(x: f64, y: f64, s: f64) -> Vec<(f64, f64)> {
         vec![(x, y), (x + s, y), (x + s, y + s), (x, y + s)]
@@ -417,4 +420,35 @@ mod tests {
         let r = Rasterizer::new(true);
         assert!(r.render(Rect::new(0, 0, 4, 4)).iter().all(|v| *v == 1.0));
     }
+
+    #[test]
+    fn pixel_bounds_clip_upper_edge_at_i32_max() {
+        let mut r = Rasterizer::new(false);
+        let max = f64::from(i32::MAX);
+        r.add_component(&[square(max - 2.0, max - 2.0, 4.0)], PathOp::Combine, FillRule::NonZero);
+
+        assert_eq!(r.pixel_bounds(), Some(Rect::new(i32::MAX - 2, i32::MAX - 2, i32::MAX, i32::MAX)));
+
+        // Raster work clipped to a normal document area stays empty and does not use the
+        // extreme bounds as an allocation size.
+        let clipped = coverage_surface(&r, PixelFormat::GRAY8, Rect::new(0, 0, 8, 8));
+        assert_eq!(clipped.tile_count(), 0);
+    }
+
+    #[test]
+    fn pixel_bounds_beyond_i32_max_are_empty() {
+        let mut r = Rasterizer::new(false);
+        let max = f64::from(i32::MAX);
+        r.add_component(&[square(max + 10.0, max + 10.0, 4.0)], PathOp::Combine, FillRule::NonZero);
+
+        assert_eq!(r.pixel_bounds(), None);
+    }
+
+    #[test]
+    fn pixel_bounds_keep_normal_boundary_rounding() {
+        let mut r = Rasterizer::new(false);
+        r.add_component(&[square(1.0, 2.0, 3.0)], PathOp::Combine, FillRule::NonZero);
+
+        assert_eq!(r.pixel_bounds(), Some(Rect::new(1, 2, 5, 6)));
+    }
 }