Handle Stack Mode sources with no visible layers (#770)

* fix(engine): reject empty smart-object stack sources (#717)

* test(engine): cover stack mode with no visible layers (#717)

* style: format regression coverage for #717

* test(engine): reproduce hidden Stack Mode sources through commands

crates/engine/src/layer_menu_cmds/tests.rs

@@ -327,6 +327,29 @@ fn stack_modes_render_from_the_nested_layers() {
     assert!(!session(8, "rgb").is_enabled("layer.smartObjects.stackMode.mean"));
 }
 
+#[test]
+fn stack_mode_rejects_sources_without_visible_layers() {
+    for depth in DEPTHS {
+        let mut s = session(depth, "rgb");
+        paint(&mut s, |_, _| [0.4, 0.4, 0.4, 1.0]);
+        s.execute("layer.smartObjects.convertToSmartObject", json!({})).unwrap();
+        s.execute("layer.smartObjects.editContents", json!({})).unwrap();
+        let ids: Vec<u64> = doc(&s).layers.iter().map(|layer| layer.id.0).collect();
+        for id in ids {
+            s.execute("layer.hideLayers", json!({"layer": id})).unwrap();
+        }
+        s.execute("layer.smartObjects.saveContents", json!({})).unwrap();
+        s.execute("file.close", json!({})).unwrap();
+
+        let err = s.execute("layer.smartObjects.stackMode.mean", json!({})).unwrap_err();
+        assert!(err.to_string().contains("no visible layers"), "{err}");
+        match &active(&s).content {
+            LayerContent::Smart(sm) => assert_eq!(sm.stack_mode, None, "failed edit must roll back"),
+            _ => panic!(),
+        }
+    }
+}
+
 #[test]
 fn reveal_in_finder_dry_run_and_enabled() {
     let mut s = session(8, "rgb");

crates/engine/src/smart_cmds.rs

@@ -194,6 +194,9 @@ pub fn stack_image(file_name: &str, bytes: &[u8], fmt: PixelFormat, mode: photoc
             photocraft_compose::render_layer(&one, bounds).px
         })
         .collect();
+    if frames.is_empty() {
+        return Err(EngineError::Other("the smart object's contents have no visible layers".into()));
+    }
     let stat = match mode {
         photocraft_doc::StackMode::Entropy => Stat::Entropy,
         photocraft_doc::StackMode::Kurtosis => Stat::Kurtosis,