fix(engine): allow targeting layer mask and vector mask via channel.target (#2458) (#2705)

* fix(engine): allow targeting layer mask and vector mask via channel.target (#2458)

* fix(ui): apply channel target actions before syncing mask targets in channels panel

---------

Co-authored-by: huudinh1401 <huudinh1401@gmail.com>
Co-authored-by: Brandon Thomas <echelon@gmail.com>

crates/engine/src/channel_clip.rs

@@ -38,7 +38,7 @@ pub(crate) fn copy(s: &mut Session) -> Result<Option<Value>> {
     let canvas = doc.bounds();
     // The surface the values come from, and which of its samples is the channel.
     let (src, k) = match d.channel_view.target {
-        ChannelTarget::Composite => return Ok(None),
+        ChannelTarget::Composite | ChannelTarget::LayerMask | ChannelTarget::VectorMask => return Ok(None),
         ChannelTarget::Color(k) => {
             let id = d.active_layer.ok_or_else(|| EngineError::Other("no active layer".into()))?;
             let surf = doc.layer(id).and_then(Layer::surface).ok_or_else(|| EngineError::Other("the active layer has no pixels".into()))?;

crates/engine/src/channel_cmds.rs

@@ -40,6 +40,10 @@ pub enum ChannelTarget {
     Color(usize),
     /// An alpha or spot channel (index into [`Document::channels`]).
     Alpha(usize),
+    /// The active layer's layer mask.
+    LayerMask,
+    /// The active layer's vector mask.
+    VectorMask,
 }
 
 /// Per-document Channels panel state: the target and the eye toggles. Not part of history.
@@ -121,6 +125,12 @@ pub(crate) fn fix_view(st: &mut DocState) {
         match v.target {
             ChannelTarget::Alpha(i) if i >= n => v.target = ChannelTarget::Composite,
             ChannelTarget::Color(k) if k >= colors || colors < 2 => v.target = ChannelTarget::Composite,
+            ChannelTarget::LayerMask if !st.active_layer.and_then(|id| st.doc.layer(id)).is_some_and(|l| l.mask.is_some()) => {
+                v.target = ChannelTarget::Composite;
+            }
+            ChannelTarget::VectorMask if !st.active_layer.and_then(|id| st.doc.layer(id)).is_some_and(|l| l.vector_mask.is_some()) => {
+                v.target = ChannelTarget::Composite;
+            }
             _ => {}
         }
     }
@@ -615,6 +625,7 @@ pub(crate) fn inject_target(s: &Session, id: &str, params: Value) -> Value {
     let t = match st.channel_view.target {
         ChannelTarget::Alpha(i) if i < st.doc.channels.len() => json!({ "channel": i }),
         ChannelTarget::Composite if st.doc.quick_mask.is_some() => json!("quickMask"),
+        ChannelTarget::LayerMask => json!("mask"),
         // Viewing the active layer's mask (⌥-click): pixel commands edit the mask.
         ChannelTarget::Composite if crate::mask_view_cmds::current(st).is_some() => json!("mask"),
         _ => return params,
@@ -1148,23 +1159,25 @@ fn move_channel(s: &mut Session, p: &Value) -> Result<Value> {
 
 /// Make `t` the edit target and show it the way Photoshop does when its row is clicked.
 fn target(s: &mut Session, t: ChannelTarget) {
-    let Some(st) = s.active_mut() else { return };
-    let colors = color_count(&st.doc);
-    let n = st.doc.channels.len();
-    let v = &mut st.channel_view;
-    v.target = t;
-    match t {
-        ChannelTarget::Composite => v.color_hidden.clear(),
-        ChannelTarget::Color(k) => {
-            v.color_hidden = (0..colors).map(|c| c != k).collect();
-            v.alpha_visible.clear();
-        }
-        ChannelTarget::Alpha(i) => {
-            v.color_hidden = vec![true; colors];
-            v.alpha_visible = (0..n).map(|c| c == i).collect();
+    if let Some(st) = s.active_mut() {
+        let colors = color_count(&st.doc);
+        let n = st.doc.channels.len();
+        let v = &mut st.channel_view;
+        v.target = t;
+        match t {
+            ChannelTarget::Composite | ChannelTarget::LayerMask | ChannelTarget::VectorMask => v.color_hidden.clear(),
+            ChannelTarget::Color(k) => {
+                v.color_hidden = (0..colors).map(|c| c != k).collect();
+                v.alpha_visible.clear();
+            }
+            ChannelTarget::Alpha(i) => {
+                v.color_hidden = vec![true; colors];
+                v.alpha_visible = (0..n).map(|c| c == i).collect();
+            }
         }
+        fix_view_now(st);
     }
-    fix_view_now(st);
+    s.tools.target_mask(t == ChannelTarget::LayerMask);
 }
 
 fn parse_target(s: &Session, p: &Value, cmd: &str) -> Result<ChannelTarget> {
@@ -1173,7 +1186,21 @@ fn parse_target(s: &Session, p: &Value, cmd: &str) -> Result<ChannelTarget> {
         Some(ChanRef::Composite) => Ok(ChannelTarget::Composite),
         Some(ChanRef::Color(k)) if k < color_count(doc) && color_count(doc) > 1 => Ok(ChannelTarget::Color(k)),
         Some(ChanRef::Alpha(i)) if i < doc.channels.len() => Ok(ChannelTarget::Alpha(i)),
-        _ => Err(bad(cmd, "`channel` must be \"composite\", a colour channel name or an alpha channel")),
+        Some(ChanRef::LayerMask) => {
+            let active = crate::active_layer_of(s).map_err(|e| bad(cmd, e))?;
+            if active.mask.is_none() {
+                return Err(bad(cmd, "the active layer has no layer mask"));
+            }
+            Ok(ChannelTarget::LayerMask)
+        }
+        Some(ChanRef::VectorMask) => {
+            let active = crate::active_layer_of(s).map_err(|e| bad(cmd, e))?;
+            if active.vector_mask.is_none() {
+                return Err(bad(cmd, "the active layer has no vector mask"));
+            }
+            Ok(ChannelTarget::VectorMask)
+        }
+        _ => Err(bad(cmd, "`channel` must be \"composite\", \"mask\", \"vectorMask\", a colour channel name or an alpha channel")),
     }
 }
 
@@ -1671,7 +1698,15 @@ pub fn specs() -> Vec<CommandSpec> {
         spec!("channel.delete", "Delete Channel", [], None, r##"{"channel":index|name?=targeted}"##, has_channels, delete_channel),
         spec!("channel.rename", "Rename Channel", [], None, r##"{"channel":index|name,"name":str}"##, has_channels, rename_channel),
         spec!("channel.move", "Reorder Channel", [], None, r##"{"channel":index|name,"to":index}"##, has_channels, move_channel),
-        spec!("channel.target", "Target Channel", [], None, r##"{"channel":"composite"|"red|green|blue|…"|index|name="composite"}"##, has_doc, target_cmd),
+        spec!(
+            "channel.target",
+            "Target Channel",
+            [],
+            None,
+            r##"{"channel":"composite"|"mask"|"vectorMask"|"red|green|blue|…"|index|name="composite"}"##,
+            has_doc,
+            target_cmd
+        ),
         spec!(
             "channel.setVisible",
             "Channel Visibility",

crates/engine/src/channel_cmds/tests.rs

@@ -565,6 +565,49 @@ fn apply_image_and_calculations_fail_gracefully_never_panic() {
     assert!(s.execute("image.applyImage", json!({"source": {"channel": "composite"}})).is_ok());
 }
 
+#[test]
+fn channel_target_layer_mask_and_vector_mask() {
+    let mut s = session();
+    // Initially the layer has no mask: targeting mask or vectorMask fails.
+    let err_mask = s.execute("channel.target", json!({"channel": "mask"})).unwrap_err().to_string();
+    assert!(err_mask.contains("no layer mask"), "{err_mask}");
+    let err_vec = s.execute("channel.target", json!({"channel": "vectorMask"})).unwrap_err().to_string();
+    assert!(err_vec.contains("no vector mask"), "{err_vec}");
+
+    // Add a layer mask.
+    s.execute("layer.layerMask.revealAll", json!({})).unwrap();
+    // Target the mask explicitly via channel.target.
+    let j = s.execute("channel.target", json!({"channel": "mask"})).unwrap();
+    assert_eq!(j["target"]["kind"], "layerMask");
+    assert_eq!(s.active().unwrap().channel_view.target, ChannelTarget::LayerMask);
+    assert!(s.tools.mask_targeted);
+
+    // Commands that follow target (invert, paint) edit the mask without explicit target.
+    s.execute("image.adjustments.invert", json!({})).unwrap();
+    let l_id = s.active().unwrap().active_layer.unwrap();
+    let mask_sf = &s.active().unwrap().doc.layer(l_id).unwrap().mask.as_ref().unwrap().surface;
+    assert_eq!(mask_sf.pixel(10, 10)[0], 0.0, "inverted mask is black");
+
+    // Target composite again.
+    let j2 = s.execute("channel.target", json!({"channel": "composite"})).unwrap();
+    assert_eq!(j2["target"]["kind"], "composite");
+    assert_eq!(s.active().unwrap().channel_view.target, ChannelTarget::Composite);
+    assert!(!s.tools.mask_targeted);
+
+    // Issue #2458 repro: new adjustment layer then channel.target {"channel": "mask"}
+    s.execute("layer.newAdjustmentLayer.curves", json!({})).unwrap();
+    let j_adj = s.execute("channel.target", json!({"channel": "mask"})).unwrap();
+    assert_eq!(j_adj["target"]["kind"], "layerMask");
+    assert_eq!(s.active().unwrap().channel_view.target, ChannelTarget::LayerMask);
+
+    // Vector mask targeting.
+    s.execute("layer.new.layer", json!({})).unwrap();
+    s.execute("layer.vectorMask.revealAll", json!({})).unwrap();
+    let j_vec = s.execute("channel.target", json!({"channel": "vectorMask"})).unwrap();
+    assert_eq!(j_vec["target"]["kind"], "vectorMask");
+    assert_eq!(s.active().unwrap().channel_view.target, ChannelTarget::VectorMask);
+}
+
 #[test]
 fn clear_with_a_colour_channel_targeted_clears_only_that_channel() {
     let near = |a: &[f32], b: &[f32]| a.len() == b.len() && a.iter().zip(b).all(|(x, y)| (x - y).abs() < 1e-3);

crates/engine/src/lib.rs

@@ -421,12 +421,18 @@ impl Session {
     pub fn set_active(&mut self, index: usize) -> bool {
         if index < self.docs.len() {
             self.active = Some(index);
+            self.sync_tools_mask_target();
             true
         } else {
             false
         }
     }
 
+    pub fn sync_tools_mask_target(&mut self) {
+        let mask = self.active().is_some_and(|st| st.channel_view.target == crate::channel_cmds::ChannelTarget::LayerMask);
+        self.tools.target_mask(mask);
+    }
+
     /// Move the document at `from` to tab position `to` (clamped to the last), keeping the active
     /// document active. Returns its new index; `None` when `from` is out of range.
     pub fn move_document(&mut self, from: usize, to: usize) -> Option<usize> {
@@ -595,6 +601,7 @@ impl Session {
         st.active_layer = Some(id);
         st.selected_layers = vec![id];
         st.layer_anchor = Some(id);
+        channel_cmds::fix_view(st);
         // Selecting a layer is not an edit: keep a clean document clean.
         let clean = st.saved_revision == st.revision;
         st.revision += 1;

crates/ui-egui/src/canvas.rs

@@ -4997,6 +4997,7 @@ pub fn paint_target(app: &PhotocraftApp) -> serde_json::Value {
     match st.channel_view.target {
         ChannelTarget::Alpha(i) if i < st.doc.channels.len() => return json!({ "channel": i }),
         ChannelTarget::Composite if st.doc.quick_mask.is_some() => return json!("quickMask"),
+        ChannelTarget::LayerMask => return json!("mask"),
         _ => {}
     }
     let has_mask = st.active_layer.and_then(|id| st.doc.layer(id)).is_some_and(|l| l.mask.is_some());

crates/ui-egui/src/channels_panel.rs

@@ -65,7 +65,10 @@ pub fn show(app: &mut PhotocraftApp, ui: &mut egui::Ui) {
     // The active layer's mask (Photoshop lists it, in italics, below the colour channels).
     let masked = active_layer.and_then(|id| doc.layer(id)).filter(|l| l.mask.is_some()).cloned();
     let mask_tex = masked.as_ref().and_then(|l| Some(app.mask_thumb(&ctx, &doc, l.id, l.mask.as_ref()?)));
-    let mask_targeted = masked.is_some() && app.ui.mask_target && view.target == ChannelTarget::Composite && !quick;
+    let mask_targeted = masked.is_some()
+        && (app.ui.mask_target || view.target == ChannelTarget::LayerMask)
+        && (view.target == ChannelTarget::Composite || view.target == ChannelTarget::LayerMask)
+        && !quick;
     // Multichannel images are their ink channels only (no composite or colour rows).
     let multichannel = doc.mode == photocraft_doc::ColorMode::Multichannel;
     let mut rows = if multichannel { Vec::new() } else { vec![Row::Composite] };
@@ -195,7 +198,7 @@ pub fn show(app: &mut PhotocraftApp, ui: &mut egui::Ui) {
                 } else if row == Row::LayerMask {
                     // Targets the mask, as clicking its thumbnail in the Layers panel does.
                     mask_click = Some(true);
-                    actions.push(("channel.target".into(), json!({ "channel": "composite" })));
+                    actions.push(("channel.target".into(), json!({ "channel": "mask" })));
                 } else {
                     if matches!(row, Row::Composite | Row::Color(_)) {
                         mask_click = Some(false);
@@ -276,6 +279,8 @@ pub fn show(app: &mut PhotocraftApp, ui: &mut egui::Ui) {
             ChannelTarget::Alpha(i) => json!(i),
             ChannelTarget::Color(k) => json!({ "color": k }),
             ChannelTarget::Composite => json!("composite"),
+            ChannelTarget::LayerMask => json!("mask"),
+            ChannelTarget::VectorMask => json!("vectorMask"),
         };
         if icons::button(ui, "trash", 26.0, false, tl!("Delete current channel")).clicked() {
             actions.push(("channel.delete".into(), json!({})));
@@ -291,10 +296,6 @@ pub fn show(app: &mut PhotocraftApp, ui: &mut egui::Ui) {
         }
     });
     ctx.data_mut(|d| d.insert_temp(thumbs_id(), drawn));
-    if let Some(on) = mask_click {
-        app.ui.mask_target = on;
-        app.sync_mask_targets();
-    }
     for (id, p) in actions {
         if id == "ui.renameChannel" {
             if let Some(i) = p.as_u64().and_then(|i| doc.channels.get(i as usize)) {
@@ -305,6 +306,10 @@ pub fn show(app: &mut PhotocraftApp, ui: &mut egui::Ui) {
         }
         let _ = app.run(&id, p);
     }
+    if let Some(on) = mask_click {
+        app.ui.mask_target = on;
+        app.sync_mask_targets();
+    }
 }
 
 fn thumbs_id() -> egui::Id {

crates/ui-egui/src/lib.rs

@@ -1700,12 +1700,13 @@ impl PhotocraftApp {
     /// as the engine routes those itself.
     pub fn layer_mask_targeted(&self) -> bool {
         let Some(st) = self.session.active() else { return false };
-        let composite = st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::Composite && st.doc.quick_mask.is_none();
+        let layer_mask = st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::LayerMask;
+        let composite = (st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::Composite || layer_mask) && st.doc.quick_mask.is_none();
         let has_mask = st.active_layer.and_then(|id| st.doc.layer(id)).is_some_and(|l| l.mask.is_some());
         if !composite || !has_mask {
             return false;
         }
-        if self.ui.mask_target {
+        if self.ui.mask_target || layer_mask {
             return true;
         }
         let colors_off = st.channel_view.visible_colors(photocraft_engine::channel_cmds::color_count(&st.doc)) == 0;
@@ -1717,10 +1718,13 @@ impl PhotocraftApp {
     /// active layer (a shape layer's path is its content, not a mask). Targeting a mask or the
     /// pixels brings back that target's foreground/background pair, as in Photoshop (#2166).
     pub(crate) fn sync_mask_targets(&mut self) {
-        if let Some(st) = self.session.active() {
-            if photocraft_engine::mask_view_cmds::current(st).is_some() {
+        if let Some(st) = self.session.active_mut() {
+            if photocraft_engine::mask_view_cmds::current(st).is_some() || st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::LayerMask {
                 self.ui.mask_target = true;
                 self.ui.vector_mask_target = false;
+            } else if st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::VectorMask {
+                self.ui.vector_mask_target = true;
+                self.ui.mask_target = false;
             }
             if self.ui.vector_mask_target && !mask_thumbs_ui::has_vector_mask(st) {
                 self.ui.vector_mask_target = false;
@@ -1728,6 +1732,9 @@ impl PhotocraftApp {
             if self.ui.mask_target && st.active_layer.and_then(|id| st.doc.layer(id)).is_none_or(|l| l.mask.is_none()) {
                 self.ui.mask_target = false;
             }
+            if !self.ui.mask_target && st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::LayerMask {
+                st.channel_view.target = photocraft_engine::channel_cmds::ChannelTarget::Composite;
+            }
         }
         let mask = self.layer_mask_targeted();
         self.session.tools.target_mask(mask);

crates/ui-egui/src/shortcut_dispatch.rs

@@ -268,8 +268,9 @@ fn delete_key_command(app: &PhotocraftApp, id: String) -> String {
     let colors_off = st.channel_view.visible_colors(photocraft_engine::channel_cmds::color_count(&st.doc)) == 0;
     let alpha_on = (0..st.doc.channels.len()).any(|i| st.channel_view.alpha_shown(i));
     let solo_mask = colors_off && has_mask && !alpha_on && st.doc.quick_mask.is_none();
-    let mask = (app.ui.mask_target || solo_mask) && has_mask;
-    if mask && composite {
+    let layer_mask = st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::LayerMask;
+    let mask = (app.ui.mask_target || solo_mask || layer_mask) && has_mask;
+    if mask && (composite || layer_mask) {
         let cmd = if st.doc.selection.is_some() { "edit.fillBackground" } else { "layer.layerMask.delete" };
         return cmd.into();
     }

crates/ui-egui/src/vector_ui.rs

@@ -462,7 +462,8 @@ enum PathTarget {
 pub(crate) fn targeted_vector_mask(app: &PhotocraftApp) -> Option<(u64, Path)> {
     let st = app.session.active()?;
     let l = st.active_layer.and_then(|id| st.doc.layer(id))?;
-    (app.ui.vector_mask_target && !matches!(l.content, LayerContent::Shape(_))).then_some(())?;
+    let targeted = app.ui.vector_mask_target || st.channel_view.target == photocraft_engine::channel_cmds::ChannelTarget::VectorMask;
+    (targeted && !matches!(l.content, LayerContent::Shape(_))).then_some(())?;
     l.vector_mask.as_ref().map(|m| (l.id.0, m.path.clone()))
 }