Paste pictures from the system clipboard (screenshots, Copy Image, copied files) (#45)

egui's paste event only carries text, so a clipboard holding just a picture
sent no paste at all on Ctrl+V, and the ribbon/context-menu Paste only knew
WordCraft's own copy ("the clipboard is empty").

Desktop: the app reads the clipboard's picture through a new
Services::clipboard_picture (arboard, already in the build via egui-winit):
copied files first, else image pixels. Mod+V whose release arrives without a
paste event, and Paste from the ribbon or menus, insert that picture through
insert.picture (one undo step); text naming copied picture files inserts the
pictures. Pixel sizes are hostile: at most 16,000 px a side and 50 Mpx, the
buffer must match, files at most 200 MB.

Web: a document paste listener reads image files from the paste event when
there's no text and delivers them through the inbox like dropped pictures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Cargo.lock

@@ -4865,6 +4865,7 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
 name = "wordcraft"
 version = "0.4.0"
 dependencies = [
+ "arboard",
  "eframe",
  "egui",
  "fmv-macos-events",

apps/wordcraft-web/Cargo.toml

@@ -28,4 +28,4 @@ rfd = { workspace = true }
 wasm-bindgen = "0.2"
 wasm-bindgen-futures = "0.4"
 js-sys = "0.3"
-web-sys = { version = "0.3", features = ["Window", "Document", "Element", "HtmlElement", "HtmlCanvasElement", "HtmlAnchorElement", "HtmlIFrameElement", "Blob", "BlobPropertyBag", "Url", "Location", "CssStyleDeclaration", "Event", "EventTarget", "BeforeUnloadEvent"] }
+web-sys = { version = "0.3", features = ["Window", "Document", "Element", "HtmlElement", "HtmlCanvasElement", "HtmlAnchorElement", "HtmlIFrameElement", "Blob", "BlobPropertyBag", "Url", "Location", "CssStyleDeclaration", "Event", "EventTarget", "BeforeUnloadEvent", "ClipboardEvent", "DataTransfer", "FileList", "File"] }

apps/wordcraft-web/src/web.rs

@@ -44,6 +44,9 @@ pub fn start() {
                         log::info!("wordcraft-web: wgpu backend {:?}", rs.adapter.get_info().backend);
                     }
                     let inbox: Inbox = Inbox::default();
+                    if let Err(e) = listen_for_pasted_pictures(inbox.clone(), cc.egui_ctx.clone()) {
+                        log::error!("pasting pictures is unavailable: {e}");
+                    }
                     let doc = if query().contains("sample") { wordcraft_engine::sample::sample_document() } else { wordcraft_doc::Document::new() };
                     let mut app = WordApp::new(Session::new(doc), services(inbox.clone(), cc.egui_ctx.clone(), dirty));
                     app.autosave = false;
@@ -100,6 +103,52 @@ impl eframe::App for WebShell {
     }
 }
 
+/// Largest picture pasted from the browser's clipboard (as `insert.picture` allows).
+const MAX_PASTED_BYTES: f64 = (200u32 << 20) as f64;
+/// Most pictures one paste inserts.
+const MAX_PASTED_PICTURES: u32 = 20;
+
+/// Pasting a picture (#45): eframe's paste handler only reads text, so a clipboard holding just
+/// pictures (a screenshot, a browser's Copy Image) is read here and the pictures arrive through
+/// the inbox like dropped ones. A clipboard with text is left to eframe.
+fn listen_for_pasted_pictures(inbox: Inbox, ctx: egui::Context) -> Result<(), String> {
+    let document = web_sys::window().and_then(|w| w.document()).ok_or("no document")?;
+    let on_paste = wasm_bindgen::closure::Closure::<dyn FnMut(web_sys::ClipboardEvent)>::new(move |e: web_sys::ClipboardEvent| {
+        let Some(data) = e.clipboard_data() else { return };
+        if !data.get_data("text").unwrap_or_default().is_empty() {
+            return;
+        }
+        let Some(files) = data.files() else { return };
+        for i in 0..files.length().min(MAX_PASTED_PICTURES) {
+            let Some(file) = files.get(i) else { continue };
+            if !file.type_().starts_with("image/") {
+                continue;
+            }
+            if file.size() > MAX_PASTED_BYTES {
+                log::warn!("pasted picture is larger than 200 MB; not inserted");
+                continue;
+            }
+            let inbox = inbox.clone();
+            let ctx = ctx.clone();
+            wasm_bindgen_futures::spawn_local(async move {
+                match wasm_bindgen_futures::JsFuture::from(file.array_buffer()).await {
+                    Ok(buf) => {
+                        let bytes = js_sys::Uint8Array::new(&buf).to_vec();
+                        // The inbox inserts files named like pictures; the engine reads the format from the bytes.
+                        inbox.lock().unwrap_or_else(|e| e.into_inner()).push(("pasted.png".into(), bytes));
+                        ctx.request_repaint();
+                    }
+                    Err(e) => log::error!("couldn't read the pasted picture: {e:?}"),
+                }
+            });
+        }
+    });
+    document.add_event_listener_with_callback("paste", on_paste.as_ref().unchecked_ref()).map_err(|e| format!("{e:?}"))?;
+    // The listener lives as long as the page.
+    on_paste.forget();
+    Ok(())
+}
+
 /// `dirty` is the flag the `beforeunload` guard reads ([`guard_unload`]).
 fn services(inbox: Inbox, ctx: egui::Context, dirty: Rc<Cell<bool>>) -> Services {
     let open_inbox = inbox.clone();

apps/wordcraft/Cargo.toml

@@ -37,5 +37,8 @@ rfd = { workspace = true }
 # Waits for an `rfd::AsyncFileDialog` on a background thread (the dialogs don't block the UI, #94);
 # rfd itself already depends on it.
 pollster = "0.4"
+# Pictures and copied files on the system clipboard (#45); egui's paste only carries text. Already
+# in the build through egui-winit, with the same features.
+arboard = { workspace = true, features = ["image-data"] }
 # The control server's random key.
 getrandom = "0.3"

apps/wordcraft/src/main.rs

@@ -20,6 +20,7 @@ mod logging;
 use wordcraft_engine::Session;
 use wordcraft_ui_egui::{
     Services, UiState, WordApp,
+    paste_picture::ClipboardPicture,
     window_geometry::{WindowGeometry, take_rescue},
 };
 
@@ -118,9 +119,23 @@ fn save_prefs(app: &WordApp) {
     }
 }
 
-/// Native file dialogs, shown without blocking the window (`file_dialogs`, #94).
+/// Native file dialogs, shown without blocking the window (`file_dialogs`, #94), and pictures on
+/// the system clipboard (#45).
 fn services(file_dialog: file_dialogs::Hook) -> Services {
-    Services { file_dialog: Some(file_dialog), ..Default::default() }
+    Services { file_dialog: Some(file_dialog), clipboard_picture: Some(Box::new(clipboard_picture)), ..Default::default() }
+}
+
+/// The picture on the system clipboard (#45): copied files, else image pixels. The UI checks the
+/// sizes before inserting anything.
+fn clipboard_picture() -> Option<ClipboardPicture> {
+    let mut clipboard = arboard::Clipboard::new().map_err(|e| log::warn!("clipboard: {e}")).ok()?;
+    if let Ok(files) = clipboard.get().file_list()
+        && !files.is_empty()
+    {
+        return Some(ClipboardPicture::Files(files));
+    }
+    let image = clipboard.get_image().ok()?;
+    Some(ClipboardPicture::Pixels { width: image.width, height: image.height, rgba: image.bytes.into_owned() })
 }
 
 /// Window, Dock and taskbar icon.

crates/ui-egui/Cargo.toml

@@ -20,6 +20,8 @@ egui = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 log = { workspace = true }
+# Pictures pasted from the system clipboard arrive as pixels and are inserted as PNG (#45).
+image = { workspace = true }
 # The system's preferred interface languages (#8). Native: OS APIs; wasm: none (English).
 sys-locale = "0.3"
 
@@ -29,7 +31,6 @@ js-sys = "0.3"
 
 [dev-dependencies]
 egui_kittest = { workspace = true, features = ["wgpu", "snapshot"] }
-image = { workspace = true }
 
 [build-dependencies]
 serde_json = { workspace = true }

crates/ui-egui/src/canvas.rs

@@ -46,6 +46,8 @@ pub struct CanvasState {
     pub(crate) obj_drag: Option<crate::objects::ObjectDrag>,
     /// Pages per row last frame; when it changes the caret's page is scrolled back into view.
     pub cols: usize,
+    /// A paste event arrived since the last Mod+V release (see `keys::canvas_events`).
+    pub(crate) pasted: bool,
 }
 
 impl CanvasState {
@@ -78,6 +80,7 @@ impl Default for CanvasState {
             obj_drag: None,
             context_menu_open: false,
             mini_anchor: None,
+            pasted: false,
             cols: 1,
         }
     }

crates/ui-egui/src/keys.rs

@@ -140,6 +140,24 @@ pub fn canvas_events(app: &mut WordApp, ctx: &egui::Context) {
     }
     let events = ctx.input(|i| i.events.clone());
     for e in events {
+        if let egui::Event::Paste(t) = &e {
+            app.canvas.pasted = true;
+            // Copied picture files arrive as their paths: insert the pictures (#45).
+            if app.session.math.is_none() && !ctx.input(|i| i.modifiers.alt) && crate::paste_picture::paste_named_files(app, t) {
+                continue;
+            }
+        }
+        // Desktop: Mod+V with no text on the clipboard sends no paste event, only the key's
+        // release; paste the clipboard's picture then (#45). A paste event before the release
+        // means it was text. (The web reads pasted pictures itself.)
+        if let egui::Event::Key { key: key @ (Key::V | Key::Paste), pressed: false, modifiers, .. } = &e {
+            let paste_keys = (modifiers.command || *key == Key::Paste) && !modifiers.alt;
+            let pasted = std::mem::take(&mut app.canvas.pasted);
+            if !pasted && paste_keys && app.session.math.is_none() && app.services.clipboard_picture.is_some() {
+                let _ = app.run("edit.paste", json!({}));
+            }
+            continue;
+        }
         // Editing an equation: text and editing keys go into it.
         if app.session.math.is_some() {
             match &e {

crates/ui-egui/src/lib.rs

@@ -29,6 +29,7 @@ pub mod keytips;
 pub mod mini_toolbar;
 pub mod objects;
 pub mod panes;
+pub mod paste_picture;
 pub mod previews;
 pub mod read_aloud;
 pub mod ribbon;
@@ -69,6 +70,9 @@ pub struct Services {
     /// Web: hand PDF bytes to the browser's print flow directly (no download, no intermediate
     /// file) — opens the system print dialog on that PDF. An error means no dialog was opened.
     pub print: Option<Box<dyn Fn(&[u8]) -> Result<(), String>>>,
+    /// Desktop: the picture on the system clipboard, if any — egui's paste only carries text
+    /// (#45). Read when Paste finds no text; see [`paste_picture`].
+    pub clipboard_picture: Option<Box<dyn Fn() -> Option<paste_picture::ClipboardPicture>>>,
 }
 
 /// Files delivered asynchronously.
@@ -302,6 +306,14 @@ impl WordApp {
             self.dialog = Some(dialogs::Dialog::SaveChanges { name, then: id.to_string(), params, document: self.session.document_id() });
             return Ok(json!({"pending": "saveChanges"}));
         }
+        // Paste without system clipboard text: a picture there is newer than our own copy (copying
+        // here replaces it with text), so it wins (#45).
+        if id == "edit.paste"
+            && params.get("text").is_none()
+            && let Some(r) = paste_picture::paste(self)
+        {
+            return r;
+        }
         let r = self.execute(id, params);
         // Match Fields and Check for Errors show what they found.
         if let Ok(v) = &r

crates/ui-egui/src/paste_picture.rs

@@ -0,0 +1,163 @@
+//! Pasting a picture from the system clipboard (#45).
+//!
+//! egui's paste event only carries text, so a clipboard holding just a picture (a screenshot, a
+//! browser's Copy Image) or a copied picture file used to paste nothing. The host reads the
+//! clipboard through [`crate::Services::clipboard_picture`]; this module checks what it got — the
+//! clipboard comes from other programs, so sizes are hostile — and inserts it with
+//! `insert.picture`, one undoable step like any inserted picture.
+
+use std::path::{Path, PathBuf};
+
+use serde_json::{Value, json};
+
+use crate::WordApp;
+
+/// What the host found on the system clipboard.
+pub enum ClipboardPicture {
+    /// Straight (not premultiplied) RGBA8 pixels, row by row.
+    Pixels { width: usize, height: usize, rgba: Vec<u8> },
+    /// Copied files (a file manager's Copy); the pictures among them are inserted.
+    Files(Vec<PathBuf>),
+}
+
+/// Longest side of a pasted picture, in pixels.
+pub const MAX_SIDE: usize = 16_000;
+/// Most pixels in a pasted picture (200 MB of RGBA).
+pub const MAX_PIXELS: usize = 50_000_000;
+/// Largest picture file pasted (as `insert.picture` allows).
+pub const MAX_FILE_BYTES: u64 = 200 << 20;
+/// Most picture files one paste inserts.
+const MAX_FILES: usize = 20;
+const IMAGE_EXTS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "bmp"];
+
+/// PNG bytes for clipboard pixels, or why they can't be pasted.
+pub fn png_from_rgba(width: usize, height: usize, rgba: Vec<u8>) -> Result<Vec<u8>, String> {
+    if width == 0 || height == 0 {
+        return Err("the clipboard picture is empty".into());
+    }
+    let pixels = width.checked_mul(height).filter(|&n| width <= MAX_SIDE && height <= MAX_SIDE && n <= MAX_PIXELS);
+    let Some(pixels) = pixels else {
+        return Err(format!("the clipboard picture is too large to paste ({width} × {height} pixels)"));
+    };
+    if Some(rgba.len()) != pixels.checked_mul(4) {
+        return Err("the clipboard picture is damaged (its size doesn't match its pixels)".into());
+    }
+    let (Ok(w), Ok(h)) = (u32::try_from(width), u32::try_from(height)) else {
+        return Err("the clipboard picture is too large to paste".into());
+    };
+    let img = image::RgbaImage::from_raw(w, h, rgba).ok_or("the clipboard picture is damaged")?;
+    let mut png = Vec::new();
+    img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).map_err(|e| format!("couldn't read the clipboard picture: {e}"))?;
+    Ok(png)
+}
+
+fn is_picture_file(path: &Path) -> bool {
+    path.extension().and_then(|e| e.to_str()).is_some_and(|e| IMAGE_EXTS.iter().any(|x| x.eq_ignore_ascii_case(e)))
+}
+
+/// Pasted text that is only paths or `file://` addresses of picture files — what file managers
+/// put on the clipboard as text when files are copied.
+pub fn names_picture_files(text: &str) -> bool {
+    if text.len() > 64 << 10 {
+        return false;
+    }
+    let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).peekable();
+    lines.peek().is_some()
+        && lines.take(MAX_FILES + 1).enumerate().all(|(i, l)| {
+            let path = l.strip_prefix("file://").unwrap_or(l);
+            i < MAX_FILES && (l.starts_with("file://") || Path::new(path).is_absolute()) && is_picture_file(Path::new(path))
+        })
+}
+
+/// Paste the picture on the system clipboard. `None` when there is none (or no clipboard
+/// service): the caller pastes as usual.
+pub(crate) fn paste(app: &mut WordApp) -> Option<Result<Value, String>> {
+    let picture = app.services.clipboard_picture.as_ref()?()?;
+    let r = match picture {
+        ClipboardPicture::Pixels { width, height, rgba } => match png_from_rgba(width, height, rgba) {
+            Ok(png) => app.execute("insert.picture", json!({"data": wordcraft_engine::cmd::insert::base64_encode(&png)})),
+            Err(e) => {
+                log::warn!("clipboard picture: {e}");
+                app.status(e.clone());
+                Err(e)
+            }
+        },
+        ClipboardPicture::Files(files) => {
+            let files: Vec<PathBuf> = files.into_iter().filter(|p| is_picture_file(p)).take(MAX_FILES).collect();
+            if files.is_empty() {
+                return None;
+            }
+            let mut r = Err(String::new());
+            for path in files {
+                let size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
+                r = if size > MAX_FILE_BYTES {
+                    let e = format!("{} is larger than 200 MB", path.display());
+                    app.status(e.clone());
+                    Err(e)
+                } else {
+                    app.execute("insert.picture", json!({"path": path.to_string_lossy()}))
+                };
+                if r.is_err() {
+                    break;
+                }
+            }
+            r
+        }
+    };
+    Some(r)
+}
+
+/// Paste text naming copied picture files: insert the pictures when the clipboard holds those
+/// files. False leaves the text to the usual paste.
+pub(crate) fn paste_named_files(app: &mut WordApp, text: &str) -> bool {
+    names_picture_files(text) && paste(app).is_some()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::Services;
+    use wordcraft_engine::Session;
+
+    fn app_with(picture: impl Fn() -> Option<ClipboardPicture> + 'static) -> WordApp {
+        let services = Services { clipboard_picture: Some(Box::new(picture)), ..Default::default() };
+        WordApp::new(Session::new(wordcraft_doc::Document::new()), services)
+    }
+
+    fn pictures(a: &mut WordApp) -> usize {
+        a.session.run("arrange.selectionPane", &json!({})).unwrap().as_array().map(|x| x.len()).unwrap_or(0)
+    }
+
+    /// Paste with only a picture on the clipboard inserts it at its natural size, one undo step.
+    #[test]
+    fn paste_inserts_clipboard_pixels() {
+        let mut a = app_with(|| Some(ClipboardPicture::Pixels { width: 40, height: 20, rgba: vec![200; 40 * 20 * 4] }));
+        let r = a.run("edit.paste", json!({})).unwrap();
+        // 96 ppi: 40 × 20 px is 30 × 15 pt.
+        assert_eq!((r["width"].as_f64(), r["height"].as_f64()), (Some(30.0), Some(15.0)));
+        assert_eq!(pictures(&mut a), 1);
+        let media = a.session.doc.media.values().next().unwrap();
+        assert_eq!(wordcraft_render::image_size(media), Some((40, 20)));
+        a.run("edit.undo", json!({})).unwrap();
+        assert_eq!(pictures(&mut a), 0);
+        // Pasted text that names picture files only becomes pictures when files were copied.
+        assert!(names_picture_files("file:///home/me/shot%201.png\n/tmp/b.JPG\n"));
+        assert!(!names_picture_files("see /tmp/b.png") && !names_picture_files("notes.txt") && !names_picture_files(""));
+    }
+
+    /// Hostile clipboard sizes are refused without inserting anything (or allocating for them).
+    #[test]
+    fn hostile_clipboard_pictures_are_refused() {
+        assert!(png_from_rgba(100_000, 1, vec![0; 400_000]).is_err());
+        assert!(png_from_rgba(usize::MAX, 2, Vec::new()).is_err());
+        assert!(png_from_rgba(10_000, 10_000, Vec::new()).is_err());
+        assert!(png_from_rgba(4, 4, vec![0; 63]).is_err());
+        assert!(png_from_rgba(0, 4, Vec::new()).is_err());
+        let mut a = app_with(|| Some(ClipboardPicture::Pixels { width: 1 << 40, height: 1 << 40, rgba: Vec::new() }));
+        assert!(a.run("edit.paste", json!({})).is_err());
+        assert_eq!(pictures(&mut a), 0);
+        // Copied files that aren't pictures leave Paste to the usual clipboard.
+        let mut a = app_with(|| Some(ClipboardPicture::Files(vec![PathBuf::from("/tmp/notes.txt")])));
+        assert_eq!(a.run("edit.paste", json!({})).unwrap_err(), "the clipboard is empty");
+    }
+}