diff --git a/apps/vectorcraft/src/main.rs b/apps/vectorcraft/src/main.rs index af4726e4..07a5d94f 100644 --- a/apps/vectorcraft/src/main.rs +++ b/apps/vectorcraft/src/main.rs @@ -501,6 +501,10 @@ fn services(parent: Parent, ui: egui::Context) -> Services { write: Some(Box::new(write_file)), // Background Save and Export write from a worker thread. write_shared: Some(std::sync::Arc::new(write_file)), + // Native saves stream into the file as they are encoded. + stream_shared: Some(std::sync::Arc::new(|p: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>| { + fileio::write_file_with(p, fill).map_err(|e| e.to_string()) + })), // Every format Copy offers and Paste reads (menu-bar Paste never sees egui's Paste event). system_clipboard: Some(clipboard::system_clipboard()), // Linux checks whether Paste has something to take on a background thread: an X11 clipboard diff --git a/crates/doc/src/node.rs b/crates/doc/src/node.rs index 06db3cc8..47a6085c 100644 --- a/crates/doc/src/node.rs +++ b/crates/doc/src/node.rs @@ -394,7 +394,9 @@ pub struct ImageObject { pub placement: crate::links::PlacementOptions, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +/// Written with its `type` first, then its fields (an internally tagged enum); read by +/// [`kind_de`], which takes the fields straight from the file when `type` comes first. +#[derive(Clone, Debug, PartialEq, Serialize)] #[serde(tag = "type", rename_all = "lowercase")] pub enum NodeKind { /// A top-level layer or sublayer. @@ -1634,10 +1636,281 @@ impl Node { } } +/// Reading a [`NodeKind`]. Derived, an internally tagged enum buffers each kind (with every +/// object inside a group, again at each level of nesting) before reading it, since `type` may come +/// anywhere among its fields. Files have it first (it is written first): then the kind is read +/// straight from the file. Otherwise (hand-written files, values whose keys are sorted) the +/// fields before `type` are buffered, and only those. +mod kind_de { + use std::fmt; + use std::sync::Arc; + + use serde::de::value::{MapAccessDeserializer, StringDeserializer}; + use serde::de::{self, DeserializeSeed, Deserializer, EnumAccess, IntoDeserializer, MapAccess, VariantAccess, Visitor}; + use serde::{Deserialize, forward_to_deserialize_any}; + use vectorcraft_geom::{Affine, FillRule, PathData}; + + use super::{LayerColor, LiveShape, Node, NodeKind, yes}; + use crate::live::{BlendSpec, EnvelopeKind, GradientMesh}; + use crate::pattern::RepeatSpec; + use crate::text::TextObject; + + /// The kinds and their fields as [`NodeKind`] has them, with the defaults its field attributes + /// note (the compiler checks that every field is here), read externally tagged: the kind from + /// `type`, its fields from the rest of the object ([`Tagged`]). + #[derive(Deserialize)] + #[serde(remote = "NodeKind", rename_all = "lowercase")] + enum KindDef { + Layer { + color: LayerColor, + #[serde(default)] + template: bool, + #[serde(default = "yes")] + printable: bool, + children: Vec>, + #[serde(default)] + clip: bool, + #[serde(default = "yes")] + preview: bool, + #[serde(default)] + dim_images: Option, + }, + Group { + children: Vec>, + #[serde(default)] + clip: bool, + }, + Path { + path: PathData, + #[serde(default)] + rule: FillRule, + #[serde(default)] + live: Option>, + #[serde(default)] + clipping: bool, + #[serde(default)] + guide: bool, + }, + Compound { + children: Vec>, + #[serde(default)] + rule: FillRule, + }, + Text(Box), + Image(Box), + SymbolInstance { + symbol: String, + xf: Affine, + }, + Blend { + children: Vec>, + #[serde(default)] + spec: Box, + }, + Envelope { + content: Vec>, + kind: Box, + #[serde(default = "crate::live::default_fidelity")] + fidelity: f64, + #[serde(default)] + editing: bool, + #[serde(default)] + options: crate::live::EnvelopeOptions, + #[serde(default)] + frame: Box, + }, + Mesh(GradientMesh), + Repeat(RepeatSpec), + PlacedDocument(Box), + } + + impl<'de> Deserialize<'de> for NodeKind { + fn deserialize>(d: D) -> Result { + d.deserialize_map(KindVisitor) + } + } + + struct KindVisitor; + + impl<'de> Visitor<'de> for KindVisitor { + type Value = NodeKind; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("an object kind (a map with `type`)") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut before = vec![]; + while let Some(key) = map.next_key::()? { + if key == "type" { + let tag: String = map.next_value()?; + return KindDef::deserialize(Tagged { tag, map: Buffered { before: before.into_iter(), value: None, map } }); + } + before.push((key, map.next_value::()?)); + } + Err(de::Error::missing_field("type")) + } + } + + /// The fields read before `type` (buffered), then the rest of the object. + struct Buffered { + before: std::vec::IntoIter<(String, serde_json::Value)>, + value: Option, + map: A, + } + + impl<'de, A: MapAccess<'de>> MapAccess<'de> for Buffered { + type Error = A::Error; + + fn next_key_seed>(&mut self, seed: K) -> Result, A::Error> { + match self.before.next() { + Some((key, value)) => { + self.value = Some(value); + seed.deserialize(StringDeserializer::::new(key)).map(Some) + } + None => self.map.next_key_seed(seed), + } + } + + fn next_value_seed>(&mut self, seed: V) -> Result { + match self.value.take() { + Some(value) => seed.deserialize(value).map_err(de::Error::custom), + None => self.map.next_value_seed(seed), + } + } + } + + /// An object whose `type` was read: the kind is `tag`, its fields are the rest of `map`. + struct Tagged { + tag: String, + map: A, + } + + impl<'de, A: MapAccess<'de>> Deserializer<'de> for Tagged { + type Error = A::Error; + + fn deserialize_any>(self, visitor: V) -> Result { + visitor.visit_enum(self) + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string bytes byte_buf + option unit unit_struct newtype_struct seq tuple tuple_struct map struct enum + identifier ignored_any + } + } + + impl<'de, A: MapAccess<'de>> EnumAccess<'de> for Tagged { + type Error = A::Error; + type Variant = Fields; + + fn variant_seed>(self, seed: V) -> Result<(V::Value, Fields), A::Error> { + let kind = seed.deserialize(self.tag.into_deserializer())?; + Ok((kind, Fields(self.map))) + } + } + + /// A kind's fields: the rest of the object. + struct Fields(A); + + impl<'de, A: MapAccess<'de>> VariantAccess<'de> for Fields { + type Error = A::Error; + + fn unit_variant(self) -> Result<(), A::Error> { + Err(de::Error::invalid_type(de::Unexpected::UnitVariant, &"an object kind with fields")) + } + + fn newtype_variant_seed>(self, seed: T) -> Result { + seed.deserialize(MapAccessDeserializer::new(self.0)) + } + + fn tuple_variant>(self, _: usize, _: V) -> Result { + Err(de::Error::invalid_type(de::Unexpected::TupleVariant, &"an object kind with fields")) + } + + fn struct_variant>(self, _: &'static [&'static str], visitor: V) -> Result { + visitor.visit_map(self.0) + } + } +} + #[cfg(test)] mod tests { use super::*; + /// A node writes its rarely set parts among the others, in the file's field order, and reads + /// them back into its extras; a node without any keeps none. + #[test] + fn extras_are_written_flat_in_order() { + let mut n = Node::path(NodeId(7), shapes::rectangle(Rect::new(0.0, 0.0, 4.0, 2.0)), Appearance::default_art()); + n.name = Some("n".into()); + n.knockout_shape = true; + n.bbox_angle = 30.0; + { + let mut e = n.extras_mut(); + e.mask = Some(Box::new(OpacityMask::new(Node::group(NodeId(8), vec![]), true))); + e.trace = Some(Box::new(serde_json::json!({"preset": "p"}))); + e.series_index = Some(2); + e.graphic_style = Some(3); + e.attrs = Some(Box::default()); + e.slice = Some(Box::default()); + e.perspective = Some(Box::new(crate::PerspectiveAttachment::new("left", 1.0))); + } + /// The top-level keys of a JSON object, in the order written. + struct Keys(Vec); + impl<'de> Deserialize<'de> for Keys { + fn deserialize>(d: D) -> Result { + struct V; + impl<'de> serde::de::Visitor<'de> for V { + type Value = Keys; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("an object") + } + fn visit_map>(self, mut a: A) -> Result { + let mut out = vec![]; + while let Some(k) = a.next_key::()? { + a.next_value::()?; + out.push(k); + } + Ok(Keys(out)) + } + } + d.deserialize_map(V) + } + } + let text = serde_json::to_string(&n).unwrap(); + let keys = serde_json::from_str::(&text).unwrap().0; + let v: serde_json::Value = serde_json::from_str(&text).unwrap(); + let order = [ + "id", + "name", + "knockout_shape", + "appearance", + "mask", + "trace", + "series_index", + "kind", + "graphic_style", + "attrs", + "slice", + "bbox_angle", + "perspective", + ]; + assert_eq!(keys, order); + assert_eq!(serde_json::from_value::(v).unwrap(), n); + let plain = Node::path(NodeId(9), shapes::rectangle(Rect::new(0.0, 0.0, 4.0, 2.0)), Appearance::default_art()); + let back: Node = serde_json::from_value(serde_json::to_value(&plain).unwrap()).unwrap(); + assert!(back.extras_box().is_none()); + // Clearing the last extra lets them go; clearing one a node doesn't have allocates none. + n.extras_mut().mask = None; + assert!(n.extras_box().is_some()); + *n.extras_mut() = NodeExtras::default(); + assert!(n.extras_box().is_none()); + n.extras_mut().mask = None; + assert!(n.extras_box().is_none()); + assert_eq!(n.extras(), &NodeExtras::default()); + } + #[test] fn rect_live_shape_roundtrip() { let l = LiveShape::Rectangle { w: 10.0, h: 20.0, radii: [0.0; 4], kinds: Default::default(), xf: Affine::translate((5.0, 5.0)) }; @@ -1901,4 +2174,50 @@ mod tests { p.appearance.stroke_mut().unwrap().join = crate::appearance::LineJoin::Round; assert_eq!(p.visual_bounds(), Some(Rect::new(-0.5, -0.5, 10.5, 10.5))); } + + /// A group holding a path, a compound, an image and a nested group, as a file writes it. + fn nested_group() -> Node { + let path = |id| Node::path(NodeId(id), shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0)), Appearance::default_art()); + let image = ImageObject { key: "k".into(), width: 2, height: 3, xf: Affine::IDENTITY, link: None, placement: Default::default() }; + let inner = Node::new(NodeId(5), NodeKind::Group { children: vec![Arc::new(path(6))], clip: true }); + let compound = Node::new(NodeId(3), NodeKind::Compound { children: vec![Arc::new(path(4))], rule: FillRule::EvenOdd }); + let kids = vec![Arc::new(path(2)), Arc::new(compound), Arc::new(Node::new(NodeId(7), NodeKind::Image(Box::new(image)))), Arc::new(inner)]; + Node::new(NodeId(1), NodeKind::Group { children: kids, clip: false }) + } + + #[test] + fn kinds_read_back_with_type_first_or_anywhere() { + let n = nested_group(); + let json = serde_json::to_string(&n).unwrap(); + assert!(json.contains(r#""kind":{"type":"group","children":["#), "written with `type` first: {json}"); + assert_eq!(serde_json::from_str::(&json).unwrap(), n); + // Through a JSON value, whose keys are sorted: `type` comes after `children`. + let v = serde_json::to_value(&n).unwrap(); + assert_eq!(serde_json::from_value::(v.clone()).unwrap(), n); + assert_eq!(serde_json::from_str::(&v.to_string()).unwrap(), n); + // `type` last, in the middle, and keys a newer version added (ignored, as before). + let kind = r#"{"children":[],"x":{"deep":[1,2]},"clip":true,"type":"group"}"#; + assert_eq!(serde_json::from_str::(kind).unwrap(), NodeKind::Group { children: vec![], clip: true }); + let kind = r#"{"symbol":"S","type":"symbolinstance","later":1,"xf":[1,0,0,1,5,6]}"#; + assert_eq!( + serde_json::from_str::(kind).unwrap(), + NodeKind::SymbolInstance { symbol: "S".into(), xf: Affine::translate((5.0, 6.0)) } + ); + let kind = r#"{"width":2,"type":"image","key":"k","height":3,"xf":[1,0,0,1,0,0]}"#; + assert!(matches!(serde_json::from_str::(kind).unwrap(), NodeKind::Image(i) if i.key == "k" && i.height == 3)); + // Defaults as before. + let NodeKind::Layer { printable, preview, template, .. } = + serde_json::from_str(r#"{"type":"layer","color":{"Preset":0},"children":[]}"#).unwrap() + else { + panic!("a layer") + }; + assert!(printable && preview && !template); + } + + #[test] + fn bad_kinds_are_errors() { + for bad in [r#"{"children":[]}"#, r#"{"type":"nothing"}"#, r#"{"type":"group"}"#, r#"{"type":3,"children":[]}"#, r#"[]"#, r#""group""#] { + assert!(serde_json::from_str::(bad).is_err(), "{bad}"); + } + } } diff --git a/crates/engine/src/cmd/fileio/load.rs b/crates/engine/src/cmd/fileio/load.rs index 534d86c8..6d44effc 100644 --- a/crates/engine/src/cmd/fileio/load.rs +++ b/crates/engine/src/cmd/fileio/load.rs @@ -283,8 +283,26 @@ pub fn open_bytes(s: &mut Session, name: &str, bytes: &[u8], path: Option, p: &Value) -> Result { + open_read(s, read_for_open(name, bytes, p)?, path) +} + +/// A file read for `document.open` but not opened yet ([`read_for_open`], [`open_read`]). +pub struct ReadFile { + loaded: Loaded, + opts: LoadOptions, +} + +/// The first half of [`open_bytes_with`]: read and decode the file, which needs no session (an +/// app may do it on another thread) → what [`open_read`] opens. +pub fn read_for_open(name: &str, bytes: &[u8], p: &Value) -> Result { let opts = LoadOptions::from_params("document.open", p)?; - open_loaded(s, load_with(name, bytes, &opts)?, path, &opts, false) + Ok(ReadFile { loaded: load_with(name, bytes, &opts)?, opts }) +} + +/// The second half of [`open_bytes_with`]: open a file [`read_for_open`] read as the new active +/// document (`path`: where it was read from) → `document.open`'s result. +pub fn open_read(s: &mut Session, file: ReadFile, path: Option) -> Result { + open_loaded(s, file.loaded, path, &file.opts, false) } /// File → New from Template: any readable file (at `path`, if it is one: its links are looked for diff --git a/crates/engine/src/cmd/fileio/mod.rs b/crates/engine/src/cmd/fileio/mod.rs index bfa892dd..8d53dd3c 100644 --- a/crates/engine/src/cmd/fileio/mod.rs +++ b/crates/engine/src/cmd/fileio/mod.rs @@ -53,15 +53,18 @@ use load::err; pub(crate) use load::import_svg; pub(crate) use load::native_file; pub(crate) use load::source; -pub use load::{Loaded, RasterImage, detect, file_name, load, load_with, open_bytes, open_bytes_with, open_template, raster_image}; +pub use load::{ + Loaded, RasterImage, ReadFile, detect, file_name, load, load_with, open_bytes, open_bytes_with, open_read, open_template, raster_image, + read_for_open, +}; pub use load::{losses_summary, overwrite_losses}; pub(crate) use native::preview_png; pub use native::with_compression_pref; pub use pdfimport::{LoadOptions, page_document}; pub(crate) use save::job_for; pub use save::{ - SAVE_FORMATS, SaveJob, SaveMode, SavePlan, export_folder, save_filters, save_format, save_job, save_plan, save_with, stamp_save_dates, - templates_dialog_folder, templates_folder, + SAVE_FORMATS, SaveJob, SaveMode, SavePlan, StreamWrite, export_folder, save_filters, save_format, save_job, save_plan, save_with, + stamp_save_dates, templates_dialog_folder, templates_folder, }; pub use screens::{PRESETS as SCREEN_PRESETS, ScreenSize, preset_rows as screen_preset_rows}; pub(crate) use screens::{ @@ -769,6 +772,13 @@ pub(crate) fn write_file(path: &str, bytes: &[u8]) -> Result<()> { write_atomic(std::path::Path::new(path), bytes).map_err(|e| EngineError::Other(format!("{path}: {e}"))) } +/// [`write_file`] with what `fill` writes into the file (a save streamed into it). +#[cfg(not(target_arch = "wasm32"))] +pub fn write_file_with(path: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>) -> Result<()> { + may_write(path)?; + vectorcraft_format::write_atomic_with(std::path::Path::new(path), |f| fill(f)).map_err(|e| EngineError::Other(format!("{path}: {e}"))) +} + #[cfg(not(target_arch = "wasm32"))] pub(crate) fn create_dir(path: &str) -> Result<()> { may_write(path)?; @@ -938,6 +948,11 @@ pub(crate) fn write_file(path: &str, _: &[u8]) -> Result<()> { Err(no_fs(path)) } +#[cfg(target_arch = "wasm32")] +pub fn write_file_with(path: &str, _: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>) -> Result<()> { + Err(no_fs(path)) +} + #[cfg(target_arch = "wasm32")] pub(crate) fn create_dir(path: &str) -> Result<()> { Err(no_fs(path)) diff --git a/crates/engine/src/cmd/fileio/native.rs b/crates/engine/src/cmd/fileio/native.rs index 289bfcff..83d9ab45 100644 --- a/crates/engine/src/cmd/fileio/native.rs +++ b/crates/engine/src/cmd/fileio/native.rs @@ -135,6 +135,14 @@ pub(super) fn save_options(cmd: &str, f: &Format, doc: &Document, p: &Value) -> /// The `.vectorcraft` file of `doc` with the options in `p`. pub(super) fn encode(cmd: &str, f: &Format, doc: &Document, p: &Value) -> Result> { + let mut out = Vec::new(); + encode_to(cmd, f, doc, p, &mut out)?; + Ok(out) +} + +/// [`encode`], written to `w` as it is made ([`vectorcraft_format::save_to`]: give it a buffered +/// writer). +pub(super) fn encode_to(cmd: &str, f: &Format, doc: &Document, p: &Value, w: impl std::io::Write) -> Result<()> { let (mut so, extras) = save_options(cmd, f, doc, p)?; // Include Linked Files keeps placed documents' files; older versions get their art. let full = if so.include_linked || so.version < vectorcraft_format::VERSION { @@ -158,7 +166,7 @@ pub(super) fn encode(cmd: &str, f: &Format, doc: &Document, p: &Value) -> Result } other => other, }; - vectorcraft_format::save_with(&native, &so).map_err(|e| bad(cmd, e.to_string())) + vectorcraft_format::save_to(&native, &so, w).map_err(|e| bad(cmd, e.to_string())) } /// The ICC files of the profiles `doc` is tagged with that were loaded from files (the built-in diff --git a/crates/engine/src/cmd/fileio/save.rs b/crates/engine/src/cmd/fileio/save.rs index 75a179cf..923f7c80 100644 --- a/crates/engine/src/cmd/fileio/save.rs +++ b/crates/engine/src/cmd/fileio/save.rs @@ -6,6 +6,7 @@ //! the caller's writer (the file system here; a save panel or a browser download in the apps) and //! makes the file the document's own (path, format, options, saved state) for Save and Save As. +use std::io::Write as _; use std::sync::Arc; use serde_json::{Map, Value, json}; @@ -14,7 +15,6 @@ use vectorcraft_doc::Document; use super::super::*; use super::{ ArtboardPick, Encoded, Format, Loaded, encode_all, file_stem, format, format_for_name, load, read_file, with_compression_pref, write_encoded, - write_file, }; use crate::{DocState, Prefs}; @@ -300,6 +300,31 @@ pub struct SaveJob { boards: Vec, } +/// Writes the file at a path with what the filler it is given writes into it +/// ([`SaveJob::write_streamed`]). +pub type StreamWrite<'a> = &'a mut dyn FnMut(&str, &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>) -> Result<()>; + +/// Bytes gathered before a streamed save writes them to the file. +const STREAM_BUFFER: usize = 256 << 10; + +/// A writer that counts the bytes written through it. +struct Counted { + inner: W, + n: u64, +} + +impl std::io::Write for Counted { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let n = self.inner.write(buf)?; + self.n += n as u64; + Ok(n) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + /// The artboards a native or `.ai` save also writes to files of their own: with /// `separateArtboards`, those `range` names (default all). fn separate_boards(cmd: &str, f: &Format, doc: &Document, p: &Value) -> Result> { @@ -443,6 +468,16 @@ impl SaveJob { &self.snapshot } + /// The document the job writes (the snapshot, as the file keeps it). + pub(crate) fn doc(&self) -> &Arc { + &self.doc + } + + /// Encode `doc` (the job's document, changed) as the job encodes its own: one file. + pub(crate) fn encode_doc(&self, doc: &Document) -> Result { + self.encode_one(doc) + } + /// Encode the file → it (and, with `separateArtboards`, a file per artboard after it), with /// the warnings (what the format loses first, then the encoder's own notes: PDF options not /// applied yet…). @@ -500,6 +535,53 @@ impl SaveJob { Ok(out) } + /// Is the file written as it is encoded by [`SaveJob::write_streamed`]? A native file (or + /// template) to a path, without a file per artboard. + pub fn streams(&self) -> bool { + self.plan.path.is_some() && matches!(self.plan.format.id, "vectorcraft" | "template") && self.boards.is_empty() + } + + /// [`SaveJob::write`] through `stream`, which writes the file at a path with what the filler + /// it is given writes (atomically, as [`vectorcraft_format::write_atomic_with`]). A native + /// file goes into it as it is encoded ([`SaveJob::streams`]), never held whole in memory; + /// other files are encoded first, then written into it. + pub fn write_streamed(&self, stream: StreamWrite) -> Result { + let (Some(path), true) = (&self.plan.path, self.streams()) else { + return self.write(|p, b| stream(p, &mut |w| w.write_all(b))); + }; + let (cmd, f) = (self.plan.mode.command(), self.plan.format); + let doc = self.doc.without_edit_modes(); + let doc = match f.id { + // A native file flagged so opening it starts a new untitled document (as + // `encode_all` writes it). + "template" => { + let mut d = doc.into_owned(); + d.template = true; + std::borrow::Cow::Owned(d) + } + _ => doc, + }; + // The encoder's failure, kept apart from the writer's (which `stream` reports). + let mut failed = None; + let mut written = 0u64; + let mut fill = |w: &mut dyn std::io::Write| { + let mut out = Counted { inner: std::io::BufWriter::with_capacity(STREAM_BUFFER, w), n: 0 }; + let r = super::native::encode_to(cmd, f, &doc, &self.params, &mut out); + written = out.n; + match r { + Ok(()) => out.inner.flush(), + Err(e) => { + failed = Some(e); + Err(std::io::Error::other("the file couldn't be encoded")) + } + } + }; + let r = stream(path, &mut fill); + r.map_err(|e| failed.take().unwrap_or(e))?; + let warnings: Vec = fidelity_warning(f).into_iter().collect(); + Ok(json!({ "path": path, "bytes": written, "format": f.id, "warnings": warnings })) + } + /// Once written: Save and Save As (except to a template) make the file `st`'s own: path, /// title, format, options, and the snapshot as the saved state (edits made since the job was /// made keep the document modified). @@ -543,10 +625,14 @@ pub fn save_with(s: &mut Session, plan: SavePlan, write: impl FnMut(&str, &[u8]) Ok(out) } -/// A save command: plan, then write with the file system. +/// A save command: plan, then write with the file system (a native file as it is encoded). fn save(s: &mut Session, mode: SaveMode, p: &Value) -> Result { let plan = save_plan(s, mode, p)?; - save_with(s, plan, write_file) + let job = save_job(s, plan)?; + let uid = s.doc()?.uid; + let out = job.write_streamed(&mut super::write_file_with)?; + job.complete(s, uid); + Ok(out) } fn can_revert(s: &Session) -> std::result::Result<(), String> { diff --git a/crates/engine/src/cmd/fileio/tests.rs b/crates/engine/src/cmd/fileio/tests.rs index d4f6554e..5af1e823 100644 --- a/crates/engine/src/cmd/fileio/tests.rs +++ b/crates/engine/src/cmd/fileio/tests.rs @@ -412,3 +412,33 @@ fn new_files_never_replace_a_file() { assert_eq!(std::fs::read(&path).unwrap(), b"first"); let _ = std::fs::remove_dir_all(d); } + +#[test] +fn a_native_save_streams_the_bytes_the_encoder_makes() { + let d = tmp_dir("stream"); + let mut s = session(200.0, 100.0, 1); + s.execute("shape.rectangle", &json!({"x": 10, "y": 10, "width": 50, "height": 20})).unwrap(); + for (name, compress) in [("a.vectorcraft", false), ("b.vectorcraft", true), ("c.vctemplate", true)] { + let path = d.join(name).to_string_lossy().to_string(); + let p = json!({"path": path, "compress": compress}); + let r = s.execute("file.saveCopy", &p).unwrap(); + let written = std::fs::read(&path).unwrap(); + assert_eq!(r["bytes"], written.len(), "{r}"); + let plan = save_plan(&s, SaveMode::Copy, &p).unwrap(); + let job = save_job(&mut s, plan).unwrap(); + assert!(job.streams()); + assert_eq!(written, job.encode().unwrap().files[0].1, "{name}"); + assert_eq!(vectorcraft_format::load(&written).unwrap().template, name.ends_with("vctemplate")); + } + // A writer that fails: the save fails with its error. + let plan = save_plan(&s, SaveMode::Copy, &json!({"path": "/x/y.vectorcraft"})).unwrap(); + let job = save_job(&mut s, plan).unwrap(); + let e = job + .write_streamed(&mut |_, fill| { + fill(&mut std::io::sink()).unwrap(); + Err(EngineError::Other("disk full".into())) + }) + .unwrap_err(); + assert_eq!(e.to_string(), "disk full"); + let _ = std::fs::remove_dir_all(&d); +} diff --git a/crates/engine/src/cmd/recovery.rs b/crates/engine/src/cmd/recovery.rs index 83623a50..f466355c 100644 --- a/crates/engine/src/cmd/recovery.rs +++ b/crates/engine/src/cmd/recovery.rs @@ -26,17 +26,21 @@ //! again. //! //! Each copy is two entries: `/.vectorcraft` (the native document, written as a save -//! writes it) and `/.json` (its title, original path and format, and when it was -//! written). The store is a folder on the desktop ([`FolderStore`]: the `recoveryFolder` +//! writes it, without its images) and `/.json` (its title, original path and format, +//! when it was written, and its images). The images are entries of their own, `/img-…blob`, +//! named by their content and shared by the area's copies: a copy written again writes only the +//! images that changed, and the images no copy names any more go. Copies written before images +//! were kept apart (the images in the document entry) still restore. The store is a folder on the desktop ([`FolderStore`]: the `recoveryFolder` //! preference, else the folder the app sets), browser storage on the web, or any //! [`RecoveryStore`] the host installs (tests use [`MemoryStore`]). use std::collections::{BTreeMap, BTreeSet}; -use std::sync::{Arc, Mutex}; +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex, Weak}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use vectorcraft_doc::Document; +use vectorcraft_doc::{Document, ImageBlob}; use super::fileio::{self, SAVE_FORMATS, SaveJob, SaveMode, SavePlan}; use super::*; @@ -50,6 +54,8 @@ pub const COMPLEX_OBJECTS: usize = 20_000; const DOC_EXT: &str = ".vectorcraft"; /// The extension of a copy's details entry. const META_EXT: &str = ".json"; +/// The extension of an image entry (see [`blob_name`]). +const BLOB_EXT: &str = ".blob"; /// The heartbeat entry of an area in a store without locks: when its app last said it runs (Unix /// seconds, as text). const BEAT: &str = "heartbeat"; @@ -527,6 +533,73 @@ struct Meta { /// (see [`crate::DocState::import_losses`]): a restored copy still refuses to replace it. imported_from: Option, import_losses: Vec, + /// The document's images by key, in entries of the area (none in copies written before + /// images were kept apart: the document entry holds them). + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + images: BTreeMap, +} + +/// One of a copy's images: its entry in the copy's area. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default)] +struct ImageRef { + mime: String, + /// The entry's name in the area ([`blob_name`]). + blob: String, + /// A linked image's preview, as a save writes it ([`ImageBlob::proxy`]). + #[serde(skip_serializing_if = "std::ops::Not::not")] + proxy: bool, +} + +/// Writes of image entries and their sweeping ([`sweep`]) take turns, in this app (the background +/// worker and the UI thread), so an image is never swept between being written and named. +static WRITES: Mutex<()> = Mutex::new(()); + +/// Image bytes and the name of their entry ([`blob_name`]). +type Named = (Weak>, String); + +/// The entry names of image bytes already hashed ([`blob_name`]), so an image is hashed once, +/// not at every interval. Bytes behind a live `Weak` never change (`Arc::make_mut` copies them). +static NAMES: Mutex> = Mutex::new(Vec::new()); + +/// The name of the entry holding `bytes`: from their content (128 bits of hash and the length), so +/// identical images share one entry and one written before isn't written again. +fn blob_name(bytes: &Arc>) -> String { + let mut names = NAMES.lock().unwrap_or_else(|e| e.into_inner()); + names.retain(|(w, _)| w.strong_count() > 0); + if let Some((_, name)) = names.iter().find(|(w, _)| std::ptr::eq(w.as_ptr(), Arc::as_ptr(bytes))) { + return name.clone(); + } + let hash = |seed: u8| { + let mut h = std::collections::hash_map::DefaultHasher::new(); + seed.hash(&mut h); + bytes.as_slice().hash(&mut h); + h.finish() + }; + let name = format!("img-{:016x}{:016x}-{}{BLOB_EXT}", hash(1), hash(2), bytes.len()); + names.push((Arc::downgrade(bytes), name.clone())); + name +} + +/// The MIME type of a linked image's preview (JPEG or PNG), as a save names it. +fn proxy_mime(bytes: &[u8]) -> &'static str { + if bytes.starts_with(&[0xff, 0xd8]) { "image/jpeg" } else { vectorcraft_doc::links::PROXY_MIME } +} + +/// Remove the image entries of `area` no copy's details name any more. Best effort: an entry left +/// behind goes at the next sweep, or with the area. +fn sweep(store: &dyn RecoveryStore, area: &str) { + let Ok(names) = store.list() else { return }; + let in_area = |n: &&String| n.split_once('/').is_some_and(|(a, _)| a == area); + let mut named = BTreeSet::new(); + for meta in names.iter().filter(in_area).filter_map(|n| n.strip_suffix(META_EXT)) { + named.extend(read_meta(store, meta).images.into_values().map(|r| r.blob)); + } + for n in names.iter().filter(in_area).filter(|n| n.ends_with(BLOB_EXT)) { + if n.split_once('/').is_some_and(|(_, b)| !named.contains(b)) { + let _ = store.remove(n); + } + } } fn read_meta(store: &dyn RecoveryStore, file: &str) -> Meta { @@ -548,14 +621,44 @@ pub struct RecoveryJob { } impl RecoveryJob { - /// Encode the snapshot and store it with its details → `{file, title, bytes}`. + /// Encode the snapshot and store it with its details and the images the area doesn't have + /// yet → `{file, title, bytes (the document's), imageBytes (images written now)}`. pub fn write(&self, store: &dyn RecoveryStore) -> std::result::Result { - let enc = self.save.encode().map_err(|e| e.to_string())?; + let doc = self.save.doc(); + // A linked image keeps its preview, as a save writes it. + let linked = doc.linked_only_images(); + let mut lean = Document::clone(doc); + let blobs = std::mem::take(&mut lean.images); + let enc = self.save.encode_doc(&lean).map_err(|e| e.to_string())?; let bytes = enc.files.first().map(|f| f.1.as_slice()).ok_or("nothing encoded")?; - let meta = serde_json::to_vec_pretty(&self.meta).map_err(|e| e.to_string())?; + let mut meta = self.meta.clone(); + let mut images = vec![]; + for (key, b) in &blobs { + let (mime, data, proxy) = match b.proxy.as_ref().filter(|_| linked.contains(key)) { + Some(p) => (proxy_mime(p), p, true), + None => (b.mime.as_str(), &b.bytes, false), + }; + let blob = blob_name(data); + meta.images.insert(key.clone(), ImageRef { mime: mime.into(), blob: blob.clone(), proxy }); + images.push((blob, data)); + } + let meta_bytes = serde_json::to_vec_pretty(&meta).map_err(|e| e.to_string())?; + let area = self.file.split_once('/').map_or("", |(a, _)| a); + let _turn = WRITES.lock().unwrap_or_else(|e| e.into_inner()); + let have: BTreeSet = if images.is_empty() { BTreeSet::new() } else { store.list()?.into_iter().collect() }; + let mut image_bytes = 0; + for (blob, data) in images { + let name = format!("{area}/{blob}"); + if !have.contains(&name) { + store.write(&name, data)?; + image_bytes += data.len(); + } + } store.write(&format!("{}{DOC_EXT}", self.file), bytes)?; - store.write(&format!("{}{META_EXT}", self.file), &meta)?; - Ok(json!({ "file": self.file, "title": self.title, "bytes": bytes.len() })) + store.write(&format!("{}{META_EXT}", self.file), &meta_bytes)?; + // Images this copy no longer shows, nor any other. + sweep(store, area); + Ok(json!({ "file": self.file, "title": self.title, "bytes": bytes.len(), "imageBytes": image_bytes })) } /// Once stored: the document (if still open) remembers the copy, so it isn't written again @@ -662,6 +765,8 @@ pub fn jobs(s: &mut Session, store: &Arc) -> Result<(Vec, file: &str) -> R let bytes = store.read(&format!("{file}{DOC_EXT}")).map_err(EngineError::Other)?; let meta_bytes = store.read(&format!("{file}{META_EXT}")).unwrap_or_default(); let meta: Meta = serde_json::from_slice(&meta_bytes).unwrap_or_default(); - let (mut doc, _, warnings) = fileio::native_file(&bytes).map_err(|e| EngineError::Other(format!("recovery copy `{file}`: {e}")))?; + let (mut doc, _, mut warnings) = fileio::native_file(&bytes).map_err(|e| EngineError::Other(format!("recovery copy `{file}`: {e}")))?; + // The images, from the entries of the copy's area. + let from = file.split_once('/').map_or("", |(a, _)| a); + let mut blobs = vec![]; + for (key, r) in &meta.images { + match store.read(&format!("{from}/{}", r.blob)) { + Ok(data) => { + let mut blob = ImageBlob::new(r.mime.clone(), data); + // Until the linked file is read, the preview stands in for it. + if r.proxy { + blob.proxy = Some(blob.bytes.clone()); + } + blobs.push((r.blob.clone(), blob.bytes.clone())); + doc.images.insert(key.clone(), blob); + } + Err(e) => warnings.push(format!("the image “{key}” of the recovered document couldn't be read: {e}")), + } + } let area = claim(s, store)?; let mut taken: Vec = copies_in(&store.list().map_err(EngineError::Other)?).into_iter().map(|(_, f)| f).collect(); taken.extend(open_copies(s)); @@ -838,8 +960,15 @@ fn restore_one(s: &mut Session, store: &Arc, file: &str) -> R f if taken.contains(&f) => fresh_name(&area, stem, &taken), f => f, }; - store.write(&format!("{mine}{DOC_EXT}"), &bytes).map_err(EngineError::Other)?; - store.write(&format!("{mine}{META_EXT}"), &meta_bytes).map_err(EngineError::Other)?; + { + let _turn = WRITES.lock().unwrap_or_else(|e| e.into_inner()); + for (blob, data) in &blobs { + store.write(&format!("{area}/{blob}"), data).map_err(EngineError::Other)?; + } + store.write(&format!("{mine}{DOC_EXT}"), &bytes).map_err(EngineError::Other)?; + store.write(&format!("{mine}{META_EXT}"), &meta_bytes).map_err(EngineError::Other)?; + } + // The copy's images in its old area go with the area, once no copy is left there. store.remove(&format!("{file}{DOC_EXT}")).map_err(EngineError::Other)?; store.remove(&format!("{file}{META_EXT}")).map_err(EngineError::Other)?; if doc.title.is_empty() { @@ -901,7 +1030,7 @@ pub fn specs() -> Vec { "Save Recovery Data", [], None, - "{} write a recovery copy of every modified document changed since its last copy, in this app's own area of the store (the app does this every autosaveInterval minutes while autosaveRecovery is on; documents of more than 20000 objects are skipped while recoveryOffForComplex is on). A copy goes when its document is saved, reverted or closed → {saved: [{file, title, bytes}], skipped: [{title, reason}], location} (location: the recovery folder, or browser storage on the web)", + "{} write a recovery copy of every modified document changed since its last copy, in this app's own area of the store (the app does this every autosaveInterval minutes while autosaveRecovery is on; documents of more than 20000 objects are skipped while recoveryOffForComplex is on). Images are stored apart, each once while unchanged. A copy goes when its document is saved, reverted or closed → {saved: [{file, title, bytes, imageBytes (images written this time)}], skipped: [{title, reason}], location} (location: the recovery folder, or browser storage on the web)", has_store, save ), diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index bf859b73..5046d6ed 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -34,16 +34,19 @@ #[cfg(not(target_arch = "wasm32"))] mod atomic; mod legacy; +mod stream; +mod unpack; use std::borrow::Cow; use std::collections::{BTreeMap, HashSet}; -use std::io::{Read as _, Write as _}; +use std::io::{Read as _, Write}; use serde::{Deserialize, Serialize}; use vectorcraft_doc::{Document, ImageBlob}; #[cfg(not(target_arch = "wasm32"))] pub use atomic::{write_atomic, write_atomic_with, write_new_with}; +use stream::{Gz, Indent, KeyScan, Out}; /// v4: large data after the JSON (v1 to v3 files still load). pub const VERSION: u32 = 4; @@ -60,7 +63,17 @@ pub const PREVIEW_MAX: u32 = 256; pub const EXTENSION: &str = "vectorcraft"; /// Extension and format name from before the project was renamed (DrawCraft): still opened. pub const LEGACY_EXTENSION: &str = "drawcraft"; -/// Most bytes a compressed file may unpack to (a guard against decompression bombs). +/// Most bytes the JSON of a compressed file may unpack to (a guard against decompression bombs, +/// as are [`MAX_BLOB`] and [`MAX_BLOBS`]). A compressed save refuses a document past any of them, +/// so every file VectorCraft writes reads back. +pub const MAX_JSON: u64 = 1 << 30; +/// Most bytes one entry of the data after the JSON (an image, a placed document, the PDF, a +/// profile) may have in a compressed file. +pub const MAX_BLOB: u64 = 1 << 30; +/// Most bytes the entries after the JSON may have together in a compressed file. +pub const MAX_BLOBS: u64 = 4 << 30; +/// Most bytes a compressed file whose JSON isn't followed by the marker where VectorCraft puts it +/// (hand-made or damaged files) may unpack to: it is unpacked whole. const MAX_UNPACKED: u64 = 512 << 20; /// Is `ext` (without the dot, any case) a native document extension? @@ -85,8 +98,10 @@ pub enum FormatError { BadVersion(u32), #[error("version {0} files can't be compressed: only version {COMPRESSED_SINCE} and later open compressed files")] CompressedTooOld(u32), - #[error("the compressed file unpacks to more than {} MB", MAX_UNPACKED >> 20)] + #[error("the compressed file unpacks to more than VectorCraft reads ({} MB of document, {} MB per image)", MAX_JSON >> 20, MAX_BLOB >> 20)] TooBig, + #[error("the document is too large to save compressed ({0}): save it with Use Compression off")] + TooBigToCompress(String), #[error("can't encode the document: {0}")] Encode(String), } @@ -106,37 +121,43 @@ struct Image { } impl Image { - /// Its bytes: the base64 `data`, or its part of `tail` (the data after the JSON). - fn bytes(&self, tail: &[u8]) -> Option> { + /// Its bytes when they are in the JSON (base64 `data`); `None` when they come after it + /// ([`Image::blob`]) or don't decode. + fn inline(&self) -> Option> { match self.blob { - Some([offset, len]) => { - let start = usize::try_from(offset).ok()?; - let end = start.checked_add(usize::try_from(len).ok()?)?; - tail.get(start..end).map(<[u8]>::to_vec) - } + Some(_) => None, None => base64_decode(&self.data), } } } /// Entries written for one file: large bytes go to the data after the JSON when the version keeps -/// them there. -struct Blobs { - tail: Vec, +/// them there (in the order they are added, each at the offset it is given). +struct Blobs<'a> { + /// The bytes, and whether they are compressed already ([`is_packed`]). + tail: Vec<(&'a [u8], bool)>, + at: u64, outside: bool, } -impl Blobs { - fn image(&mut self, mime: &str, bytes: &[u8], proxy: bool) -> Image { +impl<'a> Blobs<'a> { + fn image(&mut self, mime: &str, bytes: &'a [u8], proxy: bool) -> Image { if self.outside && bytes.len() >= INLINE_MAX { - let at = self.tail.len() as u64; - self.tail.extend_from_slice(bytes); - return Image { mime: mime.into(), data: String::new(), blob: Some([at, bytes.len() as u64]), proxy }; + let len = bytes.len() as u64; + let at = self.at; + self.at += len; + self.tail.push((bytes, is_packed(mime))); + return Image { mime: mime.into(), data: String::new(), blob: Some([at, len]), proxy }; } Image { mime: mime.into(), data: base64_encode(bytes), blob: None, proxy } } } +/// Is data of this MIME type compressed already (no use compressing it again)? +fn is_packed(mime: &str) -> bool { + matches!(mime, "image/jpeg" | "image/png" | "image/gif" | "image/webp") +} + /// The MIME type of a preview's bytes (JPEG or PNG). fn preview_mime(bytes: &[u8]) -> &'static str { if bytes.starts_with(&[0xff, 0xd8]) { "image/jpeg" } else { vectorcraft_doc::links::PROXY_MIME } @@ -247,6 +268,19 @@ pub fn save_file(doc: &Document) -> Vec { /// Serialize a document as `o` says: the version, compression, layout and preview. Editing-mode /// working copies and images nothing uses are left out. pub fn save_with(doc: &Document, o: &SaveOptions) -> Result, FormatError> { + let mut out = Vec::new(); + save_to(doc, o, &mut out)?; + Ok(out) +} + +/// Bytes of JSON serialized at a time on their way to the output. +const JSON_CHUNK: usize = 64 << 10; + +/// [`save_with`], written to `w` as it is made: the JSON is serialized straight into it (through +/// the compressor when compressing) and the images are copied in from the document, so the file +/// is never held in memory as a whole. Give it a buffered writer. On an error `w` has part of a +/// file: write to a temporary file ([`write_atomic_with`]). +pub fn save_to(doc: &Document, o: &SaveOptions, w: impl Write) -> Result<(), FormatError> { o.check()?; let pretty = o.pretty && !o.compress; let mut d = doc.without_edit_modes().into_owned(); @@ -264,62 +298,103 @@ pub fn save_with(doc: &Document, o: &SaveOptions) -> Result, FormatError if o.version < 3 { d.store_legacy_color_profiles(); } - let body = if o.version == 1 { + let v1 = if o.version == 1 { let mut v = serde_json::to_value(&d).map_err(encode_err)?; legacy::anchors_to_v1(&mut v); - to_json(&v, pretty)? + Some(v) } else { - to_json(&d, pretty)? + None }; + let entries = Entries { blobs: &blobs, linked: &linked, include_linked: o.include_linked }; + let mut out_blobs = Blobs { tail: vec![], at: 0, outside: o.version >= 4 }; // Images are only referenced by image objects' `key` (wherever the objects are: layers, - // symbols, patterns, masks, or foreign data). - let used = if blobs.is_empty() { HashSet::new() } else { string_members(&body, "key") }; - let mut out_blobs = Blobs { tail: vec![], outside: o.version >= 4 }; - let images: BTreeMap<&str, Image> = blobs - .iter() - .filter(|(k, _)| used.contains(k.as_str())) - .map(|(k, b)| { - // Include Linked Files keeps the file's bytes whenever they are loaded. - let preview_only = linked.contains(k) && (!o.include_linked || b.is_proxy()); - let image = match b.proxy.as_ref().filter(|_| preview_only) { - Some(p) => out_blobs.image(preview_mime(p), p, true), - None => out_blobs.image(&b.mime, &b.bytes, false), - }; - (k.as_str(), image) - }) - .collect(); + // symbols, patterns, masks, or foreign data). The data after the JSON has the images first, + // then the profiles and the PDF, which the file lists before the document: with those, the + // images in use are looked for before the document is written (it is serialized twice). + let mut images = None; + if !blobs.is_empty() && (!o.profiles.is_empty() || o.pdf.is_some()) { + let mut scan = KeyScan::new(std::io::sink()); + write_body(std::io::BufWriter::with_capacity(JSON_CHUNK, &mut scan), v1.as_ref(), &d, pretty)?; + images = Some(entries.images(&scan.keys, &mut out_blobs)); + } let profiles: BTreeMap<&str, Image> = o.profiles.iter().map(|(name, icc)| (name.as_str(), out_blobs.image(ICC_MIME, icc, false))).collect(); let pdf = o.pdf.as_deref().map(|p| out_blobs.image("application/pdf", p, false)); let blob = |mime: &str, bytes: &[u8]| Image { mime: mime.into(), data: base64_encode(bytes), blob: None, proxy: false }; - let carried = profiles.values().chain(&pdf).map(|i| i.data.len()).sum::(); - let size = body.len() + images.values().map(|i| i.data.len()).sum::() + carried + 256; - let mut w = Envelope { out: Vec::with_capacity(size), pretty }; + let mut out = if o.compress { Out::Gz(Box::new(Gz::new(w).map_err(encode_err)?)) } else { Out::Plain(w) }; + let mut env = Envelope { pretty, first: true }; // Older apps only know the name from before the rename. - w.field("format", &if o.version < 3 { LEGACY_EXTENSION } else { EXTENSION })?; - w.field("version", &o.version)?; - w.field("generator", &format!("VectorCraft {}", env!("CARGO_PKG_VERSION")))?; + env.field(&mut out, "format", &if o.version < 3 { LEGACY_EXTENSION } else { EXTENSION })?; + env.field(&mut out, "version", &o.version)?; + env.field(&mut out, "generator", &format!("VectorCraft {}", env!("CARGO_PKG_VERSION")))?; if let Some(png) = &o.preview { - w.field("preview", &blob("image/png", png))?; + env.field(&mut out, "preview", &blob("image/png", png))?; } if !profiles.is_empty() { - w.field("profiles", &profiles)?; + env.field(&mut out, "profiles", &profiles)?; } if let Some(pdf) = &pdf { - w.field("pdf", pdf)?; + env.field(&mut out, "pdf", pdf)?; + } + env.key(&mut out, "document")?; + let mut scan = KeyScan::new(Indent { inner: &mut out, on: pretty }); + // Without blobs no key matters. + scan.active = images.is_none() && !blobs.is_empty(); + let mut buffered = std::io::BufWriter::with_capacity(JSON_CHUNK, &mut scan); + write_body(&mut buffered, v1.as_ref(), &d, pretty)?; + buffered.flush().map_err(encode_err)?; + drop(buffered); + let used = std::mem::take(&mut scan.keys); + let images = images.unwrap_or_else(|| entries.images(&used, &mut out_blobs)); + env.field(&mut out, "images", &images)?; + env.finish(&mut out)?; + if o.compress { + let json = out.written().unwrap_or(0); + let tail: Vec = out_blobs.tail.iter().map(|(b, _)| b.len() as u64).collect(); + if json > MAX_JSON { + return Err(FormatError::TooBigToCompress(format!("{} MB of document, at most {} MB", json >> 20, MAX_JSON >> 20))); + } + if let Some(big) = tail.iter().find(|&&n| n > MAX_BLOB) { + return Err(FormatError::TooBigToCompress(format!("an image of {} MB, at most {} MB", big >> 20, MAX_BLOB >> 20))); + } + if out_blobs.at > MAX_BLOBS { + return Err(FormatError::TooBigToCompress(format!("{} MB of images, at most {} MB", out_blobs.at >> 20, MAX_BLOBS >> 20))); + } } - w.raw("document", &body); - w.field("images", &images)?; - let mut out = w.finish(); if !out_blobs.tail.is_empty() { - out.extend_from_slice(BLOB_MAGIC); - out.extend_from_slice(&out_blobs.tail); + out.write_all(BLOB_MAGIC).map_err(encode_err)?; + for &(bytes, packed) in &out_blobs.tail { + out.packed(packed).map_err(encode_err)?; + out.write_all(bytes).map_err(encode_err)?; + } } - if !o.compress { - return Ok(out); + out.finish().map_err(encode_err) +} + +/// The images a file lists, from the document's blobs. +struct Entries<'a> { + blobs: &'a BTreeMap, + /// Images only linked images show (see [`Document::linked_only_images`]). + linked: &'a std::collections::BTreeSet, + include_linked: bool, +} + +impl<'a> Entries<'a> { + /// The entries of the images named in `used`, in the order of their names. + fn images(&self, used: &HashSet, out: &mut Blobs<'a>) -> BTreeMap<&'a str, Image> { + self.blobs + .iter() + .filter(|(k, _)| used.contains(k.as_str())) + .map(|(k, b)| { + // Include Linked Files keeps the file's bytes whenever they are loaded. + let preview_only = self.linked.contains(k) && (!self.include_linked || b.is_proxy()); + let image = match b.proxy.as_ref().filter(|_| preview_only) { + Some(p) => out.image(preview_mime(p), p, true), + None => out.image(&b.mime, &b.bytes, false), + }; + (k.as_str(), image) + }) + .collect() } - let mut gz = flate2::write::GzEncoder::new(Vec::with_capacity(out.len() / 4), flate2::Compression::default()); - gz.write_all(&out).map_err(encode_err)?; - gz.finish().map_err(encode_err) } fn encode_err(e: impl std::fmt::Display) -> FormatError { @@ -330,85 +405,51 @@ fn to_json(v: &impl Serialize, pretty: bool) -> Result, FormatError> { if pretty { serde_json::to_vec_pretty(v) } else { serde_json::to_vec(v) }.map_err(encode_err) } -/// The file's top-level object, written member by member (the document goes in pre-serialized). +/// The document's JSON: its version 1 form `v1`, else `d` (generic, not `dyn Write`: serde_json +/// writes token by token). +fn write_body(w: impl Write, v1: Option<&serde_json::Value>, d: &Document, pretty: bool) -> Result<(), FormatError> { + match v1 { + Some(v) => write_json(w, v, pretty), + None => write_json(w, d, pretty), + } +} + +fn write_json(w: impl Write, v: &impl Serialize, pretty: bool) -> Result<(), FormatError> { + if pretty { serde_json::to_writer_pretty(w, v) } else { serde_json::to_writer(w, v) }.map_err(encode_err) +} + +/// The file's top-level object, written member by member (the document is streamed in). struct Envelope { - out: Vec, pretty: bool, + first: bool, } impl Envelope { - fn field(&mut self, key: &str, value: &impl Serialize) -> Result<(), FormatError> { + fn field(&mut self, out: &mut impl Write, key: &str, value: &impl Serialize) -> Result<(), FormatError> { let json = to_json(value, self.pretty)?; - self.raw(key, &json); - Ok(()) + self.key(out, key)?; + Indent { inner: out, on: self.pretty }.write_all(&json).map_err(encode_err) } - /// Member `key` (a plain name) with the JSON text `json`, indented one level when pretty (as - /// serde_json would). - fn raw(&mut self, key: &str, json: &[u8]) { - let out = &mut self.out; - out.push(if out.is_empty() { b'{' } else { b',' }); - if self.pretty { - out.extend_from_slice(b"\n "); - } - out.push(b'"'); - out.extend_from_slice(key.as_bytes()); - out.extend_from_slice(if self.pretty { b"\": " } else { b"\":" }); - // Newlines only occur between tokens (inside strings they are escaped). - let mut lines = json.split(|&b| b == b'\n'); - out.extend_from_slice(lines.next().unwrap_or_default()); - for line in lines { - out.extend_from_slice(b"\n "); - out.extend_from_slice(line); - } + /// The start of member `key` (a plain name): its value follows, indented one level when + /// pretty (as serde_json would: see [`Indent`]). + fn key(&mut self, out: &mut impl Write, key: &str) -> Result<(), FormatError> { + let sep: &[u8] = if std::mem::take(&mut self.first) { b"{" } else { b"," }; + let indent: &[u8] = if self.pretty { b"\n " } else { b"" }; + let colon: &[u8] = if self.pretty { b"\": " } else { b"\":" }; + [sep, indent, b"\"", key.as_bytes(), colon].iter().try_for_each(|part| out.write_all(part)).map_err(encode_err) } - fn finish(mut self) -> Vec { - self.out.extend_from_slice(if self.pretty { b"\n}" } else { b"}" }); - self.out + fn finish(self, out: &mut impl Write) -> Result<(), FormatError> { + out.write_all(if self.pretty { b"\n}" } else { b"}" }).map_err(encode_err) } } -/// The string values of every `"": "…"` member at any depth of JSON text. Quotes inside JSON -/// strings are escaped, so every match is a real member. -fn string_members(json: &[u8], name: &str) -> HashSet { - let tag = format!("\"{name}\":"); - let tag = tag.as_bytes(); - let mut out = HashSet::new(); - let mut rest = json; - while let Some(at) = rest.windows(tag.len()).position(|w| w == tag) { - rest = rest.get(at + tag.len()..).unwrap_or_default(); - let value = rest.trim_ascii_start(); - if value.first() == Some(&b'"') - && let Some(Ok(s)) = serde_json::Deserializer::from_slice(value).into_iter::().next() - { - out.insert(s); - } - } - out -} - /// Is this gzip data (a compressed file)? pub fn is_compressed(bytes: &[u8]) -> bool { bytes.starts_with(&[0x1f, 0x8b]) } -/// The JSON text of a file, unpacked when compressed. -fn unpack(bytes: &[u8]) -> Result, FormatError> { - if !is_compressed(bytes) { - return Ok(Cow::Borrowed(bytes)); - } - let mut raw = Vec::new(); - flate2::read::GzDecoder::new(bytes) - .take(MAX_UNPACKED + 1) - .read_to_end(&mut raw) - .map_err(|e| FormatError::NotVectorcraft(format!("damaged compressed data: {e}")))?; - if raw.len() as u64 > MAX_UNPACKED { - return Err(FormatError::TooBig); - } - Ok(Cow::Owned(raw)) -} - pub fn load(bytes: &[u8]) -> Result { load_info(bytes).map(|(doc, _)| doc) } @@ -445,18 +486,24 @@ pub fn load_info(bytes: &[u8]) -> Result<(Document, FileInfo), FormatError> { /// [`load`], with everything else the file carries. pub fn load_file(bytes: &[u8]) -> Result { - let text = unpack(bytes)?; - let (json, tail) = split_tail(&text); - let f: File = serde_json::from_slice(json).map_err(|e| FormatError::NotVectorcraft(e.to_string()))?; + let file = unpack::Unpacked::new(bytes)?; + let f: File = serde_json::from_slice(file.json()).map_err(|e| FormatError::NotVectorcraft(e.to_string()))?; if f.format != EXTENSION && f.format != LEGACY_EXTENSION { return Err(FormatError::NotVectorcraft(format!("format is `{}`", f.format))); } if f.version > VERSION { return Err(FormatError::TooNew(f.version)); } + // The bytes after the JSON, read in one pass: the images' first, then the profiles'. + let entries: Vec<[u64; 2]> = f.images.values().chain(f.profiles.values()).filter_map(|i| i.blob).collect(); + let mut tail = file.entries(&entries)?.into_iter(); + let mut bytes_of = |img: &Image| match img.blob { + Some(_) => tail.next().flatten(), + None => img.inline(), + }; let mut doc = f.document; for (k, img) in f.images { - let bytes = img.bytes(tail).ok_or_else(|| FormatError::BadImage(k.clone()))?; + let bytes = bytes_of(&img).ok_or_else(|| FormatError::BadImage(k.clone()))?; let mut blob = ImageBlob::new(img.mime, bytes); // Until the linked file is read, the preview stands in for it. if img.proxy { @@ -474,7 +521,7 @@ pub fn load_file(bytes: &[u8]) -> Result { // Untrusted: capped, named and pointing at objects the file has. doc.tidy_saved_selections(); doc.tidy_graph_designs(); - let profiles = f.profiles.into_iter().filter_map(|(name, icc)| Some((name, icc.bytes(tail)?))).collect(); + let profiles = f.profiles.into_iter().filter_map(|(name, icc)| Some((name, bytes_of(&icc)?))).collect(); Ok(NativeFile { doc, info: FileInfo { version: f.version, legacy: f.format == LEGACY_EXTENSION }, profiles }) } @@ -485,24 +532,27 @@ struct Head { pdf: Option, } -/// The file's head and the data after its JSON. -fn head(bytes: &[u8]) -> Option<(Head, Vec)> { - let text = unpack(bytes).ok()?; - let (json, tail) = split_tail(&text); - Some((serde_json::from_slice(json).ok()?, tail.to_vec())) +/// The bytes of the member of the file's head `pick` picks (only that entry of the data after +/// the JSON is kept). +fn head_member(bytes: &[u8], pick: impl FnOnce(Head) -> Option) -> Option> { + let file = unpack::Unpacked::new(bytes).ok()?; + let head: Head = serde_json::from_slice(file.json()).ok()?; + let img = pick(head)?; + match img.blob { + Some(at) => file.entries(&[at]).ok()?.pop().flatten(), + None => img.inline(), + } } /// The preview PNG embedded in a native file (`None`: it has none, or isn't a native file). pub fn preview(bytes: &[u8]) -> Option> { - let (h, tail) = head(bytes)?; - h.preview?.bytes(&tail) + head_member(bytes, |h| h.preview) } /// The PDF a native file saved with Create PDF-Compatible File carries (`None`: it has none, or /// isn't a native file). pub fn pdf_content(bytes: &[u8]) -> Option> { - let (h, tail) = head(bytes)?; - h.pdf?.bytes(&tail) + head_member(bytes, |h| h.pdf) } /// Does this look like a `.vectorcraft` file (compressed or not)? diff --git a/crates/format/src/stream.rs b/crates/format/src/stream.rs new file mode 100644 index 00000000..60879c07 --- /dev/null +++ b/crates/format/src/stream.rs @@ -0,0 +1,358 @@ +//! Writing a file as a stream: the JSON is serialized straight into the output (compressed or +//! not) and the data after it is copied in from the document's own buffers, so a save holds no +//! copy of the file in memory besides what the caller's writer keeps. + +use std::collections::HashSet; +use std::io::{self, Write}; + +use flate2::{Compress, Compression, Crc, FlushCompress, Status}; + +/// The output of a save: plain, or gzip ([`Gz`]). +pub(crate) enum Out { + Plain(W), + Gz(Box>), +} + +impl Out { + /// What is written next is compressed already (JPEG, PNG…) or not ([`Gz::packed`]). + pub(crate) fn packed(&mut self, packed: bool) -> io::Result<()> { + match self { + Out::Plain(_) => Ok(()), + Out::Gz(gz) => gz.packed(packed), + } + } + + /// Bytes written so far (before compression). + pub(crate) fn written(&self) -> Option { + match self { + Out::Plain(_) => None, + Out::Gz(gz) => Some(gz.total), + } + } + + /// Everything written: the gzip trailer goes out, and the writer is flushed. + pub(crate) fn finish(self) -> io::Result<()> { + match self { + Out::Plain(mut w) => w.flush(), + Out::Gz(gz) => gz.finish(), + } + } +} + +impl Write for Out { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Out::Plain(w) => w.write(buf), + Out::Gz(gz) => gz.write(buf), + } + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + match self { + Out::Plain(w) => w.write_all(buf), + Out::Gz(gz) => gz.write_all(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Out::Plain(w) => w.flush(), + Out::Gz(gz) => gz.flush(), + } + } +} + +/// Bytes of compressed output gathered before they go to the writer. +const OUT_CHUNK: usize = 64 << 10; + +/// A gzip stream, written as [`flate2::write::GzEncoder`] at the default level writes it (the +/// same header, and the same bytes for a file without compressed images), except that data which +/// is compressed already ([`Gz::packed`]) is stored as it is: the default level spends a second +/// or more per photo on it for next to nothing, and makes it slower to unpack. Where the level +/// changes the deflate blocks so far end with a sync flush and a fresh compressor writes on, +/// which makes one valid deflate stream that every reader unpacks. +pub(crate) struct Gz { + out: W, + deflate: Compress, + /// What `deflate` writes: stored blocks (packed data), else the default level. + stored: bool, + crc: Crc, + buf: Vec, + /// Bytes written (unpacked). + total: u64, +} + +impl Gz { + pub(crate) fn new(mut out: W) -> io::Result { + // flate2's header for the default level: no name, no time, unknown OS. + out.write_all(&[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255])?; + Ok(Self { + out, + deflate: Compress::new(Compression::default(), false), + stored: false, + crc: Crc::new(), + buf: Vec::with_capacity(OUT_CHUNK), + total: 0, + }) + } + + /// Run the compressor over `input` with `flush` until it has taken it all (and, flushing, + /// written everything out). + fn deflate(&mut self, mut input: &[u8], flush: FlushCompress) -> io::Result<()> { + loop { + if self.buf.len() == self.buf.capacity() { + self.out.write_all(&self.buf)?; + self.buf.clear(); + } + let (before_in, before_out) = (self.deflate.total_in(), self.deflate.total_out()); + let status = self.deflate.compress_vec(input, &mut self.buf, flush).map_err(io::Error::other)?; + let taken = usize::try_from(self.deflate.total_in() - before_in).unwrap_or(input.len()).min(input.len()); + input = input.get(taken..).unwrap_or_default(); + let made = self.deflate.total_out() - before_out; + let full = self.buf.len() == self.buf.capacity(); + match flush { + FlushCompress::None if input.is_empty() => return Ok(()), + FlushCompress::Finish if status == Status::StreamEnd => return Ok(()), + // A flush is done once it left room in the output. + FlushCompress::Sync if input.is_empty() && !full => return Ok(()), + _ => {} + } + if taken == 0 && made == 0 && !full { + return Err(io::Error::other("the compressor made no progress")); + } + } + } + + /// What is written next is compressed already (stored as it is), or not (compressed at the + /// default level). + fn packed(&mut self, packed: bool) -> io::Result<()> { + if packed == self.stored { + return Ok(()); + } + self.deflate(&[], FlushCompress::Sync)?; + self.deflate = Compress::new(if packed { Compression::none() } else { Compression::default() }, false); + self.stored = packed; + Ok(()) + } + + fn finish(mut self) -> io::Result<()> { + self.deflate(&[], FlushCompress::Finish)?; + self.out.write_all(&self.buf)?; + self.out.write_all(&self.crc.sum().to_le_bytes())?; + self.out.write_all(&self.crc.amount().to_le_bytes())?; + self.out.flush() + } +} + +impl Write for Gz { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.crc.update(buf); + self.total += buf.len() as u64; + self.deflate(buf, FlushCompress::None) + } + + fn flush(&mut self) -> io::Result<()> { + // Flushing would end a deflate block: the stream is flushed when it is finished. + Ok(()) + } +} + +/// Indents JSON text written through it one level (two spaces after each newline), as a member +/// of the file's top-level object; newlines only occur between tokens, inside strings they are +/// escaped. +pub(crate) struct Indent { + pub(crate) inner: W, + pub(crate) on: bool, +} + +impl Write for Indent { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + if !self.on { + return self.inner.write_all(buf); + } + let mut lines = buf.split(|&b| b == b'\n'); + self.inner.write_all(lines.next().unwrap_or_default())?; + for line in lines { + self.inner.write_all(b"\n ")?; + self.inner.write_all(line)?; + } + Ok(()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// The member that names an image. +const TAG: &[u8] = b"\"key\":"; + +/// Passes JSON text through, collecting the string values of every `"key": "…"` member at any +/// depth (what the images a document uses are named by). Quotes inside JSON strings are escaped, +/// so every match is a real member. +pub(crate) struct KeyScan { + pub(crate) inner: W, + pub(crate) keys: HashSet, + /// Looking for keys (else it only passes the text on). + pub(crate) active: bool, + /// How much of [`TAG`] the last bytes matched. + matched: usize, + /// Right after a tag: skipping whitespace to its value. + after_tag: bool, + /// The value string so far, quotes included. + value: Option>, + escaped: bool, +} + +impl KeyScan { + pub(crate) fn new(inner: W) -> Self { + Self { inner, keys: HashSet::new(), active: true, matched: 0, after_tag: false, value: None, escaped: false } + } + + fn scan(&mut self, buf: &[u8]) { + if !self.active { + return; + } + let mut i = 0; + while let Some(&b) = buf.get(i) { + // Nothing in progress: skip to the next quote. + if self.matched == 0 && !self.after_tag && self.value.is_none() { + match buf.get(i..).and_then(|r| r.iter().position(|&c| c == b'"')) { + Some(at) => i += at, + None => return, + } + self.matched = 1; + i += 1; + continue; + } + self.step(b); + i += 1; + } + } + + fn step(&mut self, b: u8) { + if let Some(v) = self.value.as_mut() { + v.push(b); + if self.escaped { + self.escaped = false; + } else if b == b'\\' { + self.escaped = true; + } else if b == b'"' + && let Some(v) = self.value.take() + && let Ok(s) = serde_json::from_slice::(&v) + { + self.keys.insert(s); + } + } else if self.after_tag && !b.is_ascii_whitespace() { + self.after_tag = false; + if b == b'"' { + self.value = Some(vec![b]); + } + } + if TAG.get(self.matched) == Some(&b) { + self.matched += 1; + if self.matched == TAG.len() { + self.matched = 0; + self.after_tag = true; + self.value = None; + self.escaped = false; + } + } else { + self.matched = usize::from(b == b'"'); + } + } +} + +impl Write for KeyScan { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.scan(buf); + self.inner.write_all(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + /// How the keys were found before saves were streamed: in the whole JSON text at once. + fn string_members(json: &[u8]) -> HashSet { + let mut out = HashSet::new(); + let mut rest = json; + while let Some(at) = rest.windows(TAG.len()).position(|w| w == TAG) { + rest = rest.get(at + TAG.len()..).unwrap_or_default(); + let value = rest.trim_ascii_start(); + if value.first() == Some(&b'"') + && let Some(Ok(s)) = serde_json::Deserializer::from_slice(value).into_iter::().next() + { + out.insert(s); + } + } + out + } + + /// The keys `KeyScan` finds in `json` written in pieces of `chunk` bytes. + fn scanned(json: &[u8], chunk: usize) -> HashSet { + let mut scan = KeyScan::new(Vec::new()); + for piece in json.chunks(chunk.max(1)) { + scan.write_all(piece).unwrap(); + } + assert_eq!(scan.inner, json, "the text passes through"); + scan.keys + } + + #[test] + fn finds_the_keys_the_whole_text_search_found() { + let v = serde_json::json!({"key": "a", "x": [{"key": "b \"q\" \\ é"}, {"monkey": "no"}, {"k": "key"}], "s": "\"key\": \"no\"", "key2": "n", "o": {"key": 3}}); + for json in [serde_json::to_vec(&v).unwrap(), serde_json::to_vec_pretty(&v).unwrap()] { + let want = string_members(&json); + assert_eq!(want, HashSet::from(["a".to_string(), "b \"q\" \\ é".to_string()])); + for chunk in [1, 2, 3, 7, 64, json.len()] { + assert_eq!(scanned(&json, chunk), want, "chunk {chunk}"); + } + } + } + + fn arb_json() -> impl Strategy { + let key = prop::sample::select(vec!["key", "k", "kkey", "key ", "x"]).prop_map(str::to_string); + let leaf = prop_oneof![ + Just(serde_json::Value::Null), + any::().prop_map(serde_json::Value::from), + "[a-z\"\\\\: ké\n]{0,8}".prop_map(serde_json::Value::from), + ]; + leaf.prop_recursive(4, 32, 6, move |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..5).prop_map(serde_json::Value::from), + prop::collection::vec((key.clone(), inner), 0..5).prop_map(|m| serde_json::Value::Object(m.into_iter().collect())), + ] + }) + } + + proptest! { + #[test] + fn scanning_in_pieces_finds_what_the_whole_text_search_found(v in arb_json(), chunk in 1usize..40, pretty in any::()) { + let json = if pretty { serde_json::to_vec_pretty(&v) } else { serde_json::to_vec(&v) }.unwrap(); + prop_assert_eq!(scanned(&json, chunk), string_members(&json)); + } + } +} diff --git a/crates/format/src/unpack.rs b/crates/format/src/unpack.rs new file mode 100644 index 00000000..360b756e --- /dev/null +++ b/crates/format/src/unpack.rs @@ -0,0 +1,214 @@ +//! Reading a file: its JSON, then the entries of the data after it (v4) one by one. A compressed +//! file is unpacked as a stream: the JSON into one buffer, each entry straight into its own, so +//! the file is never held unpacked as a whole. Limits apply per part ([`MAX_JSON`], [`MAX_BLOB`], +//! [`MAX_BLOBS`]), guarding against decompression bombs. + +use std::io::{self, Read}; + +use flate2::read::GzDecoder; + +use crate::{BLOB_MAGIC, FormatError, MAX_BLOB, MAX_BLOBS, MAX_JSON, MAX_UNPACKED, is_compressed, split_tail}; + +/// Bytes unpacked at a time while looking for the end of the JSON. +const CHUNK: u64 = 64 << 10; + +fn damaged(e: impl std::fmt::Display) -> FormatError { + FormatError::NotVectorcraft(format!("damaged compressed data: {e}")) +} + +/// A file opened for reading: its JSON, and where the data after it is. +pub(crate) struct Unpacked<'a> { + json: Json<'a>, + tail: Tail<'a>, +} + +enum Json<'a> { + Borrowed(&'a [u8]), + Owned(Vec), + /// Unpacked whole (a compressed file whose JSON isn't followed by the marker where the writer + /// puts it): `buf[..json]` is the JSON ([`Tail::Whole`] says where the data after it is). + Whole { + buf: Vec, + json: usize, + }, +} + +enum Tail<'a> { + /// The data after the JSON, in memory (an uncompressed file, or one unpacked whole). + Slice(&'a [u8]), + /// In memory, in [`Json::Whole`]'s buffer from this offset. + Whole(usize), + /// Still to unpack from `file`: it starts at `at` in the unpacked stream; `read` holds what + /// was unpacked past the marker already, and `rest` the stream from there. + Stream { + file: &'a [u8], + at: u64, + read: Vec, + rest: Box>, + }, + None, +} + +impl<'a> Unpacked<'a> { + /// Open `bytes` (compressed or not). + pub(crate) fn new(bytes: &'a [u8]) -> Result { + if !is_compressed(bytes) { + let (json, tail) = split_tail(bytes); + return Ok(Self { json: Json::Borrowed(json), tail: Tail::Slice(tail) }); + } + let mut dec = GzDecoder::new(bytes); + let mut buf = Vec::new(); + let mut nul = None; + loop { + let start = buf.len(); + let n = (&mut dec).take(CHUNK).read_to_end(&mut buf).map_err(damaged)?; + if nul.is_none() { + nul = buf.get(start..).and_then(|b| b.iter().position(|&c| c == 0)).map(|p| p + start); + } + if n == 0 || nul.is_some_and(|z| buf.len() >= z + BLOB_MAGIC.len()) { + break; + } + if buf.len() as u64 > MAX_JSON + CHUNK { + return Err(FormatError::TooBig); + } + } + let Some(z) = nul else { + // No NUL, so no marker: all of it is JSON. + if buf.len() as u64 > MAX_JSON { + return Err(FormatError::TooBig); + } + return Ok(Self { json: Json::Owned(buf), tail: Tail::None }); + }; + // JSON holds no raw NUL, so the marker (which has one) starts right before the first NUL; + // the writer puts it right after the JSON's last byte. + if let Some(end) = z.checked_sub(1) + && buf.get(end..).is_some_and(|r| r.starts_with(BLOB_MAGIC)) + && end.checked_sub(1).and_then(|k| buf.get(k)).is_some_and(|b| matches!(b, b'}' | b']' | b'"')) + { + if end as u64 > MAX_JSON { + return Err(FormatError::TooBig); + } + let read = buf.split_off(end + BLOB_MAGIC.len()); + buf.truncate(end); + let at = (end + BLOB_MAGIC.len()) as u64; + return Ok(Self { json: Json::Owned(buf), tail: Tail::Stream { file: bytes, at, read, rest: Box::new(dec) } }); + } + // Anything else (hand-made or damaged files) is unpacked whole and split as an + // uncompressed file is. + dec.take(MAX_UNPACKED + 1 - (buf.len() as u64).min(MAX_UNPACKED)).read_to_end(&mut buf).map_err(damaged)?; + if buf.len() as u64 > MAX_UNPACKED { + return Err(FormatError::TooBig); + } + let (json, tail) = split_tail(&buf); + let (json, tail) = (json.len(), buf.len() - tail.len()); + Ok(Self { json: Json::Whole { buf, json }, tail: Tail::Whole(tail) }) + } + + /// The JSON text. + pub(crate) fn json(&self) -> &[u8] { + match &self.json { + Json::Borrowed(j) => j, + Json::Owned(j) => j, + Json::Whole { buf, json, .. } => buf.get(..*json).unwrap_or_default(), + } + } + + /// The entries `[offset, length]` of the data after the JSON → each one's bytes, `None` for + /// one that isn't there (the data ends before it). Errors: the compressed data is damaged, or + /// the entries are larger than [`MAX_BLOB`] each or [`MAX_BLOBS`] together. Entries are read + /// in the order of their offsets, each straight into its own buffer. + pub(crate) fn entries(self, wanted: &[[u64; 2]]) -> Result>>, FormatError> { + let mut out: Vec>> = vec![None; wanted.len()]; + let in_memory = |tail: &[u8], out: &mut Vec>>| { + for (slot, &[offset, len]) in out.iter_mut().zip(wanted) { + *slot = range(offset, len).and_then(|(a, b)| tail.get(a..b)).map(<[u8]>::to_vec); + } + }; + match self.tail { + Tail::Slice(tail) => in_memory(tail, &mut out), + Tail::Whole(at) => { + let buf = match &self.json { + Json::Whole { buf, .. } => buf.as_slice(), + _ => &[], + }; + in_memory(buf.get(at..).unwrap_or_default(), &mut out); + } + Tail::None => in_memory(&[], &mut out), + Tail::Stream { file, at, read, rest } => { + let mut total = 0u64; + for &[_, len] in wanted { + if len > MAX_BLOB { + return Err(FormatError::TooBig); + } + total = total.saturating_add(len); + } + if total > MAX_BLOBS { + return Err(FormatError::TooBig); + } + drop(self.json); + let mut order: Vec = (0..wanted.len()).collect(); + order.sort_by_key(|&i| wanted.get(i).map_or(0, |w| w[0])); + let mut s = Stream { r: Box::new(io::Cursor::new(read).chain(rest)), pos: 0, file, at }; + for i in order { + let Some(&[offset, len]) = wanted.get(i) else { continue }; + if let Some(slot) = out.get_mut(i) { + *slot = s.entry(offset, len)?; + } + } + // To the end, so the stream's checksum is verified as before. + s.skip(MAX_BLOBS)?; + } + } + Ok(out) + } +} + +/// `offset..offset + len` as indices, when they fit. +fn range(offset: u64, len: u64) -> Option<(usize, usize)> { + let a = usize::try_from(offset).ok()?; + Some((a, a.checked_add(usize::try_from(len).ok()?)?)) +} + +/// The data after a compressed file's JSON, being unpacked. +struct Stream<'a> { + r: Box, + /// Where `r` is in the data after the JSON. + pos: u64, + file: &'a [u8], + /// Where the data after the JSON starts in the unpacked stream. + at: u64, +} + +impl Stream<'_> { + /// Unpack and drop up to `n` bytes → how many there were. + fn skip(&mut self, n: u64) -> Result { + let skipped = io::copy(&mut (&mut self.r).take(n), &mut io::sink()).map_err(damaged)?; + self.pos += skipped; + Ok(skipped) + } + + /// The entry at `offset`, `len` bytes long (`None`: the data ends before it does). + fn entry(&mut self, offset: u64, len: u64) -> Result>, FormatError> { + if offset < self.pos { + // Entries out of order or overlapping (only hand-made files): unpack again from the + // start. + let mut dec = GzDecoder::new(self.file); + let skipped = io::copy(&mut (&mut dec).take(self.at), &mut io::sink()).map_err(damaged)?; + if skipped < self.at { + return Ok(None); + } + self.r = Box::new(dec); + self.pos = 0; + } + let gap = offset - self.pos; + if self.skip(gap)? < gap { + return Ok(None); + } + let mut bytes = Vec::new(); + let n = usize::try_from(len).map_err(|_| FormatError::TooBig)?; + bytes.try_reserve_exact(n).map_err(|_| FormatError::TooBig)?; + let got = (&mut self.r).take(len).read_to_end(&mut bytes).map_err(damaged)?; + self.pos += got as u64; + Ok((got as u64 == len).then_some(bytes)) + } +} diff --git a/crates/format/tests/streaming.rs b/crates/format/tests/streaming.rs new file mode 100644 index 00000000..81d935f8 --- /dev/null +++ b/crates/format/tests/streaming.rs @@ -0,0 +1,199 @@ +//! Saving and opening as streams: the bytes a save writes are those an in-memory save wrote +//! before (the same JSON and data after it, unpacked; the same gzip bytes when no image is +//! compressed already), files written the old way still open, entries are read whatever their +//! order, and the limits apply per part of the file. +// Integration tests: unwrapping and panicking on failure is fine here, unlike in shipped code (AGENTS.md › Robustness). +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::BTreeMap; +use std::io::{Read, Write}; + +use vectorcraft_doc::{Document, ImageBlob, ImageObject, Node, NodeKind}; +use vectorcraft_format::{BLOB_MAGIC, FormatError, INLINE_MAX, MAX_BLOB, SaveOptions, load, load_file, pdf_content, preview, save_to, save_with}; +use vectorcraft_geom::Affine; + +/// A document showing images `(key, mime, len)`. +fn doc_with(images: &[(&str, &str, usize)]) -> Document { + let mut d = Document::new(100.0, 100.0); + let l = d.layers[0].id; + for (n, &(key, mime, len)) in images.iter().enumerate() { + let bytes: Vec = (0..len).map(|i| ((i * 7 + n * 13) % 251) as u8).collect(); + d.images.insert(key.into(), ImageBlob::new(mime, bytes)); + let id = d.alloc_id(); + let im = ImageObject { key: key.into(), width: 10, height: 10, xf: Affine::IDENTITY, link: None, placement: Default::default() }; + d.insert(Some(l), 0, Node::new(id, NodeKind::Image(Box::new(im)))).unwrap(); + } + d +} + +/// gzip as the app compressed files before saves were streamed (the whole file at once). +fn gzip(bytes: &[u8]) -> Vec { + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(bytes).unwrap(); + gz.finish().unwrap() +} + +fn gunzip(bytes: &[u8]) -> Vec { + let mut out = vec![]; + flate2::read::GzDecoder::new(bytes).read_to_end(&mut out).unwrap(); + out +} + +fn mixed() -> Document { + doc_with(&[ + ("photo", "image/jpeg", INLINE_MAX * 3), + ("scan", "image/tiff", INLINE_MAX * 2), + ("icon", "image/png", 100), + ("art", "image/png", INLINE_MAX + 1), + ]) +} + +#[test] +fn compressed_files_unpack_to_the_plain_file() { + let pdf = vec![b'%'; INLINE_MAX * 2]; + let profiles = BTreeMap::from([("Mine".to_string(), vec![7u8; INLINE_MAX + 3])]); + for d in [Document::new(10.0, 10.0), mixed()] { + let o = SaveOptions { pdf: Some(pdf.clone()), profiles: profiles.clone(), preview: Some(vec![1, 2, 3]), ..SaveOptions::default() }; + let plain = save_with(&d, &o).unwrap(); + let packed = save_with(&d, &SaveOptions { compress: true, ..o }).unwrap(); + assert_eq!(gunzip(&packed), plain, "the same file, compressed"); + let back = load_file(&packed).unwrap(); + for (k, b) in &d.images { + assert_eq!(back.doc.images[k].bytes, b.bytes, "{k}"); + } + assert_eq!(back.profiles, profiles); + assert_eq!(pdf_content(&packed).unwrap(), pdf); + assert_eq!(preview(&packed).unwrap(), vec![1, 2, 3]); + } +} + +#[test] +fn without_compressed_images_the_gzip_bytes_are_the_old_ones() { + let mut d = doc_with(&[("scan", "image/tiff", INLINE_MAX * 2), ("icon", "image/png", 100)]); + d.title = "No photos".into(); + for o in [SaveOptions::default(), SaveOptions { pdf: Some(vec![b'%'; INLINE_MAX]), ..SaveOptions::default() }] { + let plain = save_with(&d, &o).unwrap(); + assert_eq!(save_with(&d, &SaveOptions { compress: true, ..o }).unwrap(), gzip(&plain)); + } +} + +#[test] +fn files_compressed_the_old_way_still_open() { + let d = mixed(); + let old = gzip(&save_with(&d, &SaveOptions::default()).unwrap()); + let back = load(&old).unwrap(); + assert_eq!(back.images.len(), d.images.len()); + for (k, b) in &d.images { + assert_eq!(back.images[k].bytes, b.bytes, "{k}"); + } + assert_eq!(back.node_count(), d.node_count()); +} + +#[test] +fn save_to_writes_what_save_with_returns() { + let d = mixed(); + for compress in [false, true] { + let o = SaveOptions { compress, ..SaveOptions::default() }; + let mut streamed = vec![]; + save_to(&d, &o, &mut streamed).unwrap(); + assert_eq!(streamed, save_with(&d, &o).unwrap()); + let back = load(&streamed).unwrap(); + assert_eq!(back.images["photo"].bytes, d.images["photo"].bytes); + } + // A failing writer fails the save. + struct Full; + impl Write for Full { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("disk full")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + assert!(save_to(&d, &SaveOptions::default(), Full).is_err()); + assert!(save_to(&d, &SaveOptions { compress: true, ..SaveOptions::default() }, Full).is_err()); +} + +/// `plain` with its JSON changed by `edit` (the data after it kept). +fn with_json(plain: &[u8], edit: impl FnOnce(&mut serde_json::Value)) -> Vec { + let at = plain.windows(BLOB_MAGIC.len()).position(|w| w == BLOB_MAGIC).unwrap(); + let mut v: serde_json::Value = serde_json::from_slice(&plain[..at]).unwrap(); + edit(&mut v); + let mut out = serde_json::to_vec(&v).unwrap(); + out.extend_from_slice(&plain[at..]); + out +} + +#[test] +fn entries_in_any_order_or_shared_read_back() { + let d = doc_with(&[("a", "image/png", INLINE_MAX), ("b", "image/png", INLINE_MAX + 5)]); + let plain = save_with(&d, &SaveOptions::default()).unwrap(); + // Swap the entries' places (b's bytes listed first), and point a third image at a's bytes. + let file = with_json(&plain, |v| { + let (a, b) = (v["images"]["a"]["blob"].clone(), v["images"]["b"]["blob"].clone()); + v["images"]["a"]["blob"] = b.clone(); + v["images"]["b"]["blob"] = a.clone(); + v["images"]["c"] = serde_json::json!({"mime": "image/png", "blob": b}); + }); + for bytes in [file.clone(), gzip(&file)] { + let back = load(&bytes).unwrap(); + assert_eq!(back.images["a"].bytes, d.images["b"].bytes); + assert_eq!(back.images["b"].bytes, d.images["a"].bytes); + assert_eq!(back.images["c"].bytes, d.images["b"].bytes); + } +} + +#[test] +fn limits_apply_per_entry_and_to_all_entries() { + let d = doc_with(&[("a", "image/png", INLINE_MAX), ("b", "image/png", INLINE_MAX)]); + let plain = save_with(&d, &SaveOptions::default()).unwrap(); + // One entry past the limit: refused before anything is allocated. + let huge = with_json(&plain, |v| v["images"]["a"]["blob"] = serde_json::json!([0, MAX_BLOB + 1])); + assert!(matches!(load_file(&gzip(&huge)), Err(FormatError::TooBig))); + // Each within it, together past the limit for all. + let many = with_json(&plain, |v| { + for i in 0..5 { + v["images"][format!("x{i}")] = serde_json::json!({"mime": "image/png", "blob": [0, MAX_BLOB]}); + } + }); + assert!(matches!(load_file(&gzip(&many)), Err(FormatError::TooBig))); + // A length the data doesn't have: the image is damaged, nothing more. + let short = with_json(&plain, |v| v["images"]["a"]["blob"] = serde_json::json!([0, 1u64 << 20])); + assert!(matches!(load_file(&gzip(&short)), Err(FormatError::BadImage(_)))); + // Uncompressed files have no limits to apply (the file is all there is). + assert!(matches!(load_file(&huge), Err(FormatError::BadImage(_)))); +} + +#[test] +fn a_damaged_compressed_file_is_refused() { + let bytes = save_with(&mixed(), &SaveOptions { compress: true, ..SaveOptions::default() }).unwrap(); + // The data after the JSON cut short, then its checksum wrong. + assert!(load_file(&bytes[..bytes.len() - 4000]).is_err()); + let mut bad = bytes.clone(); + let n = bad.len(); + bad[n - 6] ^= 0xff; + assert!(load_file(&bad).is_err(), "the checksum is checked to the end"); +} + +#[test] +fn documents_read_whatever_their_key_order_and_keep_keys_they_dont_know() { + let d = mixed(); + let plain = save_with(&d, &SaveOptions::default()).unwrap(); + let file = with_json(&plain, |v| { + // serde_json's map keeps keys sorted: `type` is no longer first in the node kinds (a + // layer's `children`, `clip` and `color` come before it), and the document's fields come + // in another order. + v["document"]["aaa_from_a_newer_version"] = serde_json::json!({"x": [1, 2]}); + v["document"]["zzz_too"] = serde_json::json!("kept"); + }); + let text = String::from_utf8_lossy(&file); + assert!(!text.contains(r#"{"type":"layer""#) && text.contains(r#""type":"layer""#), "`type` isn't first"); + for bytes in [file.clone(), gzip(&file)] { + let back = load(&bytes).unwrap(); + assert_eq!(back.layers, load(&plain).unwrap().layers); + assert_eq!(back.extra["aaa_from_a_newer_version"]["x"], serde_json::json!([1, 2])); + assert_eq!(back.extra["zzz_too"], "kept"); + // Saved again, they are kept. + assert_eq!(load(&save_with(&back, &SaveOptions::default()).unwrap()).unwrap().extra["zzz_too"], "kept"); + } +} diff --git a/crates/ui-egui/src/background.rs b/crates/ui-egui/src/background.rs index a1940a7b..f0245387 100644 --- a/crates/ui-egui/src/background.rs +++ b/crates/ui-egui/src/background.rs @@ -16,6 +16,11 @@ use crate::VectorcraftApp; /// Writes a file; callable from any thread. pub type SharedWriteFn = std::sync::Arc Result<(), String> + Send + Sync>; +/// Writes the file at a path with what the filler it is given writes into it (atomically, as +/// `vectorcraft_format::write_atomic_with`); callable from any thread. +pub type SharedStreamFn = + std::sync::Arc std::io::Result<()>) -> Result<(), String> + Send + Sync>; + /// A writer for the work: the host's file writer (or browser download) on the UI thread, the /// shared one on the worker. pub type Writer<'a> = &'a mut dyn FnMut(&str, &[u8]) -> Result<(), String>; diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 60c056bd..3b5b6947 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -578,10 +578,12 @@ pub fn status_bar(app: &mut VectorcraftApp, ui: &mut Ui) { egui::RichText::new(format!("X: {} Y: {}", u.readout(p.x), u.readout(p.y))).font(theme::mono(10.5)).color(t.text_dim), ); } - // Background saves and exports in progress, else the last message. - if let Some(job) = app.background.jobs.first() { - let more = app.background.jobs.len() - 1; - let label = crate::i18n::msg(&job.label); + // Background saves, exports and opens in progress, else the last message. + let busy: Vec<&str> = + app.background.jobs.iter().map(|j| j.label.as_str()).chain(app.opening.pending.iter().map(|p| p.label.as_str())).collect(); + if let Some(job) = busy.first() { + let more = busy.len() - 1; + let label = crate::i18n::msg(job); let text = if more > 0 { format!("{label}… (+{more})") } else { format!("{label}…") }; ui.add(egui::Spinner::new().size(12.0).color(t.accent)); ui.add(egui::Label::new(egui::RichText::new(text).size(11.0).color(t.text)).truncate()); diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index d10c6bcf..4d223143 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -104,6 +104,8 @@ pub fn inspect(app: &VectorcraftApp, ctx: &egui::Context) -> Value { "nativeMenuBar": app.services.native_menu.is_some(), // Saves and exports still being written in the background (Background Save / Export). "background": app.background.jobs.iter().map(|j| j.label.as_str()).collect::>(), + // Documents still being read off the UI thread (File › Open, Open Recent): their paths. + "opening": app.opening.pending.iter().map(|p| p.path.as_str()).collect::>(), // A native file dialog open off the UI thread (`open`, `place`, `save`, `folder`), waiting // for the person at the computer: what asked for it runs again when it answers. "fileDialog": app.picks.waiting_kind(), diff --git a/crates/ui-egui/src/i18n/cs.tsv b/crates/ui-egui/src/i18n/cs.tsv index 8d698932..17fa3e96 100644 --- a/crates/ui-egui/src/i18n/cs.tsv +++ b/crates/ui-egui/src/i18n/cs.tsv @@ -4493,3 +4493,6 @@ diagnostics ui rozhraní @plural {n} commit|{n} commits {n} commit|{n} commity|{n} commitů gradient-kind Radial Kruhový mark-style Japanese Japonské +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) komprimovaný soubor se rozbalí na více, než VectorCraft přečte ({_1} MB dokumentu, {_2} MB na obrázek) +@msg the document is too large to save compressed ({0}): save it with Use Compression off dokument je příliš velký pro komprimované uložení ({0}): uložte ho s vypnutou volbou Použít kompresi +@msg the file stopped being read (please report this bug) čtení souboru se přerušilo (nahlaste prosím tuto chybu) diff --git a/crates/ui-egui/src/i18n/de.tsv b/crates/ui-egui/src/i18n/de.tsv index ffa02707..e0aad1bf 100644 --- a/crates/ui-egui/src/i18n/de.tsv +++ b/crates/ui-egui/src/i18n/de.tsv @@ -4144,3 +4144,6 @@ theme Appearance Erscheinungsbild Film Grain… Filmkörnung… Grain Körnung Highlight Area Lichterbereich +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) die komprimierte Datei entpackt sich auf mehr, als VectorCraft liest ({_1} MB Dokument, {_2} MB pro Bild) +@msg the document is too large to save compressed ({0}): save it with Use Compression off das Dokument ist zu groß, um komprimiert gespeichert zu werden ({0}): speichern Sie es mit deaktivierter Option „Komprimierung verwenden“ +@msg the file stopped being read (please report this bug) das Lesen der Datei wurde abgebrochen (bitte melden Sie diesen Fehler) diff --git a/crates/ui-egui/src/i18n/es.tsv b/crates/ui-egui/src/i18n/es.tsv index 52683d4d..7399ca1d 100644 --- a/crates/ui-egui/src/i18n/es.tsv +++ b/crates/ui-egui/src/i18n/es.tsv @@ -4187,3 +4187,6 @@ lighting Light Luz Artistic Artístico Film Grain… Granulado de película… Highlight Area Área de iluminaciones +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) el archivo comprimido ocupa al descomprimirse más de lo que VectorCraft lee ({_1} MB de documento, {_2} MB por imagen) +@msg the document is too large to save compressed ({0}): save it with Use Compression off el documento es demasiado grande para guardarlo comprimido ({0}): guárdelo con Usar compresión desactivado +@msg the file stopped being read (please report this bug) se dejó de leer el archivo (informe de este error) diff --git a/crates/ui-egui/src/i18n/fr.tsv b/crates/ui-egui/src/i18n/fr.tsv index ce3fc5da..2dd06234 100644 --- a/crates/ui-egui/src/i18n/fr.tsv +++ b/crates/ui-egui/src/i18n/fr.tsv @@ -4197,3 +4197,6 @@ lighting Light Lumière Artistic Artistique Film Grain… Grain photographique… Highlight Area Zone des tons clairs +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) le fichier compressé dépasse une fois décompressé ce que VectorCraft lit ({_1} Mo de document, {_2} Mo par image) +@msg the document is too large to save compressed ({0}): save it with Use Compression off le document est trop volumineux pour être enregistré compressé ({0}) : enregistrez-le avec Utiliser la compression désactivé +@msg the file stopped being read (please report this bug) la lecture du fichier s’est interrompue (veuillez signaler ce bogue) diff --git a/crates/ui-egui/src/i18n/hu.tsv b/crates/ui-egui/src/i18n/hu.tsv index 594a70e1..b4597c0a 100644 --- a/crates/ui-egui/src/i18n/hu.tsv +++ b/crates/ui-egui/src/i18n/hu.tsv @@ -4587,3 +4587,6 @@ theme Appearance Kinézet @msg {path}: {e} {path}: {e} @msg {status} ({e}) {status} ({e}) @msg {tool} isn't in a toolbar group of several tools az eszköz nincs többeszközös eszköztár-csoportban: {tool} +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) a tömörített fájl kicsomagolva nagyobb lenne, mint amennyit a VectorCraft beolvas ({_1} MB dokumentum, képenként {_2} MB) +@msg the document is too large to save compressed ({0}): save it with Use Compression off a dokumentum túl nagy a tömörített mentéshez ({0}): mentse a Tömörítés használata beállítás kikapcsolásával +@msg the file stopped being read (please report this bug) a fájl beolvasása megszakadt (kérjük, jelentse ezt a hibát) diff --git a/crates/ui-egui/src/i18n/it.tsv b/crates/ui-egui/src/i18n/it.tsv index d234ac57..dbcbaf32 100644 --- a/crates/ui-egui/src/i18n/it.tsv +++ b/crates/ui-egui/src/i18n/it.tsv @@ -4189,3 +4189,6 @@ lighting Light Luce Artistic Artistico Film Grain… Grana della pellicola… Highlight Area Area luci +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) il file compresso, una volta decompresso, supera ciò che VectorCraft legge ({_1} MB di documento, {_2} MB per immagine) +@msg the document is too large to save compressed ({0}): save it with Use Compression off il documento è troppo grande per essere salvato compresso ({0}): salvalo con Usa compressione disattivato +@msg the file stopped being read (please report this bug) la lettura del file si è interrotta (segnala questo errore) diff --git a/crates/ui-egui/src/i18n/ja.tsv b/crates/ui-egui/src/i18n/ja.tsv index 534dd4ec..b3080a7f 100644 --- a/crates/ui-egui/src/i18n/ja.tsv +++ b/crates/ui-egui/src/i18n/ja.tsv @@ -4183,3 +4183,6 @@ lighting Light ライト Artistic アーティスティック Film Grain… フィルム粒子… Highlight Area ハイライト領域 +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) 圧縮ファイルを展開すると VectorCraft が読み込める量を超えます (ドキュメント {_1} MB、画像 1 枚あたり {_2} MB) +@msg the document is too large to save compressed ({0}): save it with Use Compression off ドキュメントが大きすぎて圧縮保存できません ({0}): 「圧縮を使用」をオフにして保存してください +@msg the file stopped being read (please report this bug) ファイルの読み込みが中断されました (このバグを報告してください) diff --git a/crates/ui-egui/src/i18n/pt-br.tsv b/crates/ui-egui/src/i18n/pt-br.tsv index 9f8a97af..3b96c208 100644 --- a/crates/ui-egui/src/i18n/pt-br.tsv +++ b/crates/ui-egui/src/i18n/pt-br.tsv @@ -3672,3 +3672,6 @@ axis Both Ambos Film Grain… Granulação de filme… Grain Grão Highlight Area Área de realces +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) o arquivo compactado se expande para mais do que o VectorCraft lê ({_1} MB de documento, {_2} MB por imagem) +@msg the document is too large to save compressed ({0}): save it with Use Compression off o documento é grande demais para ser salvo compactado ({0}): salve-o com Usar Compressão desativado +@msg the file stopped being read (please report this bug) a leitura do arquivo foi interrompida (informe este erro) diff --git a/crates/ui-egui/src/i18n/ru.tsv b/crates/ui-egui/src/i18n/ru.tsv index 48922b74..40c12825 100644 --- a/crates/ui-egui/src/i18n/ru.tsv +++ b/crates/ui-egui/src/i18n/ru.tsv @@ -4185,3 +4185,6 @@ lighting Light Свет Artistic Художественные Film Grain… Зернистость плёнки… Highlight Area Область светов +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) сжатый файл распаковывается в объём больше, чем читает VectorCraft ({_1} МБ документа, {_2} МБ на изображение) +@msg the document is too large to save compressed ({0}): save it with Use Compression off документ слишком велик для сохранения со сжатием ({0}): сохраните его с выключенным параметром «Использовать сжатие» +@msg the file stopped being read (please report this bug) чтение файла прервалось (пожалуйста, сообщите об этой ошибке) diff --git a/crates/ui-egui/src/i18n/uk.tsv b/crates/ui-egui/src/i18n/uk.tsv index c37ee4fd..e55e57db 100644 --- a/crates/ui-egui/src/i18n/uk.tsv +++ b/crates/ui-egui/src/i18n/uk.tsv @@ -4169,3 +4169,6 @@ lighting Light Світло Artistic Художні Film Grain… Зернистість плівки… Highlight Area Область світлих тонів +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) стислий файл розпаковується в обсяг більший, ніж читає VectorCraft ({_1} МБ документа, {_2} МБ на зображення) +@msg the document is too large to save compressed ({0}): save it with Use Compression off документ завеликий для збереження зі стиском ({0}): збережіть його з вимкненим параметром «Використовувати стиск» +@msg the file stopped being read (please report this bug) читання файлу перервалося (будь ласка, повідомте про цю помилку) diff --git a/crates/ui-egui/src/io.rs b/crates/ui-egui/src/io.rs index 21f7f373..331e96a6 100644 --- a/crates/ui-egui/src/io.rs +++ b/crates/ui-egui/src/io.rs @@ -104,12 +104,18 @@ pub fn opens_as(name: &str) -> Option { /// `document.open`'s result. pub fn open_document(app: &mut VectorcraftApp, name: &str, bytes: &[u8], path: Option, p: &Value) -> Result { let r = fileio::open_bytes_with(&mut app.session, name, bytes, path, p).map_err(|e| e.to_string())?; + opened(app, &r); + Ok(r) +} + +/// A document just opened (`r`: `document.open`'s result): the views follow, the status bar says +/// what didn't come in as it was, and missing links are asked about. +pub(crate) fn opened(app: &mut VectorcraftApp, r: &Value) { app.sync_views(); if let Some(w) = r["warnings"].as_array().filter(|w| !w.is_empty()) { app.status(format!("Opened with {} note(s): {}", w.len(), w[0].as_str().unwrap_or_default())); } - crate::dialogs::missing_links::after_open(app, &r); - Ok(r) + crate::dialogs::missing_links::after_open(app, r); } /// A path from the open dialog, or "cancelled". @@ -148,9 +154,12 @@ pub fn report_open_error(app: &mut VectorcraftApp, name: &str, e: &str) { dialogs::confirm::tell(app, &message, &crate::i18n::message(crate::i18n::current(), e)); } -/// [`open_path`] for a file the user asked for, a failure reported ([`report_open_error`]). +/// [`open_path`] for a file the user asked for, a failure reported ([`report_open_error`]). On +/// the desktop a document is decoded off the UI thread ([`crate::opening`]): the result is then +/// `{background: true, status}`, and the document opens in its tab (or the failure is reported) +/// once it is ready. pub fn open_reporting(app: &mut VectorcraftApp, path: &str) -> Result { - let r = open_path(app, path); + let r = open_file(app, path, crate::opening::available()); if let Err(e) = &r { report_open_error(app, &fileio::file_name(path), e); } @@ -163,7 +172,27 @@ fn read(app: &VectorcraftApp, path: &str) -> Result, String> { /// Open the file at `path` (see [`open_bytes`]) → `document.open`'s result, or null. pub fn open_path(app: &mut VectorcraftApp, path: &str) -> Result { - let bytes = read(app, path)?; + open_file(app, path, false) +} + +/// [`open_path`]; with `background`, a document (not a file that installs, imports or asks +/// first) is decoded off the UI thread ([`crate::opening::start`]). +fn open_file(app: &mut VectorcraftApp, path: &str, background: bool) -> Result { + let mut bytes = read(app, path)?; + if background && opens_as(path).is_none() { + // A PDF with several pages or a password, or a DXF drawing, asks first, as at once. + if crate::dialogs::import_pdf::offer(app, path, &bytes, Some(path.to_string()), None) + || crate::dialogs::dxf_import::offer(app, path, &bytes, Some(path.to_string()), None) + { + note_recent(app, path); + return Ok(Value::Null); + } + match crate::opening::start(app, path, bytes) { + Ok(r) => return Ok(r), + // No thread: at once. + Err(back) => bytes = back, + } + } let r = open_bytes(app, path, &bytes, Some(path.to_string()))?; note_recent(app, path); Ok(r) @@ -459,7 +488,15 @@ fn write_plan(app: &mut VectorcraftApp, plan: SavePlan) -> Result let enabled = app.session.prefs.background_save && target.is_some(); // The work writes the file; the copy makes it the document's own afterwards. let done = job.clone(); - let work = move |write: Writer| job.write(|p, b| write(p, b).map_err(EngineError::Other)).map_err(|e| e.to_string()); + // A native file streams into the file as it is encoded (desktop; not to a download). + let stream = app.services.stream_shared.clone().filter(|_| job.streams() && !is_web(app)); + let work = move |write: Writer| { + match stream { + Some(stream) => job.write_streamed(&mut |p, fill| stream(p, fill).map_err(EngineError::Other)), + None => job.write(|p, b| write(p, b).map_err(EngineError::Other)), + } + .map_err(|e| e.to_string()) + }; let then = move |app: &mut VectorcraftApp, r: Result| { let r = r?; // The document may have closed meanwhile: the file is written all the same. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f2a5cbc0..c69442d2 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -36,6 +36,7 @@ pub mod icons; pub mod io; pub mod menus; pub mod native_menu; +pub mod opening; pub mod palette; mod panel_docking; pub mod panels; @@ -231,6 +232,10 @@ pub struct Services { /// Write a file from any thread (desktop): lets Background Save and Export write off the UI /// thread ([`background`]). Without it they run at once. pub write_shared: Option, + /// Write a file from any thread with what a filler writes into it (desktop): native saves + /// stream into the file as they are encoded instead of being held whole in memory first. + /// Without it they are encoded, then written with `write` / `write_shared`. + pub stream_shared: Option, /// Open a file (or a folder) in the system's default app for it (desktop: Edit Original, Show /// Package). Without it (the web) those answer with an error. pub open_file: Option, @@ -386,6 +391,8 @@ pub struct VectorcraftApp { pub(crate) paste_chord: shortcuts::PasteChord, /// Saves and exports running in the background (Preferences → File Handling). pub background: background::Background, + /// Documents being read off the UI thread ([`opening`]). + pub opening: opening::Opening, /// Data Recovery's timer and startup question ([`recovery`]). pub recovery: recovery::Timer, /// Modifiers the keyboard holds (from the host's input), given back after synthetic input. @@ -482,6 +489,7 @@ impl VectorcraftApp { font_check: None, paste_chord: Default::default(), background: Default::default(), + opening: Default::default(), recovery: Default::default(), host_modifiers: Default::default(), synthetic_modifiers: false, @@ -516,6 +524,11 @@ impl VectorcraftApp { if let Some(write) = s.write_shared.take() { s.write_shared = Some(Arc::new(move |p: &str, b: &[u8]| file_access::check_write(p).and_then(|()| write(p, b)))); } + if let Some(stream) = s.stream_shared.take() { + s.stream_shared = Some(Arc::new(move |p: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>| { + file_access::check_write(p).and_then(|()| stream(p, fill)) + })); + } // Edit Original and Show in Folder hand a file to another app: only one automation may read. for service in [&mut s.open_file, &mut s.reveal] { if let Some(mut open) = service.take() { @@ -982,9 +995,10 @@ impl VectorcraftApp { self.poll_font_check(ctx); picks::poll(self, ctx); background::poll(self); + opening::poll(self); #[cfg(not(target_arch = "wasm32"))] self.tick_link_updates(ctx, now); - if !self.background.jobs.is_empty() { + if !self.background.jobs.is_empty() || !self.opening.pending.is_empty() { // Keep the status bar's progress moving and pick the result up when it arrives. ctx.request_repaint_after(std::time::Duration::from_millis(100)); } diff --git a/crates/ui-egui/src/opening.rs b/crates/ui-egui/src/opening.rs new file mode 100644 index 00000000..b978d11e --- /dev/null +++ b/crates/ui-egui/src/opening.rs @@ -0,0 +1,130 @@ +//! Opening documents off the UI thread (desktop): a file the user asks to open (File › Open, Open +//! Recent, the Home screen, files handed to the app) is decoded on a thread of its own while the +//! status bar shows it, and opens in a new tab once it is ready, as it would at once: the same +//! notes, recent files and errors. On wasm (no threads) and in unit tests it opens at once +//! ([`available`]). + +use std::sync::Arc; +use std::sync::mpsc::{Receiver, TryRecvError, channel}; + +use serde_json::{Value, json}; +use vectorcraft_engine::cmd::fileio::{self, ReadFile}; +use vectorcraft_engine::file_access::ScopeHandle; + +use crate::VectorcraftApp; + +/// A file being read. +pub struct Pending { + /// Where it is. + pub path: String, + /// What the status bar shows ("Opening Poster.vectorcraft"). + pub label: String, + done: Receiver>, +} + +/// The files being read. +#[derive(Default)] +pub struct Opening { + pub pending: Vec, +} + +/// Does opening happen off the UI thread here? Not on wasm (no threads), in unit tests (they look +/// at the document right after asking for it), or with `VECTORCRAFT_SYNC_OPEN` set. +pub fn available() -> bool { + !(cfg!(target_arch = "wasm32") || cfg!(test) || std::env::var_os("VECTORCRAFT_SYNC_OPEN").is_some()) +} + +/// Decode `bytes`, the file at `path`, on a thread of its own → `{background: true, status}`, or +/// the bytes back when no thread could start (the caller opens the file at once). +pub fn start(app: &mut VectorcraftApp, path: &str, bytes: Vec) -> Result> { + let (tx, done) = channel(); + let bytes = Arc::new(bytes); + let (name, file) = (path.to_string(), bytes.clone()); + // The automation roots in force here are in force there. + let scope = ScopeHandle::capture(); + let spawned = std::thread::Builder::new().name("vectorcraft-open".into()).spawn(move || { + let r = scope + .run(|| vectorcraft_engine::guard::catch_panic(|| fileio::read_for_open(&name, &file, &Value::Null).map_err(|e| e.to_string()))) + .unwrap_or_else(|msg| Err(format!("internal error: {msg} (please report this bug)"))); + // The file's bytes go before the document is handed over. + drop(file); + // The app may have quit meanwhile: nothing to hand it to. + let _ = tx.send(r); + }); + if spawned.is_err() { + return Err(Arc::try_unwrap(bytes).unwrap_or_else(|b| b.to_vec())); + } + let label = format!("Opening {}", fileio::file_name(path)); + app.opening.pending.push(Pending { path: path.to_string(), label: label.clone(), done }); + let status = format!("{label}…"); + app.status(status.clone()); + Ok(json!({ "background": true, "status": status })) +} + +/// Open the files that are ready (each frame). +pub fn poll(app: &mut VectorcraftApp) { + let mut i = 0; + while let Some(p) = app.opening.pending.get(i) { + let r = match p.done.try_recv() { + Ok(r) => r, + Err(TryRecvError::Empty) => { + i += 1; + continue; + } + Err(TryRecvError::Disconnected) => Err("the file stopped being read (please report this bug)".to_string()), + }; + let p = app.opening.pending.remove(i); + finish(app, &p.path, r); + } +} + +/// Open a file read off the UI thread, as [`crate::io::open_reporting`] opens one read at once. +fn finish(app: &mut VectorcraftApp, path: &str, r: Result) { + let opened = r.and_then(|file| fileio::open_read(&mut app.session, file, Some(path.to_string())).map_err(|e| e.to_string())); + match opened { + Ok(r) => { + crate::io::opened(app, &r); + crate::io::note_recent(app, path); + } + Err(e) => crate::io::report_open_error(app, &fileio::file_name(path), &e), + } +} + +#[cfg(test)] +mod tests { + use vectorcraft_engine::Session; + + use super::*; + use crate::Services; + + #[test] + fn a_file_read_off_the_ui_thread_opens_as_one_read_at_once() { + let mut app = VectorcraftApp::new(Session::new(), Services::default()); + let mut d = vectorcraft_doc::Document::new(120.0, 80.0); + d.title = "Poster".into(); + let bytes = vectorcraft_format::save(&d, false); + let before = app.session.documents().len(); + let r = start(&mut app, "/art/Poster.vectorcraft", bytes).unwrap(); + assert_eq!(r["background"], true); + assert_eq!(app.ui.status, "Opening Poster.vectorcraft…"); + let t0 = std::time::Instant::now(); + while !app.opening.pending.is_empty() && t0.elapsed().as_secs() < 30 { + poll(&mut app); + std::thread::yield_now(); + } + assert_eq!(app.session.documents().len(), before + 1); + let st = app.session.doc().unwrap(); + assert_eq!((st.path.as_deref(), st.doc.artboards[0].rect.width()), (Some("/art/Poster.vectorcraft"), 120.0)); + assert_eq!(app.ui.recent_files.first().map(String::as_str), Some("/art/Poster.vectorcraft")); + // A file that can't be read: reported, nothing opened, not a recent file. + start(&mut app, "/art/Broken.vectorcraft", b"{\"format\":\"vectorcraft\"".to_vec()).unwrap(); + let t0 = std::time::Instant::now(); + while !app.opening.pending.is_empty() && t0.elapsed().as_secs() < 30 { + poll(&mut app); + std::thread::yield_now(); + } + assert_eq!(app.session.documents().len(), before + 1); + assert!(app.ui.status.starts_with("Couldn't open Broken.vectorcraft"), "{}", app.ui.status); + assert_ne!(app.ui.recent_files.first().map(String::as_str), Some("/art/Broken.vectorcraft")); + } +} diff --git a/crates/ui-egui/src/tests_save.rs b/crates/ui-egui/src/tests_save.rs index 8a39c50a..cf200fe5 100644 --- a/crates/ui-egui/src/tests_save.rs +++ b/crates/ui-egui/src/tests_save.rs @@ -205,3 +205,27 @@ fn replacing_a_file_that_opening_left_things_out_of_asks_first() { assert!(!app.session.active().unwrap().is_dirty()); let _ = std::fs::remove_dir_all(d); } + +#[test] +fn native_saves_stream_into_the_file() { + let d = dir("stream"); + let streamed = std::sync::Arc::new(std::sync::Mutex::new(vec![])); + let log = streamed.clone(); + let mut app = app(); + app.services.stream_shared = Some(std::sync::Arc::new(move |p: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>| { + log.lock().unwrap().push(p.to_string()); + let mut f = std::fs::File::create(p).map_err(|e| e.to_string())?; + fill(&mut f).map_err(|e| e.to_string()) + })); + app.run("file.new", json!({"width": 100, "height": 100})).unwrap(); + rect(&mut app); + let native = d.join("a.vectorcraft").to_string_lossy().to_string(); + let r = app.run("file.saveAs", json!({"path": native, "compress": true})).unwrap(); + let bytes = std::fs::read(&native).unwrap(); + assert_eq!(r["bytes"], bytes.len(), "{r}"); + assert!(vectorcraft_format::is_compressed(&bytes)); + assert_eq!(vectorcraft_format::load(&bytes).unwrap().node_count(), objects(&app)); + assert!(!app.session.doc().unwrap().is_dirty(), "the file is the document's own"); + assert_eq!(*streamed.lock().unwrap(), [native]); + let _ = std::fs::remove_dir_all(&d); +}