From 023c8e3edd7a266cae9d6827f85cf453178b8cd1 Mon Sep 17 00:00:00 2001 From: arhammusheer Date: Mon, 12 Oct 2026 02:32:50 +0530 Subject: [PATCH] #1244: streaming open and save, per-section limits, recovery that keeps images apart, background open Save serializes JSON straight into the compressor and writes images from their Arc into the file; open reads each image into its own buffer, and NodeKind reads its tag first instead of buffering every node. Already-compressed images are stored rather than deflated again in the same gzip stream. Limits apply per section, so any file VectorCraft writes opens again. Data Recovery writes each image once, content-addressed, instead of every interval. Native files open on a background thread. Files stay readable by main both ways; JSON-only saves are identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VmMmc1J8XeNLxhRihWv9cc --- apps/vectorcraft/src/main.rs | 4 + crates/doc/src/node.rs | 321 +++++++++++++++++++++- crates/engine/src/cmd/fileio/load.rs | 20 +- crates/engine/src/cmd/fileio/mod.rs | 21 +- crates/engine/src/cmd/fileio/native.rs | 10 +- crates/engine/src/cmd/fileio/save.rs | 92 ++++++- crates/engine/src/cmd/fileio/tests.rs | 30 +++ crates/engine/src/cmd/recovery.rs | 155 ++++++++++- crates/format/src/lib.rs | 312 ++++++++++++--------- crates/format/src/stream.rs | 358 +++++++++++++++++++++++++ crates/format/src/unpack.rs | 214 +++++++++++++++ crates/format/tests/streaming.rs | 199 ++++++++++++++ crates/ui-egui/src/background.rs | 5 + crates/ui-egui/src/chrome.rs | 10 +- crates/ui-egui/src/control.rs | 2 + crates/ui-egui/src/i18n/cs.tsv | 3 + crates/ui-egui/src/i18n/de.tsv | 3 + crates/ui-egui/src/i18n/es.tsv | 3 + crates/ui-egui/src/i18n/fr.tsv | 3 + crates/ui-egui/src/i18n/hu.tsv | 3 + crates/ui-egui/src/i18n/it.tsv | 3 + crates/ui-egui/src/i18n/ja.tsv | 3 + crates/ui-egui/src/i18n/pt-br.tsv | 3 + crates/ui-egui/src/i18n/ru.tsv | 3 + crates/ui-egui/src/i18n/uk.tsv | 3 + crates/ui-egui/src/io.rs | 49 +++- crates/ui-egui/src/lib.rs | 16 +- crates/ui-egui/src/opening.rs | 130 +++++++++ crates/ui-egui/src/tests_save.rs | 24 ++ 29 files changed, 1838 insertions(+), 164 deletions(-) create mode 100644 crates/format/src/stream.rs create mode 100644 crates/format/src/unpack.rs create mode 100644 crates/format/tests/streaming.rs create mode 100644 crates/ui-egui/src/opening.rs diff --git a/apps/vectorcraft/src/main.rs b/apps/vectorcraft/src/main.rs index af4726e4..07a5d94f 100644 --- a/apps/vectorcraft/src/main.rs +++ b/apps/vectorcraft/src/main.rs @@ -501,6 +501,10 @@ fn services(parent: Parent, ui: egui::Context) -> Services { write: Some(Box::new(write_file)), // Background Save and Export write from a worker thread. write_shared: Some(std::sync::Arc::new(write_file)), + // Native saves stream into the file as they are encoded. + stream_shared: Some(std::sync::Arc::new(|p: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>| { + fileio::write_file_with(p, fill).map_err(|e| e.to_string()) + })), // Every format Copy offers and Paste reads (menu-bar Paste never sees egui's Paste event). system_clipboard: Some(clipboard::system_clipboard()), // Linux checks whether Paste has something to take on a background thread: an X11 clipboard diff --git a/crates/doc/src/node.rs b/crates/doc/src/node.rs index 06db3cc8..47a6085c 100644 --- a/crates/doc/src/node.rs +++ b/crates/doc/src/node.rs @@ -394,7 +394,9 @@ pub struct ImageObject { pub placement: crate::links::PlacementOptions, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +/// Written with its `type` first, then its fields (an internally tagged enum); read by +/// [`kind_de`], which takes the fields straight from the file when `type` comes first. +#[derive(Clone, Debug, PartialEq, Serialize)] #[serde(tag = "type", rename_all = "lowercase")] pub enum NodeKind { /// A top-level layer or sublayer. @@ -1634,10 +1636,281 @@ impl Node { } } +/// Reading a [`NodeKind`]. Derived, an internally tagged enum buffers each kind (with every +/// object inside a group, again at each level of nesting) before reading it, since `type` may come +/// anywhere among its fields. Files have it first (it is written first): then the kind is read +/// straight from the file. Otherwise (hand-written files, values whose keys are sorted) the +/// fields before `type` are buffered, and only those. +mod kind_de { + use std::fmt; + use std::sync::Arc; + + use serde::de::value::{MapAccessDeserializer, StringDeserializer}; + use serde::de::{self, DeserializeSeed, Deserializer, EnumAccess, IntoDeserializer, MapAccess, VariantAccess, Visitor}; + use serde::{Deserialize, forward_to_deserialize_any}; + use vectorcraft_geom::{Affine, FillRule, PathData}; + + use super::{LayerColor, LiveShape, Node, NodeKind, yes}; + use crate::live::{BlendSpec, EnvelopeKind, GradientMesh}; + use crate::pattern::RepeatSpec; + use crate::text::TextObject; + + /// The kinds and their fields as [`NodeKind`] has them, with the defaults its field attributes + /// note (the compiler checks that every field is here), read externally tagged: the kind from + /// `type`, its fields from the rest of the object ([`Tagged`]). + #[derive(Deserialize)] + #[serde(remote = "NodeKind", rename_all = "lowercase")] + enum KindDef { + Layer { + color: LayerColor, + #[serde(default)] + template: bool, + #[serde(default = "yes")] + printable: bool, + children: Vec>, + #[serde(default)] + clip: bool, + #[serde(default = "yes")] + preview: bool, + #[serde(default)] + dim_images: Option, + }, + Group { + children: Vec>, + #[serde(default)] + clip: bool, + }, + Path { + path: PathData, + #[serde(default)] + rule: FillRule, + #[serde(default)] + live: Option>, + #[serde(default)] + clipping: bool, + #[serde(default)] + guide: bool, + }, + Compound { + children: Vec>, + #[serde(default)] + rule: FillRule, + }, + Text(Box), + Image(Box), + SymbolInstance { + symbol: String, + xf: Affine, + }, + Blend { + children: Vec>, + #[serde(default)] + spec: Box, + }, + Envelope { + content: Vec>, + kind: Box, + #[serde(default = "crate::live::default_fidelity")] + fidelity: f64, + #[serde(default)] + editing: bool, + #[serde(default)] + options: crate::live::EnvelopeOptions, + #[serde(default)] + frame: Box, + }, + Mesh(GradientMesh), + Repeat(RepeatSpec), + PlacedDocument(Box), + } + + impl<'de> Deserialize<'de> for NodeKind { + fn deserialize>(d: D) -> Result { + d.deserialize_map(KindVisitor) + } + } + + struct KindVisitor; + + impl<'de> Visitor<'de> for KindVisitor { + type Value = NodeKind; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("an object kind (a map with `type`)") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut before = vec![]; + while let Some(key) = map.next_key::()? { + if key == "type" { + let tag: String = map.next_value()?; + return KindDef::deserialize(Tagged { tag, map: Buffered { before: before.into_iter(), value: None, map } }); + } + before.push((key, map.next_value::()?)); + } + Err(de::Error::missing_field("type")) + } + } + + /// The fields read before `type` (buffered), then the rest of the object. + struct Buffered { + before: std::vec::IntoIter<(String, serde_json::Value)>, + value: Option, + map: A, + } + + impl<'de, A: MapAccess<'de>> MapAccess<'de> for Buffered { + type Error = A::Error; + + fn next_key_seed>(&mut self, seed: K) -> Result, A::Error> { + match self.before.next() { + Some((key, value)) => { + self.value = Some(value); + seed.deserialize(StringDeserializer::::new(key)).map(Some) + } + None => self.map.next_key_seed(seed), + } + } + + fn next_value_seed>(&mut self, seed: V) -> Result { + match self.value.take() { + Some(value) => seed.deserialize(value).map_err(de::Error::custom), + None => self.map.next_value_seed(seed), + } + } + } + + /// An object whose `type` was read: the kind is `tag`, its fields are the rest of `map`. + struct Tagged { + tag: String, + map: A, + } + + impl<'de, A: MapAccess<'de>> Deserializer<'de> for Tagged { + type Error = A::Error; + + fn deserialize_any>(self, visitor: V) -> Result { + visitor.visit_enum(self) + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string bytes byte_buf + option unit unit_struct newtype_struct seq tuple tuple_struct map struct enum + identifier ignored_any + } + } + + impl<'de, A: MapAccess<'de>> EnumAccess<'de> for Tagged { + type Error = A::Error; + type Variant = Fields; + + fn variant_seed>(self, seed: V) -> Result<(V::Value, Fields), A::Error> { + let kind = seed.deserialize(self.tag.into_deserializer())?; + Ok((kind, Fields(self.map))) + } + } + + /// A kind's fields: the rest of the object. + struct Fields(A); + + impl<'de, A: MapAccess<'de>> VariantAccess<'de> for Fields { + type Error = A::Error; + + fn unit_variant(self) -> Result<(), A::Error> { + Err(de::Error::invalid_type(de::Unexpected::UnitVariant, &"an object kind with fields")) + } + + fn newtype_variant_seed>(self, seed: T) -> Result { + seed.deserialize(MapAccessDeserializer::new(self.0)) + } + + fn tuple_variant>(self, _: usize, _: V) -> Result { + Err(de::Error::invalid_type(de::Unexpected::TupleVariant, &"an object kind with fields")) + } + + fn struct_variant>(self, _: &'static [&'static str], visitor: V) -> Result { + visitor.visit_map(self.0) + } + } +} + #[cfg(test)] mod tests { use super::*; + /// A node writes its rarely set parts among the others, in the file's field order, and reads + /// them back into its extras; a node without any keeps none. + #[test] + fn extras_are_written_flat_in_order() { + let mut n = Node::path(NodeId(7), shapes::rectangle(Rect::new(0.0, 0.0, 4.0, 2.0)), Appearance::default_art()); + n.name = Some("n".into()); + n.knockout_shape = true; + n.bbox_angle = 30.0; + { + let mut e = n.extras_mut(); + e.mask = Some(Box::new(OpacityMask::new(Node::group(NodeId(8), vec![]), true))); + e.trace = Some(Box::new(serde_json::json!({"preset": "p"}))); + e.series_index = Some(2); + e.graphic_style = Some(3); + e.attrs = Some(Box::default()); + e.slice = Some(Box::default()); + e.perspective = Some(Box::new(crate::PerspectiveAttachment::new("left", 1.0))); + } + /// The top-level keys of a JSON object, in the order written. + struct Keys(Vec); + impl<'de> Deserialize<'de> for Keys { + fn deserialize>(d: D) -> Result { + struct V; + impl<'de> serde::de::Visitor<'de> for V { + type Value = Keys; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("an object") + } + fn visit_map>(self, mut a: A) -> Result { + let mut out = vec![]; + while let Some(k) = a.next_key::()? { + a.next_value::()?; + out.push(k); + } + Ok(Keys(out)) + } + } + d.deserialize_map(V) + } + } + let text = serde_json::to_string(&n).unwrap(); + let keys = serde_json::from_str::(&text).unwrap().0; + let v: serde_json::Value = serde_json::from_str(&text).unwrap(); + let order = [ + "id", + "name", + "knockout_shape", + "appearance", + "mask", + "trace", + "series_index", + "kind", + "graphic_style", + "attrs", + "slice", + "bbox_angle", + "perspective", + ]; + assert_eq!(keys, order); + assert_eq!(serde_json::from_value::(v).unwrap(), n); + let plain = Node::path(NodeId(9), shapes::rectangle(Rect::new(0.0, 0.0, 4.0, 2.0)), Appearance::default_art()); + let back: Node = serde_json::from_value(serde_json::to_value(&plain).unwrap()).unwrap(); + assert!(back.extras_box().is_none()); + // Clearing the last extra lets them go; clearing one a node doesn't have allocates none. + n.extras_mut().mask = None; + assert!(n.extras_box().is_some()); + *n.extras_mut() = NodeExtras::default(); + assert!(n.extras_box().is_none()); + n.extras_mut().mask = None; + assert!(n.extras_box().is_none()); + assert_eq!(n.extras(), &NodeExtras::default()); + } + #[test] fn rect_live_shape_roundtrip() { let l = LiveShape::Rectangle { w: 10.0, h: 20.0, radii: [0.0; 4], kinds: Default::default(), xf: Affine::translate((5.0, 5.0)) }; @@ -1901,4 +2174,50 @@ mod tests { p.appearance.stroke_mut().unwrap().join = crate::appearance::LineJoin::Round; assert_eq!(p.visual_bounds(), Some(Rect::new(-0.5, -0.5, 10.5, 10.5))); } + + /// A group holding a path, a compound, an image and a nested group, as a file writes it. + fn nested_group() -> Node { + let path = |id| Node::path(NodeId(id), shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0)), Appearance::default_art()); + let image = ImageObject { key: "k".into(), width: 2, height: 3, xf: Affine::IDENTITY, link: None, placement: Default::default() }; + let inner = Node::new(NodeId(5), NodeKind::Group { children: vec![Arc::new(path(6))], clip: true }); + let compound = Node::new(NodeId(3), NodeKind::Compound { children: vec![Arc::new(path(4))], rule: FillRule::EvenOdd }); + let kids = vec![Arc::new(path(2)), Arc::new(compound), Arc::new(Node::new(NodeId(7), NodeKind::Image(Box::new(image)))), Arc::new(inner)]; + Node::new(NodeId(1), NodeKind::Group { children: kids, clip: false }) + } + + #[test] + fn kinds_read_back_with_type_first_or_anywhere() { + let n = nested_group(); + let json = serde_json::to_string(&n).unwrap(); + assert!(json.contains(r#""kind":{"type":"group","children":["#), "written with `type` first: {json}"); + assert_eq!(serde_json::from_str::(&json).unwrap(), n); + // Through a JSON value, whose keys are sorted: `type` comes after `children`. + let v = serde_json::to_value(&n).unwrap(); + assert_eq!(serde_json::from_value::(v.clone()).unwrap(), n); + assert_eq!(serde_json::from_str::(&v.to_string()).unwrap(), n); + // `type` last, in the middle, and keys a newer version added (ignored, as before). + let kind = r#"{"children":[],"x":{"deep":[1,2]},"clip":true,"type":"group"}"#; + assert_eq!(serde_json::from_str::(kind).unwrap(), NodeKind::Group { children: vec![], clip: true }); + let kind = r#"{"symbol":"S","type":"symbolinstance","later":1,"xf":[1,0,0,1,5,6]}"#; + assert_eq!( + serde_json::from_str::(kind).unwrap(), + NodeKind::SymbolInstance { symbol: "S".into(), xf: Affine::translate((5.0, 6.0)) } + ); + let kind = r#"{"width":2,"type":"image","key":"k","height":3,"xf":[1,0,0,1,0,0]}"#; + assert!(matches!(serde_json::from_str::(kind).unwrap(), NodeKind::Image(i) if i.key == "k" && i.height == 3)); + // Defaults as before. + let NodeKind::Layer { printable, preview, template, .. } = + serde_json::from_str(r#"{"type":"layer","color":{"Preset":0},"children":[]}"#).unwrap() + else { + panic!("a layer") + }; + assert!(printable && preview && !template); + } + + #[test] + fn bad_kinds_are_errors() { + for bad in [r#"{"children":[]}"#, r#"{"type":"nothing"}"#, r#"{"type":"group"}"#, r#"{"type":3,"children":[]}"#, r#"[]"#, r#""group""#] { + assert!(serde_json::from_str::(bad).is_err(), "{bad}"); + } + } } diff --git a/crates/engine/src/cmd/fileio/load.rs b/crates/engine/src/cmd/fileio/load.rs index 534d86c8..6d44effc 100644 --- a/crates/engine/src/cmd/fileio/load.rs +++ b/crates/engine/src/cmd/fileio/load.rs @@ -283,8 +283,26 @@ pub fn open_bytes(s: &mut Session, name: &str, bytes: &[u8], path: Option, p: &Value) -> Result { + open_read(s, read_for_open(name, bytes, p)?, path) +} + +/// A file read for `document.open` but not opened yet ([`read_for_open`], [`open_read`]). +pub struct ReadFile { + loaded: Loaded, + opts: LoadOptions, +} + +/// The first half of [`open_bytes_with`]: read and decode the file, which needs no session (an +/// app may do it on another thread) → what [`open_read`] opens. +pub fn read_for_open(name: &str, bytes: &[u8], p: &Value) -> Result { let opts = LoadOptions::from_params("document.open", p)?; - open_loaded(s, load_with(name, bytes, &opts)?, path, &opts, false) + Ok(ReadFile { loaded: load_with(name, bytes, &opts)?, opts }) +} + +/// The second half of [`open_bytes_with`]: open a file [`read_for_open`] read as the new active +/// document (`path`: where it was read from) → `document.open`'s result. +pub fn open_read(s: &mut Session, file: ReadFile, path: Option) -> Result { + open_loaded(s, file.loaded, path, &file.opts, false) } /// File → New from Template: any readable file (at `path`, if it is one: its links are looked for diff --git a/crates/engine/src/cmd/fileio/mod.rs b/crates/engine/src/cmd/fileio/mod.rs index bfa892dd..8d53dd3c 100644 --- a/crates/engine/src/cmd/fileio/mod.rs +++ b/crates/engine/src/cmd/fileio/mod.rs @@ -53,15 +53,18 @@ use load::err; pub(crate) use load::import_svg; pub(crate) use load::native_file; pub(crate) use load::source; -pub use load::{Loaded, RasterImage, detect, file_name, load, load_with, open_bytes, open_bytes_with, open_template, raster_image}; +pub use load::{ + Loaded, RasterImage, ReadFile, detect, file_name, load, load_with, open_bytes, open_bytes_with, open_read, open_template, raster_image, + read_for_open, +}; pub use load::{losses_summary, overwrite_losses}; pub(crate) use native::preview_png; pub use native::with_compression_pref; pub use pdfimport::{LoadOptions, page_document}; pub(crate) use save::job_for; pub use save::{ - SAVE_FORMATS, SaveJob, SaveMode, SavePlan, export_folder, save_filters, save_format, save_job, save_plan, save_with, stamp_save_dates, - templates_dialog_folder, templates_folder, + SAVE_FORMATS, SaveJob, SaveMode, SavePlan, StreamWrite, export_folder, save_filters, save_format, save_job, save_plan, save_with, + stamp_save_dates, templates_dialog_folder, templates_folder, }; pub use screens::{PRESETS as SCREEN_PRESETS, ScreenSize, preset_rows as screen_preset_rows}; pub(crate) use screens::{ @@ -769,6 +772,13 @@ pub(crate) fn write_file(path: &str, bytes: &[u8]) -> Result<()> { write_atomic(std::path::Path::new(path), bytes).map_err(|e| EngineError::Other(format!("{path}: {e}"))) } +/// [`write_file`] with what `fill` writes into the file (a save streamed into it). +#[cfg(not(target_arch = "wasm32"))] +pub fn write_file_with(path: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>) -> Result<()> { + may_write(path)?; + vectorcraft_format::write_atomic_with(std::path::Path::new(path), |f| fill(f)).map_err(|e| EngineError::Other(format!("{path}: {e}"))) +} + #[cfg(not(target_arch = "wasm32"))] pub(crate) fn create_dir(path: &str) -> Result<()> { may_write(path)?; @@ -938,6 +948,11 @@ pub(crate) fn write_file(path: &str, _: &[u8]) -> Result<()> { Err(no_fs(path)) } +#[cfg(target_arch = "wasm32")] +pub fn write_file_with(path: &str, _: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>) -> Result<()> { + Err(no_fs(path)) +} + #[cfg(target_arch = "wasm32")] pub(crate) fn create_dir(path: &str) -> Result<()> { Err(no_fs(path)) diff --git a/crates/engine/src/cmd/fileio/native.rs b/crates/engine/src/cmd/fileio/native.rs index 289bfcff..83d9ab45 100644 --- a/crates/engine/src/cmd/fileio/native.rs +++ b/crates/engine/src/cmd/fileio/native.rs @@ -135,6 +135,14 @@ pub(super) fn save_options(cmd: &str, f: &Format, doc: &Document, p: &Value) -> /// The `.vectorcraft` file of `doc` with the options in `p`. pub(super) fn encode(cmd: &str, f: &Format, doc: &Document, p: &Value) -> Result> { + let mut out = Vec::new(); + encode_to(cmd, f, doc, p, &mut out)?; + Ok(out) +} + +/// [`encode`], written to `w` as it is made ([`vectorcraft_format::save_to`]: give it a buffered +/// writer). +pub(super) fn encode_to(cmd: &str, f: &Format, doc: &Document, p: &Value, w: impl std::io::Write) -> Result<()> { let (mut so, extras) = save_options(cmd, f, doc, p)?; // Include Linked Files keeps placed documents' files; older versions get their art. let full = if so.include_linked || so.version < vectorcraft_format::VERSION { @@ -158,7 +166,7 @@ pub(super) fn encode(cmd: &str, f: &Format, doc: &Document, p: &Value) -> Result } other => other, }; - vectorcraft_format::save_with(&native, &so).map_err(|e| bad(cmd, e.to_string())) + vectorcraft_format::save_to(&native, &so, w).map_err(|e| bad(cmd, e.to_string())) } /// The ICC files of the profiles `doc` is tagged with that were loaded from files (the built-in diff --git a/crates/engine/src/cmd/fileio/save.rs b/crates/engine/src/cmd/fileio/save.rs index 75a179cf..923f7c80 100644 --- a/crates/engine/src/cmd/fileio/save.rs +++ b/crates/engine/src/cmd/fileio/save.rs @@ -6,6 +6,7 @@ //! the caller's writer (the file system here; a save panel or a browser download in the apps) and //! makes the file the document's own (path, format, options, saved state) for Save and Save As. +use std::io::Write as _; use std::sync::Arc; use serde_json::{Map, Value, json}; @@ -14,7 +15,6 @@ use vectorcraft_doc::Document; use super::super::*; use super::{ ArtboardPick, Encoded, Format, Loaded, encode_all, file_stem, format, format_for_name, load, read_file, with_compression_pref, write_encoded, - write_file, }; use crate::{DocState, Prefs}; @@ -300,6 +300,31 @@ pub struct SaveJob { boards: Vec, } +/// Writes the file at a path with what the filler it is given writes into it +/// ([`SaveJob::write_streamed`]). +pub type StreamWrite<'a> = &'a mut dyn FnMut(&str, &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>) -> Result<()>; + +/// Bytes gathered before a streamed save writes them to the file. +const STREAM_BUFFER: usize = 256 << 10; + +/// A writer that counts the bytes written through it. +struct Counted { + inner: W, + n: u64, +} + +impl std::io::Write for Counted { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let n = self.inner.write(buf)?; + self.n += n as u64; + Ok(n) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + /// The artboards a native or `.ai` save also writes to files of their own: with /// `separateArtboards`, those `range` names (default all). fn separate_boards(cmd: &str, f: &Format, doc: &Document, p: &Value) -> Result> { @@ -443,6 +468,16 @@ impl SaveJob { &self.snapshot } + /// The document the job writes (the snapshot, as the file keeps it). + pub(crate) fn doc(&self) -> &Arc { + &self.doc + } + + /// Encode `doc` (the job's document, changed) as the job encodes its own: one file. + pub(crate) fn encode_doc(&self, doc: &Document) -> Result { + self.encode_one(doc) + } + /// Encode the file → it (and, with `separateArtboards`, a file per artboard after it), with /// the warnings (what the format loses first, then the encoder's own notes: PDF options not /// applied yet…). @@ -500,6 +535,53 @@ impl SaveJob { Ok(out) } + /// Is the file written as it is encoded by [`SaveJob::write_streamed`]? A native file (or + /// template) to a path, without a file per artboard. + pub fn streams(&self) -> bool { + self.plan.path.is_some() && matches!(self.plan.format.id, "vectorcraft" | "template") && self.boards.is_empty() + } + + /// [`SaveJob::write`] through `stream`, which writes the file at a path with what the filler + /// it is given writes (atomically, as [`vectorcraft_format::write_atomic_with`]). A native + /// file goes into it as it is encoded ([`SaveJob::streams`]), never held whole in memory; + /// other files are encoded first, then written into it. + pub fn write_streamed(&self, stream: StreamWrite) -> Result { + let (Some(path), true) = (&self.plan.path, self.streams()) else { + return self.write(|p, b| stream(p, &mut |w| w.write_all(b))); + }; + let (cmd, f) = (self.plan.mode.command(), self.plan.format); + let doc = self.doc.without_edit_modes(); + let doc = match f.id { + // A native file flagged so opening it starts a new untitled document (as + // `encode_all` writes it). + "template" => { + let mut d = doc.into_owned(); + d.template = true; + std::borrow::Cow::Owned(d) + } + _ => doc, + }; + // The encoder's failure, kept apart from the writer's (which `stream` reports). + let mut failed = None; + let mut written = 0u64; + let mut fill = |w: &mut dyn std::io::Write| { + let mut out = Counted { inner: std::io::BufWriter::with_capacity(STREAM_BUFFER, w), n: 0 }; + let r = super::native::encode_to(cmd, f, &doc, &self.params, &mut out); + written = out.n; + match r { + Ok(()) => out.inner.flush(), + Err(e) => { + failed = Some(e); + Err(std::io::Error::other("the file couldn't be encoded")) + } + } + }; + let r = stream(path, &mut fill); + r.map_err(|e| failed.take().unwrap_or(e))?; + let warnings: Vec = fidelity_warning(f).into_iter().collect(); + Ok(json!({ "path": path, "bytes": written, "format": f.id, "warnings": warnings })) + } + /// Once written: Save and Save As (except to a template) make the file `st`'s own: path, /// title, format, options, and the snapshot as the saved state (edits made since the job was /// made keep the document modified). @@ -543,10 +625,14 @@ pub fn save_with(s: &mut Session, plan: SavePlan, write: impl FnMut(&str, &[u8]) Ok(out) } -/// A save command: plan, then write with the file system. +/// A save command: plan, then write with the file system (a native file as it is encoded). fn save(s: &mut Session, mode: SaveMode, p: &Value) -> Result { let plan = save_plan(s, mode, p)?; - save_with(s, plan, write_file) + let job = save_job(s, plan)?; + let uid = s.doc()?.uid; + let out = job.write_streamed(&mut super::write_file_with)?; + job.complete(s, uid); + Ok(out) } fn can_revert(s: &Session) -> std::result::Result<(), String> { diff --git a/crates/engine/src/cmd/fileio/tests.rs b/crates/engine/src/cmd/fileio/tests.rs index d4f6554e..5af1e823 100644 --- a/crates/engine/src/cmd/fileio/tests.rs +++ b/crates/engine/src/cmd/fileio/tests.rs @@ -412,3 +412,33 @@ fn new_files_never_replace_a_file() { assert_eq!(std::fs::read(&path).unwrap(), b"first"); let _ = std::fs::remove_dir_all(d); } + +#[test] +fn a_native_save_streams_the_bytes_the_encoder_makes() { + let d = tmp_dir("stream"); + let mut s = session(200.0, 100.0, 1); + s.execute("shape.rectangle", &json!({"x": 10, "y": 10, "width": 50, "height": 20})).unwrap(); + for (name, compress) in [("a.vectorcraft", false), ("b.vectorcraft", true), ("c.vctemplate", true)] { + let path = d.join(name).to_string_lossy().to_string(); + let p = json!({"path": path, "compress": compress}); + let r = s.execute("file.saveCopy", &p).unwrap(); + let written = std::fs::read(&path).unwrap(); + assert_eq!(r["bytes"], written.len(), "{r}"); + let plan = save_plan(&s, SaveMode::Copy, &p).unwrap(); + let job = save_job(&mut s, plan).unwrap(); + assert!(job.streams()); + assert_eq!(written, job.encode().unwrap().files[0].1, "{name}"); + assert_eq!(vectorcraft_format::load(&written).unwrap().template, name.ends_with("vctemplate")); + } + // A writer that fails: the save fails with its error. + let plan = save_plan(&s, SaveMode::Copy, &json!({"path": "/x/y.vectorcraft"})).unwrap(); + let job = save_job(&mut s, plan).unwrap(); + let e = job + .write_streamed(&mut |_, fill| { + fill(&mut std::io::sink()).unwrap(); + Err(EngineError::Other("disk full".into())) + }) + .unwrap_err(); + assert_eq!(e.to_string(), "disk full"); + let _ = std::fs::remove_dir_all(&d); +} diff --git a/crates/engine/src/cmd/recovery.rs b/crates/engine/src/cmd/recovery.rs index 83623a50..f466355c 100644 --- a/crates/engine/src/cmd/recovery.rs +++ b/crates/engine/src/cmd/recovery.rs @@ -26,17 +26,21 @@ //! again. //! //! Each copy is two entries: `/.vectorcraft` (the native document, written as a save -//! writes it) and `/.json` (its title, original path and format, and when it was -//! written). The store is a folder on the desktop ([`FolderStore`]: the `recoveryFolder` +//! writes it, without its images) and `/.json` (its title, original path and format, +//! when it was written, and its images). The images are entries of their own, `/img-…blob`, +//! named by their content and shared by the area's copies: a copy written again writes only the +//! images that changed, and the images no copy names any more go. Copies written before images +//! were kept apart (the images in the document entry) still restore. The store is a folder on the desktop ([`FolderStore`]: the `recoveryFolder` //! preference, else the folder the app sets), browser storage on the web, or any //! [`RecoveryStore`] the host installs (tests use [`MemoryStore`]). use std::collections::{BTreeMap, BTreeSet}; -use std::sync::{Arc, Mutex}; +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex, Weak}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use vectorcraft_doc::Document; +use vectorcraft_doc::{Document, ImageBlob}; use super::fileio::{self, SAVE_FORMATS, SaveJob, SaveMode, SavePlan}; use super::*; @@ -50,6 +54,8 @@ pub const COMPLEX_OBJECTS: usize = 20_000; const DOC_EXT: &str = ".vectorcraft"; /// The extension of a copy's details entry. const META_EXT: &str = ".json"; +/// The extension of an image entry (see [`blob_name`]). +const BLOB_EXT: &str = ".blob"; /// The heartbeat entry of an area in a store without locks: when its app last said it runs (Unix /// seconds, as text). const BEAT: &str = "heartbeat"; @@ -527,6 +533,73 @@ struct Meta { /// (see [`crate::DocState::import_losses`]): a restored copy still refuses to replace it. imported_from: Option, import_losses: Vec, + /// The document's images by key, in entries of the area (none in copies written before + /// images were kept apart: the document entry holds them). + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + images: BTreeMap, +} + +/// One of a copy's images: its entry in the copy's area. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default)] +struct ImageRef { + mime: String, + /// The entry's name in the area ([`blob_name`]). + blob: String, + /// A linked image's preview, as a save writes it ([`ImageBlob::proxy`]). + #[serde(skip_serializing_if = "std::ops::Not::not")] + proxy: bool, +} + +/// Writes of image entries and their sweeping ([`sweep`]) take turns, in this app (the background +/// worker and the UI thread), so an image is never swept between being written and named. +static WRITES: Mutex<()> = Mutex::new(()); + +/// Image bytes and the name of their entry ([`blob_name`]). +type Named = (Weak>, String); + +/// The entry names of image bytes already hashed ([`blob_name`]), so an image is hashed once, +/// not at every interval. Bytes behind a live `Weak` never change (`Arc::make_mut` copies them). +static NAMES: Mutex> = Mutex::new(Vec::new()); + +/// The name of the entry holding `bytes`: from their content (128 bits of hash and the length), so +/// identical images share one entry and one written before isn't written again. +fn blob_name(bytes: &Arc>) -> String { + let mut names = NAMES.lock().unwrap_or_else(|e| e.into_inner()); + names.retain(|(w, _)| w.strong_count() > 0); + if let Some((_, name)) = names.iter().find(|(w, _)| std::ptr::eq(w.as_ptr(), Arc::as_ptr(bytes))) { + return name.clone(); + } + let hash = |seed: u8| { + let mut h = std::collections::hash_map::DefaultHasher::new(); + seed.hash(&mut h); + bytes.as_slice().hash(&mut h); + h.finish() + }; + let name = format!("img-{:016x}{:016x}-{}{BLOB_EXT}", hash(1), hash(2), bytes.len()); + names.push((Arc::downgrade(bytes), name.clone())); + name +} + +/// The MIME type of a linked image's preview (JPEG or PNG), as a save names it. +fn proxy_mime(bytes: &[u8]) -> &'static str { + if bytes.starts_with(&[0xff, 0xd8]) { "image/jpeg" } else { vectorcraft_doc::links::PROXY_MIME } +} + +/// Remove the image entries of `area` no copy's details name any more. Best effort: an entry left +/// behind goes at the next sweep, or with the area. +fn sweep(store: &dyn RecoveryStore, area: &str) { + let Ok(names) = store.list() else { return }; + let in_area = |n: &&String| n.split_once('/').is_some_and(|(a, _)| a == area); + let mut named = BTreeSet::new(); + for meta in names.iter().filter(in_area).filter_map(|n| n.strip_suffix(META_EXT)) { + named.extend(read_meta(store, meta).images.into_values().map(|r| r.blob)); + } + for n in names.iter().filter(in_area).filter(|n| n.ends_with(BLOB_EXT)) { + if n.split_once('/').is_some_and(|(_, b)| !named.contains(b)) { + let _ = store.remove(n); + } + } } fn read_meta(store: &dyn RecoveryStore, file: &str) -> Meta { @@ -548,14 +621,44 @@ pub struct RecoveryJob { } impl RecoveryJob { - /// Encode the snapshot and store it with its details → `{file, title, bytes}`. + /// Encode the snapshot and store it with its details and the images the area doesn't have + /// yet → `{file, title, bytes (the document's), imageBytes (images written now)}`. pub fn write(&self, store: &dyn RecoveryStore) -> std::result::Result { - let enc = self.save.encode().map_err(|e| e.to_string())?; + let doc = self.save.doc(); + // A linked image keeps its preview, as a save writes it. + let linked = doc.linked_only_images(); + let mut lean = Document::clone(doc); + let blobs = std::mem::take(&mut lean.images); + let enc = self.save.encode_doc(&lean).map_err(|e| e.to_string())?; let bytes = enc.files.first().map(|f| f.1.as_slice()).ok_or("nothing encoded")?; - let meta = serde_json::to_vec_pretty(&self.meta).map_err(|e| e.to_string())?; + let mut meta = self.meta.clone(); + let mut images = vec![]; + for (key, b) in &blobs { + let (mime, data, proxy) = match b.proxy.as_ref().filter(|_| linked.contains(key)) { + Some(p) => (proxy_mime(p), p, true), + None => (b.mime.as_str(), &b.bytes, false), + }; + let blob = blob_name(data); + meta.images.insert(key.clone(), ImageRef { mime: mime.into(), blob: blob.clone(), proxy }); + images.push((blob, data)); + } + let meta_bytes = serde_json::to_vec_pretty(&meta).map_err(|e| e.to_string())?; + let area = self.file.split_once('/').map_or("", |(a, _)| a); + let _turn = WRITES.lock().unwrap_or_else(|e| e.into_inner()); + let have: BTreeSet = if images.is_empty() { BTreeSet::new() } else { store.list()?.into_iter().collect() }; + let mut image_bytes = 0; + for (blob, data) in images { + let name = format!("{area}/{blob}"); + if !have.contains(&name) { + store.write(&name, data)?; + image_bytes += data.len(); + } + } store.write(&format!("{}{DOC_EXT}", self.file), bytes)?; - store.write(&format!("{}{META_EXT}", self.file), &meta)?; - Ok(json!({ "file": self.file, "title": self.title, "bytes": bytes.len() })) + store.write(&format!("{}{META_EXT}", self.file), &meta_bytes)?; + // Images this copy no longer shows, nor any other. + sweep(store, area); + Ok(json!({ "file": self.file, "title": self.title, "bytes": bytes.len(), "imageBytes": image_bytes })) } /// Once stored: the document (if still open) remembers the copy, so it isn't written again @@ -662,6 +765,8 @@ pub fn jobs(s: &mut Session, store: &Arc) -> Result<(Vec, file: &str) -> R let bytes = store.read(&format!("{file}{DOC_EXT}")).map_err(EngineError::Other)?; let meta_bytes = store.read(&format!("{file}{META_EXT}")).unwrap_or_default(); let meta: Meta = serde_json::from_slice(&meta_bytes).unwrap_or_default(); - let (mut doc, _, warnings) = fileio::native_file(&bytes).map_err(|e| EngineError::Other(format!("recovery copy `{file}`: {e}")))?; + let (mut doc, _, mut warnings) = fileio::native_file(&bytes).map_err(|e| EngineError::Other(format!("recovery copy `{file}`: {e}")))?; + // The images, from the entries of the copy's area. + let from = file.split_once('/').map_or("", |(a, _)| a); + let mut blobs = vec![]; + for (key, r) in &meta.images { + match store.read(&format!("{from}/{}", r.blob)) { + Ok(data) => { + let mut blob = ImageBlob::new(r.mime.clone(), data); + // Until the linked file is read, the preview stands in for it. + if r.proxy { + blob.proxy = Some(blob.bytes.clone()); + } + blobs.push((r.blob.clone(), blob.bytes.clone())); + doc.images.insert(key.clone(), blob); + } + Err(e) => warnings.push(format!("the image “{key}” of the recovered document couldn't be read: {e}")), + } + } let area = claim(s, store)?; let mut taken: Vec = copies_in(&store.list().map_err(EngineError::Other)?).into_iter().map(|(_, f)| f).collect(); taken.extend(open_copies(s)); @@ -838,8 +960,15 @@ fn restore_one(s: &mut Session, store: &Arc, file: &str) -> R f if taken.contains(&f) => fresh_name(&area, stem, &taken), f => f, }; - store.write(&format!("{mine}{DOC_EXT}"), &bytes).map_err(EngineError::Other)?; - store.write(&format!("{mine}{META_EXT}"), &meta_bytes).map_err(EngineError::Other)?; + { + let _turn = WRITES.lock().unwrap_or_else(|e| e.into_inner()); + for (blob, data) in &blobs { + store.write(&format!("{area}/{blob}"), data).map_err(EngineError::Other)?; + } + store.write(&format!("{mine}{DOC_EXT}"), &bytes).map_err(EngineError::Other)?; + store.write(&format!("{mine}{META_EXT}"), &meta_bytes).map_err(EngineError::Other)?; + } + // The copy's images in its old area go with the area, once no copy is left there. store.remove(&format!("{file}{DOC_EXT}")).map_err(EngineError::Other)?; store.remove(&format!("{file}{META_EXT}")).map_err(EngineError::Other)?; if doc.title.is_empty() { @@ -901,7 +1030,7 @@ pub fn specs() -> Vec { "Save Recovery Data", [], None, - "{} write a recovery copy of every modified document changed since its last copy, in this app's own area of the store (the app does this every autosaveInterval minutes while autosaveRecovery is on; documents of more than 20000 objects are skipped while recoveryOffForComplex is on). A copy goes when its document is saved, reverted or closed → {saved: [{file, title, bytes}], skipped: [{title, reason}], location} (location: the recovery folder, or browser storage on the web)", + "{} write a recovery copy of every modified document changed since its last copy, in this app's own area of the store (the app does this every autosaveInterval minutes while autosaveRecovery is on; documents of more than 20000 objects are skipped while recoveryOffForComplex is on). Images are stored apart, each once while unchanged. A copy goes when its document is saved, reverted or closed → {saved: [{file, title, bytes, imageBytes (images written this time)}], skipped: [{title, reason}], location} (location: the recovery folder, or browser storage on the web)", has_store, save ), diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index bf859b73..5046d6ed 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -34,16 +34,19 @@ #[cfg(not(target_arch = "wasm32"))] mod atomic; mod legacy; +mod stream; +mod unpack; use std::borrow::Cow; use std::collections::{BTreeMap, HashSet}; -use std::io::{Read as _, Write as _}; +use std::io::{Read as _, Write}; use serde::{Deserialize, Serialize}; use vectorcraft_doc::{Document, ImageBlob}; #[cfg(not(target_arch = "wasm32"))] pub use atomic::{write_atomic, write_atomic_with, write_new_with}; +use stream::{Gz, Indent, KeyScan, Out}; /// v4: large data after the JSON (v1 to v3 files still load). pub const VERSION: u32 = 4; @@ -60,7 +63,17 @@ pub const PREVIEW_MAX: u32 = 256; pub const EXTENSION: &str = "vectorcraft"; /// Extension and format name from before the project was renamed (DrawCraft): still opened. pub const LEGACY_EXTENSION: &str = "drawcraft"; -/// Most bytes a compressed file may unpack to (a guard against decompression bombs). +/// Most bytes the JSON of a compressed file may unpack to (a guard against decompression bombs, +/// as are [`MAX_BLOB`] and [`MAX_BLOBS`]). A compressed save refuses a document past any of them, +/// so every file VectorCraft writes reads back. +pub const MAX_JSON: u64 = 1 << 30; +/// Most bytes one entry of the data after the JSON (an image, a placed document, the PDF, a +/// profile) may have in a compressed file. +pub const MAX_BLOB: u64 = 1 << 30; +/// Most bytes the entries after the JSON may have together in a compressed file. +pub const MAX_BLOBS: u64 = 4 << 30; +/// Most bytes a compressed file whose JSON isn't followed by the marker where VectorCraft puts it +/// (hand-made or damaged files) may unpack to: it is unpacked whole. const MAX_UNPACKED: u64 = 512 << 20; /// Is `ext` (without the dot, any case) a native document extension? @@ -85,8 +98,10 @@ pub enum FormatError { BadVersion(u32), #[error("version {0} files can't be compressed: only version {COMPRESSED_SINCE} and later open compressed files")] CompressedTooOld(u32), - #[error("the compressed file unpacks to more than {} MB", MAX_UNPACKED >> 20)] + #[error("the compressed file unpacks to more than VectorCraft reads ({} MB of document, {} MB per image)", MAX_JSON >> 20, MAX_BLOB >> 20)] TooBig, + #[error("the document is too large to save compressed ({0}): save it with Use Compression off")] + TooBigToCompress(String), #[error("can't encode the document: {0}")] Encode(String), } @@ -106,37 +121,43 @@ struct Image { } impl Image { - /// Its bytes: the base64 `data`, or its part of `tail` (the data after the JSON). - fn bytes(&self, tail: &[u8]) -> Option> { + /// Its bytes when they are in the JSON (base64 `data`); `None` when they come after it + /// ([`Image::blob`]) or don't decode. + fn inline(&self) -> Option> { match self.blob { - Some([offset, len]) => { - let start = usize::try_from(offset).ok()?; - let end = start.checked_add(usize::try_from(len).ok()?)?; - tail.get(start..end).map(<[u8]>::to_vec) - } + Some(_) => None, None => base64_decode(&self.data), } } } /// Entries written for one file: large bytes go to the data after the JSON when the version keeps -/// them there. -struct Blobs { - tail: Vec, +/// them there (in the order they are added, each at the offset it is given). +struct Blobs<'a> { + /// The bytes, and whether they are compressed already ([`is_packed`]). + tail: Vec<(&'a [u8], bool)>, + at: u64, outside: bool, } -impl Blobs { - fn image(&mut self, mime: &str, bytes: &[u8], proxy: bool) -> Image { +impl<'a> Blobs<'a> { + fn image(&mut self, mime: &str, bytes: &'a [u8], proxy: bool) -> Image { if self.outside && bytes.len() >= INLINE_MAX { - let at = self.tail.len() as u64; - self.tail.extend_from_slice(bytes); - return Image { mime: mime.into(), data: String::new(), blob: Some([at, bytes.len() as u64]), proxy }; + let len = bytes.len() as u64; + let at = self.at; + self.at += len; + self.tail.push((bytes, is_packed(mime))); + return Image { mime: mime.into(), data: String::new(), blob: Some([at, len]), proxy }; } Image { mime: mime.into(), data: base64_encode(bytes), blob: None, proxy } } } +/// Is data of this MIME type compressed already (no use compressing it again)? +fn is_packed(mime: &str) -> bool { + matches!(mime, "image/jpeg" | "image/png" | "image/gif" | "image/webp") +} + /// The MIME type of a preview's bytes (JPEG or PNG). fn preview_mime(bytes: &[u8]) -> &'static str { if bytes.starts_with(&[0xff, 0xd8]) { "image/jpeg" } else { vectorcraft_doc::links::PROXY_MIME } @@ -247,6 +268,19 @@ pub fn save_file(doc: &Document) -> Vec { /// Serialize a document as `o` says: the version, compression, layout and preview. Editing-mode /// working copies and images nothing uses are left out. pub fn save_with(doc: &Document, o: &SaveOptions) -> Result, FormatError> { + let mut out = Vec::new(); + save_to(doc, o, &mut out)?; + Ok(out) +} + +/// Bytes of JSON serialized at a time on their way to the output. +const JSON_CHUNK: usize = 64 << 10; + +/// [`save_with`], written to `w` as it is made: the JSON is serialized straight into it (through +/// the compressor when compressing) and the images are copied in from the document, so the file +/// is never held in memory as a whole. Give it a buffered writer. On an error `w` has part of a +/// file: write to a temporary file ([`write_atomic_with`]). +pub fn save_to(doc: &Document, o: &SaveOptions, w: impl Write) -> Result<(), FormatError> { o.check()?; let pretty = o.pretty && !o.compress; let mut d = doc.without_edit_modes().into_owned(); @@ -264,62 +298,103 @@ pub fn save_with(doc: &Document, o: &SaveOptions) -> Result, FormatError if o.version < 3 { d.store_legacy_color_profiles(); } - let body = if o.version == 1 { + let v1 = if o.version == 1 { let mut v = serde_json::to_value(&d).map_err(encode_err)?; legacy::anchors_to_v1(&mut v); - to_json(&v, pretty)? + Some(v) } else { - to_json(&d, pretty)? + None }; + let entries = Entries { blobs: &blobs, linked: &linked, include_linked: o.include_linked }; + let mut out_blobs = Blobs { tail: vec![], at: 0, outside: o.version >= 4 }; // Images are only referenced by image objects' `key` (wherever the objects are: layers, - // symbols, patterns, masks, or foreign data). - let used = if blobs.is_empty() { HashSet::new() } else { string_members(&body, "key") }; - let mut out_blobs = Blobs { tail: vec![], outside: o.version >= 4 }; - let images: BTreeMap<&str, Image> = blobs - .iter() - .filter(|(k, _)| used.contains(k.as_str())) - .map(|(k, b)| { - // Include Linked Files keeps the file's bytes whenever they are loaded. - let preview_only = linked.contains(k) && (!o.include_linked || b.is_proxy()); - let image = match b.proxy.as_ref().filter(|_| preview_only) { - Some(p) => out_blobs.image(preview_mime(p), p, true), - None => out_blobs.image(&b.mime, &b.bytes, false), - }; - (k.as_str(), image) - }) - .collect(); + // symbols, patterns, masks, or foreign data). The data after the JSON has the images first, + // then the profiles and the PDF, which the file lists before the document: with those, the + // images in use are looked for before the document is written (it is serialized twice). + let mut images = None; + if !blobs.is_empty() && (!o.profiles.is_empty() || o.pdf.is_some()) { + let mut scan = KeyScan::new(std::io::sink()); + write_body(std::io::BufWriter::with_capacity(JSON_CHUNK, &mut scan), v1.as_ref(), &d, pretty)?; + images = Some(entries.images(&scan.keys, &mut out_blobs)); + } let profiles: BTreeMap<&str, Image> = o.profiles.iter().map(|(name, icc)| (name.as_str(), out_blobs.image(ICC_MIME, icc, false))).collect(); let pdf = o.pdf.as_deref().map(|p| out_blobs.image("application/pdf", p, false)); let blob = |mime: &str, bytes: &[u8]| Image { mime: mime.into(), data: base64_encode(bytes), blob: None, proxy: false }; - let carried = profiles.values().chain(&pdf).map(|i| i.data.len()).sum::(); - let size = body.len() + images.values().map(|i| i.data.len()).sum::() + carried + 256; - let mut w = Envelope { out: Vec::with_capacity(size), pretty }; + let mut out = if o.compress { Out::Gz(Box::new(Gz::new(w).map_err(encode_err)?)) } else { Out::Plain(w) }; + let mut env = Envelope { pretty, first: true }; // Older apps only know the name from before the rename. - w.field("format", &if o.version < 3 { LEGACY_EXTENSION } else { EXTENSION })?; - w.field("version", &o.version)?; - w.field("generator", &format!("VectorCraft {}", env!("CARGO_PKG_VERSION")))?; + env.field(&mut out, "format", &if o.version < 3 { LEGACY_EXTENSION } else { EXTENSION })?; + env.field(&mut out, "version", &o.version)?; + env.field(&mut out, "generator", &format!("VectorCraft {}", env!("CARGO_PKG_VERSION")))?; if let Some(png) = &o.preview { - w.field("preview", &blob("image/png", png))?; + env.field(&mut out, "preview", &blob("image/png", png))?; } if !profiles.is_empty() { - w.field("profiles", &profiles)?; + env.field(&mut out, "profiles", &profiles)?; } if let Some(pdf) = &pdf { - w.field("pdf", pdf)?; + env.field(&mut out, "pdf", pdf)?; + } + env.key(&mut out, "document")?; + let mut scan = KeyScan::new(Indent { inner: &mut out, on: pretty }); + // Without blobs no key matters. + scan.active = images.is_none() && !blobs.is_empty(); + let mut buffered = std::io::BufWriter::with_capacity(JSON_CHUNK, &mut scan); + write_body(&mut buffered, v1.as_ref(), &d, pretty)?; + buffered.flush().map_err(encode_err)?; + drop(buffered); + let used = std::mem::take(&mut scan.keys); + let images = images.unwrap_or_else(|| entries.images(&used, &mut out_blobs)); + env.field(&mut out, "images", &images)?; + env.finish(&mut out)?; + if o.compress { + let json = out.written().unwrap_or(0); + let tail: Vec = out_blobs.tail.iter().map(|(b, _)| b.len() as u64).collect(); + if json > MAX_JSON { + return Err(FormatError::TooBigToCompress(format!("{} MB of document, at most {} MB", json >> 20, MAX_JSON >> 20))); + } + if let Some(big) = tail.iter().find(|&&n| n > MAX_BLOB) { + return Err(FormatError::TooBigToCompress(format!("an image of {} MB, at most {} MB", big >> 20, MAX_BLOB >> 20))); + } + if out_blobs.at > MAX_BLOBS { + return Err(FormatError::TooBigToCompress(format!("{} MB of images, at most {} MB", out_blobs.at >> 20, MAX_BLOBS >> 20))); + } } - w.raw("document", &body); - w.field("images", &images)?; - let mut out = w.finish(); if !out_blobs.tail.is_empty() { - out.extend_from_slice(BLOB_MAGIC); - out.extend_from_slice(&out_blobs.tail); + out.write_all(BLOB_MAGIC).map_err(encode_err)?; + for &(bytes, packed) in &out_blobs.tail { + out.packed(packed).map_err(encode_err)?; + out.write_all(bytes).map_err(encode_err)?; + } } - if !o.compress { - return Ok(out); + out.finish().map_err(encode_err) +} + +/// The images a file lists, from the document's blobs. +struct Entries<'a> { + blobs: &'a BTreeMap, + /// Images only linked images show (see [`Document::linked_only_images`]). + linked: &'a std::collections::BTreeSet, + include_linked: bool, +} + +impl<'a> Entries<'a> { + /// The entries of the images named in `used`, in the order of their names. + fn images(&self, used: &HashSet, out: &mut Blobs<'a>) -> BTreeMap<&'a str, Image> { + self.blobs + .iter() + .filter(|(k, _)| used.contains(k.as_str())) + .map(|(k, b)| { + // Include Linked Files keeps the file's bytes whenever they are loaded. + let preview_only = self.linked.contains(k) && (!self.include_linked || b.is_proxy()); + let image = match b.proxy.as_ref().filter(|_| preview_only) { + Some(p) => out.image(preview_mime(p), p, true), + None => out.image(&b.mime, &b.bytes, false), + }; + (k.as_str(), image) + }) + .collect() } - let mut gz = flate2::write::GzEncoder::new(Vec::with_capacity(out.len() / 4), flate2::Compression::default()); - gz.write_all(&out).map_err(encode_err)?; - gz.finish().map_err(encode_err) } fn encode_err(e: impl std::fmt::Display) -> FormatError { @@ -330,85 +405,51 @@ fn to_json(v: &impl Serialize, pretty: bool) -> Result, FormatError> { if pretty { serde_json::to_vec_pretty(v) } else { serde_json::to_vec(v) }.map_err(encode_err) } -/// The file's top-level object, written member by member (the document goes in pre-serialized). +/// The document's JSON: its version 1 form `v1`, else `d` (generic, not `dyn Write`: serde_json +/// writes token by token). +fn write_body(w: impl Write, v1: Option<&serde_json::Value>, d: &Document, pretty: bool) -> Result<(), FormatError> { + match v1 { + Some(v) => write_json(w, v, pretty), + None => write_json(w, d, pretty), + } +} + +fn write_json(w: impl Write, v: &impl Serialize, pretty: bool) -> Result<(), FormatError> { + if pretty { serde_json::to_writer_pretty(w, v) } else { serde_json::to_writer(w, v) }.map_err(encode_err) +} + +/// The file's top-level object, written member by member (the document is streamed in). struct Envelope { - out: Vec, pretty: bool, + first: bool, } impl Envelope { - fn field(&mut self, key: &str, value: &impl Serialize) -> Result<(), FormatError> { + fn field(&mut self, out: &mut impl Write, key: &str, value: &impl Serialize) -> Result<(), FormatError> { let json = to_json(value, self.pretty)?; - self.raw(key, &json); - Ok(()) + self.key(out, key)?; + Indent { inner: out, on: self.pretty }.write_all(&json).map_err(encode_err) } - /// Member `key` (a plain name) with the JSON text `json`, indented one level when pretty (as - /// serde_json would). - fn raw(&mut self, key: &str, json: &[u8]) { - let out = &mut self.out; - out.push(if out.is_empty() { b'{' } else { b',' }); - if self.pretty { - out.extend_from_slice(b"\n "); - } - out.push(b'"'); - out.extend_from_slice(key.as_bytes()); - out.extend_from_slice(if self.pretty { b"\": " } else { b"\":" }); - // Newlines only occur between tokens (inside strings they are escaped). - let mut lines = json.split(|&b| b == b'\n'); - out.extend_from_slice(lines.next().unwrap_or_default()); - for line in lines { - out.extend_from_slice(b"\n "); - out.extend_from_slice(line); - } + /// The start of member `key` (a plain name): its value follows, indented one level when + /// pretty (as serde_json would: see [`Indent`]). + fn key(&mut self, out: &mut impl Write, key: &str) -> Result<(), FormatError> { + let sep: &[u8] = if std::mem::take(&mut self.first) { b"{" } else { b"," }; + let indent: &[u8] = if self.pretty { b"\n " } else { b"" }; + let colon: &[u8] = if self.pretty { b"\": " } else { b"\":" }; + [sep, indent, b"\"", key.as_bytes(), colon].iter().try_for_each(|part| out.write_all(part)).map_err(encode_err) } - fn finish(mut self) -> Vec { - self.out.extend_from_slice(if self.pretty { b"\n}" } else { b"}" }); - self.out + fn finish(self, out: &mut impl Write) -> Result<(), FormatError> { + out.write_all(if self.pretty { b"\n}" } else { b"}" }).map_err(encode_err) } } -/// The string values of every `"": "…"` member at any depth of JSON text. Quotes inside JSON -/// strings are escaped, so every match is a real member. -fn string_members(json: &[u8], name: &str) -> HashSet { - let tag = format!("\"{name}\":"); - let tag = tag.as_bytes(); - let mut out = HashSet::new(); - let mut rest = json; - while let Some(at) = rest.windows(tag.len()).position(|w| w == tag) { - rest = rest.get(at + tag.len()..).unwrap_or_default(); - let value = rest.trim_ascii_start(); - if value.first() == Some(&b'"') - && let Some(Ok(s)) = serde_json::Deserializer::from_slice(value).into_iter::().next() - { - out.insert(s); - } - } - out -} - /// Is this gzip data (a compressed file)? pub fn is_compressed(bytes: &[u8]) -> bool { bytes.starts_with(&[0x1f, 0x8b]) } -/// The JSON text of a file, unpacked when compressed. -fn unpack(bytes: &[u8]) -> Result, FormatError> { - if !is_compressed(bytes) { - return Ok(Cow::Borrowed(bytes)); - } - let mut raw = Vec::new(); - flate2::read::GzDecoder::new(bytes) - .take(MAX_UNPACKED + 1) - .read_to_end(&mut raw) - .map_err(|e| FormatError::NotVectorcraft(format!("damaged compressed data: {e}")))?; - if raw.len() as u64 > MAX_UNPACKED { - return Err(FormatError::TooBig); - } - Ok(Cow::Owned(raw)) -} - pub fn load(bytes: &[u8]) -> Result { load_info(bytes).map(|(doc, _)| doc) } @@ -445,18 +486,24 @@ pub fn load_info(bytes: &[u8]) -> Result<(Document, FileInfo), FormatError> { /// [`load`], with everything else the file carries. pub fn load_file(bytes: &[u8]) -> Result { - let text = unpack(bytes)?; - let (json, tail) = split_tail(&text); - let f: File = serde_json::from_slice(json).map_err(|e| FormatError::NotVectorcraft(e.to_string()))?; + let file = unpack::Unpacked::new(bytes)?; + let f: File = serde_json::from_slice(file.json()).map_err(|e| FormatError::NotVectorcraft(e.to_string()))?; if f.format != EXTENSION && f.format != LEGACY_EXTENSION { return Err(FormatError::NotVectorcraft(format!("format is `{}`", f.format))); } if f.version > VERSION { return Err(FormatError::TooNew(f.version)); } + // The bytes after the JSON, read in one pass: the images' first, then the profiles'. + let entries: Vec<[u64; 2]> = f.images.values().chain(f.profiles.values()).filter_map(|i| i.blob).collect(); + let mut tail = file.entries(&entries)?.into_iter(); + let mut bytes_of = |img: &Image| match img.blob { + Some(_) => tail.next().flatten(), + None => img.inline(), + }; let mut doc = f.document; for (k, img) in f.images { - let bytes = img.bytes(tail).ok_or_else(|| FormatError::BadImage(k.clone()))?; + let bytes = bytes_of(&img).ok_or_else(|| FormatError::BadImage(k.clone()))?; let mut blob = ImageBlob::new(img.mime, bytes); // Until the linked file is read, the preview stands in for it. if img.proxy { @@ -474,7 +521,7 @@ pub fn load_file(bytes: &[u8]) -> Result { // Untrusted: capped, named and pointing at objects the file has. doc.tidy_saved_selections(); doc.tidy_graph_designs(); - let profiles = f.profiles.into_iter().filter_map(|(name, icc)| Some((name, icc.bytes(tail)?))).collect(); + let profiles = f.profiles.into_iter().filter_map(|(name, icc)| Some((name, bytes_of(&icc)?))).collect(); Ok(NativeFile { doc, info: FileInfo { version: f.version, legacy: f.format == LEGACY_EXTENSION }, profiles }) } @@ -485,24 +532,27 @@ struct Head { pdf: Option, } -/// The file's head and the data after its JSON. -fn head(bytes: &[u8]) -> Option<(Head, Vec)> { - let text = unpack(bytes).ok()?; - let (json, tail) = split_tail(&text); - Some((serde_json::from_slice(json).ok()?, tail.to_vec())) +/// The bytes of the member of the file's head `pick` picks (only that entry of the data after +/// the JSON is kept). +fn head_member(bytes: &[u8], pick: impl FnOnce(Head) -> Option) -> Option> { + let file = unpack::Unpacked::new(bytes).ok()?; + let head: Head = serde_json::from_slice(file.json()).ok()?; + let img = pick(head)?; + match img.blob { + Some(at) => file.entries(&[at]).ok()?.pop().flatten(), + None => img.inline(), + } } /// The preview PNG embedded in a native file (`None`: it has none, or isn't a native file). pub fn preview(bytes: &[u8]) -> Option> { - let (h, tail) = head(bytes)?; - h.preview?.bytes(&tail) + head_member(bytes, |h| h.preview) } /// The PDF a native file saved with Create PDF-Compatible File carries (`None`: it has none, or /// isn't a native file). pub fn pdf_content(bytes: &[u8]) -> Option> { - let (h, tail) = head(bytes)?; - h.pdf?.bytes(&tail) + head_member(bytes, |h| h.pdf) } /// Does this look like a `.vectorcraft` file (compressed or not)? diff --git a/crates/format/src/stream.rs b/crates/format/src/stream.rs new file mode 100644 index 00000000..60879c07 --- /dev/null +++ b/crates/format/src/stream.rs @@ -0,0 +1,358 @@ +//! Writing a file as a stream: the JSON is serialized straight into the output (compressed or +//! not) and the data after it is copied in from the document's own buffers, so a save holds no +//! copy of the file in memory besides what the caller's writer keeps. + +use std::collections::HashSet; +use std::io::{self, Write}; + +use flate2::{Compress, Compression, Crc, FlushCompress, Status}; + +/// The output of a save: plain, or gzip ([`Gz`]). +pub(crate) enum Out { + Plain(W), + Gz(Box>), +} + +impl Out { + /// What is written next is compressed already (JPEG, PNG…) or not ([`Gz::packed`]). + pub(crate) fn packed(&mut self, packed: bool) -> io::Result<()> { + match self { + Out::Plain(_) => Ok(()), + Out::Gz(gz) => gz.packed(packed), + } + } + + /// Bytes written so far (before compression). + pub(crate) fn written(&self) -> Option { + match self { + Out::Plain(_) => None, + Out::Gz(gz) => Some(gz.total), + } + } + + /// Everything written: the gzip trailer goes out, and the writer is flushed. + pub(crate) fn finish(self) -> io::Result<()> { + match self { + Out::Plain(mut w) => w.flush(), + Out::Gz(gz) => gz.finish(), + } + } +} + +impl Write for Out { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Out::Plain(w) => w.write(buf), + Out::Gz(gz) => gz.write(buf), + } + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + match self { + Out::Plain(w) => w.write_all(buf), + Out::Gz(gz) => gz.write_all(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Out::Plain(w) => w.flush(), + Out::Gz(gz) => gz.flush(), + } + } +} + +/// Bytes of compressed output gathered before they go to the writer. +const OUT_CHUNK: usize = 64 << 10; + +/// A gzip stream, written as [`flate2::write::GzEncoder`] at the default level writes it (the +/// same header, and the same bytes for a file without compressed images), except that data which +/// is compressed already ([`Gz::packed`]) is stored as it is: the default level spends a second +/// or more per photo on it for next to nothing, and makes it slower to unpack. Where the level +/// changes the deflate blocks so far end with a sync flush and a fresh compressor writes on, +/// which makes one valid deflate stream that every reader unpacks. +pub(crate) struct Gz { + out: W, + deflate: Compress, + /// What `deflate` writes: stored blocks (packed data), else the default level. + stored: bool, + crc: Crc, + buf: Vec, + /// Bytes written (unpacked). + total: u64, +} + +impl Gz { + pub(crate) fn new(mut out: W) -> io::Result { + // flate2's header for the default level: no name, no time, unknown OS. + out.write_all(&[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255])?; + Ok(Self { + out, + deflate: Compress::new(Compression::default(), false), + stored: false, + crc: Crc::new(), + buf: Vec::with_capacity(OUT_CHUNK), + total: 0, + }) + } + + /// Run the compressor over `input` with `flush` until it has taken it all (and, flushing, + /// written everything out). + fn deflate(&mut self, mut input: &[u8], flush: FlushCompress) -> io::Result<()> { + loop { + if self.buf.len() == self.buf.capacity() { + self.out.write_all(&self.buf)?; + self.buf.clear(); + } + let (before_in, before_out) = (self.deflate.total_in(), self.deflate.total_out()); + let status = self.deflate.compress_vec(input, &mut self.buf, flush).map_err(io::Error::other)?; + let taken = usize::try_from(self.deflate.total_in() - before_in).unwrap_or(input.len()).min(input.len()); + input = input.get(taken..).unwrap_or_default(); + let made = self.deflate.total_out() - before_out; + let full = self.buf.len() == self.buf.capacity(); + match flush { + FlushCompress::None if input.is_empty() => return Ok(()), + FlushCompress::Finish if status == Status::StreamEnd => return Ok(()), + // A flush is done once it left room in the output. + FlushCompress::Sync if input.is_empty() && !full => return Ok(()), + _ => {} + } + if taken == 0 && made == 0 && !full { + return Err(io::Error::other("the compressor made no progress")); + } + } + } + + /// What is written next is compressed already (stored as it is), or not (compressed at the + /// default level). + fn packed(&mut self, packed: bool) -> io::Result<()> { + if packed == self.stored { + return Ok(()); + } + self.deflate(&[], FlushCompress::Sync)?; + self.deflate = Compress::new(if packed { Compression::none() } else { Compression::default() }, false); + self.stored = packed; + Ok(()) + } + + fn finish(mut self) -> io::Result<()> { + self.deflate(&[], FlushCompress::Finish)?; + self.out.write_all(&self.buf)?; + self.out.write_all(&self.crc.sum().to_le_bytes())?; + self.out.write_all(&self.crc.amount().to_le_bytes())?; + self.out.flush() + } +} + +impl Write for Gz { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.crc.update(buf); + self.total += buf.len() as u64; + self.deflate(buf, FlushCompress::None) + } + + fn flush(&mut self) -> io::Result<()> { + // Flushing would end a deflate block: the stream is flushed when it is finished. + Ok(()) + } +} + +/// Indents JSON text written through it one level (two spaces after each newline), as a member +/// of the file's top-level object; newlines only occur between tokens, inside strings they are +/// escaped. +pub(crate) struct Indent { + pub(crate) inner: W, + pub(crate) on: bool, +} + +impl Write for Indent { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + if !self.on { + return self.inner.write_all(buf); + } + let mut lines = buf.split(|&b| b == b'\n'); + self.inner.write_all(lines.next().unwrap_or_default())?; + for line in lines { + self.inner.write_all(b"\n ")?; + self.inner.write_all(line)?; + } + Ok(()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// The member that names an image. +const TAG: &[u8] = b"\"key\":"; + +/// Passes JSON text through, collecting the string values of every `"key": "…"` member at any +/// depth (what the images a document uses are named by). Quotes inside JSON strings are escaped, +/// so every match is a real member. +pub(crate) struct KeyScan { + pub(crate) inner: W, + pub(crate) keys: HashSet, + /// Looking for keys (else it only passes the text on). + pub(crate) active: bool, + /// How much of [`TAG`] the last bytes matched. + matched: usize, + /// Right after a tag: skipping whitespace to its value. + after_tag: bool, + /// The value string so far, quotes included. + value: Option>, + escaped: bool, +} + +impl KeyScan { + pub(crate) fn new(inner: W) -> Self { + Self { inner, keys: HashSet::new(), active: true, matched: 0, after_tag: false, value: None, escaped: false } + } + + fn scan(&mut self, buf: &[u8]) { + if !self.active { + return; + } + let mut i = 0; + while let Some(&b) = buf.get(i) { + // Nothing in progress: skip to the next quote. + if self.matched == 0 && !self.after_tag && self.value.is_none() { + match buf.get(i..).and_then(|r| r.iter().position(|&c| c == b'"')) { + Some(at) => i += at, + None => return, + } + self.matched = 1; + i += 1; + continue; + } + self.step(b); + i += 1; + } + } + + fn step(&mut self, b: u8) { + if let Some(v) = self.value.as_mut() { + v.push(b); + if self.escaped { + self.escaped = false; + } else if b == b'\\' { + self.escaped = true; + } else if b == b'"' + && let Some(v) = self.value.take() + && let Ok(s) = serde_json::from_slice::(&v) + { + self.keys.insert(s); + } + } else if self.after_tag && !b.is_ascii_whitespace() { + self.after_tag = false; + if b == b'"' { + self.value = Some(vec![b]); + } + } + if TAG.get(self.matched) == Some(&b) { + self.matched += 1; + if self.matched == TAG.len() { + self.matched = 0; + self.after_tag = true; + self.value = None; + self.escaped = false; + } + } else { + self.matched = usize::from(b == b'"'); + } + } +} + +impl Write for KeyScan { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.scan(buf); + self.inner.write_all(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + /// How the keys were found before saves were streamed: in the whole JSON text at once. + fn string_members(json: &[u8]) -> HashSet { + let mut out = HashSet::new(); + let mut rest = json; + while let Some(at) = rest.windows(TAG.len()).position(|w| w == TAG) { + rest = rest.get(at + TAG.len()..).unwrap_or_default(); + let value = rest.trim_ascii_start(); + if value.first() == Some(&b'"') + && let Some(Ok(s)) = serde_json::Deserializer::from_slice(value).into_iter::().next() + { + out.insert(s); + } + } + out + } + + /// The keys `KeyScan` finds in `json` written in pieces of `chunk` bytes. + fn scanned(json: &[u8], chunk: usize) -> HashSet { + let mut scan = KeyScan::new(Vec::new()); + for piece in json.chunks(chunk.max(1)) { + scan.write_all(piece).unwrap(); + } + assert_eq!(scan.inner, json, "the text passes through"); + scan.keys + } + + #[test] + fn finds_the_keys_the_whole_text_search_found() { + let v = serde_json::json!({"key": "a", "x": [{"key": "b \"q\" \\ é"}, {"monkey": "no"}, {"k": "key"}], "s": "\"key\": \"no\"", "key2": "n", "o": {"key": 3}}); + for json in [serde_json::to_vec(&v).unwrap(), serde_json::to_vec_pretty(&v).unwrap()] { + let want = string_members(&json); + assert_eq!(want, HashSet::from(["a".to_string(), "b \"q\" \\ é".to_string()])); + for chunk in [1, 2, 3, 7, 64, json.len()] { + assert_eq!(scanned(&json, chunk), want, "chunk {chunk}"); + } + } + } + + fn arb_json() -> impl Strategy { + let key = prop::sample::select(vec!["key", "k", "kkey", "key ", "x"]).prop_map(str::to_string); + let leaf = prop_oneof![ + Just(serde_json::Value::Null), + any::().prop_map(serde_json::Value::from), + "[a-z\"\\\\: ké\n]{0,8}".prop_map(serde_json::Value::from), + ]; + leaf.prop_recursive(4, 32, 6, move |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..5).prop_map(serde_json::Value::from), + prop::collection::vec((key.clone(), inner), 0..5).prop_map(|m| serde_json::Value::Object(m.into_iter().collect())), + ] + }) + } + + proptest! { + #[test] + fn scanning_in_pieces_finds_what_the_whole_text_search_found(v in arb_json(), chunk in 1usize..40, pretty in any::()) { + let json = if pretty { serde_json::to_vec_pretty(&v) } else { serde_json::to_vec(&v) }.unwrap(); + prop_assert_eq!(scanned(&json, chunk), string_members(&json)); + } + } +} diff --git a/crates/format/src/unpack.rs b/crates/format/src/unpack.rs new file mode 100644 index 00000000..360b756e --- /dev/null +++ b/crates/format/src/unpack.rs @@ -0,0 +1,214 @@ +//! Reading a file: its JSON, then the entries of the data after it (v4) one by one. A compressed +//! file is unpacked as a stream: the JSON into one buffer, each entry straight into its own, so +//! the file is never held unpacked as a whole. Limits apply per part ([`MAX_JSON`], [`MAX_BLOB`], +//! [`MAX_BLOBS`]), guarding against decompression bombs. + +use std::io::{self, Read}; + +use flate2::read::GzDecoder; + +use crate::{BLOB_MAGIC, FormatError, MAX_BLOB, MAX_BLOBS, MAX_JSON, MAX_UNPACKED, is_compressed, split_tail}; + +/// Bytes unpacked at a time while looking for the end of the JSON. +const CHUNK: u64 = 64 << 10; + +fn damaged(e: impl std::fmt::Display) -> FormatError { + FormatError::NotVectorcraft(format!("damaged compressed data: {e}")) +} + +/// A file opened for reading: its JSON, and where the data after it is. +pub(crate) struct Unpacked<'a> { + json: Json<'a>, + tail: Tail<'a>, +} + +enum Json<'a> { + Borrowed(&'a [u8]), + Owned(Vec), + /// Unpacked whole (a compressed file whose JSON isn't followed by the marker where the writer + /// puts it): `buf[..json]` is the JSON ([`Tail::Whole`] says where the data after it is). + Whole { + buf: Vec, + json: usize, + }, +} + +enum Tail<'a> { + /// The data after the JSON, in memory (an uncompressed file, or one unpacked whole). + Slice(&'a [u8]), + /// In memory, in [`Json::Whole`]'s buffer from this offset. + Whole(usize), + /// Still to unpack from `file`: it starts at `at` in the unpacked stream; `read` holds what + /// was unpacked past the marker already, and `rest` the stream from there. + Stream { + file: &'a [u8], + at: u64, + read: Vec, + rest: Box>, + }, + None, +} + +impl<'a> Unpacked<'a> { + /// Open `bytes` (compressed or not). + pub(crate) fn new(bytes: &'a [u8]) -> Result { + if !is_compressed(bytes) { + let (json, tail) = split_tail(bytes); + return Ok(Self { json: Json::Borrowed(json), tail: Tail::Slice(tail) }); + } + let mut dec = GzDecoder::new(bytes); + let mut buf = Vec::new(); + let mut nul = None; + loop { + let start = buf.len(); + let n = (&mut dec).take(CHUNK).read_to_end(&mut buf).map_err(damaged)?; + if nul.is_none() { + nul = buf.get(start..).and_then(|b| b.iter().position(|&c| c == 0)).map(|p| p + start); + } + if n == 0 || nul.is_some_and(|z| buf.len() >= z + BLOB_MAGIC.len()) { + break; + } + if buf.len() as u64 > MAX_JSON + CHUNK { + return Err(FormatError::TooBig); + } + } + let Some(z) = nul else { + // No NUL, so no marker: all of it is JSON. + if buf.len() as u64 > MAX_JSON { + return Err(FormatError::TooBig); + } + return Ok(Self { json: Json::Owned(buf), tail: Tail::None }); + }; + // JSON holds no raw NUL, so the marker (which has one) starts right before the first NUL; + // the writer puts it right after the JSON's last byte. + if let Some(end) = z.checked_sub(1) + && buf.get(end..).is_some_and(|r| r.starts_with(BLOB_MAGIC)) + && end.checked_sub(1).and_then(|k| buf.get(k)).is_some_and(|b| matches!(b, b'}' | b']' | b'"')) + { + if end as u64 > MAX_JSON { + return Err(FormatError::TooBig); + } + let read = buf.split_off(end + BLOB_MAGIC.len()); + buf.truncate(end); + let at = (end + BLOB_MAGIC.len()) as u64; + return Ok(Self { json: Json::Owned(buf), tail: Tail::Stream { file: bytes, at, read, rest: Box::new(dec) } }); + } + // Anything else (hand-made or damaged files) is unpacked whole and split as an + // uncompressed file is. + dec.take(MAX_UNPACKED + 1 - (buf.len() as u64).min(MAX_UNPACKED)).read_to_end(&mut buf).map_err(damaged)?; + if buf.len() as u64 > MAX_UNPACKED { + return Err(FormatError::TooBig); + } + let (json, tail) = split_tail(&buf); + let (json, tail) = (json.len(), buf.len() - tail.len()); + Ok(Self { json: Json::Whole { buf, json }, tail: Tail::Whole(tail) }) + } + + /// The JSON text. + pub(crate) fn json(&self) -> &[u8] { + match &self.json { + Json::Borrowed(j) => j, + Json::Owned(j) => j, + Json::Whole { buf, json, .. } => buf.get(..*json).unwrap_or_default(), + } + } + + /// The entries `[offset, length]` of the data after the JSON → each one's bytes, `None` for + /// one that isn't there (the data ends before it). Errors: the compressed data is damaged, or + /// the entries are larger than [`MAX_BLOB`] each or [`MAX_BLOBS`] together. Entries are read + /// in the order of their offsets, each straight into its own buffer. + pub(crate) fn entries(self, wanted: &[[u64; 2]]) -> Result>>, FormatError> { + let mut out: Vec>> = vec![None; wanted.len()]; + let in_memory = |tail: &[u8], out: &mut Vec>>| { + for (slot, &[offset, len]) in out.iter_mut().zip(wanted) { + *slot = range(offset, len).and_then(|(a, b)| tail.get(a..b)).map(<[u8]>::to_vec); + } + }; + match self.tail { + Tail::Slice(tail) => in_memory(tail, &mut out), + Tail::Whole(at) => { + let buf = match &self.json { + Json::Whole { buf, .. } => buf.as_slice(), + _ => &[], + }; + in_memory(buf.get(at..).unwrap_or_default(), &mut out); + } + Tail::None => in_memory(&[], &mut out), + Tail::Stream { file, at, read, rest } => { + let mut total = 0u64; + for &[_, len] in wanted { + if len > MAX_BLOB { + return Err(FormatError::TooBig); + } + total = total.saturating_add(len); + } + if total > MAX_BLOBS { + return Err(FormatError::TooBig); + } + drop(self.json); + let mut order: Vec = (0..wanted.len()).collect(); + order.sort_by_key(|&i| wanted.get(i).map_or(0, |w| w[0])); + let mut s = Stream { r: Box::new(io::Cursor::new(read).chain(rest)), pos: 0, file, at }; + for i in order { + let Some(&[offset, len]) = wanted.get(i) else { continue }; + if let Some(slot) = out.get_mut(i) { + *slot = s.entry(offset, len)?; + } + } + // To the end, so the stream's checksum is verified as before. + s.skip(MAX_BLOBS)?; + } + } + Ok(out) + } +} + +/// `offset..offset + len` as indices, when they fit. +fn range(offset: u64, len: u64) -> Option<(usize, usize)> { + let a = usize::try_from(offset).ok()?; + Some((a, a.checked_add(usize::try_from(len).ok()?)?)) +} + +/// The data after a compressed file's JSON, being unpacked. +struct Stream<'a> { + r: Box, + /// Where `r` is in the data after the JSON. + pos: u64, + file: &'a [u8], + /// Where the data after the JSON starts in the unpacked stream. + at: u64, +} + +impl Stream<'_> { + /// Unpack and drop up to `n` bytes → how many there were. + fn skip(&mut self, n: u64) -> Result { + let skipped = io::copy(&mut (&mut self.r).take(n), &mut io::sink()).map_err(damaged)?; + self.pos += skipped; + Ok(skipped) + } + + /// The entry at `offset`, `len` bytes long (`None`: the data ends before it does). + fn entry(&mut self, offset: u64, len: u64) -> Result>, FormatError> { + if offset < self.pos { + // Entries out of order or overlapping (only hand-made files): unpack again from the + // start. + let mut dec = GzDecoder::new(self.file); + let skipped = io::copy(&mut (&mut dec).take(self.at), &mut io::sink()).map_err(damaged)?; + if skipped < self.at { + return Ok(None); + } + self.r = Box::new(dec); + self.pos = 0; + } + let gap = offset - self.pos; + if self.skip(gap)? < gap { + return Ok(None); + } + let mut bytes = Vec::new(); + let n = usize::try_from(len).map_err(|_| FormatError::TooBig)?; + bytes.try_reserve_exact(n).map_err(|_| FormatError::TooBig)?; + let got = (&mut self.r).take(len).read_to_end(&mut bytes).map_err(damaged)?; + self.pos += got as u64; + Ok((got as u64 == len).then_some(bytes)) + } +} diff --git a/crates/format/tests/streaming.rs b/crates/format/tests/streaming.rs new file mode 100644 index 00000000..81d935f8 --- /dev/null +++ b/crates/format/tests/streaming.rs @@ -0,0 +1,199 @@ +//! Saving and opening as streams: the bytes a save writes are those an in-memory save wrote +//! before (the same JSON and data after it, unpacked; the same gzip bytes when no image is +//! compressed already), files written the old way still open, entries are read whatever their +//! order, and the limits apply per part of the file. +// Integration tests: unwrapping and panicking on failure is fine here, unlike in shipped code (AGENTS.md › Robustness). +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::BTreeMap; +use std::io::{Read, Write}; + +use vectorcraft_doc::{Document, ImageBlob, ImageObject, Node, NodeKind}; +use vectorcraft_format::{BLOB_MAGIC, FormatError, INLINE_MAX, MAX_BLOB, SaveOptions, load, load_file, pdf_content, preview, save_to, save_with}; +use vectorcraft_geom::Affine; + +/// A document showing images `(key, mime, len)`. +fn doc_with(images: &[(&str, &str, usize)]) -> Document { + let mut d = Document::new(100.0, 100.0); + let l = d.layers[0].id; + for (n, &(key, mime, len)) in images.iter().enumerate() { + let bytes: Vec = (0..len).map(|i| ((i * 7 + n * 13) % 251) as u8).collect(); + d.images.insert(key.into(), ImageBlob::new(mime, bytes)); + let id = d.alloc_id(); + let im = ImageObject { key: key.into(), width: 10, height: 10, xf: Affine::IDENTITY, link: None, placement: Default::default() }; + d.insert(Some(l), 0, Node::new(id, NodeKind::Image(Box::new(im)))).unwrap(); + } + d +} + +/// gzip as the app compressed files before saves were streamed (the whole file at once). +fn gzip(bytes: &[u8]) -> Vec { + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(bytes).unwrap(); + gz.finish().unwrap() +} + +fn gunzip(bytes: &[u8]) -> Vec { + let mut out = vec![]; + flate2::read::GzDecoder::new(bytes).read_to_end(&mut out).unwrap(); + out +} + +fn mixed() -> Document { + doc_with(&[ + ("photo", "image/jpeg", INLINE_MAX * 3), + ("scan", "image/tiff", INLINE_MAX * 2), + ("icon", "image/png", 100), + ("art", "image/png", INLINE_MAX + 1), + ]) +} + +#[test] +fn compressed_files_unpack_to_the_plain_file() { + let pdf = vec![b'%'; INLINE_MAX * 2]; + let profiles = BTreeMap::from([("Mine".to_string(), vec![7u8; INLINE_MAX + 3])]); + for d in [Document::new(10.0, 10.0), mixed()] { + let o = SaveOptions { pdf: Some(pdf.clone()), profiles: profiles.clone(), preview: Some(vec![1, 2, 3]), ..SaveOptions::default() }; + let plain = save_with(&d, &o).unwrap(); + let packed = save_with(&d, &SaveOptions { compress: true, ..o }).unwrap(); + assert_eq!(gunzip(&packed), plain, "the same file, compressed"); + let back = load_file(&packed).unwrap(); + for (k, b) in &d.images { + assert_eq!(back.doc.images[k].bytes, b.bytes, "{k}"); + } + assert_eq!(back.profiles, profiles); + assert_eq!(pdf_content(&packed).unwrap(), pdf); + assert_eq!(preview(&packed).unwrap(), vec![1, 2, 3]); + } +} + +#[test] +fn without_compressed_images_the_gzip_bytes_are_the_old_ones() { + let mut d = doc_with(&[("scan", "image/tiff", INLINE_MAX * 2), ("icon", "image/png", 100)]); + d.title = "No photos".into(); + for o in [SaveOptions::default(), SaveOptions { pdf: Some(vec![b'%'; INLINE_MAX]), ..SaveOptions::default() }] { + let plain = save_with(&d, &o).unwrap(); + assert_eq!(save_with(&d, &SaveOptions { compress: true, ..o }).unwrap(), gzip(&plain)); + } +} + +#[test] +fn files_compressed_the_old_way_still_open() { + let d = mixed(); + let old = gzip(&save_with(&d, &SaveOptions::default()).unwrap()); + let back = load(&old).unwrap(); + assert_eq!(back.images.len(), d.images.len()); + for (k, b) in &d.images { + assert_eq!(back.images[k].bytes, b.bytes, "{k}"); + } + assert_eq!(back.node_count(), d.node_count()); +} + +#[test] +fn save_to_writes_what_save_with_returns() { + let d = mixed(); + for compress in [false, true] { + let o = SaveOptions { compress, ..SaveOptions::default() }; + let mut streamed = vec![]; + save_to(&d, &o, &mut streamed).unwrap(); + assert_eq!(streamed, save_with(&d, &o).unwrap()); + let back = load(&streamed).unwrap(); + assert_eq!(back.images["photo"].bytes, d.images["photo"].bytes); + } + // A failing writer fails the save. + struct Full; + impl Write for Full { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("disk full")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + assert!(save_to(&d, &SaveOptions::default(), Full).is_err()); + assert!(save_to(&d, &SaveOptions { compress: true, ..SaveOptions::default() }, Full).is_err()); +} + +/// `plain` with its JSON changed by `edit` (the data after it kept). +fn with_json(plain: &[u8], edit: impl FnOnce(&mut serde_json::Value)) -> Vec { + let at = plain.windows(BLOB_MAGIC.len()).position(|w| w == BLOB_MAGIC).unwrap(); + let mut v: serde_json::Value = serde_json::from_slice(&plain[..at]).unwrap(); + edit(&mut v); + let mut out = serde_json::to_vec(&v).unwrap(); + out.extend_from_slice(&plain[at..]); + out +} + +#[test] +fn entries_in_any_order_or_shared_read_back() { + let d = doc_with(&[("a", "image/png", INLINE_MAX), ("b", "image/png", INLINE_MAX + 5)]); + let plain = save_with(&d, &SaveOptions::default()).unwrap(); + // Swap the entries' places (b's bytes listed first), and point a third image at a's bytes. + let file = with_json(&plain, |v| { + let (a, b) = (v["images"]["a"]["blob"].clone(), v["images"]["b"]["blob"].clone()); + v["images"]["a"]["blob"] = b.clone(); + v["images"]["b"]["blob"] = a.clone(); + v["images"]["c"] = serde_json::json!({"mime": "image/png", "blob": b}); + }); + for bytes in [file.clone(), gzip(&file)] { + let back = load(&bytes).unwrap(); + assert_eq!(back.images["a"].bytes, d.images["b"].bytes); + assert_eq!(back.images["b"].bytes, d.images["a"].bytes); + assert_eq!(back.images["c"].bytes, d.images["b"].bytes); + } +} + +#[test] +fn limits_apply_per_entry_and_to_all_entries() { + let d = doc_with(&[("a", "image/png", INLINE_MAX), ("b", "image/png", INLINE_MAX)]); + let plain = save_with(&d, &SaveOptions::default()).unwrap(); + // One entry past the limit: refused before anything is allocated. + let huge = with_json(&plain, |v| v["images"]["a"]["blob"] = serde_json::json!([0, MAX_BLOB + 1])); + assert!(matches!(load_file(&gzip(&huge)), Err(FormatError::TooBig))); + // Each within it, together past the limit for all. + let many = with_json(&plain, |v| { + for i in 0..5 { + v["images"][format!("x{i}")] = serde_json::json!({"mime": "image/png", "blob": [0, MAX_BLOB]}); + } + }); + assert!(matches!(load_file(&gzip(&many)), Err(FormatError::TooBig))); + // A length the data doesn't have: the image is damaged, nothing more. + let short = with_json(&plain, |v| v["images"]["a"]["blob"] = serde_json::json!([0, 1u64 << 20])); + assert!(matches!(load_file(&gzip(&short)), Err(FormatError::BadImage(_)))); + // Uncompressed files have no limits to apply (the file is all there is). + assert!(matches!(load_file(&huge), Err(FormatError::BadImage(_)))); +} + +#[test] +fn a_damaged_compressed_file_is_refused() { + let bytes = save_with(&mixed(), &SaveOptions { compress: true, ..SaveOptions::default() }).unwrap(); + // The data after the JSON cut short, then its checksum wrong. + assert!(load_file(&bytes[..bytes.len() - 4000]).is_err()); + let mut bad = bytes.clone(); + let n = bad.len(); + bad[n - 6] ^= 0xff; + assert!(load_file(&bad).is_err(), "the checksum is checked to the end"); +} + +#[test] +fn documents_read_whatever_their_key_order_and_keep_keys_they_dont_know() { + let d = mixed(); + let plain = save_with(&d, &SaveOptions::default()).unwrap(); + let file = with_json(&plain, |v| { + // serde_json's map keeps keys sorted: `type` is no longer first in the node kinds (a + // layer's `children`, `clip` and `color` come before it), and the document's fields come + // in another order. + v["document"]["aaa_from_a_newer_version"] = serde_json::json!({"x": [1, 2]}); + v["document"]["zzz_too"] = serde_json::json!("kept"); + }); + let text = String::from_utf8_lossy(&file); + assert!(!text.contains(r#"{"type":"layer""#) && text.contains(r#""type":"layer""#), "`type` isn't first"); + for bytes in [file.clone(), gzip(&file)] { + let back = load(&bytes).unwrap(); + assert_eq!(back.layers, load(&plain).unwrap().layers); + assert_eq!(back.extra["aaa_from_a_newer_version"]["x"], serde_json::json!([1, 2])); + assert_eq!(back.extra["zzz_too"], "kept"); + // Saved again, they are kept. + assert_eq!(load(&save_with(&back, &SaveOptions::default()).unwrap()).unwrap().extra["zzz_too"], "kept"); + } +} diff --git a/crates/ui-egui/src/background.rs b/crates/ui-egui/src/background.rs index a1940a7b..f0245387 100644 --- a/crates/ui-egui/src/background.rs +++ b/crates/ui-egui/src/background.rs @@ -16,6 +16,11 @@ use crate::VectorcraftApp; /// Writes a file; callable from any thread. pub type SharedWriteFn = std::sync::Arc Result<(), String> + Send + Sync>; +/// Writes the file at a path with what the filler it is given writes into it (atomically, as +/// `vectorcraft_format::write_atomic_with`); callable from any thread. +pub type SharedStreamFn = + std::sync::Arc std::io::Result<()>) -> Result<(), String> + Send + Sync>; + /// A writer for the work: the host's file writer (or browser download) on the UI thread, the /// shared one on the worker. pub type Writer<'a> = &'a mut dyn FnMut(&str, &[u8]) -> Result<(), String>; diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 60c056bd..3b5b6947 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -578,10 +578,12 @@ pub fn status_bar(app: &mut VectorcraftApp, ui: &mut Ui) { egui::RichText::new(format!("X: {} Y: {}", u.readout(p.x), u.readout(p.y))).font(theme::mono(10.5)).color(t.text_dim), ); } - // Background saves and exports in progress, else the last message. - if let Some(job) = app.background.jobs.first() { - let more = app.background.jobs.len() - 1; - let label = crate::i18n::msg(&job.label); + // Background saves, exports and opens in progress, else the last message. + let busy: Vec<&str> = + app.background.jobs.iter().map(|j| j.label.as_str()).chain(app.opening.pending.iter().map(|p| p.label.as_str())).collect(); + if let Some(job) = busy.first() { + let more = busy.len() - 1; + let label = crate::i18n::msg(job); let text = if more > 0 { format!("{label}… (+{more})") } else { format!("{label}…") }; ui.add(egui::Spinner::new().size(12.0).color(t.accent)); ui.add(egui::Label::new(egui::RichText::new(text).size(11.0).color(t.text)).truncate()); diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index d10c6bcf..4d223143 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -104,6 +104,8 @@ pub fn inspect(app: &VectorcraftApp, ctx: &egui::Context) -> Value { "nativeMenuBar": app.services.native_menu.is_some(), // Saves and exports still being written in the background (Background Save / Export). "background": app.background.jobs.iter().map(|j| j.label.as_str()).collect::>(), + // Documents still being read off the UI thread (File › Open, Open Recent): their paths. + "opening": app.opening.pending.iter().map(|p| p.path.as_str()).collect::>(), // A native file dialog open off the UI thread (`open`, `place`, `save`, `folder`), waiting // for the person at the computer: what asked for it runs again when it answers. "fileDialog": app.picks.waiting_kind(), diff --git a/crates/ui-egui/src/i18n/cs.tsv b/crates/ui-egui/src/i18n/cs.tsv index 8d698932..17fa3e96 100644 --- a/crates/ui-egui/src/i18n/cs.tsv +++ b/crates/ui-egui/src/i18n/cs.tsv @@ -4493,3 +4493,6 @@ diagnostics ui rozhraní @plural {n} commit|{n} commits {n} commit|{n} commity|{n} commitů gradient-kind Radial Kruhový mark-style Japanese Japonské +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) komprimovaný soubor se rozbalí na více, než VectorCraft přečte ({_1} MB dokumentu, {_2} MB na obrázek) +@msg the document is too large to save compressed ({0}): save it with Use Compression off dokument je příliš velký pro komprimované uložení ({0}): uložte ho s vypnutou volbou Použít kompresi +@msg the file stopped being read (please report this bug) čtení souboru se přerušilo (nahlaste prosím tuto chybu) diff --git a/crates/ui-egui/src/i18n/de.tsv b/crates/ui-egui/src/i18n/de.tsv index ffa02707..e0aad1bf 100644 --- a/crates/ui-egui/src/i18n/de.tsv +++ b/crates/ui-egui/src/i18n/de.tsv @@ -4144,3 +4144,6 @@ theme Appearance Erscheinungsbild Film Grain… Filmkörnung… Grain Körnung Highlight Area Lichterbereich +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) die komprimierte Datei entpackt sich auf mehr, als VectorCraft liest ({_1} MB Dokument, {_2} MB pro Bild) +@msg the document is too large to save compressed ({0}): save it with Use Compression off das Dokument ist zu groß, um komprimiert gespeichert zu werden ({0}): speichern Sie es mit deaktivierter Option „Komprimierung verwenden“ +@msg the file stopped being read (please report this bug) das Lesen der Datei wurde abgebrochen (bitte melden Sie diesen Fehler) diff --git a/crates/ui-egui/src/i18n/es.tsv b/crates/ui-egui/src/i18n/es.tsv index 52683d4d..7399ca1d 100644 --- a/crates/ui-egui/src/i18n/es.tsv +++ b/crates/ui-egui/src/i18n/es.tsv @@ -4187,3 +4187,6 @@ lighting Light Luz Artistic Artístico Film Grain… Granulado de película… Highlight Area Área de iluminaciones +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) el archivo comprimido ocupa al descomprimirse más de lo que VectorCraft lee ({_1} MB de documento, {_2} MB por imagen) +@msg the document is too large to save compressed ({0}): save it with Use Compression off el documento es demasiado grande para guardarlo comprimido ({0}): guárdelo con Usar compresión desactivado +@msg the file stopped being read (please report this bug) se dejó de leer el archivo (informe de este error) diff --git a/crates/ui-egui/src/i18n/fr.tsv b/crates/ui-egui/src/i18n/fr.tsv index ce3fc5da..2dd06234 100644 --- a/crates/ui-egui/src/i18n/fr.tsv +++ b/crates/ui-egui/src/i18n/fr.tsv @@ -4197,3 +4197,6 @@ lighting Light Lumière Artistic Artistique Film Grain… Grain photographique… Highlight Area Zone des tons clairs +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) le fichier compressé dépasse une fois décompressé ce que VectorCraft lit ({_1} Mo de document, {_2} Mo par image) +@msg the document is too large to save compressed ({0}): save it with Use Compression off le document est trop volumineux pour être enregistré compressé ({0}) : enregistrez-le avec Utiliser la compression désactivé +@msg the file stopped being read (please report this bug) la lecture du fichier s’est interrompue (veuillez signaler ce bogue) diff --git a/crates/ui-egui/src/i18n/hu.tsv b/crates/ui-egui/src/i18n/hu.tsv index 594a70e1..b4597c0a 100644 --- a/crates/ui-egui/src/i18n/hu.tsv +++ b/crates/ui-egui/src/i18n/hu.tsv @@ -4587,3 +4587,6 @@ theme Appearance Kinézet @msg {path}: {e} {path}: {e} @msg {status} ({e}) {status} ({e}) @msg {tool} isn't in a toolbar group of several tools az eszköz nincs többeszközös eszköztár-csoportban: {tool} +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) a tömörített fájl kicsomagolva nagyobb lenne, mint amennyit a VectorCraft beolvas ({_1} MB dokumentum, képenként {_2} MB) +@msg the document is too large to save compressed ({0}): save it with Use Compression off a dokumentum túl nagy a tömörített mentéshez ({0}): mentse a Tömörítés használata beállítás kikapcsolásával +@msg the file stopped being read (please report this bug) a fájl beolvasása megszakadt (kérjük, jelentse ezt a hibát) diff --git a/crates/ui-egui/src/i18n/it.tsv b/crates/ui-egui/src/i18n/it.tsv index d234ac57..dbcbaf32 100644 --- a/crates/ui-egui/src/i18n/it.tsv +++ b/crates/ui-egui/src/i18n/it.tsv @@ -4189,3 +4189,6 @@ lighting Light Luce Artistic Artistico Film Grain… Grana della pellicola… Highlight Area Area luci +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) il file compresso, una volta decompresso, supera ciò che VectorCraft legge ({_1} MB di documento, {_2} MB per immagine) +@msg the document is too large to save compressed ({0}): save it with Use Compression off il documento è troppo grande per essere salvato compresso ({0}): salvalo con Usa compressione disattivato +@msg the file stopped being read (please report this bug) la lettura del file si è interrotta (segnala questo errore) diff --git a/crates/ui-egui/src/i18n/ja.tsv b/crates/ui-egui/src/i18n/ja.tsv index 534dd4ec..b3080a7f 100644 --- a/crates/ui-egui/src/i18n/ja.tsv +++ b/crates/ui-egui/src/i18n/ja.tsv @@ -4183,3 +4183,6 @@ lighting Light ライト Artistic アーティスティック Film Grain… フィルム粒子… Highlight Area ハイライト領域 +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) 圧縮ファイルを展開すると VectorCraft が読み込める量を超えます (ドキュメント {_1} MB、画像 1 枚あたり {_2} MB) +@msg the document is too large to save compressed ({0}): save it with Use Compression off ドキュメントが大きすぎて圧縮保存できません ({0}): 「圧縮を使用」をオフにして保存してください +@msg the file stopped being read (please report this bug) ファイルの読み込みが中断されました (このバグを報告してください) diff --git a/crates/ui-egui/src/i18n/pt-br.tsv b/crates/ui-egui/src/i18n/pt-br.tsv index 9f8a97af..3b96c208 100644 --- a/crates/ui-egui/src/i18n/pt-br.tsv +++ b/crates/ui-egui/src/i18n/pt-br.tsv @@ -3672,3 +3672,6 @@ axis Both Ambos Film Grain… Granulação de filme… Grain Grão Highlight Area Área de realces +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) o arquivo compactado se expande para mais do que o VectorCraft lê ({_1} MB de documento, {_2} MB por imagem) +@msg the document is too large to save compressed ({0}): save it with Use Compression off o documento é grande demais para ser salvo compactado ({0}): salve-o com Usar Compressão desativado +@msg the file stopped being read (please report this bug) a leitura do arquivo foi interrompida (informe este erro) diff --git a/crates/ui-egui/src/i18n/ru.tsv b/crates/ui-egui/src/i18n/ru.tsv index 48922b74..40c12825 100644 --- a/crates/ui-egui/src/i18n/ru.tsv +++ b/crates/ui-egui/src/i18n/ru.tsv @@ -4185,3 +4185,6 @@ lighting Light Свет Artistic Художественные Film Grain… Зернистость плёнки… Highlight Area Область светов +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) сжатый файл распаковывается в объём больше, чем читает VectorCraft ({_1} МБ документа, {_2} МБ на изображение) +@msg the document is too large to save compressed ({0}): save it with Use Compression off документ слишком велик для сохранения со сжатием ({0}): сохраните его с выключенным параметром «Использовать сжатие» +@msg the file stopped being read (please report this bug) чтение файла прервалось (пожалуйста, сообщите об этой ошибке) diff --git a/crates/ui-egui/src/i18n/uk.tsv b/crates/ui-egui/src/i18n/uk.tsv index c37ee4fd..e55e57db 100644 --- a/crates/ui-egui/src/i18n/uk.tsv +++ b/crates/ui-egui/src/i18n/uk.tsv @@ -4169,3 +4169,6 @@ lighting Light Світло Artistic Художні Film Grain… Зернистість плівки… Highlight Area Область світлих тонів +@msg the compressed file unpacks to more than VectorCraft reads ({_1} MB of document, {_2} MB per image) стислий файл розпаковується в обсяг більший, ніж читає VectorCraft ({_1} МБ документа, {_2} МБ на зображення) +@msg the document is too large to save compressed ({0}): save it with Use Compression off документ завеликий для збереження зі стиском ({0}): збережіть його з вимкненим параметром «Використовувати стиск» +@msg the file stopped being read (please report this bug) читання файлу перервалося (будь ласка, повідомте про цю помилку) diff --git a/crates/ui-egui/src/io.rs b/crates/ui-egui/src/io.rs index 21f7f373..331e96a6 100644 --- a/crates/ui-egui/src/io.rs +++ b/crates/ui-egui/src/io.rs @@ -104,12 +104,18 @@ pub fn opens_as(name: &str) -> Option { /// `document.open`'s result. pub fn open_document(app: &mut VectorcraftApp, name: &str, bytes: &[u8], path: Option, p: &Value) -> Result { let r = fileio::open_bytes_with(&mut app.session, name, bytes, path, p).map_err(|e| e.to_string())?; + opened(app, &r); + Ok(r) +} + +/// A document just opened (`r`: `document.open`'s result): the views follow, the status bar says +/// what didn't come in as it was, and missing links are asked about. +pub(crate) fn opened(app: &mut VectorcraftApp, r: &Value) { app.sync_views(); if let Some(w) = r["warnings"].as_array().filter(|w| !w.is_empty()) { app.status(format!("Opened with {} note(s): {}", w.len(), w[0].as_str().unwrap_or_default())); } - crate::dialogs::missing_links::after_open(app, &r); - Ok(r) + crate::dialogs::missing_links::after_open(app, r); } /// A path from the open dialog, or "cancelled". @@ -148,9 +154,12 @@ pub fn report_open_error(app: &mut VectorcraftApp, name: &str, e: &str) { dialogs::confirm::tell(app, &message, &crate::i18n::message(crate::i18n::current(), e)); } -/// [`open_path`] for a file the user asked for, a failure reported ([`report_open_error`]). +/// [`open_path`] for a file the user asked for, a failure reported ([`report_open_error`]). On +/// the desktop a document is decoded off the UI thread ([`crate::opening`]): the result is then +/// `{background: true, status}`, and the document opens in its tab (or the failure is reported) +/// once it is ready. pub fn open_reporting(app: &mut VectorcraftApp, path: &str) -> Result { - let r = open_path(app, path); + let r = open_file(app, path, crate::opening::available()); if let Err(e) = &r { report_open_error(app, &fileio::file_name(path), e); } @@ -163,7 +172,27 @@ fn read(app: &VectorcraftApp, path: &str) -> Result, String> { /// Open the file at `path` (see [`open_bytes`]) → `document.open`'s result, or null. pub fn open_path(app: &mut VectorcraftApp, path: &str) -> Result { - let bytes = read(app, path)?; + open_file(app, path, false) +} + +/// [`open_path`]; with `background`, a document (not a file that installs, imports or asks +/// first) is decoded off the UI thread ([`crate::opening::start`]). +fn open_file(app: &mut VectorcraftApp, path: &str, background: bool) -> Result { + let mut bytes = read(app, path)?; + if background && opens_as(path).is_none() { + // A PDF with several pages or a password, or a DXF drawing, asks first, as at once. + if crate::dialogs::import_pdf::offer(app, path, &bytes, Some(path.to_string()), None) + || crate::dialogs::dxf_import::offer(app, path, &bytes, Some(path.to_string()), None) + { + note_recent(app, path); + return Ok(Value::Null); + } + match crate::opening::start(app, path, bytes) { + Ok(r) => return Ok(r), + // No thread: at once. + Err(back) => bytes = back, + } + } let r = open_bytes(app, path, &bytes, Some(path.to_string()))?; note_recent(app, path); Ok(r) @@ -459,7 +488,15 @@ fn write_plan(app: &mut VectorcraftApp, plan: SavePlan) -> Result let enabled = app.session.prefs.background_save && target.is_some(); // The work writes the file; the copy makes it the document's own afterwards. let done = job.clone(); - let work = move |write: Writer| job.write(|p, b| write(p, b).map_err(EngineError::Other)).map_err(|e| e.to_string()); + // A native file streams into the file as it is encoded (desktop; not to a download). + let stream = app.services.stream_shared.clone().filter(|_| job.streams() && !is_web(app)); + let work = move |write: Writer| { + match stream { + Some(stream) => job.write_streamed(&mut |p, fill| stream(p, fill).map_err(EngineError::Other)), + None => job.write(|p, b| write(p, b).map_err(EngineError::Other)), + } + .map_err(|e| e.to_string()) + }; let then = move |app: &mut VectorcraftApp, r: Result| { let r = r?; // The document may have closed meanwhile: the file is written all the same. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f2a5cbc0..c69442d2 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -36,6 +36,7 @@ pub mod icons; pub mod io; pub mod menus; pub mod native_menu; +pub mod opening; pub mod palette; mod panel_docking; pub mod panels; @@ -231,6 +232,10 @@ pub struct Services { /// Write a file from any thread (desktop): lets Background Save and Export write off the UI /// thread ([`background`]). Without it they run at once. pub write_shared: Option, + /// Write a file from any thread with what a filler writes into it (desktop): native saves + /// stream into the file as they are encoded instead of being held whole in memory first. + /// Without it they are encoded, then written with `write` / `write_shared`. + pub stream_shared: Option, /// Open a file (or a folder) in the system's default app for it (desktop: Edit Original, Show /// Package). Without it (the web) those answer with an error. pub open_file: Option, @@ -386,6 +391,8 @@ pub struct VectorcraftApp { pub(crate) paste_chord: shortcuts::PasteChord, /// Saves and exports running in the background (Preferences → File Handling). pub background: background::Background, + /// Documents being read off the UI thread ([`opening`]). + pub opening: opening::Opening, /// Data Recovery's timer and startup question ([`recovery`]). pub recovery: recovery::Timer, /// Modifiers the keyboard holds (from the host's input), given back after synthetic input. @@ -482,6 +489,7 @@ impl VectorcraftApp { font_check: None, paste_chord: Default::default(), background: Default::default(), + opening: Default::default(), recovery: Default::default(), host_modifiers: Default::default(), synthetic_modifiers: false, @@ -516,6 +524,11 @@ impl VectorcraftApp { if let Some(write) = s.write_shared.take() { s.write_shared = Some(Arc::new(move |p: &str, b: &[u8]| file_access::check_write(p).and_then(|()| write(p, b)))); } + if let Some(stream) = s.stream_shared.take() { + s.stream_shared = Some(Arc::new(move |p: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>| { + file_access::check_write(p).and_then(|()| stream(p, fill)) + })); + } // Edit Original and Show in Folder hand a file to another app: only one automation may read. for service in [&mut s.open_file, &mut s.reveal] { if let Some(mut open) = service.take() { @@ -982,9 +995,10 @@ impl VectorcraftApp { self.poll_font_check(ctx); picks::poll(self, ctx); background::poll(self); + opening::poll(self); #[cfg(not(target_arch = "wasm32"))] self.tick_link_updates(ctx, now); - if !self.background.jobs.is_empty() { + if !self.background.jobs.is_empty() || !self.opening.pending.is_empty() { // Keep the status bar's progress moving and pick the result up when it arrives. ctx.request_repaint_after(std::time::Duration::from_millis(100)); } diff --git a/crates/ui-egui/src/opening.rs b/crates/ui-egui/src/opening.rs new file mode 100644 index 00000000..b978d11e --- /dev/null +++ b/crates/ui-egui/src/opening.rs @@ -0,0 +1,130 @@ +//! Opening documents off the UI thread (desktop): a file the user asks to open (File › Open, Open +//! Recent, the Home screen, files handed to the app) is decoded on a thread of its own while the +//! status bar shows it, and opens in a new tab once it is ready, as it would at once: the same +//! notes, recent files and errors. On wasm (no threads) and in unit tests it opens at once +//! ([`available`]). + +use std::sync::Arc; +use std::sync::mpsc::{Receiver, TryRecvError, channel}; + +use serde_json::{Value, json}; +use vectorcraft_engine::cmd::fileio::{self, ReadFile}; +use vectorcraft_engine::file_access::ScopeHandle; + +use crate::VectorcraftApp; + +/// A file being read. +pub struct Pending { + /// Where it is. + pub path: String, + /// What the status bar shows ("Opening Poster.vectorcraft"). + pub label: String, + done: Receiver>, +} + +/// The files being read. +#[derive(Default)] +pub struct Opening { + pub pending: Vec, +} + +/// Does opening happen off the UI thread here? Not on wasm (no threads), in unit tests (they look +/// at the document right after asking for it), or with `VECTORCRAFT_SYNC_OPEN` set. +pub fn available() -> bool { + !(cfg!(target_arch = "wasm32") || cfg!(test) || std::env::var_os("VECTORCRAFT_SYNC_OPEN").is_some()) +} + +/// Decode `bytes`, the file at `path`, on a thread of its own → `{background: true, status}`, or +/// the bytes back when no thread could start (the caller opens the file at once). +pub fn start(app: &mut VectorcraftApp, path: &str, bytes: Vec) -> Result> { + let (tx, done) = channel(); + let bytes = Arc::new(bytes); + let (name, file) = (path.to_string(), bytes.clone()); + // The automation roots in force here are in force there. + let scope = ScopeHandle::capture(); + let spawned = std::thread::Builder::new().name("vectorcraft-open".into()).spawn(move || { + let r = scope + .run(|| vectorcraft_engine::guard::catch_panic(|| fileio::read_for_open(&name, &file, &Value::Null).map_err(|e| e.to_string()))) + .unwrap_or_else(|msg| Err(format!("internal error: {msg} (please report this bug)"))); + // The file's bytes go before the document is handed over. + drop(file); + // The app may have quit meanwhile: nothing to hand it to. + let _ = tx.send(r); + }); + if spawned.is_err() { + return Err(Arc::try_unwrap(bytes).unwrap_or_else(|b| b.to_vec())); + } + let label = format!("Opening {}", fileio::file_name(path)); + app.opening.pending.push(Pending { path: path.to_string(), label: label.clone(), done }); + let status = format!("{label}…"); + app.status(status.clone()); + Ok(json!({ "background": true, "status": status })) +} + +/// Open the files that are ready (each frame). +pub fn poll(app: &mut VectorcraftApp) { + let mut i = 0; + while let Some(p) = app.opening.pending.get(i) { + let r = match p.done.try_recv() { + Ok(r) => r, + Err(TryRecvError::Empty) => { + i += 1; + continue; + } + Err(TryRecvError::Disconnected) => Err("the file stopped being read (please report this bug)".to_string()), + }; + let p = app.opening.pending.remove(i); + finish(app, &p.path, r); + } +} + +/// Open a file read off the UI thread, as [`crate::io::open_reporting`] opens one read at once. +fn finish(app: &mut VectorcraftApp, path: &str, r: Result) { + let opened = r.and_then(|file| fileio::open_read(&mut app.session, file, Some(path.to_string())).map_err(|e| e.to_string())); + match opened { + Ok(r) => { + crate::io::opened(app, &r); + crate::io::note_recent(app, path); + } + Err(e) => crate::io::report_open_error(app, &fileio::file_name(path), &e), + } +} + +#[cfg(test)] +mod tests { + use vectorcraft_engine::Session; + + use super::*; + use crate::Services; + + #[test] + fn a_file_read_off_the_ui_thread_opens_as_one_read_at_once() { + let mut app = VectorcraftApp::new(Session::new(), Services::default()); + let mut d = vectorcraft_doc::Document::new(120.0, 80.0); + d.title = "Poster".into(); + let bytes = vectorcraft_format::save(&d, false); + let before = app.session.documents().len(); + let r = start(&mut app, "/art/Poster.vectorcraft", bytes).unwrap(); + assert_eq!(r["background"], true); + assert_eq!(app.ui.status, "Opening Poster.vectorcraft…"); + let t0 = std::time::Instant::now(); + while !app.opening.pending.is_empty() && t0.elapsed().as_secs() < 30 { + poll(&mut app); + std::thread::yield_now(); + } + assert_eq!(app.session.documents().len(), before + 1); + let st = app.session.doc().unwrap(); + assert_eq!((st.path.as_deref(), st.doc.artboards[0].rect.width()), (Some("/art/Poster.vectorcraft"), 120.0)); + assert_eq!(app.ui.recent_files.first().map(String::as_str), Some("/art/Poster.vectorcraft")); + // A file that can't be read: reported, nothing opened, not a recent file. + start(&mut app, "/art/Broken.vectorcraft", b"{\"format\":\"vectorcraft\"".to_vec()).unwrap(); + let t0 = std::time::Instant::now(); + while !app.opening.pending.is_empty() && t0.elapsed().as_secs() < 30 { + poll(&mut app); + std::thread::yield_now(); + } + assert_eq!(app.session.documents().len(), before + 1); + assert!(app.ui.status.starts_with("Couldn't open Broken.vectorcraft"), "{}", app.ui.status); + assert_ne!(app.ui.recent_files.first().map(String::as_str), Some("/art/Broken.vectorcraft")); + } +} diff --git a/crates/ui-egui/src/tests_save.rs b/crates/ui-egui/src/tests_save.rs index 8a39c50a..cf200fe5 100644 --- a/crates/ui-egui/src/tests_save.rs +++ b/crates/ui-egui/src/tests_save.rs @@ -205,3 +205,27 @@ fn replacing_a_file_that_opening_left_things_out_of_asks_first() { assert!(!app.session.active().unwrap().is_dirty()); let _ = std::fs::remove_dir_all(d); } + +#[test] +fn native_saves_stream_into_the_file() { + let d = dir("stream"); + let streamed = std::sync::Arc::new(std::sync::Mutex::new(vec![])); + let log = streamed.clone(); + let mut app = app(); + app.services.stream_shared = Some(std::sync::Arc::new(move |p: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>| { + log.lock().unwrap().push(p.to_string()); + let mut f = std::fs::File::create(p).map_err(|e| e.to_string())?; + fill(&mut f).map_err(|e| e.to_string()) + })); + app.run("file.new", json!({"width": 100, "height": 100})).unwrap(); + rect(&mut app); + let native = d.join("a.vectorcraft").to_string_lossy().to_string(); + let r = app.run("file.saveAs", json!({"path": native, "compress": true})).unwrap(); + let bytes = std::fs::read(&native).unwrap(); + assert_eq!(r["bytes"], bytes.len(), "{r}"); + assert!(vectorcraft_format::is_compressed(&bytes)); + assert_eq!(vectorcraft_format::load(&bytes).unwrap().node_count(), objects(&app)); + assert!(!app.session.doc().unwrap().is_dirty(), "the file is the document's own"); + assert_eq!(*streamed.lock().unwrap(), [native]); + let _ = std::fs::remove_dir_all(&d); +}