From c51e51fe7bea9306633f3d0dbb80bee0c5596cd6 Mon Sep 17 00:00:00 2001 From: harrisonoconnorhover <130776285+harrisonoconnorhover@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:19:36 -0400 Subject: [PATCH] #92: import XLSB worksheet data and saved formula results --- Cargo.lock | 1 + README.md | 3 +- apps/gridcraft-cli/src/main.rs | 8 + apps/gridcraft-web/src/web.rs | 2 +- apps/gridcraft/src/main.rs | 6 +- crates/engine/Cargo.toml | 3 + crates/engine/src/cmd/file.rs | 20 +- crates/engine/src/io.rs | 9 +- crates/engine/src/tests.rs | 126 ++++++ crates/mcp/src/tools.rs | 2 +- crates/xlsx/src/lib.rs | 9 +- crates/xlsx/src/package.rs | 14 +- crates/xlsx/src/tests/mod.rs | 1 + crates/xlsx/src/tests/xlsb_tests.rs | 306 +++++++++++++ crates/xlsx/src/xlsb.rs | 667 ++++++++++++++++++++++++++++ docs/cli.md | 4 + docs/xlsb-import.md | 22 + 17 files changed, 1187 insertions(+), 16 deletions(-) create mode 100644 crates/xlsx/src/tests/xlsb_tests.rs create mode 100644 crates/xlsx/src/xlsb.rs create mode 100644 docs/xlsb-import.md diff --git a/Cargo.lock b/Cargo.lock index e3a4144..d742e69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1466,6 +1466,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.21", + "zip", ] [[package]] diff --git a/README.md b/README.md index f5aa689..b27fbfb 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ - **Compatible.** XLSX is the native format: styles, themes, formulas (including dynamic arrays), tables, conditional formats, validation, comments, hyperlinks, charts, pictures, sparklines, PivotTables and print settings round-trip. CSV and TSV too. + [XLSB worksheet data can be imported](docs/xlsb-import.md), using saved values instead of formulas. - **Capable.** A dependency-graph calculation engine with dynamic arrays and spilling, 500+ worksheet functions plus LET, LAMBDA, MAP, REDUCE, SCAN, BYROW, BYCOL and MAKEARRAY, structured table references, Excel's full number-format language, sort and AutoFilter, @@ -141,7 +142,7 @@ egui frontend is a separate crate, so the UI can be swapped without touching the |---|---| | L0 | `core` (addresses, values, errors, dates, input parsing) · `numfmt` (number format codes) | | L1 | `formula` (lexer, parser, printer, reference adjustment) · `functions` (worksheet function library) | -| L2 | `model` (workbook, sheets, copy-on-write cells, styles, tables, charts…) · `calc` (dependency graph, evaluator, dynamic arrays) · `xlsx` (XLSX and CSV) | +| L2 | `model` (workbook, sheets, copy-on-write cells, styles, tables, charts…) · `calc` (dependency graph, evaluator, dynamic arrays) · `xlsx` (XLSX, CSV and XLSB data import) | | L3 | `chart` (toolkit-free chart layout and rendering) | | L5 | `engine` (session, commands, history, clipboard, fill, sort, filter, file I/O) | | L6 | `ui-egui` (Excel-style UI, control channel) · `mcp` (MCP server) | diff --git a/apps/gridcraft-cli/src/main.rs b/apps/gridcraft-cli/src/main.rs index 561162f..c97b1d6 100644 --- a/apps/gridcraft-cli/src/main.rs +++ b/apps/gridcraft-cli/src/main.rs @@ -147,6 +147,14 @@ impl Args { fn exec(s: &mut Session, id: &str, params: Value) -> Result { let r = s.execute(id, params).map_err(|e| e.to_string()); s.take_ui_requests(); + if id == "file.open" + && let Ok(value) = &r + && let Some(warnings) = value.get("warnings").and_then(Value::as_array) + { + for warning in warnings.iter().filter_map(Value::as_str) { + eprintln!("Warning: {warning}"); + } + } r } diff --git a/apps/gridcraft-web/src/web.rs b/apps/gridcraft-web/src/web.rs index c06d890..b9da27c 100644 --- a/apps/gridcraft-web/src/web.rs +++ b/apps/gridcraft-web/src/web.rs @@ -98,7 +98,7 @@ fn services(inbox: Inbox, ctx: egui::Context) -> Services { let ctx = ctx.clone(); wasm_bindgen_futures::spawn_local(async move { let Some(file) = - rfd::AsyncFileDialog::new().add_filter("Spreadsheets", &["xlsx", "xlsm", "csv", "tsv", "txt", "json"]).pick_file().await + rfd::AsyncFileDialog::new().add_filter("Spreadsheets", &["xlsx", "xlsm", "xlsb", "csv", "tsv", "txt", "json"]).pick_file().await else { return; }; diff --git a/apps/gridcraft/src/main.rs b/apps/gridcraft/src/main.rs index 6acaf51..901a6c6 100644 --- a/apps/gridcraft/src/main.rs +++ b/apps/gridcraft/src/main.rs @@ -4,7 +4,7 @@ //! //! `--control ` (or `GRIDCRAFT_CONTROL_PORT`) starts a localhost JSON-lines control //! server: `{"id":1,"method":"ui.inspect","params":{}}` → `{"id":1,"ok":true,"result":…}`. -//! See `gridcraft_ui_egui::control` and `docs/control-protocol.md` for the methods. +//! See `gridcraft_ui_egui::control` and `docs/control-protocol.md` for the methxlsb. #![cfg_attr(all(target_os = "windows", not(debug_assertions)), windows_subsystem = "windows")] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -96,8 +96,9 @@ fn services() -> Services { Services { pick_open: Some(Box::new(|| { rfd::FileDialog::new() - .add_filter("Spreadsheets", &["xlsx", "xlsm", "csv", "tsv", "txt", "json"]) + .add_filter("Spreadsheets", &["xlsx", "xlsm", "xlsb", "csv", "tsv", "txt", "json"]) .add_filter("Excel Workbook", &["xlsx", "xlsm"]) + .add_filter("Excel Binary Workbook (data import)", &["xlsb"]) .add_filter("CSV", &["csv"]) .pick_file() .and_then(|p| p.to_str().map(str::to_string)) @@ -172,6 +173,7 @@ fn main() -> eframe::Result<()> { session.new_workbook(); } let mut app = SheetApp::new(session, services()); + app.after_engine(); // Show warnings from files opened on the command line. let control_port = control_port; load_prefs(&mut app); let mut viewport = egui::ViewportBuilder::default() diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 53e2350..b559b83 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -23,3 +23,6 @@ gridcraft-xlsx = { workspace = true } gridcraft-chart = { workspace = true } gridcraft-pdf = { workspace = true } image = { workspace = true } + +[dev-dependencies] +zip = { workspace = true } diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index 37a58dd..a2448d1 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -8,7 +8,7 @@ use crate::DocState; pub fn specs() -> Vec { vec![ cmd!(noundo "file.new", "New Workbook", ["File"], Some("Cmd+N"), "{sample?: \"budget\"|\"sales\"|\"grades\"}", always, new_workbook), - cmd!(noundo "file.open", "Open…", ["File"], Some("Cmd+O"), "{path} | {name, base64} (xlsx, xlsm, csv, tsv, txt, json)", always, open), + cmd!(noundo "file.open", "Open…", ["File"], Some("Cmd+O"), "{path} | {name, base64} (xlsx, xlsm, xlsb data import, csv, tsv, txt, json)", always, open), cmd!(noundo "file.save", "Save", ["File"], Some("Cmd+S"), "{path?} (xlsx by default; .csv/.tsv/.json/.html by extension)", has_doc, save), cmd!(noundo "file.saveAs", "Save As…", ["File"], Some("Cmd+Shift+S"), "{path}", has_doc, save_as), cmd!(query "file.saveBytes", "Encode Workbook", [], None, "{format?: xlsx|csv|tsv|json|html} → {base64}", has_doc, save_bytes), @@ -47,8 +47,15 @@ fn open(s: &mut Session, p: &Json) -> Result { return ok(); }; let (wb, warnings) = crate::io::open_bytes(&name, &bytes)?; - let title = std::path::Path::new(&name).file_name().and_then(|n| n.to_str()).unwrap_or("Book").to_string(); - let path = str_param(p, "path").map(str::to_string).filter(|p| crate::io::FileKind::from_path(p) == Some(crate::io::FileKind::Xlsx) || true); + let imported = + crate::io::FileKind::from_path(&name) == Some(crate::io::FileKind::Xlsb) || gridcraft_xlsx::sniff(&bytes) == gridcraft_xlsx::Format::Xlsb; + let title = if imported { + format!("{} (imported).xlsx", std::path::Path::new(&name).file_stem().and_then(|n| n.to_str()).unwrap_or("Book")) + } else { + std::path::Path::new(&name).file_name().and_then(|n| n.to_str()).unwrap_or("Book").to_string() + }; + // An import has no save target: ordinary Save must ask for a new destination. + let path = str_param(p, "path").filter(|_| !imported).map(str::to_string); // Replace an untouched blank Book1 like Excel does. if s.documents().len() == 1 && s.active().is_some_and(|d| d.path.is_none() && !d.is_dirty() && d.undo.is_empty() && d.wb.sheets.iter().all(|sh| sh.cells.is_empty())) @@ -56,6 +63,13 @@ fn open(s: &mut Session, p: &Json) -> Result { s.close_document(0); } let i = s.add_document(DocState::new(wb, path, title)); + if !warnings.is_empty() { + let mut message = warnings.iter().take(5).cloned().collect::>().join("\n\n"); + if warnings.len() > 5 { + message.push_str(&format!("\n\n{} additional import warnings.", warnings.len() - 5)); + } + s.ui_requests.push(crate::UiRequest::Message(message)); + } let d = s.doc()?; Ok( json!({"index": i, "title": d.display_title(), "sheets": d.wb.sheets.iter().map(|s| s.name.clone()).collect::>(), "warnings": warnings}), diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index aef3681..da5d45c 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -1,4 +1,4 @@ -//! Opening and saving files: XLSX (the native format), CSV/TSV, JSON (debug), HTML export. +//! Opening and saving files: XLSX, XLSB data import, CSV/TSV, JSON (debug), HTML export. use std::fmt::Write as _; @@ -11,6 +11,7 @@ use crate::{EngineError, Result}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FileKind { Xlsx, + Xlsb, Csv, Tsv, Json, @@ -22,6 +23,7 @@ impl FileKind { let ext = std::path::Path::new(path).extension()?.to_str()?.to_ascii_lowercase(); Some(match ext.as_str() { "xlsx" | "xlsm" | "xltx" | "xltm" => FileKind::Xlsx, + "xlsb" => FileKind::Xlsb, "csv" => FileKind::Csv, "tsv" | "tab" | "txt" => FileKind::Tsv, "json" | "scjson" => FileKind::Json, @@ -36,6 +38,10 @@ impl FileKind { pub fn open_bytes(name: &str, bytes: &[u8]) -> Result<(Workbook, Vec)> { let kind = FileKind::from_path(name); let sniffed = gridcraft_xlsx::sniff(bytes); + if sniffed == gridcraft_xlsx::Format::Xlsb || kind == Some(FileKind::Xlsb) { + let (wb, report) = gridcraft_xlsx::read_xlsb(bytes).map_err(|e| EngineError::Other(format!("We can't import '{name}': {e}")))?; + return Ok((wb, report.warnings)); + } if sniffed == gridcraft_xlsx::Format::Xlsx || kind == Some(FileKind::Xlsx) { let (wb, report) = gridcraft_xlsx::read_xlsx(bytes).map_err(|e| EngineError::Other(format!("We can't open '{name}': {e}")))?; return Ok((wb, report.warnings)); @@ -81,6 +87,7 @@ pub fn save_bytes(wb: &Workbook, path: &str) -> Result> { let sheet = wb.active_sheet; match FileKind::from_path(path).unwrap_or(FileKind::Xlsx) { FileKind::Xlsx => gridcraft_xlsx::write_xlsx(wb).map_err(|e| EngineError::Other(e.to_string())), + FileKind::Xlsb => Err(EngineError::Other("XLSB is supported for data import only. Save as .xlsx or another supported export format.".into())), FileKind::Csv => Ok(wb.sheet(sheet).map(|sh| gridcraft_xlsx::write_csv(sh, wb, b',')).unwrap_or_default()), FileKind::Tsv => Ok(wb.sheet(sheet).map(|sh| gridcraft_xlsx::write_csv(sh, wb, b'\t')).unwrap_or_default()), FileKind::Json => serde_json::to_vec_pretty(wb).map_err(|e| EngineError::Other(e.to_string())), diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f3e36cd..1c49414 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -153,6 +153,71 @@ fn samples_build() { } } +/// Original minimal BIFF12 records, built in source rather than a vendor workbook fixture. +fn xlsb_fixture() -> Vec { + use std::io::{Cursor, Write}; + fn wide(s: &str) -> Vec { + let units: Vec<_> = s.encode_utf16().collect(); + (units.len() as u32).to_le_bytes().into_iter().chain(units.into_iter().flat_map(u16::to_le_bytes)).collect() + } + fn record(out: &mut Vec, id: u16, payload: &[u8]) { + if id < 128 { + out.push(id as u8); + } else { + out.extend([(id as u8 & 127) | 128, (id >> 7) as u8]); + } + let mut size = payload.len(); + while size >= 128 { + out.push((size as u8 & 127) | 128); + size >>= 7; + } + out.push(size as u8); + out.extend(payload); + } + let mut book = vec![]; + record(&mut book, 131, &[]); // BeginBook + record(&mut book, 143, &[]); // BeginBundleShs + let mut bundle = vec![0; 4]; // Visible + bundle.extend(1u32.to_le_bytes()); + bundle.extend(wide("rId1")); + bundle.extend(wide("Imported")); + record(&mut book, 156, &bundle); + record(&mut book, 144, &[]); + record(&mut book, 132, &[]); + let mut sheet = vec![]; + record(&mut sheet, 129, &[]); // BeginSheet + record(&mut sheet, 145, &[]); // BeginSheetData + record(&mut sheet, 0, &[0; 17]); // Row 0, no spans + let mut numeric = vec![0; 8]; // A1, default style + numeric.extend(42.0f64.to_le_bytes()); + numeric.extend([0; 2]); // Formula flags + numeric.extend(3u32.to_le_bytes()); + numeric.extend([0x1e, 42, 0]); // PtgInt(42) + numeric.extend(0u32.to_le_bytes()); + record(&mut sheet, 9, &numeric); // FmlaNum + let mut text = 1u32.to_le_bytes().to_vec(); // B1 + text.extend(0u32.to_le_bytes()); + text.extend(wide("Original data")); + record(&mut sheet, 6, &text); + record(&mut sheet, 146, &[]); + record(&mut sheet, 130, &[]); + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let types = br#""#; + let rels = br#""#; + let book_rels = br#""#; + for (name, data) in [ + ("[Content_Types].xml", types.as_slice()), + ("_rels/.rels", rels.as_slice()), + ("xl/_rels/workbook.bin.rels", book_rels.as_slice()), + ("xl/workbook.bin", book.as_slice()), + ("xl/worksheets/sheet1.bin", sheet.as_slice()), + ] { + zip.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + zip.write_all(data).unwrap(); + } + zip.finish().unwrap().into_inner() +} + #[test] fn every_command_survives_empty_params() { let mut s = Session::new(); @@ -190,3 +255,64 @@ fn ink_strokes_and_ink_to_shape() { assert_eq!(s.execute("draw.inkToShape", json!({})).unwrap()["kind"], "Line"); assert!(s.execute("draw.stroke", json!({"points": [[1.0, 1.0]]})).is_err()); } + +#[test] +fn xlsb_import_keeps_cached_values_and_reports_limits() { + let bytes = xlsb_fixture(); + assert_eq!(gridcraft_xlsx::sniff(&bytes), gridcraft_xlsx::Format::Xlsb); + for name in ["source.xlsb", "source.xlsx"] { + let mut s = s(); + let r = s.execute("file.open", json!({"name": name, "base64": crate::io::base64_encode(&bytes)})).unwrap(); + assert!(!r["warnings"].as_array().unwrap().is_empty()); + assert!(s.take_ui_requests().iter().any(|r| matches!(r, crate::UiRequest::Message(_)))); + assert!(s.doc().unwrap().path.is_none()); + assert!(s.doc().unwrap().display_title().ends_with(".xlsx")); + assert_eq!(v(&s, "A1"), Value::Number(42.0)); + assert_eq!(v(&s, "B1"), Value::from("Original data")); + assert!(s.doc().unwrap().wb.active().unwrap().cell(CellRef::parse("A1").unwrap()).unwrap().formula.is_none()); + s.execute("cell.set", json!({"cell": "B1", "input": "Changed"})).unwrap(); + assert_eq!(v(&s, "A1"), Value::Number(42.0)); // Imported formula caches are constants. + let saved = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + s.execute("file.open", json!({"name": "imported.xlsx", "base64": saved["base64"]})).unwrap(); + assert_eq!(v(&s, "A1"), Value::Number(42.0)); + } +} + +#[test] +#[cfg(not(target_arch = "wasm32"))] +fn xlsb_import_never_reuses_source_as_save_target() { + let nonce = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let dir = std::env::temp_dir().join(format!("gridcraft-xlsb-{}-{nonce}", std::process::id())); + std::fs::create_dir(&dir).unwrap(); + let bytes = xlsb_fixture(); + for name in ["source.xlsb", "disguised.xlsx"] { + let path = dir.join(name); + std::fs::write(&path, &bytes).unwrap(); + let mut s = s(); + s.execute("file.open", json!({"path": path})).unwrap(); + s.take_ui_requests(); + s.execute("cell.set", json!({"cell": "A1", "input": "43"})).unwrap(); + s.execute("file.save", json!({})).unwrap(); + assert!(s.take_ui_requests().iter().any(|r| matches!(r, crate::UiRequest::Dialog(name, _) if name == "saveAs"))); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + assert!(s.execute("file.saveAs", json!({"path": dir.join("source.xlsb")})).is_err()); + assert!(s.execute("file.saveBytes", json!({"format": "xlsb"})).is_err()); + let output = dir.join("converted.xlsx"); + s.execute("file.saveAs", json!({"path": output})).unwrap(); + assert!(!s.doc().unwrap().is_dirty()); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + s.execute("file.open", json!({"name": "converted.xlsx", "base64": crate::io::base64_encode(&std::fs::read(output).unwrap())})).unwrap(); + assert_eq!(v(&s, "A1"), Value::Number(43.0)); + } + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn malformed_xlsb_keeps_the_current_workbook() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "Keep me"})).unwrap(); + assert!(s.execute("file.open", json!({"name": "bad.xlsb", "base64": crate::io::base64_encode(b"not a workbook")})).is_err()); + assert_eq!(s.documents().len(), 1); + assert_eq!(v(&s, "A1"), Value::from("Keep me")); + assert!(s.doc().unwrap().is_dirty()); +} diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rs index 0fb6ecd..3e2728b 100644 --- a/crates/mcp/src/tools.rs +++ b/crates/mcp/src/tools.rs @@ -329,7 +329,7 @@ pub fn tool_definitions() -> Vec { tool( "open_workbook", "Open workbook", - "Open a workbook file (.xlsx, .xlsm, .csv, .tsv, .txt, .json) and make it active.", + "Open a workbook file (.xlsx, .xlsm, .xlsb, .csv, .tsv, .txt, .json) and make it active. XLSB imports worksheet data and cached formula values only; check the returned warnings.", obj(json!({"path": string("File path")}), &["path"]), false, ), diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index 7c72476..49bfbe7 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -1,4 +1,4 @@ -//! GridCraft file formats: XLSX (Office Open XML SpreadsheetML, transitional and strict) and CSV. +//! GridCraft file formats: XLSX (transitional and strict) and CSV import/export, plus XLSB data import. //! //! Clean-room implementation from ECMA-376 / ISO/IEC 29500 and observed behaviour. All entry //! points work on byte slices (no file system access), so the crate builds for WebAssembly. @@ -23,6 +23,7 @@ mod styles; mod tables; mod theme; mod write; +mod xlsb; mod xml; #[cfg(test)] @@ -32,6 +33,7 @@ use gridcraft_core::DateSystem; use gridcraft_model::Workbook; pub use csv::{read_csv, write_csv}; +pub use xlsb::read_xlsb; /// Errors from reading or writing files. #[derive(Debug, thiserror::Error)] @@ -92,14 +94,15 @@ impl Default for CsvOptions { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Format { Xlsx, + Xlsb, Csv, Unknown, } -/// Guesses the format from the first bytes: a zip package is XLSX, text is CSV. +/// Guesses XLSB from package content types, other ZIP packages as XLSX, and text as CSV. pub fn sniff(bytes: &[u8]) -> Format { if bytes.starts_with(b"PK\x03\x04") || bytes.starts_with(b"PK\x05\x06") || bytes.starts_with(b"PK\x07\x08") { - return Format::Xlsx; + return if xlsb::is_xlsb(bytes) { Format::Xlsb } else { Format::Xlsx }; } if bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) || bytes.starts_with(&[0xEF, 0xBB, 0xBF]) { return Format::Csv; diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index e6cade6..9a28999 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -58,22 +58,28 @@ impl<'a> Package<'a> { /// Reads a part's bytes; `Ok(None)` when it doesn't exist. pub fn read(&mut self, part: &str) -> Result>, IoError> { + self.read_limited(part, MAX_PART) + } + + /// Reads a small metadata part without allowing it to expand to the general part limit. + pub fn read_limited(&mut self, part: &str, maximum: u64) -> Result>, IoError> { + let maximum = maximum.min(MAX_PART); let Some(&idx) = self.names.get(&normalize(part)) else { return Ok(None); }; let mut f = self.zip.by_index(idx).map_err(|e| IoError::Zip(format!("{part}: {e}")))?; - if f.size() > MAX_PART { + if f.size() > maximum { return Err(IoError::TooLarge(format!("{part} expands to {} bytes", f.size()))); } let remaining = MAX_TOTAL.saturating_sub(self.total); - let limit = MAX_PART.min(remaining); + let limit = maximum.min(remaining); let mut out = Vec::with_capacity(f.size().min(16 * 1024 * 1024) as usize); (&mut f).take(limit + 1).read_to_end(&mut out).map_err(|e| IoError::Zip(format!("{part}: {e}")))?; if out.len() as u64 > limit { - return Err(IoError::TooLarge(if limit < MAX_PART { + return Err(IoError::TooLarge(if remaining < maximum { "package expands to more than 1 GB".into() } else { - format!("{part} expands to more than 512 MB") + format!("{part} expands to more than {maximum} bytes") })); } self.total += out.len() as u64; diff --git a/crates/xlsx/src/tests/mod.rs b/crates/xlsx/src/tests/mod.rs index 46bd997..9b47cc9 100644 --- a/crates/xlsx/src/tests/mod.rs +++ b/crates/xlsx/src/tests/mod.rs @@ -5,6 +5,7 @@ mod fixtures; mod malformed; mod pivot_tests; mod roundtrip; +mod xlsb_tests; use std::io::Write; diff --git a/crates/xlsx/src/tests/xlsb_tests.rs b/crates/xlsx/src/tests/xlsb_tests.rs new file mode 100644 index 0000000..7c4311e --- /dev/null +++ b/crates/xlsx/src/tests/xlsb_tests.rs @@ -0,0 +1,306 @@ +//! Original BIFF12 records and ZIP packages assembled in Rust from the public format schema. +use gridcraft_core::{CellError, CellRef, DateSystem, RangeRef, Value}; +use gridcraft_model::Visibility; + +use super::make_zip; +use crate::{Format, read_xlsb, read_xlsx, sniff, write_xlsx}; + +const ROOT: &str = r#""#; +const RELS: &str = r#""#; +const TYPES: &str = r#""#; + +fn record(id: u16, bytes: &[u8]) -> Vec { + let mut out = vec![]; + if id < 128 { + out.push(id as u8); + } else { + out.extend([(id as u8 & 0x7f) | 0x80, (id >> 7) as u8]); + } + let mut len = bytes.len(); + loop { + let byte = (len & 0x7f) as u8; + len >>= 7; + out.push(if len == 0 { byte } else { byte | 0x80 }); + if len == 0 { + break; + } + } + out.extend(bytes); + out +} +fn wide(text: &str) -> Vec { + let units: Vec<_> = text.encode_utf16().collect(); + let mut out = (units.len() as u32).to_le_bytes().to_vec(); + for unit in units { + out.extend(unit.to_le_bytes()); + } + out +} +fn rich(text: &str, extras: bool) -> Vec { + let mut out = vec![if extras { 3 } else { 0 }]; + out.extend(wide(text)); + if extras { + out.extend(1u32.to_le_bytes()); + out.extend([0, 0, 0, 0]); // One original formatting run. + out.extend(wide("sound")); + out.extend(1u32.to_le_bytes()); + out.extend([0, 0, 0, 0, 1, 0, 0, 0, 0, 0]); // One ten-byte phonetic run. + } + out +} +fn row(r: u32) -> Vec { + let mut out = r.to_le_bytes().to_vec(); + out.extend([0; 13]); + record(0, &out) +} +fn cell(id: u16, col: u32, value: &[u8]) -> Vec { + let mut data = col.to_le_bytes().to_vec(); + data.extend([0; 4]); + data.extend(value); + if (8..=11).contains(&id) { + data.extend([0; 2]); // Formula flags. + data.extend(3u32.to_le_bytes()); + data.extend([0x1e, 1, 0]); // Original PtgInt(1); ignored because we import the cache. + data.extend(0u32.to_le_bytes()); + } + record(id, &data) +} +fn worksheet(data: &[u8], tail: &[u8]) -> Vec { + let mut out = record(129, &[]); + out.extend(record(148, &[0xff; 16])); // A bogus dimension must never allocate a dense grid. + out.extend(record(145, &[])); + out.extend(data); + out.extend(record(146, &[])); + out.extend(tail); + out.extend(record(130, &[])); + out +} +fn book(date1904: bool) -> Vec { + let mut out = record(131, &[]); + let mut props = u32::from(date1904).to_le_bytes().to_vec(); + props.extend(0u32.to_le_bytes()); + props.extend(wide("")); + out.extend(record(153, &props)); + out.extend(record(143, &[])); + for (id, rel, name, state) in [(1u32, "one", "Data", 0u32), (2, "two", "Hidden", 1)] { + let mut data = state.to_le_bytes().to_vec(); + data.extend(id.to_le_bytes()); + data.extend(wide(rel)); + data.extend(wide(name)); + out.extend(record(156, &data)); + } + out.extend(record(144, &[])); + out.extend(record(132, &[])); + out +} +fn sst() -> Vec { + let mut out = record(159, &[1, 0, 0, 0, 1, 0, 0, 0]); + out.extend(record(19, &rich("shared 🦀 café", true))); + out.extend(record(160, &[])); + out +} +fn package(book: &[u8], sheet: &[u8], strings: &[u8], roots: &str, rels: &str, types: &str) -> Vec { + make_zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", roots.as_bytes()), + ("records/book.bin", book), + ("records/_rels/book.bin.rels", rels.as_bytes()), + ("data/one.bin", sheet), + ("data/two.bin", &worksheet(&[], &[])), + ("text/strings.bin", strings), + ]) +} +fn file(sheet: &[u8]) -> Vec { + package(&book(false), sheet, &sst(), ROOT, RELS, TYPES) +} +fn text(s: &str) -> Value { + Value::Text(s.into()) +} + +#[test] +fn xlsb_scalar_caches_rk_unicode_visibility_and_epochs_survive_xlsx_save() { + let mut data = row(2); + let real_rk = (3.5f64.to_bits() >> 32) as u32; + let encoded = [ + cell(1, 0, &[]), + cell(2, 1, &((-123i32 << 2) | 2).to_le_bytes()), + cell(2, 2, &((123i32 << 2) | 3).to_le_bytes()), + cell(2, 3, &real_rk.to_le_bytes()), + cell(2, 4, &(real_rk | 1).to_le_bytes()), + cell(3, 5, &[0x17]), + cell(4, 6, &[1]), + cell(5, 7, &123.5f64.to_le_bytes()), + cell(6, 8, &wide("inline λ")), + cell(7, 9, &0u32.to_le_bytes()), + cell(8, 10, &wide("cached text")), + cell(9, 11, &2.0f64.to_le_bytes()), + cell(10, 12, &[0]), + cell(11, 13, &[7]), + cell(62, 14, &rich("rich text", true)), + ]; + for cell in encoded { + data.extend(cell); + } + let mut merge = vec![]; + for n in [5u32, 6, 0, 1] { + merge.extend(n.to_le_bytes()); + } + let sheet = worksheet(&data, &record(176, &merge)); + let expected = [ + Value::Empty, + Value::Number(-123.0), + Value::Number(1.23), + Value::Number(3.5), + Value::Number(0.035), + Value::Error(CellError::Ref), + Value::Bool(true), + Value::Number(123.5), + text("inline λ"), + text("shared 🦀 café"), + text("cached text"), + Value::Number(2.0), + Value::Bool(false), + Value::Error(CellError::Div0), + text("rich text"), + ]; + for epoch in [false, true] { + let bytes = package(&book(epoch), &sheet, &sst(), ROOT, RELS, TYPES); + assert_eq!(sniff(&bytes), Format::Xlsb); + let (wb, report) = read_xlsb(&bytes).unwrap(); + assert_eq!(wb.date_system, if epoch { DateSystem::D1904 } else { DateSystem::D1900 }); + assert_eq!(wb.sheets.len(), 2); + assert_eq!(wb.sheets[1].visibility, Visibility::Hidden); + assert_eq!(wb.sheets[0].cells.len(), 14); + assert_eq!(wb.sheets[0].merges, vec![RangeRef::new(CellRef::new(5, 0), CellRef::new(6, 1))]); + assert!(report.warnings.iter().any(|w| w.contains("will not recalculate") && w.contains("may display as numbers"))); + for (col, expected) in expected.iter().enumerate() { + assert_eq!(&wb.sheets[0].value(CellRef::new(2, col as u32)), expected); + } + assert!(wb.sheets[0].cells.iter().all(|(_, c)| c.formula.is_none())); + let (roundtrip, _) = read_xlsx(&write_xlsx(&wb).unwrap()).unwrap(); + assert_eq!(roundtrip.date_system, wb.date_system); + for (col, expected) in expected.iter().enumerate() { + assert_eq!(&roundtrip.sheets[0].value(CellRef::new(2, col as u32)), expected); + } + } +} + +#[test] +fn xlsb_extension_blocks_are_skipped_with_strict_boundaries_and_record_framing() { + let mut data = row(0); + data.extend(cell(5, 0, &1.0f64.to_le_bytes())); + data.extend(record(35, &[0; 4])); + data.extend(record(37, &[1, 0, 0, 0, 0, 0])); + data.extend(cell(5, 1, &99.0f64.to_le_bytes())); + data.extend(record(130, &[])); // Looks like EndSheet, but is inside an ignored extension. + data.extend(record(38, &[])); + data.extend(record(36, &[])); + data.extend([0xe8, 0x87, 0x80, 0x80, 0x80, 0x80]); // ID1000: ignored high second-ID/fourth-length bits. + data.extend(cell(5, 1, &2.0f64.to_le_bytes())); + let (wb, report) = read_xlsb(&file(&worksheet(&data, &[]))).unwrap(); + assert_eq!(wb.sheets[0].cells.len(), 2); + assert_eq!(wb.sheets[0].value(CellRef::new(0, 1)), Value::Number(2.0)); + assert!(report.warnings.iter().any(|w| w.contains("blocks were skipped"))); + for bad in [ + vec![0x81], + vec![0x81, 0], + vec![1, 0x80], + vec![1, 10, 0], + record(36, &[]), + [record(35, &[0; 4]), record(38, &[])].concat(), + record(35, &[0; 4]), + record(37, &[0, 0]), + ] { + assert!(read_xlsb(&file(&worksheet(&bad, &[]))).is_err(), "{bad:?}"); + } + let nested = record(35, &[0; 4]).repeat(65); + assert!(read_xlsb(&file(&worksheet(&nested, &[]))).unwrap_err().to_string().contains("nested too deeply")); + let mut missing_end = worksheet(&[], &[]); + missing_end.truncate(missing_end.len() - record(130, &[]).len()); + assert!(read_xlsb(&file(&missing_end)).is_err()); +} + +#[test] +fn xlsb_bad_values_strings_and_coordinates_fail_without_silent_data_loss() { + let malformed = [ + cell(4, 0, &[2]), + cell(3, 0, &[0xff]), + cell(5, 0, &f64::INFINITY.to_le_bytes()), + cell(2, 0, &0x7ff00000u32.to_le_bytes()), + cell(7, 0, &1u32.to_le_bytes()), + cell(6, 0, &[1, 0, 0, 0, 0, 0xd8]), + cell(6, 0, &u32::MAX.to_le_bytes()), + cell(5, 16384, &1.0f64.to_le_bytes()), + cell(5, 0, &[0; 3]), + cell(62, 0, &[1, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff]), + ]; + for bad in malformed { + assert!(read_xlsb(&file(&worksheet(&[row(0), bad].concat(), &[]))).is_err()); + } + assert!(read_xlsb(&file(&worksheet(&cell(5, 0, &1.0f64.to_le_bytes()), &[]))).is_err()); + assert!(read_xlsb(&file(&worksheet(&row(1048576), &[]))).is_err()); + let repeated = [row(0), cell(5, 0, &1.0f64.to_le_bytes()), cell(5, 0, &2.0f64.to_le_bytes())].concat(); + assert!(read_xlsb(&file(&worksheet(&repeated, &[]))).is_err()); + let mut formula_payload = [0u8; 18].to_vec(); // Cell header + f64 cache + flags, but no parsed-formula structure. + assert!(read_xlsb(&file(&worksheet(&[row(0), record(9, &formula_payload)].concat(), &[]))).is_err()); + formula_payload.extend(0u32.to_le_bytes()); // Invalid zero token count. + assert!(read_xlsb(&file(&worksheet(&[row(0), record(9, &formula_payload)].concat(), &[]))).is_err()); + let bad_sst = [record(159, &[1, 0, 0, 0, 2, 0, 0, 0]), record(160, &[])].concat(); + assert!(read_xlsb(&package(&book(false), &worksheet(&[], &[]), &bad_sst, ROOT, RELS, TYPES)).is_err()); +} + +#[test] +fn xlsb_requires_internal_typed_relationships_and_sniffs_only_root_workbook() { + let sheet = worksheet(&[], &[]); + for (roots, rels, types) in [ + (ROOT.replace("officeDocument\"", "wrong\""), RELS.to_owned(), TYPES.to_owned()), + (ROOT.replace("Target=", "TargetMode=\"External\" Target="), RELS.to_owned(), TYPES.to_owned()), + (ROOT.to_owned(), RELS.replace("Target=\"../data/one.bin\"", "Target=\"missing.bin\""), TYPES.to_owned()), + ( + ROOT.to_owned(), + RELS.replace("Target=\"../data/one.bin\"", "TargetMode=\"External\" Target=\"https://example.com/data.bin\""), + TYPES.to_owned(), + ), + ( + ROOT.to_owned(), + RELS.replace("Target=\"../text/strings.bin\"", "TargetMode=\"External\" Target=\"https://example.com/strings.bin\""), + TYPES.to_owned(), + ), + (ROOT.to_owned(), RELS.to_owned(), TYPES.replace("application/vnd.ms-excel.worksheet", "application/xml")), + (ROOT.to_owned(), RELS.trim_end_matches("").to_owned(), TYPES.to_owned()), + (ROOT.to_owned(), RELS.to_owned(), TYPES.replace("http://schemas.openxmlformats.org/package/2006/content-types", "wrong")), + ] { + assert!(read_xlsb(&package(&book(false), &sheet, &sst(), &roots, &rels, &types)).is_err()); + } + let chartsheet = RELS.replace( + "Id=\"two\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet", + "Id=\"two\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/chartsheet", + ); + let (wb, report) = read_xlsb(&package(&book(false), &sheet, &sst(), ROOT, &chartsheet, TYPES)).unwrap(); + assert_eq!(wb.sheets.len(), 1); + assert!(report.warnings.iter().any(|w| w.contains("non-worksheet"))); + // An unrelated binary part must not change the format of an XML workbook package. + let xml_book = TYPES.replace( + "/records/book.bin\" ContentType=\"application/vnd.ms-excel.sheet.binary.macroEnabled.main", + "/elsewhere.bin\" ContentType=\"application/vnd.ms-excel.sheet.binary.macroEnabled.main", + ); + assert_eq!(sniff(&package(&book(false), &sheet, &sst(), ROOT, RELS, &xml_book)), Format::Xlsx); + let spec_type = TYPES.replace("application/vnd.ms-excel.sheet.binary.macroEnabled.main", "application/vnd.ms-excel.main"); + assert_eq!(sniff(&package(&book(false), &sheet, &sst(), ROOT, RELS, &spec_type)), Format::Xlsb); + assert!(read_xlsb(&package(&book(false), &sheet, &sst(), ROOT, RELS, &spec_type)).is_ok()); +} + +#[test] +fn xlsb_declared_counts_and_decoded_text_are_bounded() { + let bad_sst = record(159, &[0xff; 8]); + assert!(read_xlsb(&package(&book(false), &worksheet(&[], &[]), &bad_sst, ROOT, RELS, TYPES)).is_err()); + let mut count_mismatch = record(159, &[1, 0, 0, 0, 1, 0, 0, 0]); + count_mismatch.extend(record(160, &[])); + assert!(read_xlsb(&package(&book(false), &worksheet(&[], &[]), &count_mismatch, ROOT, RELS, TYPES)).is_err()); + let mut data = row(0); + let text = wide(&"x".repeat(32767)); + for col in 0..2050 { + data.extend(cell(6, col, &text)); + } + assert!(read_xlsb(&file(&worksheet(&data, &[]))).unwrap_err().to_string().contains("64 MiB")); +} diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs new file mode 100644 index 0000000..5574e46 --- /dev/null +++ b/crates/xlsx/src/xlsb.rs @@ -0,0 +1,667 @@ +//! Data-only XLSB import, implemented from Microsoft's public MS-XLSB specification: +//! . +//! Formula caches become constants. Styles, macros and other workbook features are not run or imported. + +use std::sync::Arc; + +use gridcraft_core::{CellError, CellRef, DateSystem, MAX_COLS, MAX_ROWS, RangeRef, Value}; +use gridcraft_model::{Cell, Sheet, Visibility, Workbook}; +use quick_xml::NsReader; +use quick_xml::events::Event; +use quick_xml::name::ResolveResult; + +use crate::package::{Package, Rel, normalize, resolve, split_dir}; +use crate::{IoError, ReadReport}; + +const MAIN: &str = "application/vnd.ms-excel.main"; +const BINARY_MAIN: &str = "application/vnd.ms-excel.sheet.binary.macroEnabled.main"; +const WORKSHEET: &str = "application/vnd.ms-excel.worksheet"; +const STRINGS: &str = "application/vnd.ms-excel.sharedStrings"; +const CT_NS: &[u8] = b"http://schemas.openxmlformats.org/package/2006/content-types"; +const REL_NS: &[u8] = b"http://schemas.openxmlformats.org/package/2006/relationships"; +const REL_TYPE: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/"; +const STRICT_REL_TYPE: &str = "http://purl.oclc.org/ooxml/officeDocument/relationships/"; +const MS_REL_TYPE: &str = "http://schemas.microsoft.com/office/2006/relationships/"; +const MAX_META: u64 = 1024 * 1024; +const MAX_TEXT: usize = 64 * 1024 * 1024; +const MAX_CELLS: usize = 1_000_000; +const MAX_STRINGS: usize = 1_000_000; +const MAX_SHEETS: usize = 1024; +const MAX_RECORDS: usize = 5_000_000; + +fn invalid(msg: impl Into) -> IoError { + IoError::Format(format!("XLSB: {}", msg.into())) +} + +#[derive(Default)] +struct Budget { + cells: usize, + text: usize, +} +impl Budget { + fn text(&mut self, len: usize) -> Result<(), IoError> { + self.text = self.text.saturating_add(len); + if self.text > MAX_TEXT { + return Err(IoError::TooLarge("XLSB decoded text exceeds 64 MiB".into())); + } + Ok(()) + } + fn cell(&mut self) -> Result<(), IoError> { + self.cells += 1; + if self.cells > MAX_CELLS { + return Err(IoError::TooLarge("XLSB exceeds 1 million imported cells".into())); + } + Ok(()) + } +} + +/// Imports worksheet scalar values and saved formula results. Number formats are omitted; +/// dates remain serial numbers with the workbook's original date system. Inspect warnings. +pub fn read_xlsb(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { + let mut package = Package::open(bytes)?; + let types = content_types(&mut package)?; + let roots = relationships(&mut package, "")?; + let roots: Vec<_> = roots.iter().filter(|r| r.kind == "officeDocument").collect(); + if roots.len() != 1 { + return Err(invalid("expected one workbook relationship")); + } + let root = roots.first().ok_or_else(|| invalid("missing workbook relationship"))?; + if root.external { + return Err(invalid("external workbook relationship")); + } + check_type(&types, &root.target, MAIN)?; + let book = required(&mut package, &root.target)?; + let mut wb = Workbook::new(); + wb.sheets.clear(); + let mut report = ReadReport::default(); + report.warn("XLSB data-only import: formulas become their saved results and will not recalculate. Formatting, charts, images, macros and other features are not imported; dates, currency and percentages may display as numbers. Save a new .xlsx file; XLSB saving is not supported."); + let sheets = workbook(&book, &mut wb, &mut report)?; + let rels = relationships(&mut package, &root.target)?; + let mut budget = Budget::default(); + let sst_rels: Vec<_> = rels.iter().filter(|r| r.kind == "sharedStrings").collect(); + if sst_rels.len() > 1 { + return Err(invalid("multiple shared-string relationships")); + } + let strings = if let Some(rel) = sst_rels.first() { + if rel.external { + return Err(invalid("external shared-string relationship")); + } + check_type(&types, &rel.target, STRINGS)?; + shared_strings(&required(&mut package, &rel.target)?, &mut budget, &mut report)? + } else { + vec![] + }; + for info in sheets { + let Some(id) = info.rel else { + if info.visibility != Visibility::VeryHidden { + return Err(invalid("worksheet relationship is null")); + } + report.warn("XLSB module sheets were skipped."); + continue; + }; + let rel = rels.iter().find(|r| r.id == id).ok_or_else(|| invalid("worksheet relationship is missing"))?; + if rel.external { + return Err(invalid("external sheet relationship")); + } + if rel.kind != "worksheet" { + if !matches!(rel.kind.as_str(), "chartsheet" | "dialogsheet" | "xlMacrosheet" | "xlIntlMacrosheet") { + return Err(invalid("unknown sheet relationship type")); + } + report.warn("XLSB non-worksheet sheets were skipped."); + continue; + } + check_type(&types, &rel.target, WORKSHEET)?; + wb.check_sheet_name(&info.name, None).map_err(invalid)?; + let mut sheet = worksheet(&required(&mut package, &rel.target)?, &info.name, &strings, &mut budget, &mut report)?; + sheet.visibility = info.visibility; + wb.sheets.push(Arc::new(sheet)); + } + if wb.sheets.is_empty() { + return Err(invalid("no supported worksheets")); + } + wb.active_sheet = wb.sheets.iter().position(|s| s.visibility == Visibility::Visible).unwrap_or(0); + Ok((wb, report)) +} + +pub(crate) fn is_xlsb(bytes: &[u8]) -> bool { + let inspect = || -> Result { + let mut package = Package::open(bytes)?; + let types = content_types(&mut package)?; + let roots = relationships(&mut package, "")?; + let mut books = roots.iter().filter(|r| r.kind == "officeDocument"); + let Some(root) = books.next() else { return Ok(false) }; + Ok(books.next().is_none() && !root.external && check_type(&types, &root.target, MAIN).is_ok()) + }; + inspect().unwrap_or(false) +} + +fn required(package: &mut Package<'_>, path: &str) -> Result, IoError> { + package.read(path)?.ok_or_else(|| invalid(format!("missing part {path}"))) +} + +struct Entry { + name: String, + attrs: Vec<(String, String)>, +} +impl Entry { + fn attr(&self, name: &str) -> Option<&str> { + self.attrs.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()) + } +} + +/// The two package metadata parts are flat, namespace-qualified XML documents. Keep only +/// direct child entries, and reject truncated XML instead of guessing a partial relationship. +fn metadata(bytes: &[u8], ns: &[u8], root_name: &[u8]) -> Result, IoError> { + let mut reader = NsReader::from_reader(bytes); + let mut depth = 0usize; + let mut seen = false; + let mut entries = vec![]; + loop { + let event = reader.read_event().map_err(|e| IoError::Xml(e.to_string()))?; + match &event { + Event::Start(e) | Event::Empty(e) => { + let (namespace, local) = reader.resolve_element(e.name()); + let valid_ns = matches!(namespace, ResolveResult::Bound(n) if n.as_ref() == ns); + if depth == 0 { + if seen || !valid_ns || local.as_ref() != root_name { + return Err(invalid("invalid package metadata root")); + } + seen = true; + } else if depth == 1 && valid_ns { + let mut attrs = vec![]; + for a in e.attributes() { + let a = a.map_err(|e| IoError::Xml(e.to_string()))?; + if !a.key.as_ref().contains(&b':') { + attrs.push(( + String::from_utf8_lossy(a.key.as_ref()).into_owned(), + a.unescape_value().map_err(|e| IoError::Xml(e.to_string()))?.into_owned(), + )); + } + } + entries.push(Entry { name: String::from_utf8_lossy(local.as_ref()).into_owned(), attrs }); + } + if matches!(event, Event::Start(_)) { + depth += 1; + if depth > crate::xml::MAX_DEPTH { + return Err(invalid("package metadata is nested too deeply")); + } + } + } + Event::End(_) => depth = depth.checked_sub(1).ok_or_else(|| invalid("unexpected metadata end"))?, + Event::Text(t) if depth == 0 && !t.xml_content().map_err(|e| IoError::Xml(e.to_string()))?.trim().is_empty() => { + return Err(invalid("text outside package metadata")); + } + Event::DocType(_) => return Err(invalid("DOCTYPE in package metadata is unsupported")), + Event::Eof => break, + _ => {} + } + } + if !seen || depth != 0 { + return Err(invalid("incomplete package metadata")); + } + Ok(entries) +} + +fn content_types(package: &mut Package<'_>) -> Result, IoError> { + let data = package.read_limited("[Content_Types].xml", MAX_META)?.ok_or_else(|| invalid("missing content types"))?; + metadata(&data, CT_NS, b"Types") +} + +fn check_type(types: &[Entry], path: &str, expected: &str) -> Result<(), IoError> { + let found = types.iter().find(|e| e.name == "Override" && e.attr("PartName").is_some_and(|p| normalize(p) == normalize(path))).or_else(|| { + types.iter().find(|e| { + e.name == "Default" && e.attr("Extension").is_some_and(|ext| path.rsplit('.').next().is_some_and(|p| p.eq_ignore_ascii_case(ext))) + }) + }); + let actual = found.and_then(|e| e.attr("ContentType")); + if actual != Some(expected) && !(expected == MAIN && actual == Some(BINARY_MAIN)) { + return Err(invalid(format!("wrong or missing content type for {path}"))); + } + Ok(()) +} + +fn relationships(package: &mut Package<'_>, path: &str) -> Result, IoError> { + let (dir, file) = split_dir(path); + let relpath = if dir.is_empty() { format!("_rels/{file}.rels") } else { format!("{dir}/_rels/{file}.rels") }; + let data = package.read_limited(&relpath, MAX_META)?.ok_or_else(|| invalid(format!("missing relationships for {path}")))?; + let mut out: Vec = vec![]; + for e in metadata(&data, REL_NS, b"Relationships")? { + if e.name != "Relationship" { + continue; + } + let id = e.attr("Id").filter(|s| !s.is_empty()).ok_or_else(|| invalid("relationship has no ID"))?; + if out.iter().any(|r| r.id == id) { + return Err(invalid("duplicate relationship ID")); + } + let ty = e.attr("Type").ok_or_else(|| invalid("relationship has no type"))?; + let kind = ty + .strip_prefix(REL_TYPE) + .or_else(|| ty.strip_prefix(STRICT_REL_TYPE)) + .or_else(|| ty.strip_prefix(MS_REL_TYPE)) + .map(str::to_owned) + .unwrap_or_else(|| format!("unsupported:{ty}")); + let target = e.attr("Target").ok_or_else(|| invalid("relationship has no target"))?; + let external = e.attr("TargetMode") == Some("External"); + out.push(Rel { id: id.into(), kind, target: if external { target.into() } else { resolve(dir, target) }, external }); + } + Ok(out) +} + +struct Bytes<'a> { + rest: &'a [u8], +} +impl<'a> Bytes<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { rest: bytes } + } + fn take(&mut self, n: usize) -> Result<&'a [u8], IoError> { + let out = self.rest.get(..n).ok_or_else(|| invalid("truncated record or string"))?; + self.rest = self.rest.get(n..).ok_or_else(|| invalid("truncated record"))?; + Ok(out) + } + fn u8(&mut self) -> Result { + self.take(1)?.first().copied().ok_or_else(|| invalid("truncated byte")) + } + fn u16(&mut self) -> Result { + Ok(u16::from_le_bytes(self.take(2)?.try_into().map_err(|_| invalid("truncated u16"))?)) + } + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().map_err(|_| invalid("truncated u32"))?)) + } + fn f64(&mut self) -> Result { + let n = f64::from_le_bytes(self.take(8)?.try_into().map_err(|_| invalid("truncated number"))?); + if !n.is_finite() { + return Err(invalid("nonfinite numeric value")); + } + Ok(n) + } + fn string(&mut self, max: u32, nullable: bool) -> Result, IoError> { + let count = self.u32()?; + if nullable && count == u32::MAX { + return Ok(None); + } + if count > max { + return Err(IoError::TooLarge("XLSB string length exceeds its limit".into())); + } + let size = (count as usize).checked_mul(2).ok_or_else(|| invalid("string length overflow"))?; + let bytes = self.take(size)?; + let mut text = String::new(); + for ch in char::decode_utf16(bytes.as_chunks::<2>().0.iter().copied().map(u16::from_le_bytes)) { + text.push(ch.map_err(|_| invalid("invalid UTF-16 string"))?); + } + Ok(Some(text)) + } + fn text(&mut self, max: u32) -> Result { + self.string(max, false)?.ok_or_else(|| invalid("null string")) + } + fn finish(self) -> Result<(), IoError> { + if !self.rest.is_empty() { + return Err(invalid("unexpected bytes after record payload")); + } + Ok(()) + } +} + +struct Records<'a> { + bytes: Bytes<'a>, + count: usize, + blocks: Vec, +} +impl<'a> Records<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes: Bytes::new(bytes), count: 0, blocks: vec![] } + } + fn next(&mut self, report: &mut ReadReport) -> Result, IoError> { + while !self.bytes.rest.is_empty() { + self.count += 1; + if self.count > MAX_RECORDS { + return Err(IoError::TooLarge("XLSB part contains too many records".into())); + } + let first = self.bytes.u8()?; + let id = if first & 0x80 == 0 { + first as u16 + } else { + let id = u16::from(first & 0x7f) | (u16::from(self.bytes.u8()? & 0x7f) << 7); + if id < 128 { + return Err(invalid("invalid two-byte record type")); + } + id + }; + let mut size = 0usize; + for shift in [0, 7, 14, 21] { + let b = self.bytes.u8()?; + size |= usize::from(b & 0x7f) << shift; + // The high bit of the fourth length byte is ignored by MS-XLSB. + if b & 0x80 == 0 || shift == 21 { + break; + } + } + let payload = self.bytes.take(size)?; + match id { + 35 | 37 => { + let mut b = Bytes::new(payload); + if id == 35 { + b.take(4)?; + } else { + let count = b.u16()?; + if count == 0 { + return Err(invalid("empty alternate-content version list")); + } + b.take(usize::from(count) * 4)?; + } + b.finish()?; + if self.blocks.len() >= 64 { + return Err(invalid("extension blocks nested too deeply")); + } + self.blocks.push(id + 1); + report.warn("Unsupported XLSB future-record and alternate-content blocks were skipped."); + } + 36 | 38 => { + if self.blocks.pop() != Some(id) || !payload.is_empty() { + return Err(invalid("mismatched extension block end")); + } + } + _ if !self.blocks.is_empty() => {} + _ => return Ok(Some((id, payload))), + } + } + if !self.blocks.is_empty() { + return Err(invalid("unterminated extension block")); + } + Ok(None) + } + fn first(&mut self, id: u16, report: &mut ReadReport) -> Result<(), IoError> { + if self.next(report)? != Some((id, &[])) { + return Err(invalid("missing part begin record")); + } + Ok(()) + } + fn end(&mut self, payload: &[u8], report: &mut ReadReport) -> Result<(), IoError> { + if !payload.is_empty() || self.next(report)?.is_some() { + return Err(invalid("records after part end")); + } + Ok(()) + } +} + +struct SheetInfo { + name: String, + rel: Option, + visibility: Visibility, +} +fn workbook(bytes: &[u8], wb: &mut Workbook, report: &mut ReadReport) -> Result, IoError> { + let mut records = Records::new(bytes); + records.first(131, report)?; + let mut sheets = vec![]; + let mut bundle = false; + let mut bundle_seen = false; + while let Some((id, payload)) = records.next(report)? { + let mut b = Bytes::new(payload); + match id { + 132 => { + if bundle || !bundle_seen { + return Err(invalid("incomplete worksheet list")); + } + records.end(payload, report)?; + return Ok(sheets); + } + 143 => { + if bundle_seen { + return Err(invalid("duplicate worksheet list")); + } + bundle_seen = true; + bundle = true; + b.finish()?; + } + 144 => { + if !bundle { + return Err(invalid("unexpected worksheet list end")); + } + bundle = false; + b.finish()?; + } + 153 => { + wb.date_system = if b.u32()? & 1 != 0 { DateSystem::D1904 } else { DateSystem::D1900 }; + b.u32()?; + b.text(31)?; + b.finish()?; + } + 156 => { + if !bundle { + return Err(invalid("worksheet outside the worksheet list")); + } + if sheets.len() >= MAX_SHEETS { + return Err(IoError::TooLarge("XLSB exceeds 1024 sheets".into())); + } + let visibility = match b.u32()? { + 0 => Visibility::Visible, + 1 => Visibility::Hidden, + 2 => Visibility::VeryHidden, + _ => return Err(invalid("invalid sheet visibility")), + }; + let sheet_id = b.u32()?; + if !(1..=65535).contains(&sheet_id) { + return Err(invalid("invalid worksheet ID")); + } + let rel = b.string(255, true)?; + let name = b.text(31)?; + b.finish()?; + sheets.push(SheetInfo { name, rel, visibility }); + } + _ => {} + } + } + Err(invalid("missing workbook end record")) +} + +fn rich_string(b: &mut Bytes<'_>) -> Result { + let flags = b.u8()?; + let text = b.text(32767)?; + if flags & 1 != 0 { + let count = b.u32()?; + if count > 32767 { + return Err(invalid("too many rich-text runs")); + } + b.take(count as usize * 4)?; + } + if flags & 2 != 0 { + b.text(32767)?; + let count = b.u32()?; + if count > 32767 { + return Err(invalid("too many phonetic runs")); + } + b.take(count as usize * 10)?; + } + Ok(text) +} + +fn shared_strings(bytes: &[u8], budget: &mut Budget, report: &mut ReadReport) -> Result>, IoError> { + let mut records = Records::new(bytes); + let Some((159, payload)) = records.next(report)? else { + return Err(invalid("missing shared-string begin record")); + }; + let mut header = Bytes::new(payload); + let total = header.u32()?; + let unique = header.u32()?; + header.finish()?; + if unique > total || unique as usize > MAX_STRINGS { + return Err(IoError::TooLarge("invalid or oversized shared-string count".into())); + } + let mut strings = vec![]; + while let Some((id, payload)) = records.next(report)? { + match id { + 160 => { + records.end(payload, report)?; + if strings.len() != unique as usize { + return Err(invalid("shared-string count does not match records")); + } + return Ok(strings); + } + 19 => { + if strings.len() >= unique as usize { + return Err(invalid("too many shared-string items")); + } + let mut b = Bytes::new(payload); + let text = rich_string(&mut b)?; + b.finish()?; + budget.text(text.len())?; + strings.push(Arc::from(text)); + } + _ => return Err(invalid("unexpected shared-string record")), + } + } + Err(invalid("missing shared-string end record")) +} + +fn error(code: u8) -> Result { + Ok(match code { + 0x00 => CellError::Null, + 0x07 => CellError::Div0, + 0x0f => CellError::Value, + 0x17 => CellError::Ref, + 0x1d => CellError::Name, + 0x24 => CellError::Num, + 0x2a => CellError::NA, + 0x2b => CellError::GettingData, + _ => return Err(invalid("unknown cell error code")), + }) +} +fn boolean(b: &mut Bytes<'_>) -> Result { + match b.u8()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(invalid("invalid boolean value")), + } +} +fn cached_formula(b: &mut Bytes<'_>) -> Result<(), IoError> { + b.take(2)?; + let cce = b.u32()?; + if !(1..=16384).contains(&cce) { + return Err(invalid("invalid formula token length")); + } + b.take(cce as usize)?; + let extra = b.u32()?; + b.take(extra as usize)?; + Ok(()) +} +fn value(id: u16, b: &mut Bytes<'_>, strings: &[Arc], budget: &mut Budget) -> Result { + let value = match id { + 1 => Value::Empty, + 2 => { + let rk = b.u32()?; + let n = if rk & 2 != 0 { ((rk as i32) >> 2) as f64 } else { f64::from_bits(u64::from(rk & !3) << 32) }; + let n = if rk & 1 != 0 { n / 100.0 } else { n }; + if !n.is_finite() { + return Err(invalid("nonfinite RK number")); + } + Value::Number(n) + } + 3 | 11 => Value::Error(error(b.u8()?)?), + 4 | 10 => Value::Bool(boolean(b)?), + 5 | 9 => Value::Number(b.f64()?), + 6 | 8 | 62 => { + let s = if id == 62 { rich_string(b)? } else { b.text(32767)? }; + budget.text(s.len())?; + Value::Text(Arc::from(s)) + } + 7 => Value::Text(strings.get(b.u32()? as usize).cloned().ok_or_else(|| invalid("shared-string index is out of bounds"))?), + _ => return Err(invalid("unsupported cell record")), + }; + if (8..=11).contains(&id) { + cached_formula(b)?; + } + Ok(value) +} + +fn worksheet(bytes: &[u8], name: &str, strings: &[Arc], budget: &mut Budget, report: &mut ReadReport) -> Result { + let mut records = Records::new(bytes); + records.first(129, report)?; + let mut sheet = Sheet::new(name); + let mut data = false; + let mut data_seen = false; + let mut row = None; + let mut col = None; + while let Some((id, payload)) = records.next(report)? { + let mut b = Bytes::new(payload); + match id { + 130 => { + if data || !data_seen { + return Err(invalid("incomplete worksheet data")); + } + records.end(payload, report)?; + return Ok(sheet); + } + 145 => { + if data_seen { + return Err(invalid("duplicate worksheet data")); + } + data_seen = true; + data = true; + b.finish()?; + } + 146 => { + if !data { + return Err(invalid("unexpected worksheet data end")); + } + data = false; + b.finish()?; + } + 0 => { + if !data { + return Err(invalid("row outside worksheet data")); + } + let r = b.u32()?; + if r >= MAX_ROWS || row.is_some_and(|previous| r <= previous) { + return Err(invalid("row is out of bounds or unordered")); + } + b.take(9)?; + let spans = b.u32()?; + if spans > 16 { + return Err(invalid("too many row spans")); + } + b.take(spans as usize * 8)?; + b.finish()?; + row = Some(r); + col = None; + } + 1..=11 | 62 => { + if !data { + return Err(invalid("cell outside worksheet data")); + } + let r = row.ok_or_else(|| invalid("cell before row header"))?; + let c = b.u32()?; + if c >= MAX_COLS || col.is_some_and(|previous| c <= previous) { + return Err(invalid("column is out of bounds or unordered")); + } + b.u32()?; // Style index and phonetic flag; formatting is not imported. + let value = value(id, &mut b, strings, budget)?; + b.finish()?; + if !value.is_empty() { + budget.cell()?; + sheet.cells.set(CellRef::new(r, c), Cell::value(value)); + } + col = Some(c); + } + 176 => { + if data { + return Err(invalid("merge inside worksheet data")); + } + let r0 = b.u32()?; + let r1 = b.u32()?; + let c0 = b.u32()?; + let c1 = b.u32()?; + b.finish()?; + if r0 > r1 || c0 > c1 || r1 >= MAX_ROWS || c1 >= MAX_COLS { + report.warn("Invalid XLSB merged ranges were skipped."); + continue; + } + if sheet.merges.len() >= 100_000 { + return Err(IoError::TooLarge("XLSB exceeds 100000 merges".into())); + } + sheet.merges.push(RangeRef::new(CellRef::new(r0, c0), CellRef::new(r1, c1))); + } + _ => {} + } + } + Err(invalid("missing worksheet end record")) +} diff --git a/docs/cli.md b/docs/cli.md index 2cf5985..164ebfb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,6 +3,10 @@ Headless GridCraft: inspect, convert, evaluate, script and serve MCP without a window. Errors go to stderr with a non-zero exit code. +`.xlsb` files support [worksheet data import](xlsb-import.md). Formula cells use their last saved +values; source formatting and other workbook features are omitted. Import warnings go to stderr. +Convert to `.xlsx` to save the imported workbook; XLSB output is not supported. + ```sh cargo install --path apps/gridcraft-cli # or: cargo run -p gridcraft-cli -- ``` diff --git a/docs/xlsb-import.md b/docs/xlsb-import.md new file mode 100644 index 0000000..f513d75 --- /dev/null +++ b/docs/xlsb-import.md @@ -0,0 +1,22 @@ +# Importing Excel binary workbooks + +Open an `.xlsb` file with File → Open, drag it onto GridCraft, or pass it to the CLI. +GridCraft imports worksheet data into a new workbook and shows the conversion limits. + +The import reads worksheet names, visibility, numbers, text, booleans, errors and merged +cells. Formula cells become their **last saved results**, not live formulas. Those results +may be out of date if the source application did not recalculate before saving. + +Source formatting is omitted: dates, times, percentages and currencies initially display as +their underlying numbers. The workbook's date system is retained. Charts, images, names, +macros and other workbook features are not imported. GridCraft does not write XLSB files. + +Save opens Save As and suggests a new `.xlsx` file. The source XLSB file is not used as the +workbook's save destination; requests to save to `.xlsb` return an error. + +```sh +gridcraft-cli info source.xlsb +gridcraft-cli convert source.xlsb imported.xlsx +``` + +CLI import warnings go to stderr, leaving structured output on stdout available for scripts.