gridcraft/crates/engine/src/cmd/data.rs

1021 lines
42 KiB
Rust

//! Data tab: sort, filter, remove duplicates, text to columns, data validation, outline.
use std::cmp::Ordering;
use std::sync::Arc;
use gridcraft_core::{CellRef, RangeRef, Value, sort_compare};
use gridcraft_model::*;
use serde_json::{Value as Json, json};
use super::*;
use crate::selection::current_region;
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!("data.sortAscending", "Sort A to Z", ["Data", "Sort & Filter"], None, "{range?, column?: \"B\"}", has_doc, |s, p| quick_sort(
s, p, true
)),
cmd!("data.sortDescending", "Sort Z to A", ["Data", "Sort & Filter"], None, "{range?, column?}", has_doc, |s, p| quick_sort(s, p, false)),
cmd!(
"data.sort",
"Sort…",
["Data", "Sort & Filter"],
None,
"{range?, header?: bool, keys: [{column: \"B\", order: asc|desc, by?: values|cellColor|fontColor, customList?: [..]}], orientation?: rows|columns, matchCase?}",
has_doc,
sort
),
cmd!(
"data.filter",
"Filter",
["Data", "Sort & Filter"],
Some("Cmd+Shift+F"),
"{range?} toggles AutoFilter on the current region",
has_doc,
toggle_filter
),
cmd!(
"data.filterBy",
"Filter Column",
[],
None,
"{column: \"B\" (or 0-based offset), values?: [..], blanks?: bool, custom?: {op: \">\", value: \"10\", op2?, value2?, and?}, top?: {count, bottom?, percent?}, aboveAverage?: bool, clear?: bool}",
has_doc,
filter_by
),
cmd!("data.clearFilter", "Clear", ["Data", "Sort & Filter"], None, "{}", has_doc, clear_filter),
cmd!("data.reapply", "Reapply", ["Data", "Sort & Filter"], None, "{}", has_doc, reapply),
cmd!(
"data.removeDuplicates",
"Remove Duplicates",
["Data", "Data Tools"],
None,
"{range?, columns?: [\"A\",\"C\"], header?: bool}",
has_doc,
remove_duplicates
),
cmd!(
"data.textToColumns",
"Text to Columns",
["Data", "Data Tools"],
None,
"{range?, delimiters?: [\",\", \"\\t\", \" \", \";\"], other?: \"|\", fixedWidths?: [5, 10], treatConsecutive?: bool, textQualifier?: \"\\\"\", destination?: \"B1\"}",
has_doc,
text_to_columns
),
cmd!(
"data.validation",
"Data Validation…",
["Data", "Data Tools"],
None,
"{range?, type: any|whole|decimal|list|date|time|textLength|custom, operator?: between|notBetween|equal|notEqual|greater|less|greaterOrEqual|lessOrEqual, formula1, formula2?, ignoreBlank?, dropdown?, inputTitle?, inputMessage?, errorStyle?: stop|warning|information, errorTitle?, errorMessage?, clear?: bool}",
has_doc,
validation
),
cmd!(query "data.validate", "Check Value Against Validation", [], None, "{cell?, input} → {ok, message?}", has_doc, validate_cmd),
cmd!(noundo "data.circleInvalid", "Circle Invalid Data", ["Data", "Data Tools", "Data Validation"], None, "{} → invalid cells", has_doc, circle_invalid),
cmd!("data.group", "Group", ["Data", "Outline"], Some("Cmd+Shift+K"), "{rows?: \"2:5\" | cols?: \"B:C\"}", has_doc, |s, p| group(s, p, true)),
cmd!("data.ungroup", "Ungroup", ["Data", "Outline"], Some("Cmd+Shift+J"), "{rows? | cols?}", has_doc, |s, p| group(s, p, false)),
cmd!("data.hideDetail", "Hide Detail", ["Data", "Outline"], None, "{rows? | cols?}", has_doc, |s, p| detail(s, p, true)),
cmd!("data.showDetail", "Show Detail", ["Data", "Outline"], None, "{rows? | cols?}", has_doc, |s, p| detail(s, p, false)),
cmd!(
"data.subtotal",
"Subtotal",
["Data", "Outline"],
None,
"{range?, groupBy: \"A\", function?: sum|count|average|max|min|product, columns: [\"C\"], header?: true}",
has_doc,
subtotal
),
cmd!("data.flashFill", "Flash Fill", ["Data", "Data Tools"], None, "{range?}", has_doc, |s, p| s.execute("edit.flashFill", p.clone())),
]
}
fn col_param(v: Option<&Json>, base: u32) -> Option<u32> {
match v? {
Json::String(s) => gridcraft_core::letters_to_col(s.trim_start_matches('$')),
Json::Number(n) => n.as_u64().map(|o| base + o as u32),
_ => None,
}
}
/// The range a data command works on: the selection when it's a block, else the current
/// region (trimmed to the used range for whole columns).
fn data_range(s: &Session, p: &Json) -> Result<RangeRef> {
let d = s.doc()?;
let sh = d.wb.active().ok_or(EngineError::NoDocument)?;
if p.get("range").is_some() {
return target_range(s, p);
}
if let Some(t) = sh.table_at(d.selection.active) {
return Ok(t.data_range().map(|dr| RangeRef::new(t.range.start, dr.end)).unwrap_or(t.range));
}
let sel = d.selection.current();
if !sel.is_single() {
let used = sh.used_range().unwrap_or(sel);
return Ok(sel.intersection(&used).unwrap_or(sel));
}
Ok(current_region(sh, d.selection.active))
}
/// Excel's header guess: the first row is text over a column of numbers, or is bold.
fn guess_header(wb: &Workbook, sh: &Sheet, r: RangeRef) -> bool {
if let Some(t) = sh.table_at(r.start) {
return t.header_row;
}
if r.height() < 2 {
return false;
}
let mut votes = 0i32;
for c in r.start.col..=r.end.col {
let a = sh.value(CellRef::new(r.start.row, c));
let b = sh.value(CellRef::new(r.start.row + 1, c));
if a.is_text() && !b.is_text() && !b.is_empty() {
votes += 1;
}
if wb.styles.get(sh.style_id(CellRef::new(r.start.row, c))).font.bold
!= wb.styles.get(sh.style_id(CellRef::new(r.start.row + 1, c))).font.bold
{
votes += 1;
}
if a.is_text() && b.is_text() {
votes -= 0;
}
}
votes > 0
}
#[derive(Clone)]
struct SortKey {
col: u32,
desc: bool,
by: String,
list: Option<Vec<String>>,
}
fn quick_sort(s: &mut Session, p: &Json, asc: bool) -> Result<Json> {
let r = data_range(s, p)?;
let col = col_param(p.get("column"), r.start.col).unwrap_or(s.doc()?.selection.active.col).clamp(r.start.col, r.end.col);
let d = s.doc()?;
let sh = d.wb.active().ok_or(EngineError::NoDocument)?;
let header = guess_header(&d.wb, sh, r);
do_sort(s, r, header, vec![SortKey { col, desc: !asc, by: "values".into(), list: None }], false, false)
}
fn sort(s: &mut Session, p: &Json) -> Result<Json> {
let r = data_range(s, p)?;
let d = s.doc()?;
let sh = d.wb.active().ok_or(EngineError::NoDocument)?;
let header = bool_param(p, "header").unwrap_or_else(|| guess_header(&d.wb, sh, r));
let by_cols = str_param(p, "orientation") == Some("columns");
let case = bool_param(p, "matchCase").unwrap_or(false);
let keys: Vec<SortKey> = p
.get("keys")
.and_then(Json::as_array)
.map(|a| {
a.iter()
.filter_map(|k| {
let col = if by_cols {
k.get("row").and_then(Json::as_u64).map(|r| r as u32 - 1).or_else(|| col_param(k.get("column"), r.start.row))?
} else {
col_param(k.get("column"), r.start.col)?
};
Some(SortKey {
col,
desc: k.get("order").and_then(Json::as_str) == Some("desc"),
by: k.get("by").and_then(Json::as_str).unwrap_or("values").to_string(),
list: k.get("customList").and_then(Json::as_array).map(|l| l.iter().filter_map(|x| x.as_str().map(str::to_string)).collect()),
})
})
.collect()
})
.unwrap_or_default();
if keys.is_empty() {
s.ui_requests.push(crate::UiRequest::Dialog("sort".into(), json!({"range": r.a1(), "header": header})));
return ok();
}
if keys.len() > 64 {
return Err(bad("data.sort", "too many sort levels (64 max)"));
}
do_sort(s, r, header, keys, by_cols, case)
}
fn do_sort(s: &mut Session, r: RangeRef, header: bool, keys: Vec<SortKey>, by_cols: bool, case: bool) -> Result<Json> {
let body = if by_cols {
if header && r.width() > 1 { RangeRef::new(CellRef::new(r.start.row, r.start.col + 1), r.end) } else { r }
} else if header && r.height() > 1 {
RangeRef::new(CellRef::new(r.start.row + 1, r.start.col), r.end)
} else {
r
};
if body.count() > 20_000_000 {
return Err(bad("data.sort", "range too large"));
}
let sheet = s.doc()?.wb.active_sheet;
edit(s, |cx| {
let Some(sh) = cx.wb.sheet(sheet) else { return Ok(Json::Null) };
if sh.merges.iter().any(|m| m.intersects(&body) && !m.is_single()) {
return Err(EngineError::Other("To do this, all the merged cells need to be the same size.".into()));
}
let n = if by_cols { body.width() } else { body.height() };
let line = |i: u32, k: u32| if by_cols { CellRef::new(k, body.start.col + i) } else { CellRef::new(body.start.row + i, k) };
let theme = cx.wb.theme.clone();
let styles = &cx.wb.styles;
let key_val = |i: u32, k: &SortKey| -> (Value, u32) {
let c = line(i, k.col);
let v = sh.value(c);
let color = match k.by.as_str() {
"cellColor" => {
styles.get(sh.style_id(c)).fill.fg.resolve(&theme).map(|[r, g, b]| u32::from_be_bytes([0, r, g, b])).unwrap_or(u32::MAX)
}
"fontColor" => {
styles.get(sh.style_id(c)).font.color.resolve(&theme).map(|[r, g, b]| u32::from_be_bytes([0, r, g, b])).unwrap_or(u32::MAX)
}
_ => 0,
};
(v, color)
};
let mut order: Vec<u32> = (0..n).collect();
let keyed: Vec<Vec<(Value, u32)>> = (0..n).map(|i| keys.iter().map(|k| key_val(i, k)).collect()).collect();
order.sort_by(|&a, &b| {
for (ki, k) in keys.iter().enumerate() {
let (va, ca) = &keyed[a as usize][ki];
let (vb, cb) = &keyed[b as usize][ki];
let o = if k.by != "values" {
ca.cmp(cb)
} else if let Some(list) = &k.list {
let pos = |v: &Value| v.as_text().and_then(|t| list.iter().position(|x| x.eq_ignore_ascii_case(t))).unwrap_or(usize::MAX);
pos(va).cmp(&pos(vb))
} else if case && va.is_text() && vb.is_text() {
va.as_text().unwrap_or("").cmp(vb.as_text().unwrap_or(""))
} else {
sort_compare(va, vb)
};
// Blanks always sort last, whatever the order.
let o = if k.desc && !(va.is_empty() || vb.is_empty()) { o.reverse() } else { o };
if o != Ordering::Equal {
return o;
}
}
Ordering::Equal
});
if order.iter().enumerate().all(|(i, &o)| i as u32 == o) {
return Ok(Json::Null);
}
// Move whole lines (cells with formulas adjusted for relative references).
let lanes: Vec<u32> = if by_cols { (body.start.row..=body.end.row).collect() } else { (body.start.col..=body.end.col).collect() };
let mut moved: Vec<(CellRef, Option<Cell>)> = Vec::with_capacity(body.count() as usize);
for (dst_i, &src_i) in order.iter().enumerate() {
for &k in &lanes {
let src = line(src_i, k);
let dst = line(dst_i as u32, k);
let mut cell = sh.cell(src).cloned();
if let Some(c) = cell.as_mut()
&& let Some(f) = &c.formula
&& let Some(e) = f.expr()
{
let shifted = gridcraft_formula::adjust::shift_relative(e, dst.row as i64 - src.row as i64, dst.col as i64 - src.col as i64);
c.formula = Some(Arc::new(Formula::from_expr(shifted)));
}
moved.push((dst, cell));
}
}
// Row heights travel with rows.
let heights: Vec<Option<LineInfo>> =
if by_cols { vec![] } else { order.iter().map(|&i| sh.rows.get(&(body.start.row + i)).copied()).collect() };
let shm = cx.sheet_mut(sheet)?;
for (c, cell) in &moved {
match cell {
Some(x) => shm.cells.set(*c, x.clone()),
None => {
shm.cells.remove(*c);
}
}
}
for (i, h) in heights.into_iter().enumerate() {
let row = body.start.row + i as u32;
match h {
Some(info) => {
shm.rows.insert(row, LineInfo { hidden: shm.rows.get(&row).is_some_and(|x| x.hidden), ..info });
}
None => {
if let Some(e) = shm.rows.get_mut(&row) {
e.size = None;
}
}
}
}
for (c, _) in moved {
cx.changed.push((sheet, c));
}
cx.structural = true;
Ok(Json::Null)
})?;
Ok(json!({"sorted": body.a1()}))
}
// ---------------------------------------------------------------- filter
fn toggle_filter(s: &mut Session, p: &Json) -> Result<Json> {
let sheet = s.doc()?.wb.active_sheet;
let has = s.doc()?.wb.sheet(sheet).is_some_and(|sh| sh.autofilter.is_some());
let in_table =
s.doc()?.wb.sheet(sheet).and_then(|sh| sh.table_at(s.doc().map(|d| d.selection.active).unwrap_or_default()).map(|t| t.name.clone()));
if let Some(name) = in_table {
return edit(s, |cx| {
let sh = cx.sheet_mut(sheet)?;
if let Some(t) = sh.tables.iter_mut().find(|t| t.name == name) {
t.filter_button = !t.filter_button;
}
Ok(Json::Null)
});
}
if has {
return edit(s, |cx| {
let sh = cx.sheet_mut(sheet)?;
sh.autofilter = None;
for info in sh.rows.values_mut() {
info.hidden = false;
}
cx.structural = true;
Ok(json!({"filter": false}))
});
}
let r = data_range(s, p)?;
edit(s, |cx| {
cx.sheet_mut(sheet)?.autofilter = Some(AutoFilter { range: r, criteria: vec![] });
Ok(json!({"filter": true, "range": r.a1()}))
})
}
fn filter_by(s: &mut Session, p: &Json) -> Result<Json> {
let sheet = s.doc()?.wb.active_sheet;
let Some(sh) = s.doc()?.wb.sheet(sheet) else { return Err(EngineError::NoDocument) };
let af = match &sh.autofilter {
Some(a) => a.clone(),
None => {
toggle_filter(s, &json!({}))?;
s.doc()?.wb.sheet(sheet).and_then(|sh| sh.autofilter.clone()).ok_or_else(|| EngineError::Other("no filter".into()))?
}
};
let col = col_param(p.get("column"), af.range.start.col).ok_or_else(|| bad("data.filterBy", "missing `column`"))?;
if col < af.range.start.col || col > af.range.end.col {
return Err(bad("data.filterBy", "column outside the filter range"));
}
let off = col - af.range.start.col;
let crit = if bool_param(p, "clear").unwrap_or(false) {
None
} else if let Some(vals) = p.get("values").and_then(Json::as_array) {
Some(FilterCriterion::Values {
values: vals.iter().map(super::edit::json_to_input).collect(),
blanks: bool_param(p, "blanks").unwrap_or(false),
})
} else if let Some(c) = p.get("custom") {
let a = (c.get("op").and_then(Json::as_str).unwrap_or("=").to_string(), c.get("value").map(super::edit::json_to_input).unwrap_or_default());
let b = c.get("op2").and_then(Json::as_str).map(|o| (o.to_string(), c.get("value2").map(super::edit::json_to_input).unwrap_or_default()));
Some(FilterCriterion::Custom { a, b, and: c.get("and").and_then(Json::as_bool).unwrap_or(true) })
} else if let Some(t) = p.get("top") {
Some(FilterCriterion::Top10 {
bottom: t.get("bottom").and_then(Json::as_bool).unwrap_or(false),
percent: t.get("percent").and_then(Json::as_bool).unwrap_or(false),
count: t.get("count").and_then(Json::as_u64).unwrap_or(10) as u32,
})
} else {
bool_param(p, "aboveAverage").map(FilterCriterion::AboveAverage)
};
edit(s, |cx| {
let sh = cx.sheet_mut(sheet)?;
if let Some(a) = sh.autofilter.as_mut() {
a.criteria.retain(|(o, _)| *o != off);
if let Some(c) = crit.clone() {
a.criteria.push((off, c));
}
}
apply_filter(&mut cx.wb, sheet);
cx.structural = true;
Ok(Json::Null)
})?;
let hidden = s.doc()?.wb.sheet(sheet).map(|sh| sh.rows.values().filter(|r| r.hidden).count()).unwrap_or(0);
Ok(json!({"hiddenRows": hidden}))
}
/// Does a value pass a custom comparison (`>`, `=*abc*`, `<>x`…)?
fn custom_ok(v: &Value, text: &str, op: &str, crit: &str) -> bool {
let num = gridcraft_core::parse::parse_number_text(crit);
let cmp = match (v.as_f64(), num) {
(Some(a), Some(b)) => a.partial_cmp(&b),
_ => Some(gridcraft_core::compare_text(text, crit)),
};
let wild = || super::edit::wildcard(&text.to_lowercase(), &crit.to_lowercase());
match op {
"=" => {
if crit.contains(['*', '?']) {
wild()
} else {
cmp == Some(Ordering::Equal)
}
}
"<>" => {
if crit.contains(['*', '?']) {
!wild()
} else {
cmp != Some(Ordering::Equal)
}
}
">" => cmp == Some(Ordering::Greater),
"<" => cmp == Some(Ordering::Less),
">=" => matches!(cmp, Some(Ordering::Greater | Ordering::Equal)),
"<=" => matches!(cmp, Some(Ordering::Less | Ordering::Equal)),
"beginsWith" => text.to_lowercase().starts_with(&crit.to_lowercase()),
"endsWith" => text.to_lowercase().ends_with(&crit.to_lowercase()),
"contains" => text.to_lowercase().contains(&crit.to_lowercase()),
"notContains" => !text.to_lowercase().contains(&crit.to_lowercase()),
_ => true,
}
}
/// Hides rows of the autofilter range that fail any criterion.
pub(crate) fn apply_filter(wb: &mut Workbook, sheet: usize) {
let Some(sh) = wb.sheet(sheet) else { return };
let Some(af) = sh.autofilter.clone() else { return };
let r = af.range;
let mut hide = Vec::new();
// Precompute per-column numbers for top10/average.
let mut col_stats: std::collections::HashMap<u32, (Vec<f64>, f64)> = std::collections::HashMap::new();
for (off, c) in &af.criteria {
if matches!(c, FilterCriterion::Top10 { .. } | FilterCriterion::AboveAverage(_)) {
let col = r.start.col + off;
let nums: Vec<f64> = (r.start.row + 1..=r.end.row).filter_map(|row| sh.value(CellRef::new(row, col)).as_f64()).collect();
let avg = if nums.is_empty() { 0.0 } else { nums.iter().sum::<f64>() / nums.len() as f64 };
col_stats.insert(*off, (nums, avg));
}
}
for row in r.start.row + 1..=r.end.row {
let mut visible = true;
for (off, crit) in &af.criteria {
let c = CellRef::new(row, r.start.col + off);
let v = sh.value(c);
let text = crate::display::cell_text(wb, sh, c);
let ok = match crit {
FilterCriterion::Values { values, blanks } => {
if v.is_empty() {
*blanks
} else {
values.iter().any(|x| x.eq_ignore_ascii_case(&text) || x.eq_ignore_ascii_case(&v.display()))
}
}
FilterCriterion::Custom { a, b, and } => {
let x = custom_ok(&v, &text, &a.0, &a.1);
match b {
Some(b) => {
let y = custom_ok(&v, &text, &b.0, &b.1);
if *and { x && y } else { x || y }
}
None => x,
}
}
FilterCriterion::Top10 { bottom, percent, count } => match (v.as_f64(), col_stats.get(off)) {
(Some(n), Some((nums, _))) => {
let mut sorted = nums.clone();
sorted.sort_by(|a, b| if *bottom { a.partial_cmp(b) } else { b.partial_cmp(a) }.unwrap_or(Ordering::Equal));
let k = if *percent { ((sorted.len() as f64) * (*count as f64) / 100.0).ceil() as usize } else { *count as usize };
let k = k.clamp(1, sorted.len().max(1));
let thr = sorted.get(k - 1).copied().unwrap_or(n);
if *bottom { n <= thr } else { n >= thr }
}
_ => false,
},
FilterCriterion::AboveAverage(above) => match (v.as_f64(), col_stats.get(off)) {
(Some(n), Some((_, avg))) => {
if *above {
n > *avg
} else {
n < *avg
}
}
_ => false,
},
FilterCriterion::FillColor(col) => wb.styles.get(sh.style_id(c)).fill.fg == *col,
FilterCriterion::FontColor(col) => wb.styles.get(sh.style_id(c)).font.color == *col,
};
if !ok {
visible = false;
break;
}
}
hide.push((row, !visible));
}
let Some(shm) = wb.sheet_mut(sheet) else { return };
for (row, h) in hide {
if h {
shm.rows.entry(row).or_default().hidden = true;
} else if let Some(e) = shm.rows.get_mut(&row) {
e.hidden = false;
}
}
}
fn clear_filter(s: &mut Session, _: &Json) -> Result<Json> {
let sheet = s.doc()?.wb.active_sheet;
edit(s, |cx| {
let sh = cx.sheet_mut(sheet)?;
if let Some(a) = sh.autofilter.as_mut() {
a.criteria.clear();
}
apply_filter(&mut cx.wb, sheet);
cx.structural = true;
Ok(Json::Null)
})
}
fn reapply(s: &mut Session, _: &Json) -> Result<Json> {
let sheet = s.doc()?.wb.active_sheet;
edit(s, |cx| {
apply_filter(&mut cx.wb, sheet);
cx.structural = true;
Ok(Json::Null)
})
}
/// Distinct display values of a filter column (for the dropdown list).
pub fn filter_values(wb: &Workbook, sheet: usize, col: u32) -> Vec<(String, bool)> {
let Some(sh) = wb.sheet(sheet) else { return vec![] };
let Some(af) = &sh.autofilter else { return vec![] };
let off = col.saturating_sub(af.range.start.col);
let selected: Option<&FilterCriterion> = af.criteria.iter().find(|(o, _)| *o == off).map(|(_, c)| c);
let mut seen = std::collections::BTreeMap::new();
for row in af.range.start.row + 1..=af.range.end.row.min(af.range.start.row + 1_000_000) {
let c = CellRef::new(row, col);
let v = sh.value(c);
let t = if v.is_empty() { "(Blanks)".to_string() } else { crate::display::cell_text(wb, sh, c) };
let on = match selected {
Some(FilterCriterion::Values { values, blanks }) => {
if v.is_empty() {
*blanks
} else {
values.iter().any(|x| x.eq_ignore_ascii_case(&t))
}
}
_ => true,
};
seen.entry((sort_key(&v), t)).or_insert(on);
}
seen.into_iter().map(|((_, t), on)| (t, on)).collect()
}
fn sort_key(v: &Value) -> (u8, i64, String) {
match v {
Value::Number(n) => (0, (n * 1e6) as i64, String::new()),
Value::Text(t) => (1, 0, t.to_lowercase()),
Value::Bool(b) => (2, *b as i64, String::new()),
Value::Empty => (9, 0, String::new()),
_ => (3, 0, v.display()),
}
}
// ---------------------------------------------------------------- tools
fn remove_duplicates(s: &mut Session, p: &Json) -> Result<Json> {
let r = data_range(s, p)?;
let d = s.doc()?;
let sh = d.wb.active().ok_or(EngineError::NoDocument)?;
let header = bool_param(p, "header").unwrap_or_else(|| guess_header(&d.wb, sh, r));
let cols: Vec<u32> = p
.get("columns")
.and_then(Json::as_array)
.map(|a| a.iter().filter_map(|c| col_param(Some(c), r.start.col)).collect())
.unwrap_or_else(|| (r.start.col..=r.end.col).collect());
let sheet = d.wb.active_sheet;
let start = r.start.row + header as u32;
let mut removed = 0;
edit(s, |cx| {
let Some(sh) = cx.wb.sheet(sheet) else { return Ok(()) };
let mut seen = std::collections::HashSet::new();
let mut keep: Vec<u32> = Vec::new();
for row in start..=r.end.row {
let key: Vec<String> = cols.iter().map(|c| sh.value(CellRef::new(row, *c)).display().to_lowercase()).collect();
if seen.insert(key) {
keep.push(row);
} else {
removed += 1;
}
}
let rows: Vec<Vec<Option<Cell>>> =
keep.iter().map(|row| (r.start.col..=r.end.col).map(|c| sh.cell(CellRef::new(*row, c)).cloned()).collect()).collect();
let shm = cx.sheet_mut(sheet)?;
for row in start..=r.end.row {
for c in r.start.col..=r.end.col {
shm.cells.remove(CellRef::new(row, c));
}
}
for (i, cells) in rows.into_iter().enumerate() {
for (j, cell) in cells.into_iter().enumerate() {
if let Some(cell) = cell {
shm.cells.set(CellRef::new(start + i as u32, r.start.col + j as u32), cell);
}
}
}
for c in RangeRef::new(CellRef::new(start, r.start.col), r.end).iter() {
cx.changed.push((sheet, c));
}
Ok(())
})?;
let remaining = (r.end.row + 1 - start) as usize - removed;
Ok(json!({"removed": removed, "remaining": remaining}))
}
fn text_to_columns(s: &mut Session, p: &Json) -> Result<Json> {
let r = target_range(s, p)?;
let sheet = target_sheet(s, p)?;
let mut delims: Vec<char> = p
.get("delimiters")
.and_then(Json::as_array)
.map(|a| a.iter().filter_map(|d| d.as_str().and_then(|x| x.chars().next())).collect())
.unwrap_or_else(|| vec!['\t']);
if let Some(o) = str_param(p, "other").and_then(|o| o.chars().next()) {
delims.push(o);
}
let fixed: Option<Vec<usize>> =
p.get("fixedWidths").and_then(Json::as_array).map(|a| a.iter().filter_map(|x| x.as_u64().map(|v| v as usize)).collect());
let consecutive = bool_param(p, "treatConsecutive").unwrap_or(false);
let quote = str_param(p, "textQualifier").and_then(|q| q.chars().next()).unwrap_or('"');
let dest = cell_param(p, "destination").unwrap_or(r.start);
edit(s, |cx| {
let Some(sh) = cx.wb.sheet(sheet) else { return Ok(()) };
let mut out: Vec<(CellRef, String)> = Vec::new();
for (i, row) in (r.start.row..=r.end.row).enumerate() {
let text = sh.value(CellRef::new(row, r.start.col)).display();
let parts: Vec<String> = match &fixed {
Some(w) => {
let chars: Vec<char> = text.chars().collect();
let mut cuts = vec![0];
cuts.extend(w.iter().copied().filter(|x| *x < chars.len()));
cuts.push(chars.len());
cuts.windows(2)
.map(|p| chars.get(p[0]..p[1]).map(|s| s.iter().collect::<String>().trim().to_string()).unwrap_or_default())
.collect()
}
None => split_delimited(&text, &delims, consecutive, quote),
};
for (j, part) in parts.into_iter().enumerate() {
out.push((CellRef::new(dest.row + i as u32, dest.col + j as u32), part));
}
}
for (c, t) in out {
let cell = super::edit::input_to_cell(&t, sheet, c, &mut cx.wb).unwrap_or(None);
let shm = cx.sheet_mut(sheet)?;
match cell {
Some(x) => shm.cells.set(c, x),
None => {
shm.cells.remove(c);
}
}
cx.changed.push((sheet, c));
}
Ok(())
})?;
ok()
}
fn split_delimited(text: &str, delims: &[char], consecutive: bool, quote: char) -> Vec<String> {
let mut out = Vec::new();
let mut cur = String::new();
let mut in_q = false;
let mut last_delim = false;
let mut chars = text.chars().peekable();
while let Some(c) = chars.next() {
if c == quote {
if in_q && chars.peek() == Some(&quote) {
cur.push(quote);
chars.next();
} else {
in_q = !in_q;
}
continue;
}
if !in_q && delims.contains(&c) {
if consecutive && last_delim {
continue;
}
out.push(std::mem::take(&mut cur));
last_delim = true;
continue;
}
last_delim = false;
cur.push(c);
}
out.push(cur);
out
}
// ---------------------------------------------------------------- validation
fn op_param(s: Option<&str>) -> CfOperator {
match s.unwrap_or("between") {
"notBetween" => CfOperator::NotBetween,
"equal" => CfOperator::Equal,
"notEqual" => CfOperator::NotEqual,
"greater" | "greaterThan" => CfOperator::Greater,
"less" | "lessThan" => CfOperator::Less,
"greaterOrEqual" | "greaterThanOrEqual" => CfOperator::GreaterOrEqual,
"lessOrEqual" | "lessThanOrEqual" => CfOperator::LessOrEqual,
_ => CfOperator::Between,
}
}
fn validation(s: &mut Session, p: &Json) -> Result<Json> {
let ranges = target_ranges(s, p)?;
let sheet = target_sheet(s, p)?;
if bool_param(p, "clear").unwrap_or(false) {
return edit(s, |cx| {
let sh = cx.sheet_mut(sheet)?;
for dv in sh.validations.iter_mut() {
dv.ranges.retain(|r| !ranges.iter().any(|x| x.contains_range(r)));
}
sh.validations.retain(|d| !d.ranges.is_empty());
Ok(Json::Null)
});
}
let Some(kind) = str_param(p, "type") else {
s.ui_requests.push(crate::UiRequest::Dialog("dataValidation".into(), json!({})));
return ok();
};
let kind = match kind {
"whole" => ValidationKind::Whole,
"decimal" => ValidationKind::Decimal,
"list" => ValidationKind::List,
"date" => ValidationKind::Date,
"time" => ValidationKind::Time,
"textLength" => ValidationKind::TextLength,
"custom" => ValidationKind::Custom,
_ => ValidationKind::Any,
};
let dv = Validation {
ranges: ranges.clone(),
kind,
op: op_param(str_param(p, "operator")),
f1: str_param(p, "formula1").unwrap_or("").trim_start_matches('=').to_string(),
f2: str_param(p, "formula2").map(|f| f.trim_start_matches('=').to_string()),
allow_blank: bool_param(p, "ignoreBlank").unwrap_or(true),
in_cell_dropdown: bool_param(p, "dropdown").unwrap_or(true),
input_title: str_param(p, "inputTitle").unwrap_or("").into(),
input_message: str_param(p, "inputMessage").unwrap_or("").into(),
show_input: true,
error_title: str_param(p, "errorTitle").unwrap_or("").into(),
error_message: str_param(p, "errorMessage").unwrap_or("").into(),
error_style: match str_param(p, "errorStyle") {
Some("warning") => ErrorStyle::Warning,
Some("information") => ErrorStyle::Information,
_ => ErrorStyle::Stop,
},
show_error: true,
};
edit(s, |cx| {
let sh = cx.sheet_mut(sheet)?;
for v in sh.validations.iter_mut() {
v.ranges.retain(|r| !ranges.iter().any(|x| x.contains_range(r)));
}
sh.validations.retain(|d| !d.ranges.is_empty());
sh.validations.push(dv.clone());
Ok(Json::Null)
})
}
/// List entries of a list validation (literal or from a range).
pub fn list_items(wb: &Workbook, sheet: usize, dv: &Validation) -> Vec<String> {
let f = dv.f1.trim();
if f.starts_with('"')
|| (!f.contains('!')
&& gridcraft_formula::parse(f).map(|e| !matches!(e, gridcraft_formula::Expr::Ref(_) | gridcraft_formula::Expr::Name(_))).unwrap_or(true))
{
return f.trim_matches('"').split(',').map(|x| x.trim().to_string()).filter(|x| !x.is_empty()).collect();
}
let at = dv.ranges.first().map(|r| r.start).unwrap_or_default();
match gridcraft_calc::evaluate(wb, sheet, at, f) {
Value::Array(a) => a.data.iter().filter(|v| !v.is_empty()).map(Value::display).collect(),
v if !v.is_empty() && !v.is_error() => vec![v.display()],
_ => vec![],
}
}
/// Checks typed input against the validation of a cell. `Ok(None)` = valid.
pub fn check_validation(wb: &Workbook, sheet: usize, at: CellRef, input: &str) -> Option<(Validation, String)> {
let sh = wb.sheet(sheet)?;
let dv = sh.validations.iter().find(|d| d.ranges.iter().any(|r| r.contains(at)))?;
if dv.kind == ValidationKind::Any || !dv.show_error {
return None;
}
if input.is_empty() && dv.allow_blank {
return None;
}
let v = gridcraft_core::parse::parse_input(input, wb.date_system).value;
let num = |f: &str| gridcraft_calc::evaluate(wb, sheet, at, f).to_number().ok();
let ok = match dv.kind {
ValidationKind::List => list_items(wb, sheet, dv).iter().any(|x| x.eq_ignore_ascii_case(input)),
ValidationKind::Custom => gridcraft_calc::evaluate(wb, sheet, at, &dv.f1).to_bool().unwrap_or(false),
ValidationKind::TextLength => {
let n = input.chars().count() as f64;
compare_op(dv.op, n, num(&dv.f1), dv.f2.as_deref().and_then(num))
}
_ => match v.as_f64() {
Some(n) => {
if dv.kind == ValidationKind::Whole && n.fract() != 0.0 {
false
} else {
compare_op(dv.op, n, num(&dv.f1), dv.f2.as_deref().and_then(num))
}
}
None => false,
},
};
if ok {
return None;
}
let msg = if dv.error_message.is_empty() {
"This value doesn't match the data validation restrictions defined for this cell.".to_string()
} else {
dv.error_message.clone()
};
Some((dv.clone(), msg))
}
fn compare_op(op: CfOperator, n: f64, a: Option<f64>, b: Option<f64>) -> bool {
let Some(a) = a else { return false };
match op {
CfOperator::Between => b.is_some_and(|b| n >= a.min(b) && n <= a.max(b)),
CfOperator::NotBetween => b.is_some_and(|b| n < a.min(b) || n > a.max(b)),
CfOperator::Equal => n == a,
CfOperator::NotEqual => n != a,
CfOperator::Greater => n > a,
CfOperator::Less => n < a,
CfOperator::GreaterOrEqual => n >= a,
CfOperator::LessOrEqual => n <= a,
}
}
fn validate_cmd(s: &mut Session, p: &Json) -> Result<Json> {
let d = s.doc()?;
let at = cell_param(p, "cell").unwrap_or(d.selection.active);
let input = str_param(p, "input").unwrap_or("");
match check_validation(&d.wb, d.wb.active_sheet, at, input) {
None => Ok(json!({"ok": true})),
Some((dv, msg)) => Ok(json!({"ok": false, "message": msg, "title": dv.error_title, "style": format!("{:?}", dv.error_style)})),
}
}
fn circle_invalid(s: &mut Session, _: &Json) -> Result<Json> {
let d = s.doc()?;
let sheet = d.wb.active_sheet;
let Some(sh) = d.wb.sheet(sheet) else { return Err(EngineError::NoDocument) };
let mut bad_cells = Vec::new();
for dv in &sh.validations {
for r in &dv.ranges {
let r = sh.used_range().and_then(|u| r.intersection(&u));
let Some(r) = r else { continue };
for c in r.iter().take(100_000) {
let text = sh.cell(c).map(|x| x.input_text()).unwrap_or_default();
if !text.is_empty() && check_validation(&d.wb, sheet, c, &text).is_some() {
bad_cells.push(c.a1());
}
}
}
}
Ok(json!({"invalid": bad_cells}))
}
// ---------------------------------------------------------------- outline
fn outline_lines(s: &Session, p: &Json) -> Result<(bool, u32, u32)> {
if let Some(r) = str_param(p, "rows").and_then(RangeRef::parse) {
return Ok((true, r.start.row, r.end.row));
}
if let Some(r) = str_param(p, "cols").and_then(RangeRef::parse) {
return Ok((false, r.start.col, r.end.col));
}
let r = s.doc()?.selection.current();
if r.is_full_cols() && !r.is_full_rows() { Ok((false, r.start.col, r.end.col)) } else { Ok((true, r.start.row, r.end.row)) }
}
fn group(s: &mut Session, p: &Json, on: bool) -> Result<Json> {
let (rows, a, b) = outline_lines(s, p)?;
edit(s, |cx| {
let si = cx.sheet_index();
let sh = cx.sheet_mut(si)?;
let map = if rows { &mut sh.rows } else { &mut sh.cols };
for i in a..=b.min(a + 1_048_575) {
let e = map.entry(i).or_default();
e.outline = if on { (e.outline + 1).min(7) } else { e.outline.saturating_sub(1) };
if !on && e.outline == 0 {
e.hidden = false;
}
}
Ok(Json::Null)
})
}
fn detail(s: &mut Session, p: &Json, hide: bool) -> Result<Json> {
let (rows, a, b) = outline_lines(s, p)?;
edit(s, |cx| {
let si = cx.sheet_index();
let sh = cx.sheet_mut(si)?;
let map = if rows { &mut sh.rows } else { &mut sh.cols };
// Expand to the whole group around the selection.
let level = (a..=b).filter_map(|i| map.get(&i).map(|e| e.outline)).max().unwrap_or(0);
if level == 0 {
return Ok(Json::Null);
}
let mut lo = a;
while lo > 0 && map.get(&(lo - 1)).is_some_and(|e| e.outline >= level) {
lo -= 1;
}
let mut hi = b;
while map.get(&(hi + 1)).is_some_and(|e| e.outline >= level) {
hi += 1;
}
for i in lo..=hi {
if let Some(e) = map.get_mut(&i) {
e.hidden = hide;
}
}
cx.structural = true;
Ok(Json::Null)
})
}
fn subtotal(s: &mut Session, p: &Json) -> Result<Json> {
let r = data_range(s, p)?;
let group_col = col_param(p.get("groupBy"), r.start.col).unwrap_or(r.start.col);
let func = str_param(p, "function").unwrap_or("sum");
let code = match func {
"count" => 3,
"average" => 1,
"max" => 4,
"min" => 5,
"product" => 6,
_ => 9,
};
let cols: Vec<u32> = p
.get("columns")
.and_then(Json::as_array)
.map(|a| a.iter().filter_map(|c| col_param(Some(c), r.start.col)).collect())
.unwrap_or_else(|| vec![r.end.col]);
let label = match func {
"count" => "Count",
"average" => "Average",
"max" => "Max",
"min" => "Min",
"product" => "Product",
_ => "Total",
};
let sheet = s.doc()?.wb.active_sheet;
// Group boundaries (header in the first row).
let d = s.doc()?;
let sh = d.wb.sheet(sheet).ok_or(EngineError::NoDocument)?;
let mut groups: Vec<(u32, u32, String)> = Vec::new();
let mut start = r.start.row + 1;
for row in r.start.row + 1..=r.end.row + 1 {
let cur = sh.value(CellRef::new(row, group_col)).display();
let prev = sh.value(CellRef::new(start, group_col)).display();
if row > r.end.row || cur != prev {
groups.push((start, row - 1, prev));
start = row;
}
}
// Insert a total row after each group, bottom-up so indices stay valid.
let mut inserted = 0;
for (g0, g1, name) in groups.iter().rev() {
s.execute("home.insertRows", json!({"rows": format!("{}:{}", g1 + 2, g1 + 2)}))?;
let row = g1 + 1;
let lbl = CellRef::new(row, group_col).a1();
s.execute("cell.set", json!({"cell": lbl, "input": format!("{name} {label}")}))?;
for c in &cols {
let range = RangeRef::new(CellRef::new(*g0, *c), CellRef::new(*g1, *c)).a1();
s.execute("cell.set", json!({"cell": CellRef::new(row, *c).a1(), "input": format!("=SUBTOTAL({code},{range})")}))?;
}
s.execute("home.bold", json!({"range": RangeRef::new(CellRef::new(row, r.start.col), CellRef::new(row, r.end.col)).a1(), "on": true}))?;
s.execute("data.group", json!({"rows": format!("{}:{}", g0 + 1, g1 + 1)}))?;
inserted += 1;
}
let last = r.end.row + inserted + 1;
let lbl = CellRef::new(last, group_col).a1();
s.execute("home.insertRows", json!({"rows": format!("{}:{}", last + 1, last + 1)}))?;
s.execute("cell.set", json!({"cell": lbl, "input": format!("Grand {label}")}))?;
for c in &cols {
let range = RangeRef::new(CellRef::new(r.start.row + 1, *c), CellRef::new(last - 1, *c)).a1();
s.execute("cell.set", json!({"cell": CellRef::new(last, *c).a1(), "input": format!("=SUBTOTAL({code},{range})")}))?;
}
Ok(json!({"groups": inserted}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split() {
assert_eq!(split_delimited("a,b,,c", &[','], false, '"'), ["a", "b", "", "c"]);
assert_eq!(split_delimited("a,b,,c", &[','], true, '"'), ["a", "b", "c"]);
assert_eq!(split_delimited("\"x,y\",z", &[','], false, '"'), ["x,y", "z"]);
}
}