Accuracy: round an exact tie at the 16th significant digit toward zero

Excel shows 15 significant digits. When a number is an exact tie at the 16th
(its exact decimal expansion is 15 digits, a 5, then only zeros), Excel
rounds it toward zero: TEXT(1234567890123455,"0") is 1234567890123450.
GridCraft took Rust's `{:.14e}`, which rounds that tie to even, so any tie
with an odd 15th digit came out one unit too high (…460).

decimal_digits now formats 16 digits (correct unless the 16th is a 5),
and only on a 5 checks the exact expansion (a 30-digit screen, then the
full one) and truncates a true tie. Non-ties, including values just above
or below a tie, round as before. One format serves almost every number.

Tests: ties of every 15th-digit parity, negatives, fractional ties, the
neighbours just above and below a tie, the carry after fifteen 9s
(999999999999999.5 shows 999999999999999), and unchanged ordinary cases.

Evidence: Windows Excel 16.0.20430 TEXT() over a 174-row probe and a 24-row
carry probe agree on every row except those under a percent format and
General over negatives (the two bugs named below). The suite's number-format
golden goes from 183,843 to 184,900 of 187,810 passing with no row that
passed before failing now.

	modified:   crates/numfmt/src/decimal.rs
	modified:   crates/numfmt/src/tests.rs
This commit is contained in:
Sebastian Whincop 2026-10-11 16:22:36 +13:00
parent 9f896c1249
commit aee898595f
2 changed files with 128 additions and 5 deletions

View File

@ -9,17 +9,70 @@ pub(crate) const MAX_DECIMALS: usize = 400;
/// The 15 significant digits of a positive finite number and the power of ten of the first
/// digit: `a ≈ d0.d1d2… × 10^exp`.
///
/// The value is rounded to the nearest 15-digit decimal. An exact tie at the 16th digit (the exact
/// decimal expansion is 15 digits, a 5, then only zeros, as for 1234567890123455 or
/// 123456789012345.5) goes toward zero, as Excel does: `1234567890123455` shows `1234567890123450`,
/// not `…460`. Excel observed (Windows 16.0.20430, TEXT()).
pub(crate) fn decimal_digits(a: f64) -> ([u8; 15], i32) {
let mut digits = [0u8; 15];
if !a.is_finite() || a <= 0.0 {
return (digits, 0);
}
let s = format!("{:.14e}", a);
let (mant, exp) = s.split_once('e').unwrap_or((s.as_str(), "0"));
for (slot, b) in digits.iter_mut().zip(mant.bytes().filter(u8::is_ascii_digit)) {
*slot = b - b'0';
// 16 digits, rounded. Unless the 16th is a 5 it decides the 15th exactly (4 or less rounds
// down, 6 or more up), so one format serves almost every number.
let (d16, exp) = significant_digits(a, 16);
for (slot, d) in digits.iter_mut().zip(&d16) {
*slot = *d;
}
(digits, exp.parse().unwrap_or(0))
match d16.get(15).copied().unwrap_or(0) {
0..=4 => (digits, exp),
5 if is_exact_tie(a) => (digits, exp),
5 => {
// Not a tie: round the exact value (the 16-digit rounding above may be a double rounding).
let (d15, exp15) = significant_digits(a, 15);
let mut out = [0u8; 15];
for (slot, d) in out.iter_mut().zip(&d15) {
*slot = *d;
}
(out, exp15)
}
_ => {
let mut exp = exp;
let mut carry = true;
for d in digits.iter_mut().rev() {
if *d == 9 {
*d = 0;
} else {
*d += 1;
carry = false;
break;
}
}
if carry {
digits = [0; 15];
digits[0] = 1;
exp += 1;
}
(digits, exp)
}
}
}
/// `n` significant digits of `a` (rounded half to even on the exact binary value) and the power of
/// ten of the first.
fn significant_digits(a: f64, n: usize) -> (Vec<u8>, i32) {
let s = format!("{:.*e}", n.saturating_sub(1), a);
let (mant, exp) = s.split_once('e').unwrap_or((s.as_str(), "0"));
(mant.bytes().filter(u8::is_ascii_digit).map(|b| b - b'0').collect(), exp.parse().unwrap_or(0))
}
/// True when `a`'s exact decimal expansion has a 5 as its 16th significant digit and only zeros after.
fn is_exact_tie(a: f64) -> bool {
let is_tie = |ds: &[u8]| ds.get(15) == Some(&5) && ds.get(16..).is_some_and(|t| t.iter().all(|&d| d == 0));
// A cheap screen first (30 digits), then the exact expansion: a double has at most 767
// significant digits and Rust prints every one of them.
is_tie(&significant_digits(a, 30).0) && is_tie(&significant_digits(a, 800).0)
}
/// Rounds the number `digits × 10^exp` (see [`decimal_digits`]) to `decimals` places, half away

View File

@ -421,3 +421,73 @@ fn fuzz_no_panic() {
}
}
}
// ---- 15 significant digits: an exact tie at the 16th digit goes toward zero ----
// Windows Excel 16.0.20430 TEXT() over the exact doubles below. An odd 15th digit is where
// round-half-even and toward-zero differ; an even one is where they agree.
#[test]
fn an_exact_tie_at_the_sixteenth_digit_goes_toward_zero() {
for (v, whole) in [
(1234567890123405.0, "1234567890123400"),
(1234567890123415.0, "1234567890123410"),
(1234567890123425.0, "1234567890123420"),
(1234567890123435.0, "1234567890123430"),
(1234567890123455.0, "1234567890123450"),
(1234567890123475.0, "1234567890123470"),
(1234567890123495.0, "1234567890123490"),
(6631679258820775.0, "6631679258820770"),
(8052991635850695.0, "8052991635850690"),
] {
assert_eq!(f("0", v), whole, "0 over {v}");
assert_eq!(f("0.00", v), format!("{whole}.00"), "0.00 over {v}");
assert_eq!(f("0", -v), format!("-{whole}"), "0 over -{v}");
}
assert_eq!(f("#,##0", 1234567890123455.0), "1,234,567,890,123,450");
assert_eq!(f("#,##0", -3267527853107975.0), "-3,267,527,853,107,970");
// A fraction is a tie as well, when it is exact in binary.
assert_eq!(f("0", 123456789012345.5), "123456789012345");
assert_eq!(f("0.00", 123456789012345.5), "123456789012345.00");
// Excel observed: no carry after fifteen 9s.
assert_eq!(f("0", 999999999999999.5), "999999999999999");
assert_eq!(f("0", -999999999999999.5), "-999999999999999");
assert_eq!(f("0", 4999999999999995.0), "4999999999999990");
assert_eq!(f("#,##0", -4999999999999995.0), "-4,999,999,999,999,990");
}
#[test]
fn only_exact_ties_go_toward_zero() {
// Just above and just below a tie round to the nearer 15-digit value (Excel observed).
assert_eq!(f("0", 1234567890123455.0 + 0.25), "1234567890123460");
assert_eq!(f("0", 1234567890123455.5), "1234567890123460");
assert_eq!(f("0", 1234567890123455.0 - 0.25), "1234567890123450");
// 16 digits that are not a tie, and the ordinary decimal cases, are untouched.
assert_eq!(f("0", 1234567890123456.0), "1234567890123460");
assert_eq!(f("0", 1234567890123454.0), "1234567890123450");
assert_eq!(f("0.00", 2.675), "2.68");
assert_eq!(f("0.00", 1.005), "1.01");
assert_eq!(f("0.0", 0.285), "0.3");
assert_eq!(f("0.0", 99999999999999.95), "100000000000000.0");
// Ten digits and a half are not a 16th-digit tie.
assert_eq!(f("0", 0.5), "1");
assert_eq!(f("0", 2.5), "3");
}
#[test]
fn decimal_digits_keeps_fifteen_digits_and_truncates_a_tie() {
use crate::decimal::decimal_digits;
let (d, e) = decimal_digits(1234567890123455.0);
assert_eq!((d, e), ([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5], 15));
let (d, e) = decimal_digits(1234567890123456.0);
assert_eq!((d, e), ([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 6], 15));
let (d, e) = decimal_digits(123456789012345.5);
assert_eq!((d, e), ([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5], 14));
// Excel observed: a tie after fifteen 9s does not carry (999999999999999.5 shows 999999999999999).
let (d, e) = decimal_digits(999999999999999.5);
assert_eq!((d, e), ([9; 15], 14));
let (d, e) = decimal_digits(4999999999999995.0);
assert_eq!((d, e), ([4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9], 15));
// A 16th digit above 5 carries as usual.
let (d, e) = decimal_digits(999999999999999.6);
assert_eq!((d, e), ([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 15));
}