From ce9a1d1014871baaac632f1e934b71742f7d5b66 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Fri, 9 Oct 2026 13:32:14 -0700 Subject: [PATCH] F2: read glyph names by the naming algorithm, and stop reading Symbol as Latin Two gaps in code-to-Unicode, both found by probing the current behaviour rather than by reading the code. A font with no /Widths and no /ToUnicode fell back to StandardEncoding whatever it was, so /Symbol code 0x61 -- `alpha` in Symbol's own built-in encoding -- decoded as "a". ZapfDingbats and MacExpert were read as Latin the same way. That map is what `redact` matches patterns against, so a wrong character there is a silent redaction failure, and a wrong character is worse than no character. Symbol, ZapfDingbats and MacExpert now yield no mapping at all unless the font names a base encoding, which is always honoured as the deliberate statement it is. This keys on the base font *name*, not the /FontDescriptor symbolic flag. A great many ordinary Latin TrueType subsets set that flag and still rely on a Latin encoding, so suppressing on the flag would lose real text across a large share of real documents. A test pins that restraint so it is not "fixed" into a regression later. Glyph names now follow the published naming algorithm instead of two of its cases: a name is cut at its first `.` (`a.sc` is `a`), split on `_` into components (`f_i` is "fi"), and `uniXXXX` carries any number of four-digit values (`uni00660069` is "fi"). Names are resolved whole or not at all, so half a ligature is never returned. `glyph_text` returns a String for this; `glyph_unicode` stays as the single-character view. Hex parsing rejects the leading `+` that `from_str_radix` would otherwise accept, and a lone surrogate half. Both halves were verified by reverting each fix and watching the matching tests fail. The parity note on protect.redact-patterns now says Symbol-encoded text is skipped rather than mis-decoded. Still open, and an owner decision: Symbol and ZapfDingbats could be mapped properly with a glyph-name-to-Unicode table, but the vendored Adobe Glyph List is not on the AGENTS.md 1.1 closed list. Co-Authored-By: Claude --- crates/fonts/src/pdf.rs | 231 ++++++++++++++++++++++++++++++++--- parity/acrobat-features.toml | 2 +- 2 files changed, 216 insertions(+), 17 deletions(-) diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index b62110d1..ef20f581 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -1,9 +1,15 @@ //! What content editing and redaction need from a PDF font: how to split a string into character //! codes, each code's advance width, the glyph height (ascent, descent), and what each code means //! (Unicode) — and back, which text the font can show. Widths come from the font dictionary -//! (`/Widths`, `/W`, `/DW`, `/MissingWidth`); the standard 14 fonts without widths use -//! approximations. Meanings come from `/ToUnicode`, else the encoding (`/Encoding` base and -//! `/Differences` glyph names, ISO 32000-2 Annex D). +//! (`/Widths`, `/W`, `/DW`, `/MissingWidth`); a standard-14 font with no `/Widths` uses the exact +//! published metrics ([`crate::Std14`]). +//! +//! Meanings come from `/ToUnicode`, else the encoding (`/Encoding` base and `/Differences` glyph +//! names, ISO 32000-2 Annex D). A code may deliberately have **no** meaning: where the font's own +//! built-in encoding governs and we hold no table for it — Symbol, ZapfDingbats, MacExpert — this +//! module reports nothing rather than the Latin letter at that code. Callers must treat a missing +//! meaning as "unknown", never as "absent": `redact` matches on this map, and a wrong character +//! there is worse than no character. use std::borrow::Cow; use std::collections::{BinaryHeap, HashMap, HashSet}; @@ -992,13 +998,49 @@ fn parse_to_unicode_within(data: &[u8], out: &mut Unicode, mut budget: usize) { out.explicit_low = bit_set_below_64k(out.explicit.keys().copied()); } -/// A glyph name's Unicode (Annex D names, `uniXXXX`, `uXXXX[XX]`, single letters). -pub fn glyph_unicode(name: &str) -> Option { - if let Ok(i) = crate::encodings::NAMES.binary_search_by(|(n, _)| (*n).cmp(name)) { - return char::from_u32(crate::encodings::NAMES[i].1); +/// A glyph name's text, by the Adobe Glyph List *algorithm* — the published naming rules, not +/// the AGL data table, which this crate does not hold. +/// +/// A name is cut at its first `.`, because a suffix marks a variant of the same character rather +/// than a different one (`a.sc` is small-capital `a`, `one.oldstyle` is still `1`). What remains +/// is split on `_` into components, so a ligature name yields several characters (`f_i` is "fi"). +/// Each component is read as an Annex D name, as `uniXXXX` with one or more four-hex-digit values, +/// or as `uXXXX` through `uXXXXXX`. +/// +/// Returns `None` rather than a partial answer: a name is either fully understood or not, and half +/// a ligature would be worse than admitting we cannot read it. +pub fn glyph_text(name: &str) -> Option { + let stem = name.split('.').next().filter(|s| !s.is_empty())?; + let mut out = String::new(); + for part in stem.split('_') { + out.push_str(&glyph_component(part)?); } - let hex = name.strip_prefix("uni").filter(|h| h.len() == 4).or_else(|| name.strip_prefix('u').filter(|h| (4..=6).contains(&h.len())))?; - u32::from_str_radix(hex, 16).ok().and_then(char::from_u32) + Some(out).filter(|s| !s.is_empty()) +} + +/// One `_`-separated component of a glyph name. +fn glyph_component(part: &str) -> Option { + if let Ok(i) = crate::encodings::NAMES.binary_search_by(|(n, _)| (*n).cmp(part)) { + return char::from_u32(crate::encodings::NAMES[i].1).map(String::from); + } + // `from_str_radix` accepts a leading `+`, which is not a glyph name, so the digits are checked. + let hexish = |h: &str| !h.is_empty() && h.bytes().all(|b| b.is_ascii_hexdigit()); + if let Some(hex) = part.strip_prefix("uni").filter(|h| h.len() >= 4 && h.len() % 4 == 0 && hexish(h)) { + let mut s = String::new(); + for i in (0..hex.len()).step_by(4) { + // A lone surrogate half is not a character; such a name is not usable text. + s.push(char::from_u32(u32::from_str_radix(hex.get(i..i.saturating_add(4))?, 16).ok()?)?); + } + return Some(s); + } + let hex = part.strip_prefix('u').filter(|h| (4..=6).contains(&h.len()) && hexish(h))?; + u32::from_str_radix(hex, 16).ok().and_then(char::from_u32).map(String::from) +} + +/// [`glyph_text`] for names that stand for exactly one character. +pub fn glyph_unicode(name: &str) -> Option { + let mut chars = glyph_text(name)?.chars().collect::>().into_iter(); + chars.next().filter(|_| chars.len() == 0) } /// Code → Unicode for a font: its ToUnicode CMap, else (simple fonts) its encoding. @@ -1011,12 +1053,33 @@ fn unicode_map(doc: &Document, font: &Dict, composite: bool) -> Unicode { Some(Object::Dict(d)) => d.name(b"BaseEncoding").map(<[u8]>::to_vec), _ => None, }; - let table: &[u32; 256] = match base_name.as_deref() { - Some(b"WinAnsiEncoding") => &crate::encodings::WIN_ANSI, - Some(b"MacRomanEncoding") => &crate::encodings::MAC_ROMAN, - _ => &crate::encodings::STANDARD, + // Symbol and ZapfDingbats carry their own built-in encodings, which are not Latin + // (ISO 32000-2 9.6.6.1): Symbol code 0x61 is `alpha`, not `a`. We do not hold those two + // tables, so with no base encoding named they get no implicit mapping at all. Reporting + // nothing is honest; reporting Latin letters silently corrupts text extraction, find, + // copy and — because `redact` matches on this map — redaction. + // + // Deliberately keyed on the *name*, not on the `/FontDescriptor` symbolic flag. A great + // many ordinary Latin TrueType subsets set that flag and still rely on a Latin encoding, + // so suppressing on the flag would lose real text on a large share of real documents. + // An explicitly named base encoding is a deliberate statement and is always honoured, + // even on Symbol. + let symbolic_builtin = matches!( + font.name(b"BaseFont").map(|b| crate::Std14::from_base_font(&String::from_utf8_lossy(b))), + Some(Some(crate::Std14::Symbol | crate::Std14::ZapfDingbats)) + ); + let table: Option<&[u32; 256]> = match base_name.as_deref() { + Some(b"WinAnsiEncoding") => Some(&crate::encodings::WIN_ANSI), + Some(b"MacRomanEncoding") => Some(&crate::encodings::MAC_ROMAN), + Some(b"StandardEncoding") => Some(&crate::encodings::STANDARD), + // MacExpert is small caps, oldstyle figures and fractions, not Latin at these codes, + // and we hold no table for it either. + Some(b"MacExpertEncoding") => None, + _ if symbolic_builtin => None, + // Standard is the default for a simple font that names no base encoding. + _ => Some(&crate::encodings::STANDARD), }; - for (code, u) in table.iter().enumerate() { + for (code, u) in table.into_iter().flatten().enumerate() { if let Some(c) = char::from_u32(*u).filter(|_| *u != 0) { out.encoding.insert(code as u32, c.to_string()); } @@ -1030,8 +1093,10 @@ fn unicode_map(doc: &Document, font: &Dict, composite: bool) -> Unicode { // Out-of-range codes map nothing a simple font can show. Object::Int(n) => code = u32::try_from((*n).max(0)).unwrap_or(u32::MAX), Object::Name(n) => { - match glyph_unicode(&String::from_utf8_lossy(n)) { - Some(c) => out.encoding.insert(code, c.to_string()), + // A name we cannot read removes any inherited meaning for that code, + // rather than leaving the base encoding's letter standing in for it. + match glyph_text(&String::from_utf8_lossy(n)) { + Some(t) => out.encoding.insert(code, t), None => out.encoding.remove(&code), }; code = code.saturating_add(1); @@ -1580,3 +1645,137 @@ mod tests { assert_eq!((u.covering(0xFFFF).is_some(), u.covering(0x1_0001).is_some(), u.covering(0x1_0002).is_none()), (true, true, true)); } } + +#[cfg(test)] +mod encoding_tests { + use pdfcraft_cos::{Dict, Document, Object}; + + use super::*; + + /// A simple Type 1 font, optionally symbolic-flagged and optionally with an `/Encoding`. + fn simple(base: &str, flags: Option, enc: Option) -> (Document, Dict) { + let doc = Document::new_empty(); + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("Font")); + d.set(b"Subtype".to_vec(), Object::name("Type1")); + d.set(b"BaseFont".to_vec(), Object::name(base)); + if let Some(f) = flags { + let mut desc = Dict::new(); + desc.set(b"Flags".to_vec(), Object::Int(f)); + d.set(b"FontDescriptor".to_vec(), Object::Dict(desc)); + } + if let Some(e) = enc { + d.set(b"Encoding".to_vec(), e); + } + (doc, d) + } + + fn differences(items: Vec) -> Object { + let mut enc = Dict::new(); + enc.set(b"Differences".to_vec(), Object::Array(items)); + Object::Dict(enc) + } + + #[test] + fn glyph_names_follow_the_naming_algorithm() { + // Annex D names and the two hex forms, as before. + assert_eq!(glyph_text("quotedblleft").as_deref(), Some("\u{201c}")); + assert_eq!(glyph_text("uni20AC").as_deref(), Some("\u{20ac}")); + assert_eq!(glyph_text("u1F600").as_deref(), Some("\u{1f600}")); + // A suffix marks a variant of the same character, so the stem still reads. + assert_eq!(glyph_text("a.sc").as_deref(), Some("a")); + assert_eq!(glyph_text("one.oldstyle").as_deref(), Some("1")); + assert_eq!(glyph_text("A.alt017").as_deref(), Some("A")); + // `_` joins components, so a ligature name is more than one character. + assert_eq!(glyph_text("f_i").as_deref(), Some("fi")); + assert_eq!(glyph_text("f_f_l").as_deref(), Some("ffl")); + assert_eq!(glyph_text("a_uni0301").as_deref(), Some("a\u{301}")); + // `uniXXXX` carries several values, so one name can be a whole cluster. + assert_eq!(glyph_text("uni00660069").as_deref(), Some("fi")); + assert_eq!(glyph_text("uni004100300041").as_deref(), Some("A0A")); + // Both forms combine with a suffix. + assert_eq!(glyph_text("uni00660069.alt").as_deref(), Some("fi")); + + // Not names: unknown, empty, a half-read ligature, bad hex, and the `+` that + // `from_str_radix` would otherwise accept. + assert_eq!(glyph_text("nonsense"), None); + assert_eq!(glyph_text(""), None); + assert_eq!(glyph_text(".notdef"), None); + assert_eq!(glyph_text("f_nonsense"), None, "half a ligature is not an answer"); + assert_eq!(glyph_text("uni00"), None, "not a whole four-digit value"); + assert_eq!(glyph_text("uni0041004"), None); + assert_eq!(glyph_text("uni+041"), None, "a sign is not a hex digit"); + assert_eq!(glyph_text("u+0041"), None); + assert_eq!(glyph_text("uniD800"), None, "a lone surrogate half is not a character"); + // Glyph-index names carry no meaning, and must not be guessed at. + assert_eq!(glyph_text("g3"), None); + assert_eq!(glyph_text("index42"), None); + + // `glyph_unicode` is the single-character view of the same algorithm. + assert_eq!(glyph_unicode("uni20AC"), Some('\u{20ac}')); + assert_eq!(glyph_unicode("a.sc"), Some('a')); + assert_eq!(glyph_unicode("f_i"), None, "a ligature is not one character"); + } + + #[test] + fn a_ligature_or_unreadable_name_in_differences_is_handled() { + // A ligature name maps one code to two characters, so text extraction reads "fi". + let (doc, d) = simple("Custom", None, Some(differences(vec![Object::Int(65), Object::name("f_i")]))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.decode(b"A"), "fi"); + + // An unreadable name must *remove* the base encoding's letter rather than leave it + // standing in: code 65 is no longer "A", because the font says it is something else. + let (doc, d) = simple("Custom", None, Some(differences(vec![Object::Int(65), Object::name("g7")]))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.text_of(65), None); + assert_eq!(m.text_of(66), Some("B"), "neighbouring codes keep their meaning"); + } + + #[test] + fn symbol_and_dingbats_are_not_read_as_latin() { + // Symbol's built-in encoding puts `alpha` at 0x61. We hold no table for it, so the honest + // answer is none at all -- never "a", which would corrupt find, copy and redaction. + for base in ["Symbol", "ZapfDingbats", "ABCDEF+Symbol"] { + let (doc, d) = simple(base, Some(4), None); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.text_of(0x61), None, "{base} must not claim 'a'"); + assert_eq!(m.decode(b"abg"), "", "{base}"); + } + + // MacExpert is small caps and oldstyle figures at these codes, and we hold no table. + let (doc, d) = simple("Times-Roman", None, Some(Object::name("MacExpertEncoding"))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.text_of(0x61), None); + + // /Differences still applies on top of a built-in encoding we cannot read. + let (doc, d) = simple("Symbol", Some(4), Some(differences(vec![Object::Int(0x61), Object::name("uni03B1")]))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.text_of(0x61), Some("\u{3b1}"), "the font told us this one"); + assert_eq!(m.text_of(0x62), None, "and nothing about this one"); + } + + #[test] + fn a_named_base_encoding_is_always_honoured() { + // Naming an encoding is a deliberate statement, so it wins even on Symbol. + for name in ["WinAnsiEncoding", "MacRomanEncoding", "StandardEncoding"] { + let mut enc = Dict::new(); + enc.set(b"BaseEncoding".to_vec(), Object::name(name)); + let (doc, d) = simple("Symbol", Some(4), Some(Object::Dict(enc))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.text_of(0x61), Some("a"), "{name} was asked for by name"); + } + } + + /// Pins a deliberate restraint, so it is not "fixed" into a regression: a great many ordinary + /// Latin TrueType subsets set the `/FontDescriptor` symbolic flag and still rely on a Latin + /// encoding. Suppressing the implicit table on the flag would lose real text on a large share + /// of real documents, so only Symbol and ZapfDingbats -- named, and unambiguous -- suppress it. + #[test] + fn the_symbolic_flag_alone_does_not_suppress_the_latin_default() { + let (doc, d) = simple("SomeSubsetFont", Some(4), None); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.text_of(0x61), Some("a")); + assert_eq!(m.decode(b"abc"), "abc"); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 74ceae6e..eeba7611 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5585,7 +5585,7 @@ status = "shipped" commands = ["redact.search"] tools = ["redact_mark"] evidence = ["crates/redact/src/patterns.rs::phone_numbers", "crates/redact/src/patterns.rs::credit_cards_need_a_valid_checksum", "crates/automation/tests/automation.rs::redacting_through_tools"] -notes = "phone (North American, E.164 with + country code, national with trunk 0), e-mail, credit card (Luhn), SSN, dates (English US forms)" +notes = "phone (North American, E.164 with + country code, national with trunk 0), e-mail, credit card (Luhn), SSN, dates (English US forms); text drawn in a font whose built-in encoding we hold no table for (Symbol, ZapfDingbats, MacExpert) has no Unicode mapping, so a pattern search cannot see it -- it is skipped rather than mis-decoded" [[feature]] id = "protect.redact-pattern-locales"