From dc7c300c69fc01d8e2d8bd69a0b1169fc24efdfb Mon Sep 17 00:00:00 2001 From: bflatastic Date: Fri, 9 Oct 2026 14:34:17 +0700 Subject: [PATCH] =?UTF-8?q?Fix=20#708:=20glyphs=20whose=20ToUnicode=20is?= =?UTF-8?q?=20U+FFFD=20don't=20import=20as=20"=EF=BF=BD"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PDF or PDF-compatible .ai whose ToUnicode map gives some glyphs U+FFFD (as some files do for the digits of an embedded CID font) imported those glyphs as type showing the replacement character, so every digit came in as "�". U+FFFD now counts as no mapping. Such a glyph takes its character from the embedded font where it can (Adobe's CID tables for CID-keyed CFF, as before, then the font program's own cmap), and otherwise keeps its outline from the embedded font, so it looks as the file draws it. The import warns which font had characters the file doesn't name. Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 2 +- crates/pdf/src/import.rs | 54 ++++++++++++++++++++++++++++++-- crates/pdf/src/tests_realtext.rs | 46 +++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 4 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 9668b234..810e07a9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,7 +4,7 @@ VectorCraft is a clean-room, open-source, pure-Rust reimplementation of the Adob This file tracks **how far we are and what's left**. Time estimates are wall-clock hours of continuous Claude Opus 5.5 agent work (including builds and the CI gate), given both for **one agent** and for **4–6 parallel agents** on disjoint crates. They are counted from the remaining work (see [Parity estimate](#parity-estimate)), calibrated against measured throughput, and updated as work lands. -_Last updated: 2026-10-09 (after placed documents (File › Place of a `.vectorcraft` file, linked), M4.14–M4.98, M8.1–M8.20, M14.4–M14.7, the MCP prompts, resource templates, completions and logging, the 2026-10-07 issue fixes, the Layers panel rework and the 2026-10-08 community contributions: Pen and shape-tool modifiers, path editing, Layers, Artboards, Swatches/Brushes and dialogs, Japanese composition and PDF text, Hebrew/Arabic type, interface languages (Spanish among them), the font menu, saved selections, MCP and EPS, the missing-font export warning (#319), open paths in the Shape Builder (#374), PDF/`.ai` import keeping strokes live and hidden and nested layers (#372), per-corner Live Corners (#381), SVG type opening at the size it draws at (#396), the Wayland clipboard with copied-file paste (#398), ruler guides that can be selected, moved and deleted (#414), two batches of Preferences options that were stored but unused (#394), skeleton reads with `document.find` for large documents over MCP (#416) and anchors dragged with Direct Selection snapping to points, segments and smart guides, clipped SVG layers opening as clipping layers (#420), PDF/`.ai` layers read on pages of tens of millions of operators (#419), Document Color Mode putting RGB greys on the black plate on request and converting pattern tiles (#421) and movable dialogs with Offset Path's Joins dropdown (#423) and numeric fields scrubbed by dragging their label (#400), stroke weight presets in the stroke unit (#426), the Appearance panel's Opacity popup (#430), type on a path brackets, flip and Type on a Path Options that keep the type in place (#429), Layers rows dragged with a ghost and dropped on the trash or New Layer (#435), tool flyouts opening on a right or corner press (#436), tool flyouts torn off into floating toolbars (#437), the Shape Builder's erase mode deleting edges too (#439), installed fonts registered outside the font folders and fonts installed while the app runs listed in the font menus (#443), alike circular corners on rectangles that aren't square (#442), tool cursors as OS cursors so the pointer no longer trails over the canvas (#444), artboards copied and pasted with their art (#446), guides dragged out of the rulers snapping to the art's side midpoints and the artboards, artboard guides (#451), the toolbar's double arrow and Window › Toolbars › Double Column switching between one and two columns in any window height (#455), the Smart Guides display options and Snapping Tolerance (#394), Font Size and Leading combo boxes with preset lists in the Character and Properties panels and the Control bar (#458), screenshots pasted from the clipboard on macOS, Windows and the web (#457), the Properties panel's Fill and Stroke matching the Control bar's, Illustrator's Window menu panel keys (Shift+F7 for Align…) and Alt-wheel zoom whatever the wheel preference (Discord feedback), the point/area type widget (Discord feedback), groups in legacy Illustrator .ai files (Discord feedback), a rotating desktop log file (#465), image controls in the Control bar and Properties panel: Image Trace with its presets, Mask, Crop Image, Embed/Unembed, and an Image Trace object's preset, Expand and Release (Discord feedback), the Pen working as the Anchor Point tool with Alt held over a selected path's handles and anchors, and anchor buttons that convert, connect and cut paths at the selected anchors (Discord feedback), Japanese hanging punctuation (burasagari, #463), Escape leaving Presentation Mode with no screen mode saved across restarts, documents dropped on the desktop app opening as tabs of their own and a note that an `.ai` file's art off its artboards isn't in the part read (#472), Add/Delete Anchor Point tool icons drawn from the Pen tool's (#479), EPS import running more of PostScript Level 3 (patterns, Type 3 fonts, mesh shadings, masked images, VM and resource probes), and a preview that stands in saying why (Discord feedback, #466, #469, #474), chords with modifiers recorded in Edit › Keyboard Shortcuts (#487), Type Object Selection by Path Only (#394), Anti-aliased Artwork (#394), Show Pixel Grid (#394), pen and touch taps pressed and lifted within one frame clicking where they tapped and the Linux/Wayland pen-tablet workaround (#491), bounding-box resizes landing on the other objects' anchors and centres with labels, and the Rotate, Scale, Shear and Reflect tools snapping their drags (#482), Shift-drag marquees that toggle the objects or anchors they reach and a Lasso whose Alt subtraction keeps the rest of the selection (#483), the mouse wheel stepping the focused numeric field, the dialogs' and preferences' drag values made such fields (#485), Show The Home Screen When No Documents Are Open (#394), Numbers Without Units Are Points (#394), Auto-Collapse Iconic Panels (#394), the Contextual Task Bar moved by its handle and its More Options menu pinning and resetting its position, a designed macOS DMG window (#493), panels, panel groups and the Tools panel undocking into floating panels that move, stack and dock again (#495), the 0.5.0 startup crash on dual-GPU Linux desktops under Wayland fixed: the desktop's own GPU by default and a restart on the next GPU when one can't show the window (#502), path editing while drawing: Cmd/Ctrl with the Pen borrowing Direct Selection and drawing on afterwards, Pen+Alt reshaping segments, handles of a whole selected path dragging, and handle drags no longer re-gathering snap targets every move (#494), the Control bar's and Properties panel's Convert buttons for a path selected as a whole with Direct Selection, the Pen and the anchor tools (#504), the Pen closing a path dragged out of its first anchor every time, without flicker (#501), the Paste menu checking the system clipboard on a background thread on Linux so an X11 clipboard owner that never answers no longer freezes the window (#510), PDF/`.ai` import keeping a type object's outline stroke on the type, wrapped lines as area type, barcode digits in place and layers in the file's stacking order (#508), the Italian interface (#514), EPS images with several data sources read in turn (#498), `restore` undoing dictionary changes in EPS import (part of #505), the macOS menu bar laid out as a Mac app's (the VectorCraft menu with Settings, Language and Appearance, the system's Hide, Services and Window items; owner's request), Object › Image Trace › Make tracing with the Default preset instead of asking for a preset's name (#543), the rulers' unit menu (#528), Live Corners on stars, polygons and pen paths (#511), the drawing tools snapping to Smart Guides as the Selection tool does, with Construction Guides (#506), the in-window menus' Select All, Cut, Copy and Paste acting on the text field they were opened from (#516), Windows rendering through DX12 before Vulkan (#545), Image Trace tracing colour layers in parallel and refusing traces too large for memory, with the canvas outlining huge selections by their bounds (#525), SVGs of chosen artboards holding only the art over each artboard (#550), the font menu scrolling only its list, under a search field and filters that stay put (#555), a click on an object of a selection making it the key object that Align aligns to and Distribute Spacing spaces from, drawn with a thicker outline (#541), Shape Builder merges taken from the planar map itself, so merged regions leave no inner edge, hairline gap or spur in their outline (#569), `.ai` written by the exports and the CLI (`--export`, `convert`) as Save As writes it (#526), SVG round trips keeping hidden objects and `data-*` attributes (#522), the gradient stop popover staying open after the click that opens it from the Gradient panel (#577), standard ligatures kept under small tracking and dropped only past about −8% / +22% of a word space or when the character asks for them (#583), the Color panel keeping its height as warnings come and go so a drag on its spectrum no longer flickers near white (#578), fonts that font services such as Adobe Fonts load on Windows, listed through DirectWrite (#579), `.ai` files writing their layers and sublayers as PDF layers, hidden ones with their art, and PDF layers nested in layers importing as sublayers (#372), file dialogs on Linux shown off the UI thread so the compositor no longer finds the window not responding, and parented to the window on Linux and Windows (#592), anchors and handle ends picked from 2 px past their drawn size before the segments through them (#593), an image copied in a browser pasting as the picture, not its address (#597), the Free Transform widget with Constrain, Free Transform, Perspective Distort and Free Distort (#598), touch gestures: a two-finger tap undoes and a three-finger tap redoes (#585), the font menus' live preview staying on the text instead of flickering on and off every frame (#563), Edit Artboards showing the active artboard in the Properties panel with New Artboard, Delete Artboard and Exit (#530), the same DirectWrite fonts in `vectorcraft-cli` and its MCP server (#579), SVG layers keeping their names when wrapped and sublayers round-tripping through SVG (#372), a Layers panel that lays out only the rows in view and keeps a document's open rows in its file (#638), a panel dropped on the dock lights up the icon column it goes back to and pops out of its icon (#649), Image Trace's View modes and an Advanced section that stays as it was left (part of #641), Image Trace's Palette: Limited, Full Tone, Automatic and Document Library (part of #641), Image Trace's Create Strokes: line art traced as stroked centre lines (part of #641), a dual-GPU Mac falling back to its other GPU when one fails to start (#651), two fingers dragged on a touch screen panning the canvas (part of #449), the Home screen's Recent Files and a double-clicked New Document preset creating the document (part of #663), quitting no longer crashing on Macs with a Touch Bar (#661), Animated Zoom, Large Tabs, Scale Cursor Proportional to UI and anti-aliased bitmaps in Pixel Preview acting (part of #394), Smart Guides › Spacing Guides (part of #394), Edit Artboards' Properties choosing an artboard's size preset and flipping it between portrait and landscape (#671), Preferences › Type › Additional Fonts Folder loading the fonts of a folder you choose (#683), Canvas Color's light, medium and dark grey (part of #663), a document tab's close box after its title on Windows and Linux, before it on macOS (part of #673), Rearrange All Artboards' layouts (a grid by row or by column, one row, one column) and right-to-left order, one click away in the Properties and Artboards panels (#681), User Interface › Show Tool Group Labels and Cmd+, opening Settings on the Mac (part of #663), the Windows taskbar and Alt+Tab showing VectorCraft's icon, a click with a selection tool making its artboard the active one, and Paste in Place, in Front and in Back pasting onto the active artboard (#693), Photoshop-style raster effects on a pixel-filter pipeline: Effect › Blur › Radial Blur and Smart Blur, Effect › Sharpen › Unsharp Mask (part of gap 2), the Properties panel's number fields all one width (#696), the rotate cursor drawn as a curved double arrow (#701), the Templates folder in the desktop's own documents folder, made when missing (#703); see [Honest assessment](#honest-assessment-2026-10-05))._ +_Last updated: 2026-10-09 (after placed documents (File › Place of a `.vectorcraft` file, linked), M4.14–M4.98, M8.1–M8.20, M14.4–M14.7, the MCP prompts, resource templates, completions and logging, the 2026-10-07 issue fixes, the Layers panel rework and the 2026-10-08 community contributions: Pen and shape-tool modifiers, path editing, Layers, Artboards, Swatches/Brushes and dialogs, Japanese composition and PDF text, Hebrew/Arabic type, interface languages (Spanish among them), the font menu, saved selections, MCP and EPS, the missing-font export warning (#319), open paths in the Shape Builder (#374), PDF/`.ai` import keeping strokes live and hidden and nested layers (#372), per-corner Live Corners (#381), SVG type opening at the size it draws at (#396), the Wayland clipboard with copied-file paste (#398), ruler guides that can be selected, moved and deleted (#414), two batches of Preferences options that were stored but unused (#394), skeleton reads with `document.find` for large documents over MCP (#416) and anchors dragged with Direct Selection snapping to points, segments and smart guides, clipped SVG layers opening as clipping layers (#420), PDF/`.ai` layers read on pages of tens of millions of operators (#419), Document Color Mode putting RGB greys on the black plate on request and converting pattern tiles (#421) and movable dialogs with Offset Path's Joins dropdown (#423) and numeric fields scrubbed by dragging their label (#400), stroke weight presets in the stroke unit (#426), the Appearance panel's Opacity popup (#430), type on a path brackets, flip and Type on a Path Options that keep the type in place (#429), Layers rows dragged with a ghost and dropped on the trash or New Layer (#435), tool flyouts opening on a right or corner press (#436), tool flyouts torn off into floating toolbars (#437), the Shape Builder's erase mode deleting edges too (#439), installed fonts registered outside the font folders and fonts installed while the app runs listed in the font menus (#443), alike circular corners on rectangles that aren't square (#442), tool cursors as OS cursors so the pointer no longer trails over the canvas (#444), artboards copied and pasted with their art (#446), guides dragged out of the rulers snapping to the art's side midpoints and the artboards, artboard guides (#451), the toolbar's double arrow and Window › Toolbars › Double Column switching between one and two columns in any window height (#455), the Smart Guides display options and Snapping Tolerance (#394), Font Size and Leading combo boxes with preset lists in the Character and Properties panels and the Control bar (#458), screenshots pasted from the clipboard on macOS, Windows and the web (#457), the Properties panel's Fill and Stroke matching the Control bar's, Illustrator's Window menu panel keys (Shift+F7 for Align…) and Alt-wheel zoom whatever the wheel preference (Discord feedback), the point/area type widget (Discord feedback), groups in legacy Illustrator .ai files (Discord feedback), a rotating desktop log file (#465), image controls in the Control bar and Properties panel: Image Trace with its presets, Mask, Crop Image, Embed/Unembed, and an Image Trace object's preset, Expand and Release (Discord feedback), the Pen working as the Anchor Point tool with Alt held over a selected path's handles and anchors, and anchor buttons that convert, connect and cut paths at the selected anchors (Discord feedback), Japanese hanging punctuation (burasagari, #463), Escape leaving Presentation Mode with no screen mode saved across restarts, documents dropped on the desktop app opening as tabs of their own and a note that an `.ai` file's art off its artboards isn't in the part read (#472), Add/Delete Anchor Point tool icons drawn from the Pen tool's (#479), EPS import running more of PostScript Level 3 (patterns, Type 3 fonts, mesh shadings, masked images, VM and resource probes), and a preview that stands in saying why (Discord feedback, #466, #469, #474), chords with modifiers recorded in Edit › Keyboard Shortcuts (#487), Type Object Selection by Path Only (#394), Anti-aliased Artwork (#394), Show Pixel Grid (#394), pen and touch taps pressed and lifted within one frame clicking where they tapped and the Linux/Wayland pen-tablet workaround (#491), bounding-box resizes landing on the other objects' anchors and centres with labels, and the Rotate, Scale, Shear and Reflect tools snapping their drags (#482), Shift-drag marquees that toggle the objects or anchors they reach and a Lasso whose Alt subtraction keeps the rest of the selection (#483), the mouse wheel stepping the focused numeric field, the dialogs' and preferences' drag values made such fields (#485), Show The Home Screen When No Documents Are Open (#394), Numbers Without Units Are Points (#394), Auto-Collapse Iconic Panels (#394), the Contextual Task Bar moved by its handle and its More Options menu pinning and resetting its position, a designed macOS DMG window (#493), panels, panel groups and the Tools panel undocking into floating panels that move, stack and dock again (#495), the 0.5.0 startup crash on dual-GPU Linux desktops under Wayland fixed: the desktop's own GPU by default and a restart on the next GPU when one can't show the window (#502), path editing while drawing: Cmd/Ctrl with the Pen borrowing Direct Selection and drawing on afterwards, Pen+Alt reshaping segments, handles of a whole selected path dragging, and handle drags no longer re-gathering snap targets every move (#494), the Control bar's and Properties panel's Convert buttons for a path selected as a whole with Direct Selection, the Pen and the anchor tools (#504), the Pen closing a path dragged out of its first anchor every time, without flicker (#501), the Paste menu checking the system clipboard on a background thread on Linux so an X11 clipboard owner that never answers no longer freezes the window (#510), PDF/`.ai` import keeping a type object's outline stroke on the type, wrapped lines as area type, barcode digits in place and layers in the file's stacking order (#508), the Italian interface (#514), EPS images with several data sources read in turn (#498), `restore` undoing dictionary changes in EPS import (part of #505), the macOS menu bar laid out as a Mac app's (the VectorCraft menu with Settings, Language and Appearance, the system's Hide, Services and Window items; owner's request), Object › Image Trace › Make tracing with the Default preset instead of asking for a preset's name (#543), the rulers' unit menu (#528), Live Corners on stars, polygons and pen paths (#511), the drawing tools snapping to Smart Guides as the Selection tool does, with Construction Guides (#506), the in-window menus' Select All, Cut, Copy and Paste acting on the text field they were opened from (#516), Windows rendering through DX12 before Vulkan (#545), Image Trace tracing colour layers in parallel and refusing traces too large for memory, with the canvas outlining huge selections by their bounds (#525), SVGs of chosen artboards holding only the art over each artboard (#550), the font menu scrolling only its list, under a search field and filters that stay put (#555), a click on an object of a selection making it the key object that Align aligns to and Distribute Spacing spaces from, drawn with a thicker outline (#541), Shape Builder merges taken from the planar map itself, so merged regions leave no inner edge, hairline gap or spur in their outline (#569), `.ai` written by the exports and the CLI (`--export`, `convert`) as Save As writes it (#526), SVG round trips keeping hidden objects and `data-*` attributes (#522), the gradient stop popover staying open after the click that opens it from the Gradient panel (#577), standard ligatures kept under small tracking and dropped only past about −8% / +22% of a word space or when the character asks for them (#583), the Color panel keeping its height as warnings come and go so a drag on its spectrum no longer flickers near white (#578), fonts that font services such as Adobe Fonts load on Windows, listed through DirectWrite (#579), `.ai` files writing their layers and sublayers as PDF layers, hidden ones with their art, and PDF layers nested in layers importing as sublayers (#372), file dialogs on Linux shown off the UI thread so the compositor no longer finds the window not responding, and parented to the window on Linux and Windows (#592), anchors and handle ends picked from 2 px past their drawn size before the segments through them (#593), an image copied in a browser pasting as the picture, not its address (#597), the Free Transform widget with Constrain, Free Transform, Perspective Distort and Free Distort (#598), touch gestures: a two-finger tap undoes and a three-finger tap redoes (#585), the font menus' live preview staying on the text instead of flickering on and off every frame (#563), Edit Artboards showing the active artboard in the Properties panel with New Artboard, Delete Artboard and Exit (#530), the same DirectWrite fonts in `vectorcraft-cli` and its MCP server (#579), SVG layers keeping their names when wrapped and sublayers round-tripping through SVG (#372), a Layers panel that lays out only the rows in view and keeps a document's open rows in its file (#638), a panel dropped on the dock lights up the icon column it goes back to and pops out of its icon (#649), Image Trace's View modes and an Advanced section that stays as it was left (part of #641), Image Trace's Palette: Limited, Full Tone, Automatic and Document Library (part of #641), Image Trace's Create Strokes: line art traced as stroked centre lines (part of #641), a dual-GPU Mac falling back to its other GPU when one fails to start (#651), two fingers dragged on a touch screen panning the canvas (part of #449), the Home screen's Recent Files and a double-clicked New Document preset creating the document (part of #663), quitting no longer crashing on Macs with a Touch Bar (#661), Animated Zoom, Large Tabs, Scale Cursor Proportional to UI and anti-aliased bitmaps in Pixel Preview acting (part of #394), Smart Guides › Spacing Guides (part of #394), Edit Artboards' Properties choosing an artboard's size preset and flipping it between portrait and landscape (#671), Preferences › Type › Additional Fonts Folder loading the fonts of a folder you choose (#683), Canvas Color's light, medium and dark grey (part of #663), a document tab's close box after its title on Windows and Linux, before it on macOS (part of #673), Rearrange All Artboards' layouts (a grid by row or by column, one row, one column) and right-to-left order, one click away in the Properties and Artboards panels (#681), User Interface › Show Tool Group Labels and Cmd+, opening Settings on the Mac (part of #663), the Windows taskbar and Alt+Tab showing VectorCraft's icon, a click with a selection tool making its artboard the active one, and Paste in Place, in Front and in Back pasting onto the active artboard (#693), Photoshop-style raster effects on a pixel-filter pipeline: Effect › Blur › Radial Blur and Smart Blur, Effect › Sharpen › Unsharp Mask (part of gap 2), the Properties panel's number fields all one width (#696), the rotate cursor drawn as a curved double arrow (#701), the Templates folder in the desktop's own documents folder, made when missing (#703), PDF and .ai glyphs whose ToUnicode names no character (U+FFFD) taking their character from the embedded font or keeping their outlines, with a warning (#708); see [Honest assessment](#honest-assessment-2026-10-05))._ ## Where we are diff --git a/crates/pdf/src/import.rs b/crates/pdf/src/import.rs index ee173375..86b7c685 100644 --- a/crates/pdf/src/import.rs +++ b/crates/pdf/src/import.rs @@ -387,6 +387,33 @@ impl CidText { } } +/// A ToUnicode value that names no character: U+FFFD alone (#708). +fn replacement(u: &hayro_interpret::hayro_cmap::BfString) -> bool { + use hayro_interpret::hayro_cmap::BfString; + match u { + BfString::Char(c) => *c == char::REPLACEMENT_CHARACTER, + BfString::String(s) => !s.is_empty() && s.chars().all(|c| c == char::REPLACEMENT_CHARACTER), + } +} + +/// The character of each glyph of an embedded TrueType or OpenType font program, from its own +/// `cmap` (the first character mapped to each glyph); `None` for a bare CFF or a font without one. +pub(crate) fn font_chars(data: &[u8]) -> Option> { + use skrifa::MetadataProvider; + // As for CidText: whole fonts are some megabytes. + if data.len() > 64 << 20 { + return None; + } + let font = skrifa::FontRef::new(data).ok()?; + let mut chars = HashMap::new(); + for (c, g) in font.charmap().mappings() { + if let Some(c) = char::from_u32(c).filter(|c| !c.is_control()) { + chars.entry(g.to_u32()).or_insert(c); + } + } + (!chars.is_empty()).then_some(chars) +} + /// The ideographs of the Kangxi radicals U+2F00–U+2FD5 (their NFKC forms): the CID → Unicode /// tables of some PDFs map 龍 to the radical ⿓, which looks the same but isn't the character. const KANGXI: &str = "一丨丶丿乙亅二亠人儿入八冂冖冫几凵刀力勹匕匚匸十卜卩厂厶又口囗土士夂夊夕大女子宀寸小尢尸屮山巛工己巾干幺广廴廾弋弓彐彡彳心戈戶手支攴文斗斤方无日曰月木欠止歹殳毋比毛氏气水火爪父爻爿片牙牛犬玄玉瓜瓦甘生用田疋疒癶白皮皿目矛矢石示禸禾穴立竹米糸缶网羊羽老而耒耳聿肉臣自至臼舌舛舟艮色艸虍虫血行衣襾見角言谷豆豕豸貝赤走足身車辛辰辵邑酉釆里金長門阜隶隹雨靑非面革韋韭音頁風飛食首香馬骨高髟鬥鬯鬲鬼魚鳥鹵鹿麥麻黃黍黑黹黽鼎鼓鼠鼻齊齒龍龜龠"; @@ -603,6 +630,9 @@ struct Builder<'p> { /// Font (cache key) → its characters by glyph, for CID-keyed fonts embedded without a /// ToUnicode map. cid_text: HashMap>>, + /// Font (cache key) → the character of each glyph its embedded font program maps + /// ([`font_chars`]), for glyphs the file's ToUnicode map doesn't name (#708). + font_chars: HashMap>>>, /// Font (cache key) → the installed face that draws its glyphs, decided from its first glyph /// that can be compared ([`matching_face`]); `None`: its glyphs differ, so it stays outlines. matched: HashMap>>, @@ -752,6 +782,7 @@ impl<'p> Builder<'p> { families: None, missing_fonts: vec![], cid_text: HashMap::new(), + font_chars: HashMap::new(), matched: HashMap::new(), mask: None, masked: vec![], @@ -1267,17 +1298,34 @@ impl<'p> Builder<'p> { return false; } let key = o.font_cache_key(); - let (unicode, from_cid) = match o.as_unicode() { + let glyph = o.glyph_id().to_u32(); + let (unicode, from_cid) = match o.as_unicode().filter(|u| !replacement(u)) { Some(u) => (Some(u), false), + // No mapping, or one to U+FFFD (#708): the font program may still name the character. None => { let cid = self.cid_text.entry(key).or_insert_with(|| o.font_data().and_then(|d| CidText::of(d.data.as_ref().as_ref())).map(Arc::new)); - (cid.as_ref().and_then(|c| c.unicode(o.glyph_id().to_u32())), true) + match cid.as_ref().and_then(|c| c.unicode(glyph)) { + Some(u) => (Some(u), true), + None => { + let chars = self + .font_chars + .entry(key) + .or_insert_with(|| o.font_data().and_then(|d| font_chars(d.data.as_ref().as_ref())).map(Arc::new)); + (chars.as_ref().and_then(|m| m.get(&glyph)).map(|c| BfString::Char(*c)), false) + } + } } }; let text: String = match unicode { Some(BfString::Char(c)) => c.to_string(), Some(BfString::String(s)) => s, - None => return false, + None => { + if o.as_unicode().is_some_and(|u| replacement(&u)) { + let name = self.font_name(key, o).family; + self.warn(&format!("text in {name} whose characters the file doesn't name (U+FFFD) was kept as outlines")); + } + return false; + } }; // Adobe's CID tables map some ideographs to their look-alike Kangxi radicals. let text: String = if from_cid { text.chars().map(unify_radical).collect() } else { text }; diff --git a/crates/pdf/src/tests_realtext.rs b/crates/pdf/src/tests_realtext.rs index d73e72f6..d0e64afc 100644 --- a/crates/pdf/src/tests_realtext.rs +++ b/crates/pdf/src/tests_realtext.rs @@ -228,3 +228,49 @@ fn a_variable_font_instance_comes_back_in_its_own_style() { }); assert_eq!(styles, ["Bold"]); } + +/// The export of `text` (one point type object) with its ToUnicode map rewritten so that the +/// digits map to U+FFFD, as some files have it (#708). Each `<003D>` of a digit becomes ``, so +/// the file's offsets still hold. +fn digits_unnamed(text: &str) -> Vec { + let d = doc(vec![TextObject::point(Point::new(20.0, 50.0), text, style(24.0))]); + let bytes = pdf(&d, false).bytes; + let at = bytes.windows(11).position(|w| w == b"beginbfchar").expect("a readable ToUnicode map"); + let end = at + bytes[at..].windows(9).position(|w| w == b"endbfchar").unwrap(); + let mut out = bytes.clone(); + for i in at..end.saturating_sub(5) { + // `<0030>` … `<0039>` after a space: a digit's character (each line's code comes first). + if &bytes[i..i + 4] == b"<003" && bytes[i + 4].is_ascii_digit() && bytes[i + 5] == b'>' && bytes[i - 1] == b' ' { + out[i + 1..i + 5].copy_from_slice(b"FFFD"); + } + } + assert_ne!(out, bytes, "digits were mapped"); + out +} + +/// #708: a glyph whose ToUnicode value is U+FFFD isn't type showing "�". This subset font has no +/// character map of its own, so the digits keep their outlines, drawn as the file draws them, and +/// a warning names the font. +#[test] +fn glyphs_mapped_to_the_replacement_character_keep_their_outlines() { + let bytes = digits_unnamed("Tel 02-12345"); + let report = import_with_report(&bytes, &ImportOptions { text_as: TextAs::Text, ..Default::default() }).unwrap(); + let all = texts(&report.document).concat(); + assert!(!all.contains('\u{FFFD}') && all.contains("Tel"), "no replacement characters: {all:?}"); + assert!(ink(&report.document).width() > 50.0, "the digits are outlines"); + assert!(report.warnings.iter().any(|w| w.contains("U+FFFD")), "{:?}", report.warnings); +} + +/// #708: where the embedded font program has a character map, a glyph's character comes from it. +#[test] +fn an_embedded_fonts_own_cmap_names_its_glyphs() { + let data = std::fs::read(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../assets/fonts/SourceSans3-Regular.ttf")).unwrap(); + let chars = crate::import::font_chars(&data).unwrap(); + use skrifa::MetadataProvider; + let font = skrifa::FontRef::new(&data).unwrap(); + for c in ['0', '7', 'A', 'z'] { + let g = font.charmap().map(c).unwrap().to_u32(); + assert_eq!(chars.get(&g), Some(&c)); + } + assert!(crate::import::font_chars(b"not a font").is_none()); +}