fix(create): read UTF-16, byte-order-marked and Windows code page text files (#903)

Text files were decoded with String::from_utf8_lossy wherever they are
read: Create PDF and Combine from a .txt file, the automation `create`
tool's text `path`, and Search & Redact's word list file. So
- a UTF-16 file (Windows Notepad's "Unicode") became NULs and replacement
  characters ("??G?r?????e?" for "Grüße"),
- a UTF-8 byte order mark became a stray character at the start, and
- a Windows-1252 or 1254 file lost every accented letter ("caf�", "K��").
For a word list that means the words were never found and stayed
unredacted.

pdfcraft_create::decode_text picks UTF-8 or UTF-16 (LE/BE) by the byte
order mark and drops it, keeps valid UTF-8, and otherwise decodes the
Windows code page: 1254 when the bytes 1254 uses for Ğ İ Ş ğ ı ş appear
without á í ó ú (which Turkish never writes but Icelandic and Faroese,
the only Western languages using those bytes, always do), else 1252.
encoding_rs is already in the build through lopdf.

Co-authored-by: Brandon Thomas <echelon@gmail.com>

Cargo.lock

@@ -3639,6 +3639,7 @@ dependencies = [
 name = "pdfcraft-create"
 version = "0.5.0"
 dependencies = [
+ "encoding_rs",
  "hayro-syntax",
  "image",
  "pdfcraft-cos",

crates/automation/src/lib.rs

@@ -1046,7 +1046,7 @@ impl Automation {
                     (None, Some(p)) => {
                         let path = self.resolve(p, false)?;
                         let t = std::fs::read(&path).map_err(|e| failed(format!("{}: {e}", path.display())))?;
-                        (path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(), String::from_utf8_lossy(&t).into_owned())
+                        (path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(), pdfcraft_engine::decode_text(&t))
                     }
                     (None, None) => return Err(ToolError::InvalidArgs("text needs `text` or `path`".into())),
                 };

crates/create/Cargo.toml

@@ -16,6 +16,8 @@ png = "0.18"
 tiff = "0.11"
 image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg"] }
 thiserror = { workspace = true }
+# Already in the graph through lopdf; used only to read plain-text files (decode_text).
+encoding_rs = "0.8"
 
 [dev-dependencies]
 image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg"] }

crates/create/src/lib.rs

@@ -583,6 +583,28 @@ pub fn from_images_with_resolution(images: &[(String, Vec<u8>)], resolution: Ima
 
 // ── text ────────────────────────────────────────────────────────────────────────────────────
 
+/// The text of a plain-text file. A byte order mark picks UTF-8 or UTF-16 (little or big
+/// endian) and is dropped; without one the bytes are UTF-8 when they are valid UTF-8, and
+/// otherwise the Windows code page they most likely are: 1254 (Turkish) or 1252 (Western).
+pub fn decode_text(bytes: &[u8]) -> String {
+    if let Some((encoding, bom)) = encoding_rs::Encoding::for_bom(bytes) {
+        return encoding.decode_without_bom_handling(bytes.get(bom..).unwrap_or_default()).0.into_owned();
+    }
+    if let Ok(text) = std::str::from_utf8(bytes) {
+        return text.to_string();
+    }
+    let encoding = if turkish_code_page(bytes) { encoding_rs::WINDOWS_1254 } else { encoding_rs::WINDOWS_1252 };
+    encoding.decode_without_bom_handling(bytes).0.into_owned()
+}
+
+/// Windows-1254 and 1252 differ only in six bytes: Ğ İ Ş ğ ı ş in 1254 are Ð Ý Þ ð ý þ in 1252.
+/// Of the Western languages only Icelandic and Faroese use those, and both also write á í ó ú,
+/// which Turkish never does.
+fn turkish_code_page(bytes: &[u8]) -> bool {
+    bytes.iter().any(|b| matches!(b, 0xD0 | 0xDD | 0xDE | 0xF0 | 0xFD | 0xFE))
+        && !bytes.iter().any(|b| matches!(b, 0xC1 | 0xCD | 0xD3 | 0xDA | 0xE1 | 0xED | 0xF3 | 0xFA))
+}
+
 /// Plain text set in Helvetica on pages of `page` size with 1-inch margins.
 pub fn from_text(title: &str, text: &str, page: (f64, f64), font_size: f64) -> Result<Document, CreateError> {
     let size = if font_size.is_finite() && font_size > 0.0 { font_size.clamp(4.0, 72.0) } else { 11.0 };

crates/create/src/tests.rs

@@ -289,6 +289,28 @@ fn jpeg_2000_images_are_embedded_as_is() {
     assert_eq!(media(&pages(&doc)[0])[2], 64.0);
 }
 
+#[test]
+fn text_files_are_decoded_by_byte_order_mark_then_utf8_then_code_page() {
+    let utf16 = |text: &str, big: bool| -> Vec<u8> {
+        let mut v = if big { vec![0xFE, 0xFF] } else { vec![0xFF, 0xFE] };
+        for u in text.encode_utf16() {
+            v.extend_from_slice(&if big { u.to_be_bytes() } else { u.to_le_bytes() });
+        }
+        v
+    };
+    assert_eq!(decode_text("Kış İstanbul".as_bytes()), "Kış İstanbul");
+    assert_eq!(decode_text(b"\xEF\xBB\xBFMerhaba"), "Merhaba", "a UTF-8 byte order mark is dropped");
+    assert_eq!(decode_text(&utf16("Kış İstanbul\r\nŞubat", false)), "Kış İstanbul\r\nŞubat");
+    assert_eq!(decode_text(&utf16("Grüße 日本", true)), "Grüße 日本");
+    // Not UTF-8: Windows-1254 for Turkish, 1252 for other Western text (Icelandic included).
+    assert_eq!(decode_text(b"K\xFD\xFE \xDDstanbul \xF0\xFC\xE7"), "Kış İstanbul ğüç");
+    assert_eq!(decode_text(b"\xDE\xF3r\xF0ur \xE1 \xEDslandi"), "Þórður á íslandi");
+    assert_eq!(decode_text(b"caf\xE9 \x80 5"), "café € 5");
+    // Odd and broken input never fails.
+    assert_eq!(decode_text(&[0xFF, 0xFE, b'A']), "\u{FFFD}");
+    assert_eq!(decode_text(b""), "");
+}
+
 #[test]
 fn source_kind_tells_pdfs_images_and_text_apart() {
     assert_eq!(source_kind("a.bin", b"%PDF-1.7\n"), Some(SourceKind::Pdf));

crates/engine/src/lib.rs

@@ -35,7 +35,7 @@ pub use pdfcraft_organize::LabelStyle;
 pub use pdfcraft_organize::view::{InitialView, Layout as InitialLayout, Magnification, Navigation};
 
 pub use pdfcraft_cos::Algorithm;
-pub use pdfcraft_create::{CONVERTIBLE, ImageResolution, SourceKind, source_kind};
+pub use pdfcraft_create::{CONVERTIBLE, ImageResolution, SourceKind, decode_text, source_kind};
 pub use pdfcraft_edit::{
     Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, MarkKind, Watermark,
 };
@@ -2703,7 +2703,7 @@ impl Session {
         let created = guard(|| match kind {
             SourceKind::Pdf => open_source(name, bytes).map(|_| bytes.clone()),
             SourceKind::Image => self.create_from_images(&[(name.to_string(), bytes.to_vec())]),
-            SourceKind::Text => self.create_from_text(title, &String::from_utf8_lossy(bytes)),
+            SourceKind::Text => self.create_from_text(title, &decode_text(bytes)),
         })
         .map_err(|_| EditError::Source(format!("{name}: the file could not be read")))?;
         Ok((kind, created?))

crates/engine/src/tests.rs

@@ -465,6 +465,24 @@ fn mixed_files_convert_and_combine_in_order() {
     assert_eq!(outline_titles(&s.get(id).unwrap().info.outline), ["notes→1", "a→2", "scan→3"]);
 }
 
+#[test]
+fn utf16_and_code_page_text_files_convert_to_their_text() {
+    let mut s = Session::new();
+    // Notepad's "Unicode": UTF-16 little endian with a byte order mark.
+    let mut utf16 = vec![0xFF, 0xFE];
+    utf16.extend("Grüße aus Köln".encode_utf16().flat_map(u16::to_le_bytes));
+    let (_, pdf) = s.convert_to_pdf("notes.txt", &Arc::new(utf16)).unwrap();
+    let id = s.open_new("notes.pdf", pdf).unwrap();
+    assert_eq!(page_texts(&s, id), ["Grüße aus Köln"]);
+    // A UTF-8 byte order mark is not text; Windows-1252 bytes are not replacement characters.
+    let (_, pdf) = s.convert_to_pdf("menu.txt", &Arc::new(b"\xEF\xBB\xBFCaf\xC3\xA9".to_vec())).unwrap();
+    let id = s.open_new("menu.pdf", pdf).unwrap();
+    assert_eq!(page_texts(&s, id), ["Café"]);
+    let (_, pdf) = s.convert_to_pdf("old.txt", &Arc::new(b"Cr\xE8me br\xFBl\xE9e".to_vec())).unwrap();
+    let id = s.open_new("old.pdf", pdf).unwrap();
+    assert_eq!(page_texts(&s, id), ["Crème brûlée"]);
+}
+
 #[test]
 fn a_file_split_around_another_combines_in_order_with_one_bookmark() {
     let mut s = Session::new();

crates/ui-egui/src/files.rs

@@ -366,7 +366,7 @@ impl PdfCraftApp {
                     self.notify_fmt("{name} is too large for a word list (at most 1 MB).", &[("name", &name)]);
                     return;
                 }
-                self.redact_search.words = pdfcraft_engine::redact_word_list(&String::from_utf8_lossy(&bytes)).join("\n");
+                self.redact_search.words = pdfcraft_engine::redact_word_list(&pdfcraft_engine::decode_text(&bytes)).join("\n");
                 self.redact_search.mode = crate::redact_ui::SearchMode::Words;
             }
         }