diff --git a/Cargo.lock b/Cargo.lock index 7c3064e7..ec521319 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3639,6 +3639,7 @@ dependencies = [ name = "pdfcraft-create" version = "0.5.0" dependencies = [ + "encoding_rs", "hayro-syntax", "image", "pdfcraft-cos", diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index bd9feedd..4d785800 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -1046,7 +1046,7 @@ impl Automation { (None, Some(p)) => { let path = self.resolve(p, false)?; let t = std::fs::read(&path).map_err(|e| failed(format!("{}: {e}", path.display())))?; - (path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(), String::from_utf8_lossy(&t).into_owned()) + (path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(), pdfcraft_engine::decode_text(&t)) } (None, None) => return Err(ToolError::InvalidArgs("text needs `text` or `path`".into())), }; diff --git a/crates/create/Cargo.toml b/crates/create/Cargo.toml index 05d76cb6..7c915ae9 100644 --- a/crates/create/Cargo.toml +++ b/crates/create/Cargo.toml @@ -16,6 +16,8 @@ png = "0.18" tiff = "0.11" image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg"] } thiserror = { workspace = true } +# Already in the graph through lopdf; used only to read plain-text files (decode_text). +encoding_rs = "0.8" [dev-dependencies] image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg"] } diff --git a/crates/create/src/lib.rs b/crates/create/src/lib.rs index bbf93709..09b07270 100644 --- a/crates/create/src/lib.rs +++ b/crates/create/src/lib.rs @@ -583,6 +583,28 @@ pub fn from_images_with_resolution(images: &[(String, Vec)], resolution: Ima // ── text ──────────────────────────────────────────────────────────────────────────────────── +/// The text of a plain-text file. A byte order mark picks UTF-8 or UTF-16 (little or big +/// endian) and is dropped; without one the bytes are UTF-8 when they are valid UTF-8, and +/// otherwise the Windows code page they most likely are: 1254 (Turkish) or 1252 (Western). +pub fn decode_text(bytes: &[u8]) -> String { + if let Some((encoding, bom)) = encoding_rs::Encoding::for_bom(bytes) { + return encoding.decode_without_bom_handling(bytes.get(bom..).unwrap_or_default()).0.into_owned(); + } + if let Ok(text) = std::str::from_utf8(bytes) { + return text.to_string(); + } + let encoding = if turkish_code_page(bytes) { encoding_rs::WINDOWS_1254 } else { encoding_rs::WINDOWS_1252 }; + encoding.decode_without_bom_handling(bytes).0.into_owned() +} + +/// Windows-1254 and 1252 differ only in six bytes: Ğ İ Ş ğ ı ş in 1254 are Ð Ý Þ ð ý þ in 1252. +/// Of the Western languages only Icelandic and Faroese use those, and both also write á í ó ú, +/// which Turkish never does. +fn turkish_code_page(bytes: &[u8]) -> bool { + bytes.iter().any(|b| matches!(b, 0xD0 | 0xDD | 0xDE | 0xF0 | 0xFD | 0xFE)) + && !bytes.iter().any(|b| matches!(b, 0xC1 | 0xCD | 0xD3 | 0xDA | 0xE1 | 0xED | 0xF3 | 0xFA)) +} + /// Plain text set in Helvetica on pages of `page` size with 1-inch margins. pub fn from_text(title: &str, text: &str, page: (f64, f64), font_size: f64) -> Result { let size = if font_size.is_finite() && font_size > 0.0 { font_size.clamp(4.0, 72.0) } else { 11.0 }; diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index e3891686..d83cefc5 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -289,6 +289,28 @@ fn jpeg_2000_images_are_embedded_as_is() { assert_eq!(media(&pages(&doc)[0])[2], 64.0); } +#[test] +fn text_files_are_decoded_by_byte_order_mark_then_utf8_then_code_page() { + let utf16 = |text: &str, big: bool| -> Vec { + let mut v = if big { vec![0xFE, 0xFF] } else { vec![0xFF, 0xFE] }; + for u in text.encode_utf16() { + v.extend_from_slice(&if big { u.to_be_bytes() } else { u.to_le_bytes() }); + } + v + }; + assert_eq!(decode_text("Kış İstanbul".as_bytes()), "Kış İstanbul"); + assert_eq!(decode_text(b"\xEF\xBB\xBFMerhaba"), "Merhaba", "a UTF-8 byte order mark is dropped"); + assert_eq!(decode_text(&utf16("Kış İstanbul\r\nŞubat", false)), "Kış İstanbul\r\nŞubat"); + assert_eq!(decode_text(&utf16("Grüße 日本", true)), "Grüße 日本"); + // Not UTF-8: Windows-1254 for Turkish, 1252 for other Western text (Icelandic included). + assert_eq!(decode_text(b"K\xFD\xFE \xDDstanbul \xF0\xFC\xE7"), "Kış İstanbul ğüç"); + assert_eq!(decode_text(b"\xDE\xF3r\xF0ur \xE1 \xEDslandi"), "Þórður á íslandi"); + assert_eq!(decode_text(b"caf\xE9 \x80 5"), "café € 5"); + // Odd and broken input never fails. + assert_eq!(decode_text(&[0xFF, 0xFE, b'A']), "\u{FFFD}"); + assert_eq!(decode_text(b""), ""); +} + #[test] fn source_kind_tells_pdfs_images_and_text_apart() { assert_eq!(source_kind("a.bin", b"%PDF-1.7\n"), Some(SourceKind::Pdf)); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 24169f94..bb124c5e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -35,7 +35,7 @@ pub use pdfcraft_organize::LabelStyle; pub use pdfcraft_organize::view::{InitialView, Layout as InitialLayout, Magnification, Navigation}; pub use pdfcraft_cos::Algorithm; -pub use pdfcraft_create::{CONVERTIBLE, ImageResolution, SourceKind, source_kind}; +pub use pdfcraft_create::{CONVERTIBLE, ImageResolution, SourceKind, decode_text, source_kind}; pub use pdfcraft_edit::{ Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, MarkKind, Watermark, }; @@ -2703,7 +2703,7 @@ impl Session { let created = guard(|| match kind { SourceKind::Pdf => open_source(name, bytes).map(|_| bytes.clone()), SourceKind::Image => self.create_from_images(&[(name.to_string(), bytes.to_vec())]), - SourceKind::Text => self.create_from_text(title, &String::from_utf8_lossy(bytes)), + SourceKind::Text => self.create_from_text(title, &decode_text(bytes)), }) .map_err(|_| EditError::Source(format!("{name}: the file could not be read")))?; Ok((kind, created?)) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ad68e816..8d106294 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -465,6 +465,24 @@ fn mixed_files_convert_and_combine_in_order() { assert_eq!(outline_titles(&s.get(id).unwrap().info.outline), ["notes→1", "a→2", "scan→3"]); } +#[test] +fn utf16_and_code_page_text_files_convert_to_their_text() { + let mut s = Session::new(); + // Notepad's "Unicode": UTF-16 little endian with a byte order mark. + let mut utf16 = vec![0xFF, 0xFE]; + utf16.extend("Grüße aus Köln".encode_utf16().flat_map(u16::to_le_bytes)); + let (_, pdf) = s.convert_to_pdf("notes.txt", &Arc::new(utf16)).unwrap(); + let id = s.open_new("notes.pdf", pdf).unwrap(); + assert_eq!(page_texts(&s, id), ["Grüße aus Köln"]); + // A UTF-8 byte order mark is not text; Windows-1252 bytes are not replacement characters. + let (_, pdf) = s.convert_to_pdf("menu.txt", &Arc::new(b"\xEF\xBB\xBFCaf\xC3\xA9".to_vec())).unwrap(); + let id = s.open_new("menu.pdf", pdf).unwrap(); + assert_eq!(page_texts(&s, id), ["Café"]); + let (_, pdf) = s.convert_to_pdf("old.txt", &Arc::new(b"Cr\xE8me br\xFBl\xE9e".to_vec())).unwrap(); + let id = s.open_new("old.pdf", pdf).unwrap(); + assert_eq!(page_texts(&s, id), ["Crème brûlée"]); +} + #[test] fn a_file_split_around_another_combines_in_order_with_one_bookmark() { let mut s = Session::new(); diff --git a/crates/ui-egui/src/files.rs b/crates/ui-egui/src/files.rs index a7854f8f..d74233fd 100644 --- a/crates/ui-egui/src/files.rs +++ b/crates/ui-egui/src/files.rs @@ -366,7 +366,7 @@ impl PdfCraftApp { self.notify_fmt("{name} is too large for a word list (at most 1 MB).", &[("name", &name)]); return; } - self.redact_search.words = pdfcraft_engine::redact_word_list(&String::from_utf8_lossy(&bytes)).join("\n"); + self.redact_search.words = pdfcraft_engine::redact_word_list(&pdfcraft_engine::decode_text(&bytes)).join("\n"); self.redact_search.mode = crate::redact_ui::SearchMode::Words; } }