diff --git a/.gitignore b/.gitignore index 68105186..7d675522 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ plan/ /log .DS_Store /fuzz-out +/assets/models diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index c415c3fd..6b4999a8 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -259,9 +259,9 @@ Every asset PrintCraft includes, bundles or uses to build its published material | `hayro-cmap` 0.1.0 | `assets/cmaps.brotli` | Adobe CMap resources (predefined CJK CMaps) (non-visual data, AGENTS.md §1.1) | Adobe | BSD-3-Clause | https://github.com/adobe-type-tools/cmap-resources (via hayro-cmap) | Character-code mapping tables required to read CJK PDFs. Non-visual data, allowed by AGENTS.md §1.1 | | `hayro-interpret` 0.7.0 | `src/font/generated/metrics.rs` | Standard-14 font metrics and encodings (as Rust tables) (non-visual data, AGENTS.md §1.1) | Adobe (Core 14 AFM metrics, PDF specification encodings); tables by The Hayro Authors | MIT OR Apache-2.0 | ISO 32000-2 Annex D; Adobe Core14 AFM files (via vendored hayro-interpret) | Glyph widths and encodings for non-embedded standard fonts. Non-visual data, allowed by AGENTS.md §1.1 | -## Downloaded at build time (20) +## Downloaded at build time (22) -Fetched by `cargo xtask demo-pdf` into `target/demo-fonts/`, verified by SHA-256 and never committed. +Fonts are fetched by `cargo xtask demo-pdf` into `target/demo-fonts/`, OCR models by `cargo xtask models` into `assets/models/`; each is verified by SHA-256 and never committed. | File | Title | Author | Licence | Source | Used for | |---|---|---|---|---|---| @@ -285,3 +285,5 @@ Fetched by `cargo xtask demo-pdf` into `target/demo-fonts/`, verified by SHA-256 | `NotoColorEmoji-Regular.ttf` | Noto Color Emoji | Google Inc. | OFL-1.1 | https://github.com/google/fonts/tree/9710da1eacb3be272583c3224dcb70f9da6eadbb/ofl/notocoloremoji | Showcase PDF (cargo xtask demo-pdf) | | `Inter[opsz,wght].ttf` | Inter (variable) | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/google/fonts/tree/9710da1eacb3be272583c3224dcb70f9da6eadbb/ofl/inter | Showcase PDF (cargo xtask demo-pdf) | | `Inter-Italic[opsz,wght].ttf` | Inter Italic (variable) | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/google/fonts/tree/9710da1eacb3be272583c3224dcb70f9da6eadbb/ofl/inter | Showcase PDF (cargo xtask demo-pdf) | +| `text-detection.rten` | Ocrs text detection model | Robert Knight (ocrs-models; trained on HierText, CC-BY-SA-4.0) | CC-BY-SA-4.0 | https://github.com/robertknight/ocrs-models (the models ocrs downloads; model card: https://huggingface.co/robertknight/ocrs, cc-by-sa-4.0) | Scan & OCR: finds words on page images (cargo xtask models) | +| `text-recognition.rten` | Ocrs text recognition model | Robert Knight (ocrs-models; trained on HierText, CC-BY-SA-4.0) | CC-BY-SA-4.0 | https://github.com/robertknight/ocrs-models (the models ocrs downloads; model card: https://huggingface.co/robertknight/ocrs, cc-by-sa-4.0) | Scan & OCR: reads the words it finds (cargo xtask models) | diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 2bdc101e..e997e074 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -2970,3 +2970,29 @@ licence_file = "assets/fonts/OFL-DancingScript.txt" kind = "font" usage = "Fill & Sign typed signatures and initials: the typed name's glyph outlines, drawn as filled paths (the font file itself is not embedded in PDFs)" sha256 = "21808625578fe8d8cd10cb684be546dca077b27cd03a53a2f1ec11dc743c924c" + +[[fetched]] +file = "text-detection.rten" +url = "https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.rten" +title = "Ocrs text detection model" +author = "Robert Knight (ocrs-models; trained on HierText, CC-BY-SA-4.0)" +source = "https://github.com/robertknight/ocrs-models (the models ocrs downloads; model card: https://huggingface.co/robertknight/ocrs, cc-by-sa-4.0)" +licence = "CC-BY-SA-4.0" +licence_url = "https://creativecommons.org/licenses/by-sa/4.0/legalcode.txt" +licence_sha256 = "28a9529c7d0bb4dc51f4bf5c116a3d16ef247a052f7591466768ddf563fd1cf5" +kind = "model" +usage = "Scan & OCR: finds words on page images (cargo xtask models)" +sha256 = "f15cfb56bd02c4bf478a20343986504a1f01e1665c2b3a0ad66340f054b1b5ca" + +[[fetched]] +file = "text-recognition.rten" +url = "https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.rten" +title = "Ocrs text recognition model" +author = "Robert Knight (ocrs-models; trained on HierText, CC-BY-SA-4.0)" +source = "https://github.com/robertknight/ocrs-models (the models ocrs downloads; model card: https://huggingface.co/robertknight/ocrs, cc-by-sa-4.0)" +licence = "CC-BY-SA-4.0" +licence_url = "https://creativecommons.org/licenses/by-sa/4.0/legalcode.txt" +licence_sha256 = "28a9529c7d0bb4dc51f4bf5c116a3d16ef247a052f7591466768ddf563fd1cf5" +kind = "model" +usage = "Scan & OCR: reads the words it finds (cargo xtask models)" +sha256 = "e484866d4cce403175bd8d00b128feb08ab42e208de30e42cd9889d8f1735a6e" diff --git a/Cargo.lock b/Cargo.lock index 96efd4b0..9a250ec1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1523,6 +1523,16 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.10" @@ -2617,6 +2627,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "num_enum" version = "0.7.6" @@ -2950,6 +2970,21 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "ocrs" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e4192a7dbc7570398ef97ab2606799703b6ea11da53f637d705d44546614435" +dependencies = [ + "anyhow", + "rayon", + "rten", + "rten-imageproc", + "rten-tensor", + "thiserror 2.0.21", + "wasm-bindgen", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3463,6 +3498,7 @@ dependencies = [ "printcraft-fonts", "printcraft-forms", "printcraft-model", + "printcraft-ocr", "printcraft-optimize", "printcraft-organize", "printcraft-print", @@ -3512,6 +3548,16 @@ dependencies = [ "printcraft-cos", ] +[[package]] +name = "printcraft-ocr" +version = "0.1.0" +dependencies = [ + "ocrs", + "printcraft-fonts", + "rten", + "thiserror 2.0.21", +] + [[package]] name = "printcraft-optimize" version = "0.1.0" @@ -4008,6 +4054,116 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rten" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7741649977427aa13009537ac9a8fe1a503b0aa58784069ce8f117e47eb6ab" +dependencies = [ + "flatbuffers", + "num_cpus", + "rayon", + "rten-base", + "rten-gemm", + "rten-model-file", + "rten-parallel", + "rten-shape-inference", + "rten-simd", + "rten-tensor", + "rten-vecmath", + "rustc-hash 2.1.3", + "smallvec", + "typeid", + "wasm-bindgen", +] + +[[package]] +name = "rten-base" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413dc975c7a3d0e6da0b146cd0387e3d389cceab0fc48328bfa716c7f2ae1b1e" + +[[package]] +name = "rten-gemm" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8efc07c654d4b3f8d81e2efd8b13aea7475568395289e5bb35393411aa6a451e" +dependencies = [ + "rayon", + "rten-base", + "rten-parallel", + "rten-simd", + "rten-tensor", +] + +[[package]] +name = "rten-imageproc" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18726eae9614e23a400560eb3a5e3fdb06526914c31aadaf767f14afde7680dd" +dependencies = [ + "rten-tensor", +] + +[[package]] +name = "rten-model-file" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ea9e630b377f9249ff3c290466fef282319f86d13d560964209104cd3af542" +dependencies = [ + "flatbuffers", + "rten-base", +] + +[[package]] +name = "rten-parallel" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba9075b130f81df7be82496ea3a9bbf20fe9e75f106a22b0ece8700850f8f664" +dependencies = [ + "rayon", + "rten-base", +] + +[[package]] +name = "rten-shape-inference" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89289c26b6a5248fccf4d67627596848a03ed31888e6ce4ba6f666c689258122" +dependencies = [ + "rten-tensor", + "smallvec", +] + +[[package]] +name = "rten-simd" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0c01294eb782adca256aa130b48a04006d03763073a68f45a9c33b982ea556" + +[[package]] +name = "rten-tensor" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf72556ba231d7af50238d26dd264856a2149c6369694eb79b22644c8280c81b" +dependencies = [ + "rayon", + "rten-base", + "rten-parallel", + "smallvec", + "typeid", +] + +[[package]] +name = "rten-vecmath" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1007f05160f9ea2ce4e9c67324e5bafb27ae1e5302d0c6b1ea4db0d35fed78ee" +dependencies = [ + "rten-base", + "rten-simd", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 05fee10b..da90424a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ printcraft-xfdf = { path = "crates/xfdf", version = "0.1.0" } printcraft-sign = { path = "crates/sign", version = "0.1.0" } printcraft-optimize = { path = "crates/optimize", version = "0.1.0" } printcraft-a11y = { path = "crates/a11y", version = "0.1.0" } +printcraft-ocr = { path = "crates/ocr", version = "0.1.0" } printcraft-edit = { path = "crates/edit", version = "0.1.0" } printcraft-create = { path = "crates/create", version = "0.1.0" } printcraft-forms = { path = "crates/forms", version = "0.1.0" } diff --git a/crates/automation/src/a11y.rs b/crates/automation/src/a11y.rs index 606939bb..adb43203 100644 --- a/crates/automation/src/a11y.rs +++ b/crates/automation/src/a11y.rs @@ -134,3 +134,40 @@ impl Automation { self.accessibility_figures(a) } } + +impl Automation { + pub(crate) fn ocr_recognize(&mut self, a: &Args) -> Result { + use printcraft_engine::ocr::OcrSettings; + let mut settings = OcrSettings::default(); + if let Some(d) = a.opt_num("dpi")? { + settings.dpi = d.clamp(72.0, 600.0) as f32; + } + if let Some(l) = a.opt_str("language")? { + if !printcraft_engine::ocr::LANGUAGES.iter().any(|x| x.0 == l) { + return Err(ToolError::InvalidArgs(format!("unsupported language {l:?}"))); + } + settings.language = l.into(); + } + if let Some(s) = a.opt_bool("skip_text_pages")? { + settings.skip_text_pages = s; + } + let pages = if a.opt_ints("pages")?.is_some() { self.pages(a, "pages")? } else { Vec::new() }; + let id = self.doc(a)?.id; + let found = self.session.recognize_text(id, &pages, settings).map_err(failed)?; + let list: Vec = found + .iter() + .map(|p| match &p.skipped { + Some(why) => json!({ "page": p.page + 1, "skipped": why }), + None => json!({ "page": p.page + 1, "words": p.words.len(), "text": p.text() }), + }) + .collect(); + Ok(json!({ "words": found.iter().map(|p| p.words.len()).sum::(), "pages": list })) + } + + pub(crate) fn ocr_status(&self) -> Result { + use printcraft_engine::ocr; + let dirs: Vec = ocr::Models::search_dirs().iter().map(|d| d.to_string_lossy().into_owned()).collect(); + let langs: Vec = ocr::LANGUAGES.iter().map(|(c, n)| json!({ "code": c, "name": n })).collect(); + Ok(json!({ "available": ocr::available(), "search_dirs": dirs, "languages": langs })) + } +} diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index d892b94d..03dd3db6 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -502,6 +502,8 @@ impl Automation { "comment_set_status" => self.comment_set_status(&a)?, "sign_list" => self.sign_list(&a)?, "accessibility_check" => self.a11y_check(&a)?, + "ocr_recognize" => self.ocr_recognize(&a)?, + "ocr_status" => self.ocr_status()?, "accessibility_report" => self.a11y_report(&a)?, "accessibility_fix" => self.a11y_fix(&a)?, "accessibility_figures" => self.accessibility_figures(&a)?, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 827e872e..d9422b29 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -737,6 +737,22 @@ pub fn tools() -> Vec { t("accessibility_set_alt", "Set alternate text", "Set a figure's alternate text (alt; empty or omitted clears it), or mark it decorative (decorative: true: its content becomes an artifact and the figure leaves the tags). figure is a number from accessibility_figures. Returns the figures. Undoable.") .cmd("a11y.alt_text") .with(schema(json!({ "doc": doc(), "figure": { "type": "integer", "minimum": 1 }, "alt": { "type": "string" }, "decorative": { "type": "boolean" } }), &["doc", "figure"])), + t("ocr_recognize", "Recognize text (OCR)", "Scan & OCR ▸ Recognize text: render pages, read the words in them and add them as invisible text over the page image, so scanned pages become searchable and selectable (a searchable image; the image is not changed). Pages that already have text are skipped unless skip_text_pages is false. Returns each page's recognised text, word count or why it was skipped. Needs the OCR models (ocr_status). Undoable as one step.") + .cmd("ocr.recognize") + .with(schema( + json!({ + "doc": doc(), + "pages": pages("to recognise (default: all)"), + "dpi": { "type": "number", "minimum": 72, "maximum": 600, "description": "Resolution pages are read at (default 300; a scanned page is read at most at its own resolution)." }, + "language": { "type": "string", "enum": ["en"], "description": "Document language (default en)." }, + "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true)." }, + }), + &["doc"], + )), + t("ocr_status", "OCR status", "Whether text recognition is available (its models are installed: run `cargo xtask models` or set PRINTCRAFT_MODELS), where it looks for them, and the languages it reads.") + .ro() + .cmd("ocr.recognize") + .with(schema(json!({}), &[])), t("doc_export_all_images", "Export all images", "Write the images that pages use into a folder (`_Page__Image_.jpg|png`), each once: JPEG images unchanged, others as PNG with their soft mask as alpha. min_size skips images with fewer pixels on their shorter side. Images that can't be decoded yet (JPEG 2000, JBIG2, CCITT, separations) are listed under skipped. Includes unsaved edits.") .cmd("export.all_images") .with(schema( diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index e9e6699d..b23f0153 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -1286,3 +1286,25 @@ fn initial_view_through_tools() { assert!(matches!(a.call("doc_initial_view", &json!({ "doc": doc, "layout": "spiral" })), Err(ToolError::InvalidArgs(_)))); assert!(matches!(a.call("doc_initial_view", &json!({ "doc": doc, "page": 9 })), Err(ToolError::Failed(_)))); } + +#[test] +fn ocr_tools_make_a_scan_searchable() { + let dir = workdir("ocr"); + let mut a = auto(&dir); + let status = ok(&mut a, "ocr_status", json!({})); + assert_eq!(status["languages"][0]["code"], "en"); + if status["available"] != true { + eprintln!("skipped: OCR models not installed"); + return; + } + let text = ok(&mut a, "doc_create", json!({ "from": "text", "text": "Searchable scans with recognised words" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_export_images", json!({ "doc": text, "folder": "scan", "dpi": 150, "pages": [1] })); + let png = std::fs::read_dir(dir.join("scan")).unwrap().next().unwrap().unwrap().path(); + let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": [png.to_string_lossy()] }))["doc"].as_u64().unwrap(); + let r = ok(&mut a, "ocr_recognize", json!({ "doc": scan })); + assert!(r["pages"][0]["text"].as_str().unwrap().to_lowercase().contains("searchable"), "{r}"); + assert!(r["words"].as_u64().unwrap() >= 4); + let again = ok(&mut a, "ocr_recognize", json!({ "doc": scan, "pages": [1] })); + assert!(again["pages"][0]["skipped"].is_string(), "{again}"); + assert!(matches!(a.call("ocr_recognize", &json!({ "doc": scan, "language": "xx" })), Err(ToolError::InvalidArgs(_)))); +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 74f5d0c0..41be5145 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -22,6 +22,7 @@ printcraft-xfdf = { workspace = true } printcraft-sign = { workspace = true } printcraft-optimize = { workspace = true } printcraft-a11y = { workspace = true } +printcraft-ocr = { workspace = true } printcraft-fonts = { workspace = true } printcraft-create = { workspace = true } printcraft-model = { workspace = true } diff --git a/crates/engine/src/catalog.rs b/crates/engine/src/catalog.rs index 7afa5cfb..8cdc62c0 100644 --- a/crates/engine/src/catalog.rs +++ b/crates/engine/src/catalog.rs @@ -239,11 +239,11 @@ pub static TOOL_GROUPS: &[ToolGroup] = &[ icon: "scan-text", hue: GREEN, badge: None, - availability: Planned("M10"), + availability: Ready, sections: &[ToolSection { title: "Recognize text", items: &[ - item("In this file", "scan-text", "ocr.recognize", Planned("M10")), + item("In this file", "scan-text", "ocr.recognize", Ready), item("In multiple files", "files", "ocr.recognize_batch", Planned("M10")), item("Enhance scanned file", "sparkles", "ocr.enhance", Planned("M10")), item("Correct recognized text", "text-select", "ocr.correct", Planned("M10")), diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index a1d2802e..0e3ff58c 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -248,6 +248,7 @@ pub const COMMANDS: &[CommandSpec] = &[ c("optimize.reduce", "Reduce file size…", FILE, None, Document, "file-down"), c("optimize.advanced", "Optimize PDF…", FILE, None, Document, "settings-2"), c("export.text", "Export to text…", FILE, None, Document, "type"), + c("ocr.recognize", "Recognize text…", None, None, Modification, "scan-text"), c("a11y.check", "Check for accessibility…", None, None, Document, "accessibility"), c("a11y.report", "Open accessibility report", None, None, Document, "file-text"), c("a11y.reading_options", "Change reading options…", None, None, Document, "book-open"), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index e446c45b..04b20d14 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -14,6 +14,7 @@ pub mod catalog; pub mod commands; pub mod export; pub mod links; +pub mod ocr; pub use printcraft_organize::{BoxSpec, PageBox, SplitBy, split_ranges}; @@ -775,6 +776,12 @@ pub enum Edit { MarkDecorative { figure: u32, }, + /// Scan & OCR ▸ Recognize text: put recognised words on `page` as invisible text over the + /// image (from [`ocr::OcrJob::run`]). + AddOcrText { + page: usize, + words: Vec, + }, /// Edit a PDF ▸ Edit text: replace the text of line `line` (from `Document::text_lines`) on /// `page`, in its own font when it can show it, else in Helvetica. EditTextLine { @@ -963,6 +970,7 @@ impl Edit { Edit::SetInitialView(_) => "Change initial view".into(), Edit::SetAltText { .. } => "Set alternate text".into(), Edit::MarkDecorative { .. } => "Mark figure as decorative".into(), + Edit::AddOcrText { .. } => "Recognize text".into(), Edit::EditTextLine { .. } | Edit::EditTextBlock { .. } => "Edit text".into(), Edit::EditPageImage { change, .. } => match change { ImageEdit::Move(_) => "Move image".into(), @@ -1130,6 +1138,7 @@ fn check_permission(edit: &Edit, p: &printcraft_cos::Permissions) -> Result<(), | Edit::SetInitialView(_) | Edit::SetAltText { .. } | Edit::MarkDecorative { .. } + | Edit::AddOcrText { .. } | Edit::EditTextLine { .. } | Edit::EditTextBlock { .. } | Edit::EditPageImage { .. } @@ -1316,6 +1325,9 @@ fn run_edit(doc: &mut printcraft_cos::Document, edit: &Edit, cx: &mut EditCtx) - Edit::EditTextLine { page, line, text } => { printcraft_edit::replace_line(doc, *page, *line, text)?; } + Edit::AddOcrText { page, words } => { + printcraft_edit::stamp(doc, *page, "OCR", printcraft_ocr::text_layer(words))?; + } Edit::EditPageImage { page, index, change } => { let img = printcraft_edit::page_images(doc, *page)? .into_iter() diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs new file mode 100644 index 00000000..2d0f499d --- /dev/null +++ b/crates/engine/src/ocr.rs @@ -0,0 +1,175 @@ +//! Scan & OCR ▸ Recognize text: render pages, read the words in them and add an invisible text +//! layer ([`Edit::AddOcrText`]), making scanned pages searchable and selectable. +//! +//! Recognition is slow (about a second a page), so it is split from the edit: [`Session::ocr_job`] +//! captures what it needs, [`OcrJob::run`] works anywhere (the UI runs it on a worker thread) and +//! [`Session::apply_ocr`] applies the result as one undoable step. + +use std::sync::{Arc, Mutex}; + +use printcraft_render::{PageInfo, PageRenderer, RenderConfig, RenderRequest, RequestKind}; + +pub use printcraft_ocr::{LANGUAGES, Models, Ocr, OcrError, PlacedWord}; + +use crate::{DocId, Edit, EditError, Session}; + +/// Recognize Text settings. +#[derive(Clone, Debug, PartialEq)] +pub struct OcrSettings { + /// Resolution pages are rendered at for recognition (Acrobat's "Downsample to" choices). + pub dpi: f32, + /// Language code from [`LANGUAGES`] (the models read the Latin alphabet, whatever this says). + pub language: String, + /// Leave pages that already have text alone (Acrobat reports "page contains renderable + /// text" and skips them). + pub skip_text_pages: bool, +} + +impl Default for OcrSettings { + fn default() -> Self { + OcrSettings { dpi: 300.0, language: "en".into(), skip_text_pages: true } + } +} + +/// What recognition found on one page. +#[derive(Clone, Debug, PartialEq)] +pub struct OcrPage { + pub page: usize, + pub words: Vec, + /// Why the page was not read, if it was skipped. + pub skipped: Option, +} + +impl OcrPage { + pub fn text(&self) -> String { + self.words.iter().map(|w| w.text.as_str()).collect::>().join(" ") + } +} + +/// The recogniser, loaded once (it takes a moment) and shared. +pub fn engine() -> Result, String> { + static OCR: Mutex>> = Mutex::new(None); + let mut slot = OCR.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(o) = slot.as_ref() { + return Ok(o.clone()); + } + let o = Arc::new(Ocr::find().map_err(|e| e.to_string())?); + *slot = Some(o.clone()); + Ok(o) +} + +/// Whether the recognition models are installed. +pub fn available() -> bool { + Models::find().is_some() +} + +/// Everything recognition needs from a document, detached from the session. +pub struct OcrJob { + bytes: Arc>, + password: Option, + infos: Vec, + /// Per page: the render scale (pixels per point) that shows its images at their own + /// resolution, if it has any. + native: Vec>, + pub pages: Vec, + pub settings: OcrSettings, +} + +impl OcrJob { + /// Read the pages. `progress(done, total)` is called before each page; returning `false` + /// stops (the pages read so far are returned). + pub fn run(self, ocr: &Ocr, mut progress: impl FnMut(usize, usize) -> bool) -> Vec { + let config = RenderConfig { password: self.password.as_deref().map(Arc::from), ..Default::default() }; + let mut r = PageRenderer::new(self.bytes.clone(), config); + let mut out = Vec::new(); + let total = self.pages.len(); + for (done, &page) in self.pages.iter().enumerate() { + if !progress(done, total) { + break; + } + let Some(info) = self.infos.get(page) else { continue }; + let skip = |why: &str| OcrPage { page, words: Vec::new(), skipped: Some(why.into()) }; + if self.settings.skip_text_pages { + let t = r.render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }); + if t.text.as_ref().is_some_and(|t| !t.plain_text().trim().is_empty()) { + out.push(skip("the page already contains text")); + continue; + } + } + // A scan is read at its own resolution (enlarging it only blurs the letters), at most + // the chosen one. + let wanted = self.settings.dpi.clamp(72.0, 600.0) / 72.0; + let scale = self.native.get(page).copied().flatten().map_or(wanted, |n| n.clamp(1.0, wanted)); + let shot = r.render(RenderRequest { page, scale, ..Default::default() }); + if let Some(e) = shot.error { + out.push(skip(&e)); + continue; + } + let lines = match ocr.recognize(&shot.rgba, shot.width, shot.height) { + Ok(l) => l, + Err(e) => { + out.push(skip(&e.to_string())); + continue; + } + }; + let scale = shot.width as f32 / info.width.max(1e-3); + let to_user = |x: f32, y: f32| { + let [u, v] = info.view_to_user(x / scale, y / scale); + [u as f64, v as f64] + }; + let words = lines.iter().flat_map(|l| &l.words).map(|w| PlacedWord::place(w, to_user)).collect(); + out.push(OcrPage { page, words, skipped: None }); + } + progress(total, total); + out + } +} + +impl Session { + /// Capture what recognising `pages` (0-based; empty = all) of document `id` needs. + pub fn ocr_job(&self, id: DocId, pages: &[usize], settings: OcrSettings) -> Option { + let doc = self.get(id)?; + let all = doc.info.pages.len(); + let pages: Vec = if pages.is_empty() { (0..all).collect() } else { pages.iter().copied().filter(|p| *p < all).collect() }; + let native = (0..all) + .map(|p| { + if !pages.contains(&p) { + return None; + } + doc.page_images(p) + .iter() + .filter(|i| i.width > 1 && i.height > 1) + .map(|i| { + let m = i.matrix; + let (w, h) = (m[0].hypot(m[1]), m[2].hypot(m[3])); + (i.width as f64 / w.max(1e-6)).max(i.height as f64 / h.max(1e-6)) as f32 + }) + .reduce(f32::max) + }) + .collect(); + Some(OcrJob { bytes: doc.bytes.clone(), password: doc.password.clone(), infos: doc.info.pages.clone(), native, pages, settings }) + } + + /// Add the words found by [`OcrJob::run`] as one undoable step. Returns the number of words. + pub fn apply_ocr(&mut self, id: DocId, found: &[OcrPage]) -> Result { + let edits: Vec = + found.iter().filter(|p| !p.words.is_empty()).map(|p| Edit::AddOcrText { page: p.page, words: p.words.clone() }).collect(); + let words = found.iter().map(|p| p.words.len()).sum(); + if !edits.is_empty() { + self.apply(id, Edit::Batch { label: "Recognize text".into(), edits })?; + } + Ok(words) + } + + /// Recognise text on `pages` and add it, in one call (the CLI and agents use this). + pub fn recognize_text(&mut self, id: DocId, pages: &[usize], settings: OcrSettings) -> Result, String> { + let job = self.ocr_job(id, pages, settings).ok_or("no such document")?; + if let Some(why) = self.get(id).and_then(|d| d.read_only_reason.clone()) { + return Err(why); + } + let ocr = engine()?; + let found = job.run(&ocr, |_, _| true); + self.apply_ocr(id, &found).map_err(|e| e.to_string())?; + Ok(found) + } +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 37576779..e537dbea 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1200,3 +1200,38 @@ fn backgrounds_and_watermarks_from_files() { let junk = MarkFile { name: "x.png".into(), bytes: Arc::new(b"not an image".to_vec()), page: 0 }; assert!(s.apply(id, Edit::AddBackground { pages: vec![0], settings: Background::default(), replace: true, file: Some(junk) }).is_err()); } + +/// Scan & OCR ▸ Recognize text on a page that is only a picture of text (needs the models: +/// `cargo xtask models`; skipped without them). +#[test] +fn recognize_text_makes_a_scanned_page_searchable() { + if !ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + let mut s = Session::new().with_clock(|| 1_700_000_000); + let text = s.create_from_text("t", "The quick brown fox jumps over the lazy dog.").unwrap(); + let id = s.open("text.pdf", None, text, None).unwrap(); + let png = export::Exporter::new(s.get(id).unwrap()).png(0, 150.0).unwrap(); + let scan = s.create_from_images(&[("scan.png".into(), png)]).unwrap(); + let id = s.open("scan.pdf", None, scan, None).unwrap(); + assert_eq!(page_texts(&s, id), [""], "a picture has no text"); + + let found = s.recognize_text(id, &[], ocr::OcrSettings::default()).unwrap(); + assert_eq!(found.len(), 1); + let text = page_texts(&s, id)[0].to_lowercase(); + for w in ["quick", "brown", "fox", "lazy"] { + assert!(text.contains(w), "{text}"); + } + assert_eq!(s.get(id).unwrap().can_undo(), Some("Recognize text")); + // The text sits over the words: "quick" is left of "lazy" and on the same line. + let words = &found[0].words; + let q = words.iter().find(|w| w.text.to_lowercase().contains("quick")).unwrap(); + let l = words.iter().find(|w| w.text.to_lowercase().contains("lazy")).unwrap(); + let top = |w: &ocr::PlacedWord| w.origin[1] + w.up[1]; + assert!(q.origin[0] < l.origin[0] && (top(q) - top(l)).abs() < 3.0, "{q:?} {l:?}"); + + // A second pass skips the page: it has text now. + let again = s.recognize_text(id, &[], ocr::OcrSettings::default()).unwrap(); + assert!(again[0].skipped.is_some()); +} diff --git a/crates/ocr/Cargo.toml b/crates/ocr/Cargo.toml new file mode 100644 index 00000000..436d3100 --- /dev/null +++ b/crates/ocr/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "printcraft-ocr" +description = "Scan & OCR: recognise text in page images (ocrs) and write it as an invisible, searchable text layer" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +printcraft-fonts = { workspace = true } +ocrs = { version = "0.13.1", default-features = false, features = ["rten"] } +rten = { version = "0.26.0", default-features = false, features = ["rten_format"] } +thiserror.workspace = true diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs new file mode 100644 index 00000000..94080209 --- /dev/null +++ b/crates/ocr/src/lib.rs @@ -0,0 +1,290 @@ +//! printcraft-ocr — Scan & OCR ▸ Recognize text (L4). +//! +//! Recognition runs the ocrs engine (MIT/Apache-2.0) with its pre-trained models (CC-BY-SA-4.0, +//! fetched by `cargo xtask models`; see ATTRIBUTION.toml). The caller renders a page to pixels; +//! [`Ocr::recognize`] finds the words in them, and [`text_layer`] turns words placed in user space +//! into page content: invisible text (rendering mode 3) over each word, so the page becomes a +//! searchable image (Acrobat's "Searchable Image (Exact)": the image is left untouched). +//! +//! The models read the Latin alphabet (English and other languages written without accents). + +use std::path::{Path, PathBuf}; + +pub use printcraft_fonts::helvetica_width; + +/// The model files, as named in ATTRIBUTION.toml. +pub const DETECTION_MODEL: &str = "text-detection.rten"; +pub const RECOGNITION_MODEL: &str = "text-recognition.rten"; + +/// The languages the models read (ISO 639-1); all use the Latin alphabet without accents. +pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; + +#[derive(Debug, thiserror::Error)] +pub enum OcrError { + #[error("the text recognition models are not installed (run `cargo xtask models`, or set PRINTCRAFT_MODELS)")] + NoModels, + #[error("loading {0}: {1}")] + Load(String, String), + #[error("text recognition failed: {0}")] + Recognize(String), + #[error("the image is empty")] + EmptyImage, +} + +/// Where the two model files are. +#[derive(Clone, Debug, PartialEq)] +pub struct Models { + pub detection: PathBuf, + pub recognition: PathBuf, +} + +impl Models { + /// The models in `dir`, if both files are there. + pub fn in_dir(dir: &Path) -> Option { + let m = Models { detection: dir.join(DETECTION_MODEL), recognition: dir.join(RECOGNITION_MODEL) }; + (m.detection.is_file() && m.recognition.is_file()).then_some(m) + } + + /// Look for the models: `$PRINTCRAFT_MODELS`, then `models/` beside the executable (and + /// `Resources/models` in a macOS bundle), then the source tree's `assets/models/`. + pub fn find() -> Option { + Self::search_dirs().iter().find_map(|d| Self::in_dir(d)) + } + + /// The directories [`Models::find`] looks in, in order. + pub fn search_dirs() -> Vec { + let mut dirs = Vec::new(); + if let Some(d) = std::env::var_os("PRINTCRAFT_MODELS") { + dirs.push(PathBuf::from(d)); + } + if let Some(exe) = std::env::current_exe().ok().and_then(|e| e.parent().map(Path::to_path_buf)) { + dirs.push(exe.join("models")); + dirs.push(exe.join("../Resources/models")); + } + dirs.push(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../assets/models")); + dirs + } +} + +/// A recognised word: its text and its box in image pixels `[left, top, right, bottom]`. +#[derive(Clone, Debug, PartialEq)] +pub struct Word { + pub text: String, + pub rect: [f32; 4], +} + +/// A recognised line of words, in reading order. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Line { + pub words: Vec, +} + +impl Line { + pub fn text(&self) -> String { + self.words.iter().map(|w| w.text.as_str()).collect::>().join(" ") + } +} + +/// A loaded recogniser. Loading takes a moment; keep one and reuse it. +pub struct Ocr { + engine: ocrs::OcrEngine, +} + +impl Ocr { + pub fn load(models: &Models) -> Result { + let load = |p: &Path| rten::Model::load_file(p).map_err(|e| OcrError::Load(p.display().to_string(), e.to_string())); + let params = ocrs::OcrEngineParams { + detection_model: Some(load(&models.detection)?), + recognition_model: Some(load(&models.recognition)?), + ..Default::default() + }; + let engine = ocrs::OcrEngine::new(params).map_err(|e| OcrError::Load("ocr engine".into(), e.to_string()))?; + Ok(Ocr { engine }) + } + + /// Load the models found by [`Models::find`]. + pub fn find() -> Result { + Self::load(&Models::find().ok_or(OcrError::NoModels)?) + } + + /// Recognise the text in an RGBA (or RGB, or grey) image, `width` × `height` pixels. + pub fn recognize(&self, pixels: &[u8], width: u32, height: u32) -> Result, OcrError> { + if width == 0 || height == 0 || pixels.is_empty() { + return Err(OcrError::EmptyImage); + } + // ocrs wants 1 or 3 channels. + let rgb: std::borrow::Cow<[u8]> = if pixels.len() == (width * height * 4) as usize { + pixels.chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect::>().into() + } else { + pixels.into() + }; + let err = |e: &dyn std::fmt::Display| OcrError::Recognize(e.to_string()); + let source = ocrs::ImageSource::from_bytes(&rgb, (width, height)).map_err(|e| err(&e))?; + let input = self.engine.prepare_input(source).map_err(|e| err(&e))?; + let found = self.engine.detect_words(&input).map_err(|e| err(&e))?; + let lines = self.engine.find_text_lines(&input, &found); + let read = self.engine.recognize_text(&input, &lines).map_err(|e| err(&e))?; + use ocrs::TextItem; + Ok(read + .into_iter() + .flatten() + .map(|line| Line { + words: line + .words() + .map(|w| { + let r = w.bounding_rect(); + Word { text: w.to_string(), rect: [r.left() as f32, r.top() as f32, r.right() as f32, r.bottom() as f32] } + }) + .filter(|w| !w.text.trim().is_empty()) + .collect(), + }) + .filter(|l| !l.words.is_empty()) + .collect()) + } +} + +/// A word placed on a page, in PDF user space: the bottom-left corner of its box, and the +/// vectors along its bottom edge (`across`, left → right as read) and its left edge (`up`). +#[derive(Clone, Debug, PartialEq)] +pub struct PlacedWord { + pub text: String, + pub origin: [f64; 2], + pub across: [f64; 2], + pub up: [f64; 2], +} + +impl PlacedWord { + /// Place a pixel-space word with `to_user`, which maps an image pixel (x right, y down) to + /// user space. + pub fn place(word: &Word, to_user: impl Fn(f32, f32) -> [f64; 2]) -> PlacedWord { + let [l, t, r, b] = word.rect; + let o = to_user(l, b); + let br = to_user(r, b); + let tl = to_user(l, t); + PlacedWord { text: word.text.clone(), origin: o, across: [br[0] - o[0], br[1] - o[1]], up: [tl[0] - o[0], tl[1] - o[1]] } + } +} + +/// Helvetica's ascender and descender (em fractions): a word box spans about this much. +const BOX_EM: f64 = 0.93; +const DESCENT: f64 = 0.21; + +fn num(v: f64) -> String { + let s = format!("{:.4}", if v.abs() < 1e-9 { 0.0 } else { v }); + let s = s.trim_end_matches('0').trim_end_matches('.'); + if s == "-0" { "0".into() } else { s.into() } +} + +/// Page content that writes `words` as invisible text in `/PCHelv` (standard Helvetica, +/// WinAnsiEncoding), each word stretched to its box so selection and search highlight the +/// right place. Marked content `/OCR` so it can be told apart from the page's own text. +pub fn text_layer(words: &[PlacedWord]) -> Vec { + let mut out = b"/OCR BMC\nBT\n3 Tr\n/PCHelv 1 Tf\n".to_vec(); + for w in words { + let width = helvetica_width(&w.text, 1.0); + let height = w.up[0].hypot(w.up[1]); + if width <= 0.0 || height <= 0.0 { + continue; + } + // The em square: its height spans the box; its width is stretched to fit the word. + let em = [w.up[0] / BOX_EM, w.up[1] / BOX_EM]; + let ax = [w.across[0] / width, w.across[1] / width]; + let o = [w.origin[0] + em[0] * DESCENT, w.origin[1] + em[1] * DESCENT]; + let m = [ax[0], ax[1], em[0], em[1], o[0], o[1]]; + let nums: Vec = m.iter().map(|v| num(*v)).collect(); + out.extend_from_slice(nums.join(" ").as_bytes()); + out.extend_from_slice(b" Tm "); + out.extend(printcraft_fonts::literal(&printcraft_fonts::win_ansi(&w.text))); + out.extend_from_slice(b" Tj\n"); + } + out.extend_from_slice(b"ET\nEMC\n"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn places_words_through_the_mapping() { + // 2 pixels per point, page 612 × 792, y flipped. + let to_user = |x: f32, y: f32| [x as f64 / 2.0, 792.0 - y as f64 / 2.0]; + let w = PlacedWord::place(&Word { text: "Hello".into(), rect: [100.0, 200.0, 300.0, 240.0] }, to_user); + assert_eq!(w.origin, [50.0, 672.0]); + assert_eq!(w.across, [100.0, 0.0]); + assert_eq!(w.up, [0.0, 20.0]); + } + + #[test] + fn text_layer_is_invisible_text_fitted_to_each_box() { + let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3] }; + let s = String::from_utf8(text_layer(&[w])).unwrap(); + assert!(s.contains("3 Tr") && s.contains("/PCHelv 1 Tf"), "{s}"); + let width = helvetica_width("Hi", 1.0); + assert!(s.contains(&format!("{} 0 0 10 10 22.1 Tm (Hi) Tj", num(30.0 / width))), "{s}"); + assert!(s.starts_with("/OCR BMC") && s.ends_with("EMC\n")); + } + + #[test] + fn rotated_pages_keep_the_reading_direction() { + // A page turned 90°: image x runs up the page. + let to_user = |x: f32, y: f32| [y as f64, x as f64]; + let w = PlacedWord::place(&Word { text: "Up".into(), rect: [0.0, 0.0, 50.0, 10.0] }, to_user); + assert_eq!(w.across, [0.0, 50.0]); + assert_eq!(w.up, [-10.0, 0.0]); + } + + /// End to end on rendered text, when the models are installed (`cargo xtask models`). + #[test] + fn reads_rendered_text() { + let Some(models) = Models::find() else { + eprintln!("skipped: OCR models not installed"); + return; + }; + let ocr = Ocr::load(&models).unwrap(); + let (w, h, px) = test_image::hello(); + let lines = ocr.recognize(&px, w, h).unwrap(); + let text: Vec = lines.iter().map(Line::text).collect(); + assert!(text.iter().any(|t| t.contains("HELL") && t.contains("WOR")), "{text:?}"); + let word = lines.iter().flat_map(|l| &l.words).find(|w| w.text.contains("HELL")).unwrap(); + assert!(word.rect[0] >= 10.0 && word.rect[0] < 60.0, "{word:?}"); + } + + mod test_image { + /// "HELLO WORLD" drawn with blocky 5×7 letters, 4 pixels per dot, black on white. + pub fn hello() -> (u32, u32, Vec) { + const GLYPHS: &[(char, [&str; 7])] = &[ + ('H', ["X...X", "X...X", "X...X", "XXXXX", "X...X", "X...X", "X...X"]), + ('E', ["XXXXX", "X....", "X....", "XXXX.", "X....", "X....", "XXXXX"]), + ('L', ["X....", "X....", "X....", "X....", "X....", "X....", "XXXXX"]), + ('O', [".XXX.", "X...X", "X...X", "X...X", "X...X", "X...X", ".XXX."]), + ('W', ["X...X", "X...X", "X...X", "X.X.X", "X.X.X", "XX.XX", "X...X"]), + ('R', ["XXXX.", "X...X", "X...X", "XXXX.", "X.X..", "X..X.", "X...X"]), + ('D', ["XXXX.", "X...X", "X...X", "X...X", "X...X", "X...X", "XXXX."]), + ]; + let text = "HELLO WORLD"; + let (dot, x0, y0) = (4u32, 40u32, 40u32); + let (w, h) = (x0 * 2 + text.len() as u32 * 6 * dot, y0 * 2 + 7 * dot); + let mut px = vec![255u8; (w * h * 4) as usize]; + for (i, c) in text.chars().enumerate() { + let Some((_, rows)) = GLYPHS.iter().find(|g| g.0 == c) else { continue }; + for (ry, row) in rows.iter().enumerate() { + for (rx, b) in row.bytes().enumerate() { + if b != b'X' { + continue; + } + for dy in 0..dot { + for dx in 0..dot { + let x = x0 + (i as u32 * 6 + rx as u32) * dot + dx; + let y = y0 + ry as u32 * dot + dy; + let o = ((y * w + x) * 4) as usize; + px[o..o + 3].copy_from_slice(&[0, 0, 0]); + } + } + } + } + } + (w, h, px) + } + } +} diff --git a/crates/ui-egui/src/commands.rs b/crates/ui-egui/src/commands.rs index 96b2a2c4..37f03ff3 100644 --- a/crates/ui-egui/src/commands.rs +++ b/crates/ui-egui/src/commands.rs @@ -223,6 +223,7 @@ impl PrintCraftApp { "export.image" => self.dialog = Some(Dialog::Export(crate::export_ui::ExportKind::Image)), "export.text" => self.dialog = Some(Dialog::Export(crate::export_ui::ExportKind::Text)), "a11y.check" => self.start_accessibility_check(), + "ocr.recognize" => self.dialog = Some(Dialog::RecognizeText), "edit.edit_text" => { self.quick_tool = crate::QuickTool::EditText; self.left = crate::LeftPanel::Tool("edit"); diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index 5698aaab..f3a64ec2 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -50,6 +50,7 @@ pub fn show(app: &mut PrintCraftApp, ctx: &egui::Context) { let mut replace_now = false; let mut open_revision: Option = None; let mut a11y_now = false; + let mut ocr_now = false; let mut combine_now = false; let mut stamp_now = false; let mut alt_now = false; @@ -818,6 +819,12 @@ pub fn show(app: &mut PrintCraftApp, ctx: &egui::Context) { close = go || cancel; return; } + Dialog::RecognizeText => { + let (go, cancel) = crate::ocr_ui::body(ui, app, &t); + ocr_now = go; + close = go || cancel; + return; + } Dialog::AccessibilityOptions => { let (start, cancel) = crate::a11y_ui::options_body(ui, app, &t); a11y_now = start; @@ -1184,6 +1191,9 @@ pub fn show(app: &mut PrintCraftApp, ctx: &egui::Context) { if combine_now { app.combine_staged(); } + if ocr_now { + app.start_ocr(); + } if a11y_now { app.a11y_skipped.clear(); app.run_accessibility_check(); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index bfca0158..ccbdb483 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -18,6 +18,7 @@ mod create_ui; mod crop; mod export_ui; mod marks_ui; +mod ocr_ui; mod optimize_ui; mod search_ui; mod sign_ui; @@ -184,6 +185,8 @@ pub enum Dialog { DuplicateField, /// Check for accessibility ▸ Accessibility Checker Options. AccessibilityOptions, + /// Scan & OCR ▸ Recognize text. + RecognizeText, /// Combine files: the files, their order and pages. Combine, /// Custom stamps ▸ Create. @@ -294,6 +297,10 @@ pub struct PrintCraftApp { pub sig_expanded: Vec, /// Accessibility Checker: options, the last check, and rules skipped by hand. pub a11y_options: a11y_ui::A11yOptions, + /// Scan & OCR: the Recognize Text choices, the running job, and (tests) run it inline. + pub ocr_draft: ocr_ui::OcrDraft, + pub ocr_run: Option, + pub ocr_sync: bool, pub a11y: a11y_ui::A11yState, pub a11y_skipped: std::collections::BTreeSet, pub alt_draft: a11y_ui::AltDraft, @@ -429,6 +436,9 @@ impl PrintCraftApp { digital_ids: Vec::new(), sig_expanded: Vec::new(), a11y_options: a11y_ui::A11yOptions::default(), + ocr_draft: ocr_ui::OcrDraft::default(), + ocr_run: None, + ocr_sync: false, a11y: a11y_ui::A11yState::default(), a11y_skipped: Default::default(), alt_draft: Default::default(), @@ -866,6 +876,7 @@ impl PrintCraftApp { "export-text" => Some(Dialog::Export(export_ui::ExportKind::Text)), "export-all-images" => Some(Dialog::Export(export_ui::ExportKind::AllImages)), "accessibility-options" => Some(Dialog::AccessibilityOptions), + "recognize-text" => Some(Dialog::RecognizeText), "signature" => Some(Dialog::Signature), "optimize" => Some(Dialog::Optimize), "sign" | "certify" => { @@ -1047,6 +1058,7 @@ impl eframe::App for PrintCraftApp { self.shortcuts(ctx); self.process_pending_edits(); self.poll_export(); + self.poll_ocr(); self.process_file_requests(); // Pull finished renders into textures for every open document. for view in &mut self.views { diff --git a/crates/ui-egui/src/ocr_ui.rs b/crates/ui-egui/src/ocr_ui.rs new file mode 100644 index 00000000..8f000673 --- /dev/null +++ b/crates/ui-egui/src/ocr_ui.rs @@ -0,0 +1,229 @@ +//! Scan & OCR ▸ Recognize text: the Recognize Text dialog (pages, language, output, resolution) +//! and the background run with its progress. Recognition happens on a worker thread; the result +//! is applied as one undoable edit when it is done. + +use std::sync::{Arc, Mutex}; + +use egui::{Align, Layout}; +use printcraft_engine::DocId; +use printcraft_engine::ocr::{LANGUAGES, OcrPage, OcrSettings}; + +use crate::theme::{self, Tokens}; +use crate::{PrintCraftApp, widgets}; + +/// Which pages to read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OcrPages { + All, + Current, + Range, +} + +/// The dialog's choices (kept between runs, as in Acrobat). +#[derive(Clone, Debug, PartialEq)] +pub struct OcrDraft { + pub pages: OcrPages, + pub from: usize, + pub to: usize, + pub language: String, + /// "Downsample to" resolution. + pub dpi: u32, +} + +impl Default for OcrDraft { + fn default() -> Self { + OcrDraft { pages: OcrPages::All, from: 1, to: 1, language: "en".into(), dpi: 300 } + } +} + +/// Progress of a run: pages done, total, the result once finished, and a cancel request. +#[derive(Default)] +pub struct OcrProgress { + pub done: usize, + pub total: usize, + pub result: Option, String>>, + pub cancel: bool, +} + +pub struct OcrRun { + pub doc: DocId, + pub progress: Arc>, +} + +pub(crate) fn body(ui: &mut egui::Ui, app: &mut PrintCraftApp, t: &Tokens) -> (bool, bool) { + let pages = app.active_ids().and_then(|(_, id)| app.session.get(id)).map_or(1, |d| d.info.pages.len().max(1)); + let available = printcraft_engine::ocr::available(); + let d = &mut app.ocr_draft; + d.to = d.to.clamp(1, pages); + d.from = d.from.clamp(1, d.to); + ui.label(egui::RichText::new("Recognize Text").font(theme::semibold(18.0))); + ui.add_space(8.0); + let group = |ui: &mut egui::Ui, title: &str, body: &mut dyn FnMut(&mut egui::Ui)| { + ui.label(egui::RichText::new(title).font(theme::semibold(13.0))); + egui::Frame::new().fill(t.hover).corner_radius(egui::CornerRadius::same(6)).inner_margin(egui::Margin::same(10)).show(ui, |ui| { + ui.set_width(ui.available_width()); + body(ui); + }); + ui.add_space(8.0); + }; + group(ui, "Pages", &mut |ui| { + ui.radio_value(&mut d.pages, OcrPages::All, "All pages"); + ui.radio_value(&mut d.pages, OcrPages::Current, "Current page"); + ui.horizontal(|ui| { + ui.radio_value(&mut d.pages, OcrPages::Range, "From"); + let on = d.pages == OcrPages::Range; + ui.add_enabled(on, egui::DragValue::new(&mut d.from).range(1..=pages)); + ui.label("to"); + ui.add_enabled(on, egui::DragValue::new(&mut d.to).range(1..=pages)); + }); + }); + group(ui, "Settings", &mut |ui| { + egui::Grid::new("ocr-settings").num_columns(2).spacing([12.0, 8.0]).show(ui, |ui| { + ui.label("Document language"); + let name = LANGUAGES.iter().find(|l| l.0 == d.language).map_or("English", |l| l.1); + egui::ComboBox::from_id_salt("ocr-language").selected_text(name).width(220.0).show_ui(ui, |ui| { + for (code, name) in LANGUAGES { + ui.selectable_value(&mut d.language, (*code).to_string(), *name); + } + }); + ui.end_row(); + ui.label("Output"); + egui::ComboBox::from_id_salt("ocr-output").selected_text("Searchable Image (Exact)").width(220.0).show_ui(ui, |ui| { + let _ = ui + .selectable_label(true, "Searchable Image (Exact)") + .on_hover_text("Adds invisible text over each word; the page image is not changed"); + }); + ui.end_row(); + ui.label("Downsample to"); + egui::ComboBox::from_id_salt("ocr-dpi").selected_text(format!("{} dpi", d.dpi)).width(220.0).show_ui(ui, |ui| { + for v in [600, 300, 150, 72] { + ui.selectable_value(&mut d.dpi, v, format!("{v} dpi")); + } + }); + ui.end_row(); + }); + }); + if !available { + ui.label( + egui::RichText::new("Text recognition isn't installed: its models are missing (run `cargo xtask models`, or set PRINTCRAFT_MODELS).") + .small() + .color(t.text_muted), + ); + ui.add_space(6.0); + } + ui.add_space(6.0); + let (mut go, mut cancel) = (false, false); + ui.horizontal(|ui| { + ui.with_layout(Layout::right_to_left(Align::Center), |ui| { + if ui.add_enabled_ui(available, |ui| widgets::pill_button(ui, "Recognize text", true)).inner.clicked() { + go = true; + } + if widgets::pill_button(ui, "Cancel", false).clicked() { + cancel = true; + } + }) + }); + (go, cancel) +} + +impl PrintCraftApp { + /// Recognize text on the pages chosen in the dialog, in the background. + pub fn start_ocr(&mut self) { + let Some((vi, id)) = self.active_ids() else { return }; + if self.ocr_run.is_some() { + self.notify("Text recognition is already running"); + return; + } + if let Some(why) = self.session.get(id).and_then(|d| d.read_only_reason.clone()) { + self.notify(why); + return; + } + let d = &self.ocr_draft; + let pages: Vec = match d.pages { + OcrPages::All => Vec::new(), + OcrPages::Current => vec![self.views[vi].current], + OcrPages::Range => (d.from.saturating_sub(1)..d.to).collect(), + }; + let settings = OcrSettings { dpi: d.dpi as f32, language: d.language.clone(), ..Default::default() }; + let Some(job) = self.session.ocr_job(id, &pages, settings) else { return }; + let progress = Arc::new(Mutex::new(OcrProgress { total: job.pages.len(), ..Default::default() })); + let p = progress.clone(); + let work = move || { + let result = printcraft_engine::ocr::engine().map(|ocr| { + job.run(&ocr, |done, total| { + let Ok(mut s) = p.lock() else { return false }; + s.done = done; + s.total = total; + !s.cancel + }) + }); + if let Ok(mut s) = p.lock() { + s.result = Some(result); + } + }; + #[cfg(not(target_arch = "wasm32"))] + if self.ocr_sync { + work(); + } else { + std::thread::Builder::new().name("printcraft-ocr".into()).spawn(work).ok(); + } + #[cfg(target_arch = "wasm32")] + work(); + self.ocr_run = Some(OcrRun { doc: id, progress }); + self.poll_ocr(); + } + + /// Stop a running recognition (the pages read so far are kept). + pub fn cancel_ocr(&mut self) { + if let Some(r) = &self.ocr_run + && let Ok(mut s) = r.progress.lock() + { + s.cancel = true; + } + } + + /// Show progress; apply the result once the worker is done. + pub(crate) fn poll_ocr(&mut self) { + let Some(run) = self.ocr_run.as_ref() else { return }; + let (doc, progress) = (run.doc, run.progress.clone()); + let Ok(mut s) = progress.lock() else { return }; + let Some(result) = s.result.take() else { + let msg = format!("Recognizing text… page {} of {}", (s.done + 1).min(s.total.max(1)), s.total.max(1)); + drop(s); + if self.toast.as_ref().is_none_or(|t| t.0 != msg) { + self.notify(msg); + } + if let Some(ctx) = &self.ctx { + ctx.request_repaint_after(std::time::Duration::from_millis(200)); + } + return; + }; + drop(s); + self.ocr_run = None; + let found = match result { + Ok(f) => f, + Err(e) => { + self.notify(e); + return; + } + }; + let read = found.iter().filter(|p| p.skipped.is_none()).count(); + let skipped = found.len() - read; + match self.session.apply_ocr(doc, &found) { + Ok(words) => { + if let Some(info) = self.session.get(doc).map(|d| d.info.clone()) + && let Some(view) = self.views.iter_mut().find(|v| v.id == doc) + { + view.document_changed(&info); + } + let mut msg = + format!("Recognized {words} word{} on {read} page{}", if words == 1 { "" } else { "s" }, if read == 1 { "" } else { "s" }); + if skipped > 0 { + msg.push_str(&format!("; {skipped} page{} already had text", if skipped == 1 { "" } else { "s" })); + } + self.notify(msg); + } + Err(e) => self.notify(e.to_string()), + } + } +} diff --git a/crates/ui-egui/tests/ocr.rs b/crates/ui-egui/tests/ocr.rs new file mode 100644 index 00000000..7decf1e4 --- /dev/null +++ b/crates/ui-egui/tests/ocr.rs @@ -0,0 +1,43 @@ +//! Scan & OCR ▸ Recognize text in the real shell (egui_kittest): the dialog, the run and its +//! result (skipped without the OCR models: `cargo xtask models`). + +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use printcraft_engine::{Session, export}; +use printcraft_ui_egui::PrintCraftApp; + +/// A one-page PDF that is only a picture of a sentence. +fn scan() -> Vec { + let mut s = Session::new(); + let text = s.create_from_text("t", "Recognize these scanned words").unwrap(); + let id = s.open("text.pdf", None, text, None).unwrap(); + let png = export::Exporter::new(s.get(id).unwrap()).png(0, 150.0).unwrap(); + s.create_from_images(&[("scan.png".into(), png)]).unwrap().to_vec() +} + +#[test] +fn recognize_text_dialog_adds_searchable_text() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PrintCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.ocr_sync = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.recognize")); + h.run_steps(2); + h.get_by_label("Document language"); + h.get_by_label("Output"); + if !printcraft_engine::ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + h.get_by_label("Recognize text").click(); + h.run_steps(3); + assert!(h.state().ocr_run.is_none(), "finished"); + let toast = h.state().toast.clone().unwrap().0; + assert!(toast.starts_with("Recognized ") && toast.contains("on 1 page"), "{toast}"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 48beb026..f6ad4cb4 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6315,11 +6315,11 @@ title = "Recognize text (OCR) in this file" area = "I" tier = "P0" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "shipped" +commands = ["ocr.recognize"] +tools = ["ocr_recognize", "ocr_status"] +evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/ocr.rs::recognize_text_dialog_adds_searchable_text"] +notes = "ocrs engine (MIT/Apache) with CC-BY-SA models fetched by `cargo xtask models`; runs on a worker thread with progress; one undoable step; pages with text are skipped." [[feature]] id = "ocr.recognize-range" @@ -6327,11 +6327,11 @@ title = "Recognize text in the current page or a range" area = "I" tier = "P0" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "shipped" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/automation/tests/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/ocr.rs::recognize_text_dialog_adds_searchable_text"] +notes = "All pages, current page or a range." [[feature]] id = "ocr.recognize-multiple" @@ -6351,11 +6351,11 @@ title = "OCR document language selection" area = "I" tier = "P0" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/ui-egui/tests/ocr.rs::recognize_text_dialog_adds_searchable_text"] +notes = "Language choice exists, but the models read only the Latin alphabet without accents (English)." [[feature]] id = "ocr.searchable-image" @@ -6363,11 +6363,11 @@ title = "Searchable image output (invisible text layer)" area = "I" tier = "P0" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable"] +notes = "Invisible text (Tr 3) fitted to each word box; the image is not deskewed or downsampled (exact mode only)." [[feature]] id = "ocr.searchable-image-exact" @@ -6375,11 +6375,11 @@ title = "Searchable image (exact): leave the image untouched" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "shipped" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable"] +notes = "The image is left untouched." [[feature]] id = "ocr.editable-text-output" diff --git a/xtask/src/assets.rs b/xtask/src/assets.rs index fbe38039..ae99790d 100644 --- a/xtask/src/assets.rs +++ b/xtask/src/assets.rs @@ -270,12 +270,12 @@ pub fn check(root: &Path, m: &Manifest, repo_files: &[String], lock: &BTreeSet<( problems } -/// Download (if needed) and verify every `[[fetched]]` entry into `dir`, with its licence text. -/// Returns the paths of the fetched files. Uses `curl`, present on macOS, Windows 10+ and Linux. -pub fn fetch_all(m: &Manifest, dir: &Path) -> Result> { +/// Download (if needed) and verify every `[[fetched]]` entry of `kind` into `dir`, with its licence +/// text. Returns the paths of the fetched files. Uses `curl`, present on macOS, Windows 10+ and Linux. +pub fn fetch_all(m: &Manifest, dir: &Path, kind: &str) -> Result> { std::fs::create_dir_all(dir)?; let mut out = Vec::new(); - for f in &m.fetched { + for f in m.fetched.iter().filter(|f| f.kind == kind) { let path = dir.join(&f.file); fetch_verified(&f.url, &path, &f.sha256)?; let licence = dir.join(format!("{}.LICENCE.txt", f.file)); @@ -304,6 +304,18 @@ fn fetch_verified(url: &str, path: &Path, sha256: &str) -> Result<()> { Ok(()) } +/// `cargo xtask models [DIR]`: fetch the OCR models (`kind = "model"`) into `assets/models/` (git-ignored), +/// where the app and tests look for them. +pub fn models(args: &[String]) -> Result<()> { + let root = root(); + let m = load(&root)?; + let dir = args.first().map(PathBuf::from).unwrap_or_else(|| root.join("assets/models")); + for p in fetch_all(&m, &dir, "model")? { + println!("models: {}", p.display()); + } + Ok(()) +} + /// ATTRIBUTION.md, generated from the manifest. pub fn render_markdown(m: &Manifest) -> String { let mut s = String::new(); @@ -343,7 +355,7 @@ pub fn render_markdown(m: &Manifest) -> String { ); } s.push_str(&format!( - "\n## Downloaded at build time ({})\n\nFetched by `cargo xtask demo-pdf` into `target/demo-fonts/`, verified by SHA-256 and never committed.\n\n| File | Title | Author | Licence | Source | Used for |\n|---|---|---|---|---|---|\n", + "\n## Downloaded at build time ({})\n\nFonts are fetched by `cargo xtask demo-pdf` into `target/demo-fonts/`, OCR models by `cargo xtask models` into `assets/models/`; each is verified by SHA-256 and never committed.\n\n| File | Title | Author | Licence | Source | Used for |\n|---|---|---|---|---|---|\n", m.fetched.len() )); for f in &m.fetched { diff --git a/xtask/src/demo_pdf/mod.rs b/xtask/src/demo_pdf/mod.rs index 1e44ed77..3354af12 100644 --- a/xtask/src/demo_pdf/mod.rs +++ b/xtask/src/demo_pdf/mod.rs @@ -46,7 +46,7 @@ pub fn run(args: &[String]) -> Result<()> { // Fonts: only the pinned, openly licensed files listed in ATTRIBUTION.toml (AGENTS.md §1). let manifest = crate::assets::load(&root)?; let fonts = root.join("target/demo-fonts"); - crate::assets::fetch_all(&manifest, &fonts)?; + crate::assets::fetch_all(&manifest, &fonts, "font")?; println!("demo-pdf: rendering raster"); let png = raster::mandelbrot_png(640)?; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index dd0f4c08..a1139dbc 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -27,6 +27,7 @@ const COMMANDS: &[(&str, &str, Command)] = &[ ("parity", "Validate parity/acrobat-features.toml against the registry, tools and tests; report progress (--json, --partial)", parity::run), ("text-oracle", "Compare text extraction with pdftotext over corpus/ (word F1; target median ≥ 0.97)", gates::text_oracle), ("screenshots", "Regenerate the README screenshots in docs/images/ and their ATTRIBUTION entries", screenshots::run), + ("models", "Fetch the OCR models (ATTRIBUTION.toml kind = \"model\") into assets/models/, verified by SHA-256", assets::models), ("demo-pdf", "Build dist/demo/printcraft-showcase.pdf (needs Google Chrome or Chromium)", demo_pdf::run), ];