diff --git a/ROADMAP.md b/ROADMAP.md index 55a20074..40532238 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -408,3 +408,5 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-10-10 | major | Restructured to the progress-docs standard (stage, two numbers, dimensions, features, languages, upcoming, log); full re-measure; assessment, gaps and milestones moved to `docs/`; log sorted newest first and a stray merged entry dated | | 2026-10-07 | minor | 806 tracked features, 51.0% shipped, effort-weighted ≈ 30–35%, 600–1,100 h | | 2026-10-05 | major | Honest assessment and "Where we're lacking" added | + +- **2026-10-12 (M8):** Redaction cleans indirect and form structure references, preserving surviving form tags. diff --git a/crates/redact/README.md b/crates/redact/README.md index 8c494b15..c156ad0f 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -33,6 +33,9 @@ A verification pass re-reads each redacted page and fails the operation if any g image is still under an area. Content that can't be decoded makes the operation fail rather than leave it unredacted. +Structure cleanup resolves named MCIDs and indirect content references. Form content +references are updated to the cleaned copy; unchanged sibling descriptions remain. + ## Remove hidden information / Sanitize ```rust diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index cd496c33..d7088083 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -17,13 +17,13 @@ //! In verify mode nothing changes; glyphs and inline images that still overlap a region are //! counted. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; use pdfcraft_content::{Matrix, Op, Pieces, contains, num, overlaps, serialize_ops, string}; use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; -use crate::{Report, image}; +use crate::{Report, image, tags}; use pdfcraft_fonts::pdf::Metrics; const MAX_DEPTH: usize = 12; @@ -67,6 +67,7 @@ pub(crate) struct Output { /// Apply mode met vertical-writing text (which it cannot remove) while a redaction region was /// in force: the operation must fail closed rather than leave that text under the mark. pub vertical: bool, + pub tag_changes: Vec<(ObjRef, ObjRef, HashSet, HashSet)>, } pub(crate) struct Scope<'a> { @@ -670,11 +671,12 @@ fn xobject( let res = own.clone().unwrap_or_else(|| resources.clone()); let saved_fonts = std::mem::take(&mut scope.fonts); scope.depth += 1; - let inner = process(doc, scope, &[data], &res, fctm); + let inner = process(doc, scope, std::slice::from_ref(&data), &res, fctm); scope.depth -= 1; scope.fonts = saved_fonts; out.residue += inner.residue; out.vertical |= inner.vertical; + out.tag_changes.extend(inner.tag_changes); if scope.mode == Mode::Verify { return None; } @@ -687,7 +689,7 @@ fn xobject( // Rebuild the copy's /Resources whenever inner content gained or lost an XObject, so a // `Do` the copy no longer makes stops keeping the original image or form reachable. if !inner.xobjects.is_empty() || !inner.replaced.is_empty() { - let mut res = res; + let mut res = res.clone(); let mut xo = res_dict(doc, &res, b"XObject"); if let Some(d) = &new_data { prune(&mut xo, &inner.replaced, std::slice::from_ref(d)); @@ -698,11 +700,13 @@ fn xobject( res.set(b"XObject".to_vec(), Object::Dict(xo)); dict.set(b"Resources".to_vec(), Object::Dict(res)); } + let (changed, empty) = tags::touched(doc, &res, &data, new_data.as_deref().unwrap_or(&data)); let stream = match new_data { Some(d) => Stream::flate(dict, &d), None => Stream { dict, raw: s.raw.clone() }, }; let nr = doc.add(Object::Stream(stream)); + out.tag_changes.push((r, nr, changed, empty)); let n = fresh_name(scope, xobjects); xobjects.set(n.clone(), Object::Ref(nr)); out.xobjects.push((n.clone(), nr)); diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index e9310165..916d605e 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -333,7 +333,7 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result) -> Result HashMap, bool)> { +fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> HashMap, bool)> { let mut out: HashMap, bool)> = HashMap::new(); let mut stack: Vec> = Vec::new(); for op in pdfcraft_content::parse(data).ops { match op.op.as_slice() { b"BDC" => { let id = match op.operands.get(1) { - Some(Object::Dict(d)) => d.get(b"MCID").and_then(Object::as_int), + Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { + let props = doc.resolve(p); + let prop = doc.resolve(props.as_dict()?.get(n)?); + prop.as_dict()?.int(b"MCID") + }), + Some(p) => doc.resolve(p).as_dict().and_then(|d| d.int(b"MCID")), _ => None, }; if let Some(id) = id { @@ -56,8 +61,8 @@ fn by_mcid(data: &[u8]) -> HashMap, bool)> { } /// The MCIDs whose content changed between `before` and `after`, and those left empty. -pub(crate) fn touched(before: &[u8], after: &[u8]) -> (HashSet, HashSet) { - let (b, a) = (by_mcid(before), by_mcid(after)); +pub(crate) fn touched(doc: &Document, resources: &Dict, before: &[u8], after: &[u8]) -> (HashSet, HashSet) { + let (b, a) = (by_mcid(doc, resources, before), by_mcid(doc, resources, after)); let mut changed = HashSet::new(); let mut empty = HashSet::new(); for (id, (ops, _)) in &b { @@ -79,20 +84,27 @@ pub(crate) fn touched(before: &[u8], after: &[u8]) -> (HashSet, HashSet, target: ObjRef) -> Option { - match o { - Object::Int(n) if page == Some(target) => Some(*n), +fn mcid_of(doc: &Document, o: &Object, page: Option, target: ObjRef, stream: Option<(ObjRef, ObjRef)>) -> Option { + match &*doc.resolve(o) { + Object::Int(n) if page == Some(target) && stream.is_none() => Some(*n), Object::Dict(m) if m.name(b"Type") == Some(b"MCR") => { let pg = m.get(b"Pg").and_then(Object::as_ref).or(page); - (pg == Some(target)).then(|| m.get(b"MCID").and_then(Object::as_int)).flatten() + let stm = m.get(b"Stm").and_then(Object::as_ref); + (pg == Some(target) && stm == stream.map(|s| s.0)).then(|| m.int(b"MCID")).flatten() } _ => None, } } /// Clean the elements owning `changed` marked content on `page`; returns how many changed. -pub(crate) fn clean(doc: &mut Document, page: ObjRef, changed: &HashSet, empty: &HashSet) -> Result { - if changed.is_empty() { +pub(crate) fn clean( + doc: &mut Document, + page: ObjRef, + stream: Option<(ObjRef, ObjRef)>, + changed: &HashSet, + empty: &HashSet, +) -> Result { + if changed.is_empty() && stream.is_none() { return Ok(0); } let Some(root) = doc @@ -105,7 +117,7 @@ pub(crate) fn clean(doc: &mut Document, page: ObjRef, changed: &HashSet, em // Indirect elements with the page they inherit. let mut todo: Vec<(Object, Option)> = root.get(b"K").map(|k| vec![(k.clone(), None)]).unwrap_or_default(); let mut seen = HashSet::new(); - let mut hits: Vec<(ObjRef, Dict, Option)> = Vec::new(); + let mut hits: Vec<(ObjRef, Dict, Option, bool)> = Vec::new(); while let Some((k, pg)) = todo.pop() { match k { Object::Array(a) => todo.extend(a.into_iter().map(|x| (x, pg))), @@ -113,18 +125,27 @@ pub(crate) fn clean(doc: &mut Document, page: ObjRef, changed: &HashSet, em if !seen.insert(r) || seen.len() > 2_000_000 { continue; } - let Some(d) = doc.get(r).as_dict().cloned() else { continue }; + let object = doc.get(r); + if let Object::Array(a) = &*object { + todo.extend(a.iter().cloned().map(|x| (x, pg))); + continue; + } + let Some(d) = object.as_dict().cloned() else { continue }; if !d.contains(b"S") { continue; } let pg = d.get(b"Pg").and_then(Object::as_ref).or(pg); - let kids: Vec = match d.get(b"K") { - Some(Object::Array(a)) => a.clone(), - Some(o) => vec![o.clone()], + let kids: Vec = match d.get(b"K").map(|k| doc.resolve(k)) { + Some(o) => match &*o { + Object::Array(a) => a.clone(), + _ => d.get(b"K").cloned().into_iter().collect(), + }, None => Vec::new(), }; - if kids.iter().any(|x| mcid_of(x, pg, page).is_some_and(|m| changed.contains(&m))) { - hits.push((r, d.clone(), pg)); + let sensitive = kids.iter().any(|x| mcid_of(doc, x, pg, page, stream).is_some_and(|m| changed.contains(&m))); + let remap = stream.is_some() && kids.iter().any(|x| mcid_of(doc, x, pg, page, stream).is_some()); + if sensitive || remap { + hits.push((r, d.clone(), pg, sensitive)); } todo.extend(kids.into_iter().map(|x| (x, pg))); } @@ -132,25 +153,52 @@ pub(crate) fn clean(doc: &mut Document, page: ObjRef, changed: &HashSet, em } } let mut n = 0; - for (r, d, pg) in hits { - let kids = d.get(b"K").cloned(); - doc.update_dict(r, |e| { - for key in [&b"Alt"[..], b"ActualText", b"E"] { - e.remove(key); + for (r, d, pg, sensitive) in hits { + let kids = d.get(b"K").map(|k| doc.resolve(k)); + let array = kids.as_ref().is_some_and(|k| matches!(&**k, Object::Array(_))); + let list = match kids.as_deref() { + Some(Object::Array(a)) => a.clone(), + Some(_) => d.get(b"K").cloned().into_iter().collect(), + None => Vec::new(), + }; + let mut keep = Vec::new(); + for mut kid in list { + let mcid = mcid_of(doc, &kid, pg, page, stream); + if mcid.is_some_and(|m| empty.contains(&m)) { + continue; } - // Marked content that is now empty no longer belongs to the element. - match kids { - Some(Object::Array(a)) => { - let keep: Vec = a.into_iter().filter(|x| !mcid_of(x, pg, page).is_some_and(|m| empty.contains(&m))).collect(); - e.set(b"K".to_vec(), Object::Array(keep)); + if let Some((original, replacement)) = stream.filter(|_| mcid.is_some()) { + if let Some(reference) = kid.as_ref() { + doc.update_dict(reference, |m| { + m.set(b"Stm".to_vec(), Object::Ref(replacement)); + if m.get(b"StmOwn").and_then(Object::as_ref) == Some(original) { + m.set(b"StmOwn".to_vec(), Object::Ref(replacement)); + } + })?; + } else if let Object::Dict(m) = &mut kid { + m.set(b"Stm".to_vec(), Object::Ref(replacement)); + if m.get(b"StmOwn").and_then(Object::as_ref) == Some(original) { + m.set(b"StmOwn".to_vec(), Object::Ref(replacement)); + } } - Some(o) if mcid_of(&o, pg, page).is_some_and(|m| empty.contains(&m)) => { - e.remove(b"K"); + } + keep.push(kid); + } + doc.update_dict(r, |e| { + if sensitive { + for key in [&b"Alt"[..], b"ActualText", b"E"] { + e.remove(key); } - _ => {} + } + if array { + e.set(b"K".to_vec(), Object::Array(keep)); + } else if let Some(k) = keep.into_iter().next() { + e.set(b"K".to_vec(), k); + } else { + e.remove(b"K"); } })?; - n += 1; + n += usize::from(sensitive && doc.get(r).as_dict() != Some(&d)); } Ok(n) } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index bd1fa5a1..fe17d14c 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -631,3 +631,71 @@ fn tags_lose_what_redaction_removed() { assert_eq!(public.get(b"K").and_then(Object::as_int), Some(1)); assert!(public.get(b"ActualText").is_some(), "untouched content keeps its tags"); } + +fn assert_no_secret(doc: &Document, secret: &[u8]) { + let saved = reopen(doc); + for (r, object) in saved.scan_objects() { + let mut bytes = Vec::new(); + pdfcraft_cos::serialize(&object, &mut bytes); + assert!(!bytes.windows(secret.len()).any(|w| w == secret), "secret remains in object {r:?}"); + if let Object::Stream(s) = &*object { + let data = s.decoded().unwrap(); + assert!(!data.windows(secret.len()).any(|w| w == secret), "secret remains in stream {r:?}"); + } + } +} + +#[test] +fn named_mcids_and_indirect_content_references_lose_descriptions() { + let mut doc = one_page( + b"/P /P0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC /P <> BDC BT /F1 10 Tf 10 100 Td (Public) Tj ET EMC", + "/Properties << /P0 7 0 R >>", + vec![ + b"<< /MCID 0 >>".to_vec(), + b"<< /Type /StructTreeRoot /K 9 0 R >>".to_vec(), + b"<< /S /Document /P 8 0 R /K [10 0 R 11 0 R] >>".to_vec(), + b"<< /S /P /P 9 0 R /Pg 3 0 R /ActualText (LEAK_TAG) /K 12 0 R >>".to_vec(), + b"<< /S /P /P 9 0 R /Pg 3 0 R /ActualText (Public) /K 1 >>".to_vec(), + b"<< /Type /MCR /Pg 3 0 R /MCID 0 >>".to_vec(), + ], + ); + doc.update_dict(doc.root().unwrap(), |d| d.set(b"StructTreeRoot".to_vec(), Object::Ref(ObjRef::new(8, 0)))).unwrap(); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().tags, 1); + assert_no_secret(&doc, b"LEAK_"); + assert!(!doc.get(ObjRef::new(10, 0)).as_dict().unwrap().contains(b"K")); + assert_eq!(doc.get(ObjRef::new(11, 0)).as_dict().unwrap().get(b"ActualText").unwrap().as_string().unwrap().to_text(), "Public"); +} + +#[test] +fn form_tags_are_cleaned_and_surviving_references_point_at_the_copy() { + let mut doc = one_page( + b"/Fm Do", + "/XObject << /Fm 7 0 R >>", + vec![ + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /StructParents 0 /Resources << /Font << /F1 5 0 R >> /Properties << /P0 << /MCID 0 >> >> >>", + b"/P /P0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC /P <> BDC BT /F1 10 Tf 10 100 Td (Public) Tj ET EMC", + ), + b"<< /Type /StructTreeRoot /K 9 0 R /ParentTree 14 0 R >>".to_vec(), + b"<< /S /Document /P 8 0 R /K [10 0 R 11 0 R] >>".to_vec(), + b"<< /S /P /P 9 0 R /Pg 3 0 R /ActualText (LEAK_FORM_TAG) /K 12 0 R >>".to_vec(), + b"<< /S /P /P 9 0 R /Pg 3 0 R /ActualText (Public) /K 13 0 R >>".to_vec(), + b"<< /Type /MCR /Pg 3 0 R /Stm 7 0 R /MCID 0 >>".to_vec(), + b"<< /Type /MCR /Pg 3 0 R /Stm 7 0 R /StmOwn 7 0 R /MCID 1 >>".to_vec(), + b"<< /Nums [0 [10 0 R 11 0 R]] >>".to_vec(), + ], + ); + doc.update_dict(doc.root().unwrap(), |d| d.set(b"StructTreeRoot".to_vec(), Object::Ref(ObjRef::new(8, 0)))).unwrap(); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, None).unwrap(); + assert_no_secret(&doc, b"LEAK_"); + assert_no_secret(&doc, b"SECRET"); + assert!(!doc.get(ObjRef::new(10, 0)).as_dict().unwrap().contains(b"K")); + let reference = doc.get(ObjRef::new(13, 0)).as_dict().unwrap().get(b"Stm").unwrap().as_ref().unwrap(); + assert_ne!(reference, ObjRef::new(7, 0)); + assert_eq!(doc.get(ObjRef::new(13, 0)).as_dict().unwrap().get(b"StmOwn").unwrap().as_ref(), Some(reference)); + let Object::Stream(s) = &*doc.get(reference) else { panic!() }; + assert!(String::from_utf8(s.decoded().unwrap()).unwrap().contains("Public")); + assert!(doc.get(ObjRef::new(11, 0)).as_dict().unwrap().contains(b"ActualText")); +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 6170fc70..56b6ecd3 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -91,7 +91,7 @@ total matches the parity estimate. | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Existing simple field-font encodings and Differences are respected when generating appearances; an unencodable value uses a separate Helvetica appearance font without changing the source font. Synthetic reopen regressions cover punctuation, subset fallback and multiline text. Exact font metrics/layout and broader Unicode fallback remain incomplete. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Tab and Shift-Tab wrapping to the same field or another widget of that field refresh the newly opened editor from the accepted value before Save. Synthetic UI regressions cover both directions, same-frame Save, validation refusal and undo/redo. | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | -| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | +| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA |