Issue #1185: Protect refuses passwords that encryption would make empty or equal (#1186)

Encryption derives its keys from a prepared password, not the typed one:
AES-256 (R6) applies SASLprep, falls back to the password as given if
SASLprep refuses it, and truncates the UTF-8 to 127 bytes; RC4/AES-128
(R2-R4) use PDFDocEncoding cut or padded to exactly 32 bytes, and the
padding starts with "(". Protection::validate checked only the typed
strings, so a permissions password of U+00AD wrote a file that opens with
owner rights and no password, and an open password "A" with a permissions
password U+FF21 (or RC4/AES-128 passwords "a"x31 and "a"x31 + "(") wrote a
file whose open password lifts the restrictions.

pdfcraft-crypt gains effective_password(algorithm, password), the bytes the
keys are derived from (it and create share the private prepared_password
helper), re-exported from pdfcraft-cos. validate refuses a password whose
effective bytes are those of "", and an open and a permissions password
whose effective bytes are equal. Opening files is unchanged.

Fixes #1185.

crates/automation/tests/automation.rs

@@ -1313,6 +1313,25 @@ fn protecting_with_an_open_password_alone_restricts_nothing() {
     }
 }
 
+/// AES-256 keys on the SASLprep form of a password, which drops soft hyphens: a password made
+/// only of them is no password at all (as the permissions password, it would give everyone owner
+/// rights), so it is refused rather than written.
+#[test]
+fn protecting_with_a_password_encryption_ignores_is_refused() {
+    let dir = workdir("protect-ignored");
+    let mut a = auto(&dir);
+    let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap();
+    let answers: Vec<String> = ["open_password", "permissions_password"]
+        .into_iter()
+        .map(|key| match a.call("doc_protect", &json!({ "doc": doc, key: "\u{AD}" })) {
+            Err(ToolError::Failed(m)) => m,
+            other => format!("{key}: {other:?}"),
+        })
+        .collect();
+    assert!(answers.iter().all(|m| m.contains("soft hyphens")), "{answers:?}");
+    assert_eq!(ok(&mut a, "doc_info", json!({ "doc": doc }))["security"]["protected"], false, "a refused call changes nothing");
+}
+
 #[test]
 fn forms_through_tools() {
     let dir = workdir("forms");

crates/cos/src/lib.rs

@@ -25,7 +25,9 @@ pub use bytes::Bytes;
 pub use document::{Document, Revision, SourceIdentity, XrefEntry};
 pub use object::{Dict, MAX_DECODED, Name, ObjRef, Object, PdfString, Stream};
 pub use parser::{Lexer, parse_indirect};
-pub use pdfcraft_crypt::{Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, pdfdoc_byte, pdfdoc_char};
+pub use pdfcraft_crypt::{
+    Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char,
+};
 pub use writer::{SaveOptions, pdf_date, serialize, write_full, write_incremental};
 
 #[derive(Debug, thiserror::Error, Clone, PartialEq)]

crates/crypt/src/lib.rs

@@ -539,6 +539,32 @@ pub enum Algorithm {
     Aes256,
 }
 
+/// A password as [`create`] prepares it: for R2–R4 its PDFDocEncoding (characters it lacks
+/// dropped), at most 32 bytes; for R6 its SASLprep (RFC 4013: soft hyphens and the like removed,
+/// other spaces made plain, then NFKC; the password as given if SASLprep refuses it), in UTF-8,
+/// at most 127 bytes.
+fn prepared_password(algorithm: Algorithm, password: &str) -> Vec<u8> {
+    match algorithm {
+        Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => legacy_password(password),
+        Algorithm::Aes256 => sasl_password(password),
+    }
+}
+
+/// The bytes new encryption with `algorithm` derives its keys from for `password`: for R2–R4 its
+/// PDFDocEncoding cut or padded to exactly 32 bytes (Algorithm 2); for R6 its SASLprep (the
+/// password as given if SASLprep refuses it), at most 127 bytes of UTF-8. Two passwords with the
+/// same bytes are the same password to the file, and a password with the bytes of `""` opens it
+/// without asking, so callers can refuse either before encrypting. Different bytes can still give
+/// the same key (RC4-40 keeps 5 bytes of a hash). `create` still uses the user password for an
+/// empty R2–R4 owner password.
+pub fn effective_password(algorithm: Algorithm, password: &str) -> Vec<u8> {
+    let prepared = prepared_password(algorithm, password);
+    match algorithm {
+        Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => pad(&prepared).to_vec(),
+        Algorithm::Aes256 => prepared,
+    }
+}
+
 /// Parameters for new encryption.
 #[derive(Clone, Debug)]
 pub struct NewEncryption<'a> {
@@ -573,6 +599,8 @@ pub fn create(params: &NewEncryption, id0: &[u8]) -> Result<SecurityHandler, Cry
     // Bits 7–8 and 13–32 must be 1; bits 1–2 must be 0 (§7.6.4.2).
     let p = (params.permissions | !0xFFF | 0xC0) & !0x3;
     let mut d = EncryptDict { filter: b"Standard".to_vec(), p, encrypt_metadata: params.encrypt_metadata, ..Default::default() };
+    let user = prepared_password(params.algorithm, params.user_password);
+    let owner = prepared_password(params.algorithm, params.owner_password);
     match params.algorithm {
         Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => {
             (d.v, d.r, d.length_bits) = match params.algorithm {
@@ -587,8 +615,7 @@ pub fn create(params: &NewEncryption, id0: &[u8]) -> Result<SecurityHandler, Cry
             } else {
                 d.encrypt_metadata = true; // V < 4 always encrypts metadata
             }
-            let user = legacy_password(params.user_password);
-            let owner = if params.owner_password.is_empty() { user.clone() } else { legacy_password(params.owner_password) };
+            let owner = if params.owner_password.is_empty() { user.clone() } else { owner };
             // Algorithm 3: /O
             let rk = owner_rc4_key(&d, &owner);
             let mut o = rc4(&rk, &pad(&user));
@@ -610,8 +637,6 @@ pub fn create(params: &NewEncryption, id0: &[u8]) -> Result<SecurityHandler, Cry
             d.str_f = b"StdCF".to_vec();
             let key = expand(&params.seed, b"file key", 32);
             let salts = expand(&params.seed, b"salts", 32);
-            let user = sasl_password(params.user_password);
-            let owner = sasl_password(params.owner_password);
             // Algorithm 8: /U, /UE
             let mut u = hash_r6(6, &user, &salts[0..8], &[]);
             u.extend_from_slice(&salts[0..16]);

crates/crypt/src/tests.rs

@@ -145,6 +145,53 @@ fn r6_passwords_are_saslprepped() {
     assert!(SecurityHandler::open(d, ID, Some("SaSL-prep")).is_err());
 }
 
+#[test]
+fn effective_passwords_are_the_bytes_the_keys_are_made_from() {
+    // AES-256: SASLprep drops soft hyphens, makes other spaces plain and folds compatibility
+    // characters (NFKC); at most 127 bytes of UTF-8.
+    let r6 = |pw: &str| effective_password(Algorithm::Aes256, pw);
+    assert_eq!(r6("\u{AD}"), r6(""), "a soft hyphen alone is no password");
+    assert_eq!(r6("\u{FF21}"), r6("A"), "fullwidth A");
+    assert_eq!(r6("\u{FB01}"), r6("fi"), "the fi ligature");
+    assert_eq!(r6("a\u{3000}b"), b"a b", "an ideographic space");
+    assert_ne!(r6("A"), r6("a"), "case matters");
+    assert_eq!(r6(&"é".repeat(100)).len(), 127);
+    // RC4 and AES-128: PDFDocEncoding, padded or cut to exactly 32 bytes.
+    let (a31, a32) = ("a".repeat(31), "a".repeat(32));
+    for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] {
+        assert_eq!(effective_password(alg, &format!("{a32}1")), a32.as_bytes(), "{alg:?}");
+        assert_eq!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("1{a32}2")), "{alg:?}");
+        assert_ne!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("2{a32}")), "{alg:?}");
+        assert_eq!(effective_password(alg, &a31), effective_password(alg, &format!("{a31}(")), "{alg:?}: the padding starts with '('");
+        assert_eq!(effective_password(alg, ""), PADDING, "{alg:?}");
+        assert_eq!(effective_password(alg, "ılık€")[..5], [0x9A, b'l', 0x9A, b'k', 0xA0], "{alg:?}");
+    }
+}
+
+#[test]
+fn passwords_with_the_same_effective_bytes_are_one_password() {
+    // A file protected with "\u{FF21}" (fullwidth A) opens with "A".
+    let d = new(Algorithm::Aes256, "\u{FF21}", "owner", 0b0100).dict().clone();
+    assert_eq!(SecurityHandler::open(d.clone(), ID, Some("A")).unwrap().auth(), Auth::User);
+    assert_eq!(SecurityHandler::open(d, ID, Some("a")).unwrap_err(), CryptError::WrongPassword);
+    // A soft hyphen is no open password at all.
+    let d = new(Algorithm::Aes256, "\u{AD}", "owner", 0b0100).dict().clone();
+    assert_eq!(SecurityHandler::open(d, ID, None).unwrap().auth(), Auth::User);
+    // Open and permissions passwords with the same bytes: the open password gives owner rights.
+    let d = new(Algorithm::Aes256, "\u{FB01}", "fi", 0b0100).dict().clone();
+    assert_eq!(effective_password(Algorithm::Aes256, "\u{FB01}"), effective_password(Algorithm::Aes256, "fi"));
+    assert_eq!(SecurityHandler::open(d, ID, Some("\u{FB01}")).unwrap().auth(), Auth::Owner);
+    let a31 = "a".repeat(31);
+    let pairs = [(format!("{a31}a1"), format!("{a31}a2")), (a31.clone(), format!("{a31}("))];
+    for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] {
+        for (open, perms) in &pairs {
+            assert_eq!(effective_password(alg, open), effective_password(alg, perms), "{alg:?}");
+            let d = new(alg, open, perms, 0b0100).dict().clone();
+            assert_eq!(SecurityHandler::open(d, ID, Some(open)).unwrap().auth(), Auth::Owner, "{alg:?} {open:?}");
+        }
+    }
+}
+
 #[test]
 fn unsupported_handlers_are_reported() {
     let mut d = new(Algorithm::Aes128, "", "o", -1).dict().clone();

crates/engine/src/lib.rs

@@ -927,6 +927,19 @@ impl Protection {
         {
             return bad("this compatibility level supports only plain ASCII passwords; use AES-256 (Acrobat X and later)");
         }
+        // Encryption keys on a password as `effective_password` gives it (AES-256: SASLprep,
+        // which drops soft hyphens and folds compatibility forms; older levels: the first 32
+        // bytes, padded). One that comes out as the empty password opens the file without asking,
+        // and an open password that comes out as the permissions password lifts the restrictions.
+        let effective = |p: &str| pdfcraft_cos::effective_password(self.algorithm, p);
+        let (open, permissions) = (self.open_password.as_deref().map(effective), self.permissions_password.as_deref().map(effective));
+        let empty = effective("");
+        if open.iter().chain(&permissions).any(|p| *p == empty) {
+            return bad("passwords can't be empty or made only of characters that passwords ignore, such as soft hyphens");
+        }
+        if open.is_some() && open == permissions {
+            return bad("the open password and the permissions password must be different (PDF readers treat these two as the same password)");
+        }
         Ok(())
     }
 }

crates/engine/src/tests.rs

@@ -1145,6 +1145,67 @@ fn protection_is_validated_undoable_and_never_logged() {
     assert_eq!(page_texts(&s, id), ["Page 1"]);
 }
 
+/// Protect a fresh document with `p` as the app does: the edit's result and, when it is
+/// accepted, how the saved file authenticates with no password, the open password and the
+/// permissions password (`None`: it doesn't open).
+fn protect_and_reopen(p: &Protection) -> (Result<(), EditError>, Option<[Option<pdfcraft_cos::Auth>; 3]>) {
+    let (mut s, id) = session_with(1);
+    let result = s.apply(id, Edit::Protect(p.clone()));
+    let reopened = result.is_ok().then(|| {
+        let saved = s.save_bytes(id).unwrap();
+        let auth = |pw: Option<&str>| pdfcraft_cos::Document::open_with_password(saved.clone(), pw).ok()?.security().map(|h| h.auth());
+        [auth(None), auth(p.open_password.as_deref()), auth(p.permissions_password.as_deref())]
+    });
+    (result, reopened)
+}
+
+#[test]
+fn passwords_that_encryption_would_make_empty_or_equal_are_refused() {
+    use pdfcraft_cos::Algorithm::{Aes128, Rc4_128};
+    use pdfcraft_cos::Auth::{Owner, User};
+    let empty = "passwords can't be empty or made only of characters that passwords ignore, such as soft hyphens";
+    let same = "the open password and the permissions password must be different (PDF readers treat these two as the same password)";
+    // AES-256 keys on the SASLprep form of a password (soft hyphens dropped, then NFKC), RC4 and
+    // AES-128 on its first 32 bytes. Written as typed, these would make a file that opens
+    // without a password, or whose open password lifts the restrictions.
+    let (a31, a32) = ("a".repeat(31), "a".repeat(32));
+    let (long1, long2) = (format!("{a32}1"), format!("{a32}2"));
+    // The 32-byte padding starts with "(": 31 characters and the same followed by "(" are one key.
+    let padded = format!("{a31}(");
+    let cases = [
+        ("soft hyphen open password", protection(Some("\u{AD}"), None), empty),
+        ("soft hyphen permissions password", protection(None, Some("\u{AD}")), empty),
+        ("soft hyphen permissions password beside an open one", protection(Some("0"), Some("\u{AD}")), empty),
+        ("fullwidth A, then A", protection(Some("\u{FF21}"), Some("A")), same),
+        ("A, then fullwidth A", protection(Some("A"), Some("\u{FF21}")), same),
+        ("fi ligature, then fi", protection(Some("\u{FB01}"), Some("fi")), same),
+        ("fi, then fi ligature", protection(Some("fi"), Some("\u{FB01}")), same),
+        ("RC4-128, same first 32 bytes", Protection { algorithm: Rc4_128, ..protection(Some(&long1), Some(&long2)) }, same),
+        ("AES-128, same first 32 bytes", Protection { algorithm: Aes128, ..protection(Some(&long1), Some(&long2)) }, same),
+        ("RC4-128, same once padded", Protection { algorithm: Rc4_128, ..protection(Some(&a31), Some(&padded)) }, same),
+        ("AES-128, same once padded", Protection { algorithm: Aes128, ..protection(Some(&a31), Some(&padded)) }, same),
+    ];
+    let got: Vec<_> = cases.iter().map(|(case, p, _)| (*case, protect_and_reopen(p))).collect();
+    let want: Vec<_> = cases.iter().map(|(case, _, message)| (*case, (Err(EditError::Protection(message.to_string())), None))).collect();
+    assert_eq!(got, want, "each is refused, so no file is written");
+    // Passwords that are empty or equal as typed keep their messages.
+    let (mut s, id) = session_with(1);
+    assert_eq!(s.apply(id, Edit::Protect(protection(Some(""), None))), Err(EditError::Protection("passwords can't be empty".into())));
+    assert_eq!(
+        s.apply(id, Edit::Protect(protection(Some("same"), Some("same")))),
+        Err(EditError::Protection("the open password and the permissions password must be different".into()))
+    );
+    // Passwords that stay different are fine: only the permissions password gives owner rights.
+    let distinct = [
+        protection(Some("A"), Some("\u{FF21}\u{FF22}")),
+        Protection { algorithm: Rc4_128, ..protection(Some(&format!("1{a32}")), Some(&format!("2{a32}"))) },
+        Protection { algorithm: Aes128, ..protection(Some(&format!("1{a32}")), Some(&format!("2{a32}"))) },
+    ];
+    for p in &distinct {
+        assert_eq!(protect_and_reopen(p), (Ok(()), Some([None, Some(User), Some(Owner)])), "{:?}", p.algorithm);
+    }
+}
+
 /// A one-page form: a text field and a check box (with appearance states).
 fn form_fixture() -> Vec<u8> {
     let objs = [