fix(plist): decode formatted XML dictionary keys

docs/native-investigation.md

@@ -53,6 +53,10 @@ complete observation.
   classes are preserved. Select `root`, `offset` and `limit` explicitly.
   UID 0 cannot distinguish conditional nil from ordinary nil. Limits are
   64 MiB of input, 200,000 objects/references and 128 nesting levels.
+  XML dictionary entries decoded as `__proto__` are omitted with an explicit
+  count because REA results cannot represent that key. XML tag whitespace and
+  comments do not change that handling. Other values and the original archive
+  digest are preserved; key interpretation follows the pinned plist decoder.
 - `decode_interface_builder` / `rea decode-interface-builder <app>` reads
   compiled storyboard and nib resources into objects, outlets, actions,
   connections and controller/class names. `verify:interface-builder` compiles

src/domain/propertyListKeys.test.ts

@@ -9,6 +9,10 @@ it.each([
   "<key>__proto__</key>",
   "<key><![CDATA[__proto__]]></key>",
   "<key>&#95;&#x5F;proto__</key>",
+  "<key >__proto__</key >",
+  "<key\n>__proto__</key>",
+  "<key>__proto__<!-- trailing comment --></key>",
+  "<key><![CDATA[__proto__]]><!-- trailing comment --></key>",
 ])("omits and counts a dictionary keyed %s", (key) => {
   const parsed = parseXmlPropertyList(
     plist(
@@ -64,3 +68,56 @@ it("omits own __proto__ entries that JSON decoding creates", () => {
     omittedPrototypeKeys: 2,
   });
 });
+
+it("preserves the pinned decoder's first-child key semantics", () => {
+  const text = plist(
+    "<dict><key>__pro<![CDATA[to__]]></key><string>kept</string><key>__proto__</key><string>omitted</string></dict>",
+  );
+  expect(parseXmlPropertyList(text)).toEqual({
+    value: { __pro: "kept" },
+    omittedPrototypeKeys: 1,
+  });
+});
+
+it("preserves OpenStep dictionary and array autodetection", () => {
+  expect(
+    parseXmlPropertyList(
+      "  { CFBundleExecutable = App; values = (one, two); __rea_prototype_key__ = kept; }",
+    ),
+  ).toEqual({
+    value: {
+      CFBundleExecutable: "App",
+      values: ["one", "two"],
+      __rea_prototype_key__: "kept",
+    },
+    omittedPrototypeKeys: 0,
+  });
+  expect(parseXmlPropertyList("(one, two)")).toEqual({
+    value: ["one", "two"],
+    omittedPrototypeKeys: 0,
+  });
+});
+
+it.each(["\n", "\r\n", "\r", "\u0085", "\u2028", "\u2029"])(
+  "preserves unrelated source values around %j line endings",
+  (newline) => {
+    const text = plist(
+      `<dict>${newline}<key>kept&#13;key</key><string>a&#13;b&#x85;c&#x2028;d</string>${newline}<key${newline}>__proto__<!--tail--></key><string>omitted</string></dict>`,
+    );
+    expect(parseXmlPropertyList(text)).toEqual({
+      value: { "kept\rkey": "a\rb\u0085c\u2028d" },
+      omittedPrototypeKeys: 1,
+    });
+  },
+);
+
+it("locates sparse omitted keys across newline-dense source", () => {
+  const lines = "\n".repeat(64 * 1024);
+  const text = plist(
+    `<dict>${lines}<key >__proto__</key><string>omitted</string><key>nested</key><dict>${lines}<key>__proto__<!--tail--></key><string>omitted</string><key>kept</key><true/></dict></dict>`,
+  );
+  expect(parseXmlPropertyList(text)).toEqual({
+    value: { nested: { kept: true } },
+    omittedPrototypeKeys: 2,
+  });
+});

src/domain/propertyListKeys.ts

@@ -1,3 +1,4 @@
+import { DOMParser, normalizeLineEndings } from "@xmldom/xmldom";
 import { parse } from "plist";
 
 /** The dictionary key REA results cannot carry: validation strips it. */
@@ -46,10 +47,6 @@ export const omitPrototypeKeys = (
   value: unknown,
 ): PropertyListWithoutPrototypeKeys => withoutKey(value, PROTOTYPE_KEY);
 
-// Comments and CDATA text are consumed whole so only element keys match.
-const KEY_TOKEN =
-  /<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<key>([^<]*)<\/key>|<key><!\[CDATA\[([\s\S]*?)\]\]><\/key>/gu;
-
 const XML_ENTITIES: Readonly<Record<string, string>> = {
   amp: "&",
   apos: "'",
@@ -87,18 +84,54 @@ export const decodeXmlText = (text: string): string =>
 export const parseXmlPropertyList = (
   text: string,
 ): PropertyListWithoutPrototypeKeys => {
-  // The decoded text holds every key the decoder can produce, including
-  // entity-encoded spellings of the placeholder, so no source key aliases it.
-  // Decoding comment or CDATA text too can only lengthen the placeholder.
-  const decoded = decodeXmlText(text);
-  let placeholder = "__rea_prototype_key__";
-  while (decoded.includes(placeholder)) placeholder = `_${placeholder}`;
-  const substituted = text.replace(
-    KEY_TOKEN,
-    (token, plain?: string, cdata?: string) =>
-      (plain === undefined ? cdata : decodeXmlText(plain)) === PROTOTYPE_KEY
-        ? `<key>${placeholder}</key>`
-        : token,
+  // Preserve the pinned decoder's existing OpenStep autodetection, also used
+  // by app Info.plist resolution. Only XML needs the key-element substitution.
+  const first = text.trimStart()[0];
+  if (first === "{" || first === "(") return omitPrototypeKeys(parse(text));
+  const { substituted, placeholder } = substitutePrototypeKeys(
+    normalizeLineEndings(text),
   );
   return withoutKey(parse(substituted), placeholder);
 };
+
+/** Locate keys with the same XML parser and first-child semantics as plist. */
+const substitutePrototypeKeys = (text: string) => {
+  // Normalize only the literal line endings the pinned decoder normalizes.
+  // Preserve character references in the source: serializing a DOM would turn
+  // e.g. an encoded carriage return into a literal newline on the second parse.
+  const document = new DOMParser().parseFromString(text, "text/xml");
+  const keys = Array.from(document.getElementsByTagName("key"));
+  const decodedKeys = new Set(keys.map((key) => key.firstChild?.nodeValue));
+  let placeholder = "__rea_prototype_key__";
+  while (decodedKeys.has(placeholder)) placeholder = `_${placeholder}`;
+
+  const omitted = keys
+    .map((key) => key.firstChild)
+    .filter((node) => node?.nodeValue === PROTOTYPE_KEY);
+  if (omitted.length === 0) return { substituted: text, placeholder };
+
+  // xmldom uses one-based locators in document order. Scan forward only to
+  // needed lines instead of retaining an offset for every source newline.
+  let line = 1;
+  let lineStart = 0;
+  const parts: string[] = [];
+  let cursor = 0;
+  for (const node of omitted) {
+    const nodeLine = node?.lineNumber;
+    const column = node?.columnNumber;
+    if (nodeLine === undefined || column === undefined)
+      throw new TypeError("XML plist key has no source location");
+    while (line < nodeLine) {
+      const newline = text.indexOf("\n", lineStart);
+      if (newline === -1)
+        throw new TypeError("XML plist key has an invalid source location");
+      lineStart = newline + 1;
+      line += 1;
+    }
+    const offset = lineStart + column - 1;
+    parts.push(text.slice(cursor, offset), `<![CDATA[${placeholder}]]>`);
+    cursor = offset;
+  }
+  parts.push(text.slice(cursor));
+  return { substituted: parts.join(""), placeholder };
+};

tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts

@@ -0,0 +1,24 @@
+import { expect } from "vitest";
+
+import { parseEvidence } from "../../../src/domain/evidence.js";
+import {
+  expectKeyedArchivePrototypeKeys,
+  keyedArchivePrototypeKeyFixture,
+} from "../../fixtures/keyedArchivePrototypeKeys.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+cliTest(
+  "compiled CLI decodes formatted XML prototype keys",
+  async ({ cli }) => {
+    const fixture = await keyedArchivePrototypeKeyFixture();
+    const output = await cli.run({
+      arguments: ["inspect-keyed-archive", fixture.path, "--json"],
+      environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" },
+    });
+    expect(output.exitCode, output.stderr).toBe(0);
+    expectKeyedArchivePrototypeKeys(
+      parseEvidence(output.json).normalized_result,
+      fixture.digest,
+    );
+  },
+);

tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts

@@ -0,0 +1,49 @@
+import { expect } from "vitest";
+import { z } from "zod";
+
+import { createBinarySession } from "../../../src/composition/binary.js";
+import { parseConfig } from "../../../src/config/parseConfig.js";
+import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js";
+import { silentLogger } from "../../../src/logger.js";
+import { createServer } from "../../../src/server/createServer.js";
+import {
+  expectKeyedArchivePrototypeKeys,
+  keyedArchivePrototypeKeyFixture,
+} from "../../fixtures/keyedArchivePrototypeKeys.js";
+import { mcpTest } from "../../support/mcp/mcpFixture.js";
+
+mcpTest(
+  "MCP retains ordinary evidence with formatted XML prototype keys",
+  async ({ mcp, onTestFinished }) => {
+    const fixture = await keyedArchivePrototypeKeyFixture();
+    const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" });
+    if (!configured.ok) throw configured.error;
+    const session = createBinarySession(
+      configured.value,
+      silentLogger,
+      process.env,
+    );
+    onTestFinished(async () => {
+      await session.close();
+    });
+    const client = await mcp.connect(
+      createServer({ kind: "session", session }),
+    );
+    const opened = await client.callTool({
+      name: "open_binary",
+      arguments: { path: fixture.path },
+    });
+    expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true);
+    const result = await client.callTool({
+      name: "inspect_keyed_archive",
+      arguments: {},
+    });
+    expect(result.isError, JSON.stringify(result.content)).not.toBe(true);
+    expectKeyedArchivePrototypeKeys(
+      z
+        .object({ normalized_result: keyedArchiveResultSchema })
+        .parse(result.structuredContent).normalized_result,
+      fixture.digest,
+    );
+  },
+);

tests/fixtures/keyedArchivePrototypeKeys.ts

@@ -0,0 +1,58 @@
+import { createHash } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { expect } from "vitest";
+
+import { keyedArchiveResultSchema } from "../../src/domain/apple/keyedArchive.js";
+import { createTestTempDirectory } from "./temporaryDirectory.js";
+
+/** Inert XML with producer-equivalent key spellings and unchanged ordinary data. */
+export const keyedArchivePrototypeKeyFixture = async () => {
+  const directory = await createTestTempDirectory("rea-keyed-xml-keys-");
+  const path = join(directory, "archive.plist");
+  const bytes = Buffer.from(`<?xml version="1.0" encoding="UTF-8"?>
+<plist version="1.0"><dict>
+  <key>$archiver</key><string>NSKeyedArchiver</string>
+  <key>$version</key><integer>100000</integer>
+  <key>$objects</key><array><string>$null</string><dict>
+    <key >__proto__</key ><string>omitted outer value</string>
+    <key>kept</key><string>a&#13;b&#x85;c&#x2028;d</string>
+    <key>&#95;&#95;rea_prototype_key__</key><string>ordinary key</string>
+    <key>nested</key><dict>
+      <key><![CDATA[__proto__]]><!-- trailing comment --></key>
+      <string>omitted nested value</string>
+      <key>value</key><integer>7</integer>
+    </dict>
+  </dict></array>
+  <key>$top</key><dict><key>root</key><dict>
+    <key>CF$UID</key><integer>1</integer>
+  </dict></dict>
+</dict></plist>`);
+  await writeFile(path, bytes);
+  return { path, digest: createHash("sha256").update(bytes).digest("hex") };
+};
+
+/** Require original source identity, ordinary values and the exact omission count. */
+export const expectKeyedArchivePrototypeKeys = (
+  value: unknown,
+  digest: string,
+) => {
+  const result = keyedArchiveResultSchema.parse(value);
+  expect(result.archive_sha256).toBe(digest);
+  expect(result.roots).toEqual({ root: { CF$UID: 1 } });
+  expect(result.objects).toHaveLength(2);
+  expect(result.objects[1]).toMatchObject({
+    id: 1,
+    kind: "dictionary",
+    value: {
+      kept: "a\rb\u0085c\u2028d",
+      __rea_prototype_key__: "ordinary key",
+      nested: { value: 7 },
+    },
+  });
+  expect(result.limitations).toContain(
+    "2 dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
+  );
+  expect(JSON.stringify(result)).not.toContain("omitted outer value");
+  expect(JSON.stringify(result)).not.toContain("omitted nested value");
+};