diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 753ebdb3e..5d97e7c6f 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -71,8 +71,12 @@ complete observation. Before decoding, it also bounds the serialized structure: expanded binary plist references, NIB table records, XML elements and text, plus their projected representations must fit the 256 MiB aggregate decode budget. - This is a format-derived representation estimate, not a process-wide heap - ceiling. + NIB string values, data, property names and class names are charged at each + projected JSON occurrence, including when the archive reuses one source + value. Binary plist data projection also accounts for the temporary byte + copy made by the pinned decoder projection. These are format-derived + representation reservations, not exact heap measurements or a process-wide + heap ceiling. Archives beyond either aggregate budget are omitted with their paths and an explicitly partial `archive_decode` facet; previously decoded documents and their identities remain available. diff --git a/docs/testing.md b/docs/testing.md index 4610c4fa0..bbfc70a3c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -526,8 +526,9 @@ runtime or PC-98 device execution. ## Apple Interface Builder archives `npm run verify:interface-builder` compiles the source-owned AppKit XIB into a -real `.nib` with Xcode `ibtool`, wraps it in a temporary app bundle, and checks -the decoded view hierarchy, outlet, action, evidence coverage, and truncation +real `.nib` with Xcode `ibtool`, wraps it in a temporary app bundle, and runs +the compiled CLI and stdio MCP server with production providers. It compares +their decoded results and checks the view hierarchy, outlet, action, evidence coverage, and truncation status. Storyboard compilation additionally requires an installed iOS platform. Keep the provider-specific acceptance path independent from optional diff --git a/scripts/verify/apple/interface-builder.mjs b/scripts/verify/apple/interface-builder.mjs index fb3346f73..a27bf656b 100644 --- a/scripts/verify/apple/interface-builder.mjs +++ b/scripts/verify/apple/interface-builder.mjs @@ -8,8 +8,11 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { parseBinaryTarget } from "../../../dist/application/BinaryTargetResolver.js"; -import { ArtifactProvider } from "../../../dist/artifacts/ArtifactProvider.js"; +import { + artifactCliEvidence, + artifactMcpResult, + withArtifactMcp, +} from "../../lib/artifact-e2e.mjs"; const exec = promisify(execFile); const fixtureRoot = await mkdtemp(join(tmpdir(), "rea-interface-builder-")); @@ -41,14 +44,19 @@ try { ]); await exec("cc", [join(sourceRoot, "fixture.c"), "-o", executablePath]); - const target = await parseBinaryTarget(appPath); - if (!target.ok) throw target.error; - const execution = await new ArtifactProvider() - .createClient(target.value) - .execute("decode_interface_builder", {}, {}); - if (!execution.ok) throw execution.error; - - const decoded = execution.value.result; + const evidence = await artifactCliEvidence( + "decode-interface-builder", + appPath, + ); + const decoded = evidence.normalized_result; + await withArtifactMcp(appPath, async (client) => { + const mcp = await artifactMcpResult(client, "decode_interface_builder"); + assert.deepEqual( + mcp, + decoded, + "CLI and MCP compiled archive results differ", + ); + }); assert.equal(decoded.documents.length, 1); assert.equal( decoded.documents[0]?.relative_path, @@ -87,7 +95,9 @@ try { report = { ok: true, fixture: "compiled-appkit-nib", - provider: execution.value.provider, + provider: evidence.provider, + public_cli: true, + public_stdio_mcp: true, target_sha256: decoded.target_sha256, archive_sha256: decoded.documents[0].archive_sha256, objects: decoded.documents[0].object_count, diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index f8358d3a8..f47c5c56c 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -278,6 +278,14 @@ const projectNibArchive = ( maxHierarchyNodes: number, ) => { const byId = new Map(archive.objects.map((object) => [object.id, object])); + const normalizedClassNames = new Map(); + const normalizeClassName = (value: string): string => { + const known = normalizedClassNames.get(value); + if (known !== undefined) return known; + const normalized = value.replace(/\0+$/u, ""); + normalizedClassNames.set(value, normalized); + return normalized; + }; const decodedStrings = new Map(); const decodedString = ( objectId: number, @@ -303,14 +311,6 @@ const projectNibArchive = ( decodedStrings.set(objectId, value); return value; }; - const reserveStringOccurrence = (value: string | null): string | null => { - if (value !== null) - budget.reserve( - 64 + value.length * 6, - "NIBArchive repeated string projection exceeds the aggregate Interface Builder decode budget", - ); - return value; - }; let projectedFields = 0; for (const object of archive.objects) projectedFields += Object.keys(object.values).length; @@ -325,13 +325,21 @@ const projectNibArchive = ( const dereference = (value: JsonValue | undefined): JsonValue | undefined => { if (typeof value !== "object" || value === null || Array.isArray(value)) return value; + const encodedData = value.$nib_data_base64; + if (typeof encodedData === "string") { + budget.reserveBase64JsonStringOccurrence(encodedData); + return value; + } const ref = value.$nib_object_ref; if (typeof ref !== "number") return value; const target = byId.get(ref); if (target === undefined) return undefined; - if (target.class_name.replace(/\0+$/u, "") === "NSString") { + if (normalizeClassName(target.class_name) === "NSString") { const string = decodedString(ref, target); - if (string !== null) return reserveStringOccurrence(string); + if (string !== null) { + budget.reserveJsonStringOccurrence(string); + return string; + } } return { objectID: String(ref) }; }; @@ -341,18 +349,31 @@ const projectNibArchive = ( const ref = value.$nib_object_ref; if (typeof ref !== "number") return null; const target = byId.get(ref); - if (target?.class_name.replace(/\0+$/u, "") !== "NSString") return null; - return reserveStringOccurrence(decodedString(ref, target)); + if ( + target === undefined || + normalizeClassName(target.class_name) !== "NSString" + ) + return null; + const string = decodedString(ref, target); + if (string === null) return null; + budget.reserveJsonStringOccurrence(string); + return string; }; + const runtimeClasses = new Map(); const runtimeClass = (objectId: number): string | null => { + if (runtimeClasses.has(objectId)) + return runtimeClasses.get(objectId) ?? null; const object = byId.get(objectId); if (object === undefined) return null; - const storedClass = object.class_name.replace(/\0+$/u, ""); - if (storedClass === "NSClassSwapper") - return referencedString(object.values.NSClassName) ?? storedClass; - return storedClass === "UIClassSwapper" - ? (referencedString(object.values.UIClassName) ?? storedClass) - : storedClass; + const storedClass = normalizeClassName(object.class_name); + const resolvedClass = + storedClass === "NSClassSwapper" + ? (referencedString(object.values.NSClassName) ?? storedClass) + : storedClass === "UIClassSwapper" + ? (referencedString(object.values.UIClassName) ?? storedClass) + : storedClass; + runtimeClasses.set(objectId, resolvedClass); + return resolvedClass; }; const objects = Object.fromEntries( archive.objects @@ -364,19 +385,22 @@ const projectNibArchive = ( }) .map((object) => { const className = - runtimeClass(object.id) ?? object.class_name.replace(/\0+$/u, ""); - const strings = Object.entries(object.values).flatMap( - ([key, value]) => { - const resolved = dereference(value); - return typeof resolved === "string" ? [[key, resolved]] : []; - }, - ); - const name = - strings.find(([key]) => - /(?:title|label|identifier|accessibility|name|contents)/iu.test( - key ?? "", - ), - )?.[1] ?? className; + runtimeClass(object.id) ?? normalizeClassName(object.class_name); + budget.reserveJsonStringOccurrence(className); + let name = className; + for (const [key, value] of Object.entries(object.values)) { + if ( + !/(?:title|label|identifier|accessibility|name|contents)/iu.test( + key, + ) + ) + continue; + const resolved = dereference(value); + if (typeof resolved !== "string") continue; + name = resolved; + break; + } + budget.reserveJsonStringOccurrence(name); return [ String(object.id), { @@ -384,10 +408,10 @@ const projectNibArchive = ( label: name, objectID: String(object.id), nibValues: Object.fromEntries( - Object.entries(object.values).map(([key, value]) => [ - key, - dereference(value) ?? null, - ]), + Object.entries(object.values).map(([key, value]) => { + budget.reserveJsonPropertyNameOccurrence(key); + return [key, dereference(value) ?? null]; + }), ), }, ]; @@ -403,7 +427,7 @@ const projectNibArchive = ( : null; }; for (const object of archive.objects) { - const className = object.class_name.replace(/\0+$/u, ""); + const className = normalizeClassName(object.class_name); // AppKit connectors use NS-prefixed keys; UIKit's runtime outlet and // event connections use UI-prefixed keys with the same roles. const uiKit = /^UIRuntime\w*Connection$/u.test(className); @@ -420,6 +444,7 @@ const projectNibArchive = ( : className.includes("Control") || className.includes("Event") ? "action" : className; + budget.reserveJsonStringOccurrence(type); const eventMask = values.UIEventMask; // The source is the outlet owner or the sending control; the destination // is the outlet value or the action target, and a nil target is the @@ -485,7 +510,7 @@ const projectNibArchive = ( return { objectID: String(objectId), children }; }; const ibData = archive.objects.find( - ({ class_name }) => class_name.replace(/\0+$/u, "") === "NSIBObjectData", + ({ class_name }) => normalizeClassName(class_name) === "NSIBObjectData", ); const rootId = reference(ibData?.values.NSRoot); const hierarchyRoots = rootId === null ? [] : [rootId]; @@ -512,7 +537,11 @@ const projectNibArchive = ( "com.apple.ibtool.document.connections": connections, "com.apple.ibtool.document.hierarchy": merged.hierarchy, "com.apple.ibtool.document.classes": Object.fromEntries( - archive.classes.map((name) => [name.replace(/\0+$/u, ""), {}]), + archive.classes.map((name) => { + const className = normalizeClassName(name); + budget.reserveJsonPropertyNameOccurrence(className); + return [className, {}]; + }), ), }), }; diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts index 58a9b1ce3..d8b6f3a93 100644 --- a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts @@ -48,6 +48,33 @@ export class InterfaceBuilderDecodeBudget { this.#remainingBytes -= bytes; } + /** Reserve an upper bound for one JSON-serialized string occurrence. */ + reserveJsonStringOccurrence(value: string): void { + // JSON escaping uses at most six ASCII bytes per UTF-16 code unit, plus + // quotes. Count occurrences even when the source string is shared. + this.reserve( + value.length * 6 + 2, + "projected JSON strings exceed the aggregate Interface Builder decode budget", + ); + } + + /** Reserve the exact JSON string length for a base64 value from Buffer. */ + reserveBase64JsonStringOccurrence(value: string): void { + // Buffer's base64 alphabet contains no JSON-escaped characters. + this.reserve( + value.length + 2, + "projected NIB base64 values exceed the aggregate Interface Builder decode budget", + ); + } + + /** Reserve an upper bound for one JSON object property name occurrence. */ + reserveJsonPropertyNameOccurrence(value: string): void { + this.reserve( + value.length * 6 + 3, + "projected JSON property names exceed the aggregate Interface Builder decode budget", + ); + } + /** Reserve bytes for an optional projection without throwing on exhaustion. */ tryReserve(bytes: number): boolean { if ( @@ -214,7 +241,7 @@ const estimateBinaryPlistExpansion = ( const byteLength = type === 6 ? size * 2 : size; if (cursor + byteLength > offsetTable) throw new TypeError("binary plist scalar object is truncated"); - estimate += type === 4 ? Math.ceil((size * 4) / 3) : byteLength * 2; + estimate += type === 4 ? size + 4 * Math.ceil(size / 3) : byteLength * 2; if (estimate > maxBytes) throw new InterfaceBuilderDecodeBudgetExceeded( "aggregate_decode_budget_exhausted", diff --git a/src/artifacts/apple/NibArchive.ts b/src/artifacts/apple/NibArchive.ts index 8ed48b765..79b8a0d05 100644 --- a/src/artifacts/apple/NibArchive.ts +++ b/src/artifacts/apple/NibArchive.ts @@ -395,7 +395,7 @@ const readValue = ( `value ${index} data`, ); budget.reserve( - 32 + Math.ceil((length * 4) / 3), + 32 + 4 * Math.ceil(length / 3), `NIBArchive data value ${index} exceeds the aggregate Interface Builder decode budget`, ); const encoded = bytes.subarray(cursor.offset, next).toString("base64"); diff --git a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts index 42d7b10a3..b5fabfcb0 100644 --- a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts +++ b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { analyzeInterfaceBuilderBundle } from "../../../src/artifacts/apple/InterfaceBuilderAnalysis.js"; import { encodeNibArchiveFixture } from "../../../src/artifacts/apple/NibArchive.fixture.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interfaceBuilderGraph.js"; @@ -96,6 +97,96 @@ const binaryPlistWithRepeatedReferences = (depth: number): Buffer => { return Buffer.concat([header, ...encodedObjects, offsetTable, trailer]); }; +const binaryPlistWithRepeatedData = ( + referenceCount: number, + dataLength: number, +): Buffer => { + const header = Buffer.from("bplist00", "ascii"); + const array = Buffer.concat([ + Buffer.from([0xaf, 0x10, referenceCount]), + Buffer.alloc(referenceCount, 1), + ]); + const data = Buffer.concat([ + Buffer.from([0x4f, 0x12]), + Buffer.from( + new Uint8Array([ + (dataLength >>> 24) & 0xff, + (dataLength >>> 16) & 0xff, + (dataLength >>> 8) & 0xff, + dataLength & 0xff, + ]), + ), + Buffer.alloc(dataLength, 7), + ]); + const offsets = Buffer.alloc(8); + offsets.writeUInt32BE(header.length, 0); + offsets.writeUInt32BE(header.length + array.length, 4); + const offsetTable = header.length + array.length + data.length; + const trailer = Buffer.alloc(32); + trailer[6] = 4; + trailer[7] = 1; + trailer.writeBigUInt64BE(2n, 8); + trailer.writeBigUInt64BE(0n, 16); + trailer.writeBigUInt64BE(BigInt(offsetTable), 24); + return Buffer.concat([header, array, data, offsets, trailer]); +}; + +const encodeNibArchiveWithSharedDataValue = (input: { + readonly className: string; + readonly fieldName: string; + readonly payload: string; + readonly objectCount: number; +}): Buffer => { + const varint = (value: number): Buffer => { + const bytes: number[] = []; + let remaining = value; + do { + let byte = remaining & 0x7f; + remaining >>>= 7; + if (remaining === 0) byte |= 0x80; + bytes.push(byte); + } while (remaining > 0); + return Buffer.from(bytes); + }; + const objects = Buffer.concat( + Array.from({ length: input.objectCount }, () => + Buffer.concat([varint(0), varint(0), varint(1)]), + ), + ); + const fieldBytes = Buffer.from(input.fieldName); + const keys = Buffer.concat([varint(fieldBytes.length), fieldBytes]); + const payloadBytes = Buffer.from(input.payload); + const values = Buffer.concat([ + varint(0), + Buffer.from([8]), + varint(payloadBytes.length), + payloadBytes, + ]); + const classBytes = Buffer.from(input.className); + const classes = Buffer.concat([ + varint(classBytes.length), + varint(0), + classBytes, + ]); + const objectsOffset = 50; + const keysOffset = objectsOffset + objects.length; + const valuesOffset = keysOffset + keys.length; + const classesOffset = valuesOffset + values.length; + const header = Buffer.alloc(50); + header.write("NIBArchive", 0, "ascii"); + header.writeUInt32LE(1, 10); + header.writeUInt32LE(10, 14); + header.writeUInt32LE(input.objectCount, 18); + header.writeUInt32LE(objectsOffset, 22); + header.writeUInt32LE(1, 26); + header.writeUInt32LE(keysOffset, 30); + header.writeUInt32LE(1, 34); + header.writeUInt32LE(valuesOffset, 38); + header.writeUInt32LE(1, 42); + header.writeUInt32LE(classesOffset, 46); + return Buffer.concat([header, objects, keys, values, classes]); +}; + describe("compiled Interface Builder bundle reader", () => { it("reads nib plist archives and reports provenance", async () => { const root = await createTestTempDirectory("rea-ib-test-"); @@ -483,24 +574,16 @@ describe("bounded Interface Builder archive decoding", () => { "RepeatedReferences.nib: omitted because aggregate decode budget exhausted.", ); }); +}); - it("budgets repeated large NSString projections before retaining each field", async () => { - const root = await createTestTempDirectory("rea-ib-repeated-string-test-"); +describe("binary plist data expansion budget", () => { + it("reserves copied binary plist data for every expanded reference", async () => { + const root = await createTestTempDirectory("rea-ib-binary-data-budget-"); const bundle = join(root, "Example.app"); await mkdir(bundle, { recursive: true }); - const largeString = "x".repeat(1024 * 1024); - const references = Object.fromEntries( - Array.from({ length: 50 }, (_, index) => [`title${index}`, { ref: 1 }]), - ); await writeFile( - join(bundle, "RepeatedString.nib"), - encodeNibArchiveFixture({ - classes: ["UIView", "NSString"], - objects: [ - { classIndex: 0, values: references }, - { classIndex: 1, values: { "NS.bytes": largeString } }, - ], - }), + join(bundle, "RepeatedData.nib"), + binaryPlistWithRepeatedData(200, 700 * 1024), ); const result = await analyzeInterfaceBuilderBundle({ @@ -519,11 +602,178 @@ describe("bounded Interface Builder archive decoding", () => { }), ); expect(result.limitations).toContain( - "RepeatedString.nib: omitted because aggregate decode budget exhausted.", + "RepeatedData.nib: omitted because aggregate decode budget exhausted.", ); }); }); +describe("NIB projection byte amplification", () => { + it("omits NIB projections whose shared strings or class names amplify beyond budget", async () => { + const repeatedString = "x".repeat(1024 * 1024); + const stringReferences = Object.fromEntries( + Array.from({ length: 50 }, (_, index) => [`title${index}`, { ref: 1 }]), + ); + const longName = "n".repeat(512 * 1024); + const longNameObjects = Array.from({ length: 700 }, () => ({ + classIndex: 0, + values: { field: true }, + })); + const cases = [ + { + name: "RepeatedString", + archive: encodeNibArchiveFixture({ + classes: ["UIView", "NSString"], + objects: [ + { classIndex: 0, values: stringReferences }, + { classIndex: 1, values: { "NS.bytes": repeatedString } }, + ], + }), + }, + { + name: "RepeatedClass", + archive: encodeNibArchiveFixture({ + classes: [longName], + objects: longNameObjects, + }), + }, + ]; + + for (const { name, archive } of cases) { + const root = await createTestTempDirectory(`rea-ib-${name}-budget-`); + const bundle = join(root, "Example.app"); + await mkdir(bundle, { recursive: true }); + await writeFile(join(bundle, `${name}.nib`), archive); + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "f".repeat(64), + }); + + expect(result.documents, name).toEqual([]); + expect(result.graph.coverage, name).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "aggregate_decode_budget_exhausted", + examined: 1, + omitted: 1, + }), + ); + expect(result.limitations, name).toContain( + `${name}.nib: omitted because aggregate decode budget exhausted.`, + ); + } + }); + + it("retains a single large base64 data field within the representation budget", async () => { + const root = await createTestTempDirectory("rea-ib-large-data-positive-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle, { recursive: true }); + const archive = encodeNibArchiveFixture({ + classes: ["UIView"], + objects: [ + { + classIndex: 0, + values: { payload: "x".repeat(29 * 1024 * 1024) }, + }, + ], + }); + await writeFile(join(bundle, "LargePayload.nib"), archive); + + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "f".repeat(64), + }); + + expect(result.documents.map(({ relative_path }) => relative_path)).toEqual([ + "LargePayload.nib", + ]); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ facet: "archive_decode", status: "complete" }), + ); + const serialized = JSON.stringify(result); + expect(serialized).toContain("$nib_data_base64"); + expect(Buffer.byteLength(serialized)).toBeLessThan(256 * 1024 * 1024); + }); +}); + +cliTest( + "compiled CLI preserves a good archive and reports repeated key and data omissions", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-ib-cli-amplification-"); + const bundle = join(root, "Example.app"); + const contents = join(bundle, "Contents"); + const resources = join(bundle, "Contents", "Resources"); + await mkdir(resources, { recursive: true }); + await mkdir(join(contents, "MacOS"), { recursive: true }); + await writeFile( + join(contents, "MacOS", "App"), + thinMach(0xfeedfacf, 0x0100000c), + ); + await writeFile( + join(contents, "Info.plist"), + "CFBundleExecutableApp", + ); + await writeFile( + join(resources, "A-Good.nib"), + encodeNibArchiveFixture({ + classes: ["UIView"], + objects: [{ classIndex: 0, values: { title: "Good" } }], + }), + ); + const longKey = "k".repeat(512 * 1024); + await writeFile( + join(resources, "B-RepeatedKey.nib"), + encodeNibArchiveFixture({ + classes: ["UIView"], + objects: Array.from({ length: 700 }, () => ({ + classIndex: 0, + values: { [longKey]: true }, + })), + }), + ); + await writeFile( + join(resources, "C-RepeatedData.nib"), + encodeNibArchiveWithSharedDataValue({ + className: "UIView", + fieldName: "payload", + payload: "d".repeat(384 * 1024), + objectCount: 700, + }), + ); + + const output = await cli.run({ + arguments: ["decode-interface-builder", bundle, "--json"], + environment: { + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }, + }); + + expect(output.exitCode, `${output.stderr}\n${output.stdout}`).toBe(0); + const result = interfaceBuilderAnalysisSchema.parse( + parseEvidence(output.json).normalized_result, + ); + expect(result.documents.map(({ relative_path }) => relative_path)).toEqual([ + "Contents/Resources/A-Good.nib", + ]); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "aggregate_decode_budget_exhausted", + examined: 3, + omitted: 2, + }), + ); + expect(result.limitations).toContain( + "Contents/Resources/B-RepeatedKey.nib: omitted because aggregate decode budget exhausted.", + ); + expect(result.limitations).toContain( + "Contents/Resources/C-RepeatedData.nib: omitted because aggregate decode budget exhausted.", + ); + }, +); + describe("aggregate Interface Builder archive retention budget", () => { it("keeps earlier documents when the aggregate decode reservation is exhausted", async () => { const root = await createTestTempDirectory("rea-ib-budget-test-");