From 090b454350591f58d3349fa992109d55414f5294 Mon Sep 17 00:00:00 2001 From: saurabh sharma <146759278+saurabh-sol@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:18:54 +0530 Subject: [PATCH 01/15] fix(javascript-semantics): keep emitted data out of dispatch listener fields (#2952) * fix(javascript-semantics): keep emitted data out of dispatch listener fields The EventEmitter operation collector read argument position 1 as the listener for every event method, so a dispatch such as bus.emit("ready", data) reported the data binding and its range as the listener. Only registration and removal name a listener there; emit passes its remaining arguments to listeners that already exist. Read the listener argument only for register and remove operations. Fixes #2574 Co-Authored-By: Claude Fable 5.1 * fix(javascript-semantics): read listener arguments only for listener-taking methods removeAllListeners accepts only an optional event name, so gating the listener argument on operation kind still reported an ignored second argument as a listener. Gate on the methods whose argument position 1 names a listener instead: the registration methods, off, and removeListener. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- ...emanticAnalysis.runtime_boundaries.test.ts | 71 +++++++++++++++++++ .../javascriptSemanticAsyncEffects.ts | 11 ++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.runtime_boundaries.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.runtime_boundaries.test.ts index 03e25032a..792b8a0ea 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.runtime_boundaries.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.runtime_boundaries.test.ts @@ -130,3 +130,74 @@ describe("JavaScript semantic analysis: runtime boundaries 1", () => { expect(ir.objectOperations.length).toBeGreaterThan(0); }); }); + +describe("JavaScript semantic analysis: event dispatch data", () => { + it("keeps emitted data out of dispatch listener fields", () => { + const ir = analyzeJavaScriptSemantics(` + import { EventEmitter } from "node:events"; + const bus = new EventEmitter(); + const data = 1; + const handler = () => {}; + bus.on("ready", handler); + bus.emit("ready", data); + bus.emit("ready"); + bus.off("ready", handler); + bus.removeAllListeners("ready", data); + `); + const handler = ir.bindings.find(({ name }) => name === "handler"); + expect(handler).toBeDefined(); + expect( + ir.eventOperations.map( + ({ kind, method, listenerBindingId, listenerLocation }) => ({ + kind, + method, + listenerBindingId, + hasListenerLocation: listenerLocation !== null, + }), + ), + ).toEqual([ + { + kind: "register", + method: "on", + listenerBindingId: handler?.bindingId, + hasListenerLocation: true, + }, + { + kind: "dispatch", + method: "emit", + listenerBindingId: null, + hasListenerLocation: false, + }, + { + kind: "dispatch", + method: "emit", + listenerBindingId: null, + hasListenerLocation: false, + }, + { + kind: "remove", + method: "off", + listenerBindingId: handler?.bindingId, + hasListenerLocation: true, + }, + { + kind: "remove", + method: "removeAllListeners", + listenerBindingId: null, + hasListenerLocation: false, + }, + ]); + expect( + ir.eventOperations.map(({ eventName, resolution }) => [ + eventName, + resolution, + ]), + ).toEqual([ + ["ready", "complete"], + ["ready", "complete"], + ["ready", "complete"], + ["ready", "complete"], + ["ready", "complete"], + ]); + }); +}); diff --git a/src/domain/javascript/javascriptSemanticAsyncEffects.ts b/src/domain/javascript/javascriptSemanticAsyncEffects.ts index f7914c20f..5b3068101 100644 --- a/src/domain/javascript/javascriptSemanticAsyncEffects.ts +++ b/src/domain/javascript/javascriptSemanticAsyncEffects.ts @@ -37,6 +37,13 @@ const EVENT_REMOVE_METHODS = [ "removeListener", "removeAllListeners", ] as const; +/** + * Methods whose argument position 1 names a listener. `removeAllListeners` + * takes only an optional event name, and `emit` passes its remaining + * arguments as data to listeners that already exist. + */ +const EVENT_LISTENER_METHODS: readonly JavaScriptSemanticEventOperation["method"][] = + [...EVENT_REGISTER_METHODS, "off", "removeListener"]; const EVENT_METHODS = [ ...EVENT_REGISTER_METHODS, ...EVENT_REMOVE_METHODS, @@ -137,7 +144,9 @@ const collectEvent = ( if (method === undefined) return; const kind = eventKind(method); const eventArgument = node.arguments[0]; - const listenerArgument = node.arguments[1]; + const listenerArgument = EVENT_LISTENER_METHODS.includes(method) + ? node.arguments[1] + : undefined; const emitter = emitterIdentity(member.object, state); const eventName = dataEffectLiteralString(eventArgument); output.push({ From c7ceaadb709ab5ac14a9487b3e170e3633cec250 Mon Sep 17 00:00:00 2001 From: chAngE <2815456271@qq.com> Date: Wed, 14 Oct 2026 02:50:03 +0800 Subject: [PATCH 02/15] fix(browser): preserve unknown storage inventory and fingerprint coverage (#2491) * fix(browser): preserve unknown storage inventory and fingerprint coverage * fix(browser): compare known storage edits with incomplete inventories * test(browser): consolidate storage completeness regression workflows * test(ida): admit aligned FAT thinning fixtures on every platform --------- Co-authored-by: chAngE <172472630+chAng-L19@users.noreply.github.com> Co-authored-by: morluto Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/browser/CdpCaptureStorage.ts | 28 +- src/browser/CdpCaptureStorageFingerprints.ts | 77 +++- src/browser/CdpPageCapture.ts | 6 + src/domain/webCaptureDiff.ts | 11 +- src/ida/IdaWorkspaceThinning.test.ts | 111 +++--- ...owserProvider.storage_completeness.test.ts | 355 ++++++++++++++++++ 6 files changed, 506 insertions(+), 82 deletions(-) create mode 100644 tests/boundary/browser/cdpBrowserProvider.storage_completeness.test.ts diff --git a/src/browser/CdpCaptureStorage.ts b/src/browser/CdpCaptureStorage.ts index 1e88ca74a..9c3eae8c5 100644 --- a/src/browser/CdpCaptureStorage.ts +++ b/src/browser/CdpCaptureStorage.ts @@ -1,5 +1,6 @@ import type { InspectWebPageInput } from "../domain/browserObservation.js"; import type { WebPageInspection } from "../domain/browserObservationSchemas.js"; +import type { BrowserExclusionReason } from "../domain/browserCompleteness.js"; import { CdpConnection } from "./CdpConnection.js"; import { optionalCdpCommand } from "./CdpOptionalCommand.js"; import { @@ -26,6 +27,7 @@ export const captureStorage = async ( limitations: string[], ): Promise<{ readonly value: WebPageInspection["storage"]; + readonly keysExclusion: BrowserExclusionReason | undefined; }> => { const origin = new URL(pageUrl).origin; const quota = recordValue( @@ -82,7 +84,18 @@ export const captureStorage = async ( // list is an incomplete enumeration rather than an empty one. Array.isArray(cacheField) && caches.length === cacheField.length; + const keysExclusion = !context.input.include_storage_keys + ? undefined + : local.exclusion === "provider_unavailable" || + session.exclusion === "provider_unavailable" || + indexedRaw === undefined || + cacheRaw === undefined + ? "provider_unavailable" + : !local.complete || !session.complete || !structuredStorageComplete + ? "invalid_protocol_value" + : undefined; return { + keysExclusion, value: { origin, usage_bytes: numberValue(quota?.usage) ?? null, @@ -99,7 +112,11 @@ export const captureStorage = async ( }; }; -const emptyStorageItems: CapturedStorageItems = { +type StorageItemsCapture = CapturedStorageItems & { + readonly exclusion?: "provider_unavailable" | "invalid_protocol_value"; +}; + +const emptyStorageItems: StorageItemsCapture = { items: [], complete: false, }; @@ -118,7 +135,7 @@ const storageItems = async ( origin: string, isLocalStorage: boolean, limitations: string[], -): Promise => { +): Promise => { const raw = await optionalCdpCommand( context, "DOMStorage.getDOMStorageItems", @@ -127,7 +144,12 @@ const storageItems = async ( ); const entries = recordValue(raw)?.entries; if (raw === undefined || !Array.isArray(entries)) - return { items: [], complete: false }; + return { + items: [], + complete: false, + exclusion: + raw === undefined ? "provider_unavailable" : "invalid_protocol_value", + }; const items = entries.flatMap((entry) => { if (!Array.isArray(entry)) return []; const key = cdpStringValue(entry[0]); diff --git a/src/browser/CdpCaptureStorageFingerprints.ts b/src/browser/CdpCaptureStorageFingerprints.ts index 02a52b3be..ec7a06850 100644 --- a/src/browser/CdpCaptureStorageFingerprints.ts +++ b/src/browser/CdpCaptureStorageFingerprints.ts @@ -193,7 +193,9 @@ const addIndexedDb = async ( scope: "indexed_db_schema", identity: databaseName, value: stores.map(indexedDbStoreSchema), - complete: indexedDbSchemaComplete(stores), + complete: + stores.length === database.objectStores.length && + indexedDbSchemaComplete(stores), }); for (const store of stores) { const objectStoreName = cdpStringValue(store.name); @@ -228,19 +230,37 @@ const indexedDbSchemaComplete = ( stores.every( (store) => cdpStringValue(store.name) !== undefined && - stableValueComplete(store.keyPath) && + indexedDbKeyPathComplete(store.keyPath) && typeof store.autoIncrement === "boolean" && Array.isArray(store.indexes) && recordsValue(store.indexes).length === store.indexes.length && recordsValue(store.indexes).every( (index) => cdpStringValue(index.name) !== undefined && - stableValueComplete(index.keyPath) && + indexedDbKeyPathComplete(index.keyPath) && typeof index.unique === "boolean" && typeof index.multiEntry === "boolean", ), ); +const indexedDbKeyPathComplete = (value: unknown): boolean => { + const keyPath = recordValue(value); + if (keyPath === undefined) return false; + switch (keyPath.type) { + case "null": + return true; + case "string": + return typeof keyPath.string === "string"; + case "array": + return ( + Array.isArray(keyPath.array) && + keyPath.array.every((key) => typeof key === "string") + ); + default: + return false; + } +}; + const addIndexedDbRecords = async ( run: FingerprintRun, input: { @@ -474,14 +494,49 @@ const stableValueComplete = (value: unknown): boolean => { const remoteObjectComplete = (value: unknown): boolean => { const record = recordValue(value); - if (record === undefined) return true; - return ( - record.objectId === undefined && - record.preview === undefined && - record.customPreview === undefined && - (record.type !== "object" || record.value !== undefined) && - stableValueComplete(record) - ); + if ( + record === undefined || + record.objectId !== undefined || + record.preview !== undefined || + record.customPreview !== undefined || + !stableValueComplete(record) + ) + return false; + switch (record.type) { + case "undefined": + return ( + record.value === undefined && record.unserializableValue === undefined + ); + case "string": + case "boolean": + return ( + typeof record.value === record.type && + record.unserializableValue === undefined + ); + case "number": + return record.unserializableValue === undefined + ? typeof record.value === "number" && Number.isFinite(record.value) + : record.value === undefined && + typeof record.unserializableValue === "string" && + ["-0", "NaN", "Infinity", "-Infinity"].includes( + record.unserializableValue, + ); + case "bigint": + return ( + record.value === undefined && + typeof record.unserializableValue === "string" && + /^-?\d+n$/u.test(record.unserializableValue) + ); + case "object": + return ( + record.unserializableValue === undefined && + (record.subtype === "null" + ? record.value === null + : record.value !== null && typeof record.value === "object") + ); + default: + return false; + } }; const decodeBase64 = (value: string): Buffer | null => { diff --git a/src/browser/CdpPageCapture.ts b/src/browser/CdpPageCapture.ts index f49d4e186..337f99dfa 100644 --- a/src/browser/CdpPageCapture.ts +++ b/src/browser/CdpPageCapture.ts @@ -155,6 +155,12 @@ const captureAuthorizedPage = async ( ); if (!input.include_storage_keys) state.events.completeness.exclude("storage_keys", "not_approved", null); + else if (storageCapture.keysExclusion !== undefined) + state.events.completeness.exclude( + "storage_keys", + storageCapture.keysExclusion, + null, + ); await report(context.progress, 3, "Normalizing browser evidence"); const completedFrameResult = await authorizedMainFrame({ connection: context.connection, diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index 72c351f31..736d609ee 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -683,16 +683,17 @@ const storageMap = ( }), ], ]); - if (!includeKeys) return map; const add = (kind: string, keys: readonly string[]) => { for (const key of keys) { map.set(`storage:${kind}:${key}`, digestCanonicalValue(key)); } }; - add("local_storage", storage.local_storage_keys); - add("session_storage", storage.session_storage_keys); - add("indexed_db", storage.indexed_db_names); - add("cache", storage.cache_names); + if (includeKeys) { + add("local_storage", storage.local_storage_keys); + add("session_storage", storage.session_storage_keys); + add("indexed_db", storage.indexed_db_names); + add("cache", storage.cache_names); + } for (const fingerprint of storage.content_fingerprints) { const identity = `${fingerprint.scope}:${fingerprint.identity_sha256}`; if (!fingerprintIdentities.has(identity)) continue; diff --git a/src/ida/IdaWorkspaceThinning.test.ts b/src/ida/IdaWorkspaceThinning.test.ts index baba6e145..3af1cd5ee 100644 --- a/src/ida/IdaWorkspaceThinning.test.ts +++ b/src/ida/IdaWorkspaceThinning.test.ts @@ -21,12 +21,13 @@ afterEach(async () => { } }); -/** Write a two-slice universal fixture and resolve it for the given host. */ -const universalTarget = async (hostArchitecture: "arm64" | "x64") => { +const resolveUniversalTarget = async ( + bytes: Buffer, + hostArchitecture: "arm64" | "x64", +) => { const directory = await createTestTempDirectory("rea-ida-thin-"); roots.push(directory); const path = join(directory, "universal"); - const bytes = fat([0x01000007, 0x0100000c]); await writeFile(path, bytes); const resolved = await parseBinaryTarget(path, { hostArchitecture }); if (!resolved.ok) throw resolved.error; @@ -35,6 +36,27 @@ const universalTarget = async (hostArchitecture: "arm64" | "x64") => { return { bytes, target: resolved.value }; }; +/** Write a two-slice universal fixture and resolve it for the given host. */ +const universalTarget = (hostArchitecture: "arm64" | "x64") => + resolveUniversalTarget(fat([0x01000007, 0x0100000c]), hostArchitecture); + +const copySelectedSlice = async ({ + bytes, + target, +}: Awaited>) => { + const workspace = await IdaWorkspace.allocate(target); + roots.push(workspace.directory); + await workspace.copyInput(); + const input = await readFile(workspace.inputPath); + const { offset, size } = target.universalSlice; + expect(input.length, "no byte after the slice is copied").toBe(size); + expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true); + expect(workspace.inputSha256).toBe( + createHash("sha256").update(input).digest("hex"), + ); + return { workspace, input }; +}; + /** One slice of a synthetic universal file: CPU type, offset and size. */ interface Layout { readonly cpu: number; @@ -43,6 +65,12 @@ interface Layout { readonly alignmentExponent?: number; } +const multiSliceLayout: readonly Layout[] = [ + { cpu: 0x01000007, offset: 4096, size: 8480, alignmentExponent: 12 }, + { cpu: 0x0100000c, offset: 16384, size: 33464 }, + { cpu: 0x00000007, offset: 65536, size: 33464 }, +]; + /** * Write a universal file whose slices span more than one 64 KiB read, each * filled with its own byte so bytes outside the selected slice are visible. @@ -51,9 +79,6 @@ const largeUniversalTarget = async ( layout: readonly Layout[], hostArchitecture: "arm64" | "x64", ) => { - const directory = await createTestTempDirectory("rea-ida-thin-large-"); - roots.push(directory); - const path = join(directory, "universal"); const total = Math.max(...layout.map(({ offset, size }) => offset + size)); const bytes = Buffer.alloc(total); bytes.writeUInt32BE(0xcafebabe, 0); @@ -67,37 +92,27 @@ const largeUniversalTarget = async ( bytes.fill(0x11 * (index + 1), offset, offset + size); thinMach(0xfeedfacf, cpu).copy(bytes, offset); }); - await writeFile(path, bytes); - const resolved = await parseBinaryTarget(path, { hostArchitecture }); - if (!resolved.ok) throw resolved.error; - if (resolved.value.format !== "mach-o-universal") - throw new Error("the universal fixture did not resolve as a FAT container"); - return { bytes, target: resolved.value }; + return await resolveUniversalTarget(bytes, hostArchitecture); }; describe("IDA workspace input thinning", () => { + it("admits the multi-read FAT fixture on every host platform", async () => { + const { target } = await largeUniversalTarget(multiSliceLayout, "arm64"); + expect(target.universalSlice).toMatchObject({ offset: 16384, size: 33464 }); + }); + it.skipIf(process.platform === "win32")( "thins the selected arm64 slice into the private headless input", async () => { const { bytes, target } = await universalTarget("arm64"); - const workspace = await IdaWorkspace.allocate(target); - roots.push(workspace.directory); - await workspace.copyInput(); - - const input = await readFile(workspace.inputPath); - const { offset, size } = target.universalSlice; - expect(offset).toBeGreaterThan(0); - expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true); - expect(input.length).toBe(size); + const { workspace, input } = await copySelectedSlice({ bytes, target }); + expect(target.universalSlice.offset).toBeGreaterThan(0); expect(input.readUInt32BE(0)).toBe(0xfeedfacf); expect(input.readUInt32BE(4)).toBe(0x0100000c); expect(parseExecutableHeader(input, "arm64")).toMatchObject({ ok: true, value: { format: "mach-o", architecture: "arm64" }, }); - expect(createHash("sha256").update(input).digest("hex")).toBe( - workspace.inputSha256, - ); await workspace.remove(); }, ); @@ -106,13 +121,7 @@ describe("IDA workspace input thinning", () => { "thins the x86_64 slice when the host is intel", async () => { const { bytes, target } = await universalTarget("x64"); - const workspace = await IdaWorkspace.allocate(target); - roots.push(workspace.directory); - await workspace.copyInput(); - - const input = await readFile(workspace.inputPath); - const { offset, size } = target.universalSlice; - expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true); + const { workspace, input } = await copySelectedSlice({ bytes, target }); expect(input.readUInt32BE(4)).toBe(0x01000007); await workspace.remove(); }, @@ -122,31 +131,12 @@ describe("IDA workspace input thinning", () => { "copies exactly the selected slice when later slices follow it across read boundaries", async () => { const { bytes, target } = await largeUniversalTarget( - [ - { - cpu: 0x01000007, - offset: 4096, - size: 8480, - alignmentExponent: 12, - }, - { cpu: 0x0100000c, offset: 16384, size: 33464 }, - { cpu: 0x00000007, offset: 65536, size: 33464 }, - ], + multiSliceLayout, "arm64", ); - const workspace = await IdaWorkspace.allocate(target); - roots.push(workspace.directory); - await workspace.copyInput(); - - const input = await readFile(workspace.inputPath); - const { offset, size } = target.universalSlice; - expect(size, "the arm64 slice is selected").toBe(33464); - expect(input.length, "no byte after the slice is copied").toBe(size); - expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true); - expect(workspace.inputSha256).toBe( - createHash("sha256") - .update(bytes.subarray(offset, offset + size)) - .digest("hex"), + const { workspace } = await copySelectedSlice({ bytes, target }); + expect(target.universalSlice.size, "the arm64 slice is selected").toBe( + 33464, ); await workspace.remove(); }, @@ -162,15 +152,10 @@ describe("IDA workspace input thinning", () => { ], "x64", ); - const workspace = await IdaWorkspace.allocate(target); - roots.push(workspace.directory); - await workspace.copyInput(); - - const input = await readFile(workspace.inputPath); - const { offset, size } = target.universalSlice; - expect(size, "the x86_64 slice is selected").toBe(37456); - expect(input.length, "no byte after the slice is copied").toBe(size); - expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true); + const { workspace } = await copySelectedSlice({ bytes, target }); + expect(target.universalSlice.size, "the x86_64 slice is selected").toBe( + 37456, + ); await workspace.remove(); }, ); diff --git a/tests/boundary/browser/cdpBrowserProvider.storage_completeness.test.ts b/tests/boundary/browser/cdpBrowserProvider.storage_completeness.test.ts new file mode 100644 index 000000000..b1c5d7495 --- /dev/null +++ b/tests/boundary/browser/cdpBrowserProvider.storage_completeness.test.ts @@ -0,0 +1,355 @@ +import { expect, it } from "vitest"; + +import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; +import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js"; +import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js"; +import { compareWebCaptures } from "../../../src/domain/webCaptureDiff.js"; +import { compareWebCapturesInputSchema } from "../../../src/domain/webCaptureDiffSchemas.js"; +import { + startFakeCdpBrowser, + type FakeOptions, +} from "../../fixtures/fakeCdpBrowser.js"; +import { trackBrowser } from "./cdpBrowserProvider.support.js"; + +const inspect = async (options: FakeOptions, includeFingerprints = true) => { + const browser = await startFakeCdpBrowser(options); + trackBrowser(browser); + const input = inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + include_storage_keys: true, + include_storage_fingerprints: includeFingerprints, + }); + const capture = async () => { + const result = await new CdpBrowserProvider().inspectPage(input); + if (!result.ok) throw result.error; + return result.value; + }; + return { capture }; +}; + +const compareStorage = (before: WebPageInspection, after: WebPageInspection) => + compareWebCaptures( + compareWebCapturesInputSchema.parse({ + before: { inspection: before }, + after: { inspection: after }, + }), + ).dimensions.storage; + +it.each([ + ["CacheStorage.requestCacheNames", {}], + ["CacheStorage.requestCacheNames", { caches: null }], + ["CacheStorage.requestCacheNames", { caches: [null] }], + [ + "CacheStorage.requestCacheNames", + { caches: [{ cacheName: "assets-v1", cacheId: "secret-id" }, null] }, + ], + ["CacheStorage.requestCacheNames", { caches: [{ cacheName: "assets-v1" }] }], + ["IndexedDB.requestDatabaseNames", { databaseNames: [null] }], + ["DOMStorage.getDOMStorageItems", { entries: [null] }], +] as const)( + "preserves unknown inventory coverage from %s: %j", + async (method, reply) => { + let malformed = false; + const { capture } = await inspect({ + commandResult: (command) => + malformed && command.method === method ? reply : undefined, + }); + const before = await capture(); + malformed = true; + const after = await capture(); + expect(after.storage.fingerprints_complete).toBe(false); + expect(after.completeness.unavailable_sections).toContain("storage_keys"); + expect(after.completeness.excluded).toContainEqual({ + section: "storage_keys", + reason: "invalid_protocol_value", + count: null, + }); + expect(compareStorage(before, after)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + }, +); + +it("preserves unavailable storage-key coverage without requesting fingerprints", async () => { + const { capture } = await inspect( + { unsupportedMethods: ["CacheStorage.requestCacheNames"] }, + false, + ); + const captured = await capture(); + expect(captured.completeness.excluded).toContainEqual({ + section: "storage_keys", + reason: "provider_unavailable", + count: null, + }); + expect(captured.storage.local_storage_keys).toEqual(["public-key"]); +}); + +const validKey = { type: "string", value: "row-1" }; + +it.each(["unavailable", "malformed"] as const)( + "compares shared complete values with %s cache inventory", + async (inventory) => { + let entries: unknown[][] = [["shared-key", "before"]]; + const { capture } = await inspect({ + ...(inventory === "unavailable" + ? { unsupportedMethods: ["CacheStorage.requestCacheNames"] } + : {}), + commandResult: (command) => { + if (command.method === "CacheStorage.requestCacheNames") + return { caches: [null] }; + if (command.method === "DOMStorage.getDOMStorageItems") + return { entries }; + return undefined; + }, + }); + const before = await capture(); + expect(before.completeness.unavailable_sections).toContain("storage_keys"); + expect(before.storage.fingerprints_complete).toBe(false); + expect(compareStorage(before, await capture())).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + + entries = [["shared-key", "after"]]; + const changed = await capture(); + expect(changed.storage.usage_bytes).toBe(before.storage.usage_bytes); + expect(changed.storage.quota_bytes).toBe(before.storage.quota_bytes); + for (const [left, right] of [ + [before, changed], + [changed, before], + ] as const) + expect(compareStorage(left, right)).toMatchObject({ + status: "changed", + total_changes: 2, + changes: [ + expect.objectContaining({ change: "modified" }), + expect.objectContaining({ change: "modified" }), + ], + }); + + const incompleteValues = structuredClone(changed); + for (const fingerprint of incompleteValues.storage.content_fingerprints) + if (["local_storage", "session_storage"].includes(fingerprint.scope)) + fingerprint.complete = false; + expect(compareStorage(before, incompleteValues)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + + // Missing or invalid peer values establish neither removal nor addition. + for (const partial of [[], [["shared-key", null]]]) { + entries = partial; + const incomplete = await capture(); + for (const [left, right] of [ + [before, incomplete], + [incomplete, before], + ] as const) + expect(compareStorage(left, right)).toMatchObject({ + status: "unknown", + total_changes: 0, + changes: [], + }); + } + }, +); + +it.each(["key", "primaryKey"])( + "requires IndexedDB %s identity evidence", + async (field) => { + const { capture } = await inspect({ + commandResult: (command) => + command.method === "IndexedDB.requestData" + ? { + objectStoreDataEntries: [ + { + key: validKey, + primaryKey: validKey, + value: validKey, + [field]: {}, + }, + ], + hasMore: false, + } + : undefined, + }); + const captured = await capture(); + expect(captured.storage.fingerprints_complete).toBe(false); + expect( + captured.storage.content_fingerprints.find( + ({ scope }) => scope === "indexed_db_record", + )?.complete, + ).toBe(false); + }, +); + +it("does not fingerprint a filtered partial object-store schema as complete", async () => { + let malformed = false; + const store = { + name: "records", + keyPath: { type: "string", string: "id" }, + autoIncrement: false, + indexes: [], + }; + const { capture } = await inspect({ + commandResult: (command) => + command.method === "IndexedDB.requestDatabase" + ? { + databaseWithObjectStores: { + name: "app-db", + version: 1, + objectStores: [ + store, + malformed ? null : { ...store, name: "other" }, + ], + }, + } + : undefined, + }); + const before = await capture(); + malformed = true; + const after = await capture(); + expect( + after.storage.content_fingerprints.find( + ({ scope }) => scope === "indexed_db_schema", + )?.complete, + ).toBe(false); + expect(compareStorage(before, after)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); +}); + +const incompleteRemoteObjects = [ + undefined, + null, + "raw", + {}, + { type: "string" }, + { type: "string", value: 1 }, + { type: "boolean", value: "false" }, + { type: "number" }, + { type: "number", unserializableValue: "invalid" }, + { type: "number", unserializableValue: ["NaN"] }, + { type: "function", description: "function f() {}" }, + { type: "symbol", description: "Symbol(x)" }, + { type: "object", subtype: "null" }, + { type: "bigint", unserializableValue: "broken" }, +] as const; + +const completeRemoteObjects = [ + { type: "string", value: "" }, + { type: "boolean", value: false }, + { type: "number", value: 0 }, + { type: "undefined" }, + { type: "number", unserializableValue: "NaN" }, + { type: "number", unserializableValue: "-0" }, + { type: "bigint", unserializableValue: "-42n" }, + { type: "object", subtype: "null", value: null }, + { type: "object", value: { nested: [false, null, 0] } }, +] as const; + +it.each([ + ...incompleteRemoteObjects.map((value) => [value, false] as const), + ...completeRemoteObjects.map((value) => [value, true] as const), +])( + "validates IndexedDB RemoteObject completeness: %j => %s", + async (value, complete) => { + const { capture } = await inspect({ + commandResult: (command) => + command.method === "IndexedDB.requestData" + ? { + objectStoreDataEntries: [ + { key: validKey, primaryKey: validKey, value }, + ], + hasMore: false, + } + : undefined, + }); + const before = await capture(); + const after = await capture(); + const records = before.storage.content_fingerprints.filter( + ({ scope }) => scope === "indexed_db_record", + ); + expect(records).toHaveLength(1); + expect(records[0]?.complete).toBe(complete); + expect(before.storage.fingerprints_complete).toBe(complete); + expect(compareStorage(before, after)).toMatchObject({ + status: complete ? "unchanged" : "unknown", + total_changes: 0, + }); + }, +); + +it.each([ + [undefined, false], + [{}, false], + [{ type: "string" }, false], + [{ type: "array", array: [null] }, false], + [{ type: "null" }, true], + [{ type: "string", string: "id" }, true], + [{ type: "array", array: ["id", "name"] }, true], +] as const)( + "requires complete IndexedDB key paths: %j", + async (keyPath, complete) => { + const { capture } = await inspect({ + commandResult: (command) => + command.method === "IndexedDB.requestDatabase" + ? { + databaseWithObjectStores: { + name: "app-db", + version: 1, + objectStores: [ + { + name: "records", + keyPath, + autoIncrement: false, + indexes: [ + { + name: "by-id", + keyPath, + unique: false, + multiEntry: false, + }, + ], + }, + ], + }, + } + : undefined, + }); + const captured = await capture(); + expect( + captured.storage.content_fingerprints.find( + ({ scope }) => scope === "indexed_db_schema", + )?.complete, + ).toBe(complete); + expect(captured.storage.fingerprints_complete).toBe(complete); + }, +); + +it("accepts genuinely empty inventories as complete", async () => { + const { capture } = await inspect({ + commandResult: (command) => { + switch (command.method) { + case "CacheStorage.requestCacheNames": + return { caches: [] }; + case "IndexedDB.requestDatabaseNames": + return { databaseNames: [] }; + case "DOMStorage.getDOMStorageItems": + return { entries: [] }; + case "Network.getCookies": + return { cookies: [] }; + default: + return undefined; + } + }, + }); + const before = await capture(); + const after = await capture(); + expect(before.storage.fingerprints_complete).toBe(true); + expect(compareStorage(before, after).status).toBe("unchanged"); +}); From 0e6eca40857471341139e4203bfb32925b5333c5 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Tue, 13 Oct 2026 11:50:30 -0700 Subject: [PATCH 03/15] feat(native): read embedded cargo-auditable Rust dependency lists (#2456) * feat(native): read embedded cargo-auditable Rust dependency lists * fix(native): trim PE section padding and compute dependency cycles iteratively * fix(native): ignore SHT_NOBITS sections and validate dependency edges linearly * fix(native): bound compiler path scanning and retained hashes * fix(rust): pass package arrays as named parameterized test cases * fix(rust): admit graph expansion and preserve resource units * fix(rust): reject malformed packages, false cycles, and hidden compiler paths Fail the first malformed cargo-auditable package before Zod collects an issue per field. Require cyclic dependency indices to match the cycles in package dependencies. Keep read-only ELF ranges when a section name does not fit the bounded name window, so a rustc path in that range is still reported. * fix(ci): repair integrated checks and Hopper test baseline * fix(go): verify Mach-O sections against segments --------- Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Cursor Agent --- .../skills/reverse-engineer-anything/SKILL.md | 8 + .../references/native-and-artifacts.md | 12 + README.md | 1 + docs/.vitepress/config.ts | 1 + docs/cli.md | 12 + docs/index.md | 2 +- docs/readme/README_ar.md | 1 + docs/readme/README_de.md | 1 + docs/readme/README_es.md | 1 + docs/readme/README_fa.md | 1 + docs/readme/README_fr.md | 1 + docs/readme/README_id.md | 1 + docs/readme/README_it.md | 1 + docs/readme/README_ja.md | 1 + docs/readme/README_ko.md | 1 + docs/readme/README_pl.md | 1 + docs/readme/README_pt-BR.md | 1 + docs/readme/README_ru.md | 1 + docs/readme/README_th.md | 1 + docs/readme/README_uk.md | 1 + docs/readme/README_vi.md | 1 + docs/readme/README_zh-TW.md | 1 + docs/readme/README_zh.md | 1 + docs/rust-binaries.md | 67 +++ scripts/lib/catalog-builders.mjs | 5 + scripts/lib/catalog-core.mjs | 2 + scripts/lib/module-boundaries.mjs | 1 + src/application/CapabilityInventory.ts | 2 +- src/application/rust/RustBinaryPort.ts | 19 + .../rust/RustBinaryService.test.ts | 132 ++++++ src/application/rust/RustBinaryService.ts | 113 +++++ src/cli.ts | 2 + src/cli/rustCommands.ts | 34 ++ src/cliCommandNames.ts | 1 + src/composition/rust.ts | 6 + src/contracts/rust/rustToolContracts.ts | 26 ++ src/contracts/toolContracts.ts | 2 + src/contracts/toolEffects.test.ts | 2 + src/contracts/toolEffects.ts | 1 + src/domain/rust/rustBinary.ts | 294 ++++++++++++ src/go/GoBinaryContainer.ts | 37 ++ src/go/GoBinaryImage.ts | 5 +- src/go/GoElfImage.ts | 107 +++++ src/go/GoMachoImage.ts | 91 +++- src/go/GoPeImage.ts | 51 ++ src/rust/RustAuditableReader.test.ts | 221 +++++++++ src/rust/RustAuditableReader.ts | 440 ++++++++++++++++++ src/rust/RustBinaryImage.ts | 181 +++++++ src/rust/RustBinaryProvider.ts | 216 +++++++++ src/server/createServer.ts | 11 + src/server/registerRustTools.ts | 31 ++ tests/acceptance/rustBinaryCli.test.ts | 71 +++ tests/boundary/go/goBinaryImage.test.ts | 21 + tests/boundary/mcp/ghidraInventoryMcp.test.ts | 3 +- tests/boundary/mcp/rustBinaryMcp.test.ts | 122 +++++ tests/boundary/rust/rustBinaryImage.test.ts | 141 ++++++ tests/fixtures/rust/image.ts | 263 +++++++++++ .../cleanLinuxHopperRuntime.test.ts | 3 + vitest.config.ts | 2 +- 59 files changed, 2767 insertions(+), 12 deletions(-) create mode 100644 docs/rust-binaries.md create mode 100644 src/application/rust/RustBinaryPort.ts create mode 100644 src/application/rust/RustBinaryService.test.ts create mode 100644 src/application/rust/RustBinaryService.ts create mode 100644 src/cli/rustCommands.ts create mode 100644 src/composition/rust.ts create mode 100644 src/contracts/rust/rustToolContracts.ts create mode 100644 src/domain/rust/rustBinary.ts create mode 100644 src/rust/RustAuditableReader.test.ts create mode 100644 src/rust/RustAuditableReader.ts create mode 100644 src/rust/RustBinaryImage.ts create mode 100644 src/rust/RustBinaryProvider.ts create mode 100644 src/server/registerRustTools.ts create mode 100644 tests/acceptance/rustBinaryCli.test.ts create mode 100644 tests/boundary/mcp/rustBinaryMcp.test.ts create mode 100644 tests/boundary/rust/rustBinaryImage.test.ts create mode 100644 tests/fixtures/rust/image.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index de9f6b7c1..39f9ae20b 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -122,6 +122,14 @@ explicit path or endpoint and do not need it. absent metadata does not prove a binary is not Go, and unparsed text records remain explicit. See the [Go binary guide](https://github.com/morluto/rea/blob/main/docs/go-binaries.md). +- Embedded cargo-auditable Rust dependency metadata: `inspect_rust_binary` + with the explicit local ELF, PE, or thin Mach-O file. This built-in + target-free inspection requires no Rust installation or disassembler. Preserve + package order, dependency indices, and the section file range. A missing + `.dep-v0` section is null metadata, not proof the binary has no dependencies. + A rustc commit hash is an inference from a read-only `/rustc/<40-hex>/` path. + See the + [Rust binary guide](https://github.com/morluto/rea/blob/main/docs/rust-binaries.md). - Explicit local SQLite database snapshot: `inspect_sqlite_database`. Inspect with Node.js 24.x >=24.15 or 26+ providing native SQLite limits. Read schema diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 3e185a8a2..960e46094 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -11,6 +11,18 @@ unparsed module-text line. A missing record is an unknown, not proof of a non-Go binary; observed metadata does not authenticate its claims. Function, source and type recovery remain a separate native-analysis question. +## Rust dependency metadata + +Use `inspect_rust_binary` for an explicit local ELF, PE, or thin Mach-O when +the question concerns the cargo-auditable crate list embedded in `.dep-v0` +(Mach-O: `__DATA,.dep-v0`). No `open_binary`, Rust installation, or target +execution is required. The CLI equivalent is `inspect-rust-binary PATH`. +Preserve artifact SHA-256, package order, dependency indices, and the section +file range. A missing section is null metadata, not proof the binary has no +dependencies. An omitted package kind stays null. A rustc commit hash is +inferred from a read-only `/rustc/<40-hex>/` path and does not by itself +identify the compiler. See [Rust binaries](https://github.com/morluto/rea/blob/main/docs/rust-binaries.md). + ## SQLite snapshots Use `inspect_sqlite_database` with an explicit local database snapshot for its diff --git a/README.md b/README.md index 53a86c5fc..8129813d3 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ Additional tools and host support depend on the target: | Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](https://rea.tools/guides/native/) | | Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | | Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required | +| Rust binaries | Embedded cargo-auditable crate names, versions, sources and dependency edges | Built-in static ELF/PE/thin Mach-O inspection; [Rust binaries](docs/rust-binaries.md) | | WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` | | EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) | | SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits | diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 5d7fcc2f3..b4cf2e4ad 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -29,6 +29,7 @@ export default defineConfig({ items: [ { text: "Native binaries", link: "/native-investigation" }, { text: "Go build metadata", link: "/go-binaries" }, + { text: "Rust dependencies", link: "/rust-binaries" }, { text: "JavaScript and Electron", link: "/javascript-artifact-reconstruction", diff --git a/docs/cli.md b/docs/cli.md index c43ec2181..c801bf006 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -83,6 +83,18 @@ The shared CLI/MCP workflow reports compiler version, modules, replacements, build settings, original bytes and SHA-256. See [Go binaries](go-binaries.md) for supported containers and absent or malformed metadata. +Inspect an embedded cargo-auditable Rust dependency list without installing +Rust or configuring a disassembler: + +```bash +rea inspect-rust-binary /absolute/path/to/program --json +``` + +The same workflow reports package names, versions, sources, the root package, +dependency edges, and the section's file range. A missing `.dep-v0` section is +null metadata, not proof that the binary has no dependencies. See +[Rust binaries](rust-binaries.md). + Configure [Hopper or Ghidra](installation.md#hopper), or the [IDA adapter](ida-provider.md), before analyzing a native target. Substitute your target, search text and function name or address in these examples: diff --git a/docs/index.md b/docs/index.md index 732ad9f05..9e0912dd1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -49,7 +49,7 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). | Task | Guides | | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | +| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | | Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [JEB](jeb-analysis.md) | | Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | | Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md) | diff --git a/docs/readme/README_ar.md b/docs/readme/README_ar.md index 8df2522bc..db261f80b 100644 --- a/docs/readme/README_ar.md +++ b/docs/readme/README_ar.md @@ -135,6 +135,7 @@ npx rea-agents@latest setup | الملفات الثنائية للشيفرة الأصلية | شيفرة شبه برمجية، وتعليمات تجميع، وسلاسل نصية، ورموز، واستدعاءات، ومراجع | Hopper أو Ghidra أو IDA؛ [تحليل الشيفرة الأصلية](https://rea.tools/ar/guides/native/) | | بنية ELF دون تشغيل | الأقسام والمقاطع والرموز ومعلومات إعادة التموضع الأصلية، وآليات الحماية المحتملة المستدلّ عليها بالفحص الثابت | pwntools يوفّره المستدعي على Linux x64؛ [تشخيص الملفات الثنائية](../binary-diagnostics.md) | | شيفرة EVM البايتية | محددات التوجيه، وإزاحات البايتات، والمعاملات المستنتجة، وقابلية تغيير الحالة | مدخل محلي يحمل بايتات خامًا أو تمثيلًا سداسيًا عشريًا على Linux x64؛ [دليل EVM دون تشغيل](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | أعطال Linux المسجّلة | سجلات note الخام، ومسجّلات المعالج/الإشارات لكل خيط مسجّل، وترشيحات اختيارية لخرائط الذاكرة | pwntools يوفّره المستدعي على Linux x64؛ GDB/pwndbg اختياريان؛ [الأعطال المسجّلة](../recorded-crashes.md) | | JavaScript / Electron | الوحدات، وعمليات الاستيراد، وخرائط المصدر، والمسارات، وIPC، وعلاقات إضافات الشيفرة الأصلية | Node.js وnpm؛ [تحليل التطبيقات](https://rea.tools/ar/guides/javascript/) | | المواقع | بنية الصفحة، والسكربتات، وملاحظات الشبكة، ولقطات الشاشة المطلوبة | متصفح من عائلة Chrome؛ [تحليل المتصفح](https://rea.tools/ar/guides/browser/) | diff --git a/docs/readme/README_de.md b/docs/readme/README_de.md index 2dbb1d41a..f8e81658b 100644 --- a/docs/readme/README_de.md +++ b/docs/readme/README_de.md @@ -135,6 +135,7 @@ REA benötigt Node.js 22.x (>=22.19), 24.x (>=24.11) oder 26+ sowie npm. Zusätz | Native Binärdateien | Pseudocode, Assemblercode, Zeichenketten, Symbole, Aufrufe und Verweise | Hopper, Ghidra oder IDA; [native Analyse](https://rea.tools/de/guides/native/) | | Offline-ELF-Struktur | Sektionen, Segmente, ursprüngliche Symbole/Relokationen und mögliche Schutzmaßnahmen aus statischer Analyse | Vom Aufrufer bereitgestelltes pwntools unter Linux x64; [Binärdiagnostik](../binary-diagnostics.md) | | EVM-Bytecode | Dispatch-Selektoren, Byte-Offsets, abgeleitete Argumente und Veränderbarkeit des Zustands | Lokale Eingabe mit Rohbytes oder Hexdarstellung unter Linux x64; [Offline-EVM-Leitfaden](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Aufgezeichnete Linux-Abstürze | Unverarbeitete Note-Einträge, Register/Signale aller aufgezeichneten Threads und optionale Zuordnungskandidaten | Vom Aufrufer bereitgestelltes pwntools unter Linux x64; optional GDB/pwndbg; [aufgezeichnete Abstürze](../recorded-crashes.md) | | JavaScript / Electron | Module, Imports, Source Maps, Routen, IPC und Beziehungen zu nativen Add-ons | Node.js und npm; [Anwendungsanalyse](https://rea.tools/de/guides/javascript/) | | Websites | Seitenstruktur, Skripte, Netzwerkbeobachtungen und angeforderte Screenshots | Ein Browser aus der Chrome-Familie; [Browseranalyse](https://rea.tools/de/guides/browser/) | diff --git a/docs/readme/README_es.md b/docs/readme/README_es.md index d09be25e2..6bc927b18 100644 --- a/docs/readme/README_es.md +++ b/docs/readme/README_es.md @@ -135,6 +135,7 @@ REA requiere Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, además de npm. Las h | Binarios nativos | Pseudocódigo, ensamblador, cadenas, símbolos, llamadas y referencias | Hopper, Ghidra o IDA; [análisis nativo](https://rea.tools/es/guides/native/) | | Estructura ELF sin ejecutar | Secciones, segmentos, símbolos/reubicaciones originales y posibles medidas de mitigación detectadas mediante análisis estático | pwntools proporcionado por quien llama, en Linux x64; [diagnóstico de binarios](../binary-diagnostics.md) | | Bytecode EVM | Selectores de despacho, desplazamientos en bytes, argumentos inferidos y mutabilidad | Entrada local con bytes sin procesar o representación hexadecimal, en Linux x64; [guía de EVM sin ejecutar](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Fallos de Linux registrados | Registros note sin procesar, registros/señales de cada hilo registrado y posibles correspondencias de memoria opcionales | pwntools proporcionado por quien llama, en Linux x64; GDB/pwndbg opcionales; [fallos registrados](../recorded-crashes.md) | | JavaScript / Electron | Módulos, importaciones, mapas de código fuente, rutas, IPC y relaciones con complementos nativos | Node.js y npm; [análisis de aplicaciones](https://rea.tools/es/guides/javascript/) | | Sitios web | Estructura de la página, scripts, observaciones de red y capturas de pantalla solicitadas | Un navegador de la familia Chrome; [análisis del navegador](https://rea.tools/es/guides/browser/) | diff --git a/docs/readme/README_fa.md b/docs/readme/README_fa.md index b24e4380c..96364a10f 100644 --- a/docs/readme/README_fa.md +++ b/docs/readme/README_fa.md @@ -127,6 +127,7 @@ REA به Node.js 22.x (>=22.19)، 24.x (>=24.11) یا 26+، به‌همراه np | باینری‌های بومی | شبه‌کد، اسمبلی، رشته‌ها، نمادها، فراخوانی‌ها و ارجاعات | Hopper، Ghidra یا IDA؛ [تحلیل بومی](https://rea.tools/fa/guides/native/) | | ساختار ELF آفلاین | بخش‌ها، سگمنت‌ها، نمادها و relocationهای اصلی و گزینه‌های احتمالی کاهش آسیب‌پذیری ایستا | pwntools تأمین‌شده توسط فراخواننده روی Linux x64؛ [تشخیص باینری](../binary-diagnostics.md) | | بایت‌کد EVM | selectorهای dispatch، offsetهای بایتی، آرگومان‌های استنباط‌شده و قابلیت تغییر وضعیت | دادهٔ خام/hex محلی روی Linux x64؛ [راهنمای EVM آفلاین](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | کرش‌های ثبت‌شدهٔ لینوکس | یادداشت‌های خام، رجیسترها/سیگنال‌های تمام threadهای ثبت‌شده و گزینه‌های احتمالی نگاشت | pwntools تأمین‌شده توسط فراخواننده روی Linux x64؛ GDB/pwndbg اختیاری؛ [کرش‌های ثبت‌شده](../recorded-crashes.md) | | JavaScript / Electron | ماژول‌ها، importها، source mapها، مسیرها، IPC و ارتباط افزونه‌های بومی | Node.js و npm؛ [تحلیل برنامه](https://rea.tools/fa/guides/javascript/) | | وب‌سایت‌ها | ساختار صفحه، اسکریپت‌ها، مشاهدات شبکه و اسکرین‌شات‌های درخواستی | مرورگر مبتنی بر Chrome؛ [تحلیل مرورگر](https://rea.tools/fa/guides/browser/) | diff --git a/docs/readme/README_fr.md b/docs/readme/README_fr.md index e6278ac93..09fc9ca58 100644 --- a/docs/readme/README_fr.md +++ b/docs/readme/README_fr.md @@ -139,6 +139,7 @@ REA nécessite Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, ainsi que npm. Les | Structure ELF hors ligne | Sections, segments, symboles et relocations d'origine, et protections statiques candidates | pwntools fourni par l'utilisateur sous Linux x64 ; [diagnostics binaires](../binary-diagnostics.md) | | Artefact WASM | Validation, sections, formes WAT exactes d'import/export et WAT décodé | WABT 1.0.42 fourni par l'utilisateur ; `inspect_wasm_artifact` | | Bytecode EVM | Sélecteurs de dispatch, offsets en octets, arguments inférés et mutabilité | Fichier local brut ou hexadécimal sous Linux x64 ; [guide EVM hors ligne](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Crashs Linux enregistrés | Notes brutes, registres et signaux de chaque thread enregistré, et correspondances de mappings facultatives | pwntools fourni par l'utilisateur sous Linux x64 ; GDB/pwndbg en option ; [crashs enregistrés](../recorded-crashes.md) | | JavaScript / Electron | Modules, imports, source maps, routes, IPC et relations avec les modules natifs | Node.js et npm ; [analyse d'applications](https://rea.tools/fr/guides/javascript/) | | Sites web | Structure des pages, scripts, observations réseau et captures d'écran demandées | Un navigateur de la famille Chrome ; [analyse du navigateur](https://rea.tools/fr/guides/browser/) | diff --git a/docs/readme/README_id.md b/docs/readme/README_id.md index fba947fb2..0eab5d71d 100644 --- a/docs/readme/README_id.md +++ b/docs/readme/README_id.md @@ -142,6 +142,7 @@ Alat tambahan dan dukungan sistem host bergantung pada target yang dianalisis: | Tata letak ELF offline | Bagian, segmen, simbol/relokasi asli, dan kandidat mitigasi statis | pwntools yang disediakan pemanggil pada Linux x64; [diagnostik biner](../binary-diagnostics.md) | | Artefak WASM | Validasi, bagian, bentuk WAT import/ekspor yang tepat, dan WAT yang didekode | WABT 1.0.42 yang disediakan pemanggil; `inspect_wasm_artifact` | | Bytecode EVM | Selector dispatch, offset byte, argumen yang disimpulkan, dan sifat mutabilitas | Pembawa data raw/hex lokal; [panduan EVM offline](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Crash Linux yang direkam | Catatan mentah, register/sinyal setiap thread yang direkam, dan kandidat pemetaan opsional | pwntools yang disediakan pemanggil; GDB/pwndbg opsional; [crash yang direkam](../recorded-crashes.md) | | JavaScript / Electron | Modul, impor, source map, rute, IPC, dan hubungan add-on native | Node.js dan npm; [analisis aplikasi](https://rea.tools/id/guides/javascript/) | | Situs web | Struktur halaman, skrip, pengamatan jaringan, dan tangkapan layar yang diminta | Peramban berbasis Chrome; [analisis peramban](https://rea.tools/id/guides/browser/) | diff --git a/docs/readme/README_it.md b/docs/readme/README_it.md index b7e7c6100..e2027f660 100644 --- a/docs/readme/README_it.md +++ b/docs/readme/README_it.md @@ -139,6 +139,7 @@ REA richiede Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, oltre a npm. Strument | Layout ELF offline | Sezioni, segmenti, simboli/relocazioni originali e candidati mitigazioni statiche | pwntools fornito dal chiamante su Linux x64; [diagnostica binaria](../binary-diagnostics.md) | | Artefatto WASM | Convalida, sezioni, forme WAT di import/export esatte e WAT decodificato | WABT 1.0.42 fornito dal chiamante; `inspect_wasm_artifact` | | Bytecode EVM | Selettori di dispatch, offset dei byte, argomenti inferiti e mutabilità | Carrier raw/hex locale; [guida EVM offline](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Crash Linux registrati | Note grezze, registri/segnai di ogni thread registrato e candidati mapping opzionali | pwntools fornito dal chiamante; GDB/pwndbg opzionale; [crash registrati](../recorded-crashes.md) | | JavaScript / Electron | Moduli, import, source map, route, IPC e relazioni con add-on nativi | Node.js e npm; [analisi delle applicazioni](https://rea.tools/it/guides/javascript/) | | Siti web | Struttura della pagina, script, osservazioni di rete e screenshot richiesti | Un browser della famiglia Chrome; [analisi browser](https://rea.tools/it/guides/browser/) | diff --git a/docs/readme/README_ja.md b/docs/readme/README_ja.md index c9d7981ba..250883ebd 100644 --- a/docs/readme/README_ja.md +++ b/docs/readme/README_ja.md @@ -135,6 +135,7 @@ REA には Node.js 22.x（>=22.19）、24.x（>=24.11）、または 26+ と npm | ネイティブバイナリ | 疑似コード、アセンブリ、文字列、シンボル、呼び出し、参照 | Hopper、Ghidra、または IDA；[ネイティブ分析](https://rea.tools/ja/guides/native/) | | オフライン ELF レイアウト | セクション、セグメント、元のシンボル／再配置情報、静的解析によるセキュリティ対策の候補 | Linux x64 で利用者が用意する pwntools；[バイナリ診断](../binary-diagnostics.md) | | EVM バイトコード | ディスパッチセレクター、バイトオフセット、推定された引数と状態変更の可否 | Linux x64 上のローカルの生バイト／16 進数の入力ファイル；[オフライン EVM ガイド](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | 記録済みの Linux クラッシュ | 生の note レコード、記録された全スレッドのレジスター／シグナル、任意のマッピング候補 | Linux x64 で利用者が用意する pwntools；GDB/pwndbg は任意；[記録済みクラッシュ](../recorded-crashes.md) | | JavaScript / Electron | モジュール、インポート、ソースマップ、ルート、IPC、ネイティブアドオンの関係 | Node.js と npm；[アプリケーション分析](https://rea.tools/ja/guides/javascript/) | | ウェブサイト | ページ構造、スクリプト、ネットワークの観測結果、依頼されたスクリーンショット | Chrome 系ブラウザー；[ブラウザー分析](https://rea.tools/ja/guides/browser/) | diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index 26db0d824..99f49ee3b 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -135,6 +135,7 @@ REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니 | 네이티브 바이너리 | 의사 코드, 어셈블리, 문자열, 심볼, 호출, 참조 | Hopper, Ghidra 또는 IDA; [네이티브 분석](https://rea.tools/ko/guides/native/) | | 오프라인 ELF 레이아웃 | 섹션, 세그먼트, 원본 심볼/재배치 정보, 정적 분석으로 얻은 보안 완화 기법 후보 | Linux x64에서 호출자가 제공하는 pwntools; [바이너리 진단](../binary-diagnostics.md) | | EVM 바이트코드 | 디스패치 선택자, 바이트 오프셋, 추론한 인자와 상태 변경 가능 여부 | Linux x64의 로컬 원시 바이트/16진수 입력 파일; [오프라인 EVM 가이드](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | 기록된 Linux 크래시 | 원시 note 레코드, 기록된 모든 스레드의 레지스터/시그널, 선택적 매핑 후보 | Linux x64에서 호출자가 제공하는 pwntools; GDB/pwndbg는 선택 사항; [기록된 크래시](../recorded-crashes.md) | | JavaScript / Electron | 모듈, 임포트, 소스 맵, 라우트, IPC, 네이티브 애드온 관계 | Node.js와 npm; [애플리케이션 분석](https://rea.tools/ko/guides/javascript/) | | 웹사이트 | 페이지 구조, 스크립트, 네트워크 관찰 결과, 요청한 스크린샷 | Chrome 계열 브라우저; [브라우저 분석](https://rea.tools/ko/guides/browser/) | diff --git a/docs/readme/README_pl.md b/docs/readme/README_pl.md index 48200087d..8f2f2ab07 100644 --- a/docs/readme/README_pl.md +++ b/docs/readme/README_pl.md @@ -135,6 +135,7 @@ REA wymaga Node.js 22.x (>=22.19), 24.x (>=24.11) lub 26+ oraz npm. Dodatkowe na | Natywne pliki binarne | Pseudokod, kod asemblera, ciągi znaków, symbole, wywołania i odwołania | Hopper, Ghidra lub IDA; [analiza kodu natywnego](https://rea.tools/pl/guides/native/) | | Układ pliku ELF w trybie offline | Sekcje, segmenty, oryginalne symbole/relokacje i potencjalne mechanizmy zabezpieczeń wskazane przez analizę statyczną | pwntools dostarczone przez użytkownika w systemie Linux x64; [diagnostyka plików binarnych](../binary-diagnostics.md) | | Kod bajtowy EVM | Selektory wywołań, przesunięcia bajtowe, wywnioskowane argumenty i mutowalność | Lokalny plik z surowymi bajtami lub zapisem szesnastkowym w systemie Linux x64; [przewodnik EVM offline](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Zarejestrowane awarie systemu Linux | Surowe rekordy note, rejestry/sygnały każdego zarejestrowanego wątku i opcjonalni kandydaci na mapowania | pwntools dostarczone przez użytkownika w systemie Linux x64; opcjonalnie GDB/pwndbg; [zarejestrowane awarie](../recorded-crashes.md) | | JavaScript / Electron | Moduły, importy, mapy źródeł, trasy, IPC i powiązania z natywnymi dodatkami | Node.js i npm; [analiza aplikacji](https://rea.tools/pl/guides/javascript/) | | Strony internetowe | Struktura strony, skrypty, obserwacje sieciowe i zamówione zrzuty ekranu | Przeglądarka z rodziny Chrome; [analiza w przeglądarce](https://rea.tools/pl/guides/browser/) | diff --git a/docs/readme/README_pt-BR.md b/docs/readme/README_pt-BR.md index 6c3f34e35..4650d4eca 100644 --- a/docs/readme/README_pt-BR.md +++ b/docs/readme/README_pt-BR.md @@ -135,6 +135,7 @@ O REA requer Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, além do npm. As fer | Binários nativos | Pseudocódigo, assembly, strings, símbolos, chamadas e referências | Hopper, Ghidra ou IDA; [análise nativa](https://rea.tools/pt-BR/guides/native/) | | Estrutura ELF sem execução | Seções, segmentos, símbolos/relocações originais e possíveis mecanismos de mitigação identificados por análise estática | pwntools fornecido pelo chamador no Linux x64; [diagnóstico de binários](../binary-diagnostics.md) | | Bytecode EVM | Seletores de despacho, offsets de bytes, argumentos inferidos e mutabilidade | Entrada local com bytes brutos ou representação hexadecimal no Linux x64; [guia de EVM sem execução](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Falhas do Linux registradas | Registros note brutos, registradores/sinais de cada thread registrada e candidatos opcionais a mapeamentos | pwntools fornecido pelo chamador no Linux x64; GDB/pwndbg opcionais; [falhas registradas](../recorded-crashes.md) | | JavaScript / Electron | Módulos, importações, source maps, rotas, IPC e relações com complementos nativos | Node.js e npm; [análise de aplicativos](https://rea.tools/pt-BR/guides/javascript/) | | Sites | Estrutura da página, scripts, observações de rede e capturas de tela solicitadas | Um navegador da família Chrome; [análise do navegador](https://rea.tools/pt-BR/guides/browser/) | diff --git a/docs/readme/README_ru.md b/docs/readme/README_ru.md index 744e26f6c..68ebf1a83 100644 --- a/docs/readme/README_ru.md +++ b/docs/readme/README_ru.md @@ -135,6 +135,7 @@ REA требует Node.js 22.x (>=22.19), 24.x (>=24.11) или 26+, а так | Нативные бинарные файлы | Псевдокод, ассемблер, строки, символы, вызовы и ссылки | Hopper, Ghidra или IDA; [нативный анализ](https://rea.tools/ru/guides/native/) | | Структура ELF без запуска | Секции, сегменты, исходные символы/релокации и возможные механизмы защиты по данным статического анализа | pwntools, предоставленный вызывающей стороной, в Linux x64; [диагностика бинарных файлов](../binary-diagnostics.md) | | Байт-код EVM | Селекторы диспетчеризации, смещения байтов, предполагаемые аргументы и изменяемость состояния | Локальные необработанные байты или шестнадцатеричное представление в Linux x64; [руководство по EVM без запуска](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Записанные сбои Linux | Необработанные записи note, регистры/сигналы каждого записанного потока и необязательные кандидаты сопоставления с памятью | pwntools, предоставленный вызывающей стороной, в Linux x64; необязательные GDB/pwndbg; [записанные сбои](../recorded-crashes.md) | | JavaScript / Electron | Модули, импорты, карты исходного кода, маршруты, IPC и связи с нативными дополнениями | Node.js и npm; [анализ приложений](https://rea.tools/ru/guides/javascript/) | | Сайты | Структура страницы, скрипты, наблюдения за сетью и запрошенные снимки экрана | Браузер семейства Chrome; [анализ браузера](https://rea.tools/ru/guides/browser/) | diff --git a/docs/readme/README_th.md b/docs/readme/README_th.md index 734b28c86..60818d2b6 100644 --- a/docs/readme/README_th.md +++ b/docs/readme/README_th.md @@ -135,6 +135,7 @@ REA ต้องใช้ Node.js 22.x (>=22.19), 24.x (>=24.11) หรือ 26 | ไบนารีเนทีฟ | โค้ดเทียม แอสเซมบลี สตริง สัญลักษณ์ การเรียก และการอ้างอิง | Hopper, Ghidra หรือ IDA; [การวิเคราะห์เนทีฟ](https://rea.tools/th/guides/native/) | | โครงสร้าง ELF แบบออฟไลน์ | เซกชัน เซกเมนต์ สัญลักษณ์/รีโลเคชันต้นฉบับ และตัวเลือกกลไกป้องกันที่พบจากการวิเคราะห์แบบสถิต | pwntools ที่ผู้ใช้จัดเตรียมบน Linux x64; [การวินิจฉัยไบนารี](../binary-diagnostics.md) | | ไบต์โค้ด EVM | ตัวเลือกฟังก์ชันสำหรับการเรียก ออฟเซ็ตไบต์ อาร์กิวเมนต์และความสามารถในการเปลี่ยนสถานะที่อนุมานได้ | ไฟล์บนเครื่องที่มีไบต์ดิบหรือข้อมูลเลขฐานสิบหกบน Linux x64; [คู่มือ EVM แบบออฟไลน์](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | ข้อมูลการแครชของ Linux ที่บันทึกไว้ | ระเบียน note ดิบ รีจิสเตอร์/สัญญาณของทุกเธรดที่บันทึกไว้ และตัวเลือกการแมปเพิ่มเติม | pwntools ที่ผู้ใช้จัดเตรียมบน Linux x64; เลือกใช้ GDB/pwndbg เพิ่มเติมได้; [การแครชที่บันทึกไว้](../recorded-crashes.md) | | JavaScript / Electron | โมดูล การนำเข้า ซอร์สแมป เส้นทาง IPC และความสัมพันธ์กับส่วนเสริมเนทีฟ | Node.js และ npm; [การวิเคราะห์แอปพลิเคชัน](https://rea.tools/th/guides/javascript/) | | เว็บไซต์ | โครงสร้างหน้า สคริปต์ ข้อมูลเครือข่ายที่สังเกตได้ และภาพหน้าจอที่ร้องขอ | เบราว์เซอร์ในตระกูล Chrome; [การวิเคราะห์ผ่านเบราว์เซอร์](https://rea.tools/th/guides/browser/) | diff --git a/docs/readme/README_uk.md b/docs/readme/README_uk.md index 5c77d80b0..2ed389251 100644 --- a/docs/readme/README_uk.md +++ b/docs/readme/README_uk.md @@ -135,6 +135,7 @@ REA потребує Node.js 22.x (>=22.19), 24.x (>=24.11) або 26+, а та | Нативні бінарні файли | Псевдокод, асемблерний код, рядки, символи, виклики й посилання | Hopper, Ghidra або IDA; [нативний аналіз](https://rea.tools/uk/guides/native/) | | Структура ELF без запуску | Секції, сегменти, початкові символи/релокації та можливі захисні механізми за даними статичного аналізу | pwntools, наданий стороною, що викликає інструмент, у Linux x64; [діагностика бінарних файлів](../binary-diagnostics.md) | | Байт-код EVM | Селектори диспетчеризації, зміщення байтів, виведені припущення про аргументи та змінюваність стану | Локальні необроблені байти або шістнадцяткове представлення у Linux x64; [посібник EVM без запуску](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Записані аварійні завершення в Linux | Необроблені записи note, регістри/сигнали кожного записаного потоку й необов'язкові кандидати зіставлення пам'яті | pwntools, наданий стороною, що викликає інструмент, у Linux x64; необов'язкові GDB/pwndbg; [записані аварійні завершення](../recorded-crashes.md) | | JavaScript / Electron | Модулі, імпорти, карти вихідного коду, маршрути, IPC і зв'язки з нативними доповненнями | Node.js і npm; [аналіз застосунків](https://rea.tools/uk/guides/javascript/) | | Сайти | Структура сторінки, скрипти, мережеві спостереження й запитані знімки екрана | Браузер сімейства Chrome; [аналіз браузера](https://rea.tools/uk/guides/browser/) | diff --git a/docs/readme/README_vi.md b/docs/readme/README_vi.md index 3d2062da6..cf0bd6592 100644 --- a/docs/readme/README_vi.md +++ b/docs/readme/README_vi.md @@ -135,6 +135,7 @@ REA yêu cầu Node.js 22.x (>=22.19), 24.x (>=24.11) hoặc 26+, cùng với np | Tệp nhị phân mã máy | Mã giả, hợp ngữ, chuỗi, ký hiệu, lời gọi và tham chiếu | Hopper, Ghidra hoặc IDA; [phân tích mã máy](https://rea.tools/vi/guides/native/) | | Bố cục ELF ngoại tuyến | Các section, segment, ký hiệu/thông tin tái định vị gốc và các biện pháp giảm thiểu có thể có từ phân tích tĩnh | pwntools do bên gọi cung cấp trên Linux x64; [chẩn đoán tệp nhị phân](../binary-diagnostics.md) | | Bytecode EVM | Bộ chọn điều phối, độ lệch byte, tham số suy luận và khả năng thay đổi trạng thái | Đầu vào cục bộ chứa byte thô hoặc dạng thập lục phân trên Linux x64; [hướng dẫn EVM ngoại tuyến](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | Sự cố Linux đã ghi lại | Bản ghi note thô, thanh ghi/tín hiệu của mọi luồng được ghi lại và các ứng viên ánh xạ tùy chọn | pwntools do bên gọi cung cấp trên Linux x64; GDB/pwndbg tùy chọn; [sự cố đã ghi lại](../recorded-crashes.md) | | JavaScript / Electron | Mô-đun, các khai báo import, source map, tuyến, IPC và quan hệ với tiện ích mở rộng mã máy | Node.js và npm; [phân tích ứng dụng](https://rea.tools/vi/guides/javascript/) | | Trang web | Cấu trúc trang, script, quan sát mạng và ảnh chụp màn hình được yêu cầu | Trình duyệt thuộc họ Chrome; [phân tích trình duyệt](https://rea.tools/vi/guides/browser/) | diff --git a/docs/readme/README_zh-TW.md b/docs/readme/README_zh-TW.md index 1ddbbd26e..c8ad64e6a 100644 --- a/docs/readme/README_zh-TW.md +++ b/docs/readme/README_zh-TW.md @@ -139,6 +139,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | Go 二進位檔案 | 內嵌編譯器版本、模組、替換項和建置設定 | 內建靜態 ELF/PE/thin Mach-O 檢查；無需安裝 Go | | WASM 工件 | 驗證、節、精確的匯入/匯出 WAT 形式及解碼後的 WAT | 呼叫方提供的 WABT 1.0.42；`inspect_wasm_artifact` | | EVM 位元組碼 | 分派選擇器、位元組偏移、推斷的參數和狀態可變性 | Linux x64 上的本地原始位元組/十六進位輸入載體；[離線 EVM 指南](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | 已記錄的 Linux 崩潰 | 原始 note 記錄、每個已記錄執行緒的暫存器/訊號及可選的對映候選項 | Linux x64 上由呼叫方提供的 pwntools；可選的 GDB/pwndbg；[已記錄崩潰](../recorded-crashes.md) | | JavaScript / Electron | 模組、匯入、source map、路由、IPC 和原生附加元件關係 | Node.js 和 npm；[應用分析](https://rea.tools/zh-TW/guides/javascript/) | | 網站 | 頁面結構、腳本、網路觀察結果和按請求獲取的螢幕擷取畫面 | Chrome 系瀏覽器；[瀏覽器分析](https://rea.tools/zh-TW/guides/browser/) | diff --git a/docs/readme/README_zh.md b/docs/readme/README_zh.md index 6e70fd3c3..e5251eb36 100644 --- a/docs/readme/README_zh.md +++ b/docs/readme/README_zh.md @@ -139,6 +139,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | Go 二进制文件 | 内嵌编译器版本、模块、替换项和构建设置 | 内置静态 ELF/PE/thin Mach-O 检查；无需安装 Go | | WASM 工件 | 验证、节、精确的导入/导出 WAT 形式及解码后的 WAT | 调用方提供的 WABT 1.0.42；`inspect_wasm_artifact` | | EVM 字节码 | 分派选择器、字节偏移、推断的参数和状态可变性 | Linux x64 上的本地原始字节/十六进制输入载体；[离线 EVM 指南](../evm-bytecode.md) | +| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | | 已记录的 Linux 崩溃 | 原始 note 记录、每个已记录线程的寄存器/信号及可选的映射候选项 | Linux x64 上由调用方提供的 pwntools；可选的 GDB/pwndbg；[已记录崩溃](../recorded-crashes.md) | | JavaScript / Electron | 模块、导入、source map、路由、IPC 和原生扩展关系 | Node.js 和 npm；[应用分析](https://rea.tools/zh-CN/guides/javascript/) | | 网站 | 页面结构、脚本、网络观察结果和按请求获取的截图 | Chrome 系浏览器；[浏览器分析](https://rea.tools/zh-CN/guides/browser/) | diff --git a/docs/rust-binaries.md b/docs/rust-binaries.md new file mode 100644 index 000000000..d7cb7a750 --- /dev/null +++ b/docs/rust-binaries.md @@ -0,0 +1,67 @@ +# Rust binary dependency metadata + +`inspect_rust_binary` / `inspect-rust-binary` answers which crate names, +versions, sources, and dependency edges are embedded by cargo-auditable in one +selected local executable or library. It reads the file without running it and +requires no Rust installation, active binary target, Hopper, Ghidra, or IDA. + +```bash +rea inspect-rust-binary /absolute/path/to/application --json +``` + +The CLI resolves relative paths against the caller's working directory. MCP +requires an absolute filesystem path and preserves that exact selection: + +```json +{ + "name": "inspect_rust_binary", + "arguments": { "path": "/artifacts/application" } +} +``` + +## Observations + +The inline Evidence result contains the artifact path, SHA-256, byte length, +container format, architecture, word size, and byte order. When a `.dep-v0` +section is present, `auditable_metadata` contains: + +- `section_name`, `.dep-v0` for ELF and PE or `__DATA,.dep-v0` for thin Mach-O; +- `file_offset` and `section_bytes`, the exact file range of the compressed section; +- `compressed_bytes` and `uncompressed_bytes`; +- `format_version`, the embedded integer, or `null` when the field is omitted; +- `packages`, in the section's original order, with `name`, `version`, `source`, + `kind`, `dependencies`, and `root`; +- `root_package_index`, the single package marked `root`, or `null` when none is marked; +- `dependency_edges`, pairs of package indices in package-then-dependency order; +- `cyclic_dependency_indices`, sorted unique indices that participate in a cycle. + +An omitted `kind` stays `null`. The embedded format treats that omission as +runtime, and that default is not stored in the section. An omitted dependency +list is an empty array. An omitted `root` flag is false. + +`rustc_commit` is present only when a `/rustc/<40-hex>/` path is found in +read-only data. Its `evidence_kind` is `inferred`. The hash is not a claim that +that compiler produced the binary. Several distinct hashes leave a single +commit unknown. + +## Limits and failures + +The selected file is limited to 256 MiB. A dependency section is limited to 1 +MiB compressed. Inflated JSON is limited to the smaller of 8 MiB and 64 times +the compressed size. The complete report is limited to 8 MiB. Empty sections, +truncated or trailing zlib bytes, invalid JSON, unknown fields, dependency +indices outside the package list, more than one root package, and compressed +dependency sections are typed failures. A cycle is reported and is not walked. + +A missing section returns success with `auditable_metadata: null` and a +limitation. Absence of `.dep-v0` does not mean the binary has no dependencies. +Embedded names and versions are file content. They do not authenticate the +packages or the compiler. + +Universal Mach-O containers are unsupported. Select a thin slice first. ELF, +PE, and thin Mach-O are the supported containers. + +Package and dependency counts are checked against the minimum required JSON +representation before graph allocation or schema cloning. Oversized graphs +report the output byte budget; the compiler-path capacity reports +`maximum_rustc_hashes` in hashes, independently of byte limits. diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index d924ef49c..e480665e3 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -28,6 +28,7 @@ export const toolFamilyCatalog = (sources) => { ...sources.artifactContracts.ARTIFACT_TOOL_CONTRACTS, ...sources.evmContracts.EVM_TOOL_CONTRACTS, ...sources.goContracts.GO_TOOL_CONTRACTS, + ...sources.rustContracts.RUST_TOOL_CONTRACTS, ...sources.wasmContracts.WASM_TOOL_CONTRACTS, ...sources.sqliteContracts.SQLITE_TOOL_CONTRACTS, ...sources.captureContracts.CAPTURE_TOOL_CONTRACTS, @@ -225,6 +226,10 @@ export const providerCatalog = (sources) => { identity: sources.goProvider.GO_BINARY_PROVIDER_IDENTITY, contracts: sources.goContracts.GO_TOOL_CONTRACTS, }, + { + identity: sources.rustProvider.RUST_BINARY_PROVIDER_IDENTITY, + contracts: sources.rustContracts.RUST_TOOL_CONTRACTS, + }, { identity: sources.sqliteDomain.SQLITE_PROVIDER_IDENTITY, contracts: sources.sqliteContracts.SQLITE_TOOL_CONTRACTS, diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index f5488a9e6..1d402ed4e 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -59,6 +59,8 @@ const SOURCE_PATHS = { evmProvider: "dist/evm/EvmoleRelease.js", goContracts: "dist/contracts/go/goToolContracts.js", goProvider: "dist/go/GoBinaryProvider.js", + rustContracts: "dist/contracts/rust/rustToolContracts.js", + rustProvider: "dist/rust/RustBinaryProvider.js", artifactContracts: "dist/contracts/artifactToolContracts.js", managedContracts: "dist/contracts/managed/managedToolContracts.js", firmwareContracts: "dist/contracts/firmware/firmwareToolContracts.js", diff --git a/scripts/lib/module-boundaries.mjs b/scripts/lib/module-boundaries.mjs index 160dad7c0..a6c74491c 100644 --- a/scripts/lib/module-boundaries.mjs +++ b/scripts/lib/module-boundaries.mjs @@ -8,6 +8,7 @@ const MIGRATED_PROVIDER_ROOTS = new Set([ "sqlite", "evm", "go", + "rust", "wasm", "captures", "signals", diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index f4584eb29..925ed9ee3 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -343,7 +343,7 @@ const workflowAvailabilityFor = ({ kind, policy, }: AvailabilityContext): Availability | null => { - if (name === "inspect_go_binary") + if (name === "inspect_go_binary" || name === "inspect_rust_binary") return { reason: "available", remediation: null }; if (name === "inspect_sqlite_database") { const capability = sqliteDatabaseCapability(process.versions.node); diff --git a/src/application/rust/RustBinaryPort.ts b/src/application/rust/RustBinaryPort.ts new file mode 100644 index 000000000..b0956aea1 --- /dev/null +++ b/src/application/rust/RustBinaryPort.ts @@ -0,0 +1,19 @@ +import type { + ExecutionOptions, + ProviderIdentity, +} from "../AnalysisProvider.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { + InspectRustBinaryInput, + RustBinary, +} from "../../domain/rust/rustBinary.js"; +import type { Result } from "../../domain/result.js"; + +/** Replaceable offline dependency reader with no target-execution authority. */ +export interface RustBinaryPort { + readonly identity: ProviderIdentity; + inspect( + input: InspectRustBinaryInput, + options?: ExecutionOptions, + ): Promise>; +} diff --git a/src/application/rust/RustBinaryService.test.ts b/src/application/rust/RustBinaryService.test.ts new file mode 100644 index 000000000..7fbee615d --- /dev/null +++ b/src/application/rust/RustBinaryService.test.ts @@ -0,0 +1,132 @@ +import { expect, it } from "vitest"; +import { + AnalysisInputError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import type { RustBinary } from "../../domain/rust/rustBinary.js"; +import { ok } from "../../domain/result.js"; +import { RustBinaryService } from "./RustBinaryService.js"; + +const identity = { + id: "fixture-rust", + name: "Source-owned Rust reader seam", + version: "1", +}; +const report: RustBinary = { + artifact: { path: "/artifacts/program", sha256: "a".repeat(64), bytes: 128 }, + format: "elf", + architecture: "amd64", + bits: 64, + byte_order: "little", + auditable_metadata: { + section_name: ".dep-v0", + file_offset: 16, + section_bytes: 8, + compressed_bytes: 8, + uncompressed_bytes: 20, + format_version: 1, + packages: [ + { + name: "sample", + version: "0.1.0", + source: "local", + kind: null, + dependencies: [], + root: true, + }, + ], + root_package_index: 0, + dependency_edges: [], + cyclic_dependency_indices: [], + }, + rustc_commit: null, + limitations: [ + "No /rustc/<40-hex>/ path was found in read-only data, so a rustc commit hash is unknown.", + ], +}; + +it("rejects a relative path and extra arguments before reading", async () => { + let calls = 0; + const service = new RustBinaryService({ + identity, + inspect: () => { + calls += 1; + return Promise.resolve(ok(report)); + }, + }); + const relative = await service.inspect({ path: "program" }); + const extra = await service.inspect({ + path: report.artifact.path, + detail: "complete", + }); + expect(relative.ok).toBe(false); + expect(extra.ok).toBe(false); + if (!relative.ok) expect(relative.error).toBeInstanceOf(AnalysisInputError); + if (!extra.ok) expect(extra.error).toBeInstanceOf(AnalysisInputError); + expect(calls).toBe(0); +}); + +it("records the selected artifact and the dependency section range", async () => { + const result = await new RustBinaryService({ + identity, + inspect: () => Promise.resolve(ok(report)), + }).inspect({ path: report.artifact.path }); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + confidence: "observed", + subject: { format: "elf", architecture: "x86_64" }, + normalized_result: report, + locations: [ + { kind: "artifact-path", path: report.artifact.path }, + { kind: "file-offset-range", start: 16, end: 24 }, + ], + }); + const mismatched: RustBinary = { + ...report, + artifact: { ...report.artifact, path: "/artifacts/other" }, + }; + const invalid = await new RustBinaryService({ + identity, + inspect: () => Promise.resolve(ok(mismatched)), + }).inspect({ path: report.artifact.path }); + expect(invalid.ok).toBe(false); + if (!invalid.ok) expect(invalid.error).toBeInstanceOf(AnalysisOutputError); +}); + +it("rejects a cycle claim the package dependencies do not contain", async () => { + if (report.auditable_metadata === null) throw new Error("expected metadata"); + const invalidReport: RustBinary = { + ...report, + auditable_metadata: { + ...report.auditable_metadata, + cyclic_dependency_indices: [0], + }, + }; + const result = await new RustBinaryService({ + identity, + inspect: () => Promise.resolve(ok(invalidReport)), + }).inspect({ path: report.artifact.path }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toBeInstanceOf(AnalysisOutputError); +}); + +it.each([ + { from_index: 1, to_index: 0 }, + { from_index: 0, to_index: 1 }, +])("rejects provider edges referencing missing packages: %j", async (edge) => { + if (report.auditable_metadata === null) throw new Error("expected metadata"); + const invalidReport: RustBinary = { + ...report, + auditable_metadata: { + ...report.auditable_metadata, + dependency_edges: [edge], + }, + }; + const result = await new RustBinaryService({ + identity, + inspect: () => Promise.resolve(ok(invalidReport)), + }).inspect({ path: report.artifact.path }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toBeInstanceOf(AnalysisOutputError); +}); diff --git a/src/application/rust/RustBinaryService.ts b/src/application/rust/RustBinaryService.ts new file mode 100644 index 000000000..b112b366d --- /dev/null +++ b/src/application/rust/RustBinaryService.ts @@ -0,0 +1,113 @@ +import { isAbsolute } from "node:path"; +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import type { RustBinaryPort } from "./RustBinaryPort.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import { createEvidence, type Evidence } from "../../domain/evidence.js"; +import { BINARY_ARCHITECTURES } from "../../domain/binaryTargetTypes.js"; +import { + inspectRustBinaryInputSchema, + rustBinarySchema, +} from "../../domain/rust/rustBinary.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; + +const OPERATION = "inspect_rust_binary"; + +const architectureForSubject = (architecture: string) => { + switch (architecture) { + case "386": + return "x86"; + case "amd64": + return "x86_64"; + default: + return BINARY_ARCHITECTURES.find( + (supported) => supported === architecture, + ); + } +}; + +/** Shared exact input validation and artifact-bound Evidence for CLI and MCP. */ +export class RustBinaryService { + constructor(readonly provider: RustBinaryPort) {} + + /** Inspect one explicitly selected file without acquiring a binary session. */ + async inspect( + rawInput: unknown, + options?: ExecutionOptions, + ): Promise> { + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const input = inspectRustBinaryInputSchema.safeParse(rawInput); + if (!input.success) + return err( + analysisInputErrorFromIssues(OPERATION, input.error.issues, rawInput, { + cause: input.error, + }), + ); + if (!isAbsolute(input.data.path)) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["path"], + reason: "invalid_format", + message: "Expected an absolute filesystem path on this host.", + }, + ]), + ); + const inspected = await this.provider.inspect(input.data, options); + if (!inspected.ok) return inspected; + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const report = rustBinarySchema.safeParse(inspected.value); + if (!report.success || report.data.artifact.path !== input.data.path) + return err( + new AnalysisOutputError( + OPERATION, + "Rust dependency reader returned malformed metadata or changed the selected artifact identity.", + ), + ); + return ok(evidenceFor(this.provider.identity, input.data, report.data)); + } +} + +const evidenceFor = ( + provider: RustBinaryPort["identity"], + parameters: { readonly path: string }, + value: ReturnType, +): Evidence => { + const architecture = architectureForSubject(value.architecture); + const section = value.auditable_metadata; + return createEvidence( + { + path: value.artifact.path, + sha256: value.artifact.sha256, + format: value.format === "macho" ? "mach-o" : value.format, + ...(architecture === undefined ? {} : { architecture }), + }, + provider, + { + operation: OPERATION, + parameters, + result: value, + confidence: "observed", + limitations: value.limitations, + locations: [ + { kind: "artifact-path", path: value.artifact.path }, + ...(section === null + ? [] + : [ + { + kind: "file-offset-range" as const, + start: section.file_offset, + end: section.file_offset + section.section_bytes, + }, + ]), + ], + }, + ); +}; diff --git a/src/cli.ts b/src/cli.ts index d1bdf2912..d5b009241 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,6 +5,7 @@ import { snapshotEnvironment } from "./process/snapshotEnvironment.js"; import { createDirectAnalysis } from "./composition/directAnalysis.js"; import { registerEvmCommands } from "./cli/evmCommands.js"; import { registerGoCommands } from "./cli/goCommands.js"; +import { registerRustCommands } from "./cli/rustCommands.js"; import { registerSqliteCommands } from "./cli/sqliteCommands.js"; import { registerAnalysisViewCommands } from "./cli/analysisViewCommands.js"; import { registerBinaryDiagnosticsCommands } from "./cli/binaryDiagnosticsCommands.js"; @@ -101,6 +102,7 @@ export const createCli = ( registerSignalCommands(cli, logger, environment); registerEvmCommands(cli, logger, environment); registerGoCommands(cli, logger); + registerRustCommands(cli, logger); registerSqliteCommands(cli, logger, environment); registerArtifactCommands(cli, logger, analysis.runProviderAnalysis); registerManagedCommands(cli, logger, analysis.runProviderAnalysis); diff --git a/src/cli/rustCommands.ts b/src/cli/rustCommands.ts new file mode 100644 index 000000000..fc3f8ee85 --- /dev/null +++ b/src/cli/rustCommands.ts @@ -0,0 +1,34 @@ +import { z } from "incur"; +import { resolve } from "node:path"; +import { createRustBinaryService } from "../composition/rust.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { logCliCommand } from "../cliLogging.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { Logger } from "pino"; +import type { CliInstance } from "./types.js"; +import { withCommandCancellation } from "./commandCancellation.js"; + +/** Resolve caller paths and inspect embedded Rust dependencies through the shared workflow. */ +export const registerRustCommands = ( + cli: CliInstance, + logger: Logger, +): void => { + const service = createRustBinaryService(); + cli.command(CLI_COMMANDS.inspectRustBinary, { + description: + "Inspect an embedded cargo-auditable Rust dependency list without target execution", + args: z.object({ + path: z.string().describe("Local ELF, PE or thin Mach-O image"), + }), + run: ({ args }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectRustBinary, async () => { + const result = await service.inspect( + { path: resolve(args.path) }, + { signal }, + ); + return result.ok ? result.value : projectAnalysisError(result.error); + }), + ), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index ff208c62e..70a3a4d37 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -50,6 +50,7 @@ export const CLI_COMMANDS = Object.freeze({ inspectRecordedCrash: "inspect-recorded-crash", inspectBinaryLayout: "inspect-binary-layout", inspectGoBinary: "inspect-go-binary", + inspectRustBinary: "inspect-rust-binary", inspectPeResources: "inspect-pe-resources", inspectAnalysisView: "inspect-analysis-view", listFridaDevices: "list-frida-devices", diff --git a/src/composition/rust.ts b/src/composition/rust.ts new file mode 100644 index 000000000..5a05eceb3 --- /dev/null +++ b/src/composition/rust.ts @@ -0,0 +1,6 @@ +import { RustBinaryService } from "../application/rust/RustBinaryService.js"; +import { RustBinaryProvider } from "../rust/RustBinaryProvider.js"; + +/** Compose bundled offline inspection without a Rust toolchain or any child process. */ +export const createRustBinaryService = (): RustBinaryService => + new RustBinaryService(new RustBinaryProvider()); diff --git a/src/contracts/rust/rustToolContracts.ts b/src/contracts/rust/rustToolContracts.ts new file mode 100644 index 000000000..7f827dfbe --- /dev/null +++ b/src/contracts/rust/rustToolContracts.ts @@ -0,0 +1,26 @@ +import { + inspectRustBinaryInputSchema, + rustBinarySchema, +} from "../../domain/rust/rustBinary.js"; +import type { ToolContract } from "../toolContractTypes.js"; +import { toolContractMetadata } from "../toolEffects.js"; +import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; + +/** Static Rust dependency metadata is independent of active native-analysis providers. */ +export const RUST_TOOL_CONTRACTS = [ + { + name: "inspect_rust_binary", + ...toolContractMetadata("inspect_rust_binary"), + kind: "artifact-provider", + description: + "Inspect the cargo-auditable crate dependency list embedded in an explicit local ELF, PE or thin Mach-O image. Returns the artifact SHA-256, the .dep-v0 section file offset and size, compressed and uncompressed sizes, format version, packages in their original order, the root package index, and dependency edges by index. Reads zlib-compressed JSON without installing Rust, opening a disassembler, executing the target, writing temporary files or accessing the network. A missing section is null metadata with a limitation and is not proof that the binary has no dependencies. A rustc commit hash is reported only as an inference when a /rustc/<40-hex>/ path is present in read-only data. Resource guards bound files to 256 MiB, dependency sections to 1 MiB, inflated JSON to the smaller of 8 MiB and 64 times the compressed size, and the complete report to 8 MiB. Malformed, unsupported and oversized inputs receive typed failures rather than partial success.", + inputSchema: inspectRustBinaryInputSchema, + outputSchema: evidenceResultOf(rustBinarySchema), + examples: [ + { + title: "Inspect an embedded Rust dependency list", + input: { path: "/artifacts/application" }, + }, + ], + }, +] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index 2fdac1fbe..39b65b3d2 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -3,6 +3,7 @@ import { SIGNAL_TOOL_CONTRACTS } from "./signals/signalToolContracts.js"; import { WASM_TOOL_CONTRACTS } from "./wasm/wasmToolContracts.js"; import { EVM_TOOL_CONTRACTS } from "./evm/evmToolContracts.js"; import { GO_TOOL_CONTRACTS } from "./go/goToolContracts.js"; +import { RUST_TOOL_CONTRACTS } from "./rust/rustToolContracts.js"; import { SQLITE_TOOL_CONTRACTS } from "./sqlite/sqliteToolContracts.js"; import { ANALYSIS_VIEW_TOOL_CONTRACTS } from "./analysisViewToolContracts.js"; import { BINARY_DIAGNOSTICS_TOOL_CONTRACTS } from "./native/binaryDiagnosticsToolContracts.js"; @@ -40,6 +41,7 @@ export const TOOL_CONTRACTS = [ ...ARTIFACT_TOOL_CONTRACTS, ...EVM_TOOL_CONTRACTS, ...GO_TOOL_CONTRACTS, + ...RUST_TOOL_CONTRACTS, ...WASM_TOOL_CONTRACTS, ...SQLITE_TOOL_CONTRACTS, ...CAPTURE_TOOL_CONTRACTS, diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts index 3a4acca99..adf4d0932 100644 --- a/src/contracts/toolEffects.test.ts +++ b/src/contracts/toolEffects.test.ts @@ -19,6 +19,8 @@ const EXECUTION_WORDING_EXCEPTIONS: Readonly> = { resolve_native_call_targets: "states that it does not execute code", inspect_pe_resources: "states that it does not execute the image", inspect_go_binary: "reads embedded metadata without executing the target", + inspect_rust_binary: + "reads an embedded dependency list without executing the target", inspect_sqlite_database: "uses a trusted inspector worker without executing application code or caller SQL", inspect_binary_layout: diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 377905480..137e54801 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -54,6 +54,7 @@ const sessionEvidence = effects({ mutatesSession: true }); /** Explicit effect audit for every public tool. */ export const TOOL_EFFECTS: Readonly> = { inspect_go_binary: evidence, + inspect_rust_binary: evidence, inspect_sqlite_database: effects({ mutatesSession: true, writesFilesystem: true, diff --git a/src/domain/rust/rustBinary.ts b/src/domain/rust/rustBinary.ts new file mode 100644 index 000000000..f09cde222 --- /dev/null +++ b/src/domain/rust/rustBinary.ts @@ -0,0 +1,294 @@ +import { z } from "zod"; +import { digestSchema } from "../digests.js"; +import { localPathStringSchema } from "../localPath.js"; + +export const RUST_AUDITABLE_FORMAT_VERSION = 1; + +const index = z.number().int().nonnegative(); +const rustcHash = z.string().regex(/^[0-9a-f]{40}$/u); + +/** One embedded package, with omitted kind and root preserved as null and false. */ +export const rustPackageSchema = z.strictObject({ + name: z.string().min(1), + version: z.string().min(1), + source: z.string().min(1), + kind: z.enum(["runtime", "build"]).nullable(), + dependencies: z.array(index), + root: z.boolean(), +}); + +const dependencyEdgeSchema = z.strictObject({ + from_index: index, + to_index: index, +}); + +/** zlib JSON from one `.dep-v0` section, in the section's package order. */ +export const rustAuditableMetadataSchema = z.strictObject({ + section_name: z.enum([".dep-v0", "__DATA,.dep-v0"]), + file_offset: index, + section_bytes: z.number().int().positive(), + compressed_bytes: z.number().int().positive(), + uncompressed_bytes: z.number().int().positive(), + format_version: z.literal(RUST_AUDITABLE_FORMAT_VERSION).nullable(), + packages: z.array(rustPackageSchema), + root_package_index: index.nullable(), + dependency_edges: z.array(dependencyEdgeSchema), + cyclic_dependency_indices: z.array(index), +}); + +/** Inferred compiler id from read-only `/rustc/<40-hex>/` paths. */ +export const rustcCommitSchema = z.strictObject({ + evidence_kind: z.literal("inferred"), + hashes: z + .array( + z.strictObject({ + hash: rustcHash, + file_offset: index, + }), + ) + .min(1), +}); + +/** One explicit local executable or library, independent of a disassembler target. */ +export const inspectRustBinaryInputSchema = z.strictObject({ + path: localPathStringSchema.describe( + "Absolute filesystem path to the selected executable or library", + ), +}); + +/** Bounded cargo-auditable observation. A missing section is null, not an empty dependency list. */ +export const rustBinarySchema = z + .strictObject({ + artifact: z.strictObject({ + path: localPathStringSchema, + sha256: digestSchema, + bytes: index, + }), + format: z.enum(["elf", "pe", "macho"]), + architecture: z.string(), + bits: z.union([z.literal(32), z.literal(64)]), + byte_order: z.enum(["little", "big"]), + auditable_metadata: rustAuditableMetadataSchema.nullable(), + rustc_commit: rustcCommitSchema.nullable(), + limitations: z.array(z.string()), + }) + .superRefine((report, context) => { + const metadata = report.auditable_metadata; + if (metadata !== null) { + checkSection(report.artifact.bytes, metadata, context); + checkPackages(metadata, context); + } + checkRustc(report.artifact.bytes, report.rustc_commit, context); + }); + +export type RustPackage = z.infer; +export type RustAuditableMetadata = z.infer; +export type RustBinary = z.infer; +export type InspectRustBinaryInput = z.infer< + typeof inspectRustBinaryInputSchema +>; + +const issue = ( + context: z.RefinementCtx, + path: (string | number)[], + message: string, +): void => { + context.addIssue({ code: "custom", path, message }); +}; + +const inside = (fileBytes: number, offset: number, size: number): boolean => + offset <= fileBytes && size <= fileBytes - offset; + +const checkSection = ( + fileBytes: number, + metadata: RustAuditableMetadata, + context: z.RefinementCtx, +): void => { + if ( + !inside(fileBytes, metadata.file_offset, metadata.section_bytes) || + metadata.compressed_bytes !== metadata.section_bytes + ) + issue( + context, + ["auditable_metadata", "file_offset"], + "Dependency section range is outside the selected artifact or disagrees with its compressed size.", + ); +}; + +const checkPackages = ( + metadata: RustAuditableMetadata, + context: z.RefinementCtx, +): void => { + const roots: number[] = []; + const edgesByPackage = new Map< + number, + z.infer[] + >(); + for (const [position, edge] of metadata.dependency_edges.entries()) { + if ( + edge.from_index >= metadata.packages.length || + edge.to_index >= metadata.packages.length + ) { + issue( + context, + ["auditable_metadata", "dependency_edges", position], + "Dependency edge index is outside the package list.", + ); + continue; + } + const grouped = edgesByPackage.get(edge.from_index); + if (grouped === undefined) edgesByPackage.set(edge.from_index, [edge]); + else grouped.push(edge); + } + for (const [packageIndex, pkg] of metadata.packages.entries()) { + if (pkg.root) roots.push(packageIndex); + const edges = edgesByPackage.get(packageIndex) ?? []; + if ( + edges.length !== pkg.dependencies.length || + edges.some((edge, index) => edge.to_index !== pkg.dependencies[index]) + ) + issue( + context, + ["auditable_metadata", "dependency_edges"], + "Dependency edges do not match package dependency indices.", + ); + if (pkg.dependencies.some((target) => target >= metadata.packages.length)) + issue( + context, + ["auditable_metadata", "packages", packageIndex, "dependencies"], + "Dependency index is outside the package list.", + ); + } + const root = metadata.root_package_index; + if ( + root === null ? roots.length !== 0 : roots.length !== 1 || roots[0] !== root + ) + issue( + context, + ["auditable_metadata", "root_package_index"], + "Root package index does not match packages marked root.", + ); + const cycles = cyclicDependencyIndices(metadata.packages); + if ( + cycles.length !== metadata.cyclic_dependency_indices.length || + cycles.some( + (index, position) => + metadata.cyclic_dependency_indices[position] !== index, + ) + ) + issue( + context, + ["auditable_metadata", "cyclic_dependency_indices"], + "Cyclic dependency indices do not match the cycles in package dependencies.", + ); +}; + +/** Package indices that participate in a dependency cycle, in ascending order. */ +export const cyclicDependencyIndices = ( + packages: readonly { readonly dependencies: readonly number[] }[], +): number[] => { + const edges = packages.map((pkg) => pkg.dependencies); + const order = dependencyPostorder(edges); + const reversed = reversedDependencyEdges(edges); + const seen = new Uint8Array(edges.length); + const cyclic = new Set(); + for (let cursor = order.length - 1; cursor >= 0; cursor -= 1) { + const start = order[cursor]; + if (start === undefined || seen[start] === 1) continue; + const component = dependencyComponent(start, reversed, seen); + if (dependencyComponentInCycle(component, edges)) + for (const index of component) cyclic.add(index); + } + return [...cyclic].sort((left, right) => left - right); +}; + +interface DependencyWalkFrame { + readonly index: number; + edge: number; +} + +const dependencyPostorder = ( + edges: readonly (readonly number[])[], +): number[] => { + const seen = new Uint8Array(edges.length); + const order: number[] = []; + for (let start = 0; start < edges.length; start += 1) { + if (seen[start] === 1) continue; + const stack: DependencyWalkFrame[] = [{ index: start, edge: 0 }]; + seen[start] = 1; + while (stack.length > 0) { + const frame = stack[stack.length - 1]; + if (frame === undefined) break; + const next = edges[frame.index]?.[frame.edge]; + if (next === undefined) { + stack.pop(); + order.push(frame.index); + continue; + } + frame.edge += 1; + if (seen[next] === 0) { + seen[next] = 1; + stack.push({ index: next, edge: 0 }); + } + } + } + return order; +}; + +const reversedDependencyEdges = ( + edges: readonly (readonly number[])[], +): number[][] => { + const reversed = Array.from({ length: edges.length }, () => [] as number[]); + for (let from = 0; from < edges.length; from += 1) + for (const to of edges[from] ?? []) reversed[to]?.push(from); + return reversed; +}; + +const dependencyComponent = ( + start: number, + reversed: readonly (readonly number[])[], + seen: Uint8Array, +): number[] => { + const component: number[] = []; + const stack = [start]; + seen[start] = 1; + while (stack.length > 0) { + const index = stack.pop(); + if (index === undefined) break; + component.push(index); + for (const next of reversed[index] ?? []) { + if (seen[next] === 0) { + seen[next] = 1; + stack.push(next); + } + } + } + return component; +}; + +const dependencyComponentInCycle = ( + component: readonly number[], + edges: readonly (readonly number[])[], +): boolean => { + if (component.length > 1) return true; + const only = component[0]; + return only !== undefined && (edges[only]?.includes(only) ?? false); +}; + +const checkRustc = ( + fileBytes: number, + commit: RustBinary["rustc_commit"], + context: z.RefinementCtx, +): void => { + if (commit === null) return; + const seen = new Set(); + for (const [position, entry] of commit.hashes.entries()) { + if (seen.has(entry.hash) || !inside(fileBytes, entry.file_offset, 48)) + issue( + context, + ["rustc_commit", "hashes", position], + "rustc path hash is duplicated or its file offset is outside the artifact.", + ); + seen.add(entry.hash); + } +}; diff --git a/src/go/GoBinaryContainer.ts b/src/go/GoBinaryContainer.ts index 56fcb2b74..96b4cb146 100644 --- a/src/go/GoBinaryContainer.ts +++ b/src/go/GoBinaryContainer.ts @@ -34,6 +34,16 @@ export interface GoZeroFillMapping { readonly size: bigint; } +/** One named file section. Invalid declared ranges stay visible with `file_backed: false`. */ +export interface GoImageSection { + readonly name: string; + readonly offset: number; + readonly size: number; + readonly file_backed: boolean; + readonly read_only: boolean; + readonly compressed: boolean; +} + /** Native container facts and its producer-defined build-info search area. */ export interface GoBinaryContainer { readonly format: "elf" | "pe" | "macho"; @@ -42,6 +52,7 @@ export interface GoBinaryContainer { readonly byte_order: "little" | "big"; readonly mappings: readonly GoFileMapping[]; readonly zero_fills: readonly GoZeroFillMapping[]; + readonly sections: readonly GoImageSection[]; readonly search: GoFileMapping | null; } @@ -110,6 +121,32 @@ export class GoBinaryReader { } } +/** Admit a declared file span, or mark it unbacked when it leaves the file. */ +export const declaredFileRange = ( + fileBytes: number, + offset: bigint, + size: bigint, +): { + readonly offset: number; + readonly size: number; + readonly file_backed: boolean; +} => { + if ( + offset < 0n || + size < 0n || + offset > BigInt(Number.MAX_SAFE_INTEGER) || + size > BigInt(Number.MAX_SAFE_INTEGER) || + offset > BigInt(fileBytes) || + size > BigInt(fileBytes) - offset + ) + return { offset: 0, size: 0, file_backed: false }; + return { + offset: Number(offset), + size: Number(size), + file_backed: size > 0n, + }; +}; + /** Map a virtual span without losing precision or selecting an ambiguous alias. */ export const mappedFileOffset = ( mappings: readonly GoFileMapping[], diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index 4a845c197..e94a80ab6 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -55,7 +55,7 @@ export interface GoBinaryImage { /** Inspect bounded executable bytes using native mappings, without loading or executing code. */ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { - const container = readContainer(bytes); + const container = readGoBinaryContainer(bytes); const { format, architecture, bits, byte_order } = container; const search = container.search; const header = findHeader(bytes, search); @@ -184,7 +184,8 @@ interface LocatedString { readonly location: GoStringLocation; } -const readContainer = (bytes: Buffer): GoBinaryContainer => { +/** Identify an ELF, PE, or thin Mach-O image and its named file sections. */ +export const readGoBinaryContainer = (bytes: Buffer): GoBinaryContainer => { if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return readGoElfImage(bytes); if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) diff --git a/src/go/GoElfImage.ts b/src/go/GoElfImage.ts index fae00c40c..de4388114 100644 --- a/src/go/GoElfImage.ts +++ b/src/go/GoElfImage.ts @@ -1,9 +1,11 @@ import { + declaredFileRange, GoBinaryFormatFailure, GoBinaryReader, mappedFileOffset, type GoBinaryContainer, type GoFileMapping, + type GoImageSection, type GoZeroFillMapping, } from "./GoBinaryContainer.js"; import { readElfFixedHeader } from "../domain/elfFixedHeader.js"; @@ -100,6 +102,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { ); const mappings: GoFileMapping[] = []; const zeroFills: GoZeroFillMapping[] = []; + const sections: GoImageSection[] = []; let fallback: GoFileMapping | null = null; for (let index = 0; index < phCount; index++) { const offset = phStart + index * phWidth; @@ -158,6 +161,15 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { "malformed", "ELF section name offset is outside its string table", ); + recordElfSection(sections, { + bytes, + names, + nameOffset, + reader, + bits, + wide, + header: offset, + }); // Comparing the complete fixed name avoids quadratic scans through malicious string tables. const expectedName = Buffer.from(".go.buildinfo\0"); if ( @@ -213,10 +225,105 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { byte_order: reader.little ? "little" : "big", mappings, zero_fills: zeroFills, + sections, search: search ?? fallback, }; }; +const recordElfSection = ( + sections: GoImageSection[], + input: { + readonly bytes: Buffer; + readonly names: { readonly offset: number; readonly size: number }; + readonly nameOffset: number; + readonly reader: GoBinaryReader; + readonly bits: 32 | 64; + readonly wide: boolean; + readonly header: number; + }, +): void => { + const decoded = elfSectionName(input.bytes, input.names, input.nameOffset); + const header = { + reader: input.reader, + bits: input.bits, + wide: input.wide, + header: input.header, + }; + switch (decoded.status) { + case "named": + sections.push(elfImageSection({ ...header, name: decoded.name })); + return; + case "truncated": { + // Names past this window stay out of .dep-v0 matching. Read-only + // bytes can still hold the only compiler path. + const section = elfImageSection({ ...header, name: "" }); + if (section.read_only) sections.push(section); + return; + } + case "unnamed": + return; + default: { + const unreachable: never = decoded; + return unreachable; + } + } +}; + +const elfSectionName = ( + bytes: Buffer, + names: { readonly offset: number; readonly size: number }, + nameOffset: number, +): + | { readonly status: "named"; readonly name: string } + | { readonly status: "unnamed" } + | { readonly status: "truncated" } => { + const start = names.offset + nameOffset; + const limit = Math.min(start + 64, names.offset + names.size); + const nul = bytes.subarray(start, limit).indexOf(0); + if (nul === 0) return { status: "unnamed" }; + if (nul < 0) return { status: "truncated" }; + return { + status: "named", + name: bytes.toString("latin1", start, start + nul), + }; +}; + +const elfImageSection = ({ + reader, + bits, + wide, + header, + name, +}: { + readonly reader: GoBinaryReader; + readonly bits: 32 | 64; + readonly wide: boolean; + readonly header: number; + readonly name: string; +}): GoImageSection => { + const type = reader.u32(header + 4); + const flags = reader.word(header + 8, bits); + const declared = declaredFileRange( + reader.bytes.length, + reader.word(header + (wide ? 24 : 16), bits), + reader.word(header + (wide ? 32 : 20), bits), + ); + // SHT_NOBITS offsets and sizes do not describe stored bytes. + const range = + type === SHT_NOBITS ? { ...declared, file_backed: false } : declared; + const compressed = (flags & 0x800n) !== 0n; + const readOnly = + type === 1 && + range.file_backed && + !compressed && + (flags & 1n) === 0n && + (flags & 4n) === 0n && + (flags & 2n) !== 0n; + return { name, ...range, read_only: readOnly, compressed }; +}; + +const SHT_NOBITS = 8; + const elfArchitecture = (machine: number): string => { const names = new Map([ [3, "386"], diff --git a/src/go/GoMachoImage.ts b/src/go/GoMachoImage.ts index a178a0616..5184a5829 100644 --- a/src/go/GoMachoImage.ts +++ b/src/go/GoMachoImage.ts @@ -1,9 +1,11 @@ import { + declaredFileRange, GoBinaryFormatFailure, GoBinaryReader, mappedFileOffset, type GoBinaryContainer, type GoFileMapping, + type GoImageSection, type GoZeroFillMapping, } from "./GoBinaryContainer.js"; @@ -30,6 +32,7 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { ); const mappings: GoFileMapping[] = []; const zeroFills: GoZeroFillMapping[] = []; + const sections: GoImageSection[] = []; const named: GoFileMapping[] = []; let fallback: GoFileMapping | null = null; let offset = commandStart; @@ -105,11 +108,25 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { fallback = mapping; for (let sectionIndex = 0; sectionIndex < sectionCount; sectionIndex++) { const section = offset + segmentHeader + sectionIndex * sectionWidth; - if ( - fixedName(reader.range(section, 16, "Mach-O section name")) !== - "__go_buildinfo" - ) - continue; + const sectionName = fixedName( + reader.range(section, 16, "Mach-O section name"), + ); + sections.push( + machoImageSection({ + reader, + bits, + wide, + header: section, + name: `${fixedName(reader.range(section + 16, 16, "Mach-O section segment"))},${sectionName}`, + protection, + unmapped: unmapped || mapping === undefined, + segmentAddress: address, + segmentMemorySize: memorySize, + segmentFileOffset: fileOffset, + segmentFileSize: fileSize, + }), + ); + if (sectionName !== "__go_buildinfo") continue; if (unmapped) throw new GoBinaryFormatFailure( "malformed", @@ -175,10 +192,74 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { byte_order: little ? "little" : "big", mappings, zero_fills: zeroFills, + sections, search: named[0] ?? fallback, }; }; +const machoImageSection = ({ + reader, + bits, + wide, + header, + name, + protection, + unmapped, + segmentAddress, + segmentMemorySize, + segmentFileOffset, + segmentFileSize, +}: { + readonly reader: GoBinaryReader; + readonly bits: 32 | 64; + readonly wide: boolean; + readonly header: number; + readonly name: string; + readonly protection: number; + readonly unmapped: boolean; + readonly segmentAddress: bigint; + readonly segmentMemorySize: bigint; + readonly segmentFileOffset: bigint; + readonly segmentFileSize: bigint; +}): GoImageSection => { + const flags = reader.u32(header + (wide ? 64 : 56)); + const zeroFill = [1, 0xc, 0x12].includes(flags & 0xff); + const sectionAddress = reader.word(header + 32, bits); + const sectionSize = reader.word(header + (wide ? 40 : 36), bits); + const sectionOffset = BigInt(reader.u32(header + (wide ? 48 : 40))); + const addressDelta = sectionAddress - segmentAddress; + const fileDelta = sectionOffset - segmentFileOffset; + const inSegmentMemory = + addressDelta >= 0n && + addressDelta <= segmentMemorySize && + sectionSize <= segmentMemorySize - addressDelta; + const inSegmentFile = + fileDelta >= 0n && + fileDelta <= segmentFileSize && + sectionSize <= segmentFileSize - fileDelta; + const range = unmapped + ? { offset: 0, size: 0, file_backed: false } + : declaredFileRange( + reader.bytes.length, + BigInt(reader.u32(header + (wide ? 48 : 40))), + reader.word(header + (wide ? 40 : 36), bits), + ); + const fileBacked = + range.file_backed && !zeroFill && inSegmentMemory && inSegmentFile; + const readOnly = + fileBacked && + (protection & 1) !== 0 && + (protection & 2) === 0 && + (flags & 0x80000400) === 0; + return { + name, + ...range, + file_backed: fileBacked, + read_only: readOnly, + compressed: false, + }; +}; + const fixedName = (bytes: Buffer): string => { const nul = bytes.indexOf(0); return bytes.subarray(0, nul < 0 ? bytes.length : nul).toString("latin1"); diff --git a/src/go/GoPeImage.ts b/src/go/GoPeImage.ts index 3d7ad95a8..0787bc93d 100644 --- a/src/go/GoPeImage.ts +++ b/src/go/GoPeImage.ts @@ -1,9 +1,11 @@ import { observeMzWindowsHeader, parseDosMzHeader } from "../domain/dosMz.js"; import { + declaredFileRange, GoBinaryFormatFailure, GoBinaryReader, type GoBinaryContainer, type GoFileMapping, + type GoImageSection, type GoZeroFillMapping, } from "./GoBinaryContainer.js"; @@ -86,14 +88,30 @@ export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { ); const mappings: GoFileMapping[] = []; const zeroFills: GoZeroFillMapping[] = []; + const sections: GoImageSection[] = []; let search: GoFileMapping | null = null; for (let index = 0; index < count; index++) { const entry = start + index * 40; + const sectionName = peSectionName( + reader.range(entry, 8, "PE section name"), + ); const virtualSize = reader.u32(entry + 8); const virtualAddress = reader.u32(entry + 12); const size = reader.u32(entry + 16); const offset = reader.u32(entry + 20); const characteristics = reader.u32(entry + 36); + // SizeOfRawData includes file-alignment padding. VirtualSize is the content. + const contentSize = virtualSize === 0 ? size : Math.min(virtualSize, size); + if (sectionName.length > 0) + sections.push( + peImageSection({ + reader, + name: sectionName, + size: contentSize, + offset, + characteristics, + }), + ); const address = imageBase + BigInt(virtualAddress); if (address + BigInt(Math.max(size, virtualSize)) > 1n << BigInt(bits)) throw new GoBinaryFormatFailure( @@ -138,6 +156,39 @@ export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { byte_order: "little", mappings, zero_fills: zeroFills, + sections, search, }; }; + +const peSectionName = (bytes: Buffer): string => { + const nul = bytes.indexOf(0); + return bytes.toString("latin1", 0, nul < 0 ? bytes.length : nul); +}; + +const peImageSection = ({ + reader, + name, + size, + offset, + characteristics, +}: { + readonly reader: GoBinaryReader; + readonly name: string; + readonly size: number; + readonly offset: number; + readonly characteristics: number; +}): GoImageSection => { + const range = declaredFileRange( + reader.bytes.length, + BigInt(offset), + BigInt(size), + ); + const readOnly = + range.file_backed && + (characteristics & 0x40000000) !== 0 && + (characteristics & 0x80000000) === 0 && + (characteristics & 0x20000000) === 0 && + (characteristics & 0x40) !== 0; + return { name, ...range, read_only: readOnly, compressed: false }; +}; diff --git a/src/rust/RustAuditableReader.test.ts b/src/rust/RustAuditableReader.test.ts new file mode 100644 index 000000000..cb281185a --- /dev/null +++ b/src/rust/RustAuditableReader.test.ts @@ -0,0 +1,221 @@ +import { deflateSync } from "node:zlib"; +import { expect, it } from "vitest"; +import { + RUST_SECTION_BYTES, + readAuditableSection, + RustAuditableFormatFailure, + RustAuditableResourceFailure, +} from "./RustAuditableReader.js"; + +const section = (document: unknown) => + readAuditableSection( + ".dep-v0", + 0, + deflateSync(Buffer.from(JSON.stringify(document))), + ); + +it("preserves package order, omitted kind, and the root index", () => { + const parsed = section({ + packages: [ + { name: "itoa", version: "1.0.18", source: "crates.io" }, + { + name: "sample", + version: "0.1.0", + source: "local", + dependencies: [0], + root: true, + kind: "runtime", + }, + ], + format: 1, + }); + expect(parsed).toMatchObject({ + section_name: ".dep-v0", + format_version: 1, + root_package_index: 1, + compressed_bytes: parsed.section_bytes, + packages: [ + { + name: "itoa", + version: "1.0.18", + source: "crates.io", + kind: null, + dependencies: [], + root: false, + }, + { + name: "sample", + version: "0.1.0", + kind: "runtime", + dependencies: [0], + root: true, + }, + ], + dependency_edges: [{ from_index: 1, to_index: 0 }], + cyclic_dependency_indices: [], + }); + expect(parsed.uncompressed_bytes).toBeGreaterThan(parsed.compressed_bytes); +}); + +it("reports dependency cycles and keeps an omitted format version unknown", () => { + const parsed = section({ + packages: [ + { name: "a", version: "1", source: "local", dependencies: [1] }, + { name: "b", version: "1", source: "local", dependencies: [0] }, + ], + }); + expect(parsed.cyclic_dependency_indices).toEqual([0, 1]); + expect(parsed.format_version).toBeNull(); + expect(parsed.root_package_index).toBeNull(); +}); + +it("includes every package that participates in a dependency cycle", () => { + const parsed = section({ + packages: [ + { name: "a", version: "1", source: "local", dependencies: [1, 2] }, + { name: "b", version: "1", source: "local", dependencies: [0] }, + { name: "c", version: "1", source: "local", dependencies: [1] }, + ], + }); + expect(parsed.cyclic_dependency_indices).toEqual([0, 1, 2]); + const self = section({ + packages: [{ name: "a", version: "1", source: "local", dependencies: [0] }], + }); + expect(self.cyclic_dependency_indices).toEqual([0]); +}); + +it("finishes a long acyclic dependency chain", () => { + const count = 20_000; + const parsed = section({ + packages: Array.from({ length: count }, (_, index) => ({ + name: `p${index.toString(36)}`, + version: "1", + source: "local", + ...(index + 1 < count ? { dependencies: [index + 1] } : {}), + })), + }); + expect(parsed.packages).toHaveLength(count); + expect(parsed.cyclic_dependency_indices).toEqual([]); +}); + +it("rejects out-of-range indices, multiple roots, and schema drift", () => { + expect(() => + section({ + packages: [ + { name: "a", version: "1", source: "local", dependencies: [2] }, + ], + }), + ).toThrow(RustAuditableFormatFailure); + expect(() => + section({ + packages: [ + { name: "a", version: "1", source: "local", root: true }, + { name: "b", version: "1", source: "local", root: true }, + ], + }), + ).toThrow(/more than one root/); + expect(() => + section({ + packages: [{ name: "a", version: "1", source: "local" }], + extra: true, + }), + ).toThrow(/package schema/); +}); + +it("rejects invalid JSON, trailing zlib bytes, and an empty section", () => { + expect(() => + readAuditableSection(".dep-v0", 0, deflateSync(Buffer.from("{"))), + ).toThrow(/JSON is invalid/); + const trailed = Buffer.concat([ + deflateSync(Buffer.from('{"packages":[]}')), + Buffer.from([0]), + ]); + expect(() => readAuditableSection(".dep-v0", 0, trailed)).toThrow( + /zlib stream/, + ); + expect(() => readAuditableSection(".dep-v0", 0, Buffer.alloc(0))).toThrow( + /empty/, + ); + expect(() => + readAuditableSection(".dep-v0", 0, Buffer.from([1, 2, 3])), + ).toThrow(RustAuditableFormatFailure); +}); + +it("stops an inflate that exceeds the section ratio", () => { + const raw = Buffer.alloc(8000); + const compressed = deflateSync(raw); + expect(compressed.length * 64).toBeLessThan(raw.length); + expect(() => readAuditableSection(".dep-v0", 0, compressed)).toThrow( + RustAuditableResourceFailure, + ); + expect(() => + readAuditableSection(".dep-v0", 0, Buffer.alloc(RUST_SECTION_BYTES + 1)), + ).toThrow(/1 MiB/); +}); + +it.each([{ packages: [] }, { packages: [{ future_package_field: true }] }])( + "rejects an unknown format before interpreting its packages: %j", + ({ packages }) => { + expect(() => section({ format: 99, packages })).toThrow( + expect.objectContaining({ + name: "RustAuditableFormatFailure", + kind: "unsupported", + message: "Unsupported cargo-auditable format version 99", + }), + ); + }, +); + +it("rejects the first malformed package before schema issues accumulate", () => { + const stored = (document: unknown) => + deflateSync(Buffer.from(JSON.stringify(document)), { level: 0 }); + const emptyPackages = stored({ + format: 1, + packages: Array.from({ length: 80_000 }, () => ({})), + }); + expect(emptyPackages.length).toBeLessThan(RUST_SECTION_BYTES); + expect(() => readAuditableSection(".dep-v0", 0, emptyPackages)).toThrow( + /Dependency package 0 does not match the package schema/, + ); + const invalidDependencies = stored({ + format: 1, + packages: [ + { + name: "a", + version: "1", + source: "local", + dependencies: Array.from({ length: 1_000 }, () => null), + }, + ], + }); + expect(() => readAuditableSection(".dep-v0", 0, invalidDependencies)).toThrow( + /Dependency package 0 does not match the package schema/, + ); +}); + +it("admits dependency expansion before allocating normalized or reverse edges", () => { + const document = { + packages: [ + { + name: "a", + version: "1", + source: "local", + dependencies: Array(400_000).fill(0), + }, + ], + }; + // An uncompressed zlib stream clears the ratio guard while remaining below + // the one-MiB section limit, isolating the graph representation admission. + const payload = deflateSync(Buffer.from(JSON.stringify(document)), { + level: 0, + }); + expect(payload.length).toBeLessThan(RUST_SECTION_BYTES); + expect(() => readAuditableSection(".dep-v0", 0, payload)).toThrow( + expect.objectContaining({ + constraint: { + resource: "transport", + reported_limits: { maximum_output_bytes: 8 * 1024 * 1024 }, + }, + }), + ); +}); diff --git a/src/rust/RustAuditableReader.ts b/src/rust/RustAuditableReader.ts new file mode 100644 index 000000000..d28b95704 --- /dev/null +++ b/src/rust/RustAuditableReader.ts @@ -0,0 +1,440 @@ +import { deflateSync, inflateSync } from "node:zlib"; +import { z } from "zod"; +import { + RUST_AUDITABLE_FORMAT_VERSION, + cyclicDependencyIndices, + type RustAuditableMetadata, + type RustPackage, +} from "../domain/rust/rustBinary.js"; + +/** Selected-file admission limit shared with the stable artifact reader. */ +export const RUST_BINARY_INPUT_BYTES = 256 * 1024 * 1024; +/** Compressed `.dep-v0` section limit. */ +export const RUST_SECTION_BYTES = 1024 * 1024; +/** Inflated JSON may grow by at most this multiple of the section size. */ +export const RUST_INFLATE_RATIO = 64; +/** Absolute inflated JSON limit, applied together with the ratio. */ +export const RUST_UNCOMPRESSED_BYTES = 8 * 1024 * 1024; +/** Complete normalized report limit. */ +export const RUST_OUTPUT_BYTES = 8 * 1024 * 1024; + +const JSON_DEPTH = 32; +const packageSchema = z.strictObject({ + name: z.string().min(1), + version: z.string().min(1), + source: z.string().min(1), + kind: z.enum(["runtime", "build"]).optional(), + dependencies: z.array(z.number().int().nonnegative()).optional(), + root: z.boolean().optional(), +}); +const documentVersionSchema = z.object({ + format: z.number().int().nonnegative().optional(), +}); +const documentSchema = documentVersionSchema + .extend({ + format: z.literal(RUST_AUDITABLE_FORMAT_VERSION).optional(), + packages: z.array(packageSchema), + }) + .strict(); + +/** Malformed or unsupported dependency-section bytes. */ +export class RustAuditableFormatFailure extends Error { + constructor( + readonly kind: "malformed" | "unsupported", + message: string, + options?: ErrorOptions, + ) { + super(message, options); + this.name = "RustAuditableFormatFailure"; + } +} + +/** A resource admission limit with explicit semantic units. */ +export type RustAuditableConstraint = + | { + readonly resource: "memory"; + readonly reported_limits: { readonly maximum_bytes: number }; + } + | { + readonly resource: "transport"; + readonly reported_limits: { readonly maximum_output_bytes: number }; + } + | { + readonly resource: "capacity"; + readonly reported_limits: { readonly maximum_rustc_hashes: number }; + }; + +/** Admission refused before retained output or graph expansion. */ +export class RustAuditableResourceFailure extends Error { + constructor( + readonly constraint: RustAuditableConstraint, + message: string, + options?: ErrorOptions, + ) { + super(message, options); + this.name = "RustAuditableResourceFailure"; + } +} + +/** Parsed section payload. Cycles are listed and are not walked. */ +export const readAuditableSection = ( + sectionName: RustAuditableMetadata["section_name"], + fileOffset: number, + bytes: Buffer, +): RustAuditableMetadata => { + if (bytes.length === 0) + throw new RustAuditableFormatFailure( + "malformed", + "Dependency section is empty", + ); + if (bytes.length > RUST_SECTION_BYTES) + throw new RustAuditableResourceFailure( + { + resource: "memory", + reported_limits: { maximum_bytes: RUST_SECTION_BYTES }, + }, + "Dependency section exceeds the 1 MiB compressed limit", + ); + const inflated = inflateSection(bytes); + const document = parseDocument(inflated); + const packages = normalizePackages(document.packages); + const root = rootIndex(packages); + return { + section_name: sectionName, + file_offset: fileOffset, + section_bytes: bytes.length, + compressed_bytes: bytes.length, + uncompressed_bytes: inflated.length, + format_version: document.format ?? null, + packages, + root_package_index: root, + dependency_edges: packages.flatMap((pkg, fromIndex) => + pkg.dependencies.map((toIndex) => ({ + from_index: fromIndex, + to_index: toIndex, + })), + ), + cyclic_dependency_indices: cyclicDependencyIndices(packages), + }; +}; + +const inflateSection = (bytes: Buffer): Buffer => { + const maximum = Math.min( + RUST_UNCOMPRESSED_BYTES, + bytes.length * RUST_INFLATE_RATIO, + ); + try { + const inflated = inflateSync(bytes, { + maxOutputLength: maximum, + ...(rejectsTrailingJunk() ? { rejectGarbageAfterEnd: true } : {}), + }); + if ( + !rejectsTrailingJunk() && + shortestPrefix(bytes, inflated, maximum) !== bytes.length + ) + throw new RustAuditableFormatFailure( + "malformed", + "Dependency section has bytes after the zlib stream", + ); + return inflated; + } catch (cause) { + if (cause instanceof RustAuditableFormatFailure) throw cause; + throw inflateFailure(cause, maximum); + } +}; + +let trailingJunkRejected: boolean | undefined; + +const rejectsTrailingJunk = (): boolean => { + if (trailingJunkRejected !== undefined) return trailingJunkRejected; + const trailed = Buffer.concat([ + deflateSync(Buffer.from("a")), + Buffer.from([0]), + ]); + try { + inflateSync(trailed, { rejectGarbageAfterEnd: true }); + trailingJunkRejected = false; + } catch (cause) { + trailingJunkRejected = + cause instanceof Error && + "code" in cause && + cause.code === "ERR_TRAILING_JUNK_AFTER_STREAM_END"; + } + return trailingJunkRejected; +}; + +const shortestPrefix = ( + input: Buffer, + output: Buffer, + maximum: number, +): number => { + let low = 0; + let high = input.length; + while (low < high) { + const mid = Math.floor((low + high) / 2); + if (prefixMatches(input.subarray(0, mid), output, maximum)) high = mid; + else low = mid + 1; + } + return high; +}; + +const prefixMatches = ( + prefix: Buffer, + output: Buffer, + maximum: number, +): boolean => { + try { + return inflateSync(prefix, { maxOutputLength: maximum }).equals(output); + } catch { + return false; + } +}; + +const inflateFailure = (cause: unknown, maximum: number): Error => { + const code = + cause instanceof Error && "code" in cause ? String(cause.code) : ""; + if (code === "ERR_BUFFER_TOO_LARGE") + return new RustAuditableResourceFailure( + { resource: "memory", reported_limits: { maximum_bytes: maximum } }, + "Inflated dependency JSON exceeds the size limit", + ); + if ( + code === "ERR_TRAILING_JUNK_AFTER_STREAM_END" || + code.startsWith("Z_") || + code === "ERR_INVALID_ARG_TYPE" + ) + return new RustAuditableFormatFailure( + "malformed", + "Dependency section is not a complete zlib stream", + ); + return cause instanceof Error ? cause : new Error(String(cause)); +}; + +const parseDocument = (bytes: Buffer): z.infer => { + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (cause) { + throw new RustAuditableFormatFailure( + "malformed", + "Dependency JSON is not valid UTF-8", + { cause }, + ); + } + assertDepth(text); + let value: unknown; + try { + value = JSON.parse(text); + } catch (cause) { + throw new RustAuditableFormatFailure( + "malformed", + "Dependency section JSON is invalid", + { cause }, + ); + } + const version = documentVersionSchema.safeParse(value); + if (!version.success) + throw new RustAuditableFormatFailure( + "malformed", + "Dependency JSON does not match the format schema", + { cause: version.error }, + ); + if ( + version.data.format !== undefined && + version.data.format !== RUST_AUDITABLE_FORMAT_VERSION + ) + throw new RustAuditableFormatFailure( + "unsupported", + `Unsupported cargo-auditable format version ${version.data.format}`, + ); + admitGraphExpansion(value); + // One malformed package must fail before Zod retains an issue for every field. + rejectMalformedPackages(value); + const parsed = documentSchema.safeParse(value); + if (!parsed.success) + throw new RustAuditableFormatFailure( + "malformed", + "Dependency JSON does not match the package schema", + { cause: parsed.error }, + ); + return parsed.data; +}; + +// These are lower bounds on the required normalized representation, not +// guessed per-object heap sizes. Count source arrays before Zod clones them, +// normalization builds edge objects, or SCC analysis creates reverse edges. +const MIN_PACKAGE_OUTPUT_BYTES = JSON.stringify({ + name: "a", + version: "a", + source: "a", + kind: null, + dependencies: [], + root: true, +}).length; +const MIN_EDGE_OUTPUT_BYTES = JSON.stringify({ + from_index: 0, + to_index: 0, +}).length; +const admitGraphExpansion = (value: unknown): void => { + if ( + typeof value !== "object" || + value === null || + !("packages" in value) || + !Array.isArray(value.packages) + ) + return; + const packages: readonly unknown[] = value.packages; + let minimumBytes = + packages.length * MIN_PACKAGE_OUTPUT_BYTES + + Math.max(0, packages.length - 1); + let edges = 0; + const admit = () => { + const edgeBytes = edges * MIN_EDGE_OUTPUT_BYTES + Math.max(0, edges - 1); + if (minimumBytes + edgeBytes > RUST_OUTPUT_BYTES) + throw new RustAuditableResourceFailure( + { + resource: "transport", + reported_limits: { maximum_output_bytes: RUST_OUTPUT_BYTES }, + }, + "Dependency graph exceeds the complete report byte budget before expansion", + ); + }; + admit(); + for (const pkg of packages) { + // Shape and index validity remain owned by documentSchema/rootIndex. + if ( + typeof pkg !== "object" || + pkg === null || + !("dependencies" in pkg) || + !Array.isArray(pkg.dependencies) + ) + continue; + const count = pkg.dependencies.length; + edges += count; + minimumBytes += Math.max(0, count * 2 - 1); + admit(); + } +}; + +const PACKAGE_FIELDS = new Set([ + "name", + "version", + "source", + "kind", + "dependencies", + "root", +]); + +const rejectMalformedPackages = (value: unknown): void => { + if ( + typeof value !== "object" || + value === null || + !("packages" in value) || + !Array.isArray(value.packages) + ) + return; + for (const [index, pkg] of value.packages.entries()) { + if (!packageRecord(pkg)) + throw new RustAuditableFormatFailure( + "malformed", + `Dependency package ${String(index)} does not match the package schema`, + ); + } +}; + +const packageRecord = ( + pkg: unknown, +): pkg is { + readonly dependencies?: readonly unknown[]; +} => { + if (typeof pkg !== "object" || pkg === null || Array.isArray(pkg)) + return false; + const record = pkg as Record; + for (const key of Object.keys(record)) { + if (!PACKAGE_FIELDS.has(key)) return false; + } + if ( + !nonEmptyString(record.name) || + !nonEmptyString(record.version) || + !nonEmptyString(record.source) + ) + return false; + if ( + record.kind !== undefined && + record.kind !== "runtime" && + record.kind !== "build" + ) + return false; + if (record.root !== undefined && typeof record.root !== "boolean") + return false; + const dependencies = record.dependencies; + if (dependencies === undefined) return true; + return ( + Array.isArray(dependencies) && + dependencies.every( + (entry) => + typeof entry === "number" && Number.isInteger(entry) && entry >= 0, + ) + ); +}; + +const nonEmptyString = (value: unknown): value is string => + typeof value === "string" && value.length > 0; + +const assertDepth = (text: string): void => { + let depth = 0; + let quoted = false; + let escaped = false; + for (const char of text) { + if (quoted) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') quoted = false; + continue; + } + if (char === '"') { + quoted = true; + continue; + } + if (char === "{" || char === "[") { + depth += 1; + if (depth > JSON_DEPTH) + throw new RustAuditableFormatFailure( + "malformed", + "Dependency JSON nesting exceeds 32 levels", + ); + } else if ((char === "}" || char === "]") && depth > 0) depth -= 1; + } +}; + +const normalizePackages = ( + packages: z.infer["packages"], +): RustPackage[] => + packages.map((pkg) => ({ + name: pkg.name, + version: pkg.version, + source: pkg.source, + kind: pkg.kind ?? null, + dependencies: pkg.dependencies ?? [], + root: pkg.root ?? false, + })); + +const rootIndex = (packages: readonly RustPackage[]): number | null => { + const roots = packages.flatMap((pkg, index) => (pkg.root ? [index] : [])); + if (roots.length > 1) + throw new RustAuditableFormatFailure( + "malformed", + "Dependency JSON declares more than one root package", + ); + for (const [index, pkg] of packages.entries()) { + const outside = pkg.dependencies.find( + (target) => target >= packages.length, + ); + if (outside !== undefined) + throw new RustAuditableFormatFailure( + "malformed", + `Dependency index ${String(outside)} from package ${String(index)} is outside the package list`, + ); + } + return roots[0] ?? null; +}; diff --git a/src/rust/RustBinaryImage.ts b/src/rust/RustBinaryImage.ts new file mode 100644 index 000000000..ae077d21c --- /dev/null +++ b/src/rust/RustBinaryImage.ts @@ -0,0 +1,181 @@ +import { + GoBinaryFormatFailure, + GoBinaryResourceFailure, + type GoImageSection, +} from "../go/GoBinaryContainer.js"; +import { readGoBinaryContainer } from "../go/GoBinaryImage.js"; +import type { + RustAuditableMetadata, + RustBinary, +} from "../domain/rust/rustBinary.js"; +import { + readAuditableSection, + RustAuditableFormatFailure, + RustAuditableResourceFailure, +} from "./RustAuditableReader.js"; + +const RUSTC_PREFIX = Buffer.from("/rustc/"); +const HASH_BYTES = 40; +/** One toolchain normally leaves one hash; a small cap bounds retained output. */ +const MAX_RUSTC_HASHES = 64; + +/** Container identity plus optional dependency metadata and inferred rustc commits. */ +export interface RustBinaryImage { + readonly format: RustBinary["format"]; + readonly architecture: string; + readonly bits: 32 | 64; + readonly byte_order: RustBinary["byte_order"]; + readonly auditable_metadata: RustAuditableMetadata | null; + readonly rustc_commit: RustBinary["rustc_commit"]; +} + +/** Read dependency metadata from container sections without executing the image. */ +export const readRustBinaryImage = (bytes: Buffer): RustBinaryImage => { + const container = readContainer(bytes); + const dependency = dependencySection(container.format, container.sections); + return { + format: container.format, + architecture: container.architecture, + bits: container.bits, + byte_order: container.byte_order, + auditable_metadata: + dependency === null + ? null + : readDependency(bytes, container.format, dependency), + rustc_commit: rustcCommit(bytes, container.sections), + }; +}; + +const readContainer = (bytes: Buffer) => { + try { + return readGoBinaryContainer(bytes); + } catch (cause) { + if (cause instanceof GoBinaryFormatFailure) + throw new RustAuditableFormatFailure(cause.kind, cause.message, { + cause, + }); + if (cause instanceof GoBinaryResourceFailure) + throw new RustAuditableResourceFailure( + { + resource: "memory", + reported_limits: { maximum_bytes: cause.maximum_bytes }, + }, + cause.message, + { + cause, + }, + ); + throw cause; + } +}; + +const dependencySection = ( + format: RustBinaryImage["format"], + sections: readonly GoImageSection[], +): GoImageSection | null => { + const expected = format === "macho" ? "__DATA,.dep-v0" : ".dep-v0"; + const matches = sections.filter((section) => section.name === expected); + if (matches.length > 1) + throw new RustAuditableFormatFailure( + "malformed", + "Image contains more than one dependency section", + ); + const section = matches[0]; + if (section === undefined) return null; + if (section.compressed) + throw new RustAuditableFormatFailure( + "unsupported", + "Compressed dependency sections are unsupported", + ); + if (!section.file_backed) + throw new RustAuditableFormatFailure( + "malformed", + "Dependency section is not file-backed", + ); + return section; +}; + +const readDependency = ( + bytes: Buffer, + format: RustBinaryImage["format"], + section: GoImageSection, +): RustAuditableMetadata => + readAuditableSection( + format === "macho" ? "__DATA,.dep-v0" : ".dep-v0", + section.offset, + bytes.subarray(section.offset, section.offset + section.size), + ); + +const rustcCommit = ( + bytes: Buffer, + sections: readonly GoImageSection[], +): RustBinaryImage["rustc_commit"] => { + const hashes: { hash: string; file_offset: number }[] = []; + const seen = new Set(); + // Overlapping declared sections are scanned once, so total work stays + // bounded by the file size however many sections reference a range. + for (const range of mergedReadOnlyRanges(sections)) + collectHashes(bytes, range, seen, hashes); + return hashes.length === 0 ? null : { evidence_kind: "inferred", hashes }; +}; + +const mergedReadOnlyRanges = ( + sections: readonly GoImageSection[], +): { readonly offset: number; readonly end: number }[] => { + const ranges = sections + .filter((section) => section.read_only && section.size > 0) + .map((section) => ({ + offset: section.offset, + end: section.offset + section.size, + })) + .sort((left, right) => left.offset - right.offset); + const merged: { offset: number; end: number }[] = []; + for (const range of ranges) { + const last = merged.at(-1); + if (last !== undefined && range.offset <= last.end) + last.end = Math.max(last.end, range.end); + else merged.push({ ...range }); + } + return merged; +}; + +const collectHashes = ( + bytes: Buffer, + range: { readonly offset: number; readonly end: number }, + seen: Set, + hashes: { hash: string; file_offset: number }[], +): void => { + const region = bytes.subarray(range.offset, range.end); + let cursor = 0; + while (cursor < region.length) { + const at = region.indexOf(RUSTC_PREFIX, cursor); + if (at < 0) return; + const start = at + RUSTC_PREFIX.length; + const hash = hashAt(region, start); + if (hash !== null && !seen.has(hash)) { + if (seen.size >= MAX_RUSTC_HASHES) + throw new RustAuditableResourceFailure( + { + resource: "capacity", + reported_limits: { maximum_rustc_hashes: MAX_RUSTC_HASHES }, + }, + `Read-only data contains more than ${MAX_RUSTC_HASHES} distinct rustc commit paths.`, + ); + seen.add(hash); + hashes.push({ hash, file_offset: range.offset + at }); + } + cursor = at + 1; + } +}; + +const hashAt = (region: Buffer, start: number): string | null => { + const end = start + HASH_BYTES; + if (end >= region.length || region[end] !== 0x2f) return null; + for (let index = start; index < end; index += 1) { + const byte = region[index] ?? 0; + const hex = + (byte >= 0x30 && byte <= 0x39) || (byte >= 0x61 && byte <= 0x66); + if (!hex) return null; + } + return region.toString("latin1", start, end); +}; diff --git a/src/rust/RustBinaryProvider.ts b/src/rust/RustBinaryProvider.ts new file mode 100644 index 000000000..d0ce690f0 --- /dev/null +++ b/src/rust/RustBinaryProvider.ts @@ -0,0 +1,216 @@ +import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { ExecutionOptions } from "../application/AnalysisProvider.js"; +import type { RustBinaryPort } from "../application/rust/RustBinaryPort.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, + AnalysisUnsupportedTargetError, +} from "../domain/analysisErrorCore.js"; +import { + rustBinarySchema, + type InspectRustBinaryInput, + type RustBinary, +} from "../domain/rust/rustBinary.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { readRustBinaryImage } from "./RustBinaryImage.js"; +import { + RUST_BINARY_INPUT_BYTES, + RUST_OUTPUT_BYTES, + RustAuditableFormatFailure, + RustAuditableResourceFailure, +} from "./RustAuditableReader.js"; + +/** Bundled reader identity. Embedded package versions are separate observed facts. */ +export const RUST_BINARY_PROVIDER_IDENTITY = { + id: "rust-auditable", + name: "REA Rust dependency reader", + version: "1", +} as const; +const OPERATION = "inspect_rust_binary"; + +/** Stable in-process dependency inspection without subprocesses, temporary files, or a Rust toolchain. */ +export class RustBinaryProvider implements RustBinaryPort { + readonly identity = RUST_BINARY_PROVIDER_IDENTITY; + + /** Read one dependency section and any inferred rustc commit from the selected file. */ + async inspect( + input: InspectRustBinaryInput, + options?: ExecutionOptions, + ): Promise> { + try { + const snapshot = await readStableArtifact( + input.path, + RUST_BINARY_INPUT_BYTES, + options?.signal, + ); + options?.signal?.throwIfAborted(); + const image = readRustBinaryImage(snapshot.bytes); + const report = { + ...image, + artifact: { + path: input.path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }, + limitations: limitationsFor(image), + }; + if (Buffer.byteLength(JSON.stringify(report), "utf8") > RUST_OUTPUT_BYTES) + throw outputLimit(input.path); + const parsed = rustBinarySchema.safeParse(report); + if (!parsed.success) + throw new AnalysisOutputError( + OPERATION, + "Rust reader produced an invalid dependency report.", + { cause: parsed.error }, + ); + options?.signal?.throwIfAborted(); + return ok(parsed.data); + } catch (cause: unknown) { + return err(failureFor(cause, input.path, options?.signal)); + } + } +} + +const outputLimit = (path: string) => + new AnalysisResourceConstraintError( + OPERATION, + "transport", + `Complete Rust metadata for ${path} exceeds the inline output budget; no partial success is returned.`, + { maximum_output_bytes: RUST_OUTPUT_BYTES }, + ); + +const limitationsFor = ( + image: ReturnType, +): string[] => [ + "Embedded dependency metadata is file content. It does not authenticate package names, versions, sources, or the compiler that produced the binary.", + "Only the .dep-v0 dependency section and read-only compiler path strings are inspected. Functions, types, and runtime behavior are not recovered.", + "Resource guards bound the selected file to 256 MiB, a dependency section to 1 MiB, inflated JSON to the smaller of 8 MiB and 64 times the section size, and the complete report to 8 MiB. Package and dependency counts are admitted against that output budget before schema cloning, graph objects, or reverse-edge allocation.", + ...(image.auditable_metadata === null + ? [ + "No .dep-v0 section was found. Absence of .dep-v0 does not mean the binary has no dependencies.", + ] + : []), + ...(image.auditable_metadata?.packages.some((pkg) => pkg.kind === null) + ? [ + "An omitted package kind is left null. The embedded format treats an omitted kind as runtime, and that default was not stored in the section.", + ] + : []), + ...(image.auditable_metadata !== null && + image.auditable_metadata.cyclic_dependency_indices.length > 0 + ? [ + "Dependency indices contain a cycle. The cycle is reported and was not traversed.", + ] + : []), + ...rustcLimitation(image.rustc_commit), +]; + +const rustcLimitation = ( + commit: ReturnType["rustc_commit"], +): string[] => { + if (commit === null) + return [ + "No /rustc/<40-hex>/ path was found in read-only data, so a rustc commit hash is unknown.", + ]; + if (commit.hashes.length === 1) + return [ + "The rustc commit hash is inferred from a /rustc/<40-hex>/ path in read-only data. It does not by itself prove which compiler produced the binary.", + ]; + return [ + "Read-only data contains more than one /rustc/<40-hex>/ path, so a single rustc commit is unknown.", + ]; +}; + +const failureFor = ( + cause: unknown, + path: string, + signal?: AbortSignal, +): AnalysisError => { + if (cause instanceof AnalysisError) return cause; + if (signal?.aborted) return new AnalysisCancelledError(OPERATION); + if (cause instanceof RustAuditableResourceFailure) + return new AnalysisResourceConstraintError( + OPERATION, + cause.constraint.resource, + `Selected artifact ${path}: ${cause.message}`, + cause.constraint.reported_limits, + { cause }, + ); + if (cause instanceof RustAuditableFormatFailure) + return cause.kind === "unsupported" + ? new AnalysisUnsupportedTargetError(OPERATION, path, cause.message, { + cause, + }) + : new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["path"], + reason: "invalid_format", + message: `Selected artifact ${path}: ${cause.message}`, + }, + ]); + return artifactFailure(cause, path); +}; + +const artifactFailure = (cause: unknown, path: string): AnalysisError => { + if (cause instanceof ArtifactReaderFailure) + return artifactReaderFailure(cause, path); + if (cause instanceof Error && "code" in cause) { + if (cause.code === "EACCES" || cause.code === "EPERM") + return new AnalysisAccessDeniedError(OPERATION, path, cause.code, { + cause, + }); + if ( + ["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"].includes( + String(cause.code), + ) + ) + return new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["path"], + reason: "invalid_value", + message: `Selected artifact could not be read (${String(cause.code)}): ${path}.`, + }, + ]); + } + return new ProviderAdapterError(RUST_BINARY_PROVIDER_IDENTITY.id, OPERATION, { + cause, + diagnostics: { + path, + reason: cause instanceof Error ? cause.message : String(cause), + }, + }); +}; + +const artifactReaderFailure = ( + cause: ArtifactReaderFailure, + path: string, +): AnalysisError => { + if (cause.reason === "integrity") + return new AnalysisArtifactChangedError(OPERATION, path, cause.message, { + cause, + }); + if (cause.reason === "cancelled") + return new AnalysisCancelledError(OPERATION); + if (cause.reason === "limit") + return new AnalysisResourceConstraintError( + OPERATION, + "file-size", + cause.message, + { maximum_input_bytes: RUST_BINARY_INPUT_BYTES }, + { cause }, + ); + if (cause.reason === "path") + return new AnalysisInputError(OPERATION, { cause }, [ + { path: ["path"], reason: "invalid_format", message: cause.message }, + ]); + return new ProviderAdapterError(RUST_BINARY_PROVIDER_IDENTITY.id, OPERATION, { + cause, + diagnostics: { path, reason: cause.message }, + }); +}; diff --git a/src/server/createServer.ts b/src/server/createServer.ts index c73071bea..8acc5ac84 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -22,6 +22,9 @@ import { createRecordedCrashService } from "../composition/binaryDiagnostics.js" import { createGoBinaryService } from "../composition/go.js"; import type { GoBinaryService } from "../application/go/GoBinaryService.js"; import { registerGoTools } from "./registerGoTools.js"; +import { createRustBinaryService } from "../composition/rust.js"; +import type { RustBinaryService } from "../application/rust/RustBinaryService.js"; +import { registerRustTools } from "./registerRustTools.js"; import type { RecordedCrashService } from "../application/binaryDiagnostics/RecordedCrashService.js"; import { registerAnalysisViewTool } from "./registerAnalysisViewTool.js"; import { registerBinaryDiagnosticsTools } from "./registerBinaryDiagnosticsTools.js"; @@ -141,6 +144,7 @@ export interface CreateServerOptions { readonly logger?: Logger; readonly binaryLayout?: BinaryLayoutService; readonly goBinary?: GoBinaryService; + readonly rustBinary?: RustBinaryService; readonly recordedCrash?: RecordedCrashService; readonly firmwareAnalysis?: FirmwareAnalysisPort; readonly javascriptRecovery?: JavaScriptRecoveryPort; @@ -582,6 +586,13 @@ const registerConfiguredAnalysisTools = ( recordEvidence, withAdmittedAnalysis, ); + registerRustTools( + server, + options.rustBinary ?? createRustBinaryService(), + toolLogger, + recordEvidence, + withAdmittedAnalysis, + ); registerWasmTools( server, wasmArtifact, diff --git a/src/server/registerRustTools.ts b/src/server/registerRustTools.ts new file mode 100644 index 000000000..4f674cd82 --- /dev/null +++ b/src/server/registerRustTools.ts @@ -0,0 +1,31 @@ +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { RustBinaryService } from "../application/rust/RustBinaryService.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import type { Logger } from "pino"; +import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; +import { createEvidenceToolHandler } from "./contractToolHandler.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; + +/** Bind the exact Rust contract to its shared workflow and owning session Evidence ledger. */ +export const registerRustTools = ( + server: EvidenceMcpServer, + service: RustBinaryService, + logger: Logger, + recordEvidence?: EvidenceWriter["recordEvidence"], + withAdmittedAnalysis?: WithAdmittedAnalysis, +): void => { + const contract = toolContract("inspect_rust_binary"); + const handler = createEvidenceToolHandler( + server, + (input, options) => service.inspect(input, options), + logger, + recordEvidence, + withAdmittedAnalysis, + ); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + handler(contract), + ); +}; diff --git a/tests/acceptance/rustBinaryCli.test.ts b/tests/acceptance/rustBinaryCli.test.ts new file mode 100644 index 000000000..2f984c618 --- /dev/null +++ b/tests/acceptance/rustBinaryCli.test.ts @@ -0,0 +1,71 @@ +import { deflateSync } from "node:zlib"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { createRustImage } from "../fixtures/rust/image.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest( + "reports cargo-auditable packages through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-cli-"); + const path = join(root, "sample"); + await writeFile(path, createRustImage({ format: "macho" })); + const result = await cli.run({ + arguments: ["inspect-rust-binary", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_rust_binary").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.auditable_metadata).toMatchObject({ + section_name: "__DATA,.dep-v0", + format_version: 1, + root_package_index: 1, + packages: [ + { name: "itoa", version: "1.0.18" }, + { name: "sample", version: "0.1.0", root: true }, + ], + }); + }, +); + +cliTest("returns null metadata when .dep-v0 is absent", async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-cli-missing-"); + const path = join(root, "plain"); + await writeFile(path, createRustImage({ format: "elf", omitSection: true })); + const result = await cli.run({ + arguments: ["inspect-rust-binary", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_rust_binary").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.auditable_metadata).toBeNull(); + expect(evidence.normalized_result.limitations.join("\n")).toContain( + "does not mean the binary has no dependencies", + ); +}); + +cliTest("reports malformed zlib as invalid input", async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-cli-zlib-"); + const path = join(root, "broken.elf"); + await writeFile( + path, + createRustImage({ + format: "elf", + payload: deflateSync(Buffer.from("{")), + includeRustc: false, + }), + ); + const result = await cli.run({ + arguments: ["inspect-rust-binary", path, "--json"], + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { operation: "inspect_rust_binary" }, + }); +}); diff --git a/tests/boundary/go/goBinaryImage.test.ts b/tests/boundary/go/goBinaryImage.test.ts index 29f0e952c..c8069be98 100644 --- a/tests/boundary/go/goBinaryImage.test.ts +++ b/tests/boundary/go/goBinaryImage.test.ts @@ -6,6 +6,7 @@ import { machoImage, } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { readGoBinaryImage } from "../../../src/go/GoBinaryImage.js"; +import { readGoMachoImage } from "../../../src/go/GoMachoImage.js"; import { createGoBinaryFixture, GO_MODULE_TEXT, @@ -519,6 +520,26 @@ it("rejects build-info declared inside an unmapped Mach-O segment", () => { expect(() => readGoBinaryImage(fixture.bytes)).toThrowError(/unmapped/i); }); +it("does not mark Mach-O section bytes outside their owning segment as read-only", () => { + const fixture = createGoBinaryFixture({ format: "macho" }); + const command = 32; + const section = command + 72; + fixture.bytes.write("__TEXT", command + 8, "ascii"); + fixture.bytes.writeBigUInt64LE(1024n, command + 48); + fixture.bytes.writeUInt32LE(1, command + 56); + fixture.bytes.writeUInt32LE(1, command + 60); + fixture.bytes.write("__debug_info", section, "ascii"); + fixture.bytes.write("__TEXT", section + 16, "ascii"); + fixture.bytes.writeBigUInt64LE(0x10c00n, section + 32); + fixture.bytes.writeBigUInt64LE(128n, section + 40); + fixture.bytes.writeUInt32LE(3072, section + 48); + const image = readGoMachoImage(fixture.bytes); + expect(image.sections[0]).toMatchObject({ + file_backed: false, + read_only: false, + }); +}); + it.each(["elf", "pe", "macho"] as const)( "refuses file aliases overlapping only part of a zero-filled %s module header", (format) => { diff --git a/tests/boundary/mcp/ghidraInventoryMcp.test.ts b/tests/boundary/mcp/ghidraInventoryMcp.test.ts index ac42b9388..97ad30341 100644 --- a/tests/boundary/mcp/ghidraInventoryMcp.test.ts +++ b/tests/boundary/mcp/ghidraInventoryMcp.test.ts @@ -1,6 +1,6 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { expect, it } from "vitest"; -import { writeFile } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { nativeAotPeFixture } from "../../fixtures/nativeaotPe.js"; @@ -13,7 +13,6 @@ import { sessionEvidence, } from "./ghidraMcpHarness.js"; import { spawn } from "node:child_process"; -import { mkdir } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { GhidraClient } from "../../../src/ghidra/GhidraClient.js"; diff --git a/tests/boundary/mcp/rustBinaryMcp.test.ts b/tests/boundary/mcp/rustBinaryMcp.test.ts new file mode 100644 index 000000000..d903affd4 --- /dev/null +++ b/tests/boundary/mcp/rustBinaryMcp.test.ts @@ -0,0 +1,122 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { + createRustImage, + createRustHashOverflowImage, +} from "../../fixtures/rust/image.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { mcpTest, type TestMcp } from "../../support/mcp/mcpFixture.js"; + +const connect = async (mcp: TestMcp) => { + const session = createTestBinarySession(() => { + throw new Error("Rust metadata must not start a deep provider"); + }); + const client = await mcp.connect(createServer({ kind: "session", session })); + return { client, session }; +}; + +mcpTest( + "advertises inspect_rust_binary and reads an embedded dependency list", + async ({ mcp }) => { + const { client, session } = await connect(mcp); + try { + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_rust_binary", + ); + expect(advertised?.annotations).toMatchObject({ + readOnlyHint: false, + idempotentHint: true, + }); + const root = await createTestTempDirectory("rea-rust-mcp-"); + const path = join(root, "sample.elf"); + await writeFile(path, createRustImage({ format: "elf" })); + const response = await client.callTool({ + name: "inspect_rust_binary", + arguments: { path }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = toolContract("inspect_rust_binary").outputSchema.parse( + response.structuredContent, + ); + expect(evidence.normalized_result.auditable_metadata?.packages).toEqual([ + expect.objectContaining({ name: "itoa", version: "1.0.18" }), + expect.objectContaining({ + name: "sample", + version: "0.1.0", + root: true, + }), + ]); + const missing = join(root, "plain.elf"); + await writeFile( + missing, + createRustImage({ format: "pe", omitSection: true }), + ); + const absent = await client.callTool({ + name: "inspect_rust_binary", + arguments: { path: missing }, + }); + expect(absent.isError).not.toBe(true); + const absentEvidence = toolContract( + "inspect_rust_binary", + ).outputSchema.parse(absent.structuredContent); + expect(absentEvidence.normalized_result.auditable_metadata).toBeNull(); + } finally { + await session.close(); + } + }, +); + +mcpTest( + "requires an absolute path for inspect_rust_binary", + async ({ mcp }) => { + const { client, session } = await connect(mcp); + try { + const response = await client.callTool({ + name: "inspect_rust_binary", + arguments: { path: "sample" }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "invalid_request", + details: { operation: "inspect_rust_binary" }, + }); + } finally { + await session.close(); + } + }, +); + +mcpTest( + "preserves rustc hash capacity units through the public error contract", + async ({ mcp }) => { + const { client, session } = await connect(mcp); + try { + const root = await createTestTempDirectory("rea-rust-mcp-"); + const path = join(root, "many-hashes.elf"); + await writeFile(path, createRustHashOverflowImage()); + const response = await client.callTool({ + name: "inspect_rust_binary", + arguments: { path }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "resource_constraint", + details: { + operation: "inspect_rust_binary", + resource: "capacity", + reported_limits: { maximum_rustc_hashes: 64 }, + }, + }); + expect(parseMcpToolError(response).error.details).not.toHaveProperty( + "reported_limits.maximum_bytes", + ); + } finally { + await session.close(); + } + }, +); diff --git a/tests/boundary/rust/rustBinaryImage.test.ts b/tests/boundary/rust/rustBinaryImage.test.ts new file mode 100644 index 000000000..d4aa12283 --- /dev/null +++ b/tests/boundary/rust/rustBinaryImage.test.ts @@ -0,0 +1,141 @@ +import { expect, it } from "vitest"; +import { fat } from "../../../src/domain/binaryTarget.fixture.js"; +import { + createRustImage, + createRustHashOverflowImage, + RUSTC_HASH, + rustDependencyPayload, +} from "../../fixtures/rust/image.js"; +import { readRustBinaryImage } from "../../../src/rust/RustBinaryImage.js"; +import { RustAuditableFormatFailure } from "../../../src/rust/RustAuditableReader.js"; + +const packages = [ + { + name: "itoa", + version: "1.0.18", + source: "crates.io", + kind: null, + dependencies: [], + root: false, + }, + { + name: "sample", + version: "0.1.0", + source: "local", + kind: null, + dependencies: [0], + root: true, + }, +]; + +it.each(["elf", "pe"] as const)( + "reads a .dep-v0 section and an inferred rustc hash from %s", + (format) => { + const image = readRustBinaryImage(createRustImage({ format })); + expect(image.format).toBe(format); + expect(image.auditable_metadata).toMatchObject({ + section_name: ".dep-v0", + file_offset: 1024, + section_bytes: rustDependencyPayload().length, + format_version: 1, + root_package_index: 1, + packages, + dependency_edges: [{ from_index: 1, to_index: 0 }], + }); + expect(image.rustc_commit).toEqual({ + evidence_kind: "inferred", + hashes: [{ hash: RUSTC_HASH, file_offset: 2048 }], + }); + }, +); + +it("reads a Mach-O __DATA,.dep-v0 section", () => { + const image = readRustBinaryImage(createRustImage({ format: "macho" })); + expect(image).toMatchObject({ + format: "macho", + architecture: "arm64", + auditable_metadata: { + section_name: "__DATA,.dep-v0", + file_offset: 1024, + root_package_index: 1, + packages, + }, + rustc_commit: { + evidence_kind: "inferred", + hashes: [{ hash: RUSTC_HASH, file_offset: 2048 }], + }, + }); +}); + +it("ignores PE file-alignment padding after the dependency content", () => { + const bytes = createRustImage({ format: "pe", includeRustc: false }); + const payloadBytes = rustDependencyPayload().length; + const entry = 264; + expect(bytes.readUInt32LE(entry + 8)).toBe(payloadBytes); + bytes.writeUInt32LE(512, entry + 16); + const image = readRustBinaryImage(bytes); + expect(image.auditable_metadata).toMatchObject({ + section_name: ".dep-v0", + section_bytes: payloadBytes, + packages, + }); +}); + +it("returns null metadata when the dependency section is absent", () => { + const image = readRustBinaryImage( + createRustImage({ format: "elf", omitSection: true }), + ); + expect(image.auditable_metadata).toBeNull(); + expect(image.rustc_commit?.hashes).toEqual([ + { hash: RUSTC_HASH, file_offset: 2048 }, + ]); +}); + +it("refuses a universal Mach-O container", () => { + expect(() => readRustBinaryImage(fat([0x0100000c]))).toThrow( + RustAuditableFormatFailure, + ); + expect(() => readRustBinaryImage(fat([0x0100000c]))).toThrow(/thin slice/); +}); + +it("does not read SHT_NOBITS dependency sections as stored bytes", () => { + const bytes = createRustImage({ format: "elf" }); + // The fixture writes the .dep-v0 section header at byte 384. + bytes.writeUInt32LE(8, 384 + 4); + let metadata: unknown; + try { + metadata = readRustBinaryImage(bytes).auditable_metadata; + } catch (cause: unknown) { + expect(cause).toBeInstanceOf(RustAuditableFormatFailure); + return; + } + expect(metadata).toBeNull(); +}); + +it("reads a rustc path from a read-only ELF section whose name exceeds 64 bytes", () => { + const bytes = createRustImage({ format: "elf", omitSection: true }); + // omitSection places the rustc section header at 384 and shstrtab at 320. + const nameOffset = bytes.readUInt32LE(384); + const tableOffset = Number(bytes.readBigUInt64LE(320 + 24)); + const longName = Buffer.concat([Buffer.alloc(65, 0x61), Buffer.from([0])]); + longName.copy(bytes, tableOffset + nameOffset); + bytes.writeBigUInt64LE(BigInt(nameOffset + longName.length), 320 + 32); + const image = readRustBinaryImage(bytes); + expect(image.auditable_metadata).toBeNull(); + expect(image.rustc_commit).toEqual({ + evidence_kind: "inferred", + hashes: [{ hash: RUSTC_HASH, file_offset: 2048 }], + }); +}); + +it("refuses read-only data with more distinct rustc paths than it retains", () => { + const bytes = createRustHashOverflowImage(); + expect(() => readRustBinaryImage(bytes)).toThrow( + expect.objectContaining({ + constraint: { + resource: "capacity", + reported_limits: { maximum_rustc_hashes: 64 }, + }, + }), + ); +}); diff --git a/tests/fixtures/rust/image.ts b/tests/fixtures/rust/image.ts new file mode 100644 index 000000000..6bbac25ab --- /dev/null +++ b/tests/fixtures/rust/image.ts @@ -0,0 +1,263 @@ +import { deflateSync } from "node:zlib"; + +/** zlib payload matching the embedded dependency JSON shape. */ +export const rustDependencyPayload = ( + document: unknown = { + packages: [ + { name: "itoa", version: "1.0.18", source: "crates.io" }, + { + name: "sample", + version: "0.1.0", + source: "local", + dependencies: [0], + root: true, + }, + ], + format: 1, + }, +): Buffer => deflateSync(Buffer.from(JSON.stringify(document))); + +export const RUSTC_HASH = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const RUSTC_PATH = `/rustc/${RUSTC_HASH}/library/core`; + +/** Synthetic ELF, PE, or thin Mach-O with an optional dependency section. */ +export const createRustImage = (options: { + readonly format: "elf" | "pe" | "macho"; + readonly payload?: Buffer; + readonly omitSection?: boolean; + readonly includeRustc?: boolean; +}): Buffer => { + const payload = options.payload ?? rustDependencyPayload(); + const rustc = options.includeRustc === false ? null : Buffer.from(RUSTC_PATH); + const bytes = Buffer.alloc(4096); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const little = (offset: number, value: number, width: 16 | 32 | 64) => { + if (width === 16) view.setUint16(offset, value, true); + else if (width === 32) view.setUint32(offset, value, true); + else view.setBigUint64(offset, BigInt(value), true); + }; + const image = { + bytes, + little, + payload, + omitSection: options.omitSection === true, + rustc, + }; + if (options.format === "elf") writeElf(image); + else if (options.format === "pe") writePe(image); + else writeMacho(image); + return bytes; +}; + +interface RustImageWrite { + readonly bytes: Buffer; + readonly little: (offset: number, value: number, width: 16 | 32 | 64) => void; + readonly payload: Buffer; + readonly omitSection: boolean; + readonly rustc: Buffer | null; +} + +const writeElf = ({ + bytes, + little, + payload, + omitSection, + rustc, +}: RustImageWrite): void => { + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1, 1]); + little(16, 2, 16); + little(18, 62, 16); + little(20, 1, 32); + little(32, 128, 64); + little(40, 256, 64); + little(52, 64, 16); + little(54, 56, 16); + little(56, 1, 16); + little(58, 64, 16); + const names = omitSection + ? Buffer.from("\0.shstrtab\0.rodata\0") + : Buffer.from("\0.shstrtab\0.dep-v0\0.rodata\0"); + little(60, rustc === null && omitSection ? 0 : omitSection ? 3 : 4, 16); + little(62, rustc === null && omitSection ? 0 : 1, 16); + little(128, 1, 32); + little(132, 6, 32); + little(144, 0x400000, 64); + little(160, bytes.length, 64); + little(168, bytes.length, 64); + if (rustc === null && omitSection) return; + names.copy(bytes, 768); + const shstr = 320; + little(shstr, 1, 32); + little(shstr + 4, 3, 32); + little(shstr + 24, 768, 64); + little(shstr + 32, names.length, 64); + let next = 384; + if (!omitSection) { + little(next, 11, 32); + little(next + 4, 1, 32); + little(next + 24, 1024, 64); + little(next + 32, payload.length, 64); + payload.copy(bytes, 1024); + next += 64; + } + if (rustc !== null) { + const name = omitSection ? 11 : 19; + little(next, name, 32); + little(next + 4, 1, 32); + little(next + 8, 2, 64); + little(next + 16, 0x400000 + 2048, 64); + little(next + 24, 2048, 64); + little(next + 32, rustc.length, 64); + rustc.copy(bytes, 2048); + } +}; + +const writePe = ({ + bytes, + little, + payload, + omitSection, + rustc, +}: RustImageWrite): void => { + bytes.write("MZ", 0, "ascii"); + little(60, 128, 32); + bytes.write("PE\0\0", 128, "ascii"); + little(132, 0x8664, 16); + const count = Number(!omitSection) + Number(rustc !== null); + little(134, count, 16); + little(148, 112, 16); + little(152, 0x20b, 16); + little(176, 0x140000000, 64); + const writeSection = (section: { + readonly entry: number; + readonly name: string; + readonly offset: number; + readonly body: Buffer; + readonly address: number; + }) => { + bytes.write(section.name, section.entry, "ascii"); + little(section.entry + 8, section.body.length, 32); + little(section.entry + 12, section.address, 32); + little(section.entry + 16, section.body.length, 32); + little(section.entry + 20, section.offset, 32); + little(section.entry + 36, 0x40000040, 32); + section.body.copy(bytes, section.offset); + }; + let entry = 264; + if (!omitSection) { + writeSection({ + entry, + name: ".dep-v0", + offset: 1024, + body: payload, + address: 0x1000, + }); + entry += 40; + } + if (rustc !== null) + writeSection({ + entry, + name: ".rdata", + offset: 2048, + body: rustc, + address: 0x2000, + }); +}; + +const writeMacho = ({ + bytes, + little, + payload, + omitSection, + rustc, +}: RustImageWrite): void => { + const commands = Number(rustc !== null) + Number(!omitSection); + little(0, 0xfeedfacf, 32); + little(4, 0x0100000c, 32); + little(12, 2, 32); + little(16, commands, 32); + little(20, commands * 152, 32); + let command = 32; + if (rustc !== null) { + writeMachoSegment({ + bytes, + little, + command, + segment: "__TEXT", + section: "__cstring", + body: rustc, + fileOffset: 2048, + protection: 5, + flags: 2, + }); + command += 152; + } + if (!omitSection) + writeMachoSegment({ + bytes, + little, + command, + segment: "__DATA", + section: ".dep-v0", + body: payload, + fileOffset: 1024, + protection: 3, + flags: 0, + }); +}; + +const writeMachoSegment = ({ + bytes, + little, + command, + segment, + section, + body, + fileOffset, + protection, + flags, +}: { + readonly bytes: Buffer; + readonly little: RustImageWrite["little"]; + readonly command: number; + readonly segment: string; + readonly section: string; + readonly body: Buffer; + readonly fileOffset: number; + readonly protection: number; + readonly flags: number; +}): void => { + little(command, 0x19, 32); + little(command + 4, 152, 32); + bytes.write(segment, command + 8, "ascii"); + little(command + 24, 0x100000000, 64); + little(command + 32, 0x2000, 64); + little(command + 40, fileOffset, 64); + little(command + 48, body.length, 64); + little(command + 56, protection, 32); + little(command + 60, protection, 32); + little(command + 64, 1, 32); + const header = command + 72; + bytes.write(section, header, "ascii"); + bytes.write(segment, header + 16, "ascii"); + little(header + 32, 0x100000000, 64); + little(header + 40, body.length, 64); + little(header + 48, fileOffset, 32); + little(header + 64, flags, 32); + body.copy(bytes, fileOffset); +}; + +/** Read-only ELF data exceeding the distinct rustc-hash retention capacity. */ +export const createRustHashOverflowImage = (): Buffer => { + const bytes = createRustImage({ format: "elf", omitSection: true }); + // Without .dep-v0 the read-only section header is the second one, at 384. + // Section names end before byte 800, so the hash block starts there. + bytes.writeBigUInt64LE(800n, 384 + 24); + bytes.writeBigUInt64LE(3296n, 384 + 32); + bytes.fill(0, 800); + for (let index = 0; index < 65; index += 1) { + const hash = index.toString(16).padStart(40, "0"); + bytes.write(`/rustc/${hash}/`, 800 + index * 48, "latin1"); + } + return bytes; +}; diff --git a/tests/process-global/cleanLinuxHopperRuntime.test.ts b/tests/process-global/cleanLinuxHopperRuntime.test.ts index 8b7be17da..fdd19e8ee 100644 --- a/tests/process-global/cleanLinuxHopperRuntime.test.ts +++ b/tests/process-global/cleanLinuxHopperRuntime.test.ts @@ -41,6 +41,9 @@ it } // A rejection must come from the fixture we are about to launch. An // existing Xvfb would otherwise mask a broken executable-identity check. + // The copied launcher has a unique file identity and has not run yet. + // The synthetic proc root isolates the positive identity check from + // unrelated restricted host processes while retaining display-name checks. await assertCleanLinuxHopperRuntime( alternateMode ? [launcher] : [], procRoot, diff --git a/vitest.config.ts b/vitest.config.ts index bfef8e139..a8bf6b045 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -51,7 +51,7 @@ const TEST_PROJECTS = [ name: "adapters", include: [ "src/*.test.ts", - "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,server,wasm,windows}/**/*.test.ts", + "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", ], pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS, From 388b907c923133096a3387272367cbfb04ff1c04 Mon Sep 17 00:00:00 2001 From: Shenrui Ma <115921185+Shenrui-Ma@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:52:13 +0800 Subject: [PATCH 04/15] perf(javascript): defer ambiguity expansion in version comparisons (#2962) perf(javascript): avoid quadratic temporary matching state Repeated source groups can allocate candidates that later identity matches discard. Retain the original ambiguous groups until all tiers finish, expand only unmatched survivors, and append to locally owned key buckets in place. Constraint: Preserve matching priority, full ambiguity, and public output. Directive: Do not re-match residual groups that become unique after later tiers. Tested: 59 relevant tests, npm run check, and test discovery on current main. Tested: CLI benchmarks and full-suite failure comparison on the recorded base. Not-tested: Full suite on the updated main; native Linux and Windows workflows. --- .../javascriptApplicationVersionKeys.ts | 62 +++--- ...vascriptApplicationVersionMatching.test.ts | 181 ++++++++++++++++++ .../applicationVersionMatchingCli.test.ts | 98 ++++++++++ 3 files changed, 314 insertions(+), 27 deletions(-) create mode 100644 src/domain/javascript/javascriptApplicationVersionMatching.test.ts create mode 100644 tests/acceptance/applications/applicationVersionMatchingCli.test.ts diff --git a/src/domain/javascript/javascriptApplicationVersionKeys.ts b/src/domain/javascript/javascriptApplicationVersionKeys.ts index 306e8d316..11dbf62de 100644 --- a/src/domain/javascript/javascriptApplicationVersionKeys.ts +++ b/src/domain/javascript/javascriptApplicationVersionKeys.ts @@ -77,15 +77,23 @@ export const matchJavaScriptApplicationVersions = ( ): ApplicationVersionMatchingProjection => { const left = new Map(leftNodes.map((node) => [node.node_id, node])); const right = new Map(rightNodes.map((node) => [node.node_id, node])); - const leftCandidates = new Map>(); - const rightCandidates = new Map>(); const pairs: ApplicationVersionNodePair[] = []; + const ambiguousGroups: AmbiguousGroup[] = []; for (const tier of matchTiers()) - applyTier({ left, right, pairs, leftCandidates, rightCandidates, tier }); + applyTier({ left, right, pairs, ambiguousGroups, tier }); const remainingLeft = sortedNodes(left.values()); const remainingRight = sortedNodes(right.values()); - const leftIds = new Set(remainingLeft.map(({ node_id: id }) => id)); - const rightIds = new Set(remainingRight.map(({ node_id: id }) => id)); + const leftCandidates = new Map>(); + const rightCandidates = new Map>(); + // Later tiers can pair members of an ambiguous group. Expand only survivors, + // without pairing groups that became unique after those later matches. + for (const group of ambiguousGroups) { + const leftGroup = group.left.filter((node) => left.has(node.node_id)); + if (leftGroup.length === 0) continue; + const rightGroup = group.right.filter((node) => right.has(node.node_id)); + if (rightGroup.length === 0) continue; + recordCandidates(leftGroup, rightGroup, leftCandidates, rightCandidates); + } return { pairs: pairs.sort((a, b) => compareUnicodeCodePoints( @@ -95,17 +103,21 @@ export const matchJavaScriptApplicationVersions = ( ), unmatchedLeft: remainingLeft, unmatchedRight: remainingRight, - candidatesForLeft: filteredCandidates(leftCandidates, rightIds), - candidatesForRight: filteredCandidates(rightCandidates, leftIds), + candidatesForLeft: sortedCandidates(leftCandidates), + candidatesForRight: sortedCandidates(rightCandidates), }; }; +interface AmbiguousGroup { + readonly left: readonly ApplicationNode[]; + readonly right: readonly ApplicationNode[]; +} + interface TierContext { readonly left: Map; readonly right: Map; readonly pairs: ApplicationVersionNodePair[]; - readonly leftCandidates: Map>; - readonly rightCandidates: Map>; + readonly ambiguousGroups: AmbiguousGroup[]; readonly tier: MatchTier; } @@ -114,15 +126,12 @@ const applyTier = (context: TierContext): void => { const rightByKey = nodesByKey(context.right.values(), context.tier.key); for (const key of [...leftByKey.keys()].sort(compareUnicodeCodePoints)) { const leftGroup = leftByKey.get(key) ?? []; - const rightGroup = rightByKey.get(key) ?? []; - if (rightGroup.length === 0) continue; - recordCandidates( - leftGroup, - rightGroup, - context.leftCandidates, - context.rightCandidates, - ); - if (leftGroup.length !== 1 || rightGroup.length !== 1) continue; + const rightGroup = rightByKey.get(key); + if (rightGroup === undefined) continue; + if (leftGroup.length !== 1 || rightGroup.length !== 1) { + context.ambiguousGroups.push({ left: leftGroup, right: rightGroup }); + continue; + } const leftNode = leftGroup[0]; const rightNode = rightGroup[0]; if (leftNode === undefined || rightNode === undefined) continue; @@ -144,7 +153,9 @@ const nodesByKey = ( for (const node of nodes) { const key = keyFor(node); if (key === null) continue; - output.set(key, [...(output.get(key) ?? []), node]); + const group = output.get(key); + if (group === undefined) output.set(key, [node]); + else group.push(node); } return output; }; @@ -169,17 +180,14 @@ const recordCandidates = ( } }; -const filteredCandidates = ( +const sortedCandidates = ( candidates: ReadonlyMap>, - retained: ReadonlySet, ): ReadonlyMap => new Map( - [...candidates.entries()].flatMap(([nodeId, values]) => { - const filtered = [...values] - .filter((value) => retained.has(value)) - .sort(compareUnicodeCodePoints); - return filtered.length === 0 ? [] : [[nodeId, filtered] as const]; - }), + [...candidates.entries()].map(([nodeId, values]) => [ + nodeId, + [...values].sort(compareUnicodeCodePoints), + ]), ); const nodeIdentityKey = (node: ApplicationNode): string => diff --git a/src/domain/javascript/javascriptApplicationVersionMatching.test.ts b/src/domain/javascript/javascriptApplicationVersionMatching.test.ts new file mode 100644 index 000000000..9299d4184 --- /dev/null +++ b/src/domain/javascript/javascriptApplicationVersionMatching.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from "vitest"; + +import { + artifactEvidence, + completeCoverage, + fixtureNode, +} from "./javascriptApplicationGraph.fixture.js"; +import { + createJavaScriptApplicationGraph, + type JavaScriptApplicationGraph, +} from "./javascriptApplicationGraph.js"; +import type { ApplicationNode } from "./javascriptApplicationGraphSchemas.js"; +import { compareJavaScriptApplicationVersions } from "./javascriptApplicationVersionComparison.js"; + +const ARTIFACT_SHA = "a".repeat(64); + +const moduleNode = (name: string, source: string, structure?: string) => + fixtureNode( + "javascript-module", + { + strategy: "canonical-path", + stability: "artifact-version", + artifact_sha256: ARTIFACT_SHA, + path: `${name}.js`, + }, + name, + { + source_sha256: source.repeat(64), + ...(structure === undefined + ? {} + : { + structural_fingerprint_sha256: structure.repeat(64), + structural_fingerprint_algorithm: "fixture", + }), + }, + artifactEvidence(ARTIFACT_SHA, `${name}.js`), + ); + +const graph = (nodes: ApplicationNode[]): JavaScriptApplicationGraph => + createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: nodes.map(({ node_id }) => node_id), + nodes, + edges: [], + coverage: completeCoverage, + limitations: [], + }); + +const compare = ( + left: JavaScriptApplicationGraph, + right: JavaScriptApplicationGraph, +) => + compareJavaScriptApplicationVersions({ + left: { + evidenceId: `ev_${"1".repeat(64)}`, + rootArtifactSha256: ARTIFACT_SHA, + graph: left, + }, + right: { + evidenceId: `ev_${"2".repeat(64)}`, + rootArtifactSha256: ARTIFACT_SHA, + graph: right, + }, + leftNativeEvidence: [], + rightNativeEvidence: [], + }); + +describe("application version matching", () => { + it("does not revisit a source group that becomes unique after identity matching", () => { + const shared = moduleNode("shared", "1"); + const leftOnly = moduleNode("left", "1"); + const rightOnly = moduleNode("right", "1"); + const result = compare( + graph([shared, leftOnly]), + graph([shared, rightOnly]), + ); + + expect(result.matching).toMatchObject({ + exact_node_identity: 1, + exact_module_source_digest: 0, + ambiguous: 2, + unmatched: 0, + }); + expect( + result.items.find(({ left_node_id }) => left_node_id === shared.node_id), + ).toMatchObject({ + right_node_id: shared.node_id, + match: { status: "matched", basis: "exact-node-identity" }, + }); + expect( + result.items.find( + ({ left_node_id }) => left_node_id === leftOnly.node_id, + ), + ).toMatchObject({ + status: "unknown", + right_node_id: null, + match: { + status: "ambiguous", + candidate_right_node_ids: [rightOnly.node_id], + }, + }); + expect( + result.items.find( + ({ right_node_id }) => right_node_id === rightOnly.node_id, + ), + ).toMatchObject({ + status: "unknown", + left_node_id: null, + match: { + status: "ambiguous", + candidate_left_node_ids: [leftOnly.node_id], + }, + }); + }); + + it("unions all unresolved source and structural candidates without forcing pairs", () => { + const a = moduleNode("a", "1", "a"); + const b = moduleNode("b", "1", "a"); + const c = moduleNode("c", "2", "a"); + const x = moduleNode("x", "1", "a"); + const y = moduleNode("y", "1", "b"); + const z = moduleNode("z", "3", "a"); + const left = graph([a, b, c]); + const right = graph([x, y, z]); + const result = compare(left, right); + + expect(result.summary).toEqual({ + unchanged: 0, + added: 0, + removed: 0, + changed: 0, + unknown: 6, + }); + expect(result.matching).toMatchObject({ ambiguous: 6, unmatched: 0 }); + for (const [node, candidates] of [ + [a, [x, y, z]], + [b, [x, y, z]], + [c, [x, z]], + ] as const) { + expect( + result.items.find(({ left_node_id }) => left_node_id === node.node_id) + ?.match, + ).toEqual({ + status: "ambiguous", + basis: "none", + confidence: "unknown", + candidate_left_node_ids: [], + candidate_right_node_ids: candidates + .map(({ node_id }) => node_id) + .sort(), + }); + } + for (const [node, candidates] of [ + [x, [a, b, c]], + [y, [a, b]], + [z, [a, b, c]], + ] as const) { + expect( + result.items.find(({ right_node_id }) => right_node_id === node.node_id) + ?.match, + ).toEqual({ + status: "ambiguous", + basis: "none", + confidence: "unknown", + candidate_left_node_ids: candidates + .map(({ node_id }) => node_id) + .sort(), + candidate_right_node_ids: [], + }); + } + + // Permute the already validated graph so matching sees the changed order. + // Compare the full result, including item, comparison and change-graph IDs. + expect( + compare( + { ...left, nodes: [...left.nodes].reverse() }, + { ...right, nodes: [...right.nodes].reverse() }, + ), + ).toEqual(result); + }); +}); diff --git a/tests/acceptance/applications/applicationVersionMatchingCli.test.ts b/tests/acceptance/applications/applicationVersionMatchingCli.test.ts new file mode 100644 index 000000000..64e438ed3 --- /dev/null +++ b/tests/acceptance/applications/applicationVersionMatchingCli.test.ts @@ -0,0 +1,98 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { applicationVersionComparisonResultSchema } from "../../../src/domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "pairs repeated factory bodies by identity when application versions retain a shared bundle", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-version-matching-"); + const factoryCount = 32; + const factories = Array.from( + { length: factoryCount }, + (_, index) => `${index}: function() {}`, + ).join(","); + const bundle = `globalThis.webpackChunkApp.push([[1], {${factories}}]);`; + const versions = await Promise.all( + [1, 2].map(async (version) => { + const application = join(root, String(version)); + await mkdir(application); + await writeFile( + join(application, "package.json"), + JSON.stringify({ name: "shared-bundle", version: `${version}.0.0` }), + ); + await writeFile(join(application, "chunk.js"), bundle); + const response = await cli.run({ + arguments: ["analyze-javascript-application", application, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(response.exitCode, response.stderr).toBe(0); + return parseEvidence(response.json); + }), + ); + const [left, right] = versions; + if (left === undefined || right === undefined) + throw new Error("Missing application version Evidence"); + const analyses = versions.map(({ normalized_result }) => + javascriptApplicationAnalysisResultSchema.parse(normalized_result), + ); + const factoryNodes = analyses.map(({ graph }) => { + expect(graph.coverage.status).toBe("complete"); + const nodes = graph.nodes.filter( + ({ kind, observations }) => + kind === "javascript-module" && + observations.some( + ({ properties }) => properties.module_key !== undefined, + ), + ); + expect(nodes).toHaveLength(factoryCount); + const sourceDigests = nodes.flatMap(({ observations }) => + observations.map(({ properties }) => properties.source_sha256), + ); + expect(sourceDigests.every((digest) => typeof digest === "string")).toBe( + true, + ); + expect(new Set(sourceDigests).size).toBe(1); + return nodes; + }); + const inputPath = join(root, "comparison.json"); + await writeFile(inputPath, JSON.stringify({ left, right })); + const response = await cli.run({ + arguments: ["compare-application-versions", inputPath, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(response.exitCode, response.stderr).toBe(0); + const result = applicationVersionComparisonResultSchema.parse( + parseEvidence(response.json).normalized_result, + ); + + expect(result.left.root_artifact_sha256).not.toBe( + result.right.root_artifact_sha256, + ); + expect(result.matching.ambiguous).toBe(0); + expect(result.coverage.status).toBe("complete-within-inputs"); + expect(factoryNodes[0]?.map(({ node_id }) => node_id)).toEqual( + factoryNodes[1]?.map(({ node_id }) => node_id), + ); + for (const node of factoryNodes[0] ?? []) { + expect( + result.items.find(({ left_node_id }) => left_node_id === node.node_id), + ).toMatchObject({ + status: "unchanged", + right_node_id: node.node_id, + match: { + status: "matched", + basis: "exact-node-identity", + candidate_left_node_ids: [], + candidate_right_node_ids: [], + }, + }); + } + }, +); From 97268f40ea21b5f2d24b08c0c068a7ccf41459d4 Mon Sep 17 00:00:00 2001 From: Yunzhou Li <154221037+YunzhouLi-hub@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:52:47 +0800 Subject: [PATCH 05/15] fix(dotnet): name the truncated instruction when an operand leaves the method body (#2704) * fix(dotnet): name the truncated instruction when an operand leaves the method body The extent guard ran after the operand read, so a truncated operand threw a raw Buffer RangeError that replaced the intended diagnostic with the runtime message and lost the instruction identity. Bound the operand extent before reading its value and carry the width beside each fixed reader so the two facts cannot drift. Fixes #2677 * test(dotnet): consolidate truncated operand coverage --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/dotnet/ManagedCilDecoding.test.ts | 243 +++++++++++++++--- src/dotnet/ManagedMemberInstructionDecoder.ts | 133 ++++++---- 2 files changed, 288 insertions(+), 88 deletions(-) diff --git a/src/dotnet/ManagedCilDecoding.test.ts b/src/dotnet/ManagedCilDecoding.test.ts index 4ba33f4c9..852762b2c 100644 --- a/src/dotnet/ManagedCilDecoding.test.ts +++ b/src/dotnet/ManagedCilDecoding.test.ts @@ -67,6 +67,30 @@ const managedBodyWithRegion = ({ .methods[0]?.body; }; +const fatBody = (il: Buffer) => { + const header = Buffer.alloc(12); + header.writeUInt16LE(0x3013, 0); + header.writeUInt16LE(8, 2); + header.writeUInt32LE(il.length, 4); + const bytes = buildManagedPeFixture({ + ilBody: Buffer.concat([header, il]), + }); + return inspectManagedMembersBytes(bytes, managedPeFixtureTarget(bytes)) + .methods[0]?.body; +}; + +/** Build a tiny method body whose declared IL extent matches its supplied bytes. */ +const tinyBody = (il: readonly number[]) => { + const bytes = buildManagedPeFixture({ + ilBody: Buffer.from([(il.length << 2) | 0x02, ...il]), + }); + const inspection = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + return { body: inspection.methods[0]?.body, coverage: inspection.coverage }; +}; + describe("managed CIL decoding", () => { it("reads fat method header size from the full flags-and-size word", () => { const il = Buffer.from([ @@ -221,6 +245,187 @@ describe("managed CIL decoding", () => { }); }); +describe("managed operand extent diagnostics", () => { + it.each([ + [[0x20], 0, "Instruction ldc.i4 at IL offset 0 leaves method body"], + [ + [0x00, 0x20, 0x01], + 1, + "Instruction ldc.i4 at IL offset 1 leaves method body", + ], + [[0x2b], 0, "Instruction br.s at IL offset 0 leaves method body"], + [[0x45, 0x02], 0, "Instruction switch at IL offset 0 leaves method body"], + [ + [0xfe, 0x09, 0x00], + 0, + "Instruction ldarg at IL offset 0 leaves method body", + ], + [[0x00, 0xfe], 1, "Two-byte CIL opcode at IL offset 1 leaves method body"], + ] as const)( + "reports the instruction and IL offset for tiny IL %j", + (il, decoded, issue) => { + const { body, coverage } = tinyBody(il); + expect(body).toMatchObject({ + status: "malformed", + header_format: "tiny", + il_size: il.length, + decoded_instruction_count: decoded, + normalized_il_sha256: null, + issue, + }); + expect(coverage.state).toBe("partial"); + }, + ); + + it.each([ + ["short variable", Buffer.from([0x0e]), "ldarg.s"], + ["wide variable", Buffer.from([0xfe, 0x09]), "ldarg"], + ["int32 constant", Buffer.from([0x20]), "ldc.i4"], + ["int64 constant", Buffer.from([0x21]), "ldc.i8"], + ["float32 constant", Buffer.from([0x22]), "ldc.r4"], + ["float64 constant", Buffer.from([0x23]), "ldc.r8"], + ["short branch", Buffer.from([0x2b]), "br.s"], + ["long branch", Buffer.from([0x38]), "br"], + ["prefixed short constant", Buffer.from([0xfe, 0x12]), "unaligned."], + ["string token", Buffer.from([0x72]), "ldstr"], + ["type token", Buffer.from([0x71]), "ldobj"], + ["field token", Buffer.from([0x7b]), "ldfld"], + ["method token", Buffer.from([0x28]), "call"], + ] as const)( + "reports a truncated %s operand at its own instruction", + (_label, il, name) => { + expect(fatBody(il)).toMatchObject({ + status: "malformed", + decoded_instruction_count: 0, + normalized_il_sha256: null, + issue: `Instruction ${name} at IL offset 0 leaves method body`, + }); + }, + ); + + // A width declared too small still passes the extent check when no operand + // bytes are present, so the zero-byte cases above cannot catch it. Supplying + // one byte less than the real width makes a wrong width let the read run, + // which surfaces as the raw Buffer error rather than the diagnostic. + it.each([ + ["wide variable", Buffer.from([0xfe, 0x09]), 2, "ldarg"], + ["int32 constant", Buffer.from([0x20]), 4, "ldc.i4"], + ["int64 constant", Buffer.from([0x21]), 8, "ldc.i8"], + ["float32 constant", Buffer.from([0x22]), 4, "ldc.r4"], + ["float64 constant", Buffer.from([0x23]), 8, "ldc.r8"], + ["long branch", Buffer.from([0x38]), 4, "br"], + ["method token", Buffer.from([0x28]), 4, "call"], + ["type token", Buffer.from([0x71]), 4, "ldobj"], + ["string token", Buffer.from([0x72]), 4, "ldstr"], + ["field token", Buffer.from([0x7b]), 4, "ldfld"], + ] as const)( + "rejects a %s operand one byte short of its declared width", + (_label, opcode, width, name) => { + expect( + fatBody(Buffer.concat([opcode, Buffer.alloc(width - 1)])), + ).toMatchObject({ + status: "malformed", + decoded_instruction_count: 0, + normalized_il_sha256: null, + issue: `Instruction ${name} at IL offset 0 leaves method body`, + }); + }, + ); + + // The extent check must also reject nothing that fits, so every multi-byte + // width is exercised at exactly its own size with a known value. Each body + // ends at its operand, so a width declared too large turns a valid operand + // into the leaves-method-body diagnostic and these value assertions catch it. + // The -0 constants keep the floating-point sign fix covered through this same + // path. + it.each([ + ["wide variable", [0xfe, 0x09], [0x01, 0x00], "variable", "1"], + ["int32 constant", [0x20], [0x01, 0x00, 0x00, 0x00], "constant", "1"], + [ + "int64 constant", + [0x21], + [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + "constant", + "1", + ], + ["float32 constant", [0x22], [0x00, 0x00, 0x00, 0x80], "constant", "-0"], + [ + "float64 constant", + [0x23], + [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80], + "constant", + "-0", + ], + ["long branch", [0x38], [0x00, 0x00, 0x00, 0x00], "branch", "5"], + ["method token", [0x28], [0x01, 0x00, 0x00, 0x00], "method", "0x00000001"], + ["type token", [0x71], [0x02, 0x00, 0x00, 0x00], "type", "0x00000002"], + ["string token", [0x72], [0x03, 0x00, 0x00, 0x00], "string", "0x00000003"], + ["field token", [0x7b], [0x04, 0x00, 0x00, 0x00], "field", "0x00000004"], + ] as const)( + "decodes a %s operand at exactly its declared width", + (_label, opcode, operand, operandKind, value) => { + expect(fatBody(Buffer.from([...opcode, ...operand]))).toMatchObject({ + status: "present", + issue: null, + decoded_instruction_count: 1, + anchors: [{ il_offset: 0, operand_kind: operandKind, operand: value }], + }); + }, + ); + + it("reports a switch whose target table is missing", () => { + expect( + fatBody(Buffer.from([0x00, 0x45, 0x02, 0x00, 0x00, 0x00])), + ).toMatchObject({ + status: "malformed", + decoded_instruction_count: 1, + opcode_counts: { nop: 1 }, + issue: "Instruction switch at IL offset 1 leaves method body", + }); + }); + + it("reports a switch whose target count is itself truncated", () => { + expect(fatBody(Buffer.from([0x45, 0x02]))).toMatchObject({ + status: "malformed", + decoded_instruction_count: 0, + issue: "Instruction switch at IL offset 0 leaves method body", + }); + }); + + it("preserves instructions decoded before a truncated operand", () => { + expect(fatBody(Buffer.from([0x00, 0x20]))).toMatchObject({ + status: "malformed", + il_size: 2, + decoded_instruction_count: 1, + opcode_counts: { nop: 1 }, + issue: "Instruction ldc.i4 at IL offset 1 leaves method body", + }); + }); + + it("keeps a complete operand-bearing body decodable", () => { + expect( + fatBody(Buffer.from([0x00, 0x20, 0x01, 0x00, 0x00, 0x00, 0x2a])), + ).toMatchObject({ + status: "present", + decoded_instruction_count: 3, + opcode_counts: { nop: 1, "ldc.i4": 1, ret: 1 }, + issue: null, + }); + }); + + it("still decodes a switch whose targets fit", () => { + const il = Buffer.from([ + 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, + ]); + expect(fatBody(il)).toMatchObject({ + status: "present", + il_size: il.length, + opcode_counts: { nop: 1, switch: 1, ret: 1 }, + issue: null, + }); + }); +}); + describe("managed exception region bounds", () => { it("marks out-of-method ranges malformed and preserves the raw EH clause", () => { expect( @@ -535,41 +740,3 @@ describe("managed exception section decoding", () => { }); }); }); - -describe("managed CIL truncated operands", () => { - const tinyBody = (il: readonly number[]) => { - const bytes = buildManagedPeFixture({ - ilBody: Buffer.from([(il.length << 2) | 0x02, ...il]), - }); - const inspection = inspectManagedMembersBytes( - bytes, - managedPeFixtureTarget(bytes), - ); - return { body: inspection.methods[0]?.body, coverage: inspection.coverage }; - }; - - it.each([ - [[0x20], 0, "Instruction ldc.i4 at IL offset 0 leaves method body"], - [ - [0x00, 0x20, 0x01], - 1, - "Instruction ldc.i4 at IL offset 1 leaves method body", - ], - [[0x2b], 0, "Instruction br.s at IL offset 0 leaves method body"], - [[0x45, 0x02], 0, "Instruction switch at IL offset 0 leaves method body"], - [ - [0xfe, 0x09, 0x00], - 0, - "Instruction ldarg at IL offset 0 leaves method body", - ], - [[0x00, 0xfe], 1, "Two-byte CIL opcode at IL offset 1 leaves method body"], - ])("names the instruction and IL offset for IL %j", (il, decoded, issue) => { - const { body, coverage } = tinyBody(il); - expect(body).toMatchObject({ - status: "malformed", - decoded_instruction_count: decoded, - issue, - }); - expect(coverage.state).toBe("partial"); - }); -}); diff --git a/src/dotnet/ManagedMemberInstructionDecoder.ts b/src/dotnet/ManagedMemberInstructionDecoder.ts index 01c2b7814..0c04b7a36 100644 --- a/src/dotnet/ManagedMemberInstructionDecoder.ts +++ b/src/dotnet/ManagedMemberInstructionDecoder.ts @@ -271,82 +271,101 @@ type OperandReader = (il: Buffer, offset: number) => OperandResult; const floatingPointOperand = (value: number): string => Object.is(value, -0) ? "-0" : String(value); -const FIXED_OPERAND_READERS = new Map([ +/** + * A fixed-width operand declares its byte width next to its reader so the + * extent check can bound the read before any value is consumed. Keeping both + * facts in one entry is what stops the width and the reader from drifting. + */ +type FixedOperand = { + readonly width: number; + readonly read: OperandReader; +}; + +const fixed = (width: number, read: OperandReader): FixedOperand => ({ + width, + read, +}); + +const FIXED_OPERAND_READERS = new Map([ [ "none", - (_il, offset) => ({ next: offset, anchorKind: "none", value: null }), + fixed(0, (_il, offset) => ({ + next: offset, + anchorKind: "none", + value: null, + })), ], [ "short-var", - (il, offset) => ({ + fixed(1, (il, offset) => ({ next: offset + 1, anchorKind: "variable", value: String(il.readUInt8(offset)), - }), + })), ], [ "var", - (il, offset) => ({ + fixed(2, (il, offset) => ({ next: offset + 2, anchorKind: "variable", value: String(il.readUInt16LE(offset)), - }), + })), ], [ "short-i", - (il, offset) => ({ + fixed(1, (il, offset) => ({ next: offset + 1, anchorKind: "constant", value: String(il.readInt8(offset)), - }), + })), ], [ "i", - (il, offset) => ({ + fixed(4, (il, offset) => ({ next: offset + 4, anchorKind: "constant", value: String(il.readInt32LE(offset)), - }), + })), ], [ "i8", - (il, offset) => ({ + fixed(8, (il, offset) => ({ next: offset + 8, anchorKind: "constant", value: il.readBigInt64LE(offset).toString(), - }), + })), ], [ "r4", - (il, offset) => ({ + fixed(4, (il, offset) => ({ next: offset + 4, anchorKind: "constant", value: floatingPointOperand(il.readFloatLE(offset)), - }), + })), ], [ "r8", - (il, offset) => ({ + fixed(8, (il, offset) => ({ next: offset + 8, anchorKind: "constant", value: floatingPointOperand(il.readDoubleLE(offset)), - }), + })), ], [ "short-branch", - (il, offset) => ({ + fixed(1, (il, offset) => ({ next: offset + 1, anchorKind: "branch", value: String(offset + 1 + il.readInt8(offset)), - }), + })), ], [ "branch", - (il, offset) => ({ + fixed(4, (il, offset) => ({ next: offset + 4, anchorKind: "branch", value: String(offset + 4 + il.readInt32LE(offset)), - }), + })), ], ]); @@ -361,13 +380,42 @@ const TOKEN_OPERAND_KINDS = new Map< ["signature", "signature"], ]); +/** Every token operand is a 4-byte metadata token. */ +const TOKEN_OPERAND_WIDTH = 4; + +/** + * Bytes the operand at `offset` occupies, or `null` when the extent itself + * cannot be read from `il`. This is the single extent authority: `readOperand` + * must never be called for an operand whose extent it cannot bound, so no + * operand read can leave the method body. + */ +const operandExtent = ( + il: Buffer, + offset: number, + kind: string, +): number | null => { + const fixedOperand = FIXED_OPERAND_READERS.get(kind); + if (fixedOperand !== undefined) return fixedOperand.width; + if (kind === "switch") { + if (offset > il.length - 4) return null; + return 4 + il.readUInt32LE(offset) * 4; + } + return TOKEN_OPERAND_KINDS.has(kind) || kind === "token" + ? TOKEN_OPERAND_WIDTH + : 0; +}; + +/** + * Read an operand whose extent is already known to fit. `next` is always + * `offset + operandExtent(il, offset, kind)`; callers bound the read first. + */ const readOperand = ( il: Buffer, offset: number, kind: string, ): OperandResult => { - const fixed = FIXED_OPERAND_READERS.get(kind); - if (fixed !== undefined) return fixed(il, offset); + const fixedOperand = FIXED_OPERAND_READERS.get(kind); + if (fixedOperand !== undefined) return fixedOperand.read(il, offset); if (kind === "switch") { const count = il.readUInt32LE(offset); return { @@ -376,30 +424,15 @@ const readOperand = ( value: String(count), }; } + if (!TOKEN_OPERAND_KINDS.has(kind) && kind !== "token") + return { next: offset, anchorKind: "unknown", value: null }; const raw = il.readUInt32LE(offset); const token = `0x${raw.toString(16).padStart(8, "0")}`; - const tokenOperandKind = TOKEN_OPERAND_KINDS.get(kind); - if (tokenOperandKind !== undefined) - return { next: offset + 4, anchorKind: tokenOperandKind, value: token }; - if (kind === "token") - return { next: offset + 4, anchorKind: tokenKind(token), value: token }; - return { next: offset, anchorKind: "unknown", value: null }; -}; - -/** An operand cut off by the end of the IL body has no decodable value. */ -const readOperandWithin = ( - il: Buffer, - offset: number, - kind: string, -): OperandResult | undefined => { - try { - return readOperand(il, offset, kind); - } catch (cause: unknown) { - // Buffer reads past the IL body throw RangeError; report it as the - // instruction leaving the method body rather than as a Buffer message. - if (cause instanceof RangeError) return undefined; - throw cause; - } + return { + next: offset + TOKEN_OPERAND_WIDTH, + anchorKind: TOKEN_OPERAND_KINDS.get(kind) ?? tokenKind(token), + value: token, + }; }; export const decodeInstructions = ( @@ -432,15 +465,15 @@ export const decodeInstructions = ( issue = `Unsupported CIL opcode 0x${code.toString(16)} at IL offset ${String(start)}`; break; } - const operand = readOperandWithin(il, offset, descriptor.operand); - if ( - operand === undefined || - operand.next < offset || - operand.next > il.length - ) { + // Bound the operand extent before reading its value. Reading first would + // throw a raw Buffer RangeError out of a truncated body and replace this + // diagnostic with the runtime message, losing the instruction identity. + const extent = operandExtent(il, offset, descriptor.operand); + if (extent === null || offset + extent > il.length) { issue = `Instruction ${descriptor.name} at IL offset ${String(start)} leaves method body`; break; } + const operand = readOperand(il, offset, descriptor.operand); // Charge each actual instruction before growing the decoded arrays. // A multi-byte operand must not consume a full record per raw IL byte. admitInstruction?.(start); From 72d5bbc96d221709e9d9c9de96061dd5bebee7f1 Mon Sep 17 00:00:00 2001 From: Jose Camargo Date: Tue, 13 Oct 2026 15:55:46 -0300 Subject: [PATCH 06/15] fix(process): keep provider deadlines independent of wall-clock changes (#2311) * fix(process): keep provider deadlines independent of wall-clock changes * fix(process): retain monotonic targets after early timers Adapt the stronger timer/poll owner from @nosmile99 PR #2793 as integrated by 21aa877e5 and captured main 94072364c7f8410b0e86e72c912c2611572b278f. Retain all contributor reliability coverage. Tests and workloads skipped by explicit request. --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/process/ProviderDeadline.test.ts | 101 ++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/src/process/ProviderDeadline.test.ts b/src/process/ProviderDeadline.test.ts index ca83d98ab..6f26c870e 100644 --- a/src/process/ProviderDeadline.test.ts +++ b/src/process/ProviderDeadline.test.ts @@ -1,8 +1,106 @@ -import { afterEach, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { ProviderStartupDeadline } from "./ProviderDeadline.js"; +beforeEach(() => { + vi.useFakeTimers({ + toFake: ["setTimeout", "clearTimeout", "Date", "performance"], + }); +}); + afterEach(() => { vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +it.each([-3_600_000, 3_600_000])( + "keeps the elapsed budget when the wall clock jumps by %i ms", + async (jump) => { + const deadline = new ProviderStartupDeadline(1000); + try { + await vi.advanceTimersByTimeAsync(250); + vi.setSystemTime(Date.now() + jump); + expect(deadline.remainingMs()).toBe(750); + expect(deadline.interruption).toBeUndefined(); + expect(deadline.signal.aborted).toBe(false); + const waiting = deadline.wait(2000); + await vi.advanceTimersByTimeAsync(749); + expect(deadline.interruption).toBeUndefined(); + await vi.advanceTimersByTimeAsync(1); + await expect(waiting).resolves.toBe("aborted"); + expect(deadline.interruption).toBe("timeout"); + expect(deadline.remainingMs()).toBe(0); + expect(vi.getTimerCount()).toBe(0); + } finally { + deadline.dispose(); + } + }, +); + +it("preserves caller cancellation after a forward wall-clock correction", async () => { + const controller = new AbortController(); + const reason = new Error("caller cancelled"); + const deadline = new ProviderStartupDeadline(1000, controller.signal); + try { + const waiting = deadline.wait(1000); + await vi.advanceTimersByTimeAsync(100); + vi.setSystemTime(Date.now() + 3_600_000); + controller.abort(reason); + await expect(waiting).resolves.toBe("aborted"); + expect(deadline.interruption).toBe("cancelled"); + expect(deadline.signal.reason).toBe(reason); + expect(vi.getTimerCount()).toBe(0); + } finally { + deadline.dispose(); + } +}); + +it("does not revive an expired budget after the wall clock moves backward", async () => { + const deadline = new ProviderStartupDeadline(1000); + try { + await vi.advanceTimersByTimeAsync(1000); + vi.setSystemTime(Date.now() - 3_600_000); + expect(deadline.remainingMs()).toBe(0); + expect(deadline.interruption).toBe("timeout"); + await expect(deadline.wait(1000)).resolves.toBe("aborted"); + expect(vi.getTimerCount()).toBe(0); + } finally { + deadline.dispose(); + } +}); + +it("classifies elapsed time before a delayed timer callback and a later abort", () => { + const controller = new AbortController(); + const deadline = new ProviderStartupDeadline(1000, controller.signal); + try { + // Model an event-loop stall: elapsed time advanced, but timers have not run. + vi.spyOn(performance, "now").mockReturnValue(1001); + vi.setSystemTime(Date.now() - 3_600_000); + expect(deadline.interruption).toBe("timeout"); + expect(deadline.remainingMs()).toBe(0); + controller.abort(); + expect(deadline.interruption).toBe("timeout"); + expect(deadline.signal.reason).toMatchObject({ name: "TimeoutError" }); + expect(vi.getTimerCount()).toBe(0); + } finally { + deadline.dispose(); + } +}); + +it("keeps remaining milliseconds integral for downstream request budgets", () => { + const now = vi.spyOn(performance, "now").mockReturnValue(10.25); + const deadline = new ProviderStartupDeadline(1000); + try { + now.mockReturnValue(260.5); + expect(deadline.remainingMs()).toBe(750); + now.mockReturnValue(1010.2); + expect(deadline.remainingMs()).toBe(1); + expect(deadline.interruption).toBeUndefined(); + now.mockReturnValue(1010.25); + expect(deadline.remainingMs()).toBe(0); + expect(deadline.interruption).toBe("timeout"); + } finally { + deadline.dispose(); + } }); it("keeps remaining time and cancellation classification stable across wall-clock jumps", () => { @@ -24,6 +122,7 @@ it("keeps remaining time and cancellation classification stable across wall-cloc }); it("aborts a bounded wait when its real monotonic deadline expires", async () => { + vi.useRealTimers(); const deadline = new ProviderStartupDeadline(40); try { await expect(deadline.wait(500)).resolves.toBe("aborted"); From 4481f861880f5cbcd864059b2509c656b5803f7c Mon Sep 17 00:00:00 2001 From: Rohit Ghumare <48523873+rohitg00@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:26:00 +0530 Subject: [PATCH 07/15] fix(frida): drain admitted operations and honor queued cancellation (#2305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(frida): drain admitted operations and honor queued cancellation * fix(frida): settle cancelled loads before queued resume * test(frida): share lifecycle fixtures without duplicating assertions * fix(frida): preserve operation admission before session close * test(apple): bound cold archive fixture compilation * fix(frida): retain cancelled script creation evidence * test(release): scope preflight schema to selected jobs * fix(process): report no remaining time after a provider deadline fires Node timers can fire while performance.now() is still a fraction of a millisecond short of the monotonic deadline. ProviderStartupDeadline then reported interruption "timeout" and an aborted signal while remainingMs() returned ceil(0.x) = 1, so polling loops guarded by remainingMs() > 0 (Hopper and Ghidra startup) could run past an elapsed deadline, and the real-timer test "aborts a bounded wait when its real monotonic deadline expires" failed intermittently in CI (expected 1 to be 0). Once the deadline's own timer has classified a timeout, report zero remaining time. A deterministic regression test fires the timer with the monotonic clock 0.4 ms short of the deadline. Co-Authored-By: Claude Opus 5.5 * fix(comparison): recheck historical calls-only changes (#2223) * fix(comparison): preserve incomplete function call evidence Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty. Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls. * fix(comparison): recheck historical calls-only changes --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> (cherry picked from commit 688defca0f4a3de04eb199f7754f323c68896c3c) * fix(ci): isolate Linux Hopper process fixture inventory (cherry picked from commit 8cac51e12184d56a9ba80a4427b3e6082ddac1ec) * fix(android): make DEX descriptor pattern portable (cherry picked from commit 1335bcddc569a1cfb34b6efedcb6f2e3aaaa3357) * fix: repair the static-check failures on the default branch (#2895) * fix: repair the static-check failures on the default branch The default branch fails its own required `Static checks` job three ways, so every pull request reports that job as failed regardless of its own change. - `npm run format:check` reports `src/domain/functionComparison.test.ts` and `src/ida/IdaOperationRunner.ts`. - `npm run lint` reports a duplicate `import { join } from "node:path"` in `tests/boundary/mcp/ghidraInventoryMcp.test.ts` (lines 4 and 17). - `npm run typecheck` reports `format: "devin"` missing from the client format union in `src/application/SupportedClients.ts`, and an unchecked index in `tests/boundary/cli/clientProfileLifecycle.test.ts`, whose pair array is inferred as `string[][]`. Apply the pinned formatter, drop the duplicate import, list the `"devin"` format that its own definition already uses, and mark the pair array `as const`. Validation: `npm run check:fast` and `npm run format:check` pass. * fix(setup): keep Devin on the JSON configuration dialect --------- Co-authored-by: 吴佳浩 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> (cherry picked from commit df3d96da801976fdbe37a726c2f4f84af8ffbf90) --------- Co-authored-by: morluto Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Lucien <164857476+dakjdakd@users.noreply.github.com> Co-authored-by: 吴佳浩Alben <35291372+XiaoBinGan@users.noreply.github.com> Co-authored-by: 吴佳浩 Co-authored-by: OpenAI Codex --- docs/frida-instrumentation.md | 4 +- .../frida/FridaInstrumentationManager.test.ts | 22 + .../frida/FridaInstrumentationPort.ts | 23 +- .../frida/FridaInstrumentationService.test.ts | 72 ++ .../frida/FridaInstrumentationService.ts | 48 +- src/frida/FridaInstrumentationManager.ts | 202 +++++- src/server/registerFridaTools.ts | 43 +- tests/boundary/mcp/fridaLifecycleMcp.test.ts | 643 ++++++++++++++++++ 8 files changed, 1013 insertions(+), 44 deletions(-) create mode 100644 tests/boundary/mcp/fridaLifecycleMcp.test.ts diff --git a/docs/frida-instrumentation.md b/docs/frida-instrumentation.md index c93c76465..718af30c2 100644 --- a/docs/frida-instrumentation.md +++ b/docs/frida-instrumentation.md @@ -6,8 +6,8 @@ For remote devices, pass a host with an optional port (`host[:port]`; bracket IP MCP exposes `list_frida_devices`, `list_frida_processes`, `start_frida_session`, `load_frida_script`, `resume_frida_session`, `unload_frida_script`, `frida_session_status`, and `close_frida_session`. A spawned process starts paused so scripts can load first. Closing unloads session scripts and detaches. If a spawn remains paused, REA resumes it only after script cleanup or session detach confirms the instrumentation is released; if both cleanup paths fail, the target stays paused and the cleanup error is retained. REA never kills the target. MCP sessions end with the server process. The one-shot CLI command `instrument-with-frida` accepts one inline script or script file, observes messages for a bounded duration, and returns Evidence with the source hash and captured messages. -Failed startup cleanup retains acquired Frida handles, including a paused spawn that has not attached, for later cleanup retries. Its remote connection stays leased until session detach and target resume complete. CLI commands await provider cleanup before returning; MCP retains these resources for cleanup when its server closes. A failed cleanup remains reported rather than treating an unregistered session as fully released. +Failed startup cleanup retains acquired Frida handles, including a paused spawn that has not attached, for later cleanup retries. Its remote connection stays leased until session detach and target resume complete. CLI commands await provider cleanup before returning; MCP retains these resources for cleanup when its server closes. Server shutdown stops admitting new Frida operations and waits for admitted startup, discovery, and queued work before releasing their resources. A failed cleanup remains reported and can be retried by closing the server again, rather than treating an unregistered session as fully released. -Frida scripts execute inside the selected target with that process's memory, filesystem, and network authority. Message capture is bounded to 16 MiB and reports truncation. Script source is not stored in Evidence; its selected path and SHA-256 digest are. Evidence retains non-authentication target selection such as device or endpoint, mode, program or PID, and spawn arguments. Authentication values remain excluded. Results report the observed session and messages, while unsupported platform metadata remains unknown. A request cancelled before a Frida operation starts has no provider effects. If cancellation arrives during a native Frida call, REA waits for that call to return, then closes any session it created and returns the observed session as a partial observation. Cancellation Evidence separately reports whether a target resume was observed, whether instrumentation cleanup completed, and that final target liveness remains unknown. REA does not kill the target. Remote discovery retains its observations and reports a separate cleanup error if the temporary connection cannot be removed. +Frida scripts execute inside the selected target with that process's memory, filesystem, and network authority. Message capture is bounded to 16 MiB and reports truncation. Script source is not stored in Evidence; its selected path and SHA-256 digest are. Evidence retains non-authentication target selection such as device or endpoint, mode, program or PID, and spawn arguments. Authentication values remain excluded. Results report the observed session and messages, while unsupported platform metadata remains unknown. A request cancelled before a Frida operation starts has no provider effects. If cancellation arrives during a native Frida call, REA waits for that call to return, then closes any session it created and returns the observed session as a partial observation. A cancelled script load on an existing session releases only its new script before queued operations can run; the session and previously loaded scripts remain available. Once the provider commits a successful load, later request cancellation does not unload that script; use `unload_frida_script` to release it. If script cleanup fails, the script remains owned and a paused target cannot resume until the pending instrumentation is released. Queued script loads, resumes, and unloads check cancellation before dispatching native effects. Cancellation Evidence separately reports whether a target resume was observed, whether instrumentation cleanup completed, and that final target liveness remains unknown. REA does not kill the target. Remote discovery retains its observations and reports a separate cleanup error if the temporary connection cannot be removed. The [Frida Core API](https://frida.re/docs/frida-core-api/) documents device management, sessions, and script loading; the [JavaScript API](https://frida.re/docs/javascript-api/) documents target-side instrumentation. [Luma](https://luma.frida.re/) is Frida's standalone GUI; REA does not automate or control it. `npm run verify:frida` requires a Linux host with the optional Frida binding and uses a short-lived local Node.js fixture created by the verifier. Add `-- --address HOST:PORT --pid PID` to run the separate remote attach lane against a caller-selected target. Mock tests do not establish real device/server compatibility. diff --git a/src/application/frida/FridaInstrumentationManager.test.ts b/src/application/frida/FridaInstrumentationManager.test.ts index 7a527a807..d66280f66 100644 --- a/src/application/frida/FridaInstrumentationManager.test.ts +++ b/src/application/frida/FridaInstrumentationManager.test.ts @@ -1023,6 +1023,28 @@ it("serializes script loading against concurrent close and unloads the loaded sc expect(bindings.device.session.isDetached()).toBe(true); }); +it("drains startup when the provider loader initiates shutdown", async () => { + const bindings = new FakeDeviceManager(); + let closing: Promise | undefined; + const manager = new FridaInstrumentationManager({ + deviceManagerLoader: async () => { + closing = manager.closeAll(); + return bindings; + }, + }); + const started = await manager.startSession({ + mode: "spawn", + deviceId: "local", + program: "/tmp/fixture", + }); + if (!started.ok) throw started.error; + expect(closing).toBeDefined(); + await closing; + expect(bindings.device.session.isDetached()).toBe(true); + expect(bindings.device.resumed).toEqual([512]); + expect(manager.status(started.value.sessionId)).toBeUndefined(); +}); + it("names the unknown session or script ID it rejects", async () => { const { manager, started } = await startLocalAttachedSession(); if (!started.ok) throw started.error; diff --git a/src/application/frida/FridaInstrumentationPort.ts b/src/application/frida/FridaInstrumentationPort.ts index 39c51eb26..39fa30936 100644 --- a/src/application/frida/FridaInstrumentationPort.ts +++ b/src/application/frida/FridaInstrumentationPort.ts @@ -1,4 +1,8 @@ -import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { + AnalysisError, + AnalysisErrorOptions, +} from "../../domain/analysisErrorBase.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import type { OptionalProviderLoadFailure } from "../OptionalObservationProviders.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import type { Result } from "../../domain/result.js"; @@ -60,6 +64,15 @@ export interface FridaScriptObservation { readonly messagesTruncated: boolean; } +export class FridaScriptLoadCancelledError extends AnalysisCancelledError { + constructor( + readonly script: FridaScriptObservation, + options?: AnalysisErrorOptions, + ) { + super("load_frida_script", options); + } +} + export interface FridaSessionStatus extends FridaSessionObservation { /** True only after REA has successfully resumed this spawned target. */ readonly targetResumeObserved: boolean; @@ -100,14 +113,20 @@ export interface FridaInstrumentationPort { startSession( input: StartFridaSessionInput, ): Promise>; + /** Own cancellation and new-script cleanup before returning a final load outcome. */ loadScript( sessionId: string, source: FridaScriptSource, + signal?: AbortSignal, ): Promise>; - resumeSession(sessionId: string): Promise>; + resumeSession( + sessionId: string, + signal?: AbortSignal, + ): Promise>; unloadScript( sessionId: string, scriptId: string, + signal?: AbortSignal, ): Promise>; status(sessionId: string): FridaSessionStatus | undefined; closeSession( diff --git a/src/application/frida/FridaInstrumentationService.test.ts b/src/application/frida/FridaInstrumentationService.test.ts index c1251f9a5..dc25bc62b 100644 --- a/src/application/frida/FridaInstrumentationService.test.ts +++ b/src/application/frida/FridaInstrumentationService.test.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { expect, it } from "vitest"; +import { FridaScriptLoadCancelledError } from "./FridaInstrumentationPort.js"; import type { FridaInstrumentationPort } from "./FridaInstrumentationPort.js"; import { FridaInstrumentationService } from "./FridaInstrumentationService.js"; import { @@ -569,3 +570,74 @@ it("cleans up a session and returns typed cancellation after a pending start", a if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); expect(closed).toEqual(["00000000-0000-4000-8000-000000000001"]); }); + +it("retains a cancelled provider load's source and messages in one-shot Evidence", async () => { + const controller = new AbortController(); + const script = { + ...createScriptObservation(), + messages: [{ type: "send", payload: "cancelled-load" }], + }; + const service = new FridaInstrumentationService( + createProvider({ + async loadScript() { + controller.abort(new Error("cancel during native load")); + return { + ok: false, + error: new FridaScriptLoadCancelledError(script, { + cause: controller.signal.reason, + }), + }; + }, + }), + ); + const result = await service.instrument( + { + mode: "spawn", + program: "/fixture", + source: { sourceKind: "inline", source: "send('cancelled-load')" }, + durationMs: 0, + }, + controller.signal, + ); + if (result.ok) throw new Error("Cancelled instrumentation succeeded"); + expect(result.error.partialObservation).toMatchObject({ + normalized_result: { + source_kind: "inline", + source_sha256: script.sourceSha256, + messages: script.messages, + instrumentation_cleanup: "released", + }, + }); +}); + +it("preserves a provider-committed load when cancellation arrives during result delivery", async () => { + const controller = new AbortController(); + let unloads = 0; + const script = createScriptObservation(); + const service = new FridaInstrumentationService( + createProvider({ + loadScript() { + const completed = Promise.resolve({ ok: true as const, value: script }); + void completed.then(() => + controller.abort(new Error("late cancellation")), + ); + return completed; + }, + async unloadScript() { + unloads += 1; + return { ok: true, value: null }; + }, + }), + ); + const loaded = await service.loadScript( + createSession().sessionId, + { sourceKind: "inline", source: "send('ready')" }, + controller.signal, + ); + expect(controller.signal.aborted).toBe(true); + if (!loaded.ok) throw loaded.error; + expect(loaded.value.normalized_result).toMatchObject({ + script_id: script.scriptId, + }); + expect(unloads).toBe(0); +}); diff --git a/src/application/frida/FridaInstrumentationService.ts b/src/application/frida/FridaInstrumentationService.ts index c359850d6..3cba20d41 100644 --- a/src/application/frida/FridaInstrumentationService.ts +++ b/src/application/frida/FridaInstrumentationService.ts @@ -12,6 +12,7 @@ import { AnalysisInputError, } from "../../domain/analysisErrorCore.js"; import { err, ok, type Result } from "../../domain/result.js"; +import { FridaScriptLoadCancelledError } from "./FridaInstrumentationPort.js"; import type { FridaDeviceSelector, FridaInstrumentationPort, @@ -181,8 +182,17 @@ export class FridaInstrumentationService { async loadScript( sessionId: string, source: FridaScriptSource, + signal?: AbortSignal, ): Promise> { - const loaded = await this.#provider.loadScript(sessionId, source); + if (signal?.aborted) + return err( + new AnalysisCancelledError("load_frida_script", { + cause: signal.reason, + }), + ); + const loaded = await this.#provider.loadScript(sessionId, source, signal); + if (!loaded.ok && loaded.error instanceof FridaScriptLoadCancelledError) + return err(cancelledScriptEvidence(sessionId, loaded.error)); return loaded.ok ? ok( fridaEvidence("load_frida_script", scriptOutput(loaded.value), { @@ -192,12 +202,12 @@ export class FridaInstrumentationService { : loaded; } - resumeSession(sessionId: string) { - return this.#provider.resumeSession(sessionId); + resumeSession(sessionId: string, signal?: AbortSignal) { + return this.#provider.resumeSession(sessionId, signal); } - unloadScript(sessionId: string, scriptId: string) { - return this.#provider.unloadScript(sessionId, scriptId); + unloadScript(sessionId: string, scriptId: string, signal?: AbortSignal) { + return this.#provider.unloadScript(sessionId, scriptId, signal); } status(sessionId: string): FridaSessionStatus | undefined { @@ -243,14 +253,19 @@ export class FridaInstrumentationService { const loaded = await this.#provider.loadScript( started.value.sessionId, input.source, + signal, ); + let scriptObservation: FridaScriptObservation | undefined; + if (loaded.ok) scriptObservation = loaded.value; + else if (loaded.error instanceof FridaScriptLoadCancelledError) + scriptObservation = loaded.error.script; if (signal?.aborted) return this.#cancelSession( "instrument_with_frida", started.value, input, signal.reason, - loaded.ok ? loaded.value : undefined, + scriptObservation, ); if (!loaded.ok) { const closed = await this.#provider.closeSession(started.value.sessionId); @@ -267,6 +282,7 @@ export class FridaInstrumentationService { if (started.value.mode === "spawn") { const resumed = await this.#provider.resumeSession( started.value.sessionId, + signal, ); if (signal?.aborted) return this.#cancelSession( @@ -462,3 +478,23 @@ const sessionStatusOutput = (status: FridaSessionStatus) => ({ messages: status.messages, messages_truncated: status.messagesTruncated, }); + +const cancelledScriptEvidence = ( + sessionId: string, + error: FridaScriptLoadCancelledError, +): AnalysisCancelledError => + new AnalysisCancelledError(error.operation, { + cause: error.cause, + cleanup: error.cleanup, + partialObservation: fridaEvidence( + error.operation, + { + ...scriptOutput(error.script), + cleanup_error: error.cleanup?.reason ?? null, + script_cleanup: error.cleanupIncomplete + ? "incomplete_or_unknown" + : "released", + }, + { session_id: sessionId }, + ), + }); diff --git a/src/frida/FridaInstrumentationManager.ts b/src/frida/FridaInstrumentationManager.ts index 22b38c442..b93cffae5 100644 --- a/src/frida/FridaInstrumentationManager.ts +++ b/src/frida/FridaInstrumentationManager.ts @@ -22,9 +22,11 @@ import type { FridaSessionStatus, StartFridaSessionInput, } from "../application/frida/FridaInstrumentationPort.js"; +import { FridaScriptLoadCancelledError } from "../application/frida/FridaInstrumentationPort.js"; import type { OptionalProviderLoadFailure } from "../application/OptionalObservationProviders.js"; import { AnalysisAccessDeniedError, + AnalysisCancelledError, AnalysisCapabilityUnavailableError, AnalysisInputError, } from "../domain/analysisErrorCore.js"; @@ -58,7 +60,7 @@ interface ManagedScript { readonly name: string; readonly script: Script; readonly onMessage: (message: FridaMessage, data: Buffer | null) => void; - released: boolean; + state: "active" | "cleanup-pending" | "released"; } interface ManagedRemoteConnection { @@ -114,6 +116,9 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { readonly #deviceManagerLoader: () => Promise; #deviceManager: DeviceManager | undefined; #availabilityFailure: OptionalProviderLoadFailure | undefined; + readonly #operations = new Set>(); + #closing = false; + #closeAllPromise: Promise | undefined; constructor(options: FridaInstrumentationManagerOptions = {}) { this.#deviceManager = options.deviceManager; @@ -126,7 +131,13 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { return this.#availabilityFailure; } - async listDevices(remote?: FridaRemoteConnection): Promise< + listDevices(remote?: FridaRemoteConnection) { + return this.#withOperation("list_frida_devices", () => + this.#listDevices(remote), + ); + } + + async #listDevices(remote?: FridaRemoteConnection): Promise< Result< { readonly devices: readonly FridaDeviceObservation[]; @@ -176,7 +187,13 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { } } - async listProcesses(selector: FridaDeviceSelector): Promise< + listProcesses(selector: FridaDeviceSelector) { + return this.#withOperation("list_frida_processes", () => + this.#listProcesses(selector), + ); + } + + async #listProcesses(selector: FridaDeviceSelector): Promise< Result< { readonly deviceId: string; @@ -235,7 +252,13 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { } } - async startSession( + startSession(input: StartFridaSessionInput) { + return this.#withOperation("start_frida_session", () => + this.#startSession(input), + ); + } + + async #startSession( input: StartFridaSessionInput, ): Promise> { let manager: DeviceManager | undefined; @@ -326,16 +349,25 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { loadScript( sessionId: string, source: FridaScriptSource, + signal?: AbortSignal, ): Promise> { - return this.#withSessionLock(sessionId, () => - this.#loadScript(sessionId, source), + return this.#withOperation( + "load_frida_script", + () => + this.#withSessionLock(sessionId, () => + this.#loadScript(sessionId, source, signal), + ), + signal, ); } async #loadScript( sessionId: string, source: FridaScriptSource, + signal?: AbortSignal, ): Promise> { + if (signal?.aborted) + return err(cancelledOperation("load_frida_script", signal)); const managed = this.#sessions.get(sessionId); if (managed === undefined || managed.state === "detached") return err(unknownFridaSession("load_frida_script", sessionId)); @@ -368,6 +400,8 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { ); } + if (signal?.aborted) + return err(cancelledOperation("load_frida_script", signal)); const sourceSha256 = createHash("sha256") .update(scriptSource) .digest("hex"); @@ -387,19 +421,36 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { name: scriptName, script: nativeScript, onMessage, - released: false, + state: "active", }; managed.scripts.set(scriptId, entry); nativeScript.message.connect(onMessage); - await nativeScript.load(); - return ok({ + if (!signal?.aborted) await nativeScript.load(); + const observation: FridaScriptObservation = { scriptId, sourceKind: source.sourceKind, sourcePath: source.sourceKind === "file" ? source.path : null, sourceSha256, messages: managed.messagesByScript.get(scriptId)?.toArray() ?? [], messagesTruncated: managed.messagesTruncated, - }); + }; + if (signal?.aborted) { + const cleanupReason = await this.#releaseScript(managed, entry); + return err( + new FridaScriptLoadCancelledError(observation, { + cause: signal.reason, + ...(cleanupReason === undefined + ? {} + : { + cleanup: { + reason: cleanupReason, + resources: [sessionId, scriptId], + }, + }), + }), + ); + } + return ok(observation); } catch (cause: unknown) { const entry = managed.scripts.get(scriptId); const cleanupReason = @@ -418,9 +469,21 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { } } - resumeSession(sessionId: string): Promise> { - return this.#withSessionLock(sessionId, () => - this.#resumeSession(sessionId), + resumeSession( + sessionId: string, + signal?: AbortSignal, + ): Promise> { + return this.#withOperation( + "resume_frida_session", + () => + this.#withSessionLock(sessionId, () => + signal?.aborted + ? Promise.resolve( + err(cancelledOperation("resume_frida_session", signal)), + ) + : this.#resumeSession(sessionId), + ), + signal, ); } @@ -438,6 +501,26 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { "Only a paused process spawned by this session can be resumed.", ), ); + const pendingScripts = [...managed.scripts.values()].filter( + (entry) => entry.state === "cleanup-pending" && !entry.script.isDestroyed, + ); + if (pendingScripts.length > 0) { + const reason = + "Frida script cleanup is incomplete; unload the retained scripts or close the session before resuming."; + return err( + new AnalysisCapabilityUnavailableError( + "frida", + "resume_frida_session", + reason, + { + cleanup: { + reason, + resources: pendingScripts.map((entry) => entry.id), + }, + }, + ), + ); + } try { await managed.device.resume(managed.pid); managed.state = "running"; @@ -452,9 +535,19 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { unloadScript( sessionId: string, scriptId: string, + signal?: AbortSignal, ): Promise> { - return this.#withSessionLock(sessionId, () => - this.#unloadScript(sessionId, scriptId), + return this.#withOperation( + "unload_frida_script", + () => + this.#withSessionLock(sessionId, () => + signal?.aborted + ? Promise.resolve( + err(cancelledOperation("unload_frida_script", signal)), + ) + : this.#unloadScript(sessionId, scriptId), + ), + signal, ); } @@ -487,13 +580,14 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { parentSessionReleased = false, ): Promise { if (parentSessionReleased || entry.script.isDestroyed) - entry.released = true; - if (!entry.released) { + entry.state = "released"; + if (entry.state !== "released") { + entry.state = "cleanup-pending"; try { await entry.script.unload(); - entry.released = true; + entry.state = "released"; } catch (cause: unknown) { - if (entry.script.isDestroyed) entry.released = true; + if (entry.script.isDestroyed) entry.state = "released"; else return sanitizeError(cause, managed.remote); } } @@ -629,7 +723,17 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { return ok({ targetResumeObserved }); } - async closeAll(): Promise { + closeAll(): Promise { + this.#closing = true; + this.#closeAllPromise ??= this.#closeAll().catch((cause: unknown) => { + this.#closeAllPromise = undefined; + throw cause; + }); + return this.#closeAllPromise; + } + + async #closeAll(): Promise { + await Promise.allSettled(this.#operations); const startupFailures = ( await Promise.all( [...this.#pendingStartups].map((pending) => @@ -720,6 +824,46 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { } } + #withOperation( + name: string, + operation: () => Promise>, + signal?: AbortSignal, + ): Promise> { + if (this.#closing) + return Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + "frida", + name, + "Frida instrumentation is shutting down.", + ), + ), + ); + let start = () => {}; + const pending = new Promise>( + (resolve, reject) => { + start = () => { + try { + resolve( + signal?.aborted + ? err(cancelledOperation(name, signal)) + : operation(), + ); + } catch (cause: unknown) { + reject(cause); + } + }; + }, + ); + this.#operations.add(pending); + const release = () => { + this.#operations.delete(pending); + }; + void pending.then(release, release); + start(); + return pending; + } + async #withSessionLock( sessionId: string, operation: () => Promise, @@ -1217,6 +1361,18 @@ const providerError = ( remote?: FridaRemoteConnection, cleanupReason?: string, ): AnalysisError => { + if (cause instanceof AnalysisCancelledError) + return new AnalysisCancelledError(operation, { + cause: cause.cause, + ...(cleanupReason === undefined + ? {} + : { + cleanup: { + reason: cleanupReason, + resources: ["Frida provider operation"], + }, + }), + }); const reason = sanitizeError(cause, remote); return new AnalysisCapabilityUnavailableError("frida", operation, reason, { cause: remote === undefined ? cause : new Error(reason), @@ -1234,6 +1390,12 @@ const providerError = ( const errorMessage = (cause: unknown): string => cause instanceof Error ? cause.message : String(cause); +const cancelledOperation = ( + operation: string, + signal: AbortSignal, +): AnalysisCancelledError => + new AnalysisCancelledError(operation, { cause: signal.reason }); + interface CapturedMessage { readonly observation: JsonValue; readonly scriptId: string; diff --git a/src/server/registerFridaTools.ts b/src/server/registerFridaTools.ts index 5125f105c..fb2e06bb4 100644 --- a/src/server/registerFridaTools.ts +++ b/src/server/registerFridaTools.ts @@ -136,24 +136,35 @@ export const registerFridaTools = ( server.registerTool( load.name, toolRegistrationOptions(load), - async (input) => { + async (input, context: ServerContext) => { const result = await logToolExecution(logger, load.name, () => input.source_kind === "inline" && input.source !== undefined - ? service.loadScript(input.session_id, { - sourceKind: "inline", - source: input.source, - }) + ? service.loadScript( + input.session_id, + { + sourceKind: "inline", + source: input.source, + }, + context.mcpReq.signal, + ) : input.source_kind === "file" && input.path !== undefined - ? service.loadScript(input.session_id, { - sourceKind: "file", - path: input.path, - }) + ? service.loadScript( + input.session_id, + { + sourceKind: "file", + path: input.path, + }, + context.mcpReq.signal, + ) : Promise.resolve({ ok: false as const, error: new AnalysisInputError(load.name), }), ); - if (!result.ok) return delivery.toCallToolResult(result, load); + if (!result.ok) { + recordPartialEvidence(result.error, recordEvidence); + return delivery.toCallToolResult(result, load); + } return delivery.toEvidenceToolResult( result.value, load, @@ -164,9 +175,9 @@ export const registerFridaTools = ( server.registerTool( resume.name, toolRegistrationOptions(resume), - async (input) => { + async (input, context: ServerContext) => { const result = await logToolExecution(logger, resume.name, () => - service.resumeSession(input.session_id), + service.resumeSession(input.session_id, context.mcpReq.signal), ); return result.ok ? delivery.toCallToolResult( @@ -179,9 +190,13 @@ export const registerFridaTools = ( server.registerTool( unload.name, toolRegistrationOptions(unload), - async (input) => { + async (input, context: ServerContext) => { const result = await logToolExecution(logger, unload.name, () => - service.unloadScript(input.session_id, input.script_id), + service.unloadScript( + input.session_id, + input.script_id, + context.mcpReq.signal, + ), ); return result.ok ? delivery.toCallToolResult( diff --git a/tests/boundary/mcp/fridaLifecycleMcp.test.ts b/tests/boundary/mcp/fridaLifecycleMcp.test.ts new file mode 100644 index 000000000..2f1853125 --- /dev/null +++ b/tests/boundary/mcp/fridaLifecycleMcp.test.ts @@ -0,0 +1,643 @@ +import { createHash } from "node:crypto"; +import { setImmediate } from "node:timers/promises"; + +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { + Device, + DeviceManager, + FridaMessage, + Script, + Session, +} from "frida"; +import { expect, it, onTestFinished } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { evidenceBundleSchema } from "../../../src/domain/evidenceBundle.js"; +import { FridaInstrumentationManager } from "../../../src/frida/FridaInstrumentationManager.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +const gate = () => { + let markEntered = () => {}; + let release = () => {}; + const entered = new Promise((resolve) => { + markEntered = resolve; + }); + const released = new Promise((resolve) => { + release = resolve; + }); + return { + entered, + release, + async wait() { + markEntered(); + await released; + }, + }; +}; + +const nativeFixture = () => { + const created: string[] = []; + const loaded: string[] = []; + const unloaded: string[] = []; + const scripts: Script[] = []; + const scriptsAtResume: string[][] = []; + const controls = { + attach: async () => {}, + create: async () => {}, + load: async () => {}, + afterLoad: () => {}, + discover: async () => {}, + unloadFailures: 0, + detachFailures: 0, + detachCalls: 0, + resumes: 0, + remoteRemovals: 0, + removalFailures: 0, + }; + let detached = false; + const nativeSession: Session = { + pid: 512, + detached: { connect() {}, disconnect() {} }, + isDetached: () => detached, + async detach() { + controls.detachCalls += 1; + if (controls.detachFailures > 0) { + controls.detachFailures -= 1; + throw new Error("transient detach failure"); + } + detached = true; + }, + async createScript(source) { + created.push(source); + await controls.create(); + let destroyed = false; + let listener: + | ((message: FridaMessage, data: Buffer | null) => void) + | undefined; + const script: Script = { + get isDestroyed() { + return destroyed; + }, + message: { + connect(handler) { + listener = handler; + }, + disconnect() { + listener = undefined; + }, + }, + async load() { + await controls.load(); + loaded.push(source); + listener?.({ type: "send", payload: source }, null); + controls.afterLoad(); + }, + async unload() { + unloaded.push(source); + if (controls.unloadFailures > 0) { + controls.unloadFailures -= 1; + throw new Error("transient unload failure"); + } + destroyed = true; + }, + }; + scripts.push(script); + return script; + }, + }; + const device: Device = { + id: "local", + name: "Local fixture", + type: "local", + async enumerateProcesses() { + await controls.discover(); + return [{ pid: 512, name: "fixture" }]; + }, + async getProcessByPid() { + return { pid: 512, name: "fixture" }; + }, + async spawn() { + return 512; + }, + async resume() { + scriptsAtResume.push( + created.filter((_, index) => scripts[index]?.isDestroyed === false), + ); + controls.resumes += 1; + }, + async attach() { + await controls.attach(); + return nativeSession; + }, + }; + const deviceManager: DeviceManager = { + async enumerateDevices() { + return [device]; + }, + async getDeviceById() { + return device; + }, + async addRemoteDevice() { + return device; + }, + async removeRemoteDevice() { + controls.remoteRemovals += 1; + if (controls.removalFailures > 0) { + controls.removalFailures -= 1; + throw new Error("transient remote removal failure"); + } + }, + }; + return { + controls, + created, + loaded, + unloaded, + scripts, + scriptsAtResume, + deviceManager, + }; +}; + +const connectedFixture = async () => { + const native = nativeFixture(); + const manager = new FridaInstrumentationManager({ + deviceManager: native.deviceManager, + }); + const session = createTestBinarySession(() => { + throw new Error("Binary provider must not start"); + }); + const server = createServer( + { kind: "session", session }, + { fridaInstrumentation: manager }, + ); + const client = new Client({ name: "frida-lifecycle", version: "1" }); + onTestFinished(async () => { + native.controls.detachFailures = 0; + native.controls.removalFailures = 0; + await client.close(); + await server.close(); + await manager.closeAll(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const start = async () => { + const result = await client.callTool({ + name: "start_frida_session", + arguments: { + mode: "spawn", + program: "/fixture", + device_id: "local", + }, + }); + return toolContract("start_frida_session").outputSchema.parse( + result.structuredContent, + ).session_id; + }; + const load = (sessionId: string, source: string, signal?: AbortSignal) => + client.callTool( + { + name: "load_frida_script", + arguments: { session_id: sessionId, source_kind: "inline", source }, + }, + signal === undefined ? undefined : { signal }, + ); + return { ...native, manager, session, server, client, start, load }; +}; + +const scriptLoadFixture = async (stage: "create" | "load" = "load") => { + const fixture = await connectedFixture(); + const id = await fixture.start(); + const original = await fixture.load(id, "original"); + const held = gate(); + onTestFinished(held.release); + fixture.controls[stage] = held.wait; + return { fixture, id, original, held }; +}; + +const pendingShutdown = async (server: { close(): Promise }) => { + let closed = false; + const closing = server + .close() + .then( + () => undefined, + (cause: unknown) => cause, + ) + .finally(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + return { closing }; +}; + +const retryCancelledScriptUnload = async ( + fixture: Awaited>, + sessionId: string, + scriptId = fixture.manager.status(sessionId)?.scripts[1]?.scriptId, +) => { + if (scriptId === undefined) + throw new Error("Cancelled script was not retained"); + const retry = await fixture.client.callTool({ + name: "unload_frida_script", + arguments: { + session_id: sessionId, + script_id: scriptId, + }, + }); + expect(retry.isError).not.toBe(true); +}; + +const retainedScriptObservation = async ( + fixture: Awaited>, + sessionId: string, + source: string, +) => { + const response = await fixture.client.callTool({ + name: "get_evidence_bundle", + arguments: {}, + }); + const bundle = evidenceBundleSchema.parse( + toolContract("get_evidence_bundle").outputSchema.parse( + response.structuredContent, + ).result, + ); + const sourceSha256 = createHash("sha256").update(source).digest("hex"); + const records = bundle.records.filter((record) => { + const result = record.normalized_result; + return ( + record.operation === "load_frida_script" && + result !== null && + typeof result === "object" && + !Array.isArray(result) && + result.source_sha256 === sourceSha256 + ); + }); + expect(records).toHaveLength(1); + expect(records[0]?.parameters).toEqual({ session_id: sessionId }); + const observation = records[0]?.normalized_result; + if ( + observation === undefined || + observation === null || + typeof observation !== "object" || + Array.isArray(observation) + ) + throw new Error("Missing retained script observation"); + expect(observation).toMatchObject({ + script_id: expect.any(String), + source_kind: "inline", + source_path: null, + source_sha256: sourceSha256, + messages_truncated: false, + }); + return observation; +}; + +it.each(["load", "resume", "unload"] as const)( + "finishes an earlier MCP %s request before closing its Frida session", + async (operation) => { + const fixture = await connectedFixture(); + const id = await fixture.start(); + const original = await fixture.load(id, "original"); + const scriptId = toolContract("load_frida_script").outputSchema.parse( + original.structuredContent, + ).normalized_result.script_id; + const pending = fixture.client.callTool({ + name: { + load: "load_frida_script", + resume: "resume_frida_session", + unload: "unload_frida_script", + }[operation], + arguments: { + session_id: id, + ...(operation === "load" + ? { source_kind: "inline", source: "last-script" } + : {}), + ...(operation === "unload" ? { script_id: scriptId } : {}), + }, + }); + const closing = fixture.client.callTool({ + name: "close_frida_session", + arguments: { session_id: id }, + }); + expect((await pending).isError).not.toBe(true); + expect((await closing).isError).not.toBe(true); + expect(fixture.loaded).toEqual( + operation === "load" ? ["original", "last-script"] : ["original"], + ); + expect(fixture.unloaded).toEqual(fixture.loaded); + expect(fixture.controls.detachCalls).toBe(1); + expect(fixture.controls.resumes).toBe(1); + expect(fixture.manager.status(id)).toBeUndefined(); + }, +); + +it.each(["load", "resume", "unload"] as const)( + "does not dispatch a cancelled queued Frida %s through MCP", + async (operation) => { + const { fixture, id, original, held } = await scriptLoadFixture(); + const scriptId = toolContract("load_frida_script").outputSchema.parse( + original.structuredContent, + ).normalized_result.script_id; + const first = fixture.load(id, "blocking"); + await held.entered; + const controller = new AbortController(); + const cancelled = + operation === "load" + ? fixture.load(id, "cancelled", controller.signal) + : fixture.client.callTool( + { + name: + operation === "resume" + ? "resume_frida_session" + : "unload_frida_script", + arguments: { + session_id: id, + ...(operation === "unload" ? { script_id: scriptId } : {}), + }, + }, + { signal: controller.signal }, + ); + const rejected = expect(cancelled).rejects.toThrow( + "cancel queued operation", + ); + await fixture.client.ping(); + controller.abort(new Error("cancel queued operation")); + await fixture.client.ping(); + await rejected; + held.release(); + await first; + await fixture.load(id, "barrier"); + expect(fixture.created).toEqual(["original", "blocking", "barrier"]); + expect(fixture.unloaded).toEqual([]); + expect(fixture.controls.resumes).toBe(0); + expect(fixture.manager.status(id)?.scripts).toHaveLength(3); + }, +); + +it.each(["create", "load"] as const)( + "releases only the new script when MCP cancellation arrives during native %s", + async (stage) => { + const { fixture, id, held } = await scriptLoadFixture(stage); + const controller = new AbortController(); + const cancelled = fixture.load(id, "cancelled-native", controller.signal); + const rejected = expect(cancelled).rejects.toThrow( + "cancel native operation", + ); + await held.entered; + controller.abort(new Error("cancel native operation")); + await fixture.client.ping(); + await rejected; + held.release(); + await fixture.load(id, "barrier"); + await fixture.client.ping(); + expect(fixture.unloaded).toEqual(["cancelled-native"]); + expect(fixture.loaded).toEqual( + stage === "create" + ? ["original", "barrier"] + : ["original", "cancelled-native", "barrier"], + ); + expect(fixture.controls.detachCalls).toBe(0); + expect(fixture.controls.resumes).toBe(0); + expect(fixture.manager.status(id)?.scripts).toHaveLength(2); + const observation = await retainedScriptObservation( + fixture, + id, + "cancelled-native", + ); + expect(observation).toMatchObject({ + messages: + stage === "create" + ? [] + : [expect.objectContaining({ payload: "cancelled-native" })], + script_cleanup: "released", + cleanup_error: null, + }); + }, +); + +it("awaits an admitted attach on server shutdown and retries failed native cleanup", async () => { + const fixture = await connectedFixture(); + const held = gate(); + onTestFinished(held.release); + fixture.controls.attach = held.wait; + fixture.controls.detachFailures = 1; + const starting = fixture.client + .callTool({ + name: "start_frida_session", + arguments: { + mode: "attach", + pid: 512, + device_id: "local", + }, + }) + .catch(() => undefined); + await held.entered; + const { closing } = await pendingShutdown(fixture.server); + expect(fixture.controls.detachCalls).toBe(0); + held.release(); + expect(await closing).toBeInstanceOf(AggregateError); + await starting; + expect(fixture.controls.detachCalls).toBe(1); + await fixture.server.close(); + expect(fixture.controls.detachCalls).toBe(2); + await fixture.server.close(); + expect(fixture.controls.detachCalls).toBe(2); + const afterClose = await fixture.manager.startSession({ + mode: "attach", + pid: 512, + }); + expect(afterClose).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError" }, + }); +}); + +it("drains remote discovery before shutdown and retains failed connection cleanup", async () => { + const fixture = await connectedFixture(); + const held = gate(); + onTestFinished(held.release); + fixture.controls.discover = held.wait; + fixture.controls.removalFailures = 2; + const discovering = fixture.client + .callTool({ + name: "list_frida_processes", + arguments: { + remote: { address: "127.0.0.1:27042" }, + }, + }) + .catch(() => undefined); + await held.entered; + const { closing } = await pendingShutdown(fixture.server); + held.release(); + expect(await closing).toBeInstanceOf(AggregateError); + await discovering; + expect(fixture.controls.remoteRemovals).toBe(2); + await fixture.server.close(); + expect(fixture.controls.remoteRemovals).toBe(3); +}); + +it.each(["create", "load"] as const)( + "retains the identity of a cancelled native %s for an explicit MCP unload retry", + async (stage) => { + const { fixture, id, held } = await scriptLoadFixture(stage); + fixture.controls.unloadFailures = 1; + const controller = new AbortController(); + const cancelled = fixture.load(id, "cancelled-native", controller.signal); + const rejected = expect(cancelled).rejects.toThrow( + "cancel retained script", + ); + await held.entered; + controller.abort(new Error("cancel retained script")); + await fixture.client.ping(); + await rejected; + held.release(); + await fixture.load(id, "barrier"); + await fixture.client.ping(); + expect(fixture.manager.status(id)?.scripts).toHaveLength(3); + const observation = await retainedScriptObservation( + fixture, + id, + "cancelled-native", + ); + expect(observation).toMatchObject({ + messages: + stage === "create" + ? [] + : [expect.objectContaining({ payload: "cancelled-native" })], + script_cleanup: "incomplete_or_unknown", + cleanup_error: expect.stringContaining("transient unload failure"), + }); + const scriptId = observation.script_id; + if (typeof scriptId !== "string") + throw new Error("Missing cancelled script identity"); + expect(fixture.manager.status(id)?.scripts[1]?.scriptId).toBe(scriptId); + const resuming = await fixture.client.callTool({ + name: "resume_frida_session", + arguments: { session_id: id }, + }); + expect(resuming.isError).toBe(true); + expect(fixture.controls.resumes).toBe(0); + await retryCancelledScriptUnload(fixture, id, scriptId); + expect(fixture.unloaded).toEqual(["cancelled-native", "cancelled-native"]); + expect(fixture.loaded).toEqual( + stage === "create" + ? ["original", "barrier"] + : ["original", "cancelled-native", "barrier"], + ); + expect(fixture.manager.status(id)?.scripts).toHaveLength(2); + expect(fixture.controls.detachCalls).toBe(0); + }, +); + +it.each([0, 1])( + "settles cancelled native-load cleanup before a queued resume (unload failures: %i)", + async (unloadFailures) => { + const { fixture, id, held } = await scriptLoadFixture(); + fixture.controls.unloadFailures = unloadFailures; + const controller = new AbortController(); + const loading = fixture.load(id, "cancelled-native", controller.signal); + const rejected = expect(loading).rejects.toThrow( + "cancel before queued resume", + ); + await held.entered; + const resume = () => + fixture.client.callTool({ + name: "resume_frida_session", + arguments: { session_id: id }, + }); + const resuming = resume(); + await fixture.client.ping(); + controller.abort(new Error("cancel before queued resume")); + await fixture.client.ping(); + await rejected; + held.release(); + const resumed = await resuming; + await fixture.client.ping(); + expect(fixture.unloaded).toEqual(["cancelled-native"]); + expect(fixture.scriptsAtResume).toEqual( + unloadFailures === 0 ? [["original"]] : [], + ); + expect(fixture.controls.detachCalls).toBe(0); + expect(fixture.session.exportEvidenceBundle().records).toContainEqual( + expect.objectContaining({ + operation: "load_frida_script", + normalized_result: expect.objectContaining({ + messages: [expect.objectContaining({ payload: "cancelled-native" })], + script_cleanup: + unloadFailures === 0 ? "released" : "incomplete_or_unknown", + }), + }), + ); + if (unloadFailures === 0) { + expect(resumed.isError).not.toBe(true); + expect(fixture.manager.status(id)?.scripts).toHaveLength(1); + return; + } + expect(resumed.isError).toBe(true); + expect(fixture.manager.status(id)?.state).toBe("paused"); + expect((await resume()).isError).toBe(true); + await retryCancelledScriptUnload(fixture, id); + expect((await resume()).isError).not.toBe(true); + expect(fixture.scriptsAtResume).toEqual([["original"]]); + }, +); + +it.each([0, 1, 2, 3, 4, 8])( + "keeps MCP load Evidence consistent with queued resume across settlement timing %i", + async (microtasks) => { + const { fixture, id, held } = await scriptLoadFixture(); + const controller = new AbortController(); + fixture.controls.afterLoad = () => { + let remaining = microtasks; + const abort = () => { + if (remaining-- > 0) queueMicrotask(abort); + else controller.abort(new Error("cancel around load completion")); + }; + abort(); + }; + const loading = fixture.load(id, "settling-native", controller.signal); + const observed = loading.catch(() => undefined); + await held.entered; + const resuming = fixture.client.callTool({ + name: "resume_frida_session", + arguments: { session_id: id }, + }); + await fixture.client.ping(); + held.release(); + await observed; + expect((await resuming).isError).not.toBe(true); + fixture.controls.afterLoad = () => {}; + await fixture.load(id, "barrier"); + await fixture.client.ping(); + const records = fixture.session + .exportEvidenceBundle() + .records.filter( + (record) => + record.operation === "load_frida_script" && + JSON.stringify(record.normalized_result).includes("settling-native"), + ); + expect(records).toHaveLength(1); + const recorded = records[0]?.normalized_result; + if ( + recorded === null || + typeof recorded !== "object" || + Array.isArray(recorded) + ) + throw new Error("Missing script load Evidence"); + if (recorded.script_cleanup === "released") { + expect(fixture.scriptsAtResume).toEqual([["original"]]); + expect(fixture.unloaded).toEqual(["settling-native"]); + } else { + expect(recorded.script_cleanup).toBeUndefined(); + expect(fixture.scriptsAtResume).toEqual([ + ["original", "settling-native"], + ]); + expect(fixture.unloaded).toEqual([]); + } + }, +); From 729ca2ab6ae0644b5a466534f9287961bf7152db Mon Sep 17 00:00:00 2001 From: Tufail Akram <72789060+akram1089@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:26:26 +0530 Subject: [PATCH 08/15] fix(ghidra): report unapplied PE CodeView PDB limitation (#2283) * fix(ghidra): report unapplied PE CodeView PDB limitation (#2259) When Ghidra analyzes a PE target that references companion debug symbols in its CodeView (IMAGE_DEBUG_TYPE_CODEVIEW / RSDS or NB10) debug directory, Ghidra's headless analyzer cannot automatically locate or apply the companion PDB from isolated runtime directories without companion admission. Procedure names remain analysis-generated (FUN_
). Parse CodeView debug metadata (RSDS GUID/age and NB10 age) during PE target resolution, and report an Evidence limitation informing callers and agents that referenced debug symbols were not applied and procedure names are analysis-generated. Fixes #2259 * test(pe): group CodeView header coverage separately * fix(ghidra): avoid contradictory PDB limitations * fix(android): make DEX descriptor pattern portable * fix(ci): isolate Linux Hopper process fixture inventory * fix(comparison): recheck historical calls-only changes (#2223) * fix(comparison): preserve incomplete function call evidence Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty. Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls. * fix(comparison): recheck historical calls-only changes --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> * style: format identity comparison and IDA runner --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Lucien <164857476+dakjdakd@users.noreply.github.com> --- src/application/BinaryTargetResolver.ts | 98 +++++++++- src/domain/binaryTarget.fixture.ts | 74 +++++++- src/domain/binaryTarget.ts | 173 +++++++++++++++++- src/domain/binaryTargetTypes.ts | 10 + src/ghidra/GhidraPeProfile.test.ts | 95 ++++++++++ src/ghidra/GhidraPeProfile.ts | 24 +++ src/ghidra/GhidraProvider.behavior.test.ts | 33 ++++ src/ghidra/GhidraProviderClient.ts | 7 +- .../boundary/filesystem/binaryTarget.test.ts | 100 ++++++++++ 9 files changed, 607 insertions(+), 7 deletions(-) create mode 100644 src/ghidra/GhidraPeProfile.test.ts create mode 100644 src/ghidra/GhidraPeProfile.ts diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index b07364adf..348454e72 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -40,10 +40,14 @@ import { } from "../domain/zipPackageFormat.js"; import { + parseCodeViewData, parseExecutableHeader, parsePeRecord, + peDebugDirectoryEntry, + peRvaToFileOffset, type ExecutableMetadata, } from "../domain/binaryTarget.js"; +import type { PeCodeViewRecord } from "../domain/binaryTargetTypes.js"; /** Options for resolving a local binary target and its stable identity. */ export interface BinaryTargetResolverOptions { @@ -549,10 +553,98 @@ const readPeMetadata = async ( return err(`unsupported ${windowsHeader.signature} executable in MZ image`); const optionalHeaderSize = fileHeader.readUInt16LE(20); if (optionalHeaderSize > 4096) return err("invalid PE optional header size"); + const numberOfSections = fileHeader.readUInt16LE(6); const record = Buffer.alloc(24 + optionalHeaderSize); const recordRead = await handle.read(record, 0, record.length, offset); throwIfTargetResolutionCancelled(signal); - return recordRead.bytesRead === record.length - ? parsePeRecord(record) - : err("invalid or truncated PE header"); + if (recordRead.bytesRead !== record.length) + return err("invalid or truncated PE header"); + const codeView = await readPeCodeView( + handle, + offset, + numberOfSections, + record.subarray(24), + optionalHeaderSize, + signal, + ); + return parsePeRecord(record, codeView); +}; + +const readPeCodeView = async ( + handle: FileHandle, + peOffset: number, + numberOfSections: number, + optionalHeader: Buffer, + optionalHeaderSize: number, + signal?: AbortSignal, +): Promise => { + throwIfTargetResolutionCancelled(signal); + const debugDir = peDebugDirectoryEntry(optionalHeader); + if (debugDir === undefined || numberOfSections > 96) return undefined; + const sectionTableOffset = peOffset + 24 + optionalHeaderSize; + const sectionTableSize = numberOfSections * 40; + const sectionTable = Buffer.alloc(sectionTableSize); + const sectionRead = await handle.read( + sectionTable, + 0, + sectionTableSize, + sectionTableOffset, + ); + throwIfTargetResolutionCancelled(signal); + if (sectionRead.bytesRead !== sectionTableSize) return undefined; + const debugFileOffset = peRvaToFileOffset( + debugDir.debugRva, + sectionTable, + numberOfSections, + debugDir.sizeOfHeaders, + Math.min(debugDir.debugSize, 4096), + ); + if (debugFileOffset === undefined) return undefined; + const maxDebugSize = Math.min(debugDir.debugSize, 4096); + const debugDirectoryBuffer = Buffer.alloc(maxDebugSize); + const debugDirRead = await handle.read( + debugDirectoryBuffer, + 0, + maxDebugSize, + debugFileOffset, + ); + throwIfTargetResolutionCancelled(signal); + const numEntries = Math.floor(debugDirRead.bytesRead / 28); + for (let j = 0; j < numEntries; j++) { + const entryOffset = j * 28; + const type = debugDirectoryBuffer.readUInt32LE(entryOffset + 12); + if (type !== 2) continue; // IMAGE_DEBUG_TYPE_CODEVIEW + const sizeOfData = debugDirectoryBuffer.readUInt32LE(entryOffset + 16); + let pointerToRawData = debugDirectoryBuffer.readUInt32LE(entryOffset + 24); + if (pointerToRawData === 0) { + const addressOfRawData = debugDirectoryBuffer.readUInt32LE( + entryOffset + 20, + ); + const mapped = peRvaToFileOffset( + addressOfRawData, + sectionTable, + numberOfSections, + debugDir.sizeOfHeaders, + Math.min(sizeOfData, 4096), + ); + if (mapped === undefined) continue; + pointerToRawData = mapped; + } + if (sizeOfData < 16) continue; + const maxCodeViewRead = Math.min(sizeOfData, 4096); + const codeViewBuffer = Buffer.alloc(maxCodeViewRead); + const codeViewRead = await handle.read( + codeViewBuffer, + 0, + maxCodeViewRead, + pointerToRawData, + ); + throwIfTargetResolutionCancelled(signal); + const parsed = parseCodeViewData( + codeViewBuffer.subarray(0, codeViewRead.bytesRead), + sizeOfData, + ); + if (parsed !== undefined) return parsed; + } + return undefined; }; diff --git a/src/domain/binaryTarget.fixture.ts b/src/domain/binaryTarget.fixture.ts index 7d8a0a7f9..345a3f24c 100644 --- a/src/domain/binaryTarget.fixture.ts +++ b/src/domain/binaryTarget.fixture.ts @@ -58,20 +58,67 @@ export const fat = ( return bytes; }; -/** Build a bounded PE image with optional managed metadata. */ +export interface PeCodeViewFixture { + readonly type?: "RSDS" | "NB10"; + readonly guid?: string; + readonly age?: number; + readonly pdbPath?: string; +} + +/** Build a bounded PE image with optional managed metadata or CodeView debug data. */ export const pe = ( machine: number, offset = 64, characteristics = 0x0002, managed = false, + codeView?: PeCodeViewFixture, ): Buffer => { const optionalHeaderSize = machine === 0x014c || machine === 0x01c0 ? 224 : 240; - const bytes = Buffer.alloc(Math.max(512, offset + 24 + optionalHeaderSize)); + const sectionTableOffset = offset + 24 + optionalHeaderSize; + const sectionCount = codeView !== undefined ? 1 : 0; + const rawDataOffset = 512; + + let codeViewData: Buffer | undefined; + if (codeView !== undefined) { + const type = codeView.type ?? "RSDS"; + const pdbPath = codeView.pdbPath ?? "app.pdb"; + const age = codeView.age ?? 1; + const pathBuf = Buffer.from(`${pdbPath}\0`, "utf8"); + if (type === "RSDS") { + codeViewData = Buffer.alloc(24 + pathBuf.length); + codeViewData.write("RSDS", 0, "ascii"); + const guid = codeView.guid ?? "01234567-89ab-cdef-0123-456789abcdef"; + const parts = guid.split("-"); + if (parts.length === 5) { + codeViewData.writeUInt32LE(parseInt(parts[0]!, 16), 4); + codeViewData.writeUInt16LE(parseInt(parts[1]!, 16), 8); + codeViewData.writeUInt16LE(parseInt(parts[2]!, 16), 10); + Buffer.from(parts[3]!, "hex").copy(codeViewData, 12); + Buffer.from(parts[4]!, "hex").copy(codeViewData, 14); + } + codeViewData.writeUInt32LE(age, 20); + pathBuf.copy(codeViewData, 24); + } else { + codeViewData = Buffer.alloc(16 + pathBuf.length); + codeViewData.write("NB10", 0, "ascii"); + codeViewData.writeUInt32LE(0, 4); + codeViewData.writeUInt32LE(0, 8); + codeViewData.writeUInt32LE(age, 12); + pathBuf.copy(codeViewData, 16); + } + } + + const minSize = + codeViewData !== undefined + ? rawDataOffset + 28 + codeViewData.length + 32 + : Math.max(512, offset + 24 + optionalHeaderSize); + const bytes = Buffer.alloc(minSize); bytes.write("MZ", 0, "ascii"); bytes.writeUInt32LE(offset, 0x3c); bytes.write("PE\0\0", offset, "binary"); bytes.writeUInt16LE(machine, offset + 4); + bytes.writeUInt16LE(sectionCount, offset + 6); bytes.writeUInt16LE(optionalHeaderSize, offset + 20); bytes.writeUInt16LE(characteristics, offset + 22); const optionalHeader = offset + 24; @@ -83,6 +130,29 @@ export const pe = ( bytes.writeUInt32LE(0x2000, optionalHeader + directoryOffset + 14 * 8); bytes.writeUInt32LE(72, optionalHeader + directoryOffset + 14 * 8 + 4); } + if (codeViewData !== undefined) { + const debugDirRva = 0x1000; + const debugDirSize = 28; + bytes.writeUInt32LE(debugDirRva, optionalHeader + directoryOffset + 6 * 8); + bytes.writeUInt32LE( + debugDirSize, + optionalHeader + directoryOffset + 6 * 8 + 4, + ); + + bytes.write(".rdata\0\0", sectionTableOffset, "ascii"); + bytes.writeUInt32LE(0x1000, sectionTableOffset + 8); + bytes.writeUInt32LE(0x1000, sectionTableOffset + 12); + bytes.writeUInt32LE(512, sectionTableOffset + 16); + bytes.writeUInt32LE(rawDataOffset, sectionTableOffset + 20); + bytes.writeUInt32LE(0x40000040, sectionTableOffset + 36); + + bytes.writeUInt32LE(2, rawDataOffset + 12); + bytes.writeUInt32LE(codeViewData.length, rawDataOffset + 16); + bytes.writeUInt32LE(debugDirRva + 28, rawDataOffset + 20); + bytes.writeUInt32LE(rawDataOffset + 28, rawDataOffset + 24); + + codeViewData.copy(bytes, rawDataOffset + 28); + } return bytes; }; diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 94828ee6e..4e668e460 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -3,6 +3,7 @@ import type { ElfFamilyMetadata, HeaderArchitecture, MipsElfMetadata, + PeCodeViewRecord, RiscvElfMetadata, UniversalMachOSlice, } from "./binaryTargetTypes.js"; @@ -50,6 +51,7 @@ export type ExecutableMetadata = | "shared-library" | "non-executable"; readonly managed: boolean; + readonly codeView?: PeCodeViewRecord; }; /** @@ -257,7 +259,8 @@ const parseMz = ( ); if (windowsHeader.signature !== "PE") return err("invalid or truncated Windows executable header in MZ image"); - return parsePeRecord(bytes.subarray(offset)); + const codeView = parsePeCodeViewFromBuffer(bytes, offset); + return parsePeRecord(bytes.subarray(offset), codeView); } const parsed = parseDosMzHeader(bytes, fileSize, signatureAtOffset); return parsed.ok @@ -272,6 +275,7 @@ const parseMz = ( /** Parse a bounded PE signature, COFF record, and optional header without I/O. */ export const parsePeRecord = ( record: Buffer, + codeView: PeCodeViewRecord | undefined, ): Result => { if ( record.length < 24 || @@ -305,9 +309,176 @@ export const parsePeRecord = ( ? "application" : "shared-library", managed: managed.value, + ...(codeView !== undefined ? { codeView } : {}), }); }; +/** Parse CodeView record data (RSDS or NB10) from raw data bytes. */ +export const parseCodeViewData = ( + buffer: Buffer, + sizeOfData: number, +): PeCodeViewRecord | undefined => { + if (sizeOfData < 16 || buffer.length < 16) return undefined; + const magic = buffer.subarray(0, 4).toString("binary"); + const pathOffset = magic === "RSDS" ? 24 : magic === "NB10" ? 16 : undefined; + if ( + pathOffset === undefined || + sizeOfData <= pathOffset || + buffer.length <= pathOffset + ) + return undefined; + const end = buffer.indexOf(0, pathOffset); + if (end <= pathOffset || end >= sizeOfData) return undefined; + const pdbPath = buffer.toString("utf8", pathOffset, end); + const pdbFileName = pdbPath.split(/[/\\]/u).pop() ?? pdbPath; + if (magic === "RSDS" && sizeOfData >= 24 && buffer.length >= 24) { + const part1 = buffer.readUInt32LE(4).toString(16).padStart(8, "0"); + const part2 = buffer.readUInt16LE(8).toString(16).padStart(4, "0"); + const part3 = buffer.readUInt16LE(10).toString(16).padStart(4, "0"); + const part4 = buffer.subarray(12, 14).toString("hex"); + const part5 = buffer.subarray(14, 20).toString("hex"); + const guid = `${part1}-${part2}-${part3}-${part4}-${part5}`; + const age = buffer.readUInt32LE(20); + return { pdbPath, pdbFileName, guid, age }; + } + if (magic === "NB10") { + const age = buffer.readUInt32LE(12); + return { pdbPath, pdbFileName, guid: null, age }; + } + return undefined; +}; + +/** Locate the debug directory RVA and size from a PE optional header. */ +export const peDebugDirectoryEntry = ( + optionalHeader: Buffer, +): + | { + readonly debugRva: number; + readonly debugSize: number; + readonly sizeOfHeaders: number; + } + | undefined => { + if (optionalHeader.length < 2) return undefined; + const magic = optionalHeader.readUInt16LE(0); + if (magic !== 0x10b && magic !== 0x20b) return undefined; + const directoryOffset = magic === 0x20b ? 112 : 96; + const directoryCountOffset = directoryOffset - 4; + if (optionalHeader.length < directoryCountOffset + 4) return undefined; + const directoryCount = optionalHeader.readUInt32LE(directoryCountOffset); + if (directoryCount <= 6) return undefined; + const debugEntryOffset = directoryOffset + 6 * 8; + if (optionalHeader.length < debugEntryOffset + 8) return undefined; + const debugRva = optionalHeader.readUInt32LE(debugEntryOffset); + const debugSize = optionalHeader.readUInt32LE(debugEntryOffset + 4); + if (debugRva === 0 || debugSize < 28) return undefined; + return { + debugRva, + debugSize, + sizeOfHeaders: optionalHeader.readUInt32LE(60), + }; +}; + +/** Translate an RVA to an absolute file offset using the PE section table. */ +export const peRvaToFileOffset = ( + rva: number, + sectionTable: Buffer, + numberOfSections: number, + sizeOfHeaders: number, + size = 1, +): number | undefined => { + if (rva < sizeOfHeaders && size <= sizeOfHeaders - rva) return rva; + for (let i = 0; i < numberOfSections; i++) { + const offset = i * 40; + if (sectionTable.length < offset + 40) break; + const virtualAddress = sectionTable.readUInt32LE(offset + 12); + const sizeOfRawData = sectionTable.readUInt32LE(offset + 16); + const pointerToRawData = sectionTable.readUInt32LE(offset + 20); + if ( + pointerToRawData > 0 && + rva >= virtualAddress && + rva - virtualAddress < sizeOfRawData && + size <= sizeOfRawData - (rva - virtualAddress) + ) { + return pointerToRawData + (rva - virtualAddress); + } + } + return undefined; +}; + +/** Parse CodeView metadata from a bounded in-memory PE image buffer. */ +export const parsePeCodeViewFromBuffer = ( + fileBytes: Buffer, + peOffset: number, +): PeCodeViewRecord | undefined => { + if (fileBytes.length < peOffset + 24) return undefined; + const numberOfSections = fileBytes.readUInt16LE(peOffset + 6); + const optionalHeaderSize = fileBytes.readUInt16LE(peOffset + 20); + if ( + optionalHeaderSize > 4096 || + fileBytes.length < peOffset + 24 + optionalHeaderSize + ) + return undefined; + const optionalHeader = fileBytes.subarray( + peOffset + 24, + peOffset + 24 + optionalHeaderSize, + ); + const debugDir = peDebugDirectoryEntry(optionalHeader); + if (debugDir === undefined) return undefined; + const sectionTableOffset = peOffset + 24 + optionalHeaderSize; + const sectionTableSize = numberOfSections * 40; + if (fileBytes.length < sectionTableOffset + sectionTableSize) + return undefined; + const sectionTable = fileBytes.subarray( + sectionTableOffset, + sectionTableOffset + sectionTableSize, + ); + const debugFileOffset = peRvaToFileOffset( + debugDir.debugRva, + sectionTable, + numberOfSections, + debugDir.sizeOfHeaders, + Math.min(debugDir.debugSize, 4096), + ); + if (debugFileOffset === undefined || fileBytes.length < debugFileOffset + 28) + return undefined; + const maxDebugBytes = Math.min( + debugDir.debugSize, + fileBytes.length - debugFileOffset, + ); + const numEntries = Math.floor(maxDebugBytes / 28); + for (let j = 0; j < numEntries; j++) { + const entryOffset = debugFileOffset + j * 28; + const type = fileBytes.readUInt32LE(entryOffset + 12); + if (type !== 2) continue; // IMAGE_DEBUG_TYPE_CODEVIEW + const sizeOfData = fileBytes.readUInt32LE(entryOffset + 16); + let pointerToRawData = fileBytes.readUInt32LE(entryOffset + 24); + if (pointerToRawData === 0) { + const addressOfRawData = fileBytes.readUInt32LE(entryOffset + 20); + const mapped = peRvaToFileOffset( + addressOfRawData, + sectionTable, + numberOfSections, + debugDir.sizeOfHeaders, + Math.min(sizeOfData, 4096), + ); + if (mapped === undefined) continue; + pointerToRawData = mapped; + } + if ( + sizeOfData < 16 || + fileBytes.length < pointerToRawData + Math.min(sizeOfData, 24) + ) + continue; + const codeViewData = fileBytes.subarray( + pointerToRawData, + Math.min(fileBytes.length, pointerToRawData + sizeOfData), + ); + const parsed = parseCodeViewData(codeViewData, sizeOfData); + if (parsed !== undefined) return parsed; + } + return undefined; +}; + const peManagedStatus = ( optionalHeader: Buffer, architecture: BinaryArchitecture, diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index 99ad8487f..8c16612bc 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -24,6 +24,14 @@ export interface ElfFamilyMetadata { readonly flags: number; } +/** Referenced CodeView debug symbols from a PE debug directory. */ +export interface PeCodeViewRecord { + readonly pdbPath: string; + readonly pdbFileName: string; + readonly guid: string | null; + readonly age: number; +} + /** Families whose parsed target variant carries `ElfFamilyMetadata`. */ export const ELF_METADATA_ARCHITECTURES = [ "mips", @@ -170,6 +178,8 @@ export type BinaryTarget = | "non-executable"; /** Whether the PE declares a non-empty CLI header data-directory entry. */ readonly managed: boolean; + /** Referenced CodeView debug symbols from the PE debug directory, if present. */ + readonly codeView?: PeCodeViewRecord; }) | (BinaryTargetIdentity & NonExecutableMetadata & { diff --git a/src/ghidra/GhidraPeProfile.test.ts b/src/ghidra/GhidraPeProfile.test.ts new file mode 100644 index 000000000..561aad701 --- /dev/null +++ b/src/ghidra/GhidraPeProfile.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { ghidraPeCodeViewLimitations } from "./GhidraPeProfile.js"; + +describe("ghidraPeCodeViewLimitations", () => { + it("returns empty limitations for non-PE targets", () => { + const elfTarget: BinaryTarget = { + path: "/test/file", + sha256: "abc", + kind: "executable", + format: "elf", + architecture: "x86_64", + availableArchitectures: ["x86_64"], + }; + expect(ghidraPeCodeViewLimitations(elfTarget)).toEqual([]); + }); + + it("does not claim CodeView symbols were unapplied when the session loaded a PDB", () => { + const peTarget: BinaryTarget = { + path: "/test/file.exe", + sha256: "abc", + kind: "executable", + architecture: "x86_64", + availableArchitectures: ["x86_64"], + executableRole: "application", + managed: false, + format: "pe", + codeView: { + pdbPath: "C:\\symbols.pdb", + pdbFileName: "symbols.pdb", + guid: "12345678-1234-1234-1234-123456789abc", + age: 1, + }, + }; + expect(ghidraPeCodeViewLimitations(peTarget, true)).toEqual([]); + }); + + it("returns empty limitations for PE target without codeView", () => { + const peTarget: BinaryTarget = { + path: "/test/file.exe", + sha256: "def", + kind: "executable", + format: "pe", + architecture: "x86_64", + availableArchitectures: ["x86_64"], + executableRole: "application", + managed: false, + }; + expect(ghidraPeCodeViewLimitations(peTarget)).toEqual([]); + }); + + it("returns limitation with GUID and age for RSDS CodeView records", () => { + const peTarget: BinaryTarget = { + path: "/test/file.exe", + sha256: "def", + kind: "executable", + format: "pe", + architecture: "x86_64", + availableArchitectures: ["x86_64"], + executableRole: "application", + managed: false, + codeView: { + pdbPath: "C:\\build\\app.pdb", + pdbFileName: "app.pdb", + guid: "01234567-89ab-cdef-0123-456789abcdef", + age: 1, + }, + }; + expect(ghidraPeCodeViewLimitations(peTarget)).toEqual([ + 'The target references debug symbols "app.pdb" (GUID 01234567-89ab-cdef-0123-456789abcdef, age 1) that were not applied; procedure names are analysis-generated.', + ]); + }); + + it("returns limitation without GUID for NB10 CodeView records", () => { + const peTarget: BinaryTarget = { + path: "/test/file.exe", + sha256: "def", + kind: "executable", + format: "pe", + architecture: "x86", + availableArchitectures: ["x86"], + executableRole: "application", + managed: false, + codeView: { + pdbPath: "D:\\test\\legacy.pdb", + pdbFileName: "legacy.pdb", + guid: null, + age: 2, + }, + }; + expect(ghidraPeCodeViewLimitations(peTarget)).toEqual([ + 'The target references debug symbols "legacy.pdb" (age 2) that were not applied; procedure names are analysis-generated.', + ]); + }); +}); diff --git a/src/ghidra/GhidraPeProfile.ts b/src/ghidra/GhidraPeProfile.ts new file mode 100644 index 000000000..217b3256f --- /dev/null +++ b/src/ghidra/GhidraPeProfile.ts @@ -0,0 +1,24 @@ +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; + +/** + * Report unapplied PE companion PDB symbols when CodeView debug information is present. + * + * Ghidra does not automatically locate or apply sidecar PDBs in REA's isolated + * runtime directories without companion admission. When the PE target header + * references debug symbols, surface this limitation so callers understand that + * procedure names remain analysis-generated. + */ +export const ghidraPeCodeViewLimitations = ( + target: BinaryTarget, + pdbLoaded = false, +): readonly string[] => { + if (target.format !== "pe" || target.codeView === undefined || pdbLoaded) { + return []; + } + const { pdbFileName, guid, age } = target.codeView; + return [ + guid !== null + ? `The target references debug symbols "${pdbFileName}" (GUID ${guid}, age ${age}) that were not applied; procedure names are analysis-generated.` + : `The target references debug symbols "${pdbFileName}" (age ${age}) that were not applied; procedure names are analysis-generated.`, + ]; +}; diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index ccdfd96f7..b485f6ab3 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -822,6 +822,39 @@ describe("Ghidra result projection", () => { }); }); +describe("Ghidra PE CodeView debug symbol limitations", () => { + it("projects unapplied companion PDB limitation for PE target with CodeView record", async () => { + const ghidra = provider(installationHost(), () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => Promise.resolve(ok(null)), + close: () => Promise.resolve(ok(null)), + })); + const target = { + ...peTarget("x86_64"), + codeView: { + pdbPath: "C:\\projects\\app\\build\\test.pdb", + pdbFileName: "test.pdb", + guid: "01234567-89ab-cdef-0123-456789abcdef", + age: 1, + }, + }; + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("expected PE profile"); + const client = ghidra.createClient(target, profile.value.profile); + const health = await client.execute("health", {}); + expect(health).toMatchObject({ + ok: true, + value: { + limitations: expect.arrayContaining([ + 'The target references debug symbols "test.pdb" (GUID 01234567-89ab-cdef-0123-456789abcdef, age 1) that were not applied; procedure names are analysis-generated.', + ]), + }, + }); + await client.close(); + }); +}); + describe("Ghidra relationship coverage", () => { it("marks the full direct call-reference response complete while keeping static caveats", async () => { const ghidra = provider(installationHost(), () => ({ diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 25838453e..9db1b0c5f 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -68,6 +68,7 @@ import { isGhidraMipsSoftFloatTarget, } from "./GhidraMipsProfile.js"; import { ghidraRiscvLimitations } from "./GhidraRiscvProfile.js"; +import { ghidraPeCodeViewLimitations } from "./GhidraPeProfile.js"; import { unverifiedGhidraBuildLimitation } from "./GhidraInstallationPolicy.js"; import { GhidraHeadlessLauncher } from "./GhidraLauncher.js"; import { attestGhidraNativeLoadImage } from "./GhidraLoadImageAttest.js"; @@ -530,6 +531,7 @@ export const createGhidraProviderClient = (input: { // Replaced by the reported outcome once the session handshake is checked. const seedLimitations: string[] = [...ghidraSeedLimitations(seeds)]; const pdbLimitations: string[] = []; + let pdbLoaded = false; // Replaced by the reported storage outcome once the handshake is checked. const softFloatLimitations: string[] = [ ...ghidraMipsSoftFloatLimitations(softFloat), @@ -654,12 +656,14 @@ export const createGhidraProviderClient = (input: { operation: AnalysisOperation, info: GhidraSessionInfo, ): Promise => { - if (target.format === "pe") + if (target.format === "pe") { + pdbLoaded = info.pdb?.loaded === true; pdbLimitations.splice( 0, pdbLimitations.length, ...ghidraPdbLimitations(info.pdb), ); + } const softFloatReason = ghidraMipsSoftFloatFailure( softFloat, info.mips_soft_float_storage, @@ -712,6 +716,7 @@ export const createGhidraProviderClient = (input: { ...ghidraExtensionLimitations(extensions), ...seedLimitations, ...pdbLimitations, + ...ghidraPeCodeViewLimitations(target, pdbLoaded), ...(releaseLimitation === undefined ? [] : [releaseLimitation]), ]; const nativeAotOwner = diff --git a/tests/boundary/filesystem/binaryTarget.test.ts b/tests/boundary/filesystem/binaryTarget.test.ts index e56055069..eb5977a54 100644 --- a/tests/boundary/filesystem/binaryTarget.test.ts +++ b/tests/boundary/filesystem/binaryTarget.test.ts @@ -26,6 +26,7 @@ import { thinMach, } from "../../../src/domain/binaryTarget.fixture.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { parseExecutableHeader } from "../../../src/domain/binaryTarget.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -424,6 +425,65 @@ describe("binary target I/O: explicit kinds and executable headers", () => { }); }); + it("extracts CodeView RSDS debug metadata from a PE target", async () => { + const directory = await createTestTempDirectory("rea-target-"); + const path = join(directory, "symbols.exe"); + await writeFile( + path, + pe(0x8664, 64, 0x0002, false, { + type: "RSDS", + pdbPath: "C:\\build\\test.pdb", + guid: "12345678-abcd-ef01-2345-6789abcdef01", + age: 3, + }), + ); + const result = await parseBinaryTarget(path, { + cwd: directory, + hostArchitecture: "x64", + }); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + format: "pe", + architecture: "x86_64", + codeView: { + pdbPath: "C:\\build\\test.pdb", + pdbFileName: "test.pdb", + guid: "12345678-abcd-ef01-2345-6789abcdef01", + age: 3, + }, + }); + }); + + it("extracts CodeView NB10 debug metadata from a PE target", async () => { + const directory = await createTestTempDirectory("rea-target-"); + const path = join(directory, "legacy.exe"); + await writeFile( + path, + pe(0x014c, 64, 0x0002, false, { + type: "NB10", + pdbPath: "D:\\old\\legacy.pdb", + age: 1, + }), + ); + const result = await parseBinaryTarget(path, { + cwd: directory, + hostArchitecture: "ia32", + }); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + format: "pe", + architecture: "x86", + codeView: { + pdbPath: "D:\\old\\legacy.pdb", + pdbFileName: "legacy.pdb", + guid: null, + age: 1, + }, + }); + }); + it("refuses a truncated Mach-O header before any provider sees it", async () => { const directory = await createTestTempDirectory("rea-target-"); const path = join(directory, "truncated"); @@ -884,3 +944,43 @@ it.each([ }); }, ); + +describe("binary target I/O: PE CodeView header ranges", () => { + it.each(["RSDS", "NB10"] as const)( + "reads bounded %s header debug records and rejects unfinished names", + async (type) => { + const directory = await createTestTempDirectory("rea-target-"); + const path = join(directory, "header-symbols.exe"); + const bytes = pe(0x8664, 64, 0x0002, false, { + type, + pdbPath: "header.pdb", + }); + const optionalHeader = 64 + 24; + bytes.writeUInt32LE(512, optionalHeader + 60); + bytes.copy(bytes, 384, 512); + bytes.writeUInt32LE(384, optionalHeader + 112 + 6 * 8); + bytes.writeUInt32LE(412, 384 + 20); + bytes.writeUInt32LE(0, 384 + 24); + for (const complete of [true, false]) { + if (!complete) { + const dataSize = bytes.readUInt32LE(384 + 16); + bytes[412 + dataSize - 1] = 0x61; + } + await writeFile(path, bytes); + const resolved = await parseBinaryTarget(path, { + hostArchitecture: "x64", + }); + const buffered = parseExecutableHeader(bytes, "x64"); + if (!resolved.ok) throw resolved.error; + if (!buffered.ok) throw new Error(buffered.error); + for (const value of [resolved.value, buffered.value]) { + if (complete) + expect(value).toMatchObject({ + codeView: { pdbPath: "header.pdb" }, + }); + else expect(value).not.toHaveProperty("codeView"); + } + } + }, + ); +}); From 42667fc1a03ffae7ffea57a611f52162ea223a7c Mon Sep 17 00:00:00 2001 From: Jeremy Levartovsky <140487034+JayOfTheKeyboard@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:58:41 +1100 Subject: [PATCH 09/15] fix(wasm): report WABT output past its budget as a resource constraint (#2117) * fix(wasm): report WABT output past its budget as a resource constraint A module whose WAT exceeds the 32 MiB output budget failed as unreadable_output, advised `rea doctor`, and echoed the full 32 MiB truncated WAT prefix in details.captured_output. Map the owned-command output-limit outcome during inspection to AnalysisResourceConstraintError naming the budget, as the apktool and native macOS providers do, with a remediation that says the fixed budget will not change on retry. Captured output for an output-limit failure keeps only the first 4,096 characters of each stream, including on the cleanup-failure path. An oversized --version probe keeps its configuration-phase mapping. * fix(wasm): keep observed sizes and whole code points in output-limit excerpts Cut each stream's output-limit excerpt with boundedUtf8Text at 4,096 UTF-8 bytes, so an astral character at the cut no longer leaves a lone surrogate. Mark the output truncated when either excerpt was cut, and report each stream's observed byte count. Word the remediation as a fact about the fixed budget. A flood of diagnostics from any WABT tool reaches this path, so it no longer says the module is too large. Replace the 32 MiB end-to-end test with a launcher seam that raises a small output-limit failure. * fix(wasm): match sibling captured-output fields for WABT output limits Report retained byte counts in stdout_bytes and stderr_bytes, as apktool and the firmware command do, and record exit_code and signal. Mark the excerpt truncated outright, since the command raises output-limit only after retention was truncated. When cleanup also fails, record the output limit and its budget as the previous error instead of the generic command message. Document that the two streams share the budget, so the excerpt holds up to 4,096 of the bytes each stream retained. * docs(wasm): scope the output-budget error to inspection commands * docs(wasm): distinguish reported warm costs from unmeasured gaps * docs(wasm): clarify the scope of reported warm measurements --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/wasm-artifacts.md | 21 +++- src/wasm/WabtArtifactProvider.test.ts | 156 ++++++++++++++++++++++++++ src/wasm/WabtFailures.ts | 51 ++++++++- 3 files changed, 219 insertions(+), 9 deletions(-) diff --git a/docs/wasm-artifacts.md b/docs/wasm-artifacts.md index 17b6c77f9..12d4da85b 100644 --- a/docs/wasm-artifacts.md +++ b/docs/wasm-artifacts.md @@ -61,9 +61,24 @@ The tool profile preserves upstream release/source identity, resolved executable paths, executable digests, exact banners and arguments. Executable digests are checked again after each command. Caller-supplied tools are trusted programs; version and digest recording are not publisher authentication or an OS sandbox. -Each subprocess has a 30-second deadline and 32 MiB combined output budget; -truncation fails. All commands use existing owned-process cancellation and -cleanup. Workspace cleanup failures remain explicit errors with resource paths. +Each subprocess has a 30-second deadline and 32 MiB combined output budget. +Inspection output past the budget fails with `resource_constraint`; an oversized +`--version` probe still reports `unreadable_output`. Both streams share the +budget, so a stream that starts after the other has filled it retains nothing. +The captured output keeps up to the first 4,096 bytes that each stream retained, +each stream's retained byte count, and the exit code and signal. Retained text +can end in U+FFFD where the budget cut a multi-byte character. + +These excerpts bound the returned diagnostic text. +[PR #2117](https://github.com/morluto/rea/pull/2117) reports elapsed time, CPU +time, and peak process-family memory, including WABT children, for warm Linux +x64 WABT 1.0.42 CLI workflows; those costs did not change outside measurement +noise. Agent token usage and truly cold workflows remain +unmeasured. The underlying error cause still retains the full captured +subprocess snapshot, so the excerpt does not establish lower retained memory +or faster analysis. +All commands use existing owned-process cancellation and cleanup. Workspace +cleanup failures remain explicit errors with resource paths. WAT is a decoded representation, not original source or proof of runtime behavior. Custom annotations can preserve payloads, but WAT does not establish original diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index 428cf8cbd..161e542d2 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -8,6 +8,8 @@ import { PrivateRuntimeRootUnavailableError } from "../process/PrivateRuntimeRoo import { spawnOwnedProviderProcess } from "../process/ProviderProcess.js"; import { OwnedCommandFailure } from "../process/OwnedCommand.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { WABT_LIMITS } from "./WabtRelease.js"; +import { wabtFailure } from "./WabtFailures.js"; const fileCloseFailure = vi.hoisted(() => ({ selectedPath: "", @@ -391,3 +393,157 @@ it.each([ expect(() => process.kill(pid, 0)).toThrow(); } }); +const outputLimitFailure = ({ + stdout = "(module", + stderr = "", + cleanupFailure = null, +}: { + readonly stdout?: string; + readonly stderr?: string; + readonly cleanupFailure?: string | null; +} = {}) => + new OwnedCommandFailure( + "output-limit", + "Command diagnostic output exceeded its complete-output budget.", + { + diagnosticTruncated: true, + // Observed sizes differ from retained sizes, as after a real overflow. + stdout: { + text: stdout, + bytes: Buffer.byteLength(stdout), + observedBytes: Buffer.byteLength(stdout) + 1000, + }, + stderr: { + text: stderr, + bytes: Buffer.byteLength(stderr), + observedBytes: Buffer.byteLength(stderr) + 1000, + }, + exitCode: null, + signal: "SIGKILL", + }, + cleanupFailure, + undefined, + cleanupFailure === null ? [] : ["/owned/runtime"], + ); +it("reports WABT output beyond the budget as a resource constraint with a bounded excerpt", async () => { + const selectedRoot = await createTestTempDirectory("rea-wabt-budget-"); + const path = join(selectedRoot, "selected.wasm"); + await writeFile(path, Buffer.alloc(76, 1)); + for (const tool of ["wasm-validate", "wasm-objdump", "wasm2wat"]) + await writeFile( + join(selectedRoot, tool + (process.platform === "win32" ? ".exe" : "")), + "source-owned executable seam", + { mode: 0o700 }, + ); + const dump = await readFile( + new URL("../../tests/fixtures/wasm/objdump.txt", import.meta.url), + "utf8", + ); + const provider = new WabtArtifactProvider( + { REA_WABT_BIN_DIRECTORY: selectedRoot }, + (input) => { + const version = input.arguments[0] === "--version"; + if (!version && input.command.includes("wasm2wat")) + throw outputLimitFailure({ stdout: "(module" + "a".repeat(5000) }); + const script = version + ? 'process.stdout.write("1.0.42\\n")' + : input.command.includes("objdump") + ? `process.stdout.write(${JSON.stringify(dump)})` + : ""; + return spawnOwnedProviderProcess({ + ...input, + command: process.execPath, + arguments: ["-e", script], + expectedCommand: null, + }); + }, + ); + const result = await provider.inspect({ + path, + glue_paths: [], + candidate_paths: [], + }); + if (result.ok) throw new Error("Expected the output budget to fail"); + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "memory", + reportedLimits: { max_output_bytes: WABT_LIMITS.outputBytes }, + }); + const projection = projectAnalysisError(result.error); + expect(projection).toMatchObject({ + code: "resource_constraint", + message: expect.stringContaining( + `${String(WABT_LIMITS.outputBytes)}-byte output budget`, + ), + remediation: { + action: expect.stringContaining("retrying with the same input"), + }, + }); + expect(projection.details?.captured_output).toEqual({ + stdout: "(module" + "a".repeat(4089), + stderr: "", + truncated: true, + stdout_bytes: 5007, + stderr_bytes: 0, + exit_code: null, + signal: "SIGKILL", + }); +}); +it("bounds an oversized stderr to an excerpt", () => { + expect( + projectAnalysisError( + wabtFailure( + outputLimitFailure({ stderr: "w".repeat(5000) }), + "producer", + "/module.wasm", + ), + ).details?.captured_output, + ).toEqual({ + stdout: "(module", + stderr: "w".repeat(4096), + truncated: true, + stdout_bytes: 7, + stderr_bytes: 5000, + exit_code: null, + signal: "SIGKILL", + }); +}); +it("cuts the excerpt on a code point boundary", () => { + const failure = wabtFailure( + outputLimitFailure({ stdout: "a".repeat(4095) + "\u{1F600}b" }), + "producer", + "/module.wasm", + ); + expect(failure.capturedOutput?.stdout).toBe("a".repeat(4095)); +}); +it("bounds the excerpt when cleanup also fails after the output budget", () => { + expect( + projectAnalysisError( + wabtFailure( + outputLimitFailure({ + stderr: "w".repeat(5000), + cleanupFailure: "Source-owned cleanup uncertainty", + }), + "producer", + "/module.wasm", + ), + ), + ).toMatchObject({ + code: "cleanup_incomplete", + details: { + diagnostics: { + previous_error: { + failure_kind: "output-limit", + message: `WABT output exceeded the ${String(WABT_LIMITS.outputBytes)}-byte output budget`, + max_output_bytes: WABT_LIMITS.outputBytes, + }, + captured_output: { stdout: "(module", stderr: "w".repeat(4096) }, + }, + }, + }); +}); +it("does not blame the input when a version probe exceeds the budget", () => { + expect( + wabtFailure(outputLimitFailure(), "configuration", "/module.wasm"), + ).toMatchObject({ _tag: "AnalysisOutputError" }); +}); diff --git a/src/wasm/WabtFailures.ts b/src/wasm/WabtFailures.ts index 7b6875203..b679f7640 100644 --- a/src/wasm/WabtFailures.ts +++ b/src/wasm/WabtFailures.ts @@ -1,4 +1,6 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, @@ -6,6 +8,7 @@ import { AnalysisAccessDeniedError, AnalysisInputError, AnalysisOutputError, + AnalysisResourceConstraintError, AnalysisTimeoutError, AnalysisCapabilityUnavailableError, } from "../domain/analysisErrorCore.js"; @@ -13,6 +16,21 @@ import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { OwnedCommandFailure } from "../process/OwnedCommand.js"; import { PrivateRuntimeRootUnavailableError } from "../process/PrivateRuntimeRootUnavailableError.js"; import { WABT_LIMITS, WABT_PROVIDER_IDENTITY } from "./WabtRelease.js"; +const OUTPUT_LIMIT_EXCERPT_BYTES = 4096; +const OUTPUT_LIMIT_REASON = `WABT output exceeded the ${String(WABT_LIMITS.outputBytes)}-byte output budget`; +/** Past the output budget, retained text is a prefix of an oversized stream; keep a bounded excerpt of each. */ +const outputLimitExcerpt = (snapshot: ProviderProcessSnapshot) => ({ + stdout: boundedUtf8Text(snapshot.stdout.text, OUTPUT_LIMIT_EXCERPT_BYTES) + .text, + stderr: boundedUtf8Text(snapshot.stderr.text, OUTPUT_LIMIT_EXCERPT_BYTES) + .text, + // The command reports output-limit only after retention was truncated. + truncated: true, + stdout_bytes: snapshot.stdout.bytes, + stderr_bytes: snapshot.stderr.bytes, + exit_code: snapshot.exitCode ?? null, + signal: snapshot.signal ?? null, +}); /** Preserve lifecycle failures and diagnostic bytes instead of returning partial success. */ export const wabtFailure = ( cause: unknown, @@ -37,11 +55,13 @@ export const wabtFailure = ( const capturedOutput = cause.snapshot === null ? undefined - : { - stdout: cause.snapshot.stdout.text, - stderr: cause.snapshot.stderr.text, - truncated: cause.snapshot.diagnosticTruncated ?? false, - }; + : cause.reason === "output-limit" + ? outputLimitExcerpt(cause.snapshot) + : { + stdout: cause.snapshot.stdout.text, + stderr: cause.snapshot.stderr.text, + truncated: cause.snapshot.diagnosticTruncated ?? false, + }; const options = capturedOutput === undefined ? { cause } : { cause, capturedOutput }; if (cause.cleanupFailure !== null) @@ -50,7 +70,14 @@ export const wabtFailure = ( cause.resources, { reason: cause.cleanupFailure, - previous_error: reason, + previous_error: + cause.reason === "output-limit" + ? { + failure_kind: "output-limit", + message: OUTPUT_LIMIT_REASON, + max_output_bytes: WABT_LIMITS.outputBytes, + } + : reason, ...(capturedOutput === undefined ? {} : { captured_output: capturedOutput }), @@ -65,6 +92,18 @@ export const wabtFailure = ( WABT_LIMITS.timeoutMs, options, ); + if (phase !== "configuration" && cause.reason === "output-limit") + return new AnalysisResourceConstraintError( + "inspect_wasm_artifact", + "memory", + OUTPUT_LIMIT_REASON, + { max_output_bytes: WABT_LIMITS.outputBytes }, + { + ...options, + remediationAction: + "The WABT output budget is fixed, so retrying with the same input will fail the same way.", + }, + ); if (phase === "configuration" && cause.reason === "process") return new AnalysisCapabilityUnavailableError( WABT_PROVIDER_IDENTITY.id, From e4186af31be855a127804a8543def43cd77404be Mon Sep 17 00:00:00 2001 From: BeqaKs <91750923+BeqaKs@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:59:54 +0400 Subject: [PATCH 10/15] fix(process): expose Darwin swiftc compile deadline via env (#2240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(process): expose Darwin swiftc compile deadline via env Busy hosts can exceed the hard-coded 60s ownership-helper compile timeout. Honor REA_SWIFTC_COMPILE_TIMEOUT_MS like the other provider startup deadlines, keeping 60000 as the default. Fixes #2184 Co-authored-by: Cursor * fix(process): keep the Swift compile deadline with its runtime owner * fix(comparison): recheck historical calls-only changes (#2223) * fix(comparison): preserve incomplete function call evidence Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty. Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls. * fix(comparison): recheck historical calls-only changes --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> (cherry picked from commit 688defca0f4a3de04eb199f7754f323c68896c3c) * fix(ci): isolate Linux Hopper process fixture inventory (cherry picked from commit 8cac51e12184d56a9ba80a4427b3e6082ddac1ec) * fix(android): make DEX descriptor pattern portable (cherry picked from commit 1335bcddc569a1cfb34b6efedcb6f2e3aaaa3357) * fix: repair the static-check failures on the default branch (#2895) * fix: repair the static-check failures on the default branch The default branch fails its own required `Static checks` job three ways, so every pull request reports that job as failed regardless of its own change. - `npm run format:check` reports `src/domain/functionComparison.test.ts` and `src/ida/IdaOperationRunner.ts`. - `npm run lint` reports a duplicate `import { join } from "node:path"` in `tests/boundary/mcp/ghidraInventoryMcp.test.ts` (lines 4 and 17). - `npm run typecheck` reports `format: "devin"` missing from the client format union in `src/application/SupportedClients.ts`, and an unchecked index in `tests/boundary/cli/clientProfileLifecycle.test.ts`, whose pair array is inferred as `string[][]`. Apply the pinned formatter, drop the duplicate import, list the `"devin"` format that its own definition already uses, and mark the pair array `as const`. Validation: `npm run check:fast` and `npm run format:check` pass. * fix(setup): keep Devin on the JSON configuration dialect --------- Co-authored-by: 吴佳浩 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> (cherry picked from commit df3d96da801976fdbe37a726c2f4f84af8ffbf90) --------- Co-authored-by: Cursor Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: morluto <17176810+morluto@users.noreply.github.com> Co-authored-by: Lucien <164857476+dakjdakd@users.noreply.github.com> Co-authored-by: 吴佳浩Alben <35291372+XiaoBinGan@users.noreply.github.com> Co-authored-by: 吴佳浩 --- docs/installation.md | 3 ++ docs/mcp-contracts.md | 1 + src/config.test.ts | 34 +++++++++++++++++++ src/config/environment.ts | 4 +++ src/config/providerStartupTimeout.ts | 4 +++ ...arwinProcessRunTokenReader.timeout.test.ts | 34 +++++++++++++++++++ src/process/DarwinProcessRunTokenReader.ts | 33 +++++++++++++++--- src/process/ProcessOwnershipObservation.ts | 5 +++ 8 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 src/process/DarwinProcessRunTokenReader.timeout.test.ts diff --git a/docs/installation.md b/docs/installation.md index a39471585..3ad6b7071 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -764,6 +764,9 @@ errors expose Error names, messages, and available codes under truncation markers. Darwin ownership-helper compilation failures also retain the compiler outcome, exit code or signal, and bounded stderr; a timed-out compiler remains distinguishable from a compiler that exited with an error. +The helper compile deadline defaults to 120,000 ms; set +`REA_SWIFTC_COMPILE_TIMEOUT_MS` to an integer between 1 and 2,147,483,647 when +a busy host needs longer. Other rejection values retain their primitive value or an explicit type. Shutdown warnings include the failure kind, message, and diagnostics, while process and temporary-project cleanup diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 876a30f6d..a2708511c 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -474,6 +474,7 @@ These deadlines have different owners: | MCP initialize | The client bounds transport/REA connection startup, before any target query. | | Hopper startup | REA allows 120,000 ms by default; `REA_HOPPER_STARTUP_TIMEOUT_MS` accepts decimal integers from 1–2,147,483,647 ms. Invalid values fail configuration validation. Startup failure is reported by the first provider query. | | Ghidra startup | REA allows 330,000 ms by default; `REA_GHIDRA_STARTUP_TIMEOUT_MS` accepts decimal integers from 1–2,147,483,647 ms. Invalid supplied values fail configuration validation. Startup failure is reported by the first provider query. | +| Darwin swiftc helper | REA allows 120,000 ms by default for ownership-helper compilation; `REA_SWIFTC_COMPILE_TIMEOUT_MS` accepts decimal integers from 1–2,147,483,647 ms. Invalid values fail configuration validation. | | Individual tool call | The client bounds its wait, including cold engine startup. The pinned client SDK 2.3.1 defaults to 60,000 ms and can cancel earlier than REA's startup deadline. | For an already connected client using the pinned SDK, request options are the diff --git a/src/config.test.ts b/src/config.test.ts index 67db7e111..efa1b48fc 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -249,6 +249,40 @@ describe("configuration failure diagnostics", () => { }, ); + it.each(["1", "90000", "2147483647"])( + "accepts Swift compile deadline %s without changing Hopper's deadline", + (value) => { + expect( + parseConfig({ REA_SWIFTC_COMPILE_TIMEOUT_MS: value }), + ).toMatchObject({ + ok: true, + value: { + hopperStartupTimeoutMs: 120_000, + }, + }); + }, + ); + + it.each(["", "0", "-1", "1.5", "1e3", "invalid", "2147483648"])( + "rejects malformed Swift compile deadline %j with its setting", + (value) => { + const result = parseConfig({ REA_SWIFTC_COMPILE_TIMEOUT_MS: value }); + if (result.ok) throw new Error("Expected invalid Swift compile deadline"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "configuration_invalid", + category: "invalid_input", + details: { + settings: [ + { + setting: "REA_SWIFTC_COMPILE_TIMEOUT_MS", + constraint: expect.any(String), + }, + ], + }, + }); + }, + ); + it.each([ ["GHIDRA_INSTALL_DIR", "./fixture"], ["JAVA_HOME", "./jdk"], diff --git a/src/config/environment.ts b/src/config/environment.ts index 7dcd955d0..121d46614 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -12,6 +12,7 @@ import { ghidraDisabledAnalyzersSchema } from "./ghidraDisabledAnalyzers.js"; import { DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS, DEFAULT_HOPPER_STARTUP_TIMEOUT_MS, + DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS, providerStartupTimeoutSchema, } from "./providerStartupTimeout.js"; import { mcpInputSchemaProfileSchema } from "./mcpInputSchemaProfile.js"; @@ -47,6 +48,9 @@ const environmentSchema = z.object({ REA_HOPPER_STARTUP_TIMEOUT_MS: providerStartupTimeoutSchema.default( DEFAULT_HOPPER_STARTUP_TIMEOUT_MS, ), + REA_SWIFTC_COMPILE_TIMEOUT_MS: providerStartupTimeoutSchema.default( + DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS, + ), REA_GHIDRA_LANGUAGE_ID: ghidraLanguageIdSchema.optional(), REA_GHIDRA_COMPILER_SPEC_ID: ghidraCompilerSpecIdSchema.optional(), REA_GHIDRA_DISABLED_ANALYZERS: ghidraDisabledAnalyzersSchema.optional(), diff --git a/src/config/providerStartupTimeout.ts b/src/config/providerStartupTimeout.ts index 27c0116de..eb3b9d52f 100644 --- a/src/config/providerStartupTimeout.ts +++ b/src/config/providerStartupTimeout.ts @@ -6,6 +6,10 @@ export const DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS = 330_000; /** Default absolute deadline for Hopper launch, analysis, bridge, and readiness. */ export const DEFAULT_HOPPER_STARTUP_TIMEOUT_MS = 120_000; +/** Default deadline for Darwin ownership-helper `swiftc` compilation. */ +/** Fresh Swift SDK cache took ~88s on the supported Intel macOS VM. */ +export const DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS = 120_000; + // Node schedules delays above this signed 32-bit boundary as a 1 ms timer. const MAX_TIMER_DELAY_MS = 2_147_483_647; diff --git a/src/process/DarwinProcessRunTokenReader.timeout.test.ts b/src/process/DarwinProcessRunTokenReader.timeout.test.ts new file mode 100644 index 000000000..224b49600 --- /dev/null +++ b/src/process/DarwinProcessRunTokenReader.timeout.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +import { resolveSwiftcCompileTimeoutMs } from "./DarwinProcessRunTokenReader.js"; + +describe("resolveSwiftcCompileTimeoutMs", () => { + it("keeps the historical default when unset", () => { + expect(resolveSwiftcCompileTimeoutMs({ environment: {} })).toBe(120_000); + }); + + it("prefers an explicit compileTimeoutMs over the environment", () => { + expect( + resolveSwiftcCompileTimeoutMs({ + compileTimeoutMs: 12_000, + environment: { REA_SWIFTC_COMPILE_TIMEOUT_MS: "300000" }, + }), + ).toBe(12_000); + }); + + it("reads a valid REA_SWIFTC_COMPILE_TIMEOUT_MS value", () => { + expect( + resolveSwiftcCompileTimeoutMs({ + environment: { REA_SWIFTC_COMPILE_TIMEOUT_MS: "300000" }, + }), + ).toBe(300_000); + }); + + it("falls back to the default for a malformed environment value", () => { + expect( + resolveSwiftcCompileTimeoutMs({ + environment: { REA_SWIFTC_COMPILE_TIMEOUT_MS: "loud" }, + }), + ).toBe(120_000); + }); +}); diff --git a/src/process/DarwinProcessRunTokenReader.ts b/src/process/DarwinProcessRunTokenReader.ts index 335e72fbe..79e9d9c11 100644 --- a/src/process/DarwinProcessRunTokenReader.ts +++ b/src/process/DarwinProcessRunTokenReader.ts @@ -5,6 +5,10 @@ import { isAbsolute, join } from "node:path"; import { fileURLToPath } from "node:url"; import { z } from "zod"; +import { + DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS, + providerStartupTimeoutSchema, +} from "../config/providerStartupTimeout.js"; import { safeParseJson } from "../domain/safeJson.js"; import { boundedUtf8Text } from "./boundedUtf8Text.js"; import { execFileOutput, execFileOutputFailure } from "./ExecFileOutput.js"; @@ -73,10 +77,27 @@ const sourceFiles = [ */ export const SWIFTC_INTERRUPT = "SIGINT"; -/** Deadline for the one-shot helper compilation. */ -// A fresh Swift SDK cache took 88 seconds on the supported Intel macOS VM. -// Keep compilation bounded independently of short provider version probes. -const COMPILER_TIMEOUT_MS = 120_000; +/** + * Resolve the Darwin ownership-helper compile deadline from an explicit option + * or `REA_SWIFTC_COMPILE_TIMEOUT_MS`. The default matches the measured fresh + * Swift SDK cache on the supported Intel macOS VM (about 88s), so compilation + * stays bounded independently of short provider version probes. Invalid ambient + * values fall back to the default; `parseConfig` still rejects them at CLI/MCP + * admission. + */ +export const resolveSwiftcCompileTimeoutMs = ( + options: { + readonly compileTimeoutMs?: number; + readonly environment?: Readonly>; + } = {}, +): number => { + if (options.compileTimeoutMs !== undefined) return options.compileTimeoutMs; + const raw = (options.environment ?? process.env) + .REA_SWIFTC_COMPILE_TIMEOUT_MS; + if (raw === undefined) return DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS; + const parsed = providerStartupTimeoutSchema.safeParse(raw); + return parsed.success ? parsed.data : DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS; +}; /** Bound on compiler diagnostics embedded in the inspection failure text. */ const STDERR_EXCERPT_LIMIT = 512; @@ -257,6 +278,8 @@ export const createDarwinProcessRunTokenReader = ( readonly compiler?: DarwinProcessReaderCompiler; /** Deadline for helper compilation, independent of provider execution. */ readonly compileTimeoutMs?: number; + /** Environment consulted when `compileTimeoutMs` is omitted. */ + readonly environment?: Readonly>; /** * Caller-owned directory holding the compiler's module cache. Unset keeps * Swift's reusable default cache; a supplied directory outlives this @@ -269,7 +292,7 @@ export const createDarwinProcessRunTokenReader = ( kind: "xcrun", executable: "/usr/bin/xcrun", }; - const compilerTimeoutMs = options.compileTimeoutMs ?? COMPILER_TIMEOUT_MS; + const compilerTimeoutMs = resolveSwiftcCompileTimeoutMs(options); const moduleCachePath = options.moduleCachePath ?? ""; let root: string | undefined; let executable: string | undefined; diff --git a/src/process/ProcessOwnershipObservation.ts b/src/process/ProcessOwnershipObservation.ts index 834335d57..5747cded8 100644 --- a/src/process/ProcessOwnershipObservation.ts +++ b/src/process/ProcessOwnershipObservation.ts @@ -225,6 +225,8 @@ export const createSystemProcessOwnershipHost = ( readonly darwinCompiler?: DarwinProcessReaderCompiler; /** Override the caller-owned module cache used compiling that helper. */ readonly darwinModuleCachePath?: string; + /** Deadline for Darwin ownership-helper compilation. */ + readonly compileTimeoutMs?: number; } = {}, ): ProcessOwnershipHost => { const darwinModuleCachePath = @@ -247,6 +249,9 @@ export const createSystemProcessOwnershipHost = ( ...(darwinModuleCachePath === undefined ? {} : { moduleCachePath: darwinModuleCachePath }), + ...(options.compileTimeoutMs === undefined + ? { environment: hostEnvironment } + : { compileTimeoutMs: options.compileTimeoutMs }), }) : undefined; const listProcesses = async (signal?: AbortSignal) => { From 45a3509db34e2eca5558b90b06fca1baefdf3479 Mon Sep 17 00:00:00 2001 From: Thang Luu Date: Wed, 14 Oct 2026 04:00:27 +0900 Subject: [PATCH 11/15] fix(artifacts): compare inventories using declared coverage (#2214) * fix(artifacts): preserve complete root inventory comparisons * fix(artifacts): declare inventory coverage before comparison * fix(artifacts): preserve traversal omissions and materialization limits * fix(artifacts): preserve coverage declarations and projection scope Remove the duplicate inventory-schema import. Compare original coverage declarations, including field omission, before adapting legacy pages. Require explicit scope for completeness and container-entry coverage for application projections. Source inspection and git diff --check only. Tests, builds and typechecks skipped by explicit user request. GitHub review writes paused globally. * fix(artifacts): require entry coverage for child changes * fix(artifacts): accept source-valid inventory limitations * style(artifacts): format mobile graph projection --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: OpenAI Codex --- docs/mcp-contracts.md | 25 + src/artifacts/ArtifactReader.ts | 33 +- src/artifacts/DirectoryArtifactReader.ts | 39 +- src/artifacts/NativeDmgArtifactReader.ts | 16 +- src/artifacts/inventory/reader.ts | 77 ++-- src/artifacts/inventory/scanCanonical.ts | 37 +- src/artifacts/inventory/scanReader.ts | 39 +- src/domain/android/androidApplication.ts | 5 +- src/domain/apple/appleApplication.ts | 30 +- src/domain/artifactComparison.ts | 65 ++- src/domain/artifactGraph.ts | 19 +- src/domain/artifactInventoryEvidence.ts | 120 +++-- src/domain/artifactInventorySnapshot.ts | 5 +- src/domain/harmony/harmonyApplication.ts | 5 +- .../mobileApplicationGraphProjection.ts | 3 +- .../artifactComparisonCoverage.test.ts | 432 ++++++++++++++++++ .../filesystem/android/application.test.ts | 8 +- .../filesystem/appleApplication.test.ts | 10 +- .../filesystem/artifactComparison.test.ts | 118 ++++- .../artifactInventoryRegularRoot.test.ts | 19 + .../filesystem/harmony/application.test.ts | 10 +- .../mcp/artifactMcp.inlineInventory.test.ts | 10 +- tests/boundary/mcp/artifactMcp.test.ts | 42 +- .../native/nativeDmgArtifactReader.test.ts | 18 +- 24 files changed, 1023 insertions(+), 162 deletions(-) create mode 100644 tests/acceptance/applications/artifactComparisonCoverage.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index a2708511c..307d323df 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -21,6 +21,31 @@ directly to `compare_functions`, and `inspect_artifact` Evidence to `compare_artifacts`. Use `get_evidence_bundle` when the task needs broader retained session history or an explicit bundle for transfer. +Artifact inventories declare `coverage.scope` (`root` or `container-entries`) +and `coverage.status` (`complete` or `unknown`) independently of diagnostic +wording. Comparison requires matching scopes, complete graph members and complete +traversal coverage before concluding equality. A fully observed root without +container children can be `unchanged`; its scope notice stays in `limitations` +without creating a gap. Content and executable metadata changes still produce +root deltas. Missing graph members yield `truncated`; unknown traversal coverage +or unavailable occurrence hashes retain uncertainty. Integrity mismatches remain +contradictions. Static artifact equality does not establish unchanged runtime +behavior. + +Exhausted traversals with deliberately omitted filesystem entries retain +`unknown` coverage and identify each omitted path/type in `limitations`; FIFO, +socket and device entries are not opened. Case-distinct archive members remain +separate observations. Their case-insensitive materialization diagnostic does +not reduce traversal coverage when both spellings were read. + +Current output also retains `coverage_status` (`complete` or `partial`) for existing clients, derived from the scoped traversal status. The owning parser normalizes retained `coverage_status` records to scoped coverage and rejects conflicting declarations. + +The coverage field is optional when reading older retained Evidence. The owning +inventory parser recognizes historical REA scope notices for compatibility; +unfamiliar legacy limitations remain conservative. Typed records do not derive +coverage from limitation text. Pages from one inventory must carry the same +coverage declaration. + Function dossiers can declare per-facet coverage in the optional `normalized_result.facet_availability` map. Each supplied facet is either `{ "available": true }` or `{ "available": false, "reason": "..." }`. diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index f072dd35c..ad40de57d 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -1,5 +1,8 @@ import type { Readable } from "node:stream"; -import type { ArtifactCommand } from "../domain/artifactGraph.js"; +import type { + ArtifactCommand, + ArtifactInventoryCoverage, +} from "../domain/artifactGraph.js"; import type { ZipPackageFormat } from "../domain/zipPackageFormat.js"; import type { AnalysisCleanupObservation, @@ -90,10 +93,36 @@ export const sameArtifactEntry = ( entry.sourceIdentity?.mtimeNs === produced.sourceIdentity?.mtimeNs && entry.sourceIdentity?.ctimeNs === produced.sourceIdentity?.ctimeNs; +/** One entry or subtree deliberately excluded from a traversal. */ +export interface ArtifactTraversalOmission { + readonly path: string; + readonly kind: + | "fifo" + | "socket" + | "block-device" + | "character-device" + | "unsupported" + | "directory-descendants"; +} + +/** Facts from an exhausted traversal, separate from entry content certainty. */ +export interface ArtifactReaderTraversal { + readonly status: ArtifactInventoryCoverage["status"]; + readonly limitations: readonly string[]; +} + +/** Per-iterator observation sink; adapters report intentional omissions without shared state. */ +export interface ArtifactTraversalOptions { + readonly onOmission?: (omission: ArtifactTraversalOmission) => void; +} + /** Read-only adapter over one directory, archive, or virtual container. */ export interface ArtifactReader { readonly format: "directory" | ZipPackageFormat | "asar" | "file"; - entries(signal?: AbortSignal): AsyncIterable; + entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable; open(entry: ArtifactEntry, signal?: AbortSignal): Promise; provenance(): readonly ArtifactCommand[]; close(): Promise; diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.ts index 4a1d67135..399bf979b 100644 --- a/src/artifacts/DirectoryArtifactReader.ts +++ b/src/artifacts/DirectoryArtifactReader.ts @@ -14,12 +14,18 @@ import { sameArtifactSourceIdentity, type ArtifactEntry, type ArtifactReader, + type ArtifactTraversalOmission, + type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { hostExecutableMarking } from "./hostExecutableMarking.js"; import { OwnedDirectoryHandle } from "../filesystem/OwnedDirectoryHandle.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js"; +interface DirectoryTraversalOptions extends ArtifactTraversalOptions { + readonly descendInto?: (path: string) => boolean; +} + /** Non-symlink-following directory reader. */ export class DirectoryArtifactReader implements ArtifactReader { readonly format = "directory" as const; @@ -35,9 +41,11 @@ export class DirectoryArtifactReader implements ArtifactReader { /** Enumerate entries, optionally restricting descent while retaining directories. */ async *entries( signal?: AbortSignal, - descendInto?: (path: string) => boolean, + selection?: DirectoryTraversalOptions | ((path: string) => boolean), ): AsyncIterable { const root = await this.#resolveRoot(); + const options = + typeof selection === "function" ? { descendInto: selection } : selection; const pending = [root]; while (pending.length > 0) { abortIfNeeded(signal); @@ -50,7 +58,7 @@ export class DirectoryArtifactReader implements ArtifactReader { directory, pending, signal, - descendInto, + options, )) yield entry; await withDirectoryIoContext("inspect directory", directory, () => @@ -75,7 +83,7 @@ export class DirectoryArtifactReader implements ArtifactReader { directory: string, pending: string[], signal?: AbortSignal, - descendInto?: (path: string) => boolean, + options?: DirectoryTraversalOptions, ): AsyncIterable { const root = await this.#resolveRoot(); const handle = new OwnedDirectoryHandle( @@ -103,10 +111,16 @@ export class DirectoryArtifactReader implements ArtifactReader { ); const path = relative(root, absolute).split(sep).join("/"); const entry = artifactEntry(path, absolute, metadata); - if (entry === undefined) continue; + if (entry === undefined) { + options?.onOmission?.({ path, kind: unsupportedEntryKind(metadata) }); + continue; + } yield entry; - if (entry.kind === "directory" && (descendInto?.(path) ?? true)) - childDirectories.push(absolute); + if (entry.kind === "directory") { + if (options?.descendInto?.(path) ?? true) + childDirectories.push(absolute); + else options?.onOmission?.({ path, kind: "directory-descendants" }); + } } } catch (cause: unknown) { traversal = { @@ -256,6 +270,19 @@ export class DirectoryArtifactReader implements ArtifactReader { } } +const unsupportedEntryKind = ( + metadata: BigIntStats, +): ArtifactTraversalOmission["kind"] => + metadata.isFIFO() + ? "fifo" + : metadata.isSocket() + ? "socket" + : metadata.isBlockDevice() + ? "block-device" + : metadata.isCharacterDevice() + ? "character-device" + : "unsupported"; + const artifactEntry = ( path: string, absolute: string, diff --git a/src/artifacts/NativeDmgArtifactReader.ts b/src/artifacts/NativeDmgArtifactReader.ts index 801ee4daf..93c12ece3 100644 --- a/src/artifacts/NativeDmgArtifactReader.ts +++ b/src/artifacts/NativeDmgArtifactReader.ts @@ -8,6 +8,7 @@ import { ArtifactReaderFailure, type ArtifactEntry, type ArtifactReader, + type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js"; import { @@ -149,13 +150,24 @@ export class NativeDmgArtifactReader implements ArtifactReader { this.host = host ?? createSystemHost(environment); } - async *entries(signal?: AbortSignal): AsyncIterable { + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { await this.#ensureAttached(signal); const directory = this.#directory; if (directory === undefined) throw new ArtifactReaderFailure("unavailable", "DMG is not attached"); const prefix = basename(this.path); - const entries = directory.entries(signal)[Symbol.asyncIterator](); + const entries = directory + .entries(signal, { + onOmission: (omission) => + options?.onOmission?.({ + ...omission, + path: `${prefix}/${omission.path}`, + }), + }) + [Symbol.asyncIterator](); try { for (;;) { if (this.#closed) return; diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.ts index f195eac27..749b6e0b5 100644 --- a/src/artifacts/inventory/reader.ts +++ b/src/artifacts/inventory/reader.ts @@ -1,5 +1,8 @@ import { AsarArtifactReader } from "../AsarArtifactReader.js"; -import type { ArtifactReader } from "../ArtifactReader.js"; +import type { + ArtifactReader, + ArtifactReaderTraversal, +} from "../ArtifactReader.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js"; import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js"; @@ -7,7 +10,6 @@ import { ZipArtifactReader } from "../ZipArtifactReader.js"; import type { ArtifactInventoryCoverage, ArtifactOccurrence, - IntegrityContradiction, } from "../../domain/artifactGraph.js"; import { isZipPackageFormat, @@ -40,42 +42,47 @@ export const createReader = ( } }; -/** Classify source coverage and preserve its user-facing explanation once. */ -export const inventoryAssessment = ( +/** Decide traversal scope and availability once, independently of diagnostic wording. */ +export const inventoryCoverage = ( format: ArtifactOccurrence["artifact_format"], reader: ArtifactReader | undefined, - integrityContradictions: readonly IntegrityContradiction[], - caseCollisionLimitation: string | undefined, + traversal: ArtifactReaderTraversal, ): { - readonly coverage_status: ArtifactInventoryCoverage; + readonly coverage: ArtifactInventoryCoverage; readonly limitations: string[]; } => { - let coverageStatus: ArtifactInventoryCoverage = "complete"; - const limitations: string[] = []; - if (reader !== undefined) { - if (isZipPackageFormat(reader.format)) - limitations.push(ZIP_NON_ENTRY_TAIL_LIMITATION); - } else if (format === "dmg" || format === "pkg") { - coverageStatus = "partial"; - limitations.push( - `${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`, - ); - } else if (format === "mach-o-universal" && process.platform !== "darwin") { - coverageStatus = "partial"; - limitations.push( - "Universal Mach-O slices require the native macOS lipo adapter.", - ); - } else { - // The graph's modeled scope is the root itself for recognized leaf formats. - limitations.push("Artifact has no child container entries."); - } - if (caseCollisionLimitation !== undefined) { - coverageStatus = "partial"; - limitations.push(caseCollisionLimitation); - } - if (integrityContradictions.length > 0) - limitations.push( - `${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`, - ); - return { coverage_status: coverageStatus, limitations }; + if (reader !== undefined) + return { + coverage: { scope: "container-entries", status: traversal.status }, + limitations: [ + ...(isZipPackageFormat(reader.format) + ? [ZIP_NON_ENTRY_TAIL_LIMITATION] + : []), + ...traversal.limitations, + ], + }; + // dmg and pkg reach here only after classifyRootFormat matched their bytes. + if (format === "dmg" || format === "pkg") + return { + coverage: { scope: "container-entries", status: "unknown" }, + limitations: [ + `${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`, + ], + }; + if (format === "mach-o-universal") + return { + coverage: { scope: "container-entries", status: "unknown" }, + limitations: [ + "Universal Mach-O slices require the native macOS lipo adapter.", + ], + }; + if (isZipPackageFormat(format) || format === "directory" || format === "asar") + return { + coverage: { scope: "container-entries", status: "unknown" }, + limitations: [`${format} child inventory reader is unavailable.`], + }; + return { + coverage: { scope: "root", status: "complete" }, + limitations: ["Artifact has no child container entries."], + }; }; diff --git a/src/artifacts/inventory/scanCanonical.ts b/src/artifacts/inventory/scanCanonical.ts index d5234bdf6..2c8679e21 100644 --- a/src/artifacts/inventory/scanCanonical.ts +++ b/src/artifacts/inventory/scanCanonical.ts @@ -10,6 +10,7 @@ import type { RootInventorySource } from "./classify.js"; import { ArtifactReaderFailure, type ArtifactReader, + type ArtifactReaderTraversal, } from "../ArtifactReader.js"; import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js"; import { @@ -40,7 +41,7 @@ import { hashStableRootArtifact, hashStableRootArtifactHandle, } from "./hashStableRootArtifact.js"; -import { createReader, inventoryAssessment } from "./reader.js"; +import { createReader, inventoryCoverage } from "./reader.js"; import { scanReader, type PendingIntegrityContradiction, @@ -99,6 +100,7 @@ export const scanCanonicalArtifactInventoryInScope = async ( occurrences, pendingContradictions, caseCollisionLimitation, + traversal, } = await scanReader(reader, ownedReaders, options.signal, integrity); outcome = { kind: "completed", @@ -115,6 +117,7 @@ export const scanCanonicalArtifactInventoryInScope = async ( occurrences, pendingContradictions, caseCollisionLimitation, + traversal, }), }; } catch (cause: unknown) { @@ -196,6 +199,7 @@ interface SnapshotBuildInput { readonly occurrences: MutableOccurrence[]; readonly pendingContradictions: PendingIntegrityContradiction[]; readonly caseCollisionLimitation: string | undefined; + readonly traversal: ArtifactReaderTraversal; } const buildInventorySnapshot = async ( @@ -268,12 +272,7 @@ const buildInventorySnapshot = async ( orderedOccurrences, orderedEdges, }); - const assessment = inventoryAssessment( - rootFormat, - reader, - integrityContradictions, - input.caseCollisionLimitation, - ); + const traversal = inventoryCoverage(rootFormat, reader, input.traversal); return { manifest, nodes: orderedNodes, @@ -281,8 +280,14 @@ const buildInventorySnapshot = async ( edges: orderedEdges, provenance: reader?.provenance() ?? [], integrity_contradictions: integrityContradictions, - coverage_status: assessment.coverage_status, - limitations: assessment.limitations, + coverage: traversal.coverage, + coverage_status: + traversal.coverage.status === "complete" ? "complete" : "partial", + limitations: buildLimitations( + traversal.limitations, + integrityContradictions, + input.caseCollisionLimitation, + ), }; }; @@ -395,3 +400,17 @@ const buildManifest = ({ occurrence_count: orderedOccurrences.length, edge_count: orderedEdges.length, }); + +const buildLimitations = ( + scopeLimitations: readonly string[], + integrityContradictions: readonly IntegrityContradiction[], + caseCollisionLimitation: string | undefined, +): string[] => [ + ...scopeLimitations, + ...(caseCollisionLimitation === undefined ? [] : [caseCollisionLimitation]), + ...(integrityContradictions.length === 0 + ? [] + : [ + `${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`, + ]), +]; diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 6873b14d7..38b67d843 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -9,9 +9,12 @@ import { ArtifactReaderFailure, type ArtifactEntry, type ArtifactReader, + type ArtifactReaderTraversal, + type ArtifactTraversalOmission, } from "../ArtifactReader.js"; import { AsarArtifactReader } from "../AsarArtifactReader.js"; import type { ArtifactNode } from "../../domain/artifactGraph.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; import { classifyArtifactContent, classifyArtifactBytes, @@ -55,6 +58,7 @@ interface ScanContext { readonly registry: ArtifactPathRegistry; readonly expandedContainerIds: Set; readonly ownedReaders: ArtifactReader[]; + readonly omissions: ArtifactTraversalOmission[]; } export const scanReader = async ( @@ -67,6 +71,7 @@ export const scanReader = async ( readonly occurrences: MutableOccurrence[]; readonly pendingContradictions: PendingIntegrityContradiction[]; readonly caseCollisionLimitation: string | undefined; + readonly traversal: ArtifactReaderTraversal; }> => { const { nodes, occurrences, pendingContradictions } = emptyScan(); if (reader === undefined) @@ -75,6 +80,7 @@ export const scanReader = async ( occurrences, pendingContradictions, caseCollisionLimitation: undefined, + traversal: { status: "complete", limitations: [] }, }; const context: ScanContext = { reader, @@ -87,6 +93,7 @@ export const scanReader = async ( registry: new ArtifactPathRegistry(), expandedContainerIds: new Set(), ownedReaders, + omissions: [], }; await visitArtifactEntries(context, reader, ""); // Archive directories may appear after their children. Resolve containment @@ -101,9 +108,37 @@ export const scanReader = async ( occurrences, pendingContradictions, caseCollisionLimitation: annotateEnglishUnicodeCaseCollisions(occurrences), + traversal: { + status: context.omissions.length === 0 ? "complete" : "unknown", + limitations: [ + ...new Set( + context.omissions.map( + ({ path, kind }) => + `Artifact traversal omitted ${kind} at ${JSON.stringify(path)}.`, + ), + ), + ].sort(compareUnicodeCodePoints), + }, }; }; +const inventoryEntries = ( + context: ScanContext, + reader: ArtifactReader, + prefix: string, +): AsyncIterator => + reader + .entries(context.signal, { + onOmission: (omission) => + context.omissions.push({ + ...omission, + path: normalizeArtifactPath( + prefix.length === 0 ? omission.path : `${prefix}/${omission.path}`, + ), + }), + }) + [Symbol.asyncIterator](); + const visitArtifactEntries = async ( context: ScanContext, currentReader: ArtifactReader, @@ -117,7 +152,7 @@ const visitArtifactEntries = async ( { reader: currentReader, prefix, - iterator: currentReader.entries(context.signal)[Symbol.asyncIterator](), + iterator: inventoryEntries(context, currentReader, prefix), }, ]; let failure: { readonly cause: unknown } | undefined; @@ -191,7 +226,7 @@ const visitArtifactEntries = async ( stack.push({ reader: nested, prefix: logicalPath, - iterator: nested.entries(context.signal)[Symbol.asyncIterator](), + iterator: inventoryEntries(context, nested, logicalPath), }); } } diff --git a/src/domain/android/androidApplication.ts b/src/domain/android/androidApplication.ts index c46fb9661..99b20c321 100644 --- a/src/domain/android/androidApplication.ts +++ b/src/domain/android/androidApplication.ts @@ -68,7 +68,7 @@ export const androidApplicationProjectionResultSchema = z.strictObject({ // Count completeness is separate from the declared coverage status. inventory_complete: z.boolean(), }), - limitations: z.array(z.string().min(1)), + limitations: z.array(z.string()), }); export type AndroidApplicationProjectionInput = z.infer< @@ -99,7 +99,8 @@ export const projectAndroidApplication = ( ); const components = classified; const limitations = [ - ...(!hasCompleteInventoryCoverage(inventory) + ...inventory.limitations, + ...(!hasCompleteInventoryCoverage(inventory, "container-entries") ? ["Source inventory coverage is partial; absence is unknown."] : []), "Manifest, resource, signing, and bytecode semantics require a dedicated Android provider; this projection reports exact inventory paths and hashes only.", diff --git a/src/domain/apple/appleApplication.ts b/src/domain/apple/appleApplication.ts index dfb34e3c5..6dac13f82 100644 --- a/src/domain/apple/appleApplication.ts +++ b/src/domain/apple/appleApplication.ts @@ -94,7 +94,7 @@ export const appleApplicationProjectionResultSchema = z.strictObject({ // Count completeness is separate from the declared coverage status. inventory_complete: z.boolean(), }), - limitations: z.array(z.string().min(1)), + limitations: z.array(z.string()), }); export type AppleApplicationProjectionInput = z.infer< @@ -169,18 +169,24 @@ export const projectAppleApplication = ( ); const uninventoriedImage = rootFormat === "dmg" && inventory.occurrences.length === 1; - const inventoryComplete = hasCompleteInventoryCoverage(inventory); + const inventoryComplete = hasCompleteInventoryCoverage( + inventory, + "container-entries", + ); const complete = inventoryComplete && !uninventoriedImage; - const limitations = projectionLimitations({ - complete: inventoryComplete, - uninventoriedImage, - roots, - bundles, - sidecars: entries.some(({ path }) => isSidecar(path)), - symlinks: entries.some(({ kind }) => kind === "symlink"), - unattributed, - bridgeCandidateCoverage: bridgeProjection.coverage, - }); + const limitations = [ + ...inventory.limitations, + ...projectionLimitations({ + complete: inventoryComplete, + uninventoriedImage, + roots, + bundles, + sidecars: entries.some(({ path }) => isSidecar(path)), + symlinks: entries.some(({ kind }) => kind === "symlink"), + unattributed, + bridgeCandidateCoverage: bridgeProjection.coverage, + }), + ]; const withoutId = { root_sha256: inventory.manifest.root_sha256, root_format: rootFormat, diff --git a/src/domain/artifactComparison.ts b/src/domain/artifactComparison.ts index 9d7a766af..72494136c 100644 --- a/src/domain/artifactComparison.ts +++ b/src/domain/artifactComparison.ts @@ -104,13 +104,51 @@ export const compareArtifacts = ( const right = parseArtifactInventoryEvidence(rightEvidence); const leftComplete = left.inventory.complete; const rightComplete = right.inventory.complete; - const leftCovered = hasCompleteInventoryCoverage(left.inventory); - const rightCovered = hasCompleteInventoryCoverage(right.inventory); + const sameScope = + left.inventory.coverage.scope === right.inventory.coverage.scope; + const rootCoverageComplete = + sameScope && + left.inventory.complete && + right.inventory.complete && + left.inventory.coverage.status === "complete" && + right.inventory.coverage.status === "complete"; + const entryCoverageComplete = + rootCoverageComplete && + left.inventory.coverage.scope === "container-entries"; + const leftCovered = + sameScope && + hasCompleteInventoryCoverage(left.inventory, left.inventory.coverage.scope); + const rightCovered = + sameScope && + hasCompleteInventoryCoverage( + right.inventory, + right.inventory.coverage.scope, + ); + const fullyCovered = + leftCovered && + rightCovered && + (entryCoverageComplete || + (rootCoverageComplete && + left.inventory.manifest.occurrence_count === 1 && + right.inventory.manifest.occurrence_count === 1)); const limitations = [ ...left.inventory.limitations.map((item) => `Left: ${item}`), ...right.inventory.limitations.map((item) => `Right: ${item}`), ...(leftComplete ? [] : ["Left artifact inventory is incomplete."]), ...(rightComplete ? [] : ["Right artifact inventory is incomplete."]), + ...(sameScope ? [] : ["Artifact inventory scopes differ."]), + ...(sameScope && + left.inventory.coverage.scope === "root" && + (left.inventory.manifest.occurrence_count > 1 || + right.inventory.manifest.occurrence_count > 1) + ? ["Container entry coverage is unavailable for child paths."] + : []), + ...(left.inventory.coverage.status === "complete" + ? [] + : ["Left artifact inventory traversal coverage is unknown."]), + ...(right.inventory.coverage.status === "complete" + ? [] + : ["Right artifact inventory traversal coverage is unknown."]), ]; const changes = compareOccurrences( left.inventory, @@ -119,7 +157,7 @@ export const compareArtifacts = ( ...left.evidence.map(({ evidence_id: id }) => id), ...right.evidence.map(({ evidence_id: id }) => id), ], - leftCovered && rightCovered, + { root: rootCoverageComplete, entries: entryCoverageComplete }, ); applyRootHashChange(left.inventory, right.inventory, changes, [ ...left.evidence.map(({ evidence_id: id }) => id), @@ -129,7 +167,7 @@ export const compareArtifacts = ( left.inventory, right.inventory, changes, - leftCovered && rightCovered, + fullyCovered, ); return artifactComparisonResultSchema.parse({ status: @@ -139,7 +177,7 @@ export const compareArtifacts = ( ({ classification }) => classification === "contradiction", ) ? "contradiction" - : !leftCovered || !rightCovered + : !fullyCovered ? "unknown" : changes.some(({ classification }) => classification === "unknown") ? "unknown" @@ -158,7 +196,7 @@ const compareOccurrences = ( left: InventorySet, right: InventorySet, evidenceLinks: readonly string[], - complete: boolean, + coverage: { readonly root: boolean; readonly entries: boolean }, ): ArtifactChange[] => { const leftByPath = new Map( left.occurrences.map((item) => [item.logical_path, item]), @@ -187,7 +225,8 @@ const compareOccurrences = ( rightNode: nodeFor(rightOccurrence, rightNodes), leftRelations: leftRelations.get(path) ?? [], rightRelations: rightRelations.get(path) ?? [], - complete, + rootCovered: coverage.root, + entriesCovered: coverage.entries, evidenceLinks, }); if (change !== null) output.push(change); @@ -203,7 +242,8 @@ interface ClassifyPathInput { readonly rightNode: ArtifactNode | undefined; readonly leftRelations: readonly RelationProjection[]; readonly rightRelations: readonly RelationProjection[]; - readonly complete: boolean; + readonly rootCovered: boolean; + readonly entriesCovered: boolean; readonly evidenceLinks: readonly string[]; } @@ -217,17 +257,18 @@ const buildArtifactChangeBase = (input: ClassifyPathInput) => ({ }); const classifyPresence = (input: ClassifyPathInput): ArtifactChange | null => { + const covered = input.path === "." ? input.rootCovered : input.entriesCovered; if (input.leftOccurrence === undefined) return { ...buildArtifactChangeBase(input), - classification: input.complete ? "added" : "unknown", - dimensions: [input.complete ? "content" : "availability"], + classification: covered ? "added" : "unknown", + dimensions: [covered ? "content" : "availability"], }; if (input.rightOccurrence === undefined) return { ...buildArtifactChangeBase(input), - classification: input.complete ? "removed" : "unknown", - dimensions: [input.complete ? "content" : "availability"], + classification: covered ? "removed" : "unknown", + dimensions: [covered ? "content" : "availability"], }; return null; }; diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 6b2709baa..5a804c8d6 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -188,8 +188,11 @@ export const artifactGraphManifestSchema = z.object({ edge_count: z.number().int().min(0), }); -/** Whether the inventory producer covered its modeled child-entry scope. */ -export const artifactInventoryCoverageSchema = z.enum(["complete", "partial"]); +/** Producer-owned traversal coverage; content certainty remains on each occurrence. */ +export const artifactInventoryCoverageSchema = z.strictObject({ + scope: z.enum(["root", "container-entries"]), + status: z.enum(["complete", "unknown"]), +}); export type ArtifactInventoryCoverage = z.infer< typeof artifactInventoryCoverageSchema >; @@ -202,15 +205,19 @@ export const artifactInventoryResultSchema = z.object({ edges: z.array(artifactEdgeSchema), provenance: z.array(artifactCommandSchema), integrity_contradictions: z.array(integrityContradictionSchema).default([]), + // Omitted by older retained Evidence; the inventory parser owns its compatibility policy. + coverage: artifactInventoryCoverageSchema.optional(), limitations: z.array(z.string()), /** Optional only for persisted inventory Evidence written before this field. */ - coverage_status: artifactInventoryCoverageSchema.optional(), + coverage_status: z.enum(["complete", "partial"]).optional(), }); -/** Current inventory producers always classify the coverage they examined. */ +/** Current producers always declare coverage; historical Evidence inputs may omit it. */ export const artifactInventoryOutputSchema = - artifactInventoryResultSchema.extend({ - coverage_status: artifactInventoryCoverageSchema, + artifactInventoryResultSchema.safeExtend({ + coverage: artifactInventoryCoverageSchema, + coverage_status: + artifactInventoryResultSchema.shape.coverage_status.unwrap(), }); export type ArtifactInventoryOutput = z.infer< typeof artifactInventoryOutputSchema diff --git a/src/domain/artifactInventoryEvidence.ts b/src/domain/artifactInventoryEvidence.ts index 213364816..5f95bdad6 100644 --- a/src/domain/artifactInventoryEvidence.ts +++ b/src/domain/artifactInventoryEvidence.ts @@ -18,7 +18,10 @@ import { } from "./artifactIdentity.js"; import { parseEvidence, type Evidence } from "./evidence.js"; import { artifactInspectionResultSchema } from "./artifactInspection.js"; -import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "./zipPackageFormat.js"; +import { + isZipPackageFormat, + ZIP_NON_ENTRY_TAIL_LIMITATION, +} from "./zipPackageFormat.js"; /** Complete or partially assembled inventory used by comparison workflows. */ export interface InventorySet { @@ -30,13 +33,17 @@ export interface InventorySet { readonly limitations: readonly string[]; /** True when merged node, occurrence, and edge counts match the manifest. */ readonly complete: boolean; - readonly coverage_status: ArtifactInventoryCoverage; + readonly coverage: ArtifactInventoryCoverage; } /** Whether all graph members are present and the producer reports full coverage. */ export const hasCompleteInventoryCoverage = ( inventory: InventorySet, -): boolean => inventory.complete && inventory.coverage_status === "complete"; + requiredScope: ArtifactInventoryCoverage["scope"], +): boolean => + inventory.complete && + inventory.coverage.scope === requiredScope && + inventory.coverage.status === "complete"; export interface ParsedInventorySet { readonly evidence: readonly Evidence[]; @@ -132,11 +139,14 @@ const assembleInventoryResults = ( const first = pages[0]; if (first === undefined) throw new TypeError("Artifact inventory requires Evidence pages"); - const coverageStatuses = new Set(pages.map(resolveInventoryCoverageStatus)); - if (coverageStatuses.size !== 1) - throw new TypeError("Artifact inventory pages disagree on coverage status"); - const coverageStatus = [...coverageStatuses][0]!; const manifestJson = canonicalJson(first.manifest, "Artifact inventory"); + const coverageJson = canonicalJson( + { + coverage: first.coverage ?? null, + coverage_status: first.coverage_status ?? null, + }, + "Artifact inventory coverage", + ); const nodes = new Map(); const occurrences = new Map(); const paths = new Map(); @@ -144,6 +154,18 @@ const assembleInventoryResults = ( for (const page of pages) { if (canonicalJson(page.manifest, "Artifact inventory") !== manifestJson) throw new TypeError("Artifact inventory pages do not share one manifest"); + if ( + canonicalJson( + { + coverage: page.coverage ?? null, + coverage_status: page.coverage_status ?? null, + }, + "Artifact inventory coverage", + ) !== coverageJson + ) + throw new TypeError( + "Artifact inventory pages do not share one coverage declaration", + ); if ( canonicalJson(page.integrity_contradictions, "Artifact inventory") !== canonicalJson(first.integrity_contradictions, "Artifact inventory") @@ -165,12 +187,11 @@ const assembleInventoryResults = ( for (const edge of page.edges) mergeExact(edges, edge.edge_id, edge, "edge"); } - const inventory = orderedInventory( - first, - pages, - { nodes, occurrences, edges }, - coverageStatus, - ); + const inventory = orderedInventory(first, pages, { + nodes, + occurrences, + edges, + }); if (inventory.complete) validateCompleteInventory(inventory); return inventory; }; @@ -186,7 +207,6 @@ const orderedInventory = ( ArtifactInventoryResult["edges"][number] >; }, - coverageStatus: ArtifactInventoryCoverage, ): InventorySet => { const nodes = [...values.nodes.values()].sort((left, right) => compareUnicodeCodePoints(left.artifact_id, right.artifact_id), @@ -197,40 +217,78 @@ const orderedInventory = ( const edges = [...values.edges.values()].sort((left, right) => compareUnicodeCodePoints(left.edge_id, right.edge_id), ); + const limitations = [ + ...new Set(pages.flatMap((inventory) => inventory.limitations)), + ].sort(compareUnicodeCodePoints); return { manifest: first.manifest, nodes, occurrences, edges, integrityContradictions: first.integrity_contradictions, - limitations: [ - ...new Set(pages.flatMap((inventory) => inventory.limitations)), - ].sort(compareUnicodeCodePoints), + limitations, + coverage: resolveInventoryCoverage({ ...first, limitations }), complete: nodes.length === first.manifest.node_count && occurrences.length === first.manifest.occurrence_count && edges.length === first.manifest.edge_count, - coverage_status: coverageStatus, }; }; -const resolveInventoryCoverageStatus = ( +// Normalize both retained coverage formats at this owning boundary. +const resolveInventoryCoverage = ( inventory: ArtifactInventoryResult, ): ArtifactInventoryCoverage => { - if (inventory.coverage_status !== undefined) return inventory.coverage_status; - // Older retained Evidence has no typed status. Keep only its known - // informational notices complete; unknown limitation text fails closed. - return inventory.limitations.every(isKnownNonCoverageNotice) - ? "complete" - : "partial"; + const legacy = legacyInventoryCoverage( + inventory.manifest.root_format, + inventory.limitations, + inventory.integrity_contradictions.length, + ); + if (inventory.coverage !== undefined) { + if ( + inventory.coverage_status !== undefined && + (inventory.coverage.status === "complete") !== + (inventory.coverage_status === "complete") + ) + throw new TypeError( + "Artifact inventory has conflicting coverage declarations", + ); + return inventory.coverage; + } + return inventory.coverage_status === undefined + ? legacy + : { + ...legacy, + status: + inventory.coverage_status === "complete" ? "complete" : "unknown", + }; }; -const isKnownNonCoverageNotice = (limitation: string): boolean => - limitation === "Artifact has no child container entries." || - limitation === ZIP_NON_ENTRY_TAIL_LIMITATION || - /^\d+ integrity contradiction\(s\) were recorded; mismatched content is observed-untrusted\.$/u.test( - limitation, - ); +// Retained pre-coverage records cannot classify arbitrary prose. Recognize only +// the historical producer vocabulary here; unfamiliar diagnostics remain unknown. +const legacyInventoryCoverage = ( + format: ArtifactInventoryResult["manifest"]["root_format"], + limitations: readonly string[], + contradictionCount: number, +): ArtifactInventoryCoverage => ({ + scope: + isZipPackageFormat(format) || + ["directory", "asar", "dmg", "pkg", "mach-o-universal"].includes(format) + ? "container-entries" + : "root", + status: limitations.some( + (limitation) => + limitation !== "Artifact has no child container entries." && + limitation !== ZIP_NON_ENTRY_TAIL_LIMITATION && + !( + contradictionCount > 0 && + limitation === + `${String(contradictionCount)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.` + ), + ) + ? "unknown" + : "complete", +}); const validateCompleteInventory = (inventory: InventorySet): void => { const nodeIds = new Set(inventory.nodes.map(({ artifact_id: id }) => id)); diff --git a/src/domain/artifactInventorySnapshot.ts b/src/domain/artifactInventorySnapshot.ts index 2157c03ed..8626d7465 100644 --- a/src/domain/artifactInventorySnapshot.ts +++ b/src/domain/artifactInventorySnapshot.ts @@ -7,7 +7,9 @@ import type { /** Immutable inventory produced by one complete artifact scan. */ export interface ArtifactInventorySnapshot { - readonly coverage_status: ArtifactInventoryCoverage; + readonly coverage_status: NonNullable< + ArtifactInventoryResult["coverage_status"] + >; readonly manifest: ArtifactInventoryResult["manifest"]; readonly nodes: readonly ArtifactNode[]; readonly occurrences: ArtifactInventoryResult["occurrences"]; @@ -16,5 +18,6 @@ export interface ArtifactInventorySnapshot { ArtifactInventoryResult["provenance"][number] >; readonly integrity_contradictions: readonly IntegrityContradiction[]; + readonly coverage?: ArtifactInventoryCoverage; readonly limitations: readonly string[]; } diff --git a/src/domain/harmony/harmonyApplication.ts b/src/domain/harmony/harmonyApplication.ts index 71755e828..fff8429cb 100644 --- a/src/domain/harmony/harmonyApplication.ts +++ b/src/domain/harmony/harmonyApplication.ts @@ -72,7 +72,7 @@ export const harmonyApplicationProjectionResultSchema = z.strictObject({ // Count completeness is separate from the declared coverage status. inventory_complete: z.boolean(), }), - limitations: z.array(z.string().min(1)), + limitations: z.array(z.string()), }); export type HarmonyApplicationProjectionInput = z.infer< @@ -112,7 +112,8 @@ export const projectHarmonyApplication = ( classified.native_libraries, ); const limitations = [ - ...(!hasCompleteInventoryCoverage(inventory) + ...inventory.limitations, + ...(!hasCompleteInventoryCoverage(inventory, "container-entries") ? ["Source inventory coverage is partial; absence is unknown."] : []), ...(unavailable.length > 0 diff --git a/src/domain/mobileApplicationGraphProjection.ts b/src/domain/mobileApplicationGraphProjection.ts index 93e4143c5..13bbe9545 100644 --- a/src/domain/mobileApplicationGraphProjection.ts +++ b/src/domain/mobileApplicationGraphProjection.ts @@ -56,7 +56,8 @@ export const projectionCoverage = ( inventory_complete: boolean; } => ({ status: - hasCompleteInventoryCoverage(inventory) && budgetComplete + hasCompleteInventoryCoverage(inventory, "container-entries") && + budgetComplete ? ("complete-within-inventory" as const) : ("partial" as const), inventory_complete: inventory.complete, diff --git a/tests/acceptance/applications/artifactComparisonCoverage.test.ts b/tests/acceptance/applications/artifactComparisonCoverage.test.ts new file mode 100644 index 000000000..db8660311 --- /dev/null +++ b/tests/acceptance/applications/artifactComparisonCoverage.test.ts @@ -0,0 +1,432 @@ +import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { createServer } from "node:net"; +import { join } from "node:path"; +import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect } from "vitest"; + +import { + artifactInventoryResultSchema, + type ArtifactInventoryResult, +} from "../../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../../src/domain/evidence.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; + +const plist = (message: string) => + `\nmessage${message}\n`; + +const inventoryEvidence = (input: unknown): Evidence => { + const inspection = parseEvidence(input); + const inventory = artifactInspectionResultSchema + .parse(inspection.normalized_result) + .substeps.find( + ({ evidence }) => evidence.operation === "inventory_artifact", + )?.evidence; + if (inventory === undefined) throw new Error("Expected inventory Evidence"); + return parseEvidence(inventory); +}; + +const inspectPair = async ( + cli: TestCli, + leftPath: string, + rightPath: string, +) => { + const result = await cli.run({ + arguments: ["inspect-artifact", leftPath, "--json"], + }); + expect(result.exitCode, `${result.stderr}\n${result.stdout}`).toBe(0); + const left = inventoryEvidence(result.json); + const { call } = await connectLocalToolsMcp(); + const opened = await call("open_binary", { path: rightPath }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspected = await call("inspect_artifact", {}); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + return { left, right: inventoryEvidence(inspected.structuredContent), call }; +}; + +const assertCoverageControls = async ({ + left, + right, + inventory, + compare, +}: { + left: Evidence; + right: Evidence; + inventory: ArtifactInventoryResult; + compare: (left: Evidence, right: Evidence) => Promise; +}) => { + // Authenticated derived controls isolate missing members and genuine coverage + // gaps; they do not claim that the complete plist producer omitted children. + const derive = (result: unknown) => + createEvidence( + { + path: right.subject?.local_path ?? "", + sha256: inventory.manifest.root_sha256, + format: "plist", + }, + right.provider, + { + operation: "inventory_artifact", + parameters: {}, + result: jsonValueSchema.parse(result), + confidence: "derived", + authority: "shipped-artifact", + }, + ); + const partial = await compare(left, derive({ ...inventory, nodes: [] })); + expect(partial.normalized_result).toMatchObject({ + status: "truncated", + summary: { unchanged: 0, unknown: 1 }, + }); + const reworded = await compare( + left, + derive({ + ...inventory, + coverage: { scope: "root", status: "complete" }, + limitations: ["Root bytes observed; no container children are modeled."], + }), + ); + expect(reworded.normalized_result).toMatchObject({ + status: "unchanged", + summary: { unchanged: 1, unknown: 0 }, + limitations: expect.arrayContaining([ + "Right: Root bytes observed; no container children are modeled.", + ]), + }); + const gap = await compare( + left, + derive({ + ...inventory, + coverage: { scope: "root", status: "unknown" }, + coverage_status: "partial", + limitations: [], + }), + ); + expect(gap.normalized_result).toMatchObject({ + status: "unknown", + summary: { unchanged: 0, unknown: 1 }, + }); + const differentScope = await compare( + left, + derive({ + ...inventory, + coverage: { scope: "container-entries", status: "complete" }, + }), + ); + expect(differentScope.normalized_result).toMatchObject({ + status: "unknown", + limitations: expect.arrayContaining(["Artifact inventory scopes differ."]), + }); + + const { coverage, coverage_status: _coverageStatus, ...legacy } = inventory; + expect(coverage).toBeDefined(); + const retained = await compare(left, derive(legacy)); + expect(retained.normalized_result).toMatchObject({ + status: "unchanged", + summary: { unchanged: 1, unknown: 0 }, + }); + for (const limitation of [ + "Child inventory is unavailable.", + "Missing child observations; 1 integrity contradiction(s) were recorded.", + ]) { + const unfamiliarLegacy = await compare( + left, + derive({ ...legacy, limitations: [limitation] }), + ); + expect(unfamiliarLegacy.normalized_result).toMatchObject({ + status: "unknown", + summary: { unchanged: 0, unknown: 1 }, + }); + } +}; + +cliTest( + "compares fully observed roots without inventing coverage gaps through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-artifact-coverage-"); + const leftPath = join(root, "left.plist"); + const rightPath = join(root, "right.plist"); + await Promise.all([ + writeFile(leftPath, plist("hello")), + writeFile(rightPath, plist("hello")), + ]); + const leftResult = await cli.run({ + arguments: ["inspect-artifact", leftPath, "--json"], + }); + expect(leftResult.exitCode, leftResult.stderr).toBe(0); + const left = inventoryEvidence(leftResult.json); + const { call } = await connectLocalToolsMcp(); + const inspect = async (path: string) => { + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const result = await call("inspect_artifact", {}); + expect(result.isError, JSON.stringify(result)).not.toBe(true); + return inventoryEvidence(result.structuredContent); + }; + const compare = async (left: Evidence, right: Evidence) => { + const result = await call("compare_artifacts", { left, right }); + expect(result.isError, JSON.stringify(result)).not.toBe(true); + return parseEvidence(result.structuredContent); + }; + const right = await inspect(rightPath); + const inventory = artifactInventoryResultSchema.parse( + right.normalized_result, + ); + expect(inventory.coverage).toEqual({ scope: "root", status: "complete" }); + expect(inventory.manifest).toMatchObject({ + root_format: "plist", + node_count: 1, + occurrence_count: 1, + edge_count: 0, + }); + expect(inventory.limitations).toEqual([ + "Artifact has no child container entries.", + ]); + const equal = await compare(left, right); + expect(equal.normalized_result).toMatchObject({ + status: "unchanged", + summary: { unchanged: 1, unknown: 0 }, + changes: [], + limitations: [ + "Left: Artifact has no child container entries.", + "Right: Artifact has no child container entries.", + ], + }); + const behavior = await call("find_changed_behavior", { + comparisons: [equal], + }); + expect(behavior.isError, JSON.stringify(behavior)).not.toBe(true); + expect(behavior.structuredContent).toMatchObject({ + normalized_result: { + behavior_status: "unknown", + summary: { unresolved: 0 }, + findings: { items: [] }, + limitations: expect.arrayContaining([ + "No process comparison Evidence was supplied.", + ]), + }, + }); + const unknowns = await call("list_unknowns", { + domain: "artifact-comparison", + }); + expect(unknowns.isError, JSON.stringify(unknowns)).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { items: [], total: 0 }, + }); + + await writeFile(rightPath, plist("world")); + const changed = await compare(left, await inspect(rightPath)); + expect(changed.normalized_result).toMatchObject({ + status: "changed", + summary: { unchanged: 0, changed: 1, unknown: 0 }, + changes: [ + expect.objectContaining({ logical_path: ".", dimensions: ["content"] }), + ], + }); + + const zipPaths = [join(root, "left.zip"), join(root, "right.zip")] as const; + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("same.txt", new TextReader("same")); + const bytes = await writer.close(); + await Promise.all(zipPaths.map((path) => writeFile(path, bytes))); + const zip = await compare( + await inspect(zipPaths[0]), + await inspect(zipPaths[1]), + ); + expect(zip.normalized_result).toMatchObject({ + status: "unchanged", + summary: { unchanged: 2, unknown: 0 }, + changes: [], + }); + + await assertCoverageControls({ left, right, inventory, compare }); + }, + 30_000, +); + +cliTest.skipIf(process.platform === "win32")( + "preserves executable metadata changes for fully observed plist roots", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-artifact-mode-"); + const leftPath = join(root, "left.plist"); + const rightPath = join(root, "right.plist"); + await Promise.all([ + writeFile(leftPath, plist("hello")), + writeFile(rightPath, plist("hello")), + ]); + await Promise.all([chmod(leftPath, 0o644), chmod(rightPath, 0o755)]); + const result = await cli.run({ + arguments: ["inspect-artifact", leftPath, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const left = inventoryEvidence(result.json); + const { call } = await connectLocalToolsMcp(); + const opened = await call("open_binary", { path: rightPath }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspected = await call("inspect_artifact", {}); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + const right = inventoryEvidence(inspected.structuredContent); + expect(right.subject?.digest.sha256).toBe(left.subject?.digest.sha256); + const comparison = await call("compare_artifacts", { left, right }); + expect(comparison.isError, JSON.stringify(comparison)).not.toBe(true); + expect( + parseEvidence(comparison.structuredContent).normalized_result, + ).toMatchObject({ + status: "changed", + summary: { unchanged: 0, changed: 1, unknown: 0 }, + changes: [ + expect.objectContaining({ + logical_path: ".", + dimensions: ["executable"], + }), + ], + }); + }, + 30_000, +); + +cliTest.skipIf(process.platform === "win32")( + "retains traversal uncertainty when a directory omits special filesystem entries", + async ({ cli, processes }) => { + const root = await createTestTempDirectory("rea-gap-"); + const leftPath = join(root, "l.app"), + rightPath = join(root, "r.app"); + await Promise.all([mkdir(leftPath), mkdir(rightPath)]); + await Promise.all([ + writeFile(join(leftPath, "same.txt"), "same"), + writeFile(join(rightPath, "same.txt"), "same"), + ]); + await Promise.all( + [leftPath, rightPath].flatMap((path) => [ + writeFile( + join(path, "Info.plist"), + "CFBundleExecutablemain.js", + ), + writeFile(join(path, "main.js"), "export const value = 1;"), + ]), + ); + const fifo = await processes.run("mkfifo", [join(leftPath, "p")]); + expect(fifo.exitCode, fifo.stderr).toBe(0); + const socket = createServer(); + await new Promise((resolve, reject) => { + socket.once("error", reject); + socket.listen(join(leftPath, "s"), resolve); + }); + try { + const { left, right, call } = await inspectPair(cli, leftPath, rightPath); + const inventory = artifactInventoryResultSchema.parse( + left.normalized_result, + ); + expect(inventory.coverage).toEqual({ + scope: "container-entries", + status: "unknown", + }); + expect(inventory.limitations).toEqual( + expect.arrayContaining([ + 'Artifact traversal omitted fifo at "p".', + 'Artifact traversal omitted socket at "s".', + ]), + ); + const compared = await call("compare_artifacts", { left, right }); + expect(compared.isError, JSON.stringify(compared)).not.toBe(true); + expect( + parseEvidence(compared.structuredContent).normalized_result, + ).toMatchObject({ status: "unknown", summary: { unchanged: 0 } }); + } finally { + await new Promise((resolve, reject) => + socket.close((cause) => + cause === undefined ? resolve() : reject(cause), + ), + ); + } + }, + 30_000, +); + +cliTest( + "compares fully traversed case-distinct ZIP entries while retaining materialization limits", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-case-coverage-"); + const leftPath = join(root, "left.zip"), + rightPath = join(root, "right.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("A", new TextReader("upper")); + await writer.add("a", new TextReader("lower")); + const bytes = await writer.close(); + await Promise.all([ + writeFile(leftPath, bytes), + writeFile(rightPath, bytes), + ]); + const { left, right, call } = await inspectPair(cli, leftPath, rightPath); + const inventory = artifactInventoryResultSchema.parse( + left.normalized_result, + ); + expect(inventory.coverage).toEqual({ + scope: "container-entries", + status: "complete", + }); + expect( + inventory.occurrences.map(({ logical_path }) => logical_path).sort(), + ).toEqual([".", "A", "a"]); + const compared = await call("compare_artifacts", { left, right }); + expect(compared.isError, JSON.stringify(compared)).not.toBe(true); + expect( + parseEvidence(compared.structuredContent).normalized_result, + ).toMatchObject({ + status: "unchanged", + summary: { unchanged: 3, unknown: 0 }, + changes: [], + limitations: expect.arrayContaining([ + expect.stringContaining("case-insensitive destination"), + ]), + }); + }, + 30_000, +); + +cliTest( + "keeps unreadable encrypted ZIP content unknown with complete traversal", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-encrypted-comparison-"); + const leftPath = join(root, "left.zip"); + const rightPath = join(root, "right.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("secret.txt", new TextReader("secret bytes"), { + password: "fixture-password", + }); + const bytes = await writer.close(); + await Promise.all([ + writeFile(leftPath, bytes), + writeFile(rightPath, bytes), + ]); + const { left, right, call } = await inspectPair(cli, leftPath, rightPath); + for (const evidence of [left, right]) + expect( + artifactInventoryResultSchema.parse(evidence.normalized_result), + ).toMatchObject({ + coverage: { scope: "container-entries", status: "complete" }, + occurrences: expect.arrayContaining([ + expect.objectContaining({ + logical_path: "secret.txt", + hash_status: "unavailable", + encrypted: true, + }), + ]), + }); + const compared = await call("compare_artifacts", { left, right }); + expect(compared.isError, JSON.stringify(compared)).not.toBe(true); + const result = parseEvidence(compared.structuredContent).normalized_result; + expect(result).toMatchObject({ + status: "unknown", + summary: { unknown: expect.any(Number) }, + }); + expect(result).toHaveProperty("summary.unknown", 1); + }, +); diff --git a/tests/boundary/filesystem/android/application.test.ts b/tests/boundary/filesystem/android/application.test.ts index 28fa4d8c8..01a5901c8 100644 --- a/tests/boundary/filesystem/android/application.test.ts +++ b/tests/boundary/filesystem/android/application.test.ts @@ -228,7 +228,7 @@ describe("case-distinct APK resource names", () => { await runProviderAnalysis(path, "inventory_artifact", {}), ); expect(inventory.normalized_result).toMatchObject({ - coverage_status: "partial", + coverage_status: "complete", }); const result = projectAndroidApplicationEvidence({ inventory_evidence: [inventory], @@ -238,11 +238,11 @@ describe("case-distinct APK resource names", () => { ); expect(projection.root_format).toBe("apk"); expect(projection.coverage).toEqual({ - status: "partial", + status: "complete-within-inventory", inventory_complete: true, }); - expect(projection.limitations).toContain( - "Source inventory coverage is partial; absence is unknown.", + expect(projection.limitations.join(" ")).toContain( + "case-insensitive destination", ); expect(projection.components.manifests).toHaveLength(1); expect(projection.components.dex).toHaveLength(1); diff --git a/tests/boundary/filesystem/appleApplication.test.ts b/tests/boundary/filesystem/appleApplication.test.ts index 00ef24ba5..31c395b55 100644 --- a/tests/boundary/filesystem/appleApplication.test.ts +++ b/tests/boundary/filesystem/appleApplication.test.ts @@ -75,7 +75,7 @@ async function createCompleteAppleProjection() { } describe("Apple application projection", () => { - it("keeps a case-colliding real IPA inventory partial", async () => { + it("retains complete traversal and case-collision limits for a real IPA", async () => { const root = await createTestTempDirectory("rea-apple-case-collision-"); const path = join(root, "CaseCollision.ipa"); const writer = new ZipWriter(new Uint8ArrayWriter()); @@ -93,7 +93,7 @@ describe("Apple application projection", () => { await runProviderAnalysis(path, "inventory_artifact", {}), ); expect(inventory.normalized_result).toMatchObject({ - coverage_status: "partial", + coverage_status: "complete", }); const result = projectAppleApplicationEvidence({ inventory_evidence: [inventory], @@ -103,11 +103,11 @@ describe("Apple application projection", () => { ); expect(projection.coverage).toEqual({ - status: "partial", + status: "complete-within-inventory", inventory_complete: true, }); - expect(projection.limitations).toContain( - "Source inventory coverage is partial; absence is unknown.", + expect(projection.limitations.join(" ")).toContain( + "case-insensitive destination", ); }); diff --git a/tests/boundary/filesystem/artifactComparison.test.ts b/tests/boundary/filesystem/artifactComparison.test.ts index ca40596d3..f298c28dd 100644 --- a/tests/boundary/filesystem/artifactComparison.test.ts +++ b/tests/boundary/filesystem/artifactComparison.test.ts @@ -77,7 +77,11 @@ describe("leaf artifact comparison", () => { // Persisted inventory Evidence predates coverage_status. The known leaf // notice remains complete when normalized at the inventory boundary. const parsed = artifactInventoryResultSchema.parse(left.normalized_result); - const { coverage_status: _coverageStatus, ...legacyInventory } = parsed; + const { + coverage: _coverage, + coverage_status: _coverageStatus, + ...legacyInventory + } = parsed; const legacy = createEvidence( { path: left.subject!.local_path, @@ -157,7 +161,7 @@ describe("artifact inventory coverage", () => { const graph = artifactInventoryResultSchema.parse( complete.normalized_result, ); - const partial = createEvidence( + const contradictory = createEvidence( { path: complete.subject!.local_path, sha256: complete.subject!.digest.sha256, @@ -168,6 +172,29 @@ describe("artifact inventory coverage", () => { operation: "inventory_artifact", parameters: {}, result: jsonValueSchema.parse({ ...graph, coverage_status: "partial" }), + confidence: "derived", + authority: "shipped-artifact", + }, + ); + expect(() => parseArtifactInventoryEvidence(contradictory)).toThrow( + "Artifact inventory has conflicting coverage declarations", + ); + + const { coverage: _coverage, ...legacyStatusGraph } = graph; + const partial = createEvidence( + { + path: complete.subject!.local_path, + sha256: complete.subject!.digest.sha256, + format: complete.subject!.format, + }, + PROVIDER, + { + operation: "inventory_artifact", + parameters: {}, + result: jsonValueSchema.parse({ + ...legacyStatusGraph, + coverage_status: "partial", + }), confidence: "observed", authority: "shipped-artifact", }, @@ -180,7 +207,8 @@ describe("artifact inventory coverage", () => { summary: { unchanged: 0, unknown: 1 }, }); - const { coverage_status: _coverageStatus, ...legacyGraph } = graph; + const { coverage_status: _coverageStatus, ...legacyGraph } = + legacyStatusGraph; const unknownLegacy = createEvidence( { path: complete.subject!.local_path, @@ -219,6 +247,7 @@ describe("artifact inventory coverage", () => { const fragmentGraph = artifactInventoryResultSchema.parse( fragment.normalized_result, ); + const { coverage: _coverage, ...legacyFragmentGraph } = fragmentGraph; const mixed = createEvidence( { path: fragment.subject!.local_path, @@ -230,7 +259,7 @@ describe("artifact inventory coverage", () => { operation: "inventory_artifact", parameters: fragment.parameters, result: jsonValueSchema.parse({ - ...fragmentGraph, + ...legacyFragmentGraph, coverage_status: "partial", }), confidence: "derived", @@ -239,7 +268,7 @@ describe("artifact inventory coverage", () => { ); expect(() => parseArtifactInventoryEvidence([firstFragment, mixed]), - ).toThrow("Artifact inventory pages disagree on coverage status"); + ).toThrow("Artifact inventory pages do not share one coverage declaration"); }); }); @@ -314,6 +343,61 @@ describe("artifact comparison", () => { }); describe("artifact comparison completeness", () => { + it("does not infer child changes from root-only coverage", async () => { + const root = await createTestTempDirectory("rea-artifact-root-coverage-"); + const leftPath = join(root, "left.app"); + const rightPath = join(root, "right.app"); + await Promise.all([mkdir(leftPath), mkdir(rightPath)]); + await Promise.all([ + writeFile(join(leftPath, "left-only.txt"), "left"), + writeFile(join(rightPath, "right-only.txt"), "right"), + ]); + const [left, right] = await Promise.all([ + observe(leftPath), + observe(rightPath), + ]); + const rootOnly = (evidence: Awaited>) => { + const inventory = artifactInventoryResultSchema.parse( + evidence.normalized_result, + ); + return createEvidence( + { + path: evidence.subject?.local_path ?? "", + sha256: inventory.manifest.root_sha256, + format: inventory.manifest.root_format, + }, + PROVIDER, + { + operation: "inventory_artifact", + parameters: {}, + result: jsonValueSchema.parse({ + ...inventory, + coverage: { scope: "root", status: "complete" }, + }), + confidence: "derived", + authority: "shipped-artifact", + }, + ); + }; + const compared = compareArtifacts(rootOnly(left), rootOnly(right)); + expect(compared.status).toBe("unknown"); + expect(compared.summary).toMatchObject({ added: 0, removed: 0 }); + expect( + compared.changes + .filter(({ logical_path }) => logical_path !== ".") + .map(({ logical_path, classification }) => ({ + logical_path, + classification, + })), + ).toEqual([ + { logical_path: "left-only.txt", classification: "unknown" }, + { logical_path: "right-only.txt", classification: "unknown" }, + ]); + expect(compared.limitations).toContain( + "Container entry coverage is unavailable for child paths.", + ); + }); + it("compares every graph member for inventories larger than 500 entries", async () => { const root = await createTestTempDirectory("rea-artifact-pages-"); const entryCount = 501; @@ -415,6 +499,30 @@ describe("artifact comparison completeness", () => { evidence_links: citations, })), }); + const fragment = leftFragments[0]; + if (fragment === undefined) throw new Error("Expected inventory fragments"); + const conflictingScope = createEvidence( + { + path: leftPath, + sha256: left.subject?.digest.sha256 ?? "", + format: "directory", + }, + PROVIDER, + { + operation: "inventory_artifact", + parameters: {}, + result: jsonValueSchema.parse({ + ...artifactInventoryResultSchema.parse(fragment.normalized_result), + coverage: { scope: "root", status: "complete" }, + }), + }, + ); + expect(() => + compareArtifacts( + [conflictingScope, ...leftFragments.slice(1)], + rightFragments, + ), + ).toThrow(/coverage declaration/u); }, 15_000); it("rejects non-inventory and tampered Evidence", async () => { diff --git a/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts b/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts index 9ea26cfda..05e6a0e1a 100644 --- a/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts +++ b/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts @@ -52,3 +52,22 @@ it("rejects a root path replaced between its initial stat and open", async () => message: `Root artifact changed before inventory: ${path}`, }); }); + +it("keeps unavailable container traversal distinct from observed root bytes", async () => { + const root = await createTestTempDirectory("rea-artifact-root-scope-"); + const path = join(root, "fixture.pkg"); + // The recognized XAR prefix selects PKG inventory; no child decoder is installed. + await writeFile(path, Buffer.from("xar!fixture")); + const inventory = await inventoryArtifact(path); + expect(inventory.manifest).toMatchObject({ + root_format: "pkg", + node_count: 1, + occurrence_count: 1, + edge_count: 0, + }); + expect(inventory.occurrences[0]?.hash_status).toBe("verified"); + expect(inventory.coverage).toEqual({ + scope: "container-entries", + status: "unknown", + }); +}); diff --git a/tests/boundary/filesystem/harmony/application.test.ts b/tests/boundary/filesystem/harmony/application.test.ts index b2870e6c5..a50c440e9 100644 --- a/tests/boundary/filesystem/harmony/application.test.ts +++ b/tests/boundary/filesystem/harmony/application.test.ts @@ -214,7 +214,7 @@ describe("HarmonyOS packaging boundaries", () => { }); describe("HarmonyOS inventory coverage", () => { - it("keeps a case-colliding inventory partial in the application projection", async () => { + it("retains complete traversal and case-collision limits in the application projection", async () => { const path = await writeApplicationZip( "rea-harmony-case-collision-", "Fixture.hap", @@ -225,7 +225,7 @@ describe("HarmonyOS inventory coverage", () => { ); const inventory = await inventoryApplicationPackage(path); expect(inventory.normalized_result).toMatchObject({ - coverage_status: "partial", + coverage_status: "complete", }); const projection = requireDeterministicProjection( @@ -234,11 +234,11 @@ describe("HarmonyOS inventory coverage", () => { harmonyApplicationProjectionResultSchema, ); expect(projection.coverage).toEqual({ - status: "partial", + status: "complete-within-inventory", inventory_complete: true, }); - expect(projection.limitations).toContain( - "Source inventory coverage is partial; absence is unknown.", + expect(projection.limitations.join(" ")).toContain( + "case-insensitive destination", ); }); diff --git a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts index de78af9d0..fa0481ac7 100644 --- a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts +++ b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts @@ -254,8 +254,11 @@ const assertHistoricalInventory = async ( const { archive, parent, inventoryEvidence, graph } = input; // Historical records may omit a later inventory field. Input parsing must // not insert defaults inside that authenticated child Evidence. - const { integrity_contradictions: _contradictions, ...legacyInventory } = - graph; + const { + integrity_contradictions: _contradictions, + coverage: _coverage, + ...legacyInventory + } = graph; const legacyTarget = { path: archive, sha256: graph.manifest.root_sha256, @@ -275,6 +278,9 @@ const assertHistoricalInventory = async ( expect( legacyInspection.substeps[0]?.evidence.normalized_result, ).not.toHaveProperty("integrity_contradictions"); + expect( + legacyInspection.substeps[0]?.evidence.normalized_result, + ).not.toHaveProperty("coverage"); const legacyParent = createEvidence(legacyTarget, parent.provider, { operation: "inspect_artifact", parameters: {}, diff --git a/tests/boundary/mcp/artifactMcp.test.ts b/tests/boundary/mcp/artifactMcp.test.ts index da8a586d2..96ba07283 100644 --- a/tests/boundary/mcp/artifactMcp.test.ts +++ b/tests/boundary/mcp/artifactMcp.test.ts @@ -1,5 +1,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; import { ok as resultOk } from "../../../src/domain/result.js"; import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -98,20 +100,6 @@ it.each([ 20_000, ); -const artifactInventoryResultSchema = z.object({ - occurrences: z.array( - z.object({ logical_path: z.string(), hash_status: z.string() }), - ), - integrity_contradictions: z.array( - z.object({ - logical_path: z.string(), - declared_sha256: z.string(), - observed_sha256: z.string(), - trust: z.literal("observed-untrusted"), - }), - ), -}); - it("extracts an active archive through MCP when requested", async () => { const root = await createTestTempDirectory("rea-artifact-extract-mcp-"); const archive = join(root, "fixture.zip"); @@ -318,6 +306,30 @@ it("records an explicitly continued mismatch, preserves verified siblings, and n }), ]), }); + const { coverage: _coverage, ...legacyInventory } = inventory; + const historical = createEvidence( + { path: archive, sha256: inventory.manifest.root_sha256, format: "asar" }, + compact.provider, + { + operation: "inventory_artifact", + parameters: {}, + result: jsonValueSchema.parse(legacyInventory), + }, + ); + const retainedComparison = await client.callTool({ + name: "compare_artifacts", + arguments: { left: evidence, right: historical }, + }); + expect( + retainedComparison.isError, + JSON.stringify(retainedComparison), + ).not.toBe(true); + expect( + toolEvidence(retainedComparison.structuredContent).normalized_result, + ).toMatchObject({ + status: "contradiction", + summary: { unchanged: 2, contradiction: 2, unknown: 0 }, + }); } finally { await Promise.allSettled([client.close(), server.close(), session.close()]); } diff --git a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts index 5d6af987d..72d11abec 100644 --- a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts +++ b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts @@ -17,6 +17,7 @@ import { execFileOutput } from "../../../../src/process/ExecFileOutput.js"; import { ArtifactReaderFailure, type ArtifactEntry, + type ArtifactTraversalOmission, } from "../../../../src/artifacts/ArtifactReader.js"; import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourceScope.js"; import { DirectoryArtifactReader } from "../../../../src/artifacts/DirectoryArtifactReader.js"; @@ -473,10 +474,10 @@ describe("native DMG traversal shutdown", () => { .mockImplementation(async function* ( this: DirectoryArtifactReader, signal?: AbortSignal, - descendInto?: (path: string) => boolean, + selection?: Parameters[1], ) { const iterator = originalEntries - .call(this, signal, descendInto) + .call(this, signal, selection) [Symbol.asyncIterator](); try { for (;;) { @@ -606,9 +607,20 @@ describe("native DMG artifact reader", () => { undefined, host, ); + if (mountPoint === undefined) throw new Error("expected mounted directory"); + if (process.platform !== "win32") + await execFileOutput("mkfifo", [join(mountPoint, "special.pipe")]); const entries = []; - for await (const entry of reader.entries()) entries.push(entry.path); + const omissions: ArtifactTraversalOmission[] = []; + for await (const entry of reader.entries(undefined, { + onOmission: (omission) => omissions.push(omission), + })) + entries.push(entry.path); expect(entries).toContain("image.dmg/Fixture/hello.txt"); + if (process.platform !== "win32") + expect(omissions).toEqual([ + { path: "image.dmg/Fixture/special.pipe", kind: "fifo" }, + ]); const provenance = reader.provenance(); const first = provenance[0]; if (first === undefined) throw new Error("expected attachment provenance"); From 26525706fcd102355d1cd765138398067f32527f Mon Sep 17 00:00:00 2001 From: Nelglor Date: Tue, 13 Oct 2026 15:01:23 -0400 Subject: [PATCH 12/15] feat(managed): decode ldstr user strings in member inspection (#2162) * feat(managed): decode ldstr user strings in member inspection inspect_managed_members reported every ldstr operand as a bare 0x70 token, so an analyst could see that a method loads a literal but not which one. Answering "where is this path or key used" required a second tool. Read the optional #US stream and add a top-level user_strings inventory: each distinct token loaded by an inspected method body, once, in token order, with its #US file offset and decoded UTF-16 value. Instruction anchors and normalized CIL digests keep their build-local tokens. A missing #US stream or malformed item is reported against its token as partial coverage. Evidence recorded before this field reads user_strings as null. Co-Authored-By: Claude Opus 5.5 * fix(managed): collect user strings only from ldstr A malformed ldtoken can carry a 0x70 token, which the operand decoder classifies as a string operand. Only ldstr loads a literal, so collect user strings by opcode. Co-Authored-By: Claude Opus 5.5 * fix(managed): admit user-string tokens and failures before collecting them Charge each distinct ldstr token to the decode budget before adding it to the collection set, and admit each #US failure before retaining it. A missing #US stream is now one stream-level issue rather than one per token. Document why the #US terminal flag is not validated: Mono and Roslyn assemblies disagree with ECMA-335 II.24.2.4 and the runtime ignores it. Co-Authored-By: Claude Opus 5.5 * docs(managed): describe missing #US reporting and the unvalidated flag byte Co-Authored-By: Claude Opus 5.5 * test(managed): cover #US items that run past the stream end An odd length whose payload crosses the end of #US, and a two-byte length prefix that starts on the last #US byte, are both rejected as "User string leaves #US" beside a readable literal. The test fails if that bounds check is removed. Co-Authored-By: Claude Opus 5.5 * fix(managed): report reserved #US length prefixes as string failures The compressed-length reader shared by #Blob and #US hard-coded invalid-blob and "Metadata blob" for a reserved prefix, so a malformed user string was published as a blob failure. The caller now names the item it reads: #Blob keeps its code and message, and #US reports invalid-string. Co-Authored-By: Claude Opus 5.5 * fix(managed): append member inspection issues without argument spreading inspectManagedMembersBytes gathered every producer's issues with issues.push(...list), passing one argument per issue. A large malformed input, such as hundreds of thousands of distinct bad ldstr operands or MemberRef rows, overflows the stack with an untyped RangeError instead of returning partial coverage. All six sites now use appendManagedParseIssues, which appends one issue at a time. Co-Authored-By: Claude Opus 5.5 * fix(managed): classify truncated #US and #Blob length prefixes by item A two- or four-byte compressed-length prefix cut off by the end of the artifact went through the generic range check and was reported as invalid-row. The prefix reader now uses the item's own code for both reserved and truncated prefixes: invalid-string for #US, invalid-blob for #Blob. The unused generic range helper is removed. Co-Authored-By: Claude Opus 5.5 * fix(managed): retain user-string terminal flag conformance issues * test(managed): write ECMA-335 #US terminal flags and cover flag issues d5b7b0704 reports a #US terminal flag that differs from ECMA-335 II.24.2.4. The fixture heap always wrote 0, so its non-Latin-1 literals now read as nonconforming. The fixture writes the spec flag, and a new test flips one flag each way ("ok" set to 1, "it's" set to 0) and checks that both literals remain beside one issue each. Co-Authored-By: Claude Opus 5.5 * fix(managed): classify ldstr operands before collapsing a missing #US With no #US stream, every collected ldstr operand was counted as a #US reference and reported only as missing-stream, while the same non-0x70 operand got invalid-heap-index when the stream existed. parseUserStrings now owns which operands are user-string tokens: any other operand is invalid-heap-index either way, and only 0x70 tokens count toward the missing-stream issue. The heap reader drops its duplicate kind check. Co-Authored-By: Claude Opus 5.5 * fix(managed): state user-string limits in member comparisons compare_managed_members ignored user_strings, so a method whose ldstr literal changed at the same token, or legacy Evidence that never examined literals, could read as unchanged with no caveat. The comparison now adds an unknown limitation when either side predates user strings and a not-compared limitation when literals were observed. ExportedType and user-string inventories share one uncomparedInventory rule. Co-Authored-By: Claude Opus 5.5 * docs(managed): advertise ldstr literals in member inspection help The MCP tool description, the type_names input, and the CLI command and --type help now say that inspect_managed_members returns the #US string literals that ldstr loads, and that a type selection limits them to the literals the selected types' method bodies load. Co-Authored-By: Claude Opus 5.5 * fix(managed): read the empty #US entry at offset 0 ECMA-335 II.24.2.4 makes the first #US entry the empty blob: a zero length with no terminal flag byte. ldstr 0x70000000 loads it, but the odd-length check rejected it as invalid-string, dropping the literal and making coverage partial. Offset 0 with length 0 now reads as "". Co-Authored-By: Claude Opus 5.5 * fix(managed): reserve #US offset 0 and admit literals before scanning Offset 0 is the reserved empty #US entry, so a nonempty item there is now invalid-string instead of a published literal. The literal's projection is reserved before the terminal-flag scan, so an inadmissible item fails at the budget without reading every code unit. The scan's loop index no longer shadows the token's heap index. Co-Authored-By: Claude Opus 5.5 * fix(managed): report #US length prefixes wider than ECMA-335 assigns compressedLength accepted an overlong encoding, such as 0x80 0x05 for a 5-byte item, without comment. It now reports whether the width is the one ECMA-335 II.23.2 assigns, and the #US reader reports a non-minimal prefix as an invalid-string conformance issue beside the readable literal, like the terminal flag. readMetadataUserString returns every issue it observes. #Blob behavior is unchanged. Co-Authored-By: Claude Opus 5.5 * fix(managed): check the #US length width before the empty entry The offset-0 empty entry returned "" before the length-width check, so an overlong empty entry such as 0x80 0x00 read as complete. The width issue is now collected first and returned with the empty literal. Co-Authored-By: Claude Opus 5.5 * fix(managed): admit #US width issues only for returned literals The overlong-prefix issue was reserved before the fatal shape checks, so an item that then failed, such as 0x80 0x00 away from offset 0, leaked 512 bytes of budget per token. The width issue is now built and admitted only on paths that return the literal. Co-Authored-By: Claude Opus 5.5 * fix(managed): keep #US issues out of member metadata status User-string issues, including terminal-flag and length-prefix conformance differences, marked metadata.status partial, so compare_managed_members turned one-sided methods and fields into unknown even though every member row was examined. #US issues now make coverage partial while metadata.status describes the member inventory. Co-Authored-By: Claude Opus 5.5 * fix(managed): give user strings their own coverage #US issues entered the member coverage, so every consumer that reads it as member coverage (application graph member facts, comparison status) reported examined members as partial over a literal conformance difference. user_strings is now { state, literals, issues }: #US issues live there and never change the member coverage or metadata.status. parseUserStrings returns that inventory directly. compare_managed_members inspects with user strings skipped, since it never reads literals, so a large literal cannot exhaust the decode budget of a comparison. Its limitations state once that literals are not compared, and the ExportedType limitations return to their main form. Co-Authored-By: Claude Opus 5.5 * fix(managed): mark user strings partial when a body is not fully decoded With user strings owning their coverage, an inspected method body that was partial, malformed, or truncated no longer reached their state, so a literal after the decode stopped was missing from a "complete" inventory. parseUserStrings now marks the inventory partial unless every inspected body was fully decoded, keeping the literals it did observe. Co-Authored-By: Claude Opus 5.5 * fix(managed): mark user strings partial when a method row is omitted A MethodDef row that failed before its body was read was recorded as an issue and omitted, so its ldstr instructions were never scanned, yet the user-string inventory could still read as complete. parseUserStrings now takes the parseMethods result and stays partial whenever a method row failed, as well as when an inspected body was not fully decoded. Co-Authored-By: Claude Opus 5.5 * fix(managed): count only method rows that may hide a selected body Any parseMethods issue marked user strings partial, including an unreadable name on a method outside the type selection, so a fully scanned selection read as incomplete. parseMethods now reads each row's owner first and reports unscannedRows: rows that produced no method while their owner is selected or unknown. parseUserStrings uses that count. Co-Authored-By: Claude Opus 5.5 * fix(managed): treat unreadably named types as possibly selected Under a type_names selection, a type whose name could not be read may be a requested type, but its methods were skipped without counting toward unscannedRows, so an unexamined body left user strings complete. A method whose owner's name is unreadable now counts as possibly selected. Co-Authored-By: Claude Opus 5.5 * fix(managed): judge user-string scans by CIL decoding, not body status body.status is malformed for an invalid exception section even when every instruction decoded, so user strings read as partial although every ldstr was examined. The body reader now reports whether its CIL was fully scanned, independent of exception-region validity, and parseMethods folds unscanned bodies and unproduced rows into one unscannedMethods count that parseUserStrings uses instead of reinterpreting body.status. Co-Authored-By: Claude Opus 5.5 * fix(managed): stop deduplicating user-string issues The inventory passed its issues through uniqueManagedParseIssues, which serializes every issue into an unbudgeted map: about 42 MB of heap and 113 ms for 125,000 issues on Node 24. The issues are unique by construction (one failure or per-offset conformance issues per distinct token, one missing-stream issue), so the inventory now returns them, and its literals, without copies. Co-Authored-By: Claude Opus 5.5 * fix(managed): preserve selected string coverage unknowns --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/managed-code-analysis.md | 30 +- .../managed/ManagedMemberComparisonService.ts | 6 +- src/cli/managedCommands.ts | 4 +- src/contracts/managed/managedToolContracts.ts | 4 +- .../managed/managedWorkflowExamples.ts | 1 + src/domain/managed/managedArtifact.ts | 21 + .../managedMemberComparisonCoverage.ts | 3 + src/dotnet/ManagedMemberInspector.test.ts | 457 +++++++++++++++++- src/dotnet/ManagedMemberInspector.ts | 37 +- src/dotnet/ManagedMemberInspectorCore.ts | 3 + src/dotnet/ManagedMemberRows.ts | 44 +- src/dotnet/ManagedMetadataHeaps.test.ts | 28 +- src/dotnet/ManagedMetadataHeaps.ts | 192 ++++++-- src/dotnet/ManagedMetadataLayout.ts | 3 + src/dotnet/ManagedMethodBodyReader.ts | 46 +- src/dotnet/ManagedPe.fixture.ts | 140 ++++-- src/dotnet/ManagedReaderFailure.ts | 11 + src/dotnet/ManagedUserStrings.ts | 120 +++++ ...anagedMemberComparisonFromMetadata.test.ts | 24 + 19 files changed, 1054 insertions(+), 120 deletions(-) create mode 100644 src/dotnet/ManagedUserStrings.ts diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index 1d1a579e6..167d947c8 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -139,14 +139,40 @@ The result preserves the requested names in `selection.type_names` and reports unmatched names separately. If metadata names are unreadable, limitations retain that uncertainty. With no selection, `selection` is null and whole-assembly inspection keeps its existing resource policy. Selection restricts types, -fields, methods, call edges and field accesses; member references remain an -assembly-wide inventory, so output size is not guaranteed to be small. +fields, methods, call edges, field accesses and user strings; member references +remain an assembly-wide inventory, so output size is not guaranteed to be small. Coverage describes the selected members. Comparisons leave members outside the opposite observation's selection unknown, and application graphs report their selected scope as partial. Neither makes an absence claim about the whole assembly. Older member Evidence without `selection` remains readable. +### User strings + +`ldstr` anchors keep their build-local `0x70` tokens. `user_strings.literals` +lists each distinct token loaded by an inspected method body once, in token +order, with its `#US` file offset and decoded value. Values keep their stored +UTF-16 code units, including unpaired surrogates. Token `0x70000000` names the +heap's empty entry at offset 0 and reads as `""`; any other item there is +malformed. The terminal flag is checked against ECMA-335 II.24.2.4, and the +length prefix against the width II.23.2 assigns. A difference is reported with +its byte offset as a user-string issue while preserving the readable literal. +Toolchain conventions can differ; this conformance observation does not +establish runtime rejection. A missing `#US` stream is one stream-level issue; +a malformed item is reported against its token. User strings carry their own +coverage: any of these makes `user_strings.state` partial and lists the issue +in `user_strings.issues`, while the member `coverage` and `metadata.status` are +unaffected, and readable literals remain. The state is also partial when an +inspected method's CIL was not fully decoded (a malformed exception section +alone does not count) or a method that may belong to the inspected types was +not inspected (its row could not be read, or its type's name could not be read +under a `type_names` selection), because `ldstr` instructions there were not +observed. Normalized CIL digests do not include decoded values. +`compare_managed_members` does not decode or compare literals: an unchanged +method does not establish unchanged strings, and the comparison says so in its +limitations. Older member Evidence without `user_strings` reads as `null`: not +examined. + ### Edge target declaring types `member_refs[].parent_name`, `call_edges[].target_declaring_type`, and diff --git a/src/application/managed/ManagedMemberComparisonService.ts b/src/application/managed/ManagedMemberComparisonService.ts index 252feb8ca..04fae187f 100644 --- a/src/application/managed/ManagedMemberComparisonService.ts +++ b/src/application/managed/ManagedMemberComparisonService.ts @@ -92,7 +92,11 @@ export const compareManagedMemberPaths = async ( throw new EvidenceIntegrityError( `Managed artifact digest changed after open: expected ${target.sha256}, observed ${snapshot.sha256} at ${target.path}`, ); - const inspection = inspectManagedMembersBytes(snapshot.bytes, target); + // Member comparison never reads literals, so it does not spend the + // decode budget on them. + const inspection = inspectManagedMembersBytes(snapshot.bytes, target, { + userStrings: false, + }); const evidence = createEvidence(target, MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_members", parameters: {}, diff --git a/src/cli/managedCommands.ts b/src/cli/managedCommands.ts index eee4eae5a..782549720 100644 --- a/src/cli/managedCommands.ts +++ b/src/cli/managedCommands.ts @@ -56,13 +56,13 @@ const registerInspectManagedMembers = ( ): void => { cli.command(CLI_COMMANDS.inspectManagedMembers, { description: - "Inspect PE/CLI metadata members, signatures, and CIL anchors without loading target code", + "Inspect PE/CLI metadata members, signatures, CIL anchors, and ldstr string literals without loading target code", args: z.object({ path: z.string().describe("Managed PE executable or assembly path"), }), options: z.object({ type: managedMembersInputSchema.shape.type_names.describe( - "Inspect only members declared by this exact type full name, such as System.Threading.Monitor; a nested type is named by its bare name, which selects every nested type of that name (repeatable)", + "Inspect only members declared by this exact type full name, and the string literals they load, such as System.Threading.Monitor; a nested type is named by its bare name, which selects every nested type of that name (repeatable)", ), }), run: ({ args, options }) => diff --git a/src/contracts/managed/managedToolContracts.ts b/src/contracts/managed/managedToolContracts.ts index 1e685a702..2c0088b9b 100644 --- a/src/contracts/managed/managedToolContracts.ts +++ b/src/contracts/managed/managedToolContracts.ts @@ -43,7 +43,7 @@ export const managedMembersInputSchema = managedTargetInputSchema.extend({ }) .meta({ uniqueItems: true }) .describe( - "Inspect only the fields, methods, call edges, and field accesses declared by these exact type full names; omit to inspect every type. Unmatched names are reported in selection.unmatched_type_names.", + "Inspect only the fields, methods, call edges, and field accesses declared by these exact type full names, and the user strings their method bodies load; omit to inspect every type. Unmatched names are reported in selection.unmatched_type_names.", ) .optional(), }); @@ -87,7 +87,7 @@ export const MANAGED_TOOL_CONTRACTS = [ name: "inspect_managed_members", ...toolContractMetadata("inspect_managed_members"), description: - "Inspect PE/CLI metadata members, signatures, raw CIL hashes, decoded-instruction-tuple hashes, separately reported exception regions, call edges, and field-access anchors without loading or executing target code. Select type_names to decode only the members of those exact declaring types, which keeps large assemblies such as a base class library within the representation budget. Metadata tokens are reported as build-local coordinates bound to the artifact SHA-256 and MVID; the tuple hash does not resolve them or fully commit control flow.", + "Inspect PE/CLI metadata members, signatures, raw CIL hashes, decoded-instruction-tuple hashes, separately reported exception regions, call edges, field-access anchors, and the #US string literals that ldstr loads, without loading or executing target code. Select type_names to decode only the members of those exact declaring types and the literals their bodies load, which keeps large assemblies such as a base class library within the representation budget. Metadata tokens are reported as build-local coordinates bound to the artifact SHA-256 and MVID; the tuple hash does not resolve them or fully commit control flow.", kind: "managed-provider", inputSchema: managedMembersInputSchema, outputSchema: memberOutputSchema, diff --git a/src/contracts/managed/managedWorkflowExamples.ts b/src/contracts/managed/managedWorkflowExamples.ts index 85297d671..6e5d836ec 100644 --- a/src/contracts/managed/managedWorkflowExamples.ts +++ b/src/contracts/managed/managedWorkflowExamples.ts @@ -53,6 +53,7 @@ const emptyManagedMembers = ( member_refs: [], call_edges: [], field_accesses: [], + user_strings: { state: "complete", literals: [], issues: [] }, selection: null, coverage: { state: "complete", issues: [] }, limitations: [], diff --git a/src/domain/managed/managedArtifact.ts b/src/domain/managed/managedArtifact.ts index 94eff2398..f41173ba6 100644 --- a/src/domain/managed/managedArtifact.ts +++ b/src/domain/managed/managedArtifact.ts @@ -219,6 +219,16 @@ const managedMemberReferenceSchema = z.object({ signature: managedSignatureSchema, }); +const managedUserStringSchema = z.object({ + token: tokenSchema, + offset: offsetSchema.describe("File offset of the #US item"), + value: z + .string() + .describe( + "Decoded UTF-16 code units, excluding the terminal flag byte; unpaired surrogates are kept", + ), +}); + const managedModuleReferenceSchema = z.object({ token: tokenSchema, row_offset: offsetSchema, @@ -457,6 +467,17 @@ export const managedMemberInspectionSchema = z.object({ member_refs: z.array(managedMemberReferenceSchema), call_edges: z.array(managedCallEdgeSchema), field_accesses: z.array(managedFieldAccessSchema), + // Distinct #US literals loaded by ldstr in the inspected method bodies, in + // token order, with their own coverage: #US issues never enter the member + // coverage. null means they were not examined, not that there were none. + user_strings: z + .object({ + state: z.enum(["complete", "partial"]), + literals: z.array(managedUserStringSchema), + issues: z.array(managedParseIssueSchema), + }) + .nullable() + .default(null), /** Caller-selected declaring types; null when every type was inspected. */ selection: z .object({ diff --git a/src/domain/managed/managedMemberComparisonCoverage.ts b/src/domain/managed/managedMemberComparisonCoverage.ts index 7f28cc88e..85da3b4eb 100644 --- a/src/domain/managed/managedMemberComparisonCoverage.ts +++ b/src/domain/managed/managedMemberComparisonCoverage.ts @@ -131,5 +131,8 @@ export const comparisonLimitations = ( limitations.push( "ExportedType rows, including type forwarders, are observed but not compared; an unchanged comparison does not establish an unchanged exported or forwarded type set.", ); + limitations.push( + "ldstr literals are not compared; normalized CIL keeps #US tokens rather than decoded text, so an unchanged method does not establish unchanged string literals.", + ); return limitations; }; diff --git a/src/dotnet/ManagedMemberInspector.test.ts b/src/dotnet/ManagedMemberInspector.test.ts index 9ada1bf55..3aee06e73 100644 --- a/src/dotnet/ManagedMemberInspector.test.ts +++ b/src/dotnet/ManagedMemberInspector.test.ts @@ -1,8 +1,18 @@ import { describe, expect, it } from "vitest"; import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js"; -import { ManagedDecodeBudget } from "./ManagedDecodeBudget.js"; +import { + ManagedDecodeBudget, + managedDecodeBudget, +} from "./ManagedDecodeBudget.js"; +import { + metadataRowOffset, + readManagedMetadataLayout, + type ManagedMetadataLayout, +} from "./ManagedMetadataLayout.js"; +import { readManagedPeLayout } from "./ManagedPeReader.js"; import { edges } from "./ManagedMemberRows.js"; +import { parseUserStrings } from "./ManagedUserStrings.js"; import { signature, type ManagedMethod } from "./ManagedMemberInspectorCore.js"; import { ManagedReaderFailure } from "./ManagedReaderFailure.js"; import { managedMemberInspectionSchema } from "../domain/managed/managedArtifact.js"; @@ -11,9 +21,23 @@ import { comparisonLimitations } from "../domain/managed/managedMemberComparison import { buildManagedPeFixture, buildNativePeFixture, + fixtureUserStringHeap, managedPeFixtureTarget, } from "./ManagedPe.fixture.js"; +/** A tiny-header body that loads each `ldstr` token, discards it, and returns. */ +const ldstrBody = (tokens: readonly string[]): Buffer => { + const il = Buffer.concat([ + ...tokens.map((token) => { + const instruction = Buffer.from([0x72, 0, 0, 0, 0, 0x26]); + instruction.writeUInt32LE(Number.parseInt(token, 16), 1); + return instruction; + }), + Buffer.from([0x2a]), + ]); + return Buffer.concat([Buffer.from([(il.length << 2) | 2]), il]); +}; + describe("managed member inspection", () => { it("keeps metadata handles distinct from calls and field accesses", () => { const il = Buffer.from([ @@ -605,6 +629,433 @@ describe("managed exported types", () => { }); }); +describe("managed user strings", () => { + const inspect = (options: Parameters[0]) => { + const bytes = buildManagedPeFixture(options); + return inspectManagedMembersBytes(bytes, managedPeFixtureTarget(bytes)); + }; + + it("decodes each loaded literal once, in token order, without changing CIL tokens", () => { + const { heap, tokens } = fixtureUserStringHeap([ + "quicksave.peak", + "unused", + "Ünïcødé 𝄞", + "\ud800", + ]); + const [path, , unicode, lone] = tokens as [string, string, string, string]; + // 0x70000000 names the heap's empty entry at offset 0. + const empty = "0x70000000"; + const ilBody = ldstrBody([unicode, path, lone, path, empty]); + const result = inspect({ ilBody, userStringHeap: heap }); + + expect(result.user_strings?.literals).toEqual([ + { token: empty, offset: expect.any(Number), value: "" }, + { token: path, offset: expect.any(Number), value: "quicksave.peak" }, + { token: unicode, offset: expect.any(Number), value: "Ünïcødé 𝄞" }, + { token: lone, offset: expect.any(Number), value: "\ud800" }, + ]); + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + expect(result.user_strings).toMatchObject({ + state: "complete", + issues: [], + }); + const body = result.methods[0]?.body; + expect(body?.anchors.map(({ operand }) => operand)).toEqual([ + unicode, + path, + lone, + path, + empty, + ]); + expect(body?.normalized_il_sha256).toBe( + inspect({ ilBody }).methods[0]?.body.normalized_il_sha256, + ); + expect(comparisonLimitations(result, result)).toContain( + "ldstr literals are not compared; normalized CIL keeps #US tokens rather than decoded text, so an unchanged method does not establish unchanged string literals.", + ); + }); + + it("ignores a 0x70 token carried by an instruction other than ldstr", () => { + const { heap, tokens } = fixtureUserStringHeap(["not loaded"]); + const token = Number.parseInt(tokens[0] ?? "", 16); + const ldtoken = Buffer.from([0xd0, 0, 0, 0, 0, 0x26, 0x2a]); + ldtoken.writeUInt32LE(token, 1); + const result = inspect({ + ilBody: Buffer.concat([ + Buffer.from([(ldtoken.length << 2) | 2]), + ldtoken, + ]), + userStringHeap: heap, + }); + + expect(result.methods[0]?.body.anchors[0]).toMatchObject({ + opcode: "ldtoken", + operand_kind: "string", + }); + expect(result.user_strings?.literals).toEqual([]); + }); + + it("admits ldstr tokens before collecting them", () => { + const layout = { userStrings: null } as unknown as ManagedMetadataLayout; + // Leave 1 KiB of the 64 MiB budget: too little for 100 distinct tokens. + managedDecodeBudget(layout).reserve(64 * 1024 * 1024 - 1024, null); + const method = { + body: { + anchors: Array.from({ length: 100 }, (_, index) => ({ + il_offset: index * 5, + opcode: "ldstr", + operand_kind: "string", + operand: `0x70${(index + 1).toString(16).padStart(6, "0")}`, + })), + }, + } as unknown as ManagedMethod; + + expect(() => + parseUserStrings(Buffer.alloc(0), layout, { + methods: [method], + unscannedMethods: 0, + }), + ).toThrow(ManagedReaderFailure); + }); + + it("retains readable literals beside malformed #US items", () => { + const { heap, tokens } = fixtureUserStringHeap(["ok", "bad", "worse"]); + const [ok, bad, worse] = tokens as [string, string, string]; + // "bad" encodes 7 bytes; an even length cannot end in the flag byte. + heap.writeUInt8(6, Number.parseInt(bad.slice(4), 16)); + // 0xe0 is a reserved compressed-length prefix. + heap.writeUInt8(0xe0, Number.parseInt(worse.slice(4), 16)); + const result = inspect({ + ilBody: ldstrBody([ok, bad, worse, "0x70ffffff", "0x06000001"]), + userStringHeap: heap, + }); + + expect(result.user_strings?.literals).toEqual([ + { token: ok, offset: expect.any(Number), value: "ok" }, + ]); + expect(result.user_strings?.state).toBe("partial"); + expect( + result.user_strings?.issues.map(({ code, scope }) => [code, scope]), + ).toEqual([ + ["invalid-heap-index", "metadata.#US:0x06000001"], + ["invalid-string", `metadata.#US:${bad}`], + ["invalid-string", `metadata.#US:${worse}`], + ["invalid-heap-index", "metadata.#US:0x70ffffff"], + ]); + }); + + it("rejects #US items that run past the end of the stream", () => { + const { heap: readable, tokens } = fixtureUserStringHeap(["ok"]); + // An odd length claiming 127 bytes with two left, then a two-byte length + // prefix whose second byte lies past the stream end. + const crossing = Buffer.from([0x7f, 0x41, 0x00]); + const heap = Buffer.concat([readable, crossing, Buffer.from([0x80])]); + const token = (offset: number) => + `0x70${offset.toString(16).padStart(6, "0")}`; + const crossingToken = token(readable.length); + const truncatedToken = token(readable.length + crossing.length); + const result = inspect({ + ilBody: ldstrBody([tokens[0] ?? "", crossingToken, truncatedToken]), + userStringHeap: heap, + }); + + expect(result.user_strings?.literals).toEqual([ + { token: tokens[0], offset: expect.any(Number), value: "ok" }, + ]); + expect(result.user_strings).toMatchObject({ + state: "partial", + issues: [crossingToken, truncatedToken].map((item) => ({ + code: "invalid-string", + scope: `metadata.#US:${item}`, + detail: "User string leaves #US", + })), + }); + }); + + it("leaves user strings unexamined when CLI metadata is unavailable or predates them", () => { + const native = buildNativePeFixture(); + expect( + inspectManagedMembersBytes(native, managedPeFixtureTarget(native)) + .user_strings, + ).toBeNull(); + + const { user_strings: _omitted, ...legacy } = inspect({}); + const parsed = managedMemberInspectionSchema.parse(legacy); + expect(parsed.user_strings).toBeNull(); + expect(comparisonLimitations(parsed, inspect({}))).toContain( + "ldstr literals are not compared; normalized CIL keeps #US tokens rather than decoded text, so an unchanged method does not establish unchanged string literals.", + ); + }); +}); + +describe("managed user string issues", () => { + const inspect = (options: Parameters[0]) => { + const bytes = buildManagedPeFixture(options); + return inspectManagedMembersBytes(bytes, managedPeFixtureTarget(bytes)); + }; + + it("reports a missing #US stream once and keeps the method", () => { + const result = inspect({ + // A MethodDef operand is not a #US reference, so it is reported alone. + ilBody: ldstrBody([ + "0x70000001", + "0x70000005", + "0x70000001", + "0x06000001", + ]), + }); + + expect(result.methods).toHaveLength(1); + expect(result.coverage.state).toBe("complete"); + expect(result.user_strings).toEqual({ + state: "partial", + literals: [], + issues: [ + { + code: "invalid-heap-index", + scope: "metadata.#US:0x06000001", + offset: null, + detail: "ldstr operand does not name an item inside #US", + }, + { + code: "missing-stream", + scope: "metadata.#US", + offset: null, + detail: + "2 distinct ldstr operand(s) reference #US, but the metadata has no #US stream", + }, + ], + }); + }); + + it("rejects a nonempty item at the reserved #US offset 0", () => { + // "A" with its flag, where only the empty entry may sit. + const result = inspect({ + ilBody: ldstrBody(["0x70000000"]), + userStringHeap: Buffer.from([3, 0x41, 0, 0]), + }); + + expect(result.user_strings?.literals).toEqual([]); + expect(result.user_strings).toMatchObject({ + state: "partial", + issues: [{ code: "invalid-string", scope: "metadata.#US:0x70000000" }], + }); + }); + + it.each([ + // "ok" is 5 bytes, which ECMA-335 encodes in one prefix byte, not 0x80 0x05. + ["0x70000001", [0, 0x80, 0x05, 0x6f, 0, 0x6b, 0, 0], "ok"], + // The empty entry at offset 0 is the single byte 0x00, not 0x80 0x00. + ["0x70000000", [0x80, 0x00], ""], + ])( + "keeps %s when its length prefix is wider than ECMA-335 assigns", + (token, heap, value) => { + const result = inspect({ + ilBody: ldstrBody([token]), + userStringHeap: Buffer.from(heap), + }); + + expect(result.user_strings?.literals).toEqual([ + { token, offset: expect.any(Number), value }, + ]); + expect(result.user_strings).toMatchObject({ + state: "partial", + issues: [ + { + code: "invalid-string", + scope: `metadata.#US:${token}`, + detail: expect.stringContaining("prefix wider than ECMA-335"), + }, + ], + }); + }, + ); + + it("admits no width issue for an item that then fails", () => { + // 64 overlong zero lengths away from offset 0: each item is fatal. + const heap = Buffer.concat([ + Buffer.from([0]), + ...Array.from({ length: 64 }, () => Buffer.from([0x80, 0x00])), + ]); + const layout = { + userStrings: { name: "#US", offset: 0, size: heap.length }, + } as unknown as ManagedMetadataLayout; + // Leave 24 KiB: enough for the failures, not for 512 bytes leaked each. + managedDecodeBudget(layout).reserve(64 * 1024 * 1024 - 24 * 1024, null); + const method = { + body: { + anchors: Array.from({ length: 64 }, (_, index) => ({ + il_offset: index * 5, + opcode: "ldstr", + operand_kind: "string", + operand: `0x70${(1 + index * 2).toString(16).padStart(6, "0")}`, + })), + }, + } as unknown as ManagedMethod; + + expect( + parseUserStrings(heap, layout, { methods: [method], unscannedMethods: 0 }) + .issues, + ).toHaveLength(64); + }); + + it("keeps literals whose terminal flag differs from ECMA-335 and reports each flag", () => { + const { heap, tokens } = fixtureUserStringHeap(["ok", "it's"]); + const [ok, quoted] = tokens as [string, string]; + // A short item is a one-byte length, the UTF-16 text, then the flag byte. + const flagAt = (token: string, value: string) => + Number.parseInt(token.slice(4), 16) + 1 + value.length * 2; + heap.writeUInt8(1, flagAt(ok, "ok")); + heap.writeUInt8(0, flagAt(quoted, "it's")); + const result = inspect({ + ilBody: ldstrBody([ok, quoted]), + userStringHeap: heap, + }); + + expect(result.coverage.state).toBe("complete"); + expect(result.user_strings?.literals.map(({ value }) => value)).toEqual([ + "ok", + "it's", + ]); + expect(result.user_strings).toMatchObject({ + state: "partial", + issues: [ok, quoted].map((token) => ({ + code: "invalid-string", + scope: `metadata.#US:${token}`, + detail: expect.stringContaining("terminal flag"), + })), + }); + }); +}); + +describe("managed user string scan completeness", () => { + const inspect = (options: Parameters[0]) => { + const bytes = buildManagedPeFixture(options); + return inspectManagedMembersBytes(bytes, managedPeFixtureTarget(bytes)); + }; + + it("marks user strings partial when a method body is not fully decoded", () => { + const { heap, tokens } = fixtureUserStringHeap(["ok", "later"]); + const [ok, later] = tokens as [string, string]; + // ldstr "ok", then an undefined opcode before ldstr "later". + const il = Buffer.from([ + 0x72, 0, 0, 0, 0, 0x26, 0xfe, 0xff, 0x72, 0, 0, 0, 0, 0x26, 0x2a, + ]); + il.writeUInt32LE(Number.parseInt(ok, 16), 1); + il.writeUInt32LE(Number.parseInt(later, 16), 9); + const result = inspect({ + ilBody: Buffer.concat([Buffer.from([(il.length << 2) | 2]), il]), + userStringHeap: heap, + }); + + expect(result.methods[0]?.body.status).toBe("malformed"); + expect(result.user_strings).toEqual({ + state: "partial", + literals: [{ token: ok, offset: expect.any(Number), value: "ok" }], + issues: [], + }); + }); + + it.each([ + ["outside", 1, "complete"], + ["inside", 2, "partial"], + ])( + "judges an unreadable method row %s the selection by its owner", + (_where, badRow, state) => { + const bytes = buildManagedPeFixture({ + additionalEmptyTypes: 1, + methodRowCount: 2, + }); + const cli = readManagedPeLayout(bytes); + if (cli.cli === null) throw new Error("fixture has no CLI header"); + const { rva, size } = cli.cli.metadata; + const layout = readManagedMetadataLayout( + bytes, + cli.rvaToOffset(rva, size, "cli.metadata"), + size, + ); + const write = (offset: number, width: number, value: number) => + width === 2 + ? bytes.writeUInt16LE(value, offset) + : bytes.writeUInt32LE(value, offset); + // Fixture.Program's methods now start at row 2, so Fixture.Empty0 owns row 1. + write( + metadataRowOffset(layout, 2, 2) + + 4 + + 2 * layout.stringIndexSize + + layout.codedIndexSize("TypeDefOrRef") + + layout.tableIndexSize(4), + layout.tableIndexSize(6), + 2, + ); + // An out-of-range name index makes that method row unreadable. + write( + metadataRowOffset(layout, 6, badRow) + 8, + layout.stringIndexSize, + 0xffff, + ); + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + { typeNames: ["Fixture.Program"] }, + ); + + expect(result.user_strings?.state).toBe(state); + }, + ); + + it("keeps user strings complete when only the exception section is malformed", () => { + const { heap, tokens } = fixtureUserStringHeap(["ok"]); + // ldstr "ok"; pop; ret all decode before an exception section that + // holds no whole clause. + const il = Buffer.from([0x72, 0, 0, 0, 0, 0x26, 0x2a]); + il.writeUInt32LE(Number.parseInt(tokens[0] ?? "", 16), 1); + const header = Buffer.alloc(12); + header.writeUInt16LE(0x300b, 0); + header.writeUInt16LE(8, 2); + header.writeUInt32LE(il.length, 4); + const section = Buffer.alloc(17); + section[0] = 0x01; + section[1] = section.length; + const result = inspect({ + ilBody: Buffer.concat([header, il, Buffer.alloc(1), section]), + userStringHeap: heap, + }); + + expect(result.methods[0]?.body.status).toBe("malformed"); + expect(result.user_strings).toMatchObject({ + state: "complete", + literals: [{ value: "ok" }], + }); + }); + + it("marks user strings partial when selected type identity is unreadable", () => { + const layout = { + userStrings: { name: "#US", offset: 0, size: 1 }, + } as unknown as ManagedMetadataLayout; + expect( + parseUserStrings(Buffer.from([0]), layout, { + methods: [], + unscannedMethods: 0, + unreadableSelectedTypes: 1, + }), + ).toEqual({ state: "partial", literals: [], issues: [] }); + }); + + it("marks user strings partial when an inspected method row was not read", () => { + const layout = { + userStrings: { name: "#US", offset: 0, size: 1 }, + } as unknown as ManagedMetadataLayout; + // The omitted row's body, and any ldstr in it, was never scanned. + expect( + parseUserStrings(Buffer.from([0]), layout, { + methods: [], + unscannedMethods: 1, + }), + ).toEqual({ state: "partial", literals: [], issues: [] }); + }); +}); + describe("managed member decoding budget", () => { it("admits a wide-instruction method whose actual records fit the budget", () => { // Five-byte branches all target the next instruction. A final ret leaves @@ -1243,7 +1694,7 @@ describe("managed member type selection", () => { expect(result.field_accesses).toEqual(everything.field_accesses); expect(result.coverage).toEqual(everything.coverage); expect(result.limitations).toContain( - "Types, fields, methods, call edges, and field accesses cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", + "Types, fields, methods, call edges, field accesses, and user strings cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", ); }); @@ -1260,6 +1711,8 @@ describe("managed member type selection", () => { expect(result.selection?.unmatched_type_names).toEqual(["Fixture.Program"]); expect(result.coverage.state).toBe("partial"); + // Its methods were never inspected, so neither were their literals. + expect(result.user_strings?.state).toBe("partial"); expect(result.limitations).toContain( "1 TypeDef name(s) could not be read; a name in selection.unmatched_type_names may belong to one of them, so its absence is not observed.", ); diff --git a/src/dotnet/ManagedMemberInspector.ts b/src/dotnet/ManagedMemberInspector.ts index dba20ac57..1b018522a 100644 --- a/src/dotnet/ManagedMemberInspector.ts +++ b/src/dotnet/ManagedMemberInspector.ts @@ -19,6 +19,7 @@ import { } from "./ManagedMetadataInventory.js"; import { readManagedMetadataLayout } from "./ManagedMetadataLayout.js"; import { + appendManagedParseIssues, ManagedReaderFailure, uniqueManagedParseIssues, } from "./ManagedReaderFailure.js"; @@ -29,6 +30,7 @@ import { } from "./ManagedMemberInspectorCore.js"; import { edgeTargets } from "./ManagedMethodSpecs.js"; import { typeNameLookup } from "./ManagedTypeNames.js"; +import { parseUserStrings } from "./ManagedUserStrings.js"; import { edges, parseExportedTypes, @@ -70,6 +72,7 @@ const unavailable = ( }, types: [], exported_types: null, + user_strings: null, fields: [], methods: [], member_refs: [], @@ -149,11 +152,19 @@ const selectTypes = ( /** * Inspect metadata members and method bodies without loading target code. * Selected type names restrict full decoding to members of those exact types. + * A caller that never reads literals, such as member comparison, can skip + * user strings; they are then reported as not examined. */ export const inspectManagedMembersBytes = ( bytes: Buffer, target: BinaryTarget, - { typeNames }: { readonly typeNames?: readonly string[] } = {}, + { + typeNames, + userStrings: readUserStrings = true, + }: { + readonly typeNames?: readonly string[]; + readonly userStrings?: boolean; + } = {}, ): ManagedMemberInspection => { const pe = readManagedPeLayout(bytes); if (pe.cli === null) @@ -161,7 +172,7 @@ export const inspectManagedMembersBytes = ( const issues: ManagedParseIssue[] = []; try { const { layout, inventory } = readMemberInventory(bytes, pe, issues); - issues.push(...inventory.issues); + appendManagedParseIssues(issues, inventory.issues); const ranges = typeRanges(bytes, layout, issues); const { tokens: selected, @@ -169,16 +180,16 @@ export const inspectManagedMembersBytes = ( unreadableNames, } = selectTypes(ranges, typeNames); const parsedTypes = parseTypes(bytes, layout, ranges); - issues.push(...parsedTypes.issues); + appendManagedParseIssues(issues, parsedTypes.issues); const exportedTypes = parseExportedTypes(bytes, layout); - issues.push(...exportedTypes.issues); + appendManagedParseIssues(issues, exportedTypes.issues); const fields = parseFields(bytes, layout, ranges, selected); const memberRefs = parseMemberRefs( bytes, layout, typeNameLookup(bytes, layout, ranges, issues), ); - issues.push(...memberRefs.issues); + appendManagedParseIssues(issues, memberRefs.issues); const methods = parseMethods({ bytes, layout, @@ -186,13 +197,21 @@ export const inspectManagedMembersBytes = ( ranges, selected, }); - issues.push(...fields.issues, ...methods.issues); + appendManagedParseIssues(issues, fields.issues, methods.issues); + // #US literals sit beside the member inventory and carry their own + // coverage, so their issues never make member facts partial. + const userStrings = readUserStrings + ? parseUserStrings(bytes, layout, { + ...methods, + unreadableSelectedTypes: unreadableNames, + }) + : null; const targets = edgeTargets(bytes, layout, methods.methods, [ fields.core, memberRefs.core, methods.core, ]); - issues.push(...targets.issues); + appendManagedParseIssues(issues, targets.issues); const coverageIssues = uniqueManagedParseIssues(issues); const incompleteMembers = [...fields.fields, ...memberRefs.refs, ...methods.methods].some( @@ -238,6 +257,7 @@ export const inspectManagedMembersBytes = ( member_refs: memberRefs.refs, call_edges: related.callEdges, field_accesses: related.fieldAccesses, + user_strings: userStrings, selection, coverage: { state: @@ -250,11 +270,12 @@ export const inspectManagedMembersBytes = ( "Metadata tokens are build-local coordinates and are only meaningful with the reported artifact SHA-256 and MVID.", "CIL instruction anchors are decoded from file-backed method bodies only; no target assembly is loaded or executed.", "Exported types are reported from ExportedType rows as declared; forwarded types are not resolved in their implementing assemblies.", + "User strings are the #US literals loaded by ldstr in the inspected method bodies, with their own coverage; instruction anchors and normalized CIL digests keep their build-local tokens.", "Member signatures decode ECMA-335 primitive, class, valuetype, generic instantiation, generic variable, pointer, byref, single-dimension and general array, function pointer, custom modifier, and typed reference forms; class and valuetype operands keep their encoded TypeDefOrRefOrSpec values, and unsupported forms such as vararg sentinels retain raw signature hashes.", ...(selection === null ? [] : [ - "Types, fields, methods, call edges, and field accesses cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", + "Types, fields, methods, call edges, field accesses, and user strings cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", ]), ...(unreadableNames === 0 ? [] diff --git a/src/dotnet/ManagedMemberInspectorCore.ts b/src/dotnet/ManagedMemberInspectorCore.ts index 7776c0090..caac2e66e 100644 --- a/src/dotnet/ManagedMemberInspectorCore.ts +++ b/src/dotnet/ManagedMemberInspectorCore.ts @@ -35,6 +35,9 @@ export type ManagedMemberRef = ManagedMemberInspection["member_refs"][number]; export type ManagedCallEdge = ManagedMemberInspection["call_edges"][number]; export type ManagedFieldAccess = ManagedMemberInspection["field_accesses"][number]; +export type ManagedUserString = NonNullable< + ManagedMemberInspection["user_strings"] +>["literals"][number]; export type ManagedSignature = ManagedMethod["signature"]; export type ManagedMethodBody = ManagedMethod["body"]; export type ManagedInstructionAnchor = ManagedMethodBody["anchors"][number]; diff --git a/src/dotnet/ManagedMemberRows.ts b/src/dotnet/ManagedMemberRows.ts index 201ff1c45..700f723e6 100644 --- a/src/dotnet/ManagedMemberRows.ts +++ b/src/dotnet/ManagedMemberRows.ts @@ -269,16 +269,36 @@ export const parseMethods = ({ readonly methods: readonly ManagedMethod[]; readonly core: ReadonlyMap; readonly issues: readonly ManagedParseIssue[]; + /** + * Methods that may belong to a selected type but whose CIL was not fully + * scanned: rows that produced no method, or bodies whose decoding stopped. + */ + readonly unscannedMethods: number; } => { const methods: ManagedMethod[] = []; const core = new Map(); const issues: ManagedParseIssue[] = []; + let unscannedMethods = 0; const table = layout.table(6); const ownerRow = readManagedValue( () => memberDeclaringTypeRowLookup(bytes, layout, 6), issues, ); for (let row = 1; row <= (table?.rowCount ?? 0); row += 1) { + // The owner is read first: a row that produced no method hides a selected + // body only when its owner is selected, unknown, or unreadably named. + const owner = + ownerRow === undefined + ? undefined + : readManagedValue(() => ownerRow(row), issues); + const declared = owner === undefined ? undefined : ranges[owner - 1]; + const mayBeSelected = + selected === null || + owner === undefined || + isSelected(selected, declared) || + // A type whose name is unreadable may be one the caller requested. + declared?.fullName === null; + const produced = methods.length; readManagedValue(() => { const cursor = metadataRowCursor(bytes, layout, 6, row); const rva = cursor.readUInt32(); @@ -290,11 +310,6 @@ export const parseMethods = ({ cursor.readIndex(layout.stringIndexSize), layout.strings.size, ); - const owner = - ownerRow === undefined - ? 0 - : (readManagedValue(() => ownerRow(row), issues) ?? 0); - const declared = ranges[owner - 1]; const token = metadataToken(6, row); const member = { token, @@ -316,6 +331,14 @@ export const parseMethods = ({ ...member, genericArity: signatureGenericArity(decoded), }); + const read = methodBody( + bytes, + pe, + rva, + { implFlags, flags }, + managedDecodeBudget(layout), + ); + if (!read.scanned) unscannedMethods += 1; methods.push({ token, row_offset: cursor.start, @@ -326,17 +349,12 @@ export const parseMethods = ({ impl_flags: implFlags, flags, signature: decoded, - body: methodBody( - bytes, - pe, - rva, - { implFlags, flags }, - managedDecodeBudget(layout), - ), + body: read.body, }); }, issues); + if (mayBeSelected && methods.length === produced) unscannedMethods += 1; } - return { methods, core, issues }; + return { methods, core, issues, unscannedMethods }; }; /** diff --git a/src/dotnet/ManagedMetadataHeaps.test.ts b/src/dotnet/ManagedMetadataHeaps.test.ts index c94ae4055..ebc2a0d49 100644 --- a/src/dotnet/ManagedMetadataHeaps.test.ts +++ b/src/dotnet/ManagedMetadataHeaps.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vitest"; -import { readMetadataBlob } from "./ManagedMetadataHeaps.js"; +import { + readMetadataBlob, + readMetadataUserString, +} from "./ManagedMetadataHeaps.js"; import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; describe("managed metadata heaps", () => { @@ -16,6 +19,7 @@ describe("managed metadata heaps", () => { strings: { name: "#Strings", offset: 0, size: 0 }, guid: { name: "#GUID", offset: 0, size: 0 }, blob: { name: "#Blob", offset: 0, size: bytes.length }, + userStrings: null, tables: new Map(), rowCounts: [], stringIndexSize: 2, @@ -30,4 +34,26 @@ describe("managed metadata heaps", () => { itemLength, ); }); + + it.each([[[0x80]], [[0xc0, 0x00, 0x00]]])( + "reports a #US length prefix cut off by the end of the artifact as a string failure", + (prefix) => { + // #US ends at the end of the artifact, inside a multi-byte length prefix. + const bytes = Buffer.from([0x00, ...prefix]); + const layout = { + userStrings: { name: "#US", offset: 0, size: bytes.length }, + } as unknown as ManagedMetadataLayout; + + expect(() => + readMetadataUserString(bytes, layout, "0x70000001"), + ).toThrowError( + expect.objectContaining({ + issue: expect.objectContaining({ + code: "invalid-string", + scope: "metadata.#US:0x70000001", + }), + }), + ); + }, + ); }); diff --git a/src/dotnet/ManagedMetadataHeaps.ts b/src/dotnet/ManagedMetadataHeaps.ts index 51b85576c..0987680f0 100644 --- a/src/dotnet/ManagedMetadataHeaps.ts +++ b/src/dotnet/ManagedMetadataHeaps.ts @@ -1,3 +1,4 @@ +import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; import { createHash } from "node:crypto"; import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; @@ -8,21 +9,6 @@ import { } from "./ManagedMetadataLayout.js"; import { managedFailure } from "./ManagedReaderFailure.js"; -const requireRange = ( - bytes: Buffer, - offset: number, - length: number, - scope: string, -): void => { - if (offset < 0 || length < 0 || offset > bytes.length - length) - throw managedFailure( - "invalid-row", - scope, - `${scope} leaves the artifact byte range`, - offset >= 0 ? offset : null, - ); -}; - /** Cursor for checked fixed-width metadata row decoding. */ export class MetadataRowCursor { #offset: number; @@ -69,36 +55,54 @@ export class MetadataRowCursor { } } +// Issue code and label for the heap item a compressed length prefixes. +const PREFIXED_ITEMS = { + blob: { code: "invalid-blob", label: "Metadata blob" }, + userString: { code: "invalid-string", label: "User string" }, +} as const; + const compressedLength = ( bytes: Buffer, offset: number, scope: string, -): { readonly length: number; readonly prefix: number } => { - requireRange(bytes, offset, 1, scope); + item: keyof typeof PREFIXED_ITEMS, +): { + readonly length: number; + readonly prefix: number; + /** Whether ECMA-335 II.23.2 assigns this width to the value. */ + readonly minimal: boolean; +} => { + const { code, label } = PREFIXED_ITEMS[item]; + const requirePrefix = (size: number): void => { + if (offset < 0 || offset > bytes.length - size) + throw managedFailure( + code, + scope, + `${label} length prefix leaves the artifact byte range`, + offset >= 0 ? offset : null, + ); + }; + requirePrefix(1); const first = bytes[offset] ?? 0; - if ((first & 0x80) === 0) return { length: first, prefix: 1 }; + if ((first & 0x80) === 0) return { length: first, prefix: 1, minimal: true }; if ((first & 0xc0) === 0x80) { - requireRange(bytes, offset, 2, scope); - return { - length: ((first & 0x3f) << 8) | (bytes[offset + 1] ?? 0), - prefix: 2, - }; + requirePrefix(2); + const length = ((first & 0x3f) << 8) | (bytes[offset + 1] ?? 0); + return { length, prefix: 2, minimal: length >= 0x80 }; } if ((first & 0xe0) === 0xc0) { - requireRange(bytes, offset, 4, scope); - return { - length: - (first & 0x1f) * 0x01_00_00_00 + - (bytes[offset + 1] ?? 0) * 0x01_00_00 + - (bytes[offset + 2] ?? 0) * 0x0100 + - (bytes[offset + 3] ?? 0), - prefix: 4, - }; + requirePrefix(4); + const length = + (first & 0x1f) * 0x01_00_00_00 + + (bytes[offset + 1] ?? 0) * 0x01_00_00 + + (bytes[offset + 2] ?? 0) * 0x0100 + + (bytes[offset + 3] ?? 0); + return { length, prefix: 4, minimal: length >= 0x4000 }; } throw managedFailure( - "invalid-blob", + code, scope, - "Metadata blob uses a reserved compressed-length prefix", + `${label} uses a reserved compressed-length prefix`, offset, ); }; @@ -172,7 +176,7 @@ export const readMetadataBlob = ( layout.blob.offset + Math.max(index, 0), ); const start = layout.blob.offset + index; - const decoded = compressedLength(bytes, start, "metadata.#Blob"); + const decoded = compressedLength(bytes, start, "metadata.#Blob", "blob"); if (decoded.length > heapExtent) throw managedFailure( "invalid-blob", @@ -191,6 +195,124 @@ export const readMetadataBlob = ( return bytes.subarray(content, content + decoded.length); }; +/** + * Read the #US literal named by a `0x70` user-string token: a compressed byte + * length, UTF-16LE code units, then one ECMA-335 terminal flag byte. Code units + * are kept as stored, including unpaired surrogates. `parseUserStrings` owns + * which `ldstr` operands are user-string tokens. + */ +export const readMetadataUserString = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + token: string, +): { + readonly offset: number; + readonly value: string; + /** Conformance differences observed beside the readable literal. */ + readonly issues: readonly ManagedParseIssue[]; +} => { + const heap = layout.userStrings; + const scope = `metadata.#US:${token}`; + if (heap === null) + throw managedFailure( + "missing-stream", + scope, + "ldstr references #US, but the metadata has no #US stream", + ); + const index = Number.parseInt(token.slice(4), 16); + if (index >= heap.size) + throw managedFailure( + "invalid-heap-index", + scope, + "ldstr operand does not name an item inside #US", + heap.offset + index, + ); + const start = heap.offset + index; + const decoded = compressedLength(bytes, start, scope, "userString"); + const content = start + decoded.prefix; + if (content > heap.offset + heap.size - decoded.length) + throw managedFailure( + "invalid-string", + scope, + "User string leaves #US", + start, + ); + const budget = managedDecodeBudget(layout); + // A wider length prefix than the value needs still reads unambiguously, so + // it is reported like the flag rather than discarding the literal. It is + // admitted only on paths that return the literal, so a fatal item leaves + // no budget behind. + const conformance = (): ManagedParseIssue[] => { + if (decoded.minimal) return []; + budget.reserve(512, start); + return [ + { + code: "invalid-string", + scope, + offset: start, + detail: `User string length ${String(decoded.length)} uses a ${String(decoded.prefix)}-byte prefix wider than ECMA-335 II.23.2 assigns; readable UTF-16 is preserved and runtime acceptance is not assessed`, + }, + ]; + }; + // ECMA-335 II.24.2.4: offset 0 is the heap's empty entry, a zero length + // with no terminal flag byte; every other item carries the flag. + if (index === 0) { + if (decoded.length === 0) + return { offset: start, value: "", issues: conformance() }; + throw managedFailure( + "invalid-string", + scope, + `#US offset 0 is the reserved empty entry, but it encodes ${String(decoded.length)} byte(s)`, + start, + ); + } + if (decoded.length % 2 !== 1) + throw managedFailure( + "invalid-string", + scope, + "User string length is not UTF-16 code units plus one terminal flag byte", + start, + ); + const units = (decoded.length - 1) / 2; + // Admit the literal before scanning it: JSON spends at most six bytes per + // UTF-16 code unit. + budget.reserveString(units, start); + // Check the serialized flag independently of literal decoding. A difference + // from ECMA-335 is observable metadata, not proof of runtime rejection. + let expectedFlag = 0; + for (let unitIndex = 0; unitIndex < units; unitIndex++) { + const unit = bytes.readUInt16LE(content + unitIndex * 2); + if ( + unit > 0xff || + (unit >= 0x01 && unit <= 0x08) || + (unit >= 0x0e && unit <= 0x1f) || + unit === 0x27 || + unit === 0x2d || + unit === 0x7f + ) { + expectedFlag = 1; + break; + } + } + const flagOffset = content + units * 2; + const actualFlag = bytes[flagOffset]; + const issues = conformance(); + if (actualFlag !== expectedFlag) { + budget.reserve(512, flagOffset); + issues.push({ + code: "invalid-string", + scope, + offset: flagOffset, + detail: `User string terminal flag ${String(actualFlag)} differs from ECMA-335 II.24.2.4 value ${String(expectedFlag)}; readable UTF-16 is preserved and runtime acceptance is not assessed`, + }); + } + return { + offset: start, + value: bytes.subarray(content, flagOffset).toString("utf16le"), + issues, + }; +}; + /** Read one conventional GUID value from the one-based #GUID heap. */ export const readMetadataGuid = ( bytes: Buffer, diff --git a/src/dotnet/ManagedMetadataLayout.ts b/src/dotnet/ManagedMetadataLayout.ts index b7dc0a7ae..b38854645 100644 --- a/src/dotnet/ManagedMetadataLayout.ts +++ b/src/dotnet/ManagedMetadataLayout.ts @@ -78,6 +78,8 @@ export interface ManagedMetadataLayout { readonly strings: MetadataStream; readonly guid: MetadataStream; readonly blob: MetadataStream; + /** Optional: assemblies without `ldstr` literals may omit #US. */ + readonly userStrings: MetadataStream | null; readonly tables: ReadonlyMap; readonly rowCounts: readonly number[]; readonly stringIndexSize: 2 | 4; @@ -431,6 +433,7 @@ export const readManagedMetadataLayout = ( strings: requireStream(streams, "#Strings"), guid: requireStream(streams, "#GUID"), blob: requireStream(streams, "#Blob"), + userStrings: streams.get("#US") ?? null, tables, rowCounts, stringIndexSize: heaps.string, diff --git a/src/dotnet/ManagedMethodBodyReader.ts b/src/dotnet/ManagedMethodBodyReader.ts index 7c507cdd9..81f3543a2 100644 --- a/src/dotnet/ManagedMethodBodyReader.ts +++ b/src/dotnet/ManagedMethodBodyReader.ts @@ -76,17 +76,25 @@ export const methodBody = ( flags: 0, }, budget?: ManagedDecodeBudget, -): ManagedMethodBody => { - if (rva === 0) return emptyMethodBody(rva, "absent", null); - if ( +): { + readonly body: ManagedMethodBody; + /** + * Whether every CIL instruction was decoded, independent of exception-region + * validity. A body with no CIL, absent or native, has nothing left unscanned. + */ + readonly scanned: boolean; +} => { + // Without CIL there is nothing left unscanned. + if (rva === 0) + return { body: emptyMethodBody(rva, "absent", null), scanned: true }; + const native = (implementation.implFlags & 7) !== 0 || - (implementation.flags & 0x2000) !== 0 - ) - return emptyMethodBody( - rva, - "partial", - "CIL decoding is unavailable for native, runtime, unmanaged, or P/Invoke implementations", - ); + (implementation.flags & 0x2000) !== 0; + if (native) { + const issue = + "CIL decoding is unavailable for native, runtime, unmanaged, or P/Invoke implementations"; + return { body: emptyMethodBody(rva, "partial", issue), scanned: true }; + } try { const offset = pe.rvaToOffset(rva, 1, "method.body"); const methodExtent = pe.rvaAvailableBytes(rva, "method.body"); @@ -181,7 +189,10 @@ export const methodBody = ( ? "Instruction decoding stopped before the end of the method body" : null), }; - return budget === undefined ? body : admitManagedProjection(body, budget); + return { + body: budget === undefined ? body : admitManagedProjection(body, budget), + scanned: decoded.issue === null && decoded.truncated === 0, + }; } catch (cause: unknown) { if ( cause instanceof ManagedReaderFailure && @@ -193,11 +204,14 @@ export const methodBody = ( !(cause instanceof RangeError) ) throw cause; - return emptyMethodBody( - rva, - "malformed", - cause instanceof Error ? cause.message : "Method body parse failed", - ); + return { + body: emptyMethodBody( + rva, + "malformed", + cause instanceof Error ? cause.message : "Method body parse failed", + ), + scanned: false, + }; } finally { // IL and exception-section bookkeeping coexist until the body is built. budget?.releaseTransient(); diff --git a/src/dotnet/ManagedPe.fixture.ts b/src/dotnet/ManagedPe.fixture.ts index 7ea2b06b4..0d4cf4978 100644 --- a/src/dotnet/ManagedPe.fixture.ts +++ b/src/dotnet/ManagedPe.fixture.ts @@ -53,6 +53,8 @@ interface ManagedPeFixtureOptions { readonly targetFramework?: string | null; readonly typeName?: string; readonly typeNamespace?: string; + /** Raw #US stream bytes; omitted means the metadata has no #US stream. */ + readonly userStringHeap?: Buffer; readonly virtualSectionSize?: number; } @@ -416,6 +418,56 @@ const fixedStringAttributeBlob = (value: string): Buffer => { return Buffer.concat([Buffer.from([1, 0, bytes.length]), bytes, u16(0)]); }; +/** + * Lay out ECMA-335 #US items (compressed length, UTF-16LE, terminal flag byte) + * and return the `ldstr` token that names each value. + */ +// ECMA-335 II.24.2.4 terminal flag: 1 when a code unit has a high byte or a +// low byte in 0x01-0x08, 0x0E-0x1F, 0x27, 0x2D, or 0x7F. +const userStringFlag = (text: Buffer): number => { + for (let index = 0; index < text.length; index += 2) { + const unit = text.readUInt16LE(index); + if ( + unit > 0xff || + (unit >= 0x01 && unit <= 0x08) || + (unit >= 0x0e && unit <= 0x1f) || + unit === 0x27 || + unit === 0x2d || + unit === 0x7f + ) + return 1; + } + return 0; +}; + +/** A #US heap whose items carry the ECMA-335 terminal flag, and their tokens. */ +export const fixtureUserStringHeap = ( + values: readonly string[], +): { readonly heap: Buffer; readonly tokens: readonly string[] } => { + const items: Buffer[] = [Buffer.from([0])]; + const tokens: string[] = []; + let size = 1; + for (const value of values) { + const text = Buffer.from(value, "utf16le"); + const length = text.length + 1; + if (length > 0x3fff) + throw new RangeError("Managed PE fixture user string is too long"); + const prefix = + length < 0x80 + ? Buffer.from([length]) + : Buffer.from([0x80 | (length >> 8), length & 0xff]); + tokens.push(`0x70${size.toString(16).padStart(6, "0")}`); + const item = Buffer.concat([ + prefix, + text, + Buffer.from([userStringFlag(text)]), + ]); + items.push(item); + size += item.length; + } + return { heap: Buffer.concat(items), tokens }; +}; + const metadataStreamHeader = ( relativeOffset: number, size: number, @@ -428,12 +480,28 @@ const metadataStreamHeader = ( }; const metadataRoot = ( - tables: Buffer, - strings: Buffer, - guid: Buffer, - blob: Buffer, + { + tables, + strings, + guid, + blob, + userStrings, + }: { + readonly tables: Buffer; + readonly strings: Buffer; + readonly guid: Buffer; + readonly blob: Buffer; + readonly userStrings?: Buffer | undefined; + }, tableStreamName: "#~" | "#-" = "#~", ): Buffer => { + const streams: readonly (readonly [string, Buffer])[] = [ + [tableStreamName, tables], + ["#Strings", strings], + ...(userStrings === undefined ? [] : [["#US", userStrings] as const]), + ["#GUID", guid], + ["#Blob", blob], + ]; const version = Buffer.from("v4.0.30319\0\0", "utf8"); const header = Buffer.concat([ u32(0x424a_5342), @@ -443,34 +511,27 @@ const metadataRoot = ( u32(version.length), version, u16(0), - u16(4), + u16(streams.length), ]); - const streamHeaderSize = - metadataStreamHeader(0, 0, tableStreamName).length + - metadataStreamHeader(0, 0, "#Strings").length + - metadataStreamHeader(0, 0, "#GUID").length + - metadataStreamHeader(0, 0, "#Blob").length; + const streamHeaderSize = streams.reduce( + (total, [name]) => total + metadataStreamHeader(0, 0, name).length, + 0, + ); let offset = align4(header.length + streamHeaderSize); - const tablesOffset = offset; - offset = align4(offset + tables.length); - const stringsOffset = offset; - offset = align4(offset + strings.length); - const guidOffset = offset; - offset = align4(offset + guid.length); - const blobOffset = offset; - const headers = Buffer.concat([ - metadataStreamHeader(tablesOffset, tables.length, tableStreamName), - metadataStreamHeader(stringsOffset, strings.length, "#Strings"), - metadataStreamHeader(guidOffset, guid.length, "#GUID"), - metadataStreamHeader(blobOffset, blob.length, "#Blob"), - ]); - const root = Buffer.alloc(align4(blobOffset + blob.length)); + const placed = streams.map(([name, data]) => { + const at = offset; + offset = align4(offset + data.length); + return { name, data, at }; + }); + const headers = Buffer.concat( + placed.map(({ name, data, at }) => + metadataStreamHeader(at, data.length, name), + ), + ); + const root = Buffer.alloc(offset); header.copy(root, 0); headers.copy(root, header.length); - tables.copy(root, tablesOffset); - strings.copy(root, stringsOffset); - guid.copy(root, guidOffset); - blob.copy(root, blobOffset); + for (const { data, at } of placed) data.copy(root, at); return root; }; @@ -951,17 +1012,18 @@ const buildManagedFixtureMetadata = ( (stringSize === 4 ? 0x01 : 0) | (guidSize === 4 ? 0x02 : 0) | (blobSize === 4 ? 0x04 : 0); - const metadata = metadataRoot( - tablesStream( + const metadata = metadataRoot({ + tables: tablesStream( rows, options.metadataValidMaskExtra ?? 0n, heapFlags, options.metadataExtraData, ), - strings.toBuffer(), - guidHeap(options.mvid), - blobs.toBuffer(), - ); + strings: strings.toBuffer(), + guid: guidHeap(options.mvid), + blob: blobs.toBuffer(), + userStrings: options.userStringHeap, + }); if (options.corruptMetadataSignature === true) metadata.writeUInt32LE(0, 0); return metadata; }; @@ -1189,10 +1251,12 @@ export const buildManagedMemberPointerFixture = ( if (fieldPointers !== null) rows.set(3, fieldPointers.map(u16)); if (methodPointers !== null) rows.set(5, methodPointers.map(u16)); return metadataRoot( - tablesStream(rows, 0n, 0), - strings.toBuffer(), - guidHeap(), - blobs.toBuffer(), + { + tables: tablesStream(rows, 0n, 0), + strings: strings.toBuffer(), + guid: guidHeap(), + blob: blobs.toBuffer(), + }, options.streamName ?? "#-", ); }); diff --git a/src/dotnet/ManagedReaderFailure.ts b/src/dotnet/ManagedReaderFailure.ts index 5e16c0ca4..e360571b4 100644 --- a/src/dotnet/ManagedReaderFailure.ts +++ b/src/dotnet/ManagedReaderFailure.ts @@ -43,3 +43,14 @@ export const uniqueManagedParseIssues = ( ): readonly ManagedParseIssue[] => [ ...new Map(issues.map((issue) => [JSON.stringify(issue), issue])).values(), ]; + +/** + * Append issue lists one issue at a time. Spreading a list into `push` passes + * one argument per issue, which overflows the stack on large malformed inputs. + */ +export const appendManagedParseIssues = ( + issues: ManagedParseIssue[], + ...found: readonly (readonly ManagedParseIssue[])[] +): void => { + for (const list of found) for (const issue of list) issues.push(issue); +}; diff --git a/src/dotnet/ManagedUserStrings.ts b/src/dotnet/ManagedUserStrings.ts new file mode 100644 index 000000000..cce2f0b65 --- /dev/null +++ b/src/dotnet/ManagedUserStrings.ts @@ -0,0 +1,120 @@ +import { + admitManagedProjection, + managedDecodeBudget, +} from "./ManagedDecodeBudget.js"; +import type { + ManagedMemberInspection, + ManagedParseIssue, +} from "../domain/managed/managedArtifact.js"; +import type { + ManagedMethod, + ManagedUserString, +} from "./ManagedMemberInspectorCore.js"; +import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; +import { readMetadataUserString } from "./ManagedMetadataHeaps.js"; +import { readManagedValue } from "./ManagedReaderFailure.js"; + +type UserStringInventory = NonNullable; + +/** + * The literals with their own coverage, which never enters member coverage. + * Issues are unique by construction, so they are not deduplicated: each + * distinct token yields one failure or per-offset conformance issues, and a + * missing stream is one issue. + */ +const inventory = ( + literals: ManagedUserString[], + issues: ManagedParseIssue[], + scanned: boolean, +): UserStringInventory => ({ + state: issues.length === 0 && scanned ? "complete" : "partial", + literals, + issues, +}); + +/** Bookkeeping per distinct ldstr token: its set entry and its sorted slot. */ +const USER_STRING_TOKEN_BYTES = 128; + +/** Decode each distinct #US literal an inspected method loads, in token order. */ +export const parseUserStrings = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + parsed: { + readonly methods: readonly ManagedMethod[]; + /** Methods that may be in scope but whose CIL was not fully scanned. */ + readonly unscannedMethods: number; + /** Unreadable selected type names may hide a requested type. */ + readonly unreadableSelectedTypes?: number; + }, +): UserStringInventory => { + const { methods } = parsed; + const budget = managedDecodeBudget(layout); + const scanned = + parsed.unscannedMethods === 0 && + (parsed.unreadableSelectedTypes ?? 0) === 0; + const userStrings: ManagedUserString[] = []; + const issues: ManagedParseIssue[] = []; + const tokens = new Set(); + try { + // Only ldstr loads a literal; a malformed ldtoken can carry a 0x70 token. + for (const method of methods) + for (const { opcode, operand } of method.body.anchors) + if (opcode === "ldstr" && operand !== null && !tokens.has(operand)) { + budget.admitTransient(USER_STRING_TOKEN_BYTES, null); + tokens.add(operand); + } + // Tokens are fixed-width lowercase hex, so string order is token order. + const references: string[] = []; + for (const token of [...tokens].sort()) { + if (token.startsWith("0x70")) references.push(token); + // An operand from another table names no #US item, with or without + // the stream. + else + issues.push( + admitManagedProjection( + { + code: "invalid-heap-index", + scope: `metadata.#US:${token}`, + offset: null, + detail: "ldstr operand does not name an item inside #US", + }, + budget, + ), + ); + } + if (references.length > 0 && layout.userStrings === null) { + // A missing stream is one fact about the metadata, not one per token. + issues.push( + admitManagedProjection( + { + code: "missing-stream", + scope: "metadata.#US", + offset: null, + detail: `${String(references.length)} distinct ldstr operand(s) reference #US, but the metadata has no #US stream`, + }, + budget, + ), + ); + return inventory(userStrings, issues, scanned); + } + for (const token of references) { + const failures: ManagedParseIssue[] = []; + readManagedValue(() => { + const { + offset, + value, + issues: observed, + } = readMetadataUserString(bytes, layout, token); + for (const issue of observed) failures.push(issue); + userStrings.push( + admitManagedProjection({ token, offset, value }, budget), + ); + }, failures); + for (const failure of failures) + issues.push(admitManagedProjection(failure, budget)); + } + return inventory(userStrings, issues, scanned); + } finally { + budget.releaseTransient(); + } +}; diff --git a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts index 56014fbd4..c4c4e1e1a 100644 --- a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts +++ b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts @@ -313,6 +313,30 @@ describe("managed member comparison uncertainty", () => { }); }); +describe("managed member comparison beside user strings", () => { + it("keeps member deltas and coverage when only #US literals are nonconforming", () => { + // The method loads "ok", stored with flag 1 where ECMA-335 expects 0. + const options = { + ilBody: Buffer.from([(7 << 2) | 2, 0x72, 1, 0, 0, 0x70, 0x26, 0x2a]), + userStringHeap: Buffer.from([0, 5, 0x6f, 0, 0x6b, 0, 1]), + }; + const left = inspect(buildManagedPeFixture(options), "/tmp/left-us.dll"); + const right = inspect(buildManagedPeFixture(options), "/tmp/right-us.dll"); + const result = compareManagedMembers( + { evidenceId: left.evidenceId, result: { ...left.result, fields: [] } }, + { + evidenceId: right.evidenceId, + result: { ...right.result, methods: [] }, + }, + ); + + expect(left.result.metadata.status).toBe("complete"); + expect(left.result.coverage.state).toBe("complete"); + expect(left.result.user_strings?.state).toBe("partial"); + expect(result.summary).toMatchObject({ added: 1, removed: 1, unknown: 0 }); + }); +}); + describe("managed member comparison of undecoded signatures", () => { it("remaps renamed methods only when raw signatures and complete normalized IL match", () => { const signature = Buffer.from([0x00, 0x00, 0x7f]); From 02095cd74a39e55227e554b37ed76e62476e8e94 Mon Sep 17 00:00:00 2001 From: xingzhou Date: Wed, 14 Oct 2026 03:02:57 +0800 Subject: [PATCH 13/15] fix(browser): report complete storage key inventory changes (#2861) * fix(browser): report complete storage key inventory changes * fix(browser): preserve unavailable storage inventory coverage Reuse the key-coverage propagation and independent fingerprint mapping proposed by chAng-L19 in morluto/rea#2491. Propagate failed or malformed inventories into the canonical completeness ledger before comparing membership; preserve independently proven value changes. Add producer-to-comparison regressions and document recapture requirements. --- docs/browser-observation.md | 1 + src/domain/webCaptureDiff.ts | 5 +- tests/boundary/browser/webCaptureDiff.test.ts | 179 ++++++++++++++++++ 3 files changed, 184 insertions(+), 1 deletion(-) diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 72dde900b..879d01520 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -243,6 +243,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex - Console observations with a stack source retain call type, argument types, timestamp, and source location. When console text capture is selected, already-delivered primitive values are retained verbatim; objects, getters, and remote properties are never expanded. - WebSocket observations retain direction, opcode, and payload byte length. When shape capture is selected, text frames are classified as text or complete value-free JSON shape; binary bytes, hashes, prefixes, and raw frames are never retained. - Storage observations always redact values. Key names, IndexedDB names, and cache names require `include_storage_keys`; stable content fingerprints require the additional `include_storage_fingerprints` selection. Cookie fingerprints cover cookies applicable to the current main-frame URL, including its path, rather than every cookie path on that origin. Other storage remains scoped to the origin. Cache bodies above 64 KiB and partial IndexedDB remote objects make fingerprint coverage incomplete, so identical observations remain `unknown`. +- Unavailable or malformed storage inventories mark `storage_keys` unavailable, even without fingerprints. Comparisons suppress inferred key additions and removals while preserving modifications proven by complete fingerprints on both sides. Previously exported captures are not retroactively corrected; recapture them to obtain these coverage markers. - Script metadata is included only when CDP supplies a URL on an allowed origin. Stable keys exclude transient CDP script IDs, and exact transient raw URLs are used only during script/resource reconciliation. URL-less scripts are excluded because their origin cannot be established. When CDP supplies an execution-context association, the accepted script retains its authorized frame ID for later attribution. Script metadata admission bounds retained identity and source-map strings to 8 MiB and 50,000 records. When a source-map URL cannot fit, the script identity remains and source-map coverage reports `resource_budget_exhausted`; rejected script observations are counted without retaining their IDs. Source content is omitted unless explicitly requested and is returned inline as a self-verifying artifact with its SHA-256 digest. Frames, resources, scripts, events, and workers outside the selected origin scope are excluded. An explicit `allowed_origins` list can include additional origins; excluded target details are counted without being exposed. diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index 736d609ee..d8ce6d9e2 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -70,9 +70,11 @@ const compareDimension = ( after: ReadonlyMap, complete: boolean, reason: string, + inventoryComplete = complete, ): Dimension => { + // Membership evidence can be complete even when selected content is unknown. const all = compareIdentities(before, after).filter( - ({ change }) => complete || change === "modified", + ({ change }) => inventoryComplete || change === "modified", ); if (all.length > 0) return { @@ -160,6 +162,7 @@ const storageDimension = ( ), storageComparable(before, after, keysComplete, usageComplete), "Storage fingerprints, inventories, usage, or quota were unavailable or truncated in at least one observation.", + keysComplete, ); }; diff --git a/tests/boundary/browser/webCaptureDiff.test.ts b/tests/boundary/browser/webCaptureDiff.test.ts index 3680f7dfa..99f347c81 100644 --- a/tests/boundary/browser/webCaptureDiff.test.ts +++ b/tests/boundary/browser/webCaptureDiff.test.ts @@ -14,6 +14,185 @@ import { } from "../../fixtures/fakeCdpBrowser.js"; const browsers: FakeCdpBrowser[] = []; + +const inspectStorage = async ( + browser: FakeCdpBrowser, + fingerprints = false, +) => { + const captured = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + include_storage_keys: true, + include_storage_fingerprints: fingerprints, + }), + ); + if (!captured.ok) throw captured.error; + return captured.value; +}; + +const compareStorage = (left: WebPageInspection, right: WebPageInspection) => + compareWebCaptures( + compareWebCapturesInputSchema.parse({ + before: { inspection: left }, + after: { inspection: right }, + }), + ).dimensions.storage; + +it.each([ + ["local", "DOMStorage.getDOMStorageItems", { entries: [null] }], + [ + "session", + "DOMStorage.getDOMStorageItems", + { entries: [["known", "value"], null] }, + ], + ["indexed", "IndexedDB.requestDatabaseNames", { databaseNames: [null] }], + ["cache", "CacheStorage.requestCacheNames", { caches: [null] }], +] as const)( + "does not invent membership changes after an incomplete %s inventory", + async (kind, method, malformed) => { + let failure: "unsupported" | "malformed" | undefined; + const matches = (command: { + method: string; + params: Record; + }) => { + if (command.method !== method) return false; + if (kind !== "local" && kind !== "session") return true; + return ( + (command.params.storageId as { isLocalStorage: boolean }) + .isLocalStorage === + (kind === "local") + ); + }; + const browser = await startFakeCdpBrowser({ + commandError: (command) => + failure === "unsupported" && matches(command) + ? { code: -32601, message: "Method not found" } + : undefined, + commandResult: (command) => + failure === "malformed" && matches(command) ? malformed : undefined, + }); + browsers.push(browser); + const before = await inspectStorage(browser); + expect(before.completeness.unavailable_sections).not.toContain( + "storage_keys", + ); + for (const [mode, reason] of [ + ["unsupported", "provider_unavailable"], + ["malformed", "invalid_protocol_value"], + ] as const) { + failure = mode; + const after = await inspectStorage(browser); + expect(after.completeness.excluded).toContainEqual({ + section: "storage_keys", + reason, + count: null, + }); + expect(after.completeness.unavailable_sections).toContain("storage_keys"); + for (const [left, right] of [ + [before, after], + [after, before], + ] as const) + expect(compareStorage(left, right)).toMatchObject({ + status: "unknown", + total_changes: 0, + changes: [], + }); + if (kind === "session" && mode === "malformed") + expect(after.storage.session_storage_keys).toEqual(["known"]); + } + }, +); + +it("preserves shared content changes when a peer inventory is unavailable", async () => { + let value = "before"; + const browser = await startFakeCdpBrowser({ + unsupportedMethods: ["CacheStorage.requestCacheNames"], + commandResult: (command) => + command.method === "DOMStorage.getDOMStorageItems" + ? { entries: [["shared-key", value]] } + : undefined, + }); + browsers.push(browser); + const before = await inspectStorage(browser, true); + expect(before.completeness.unavailable_sections).toContain("storage_keys"); + expect(compareStorage(before, before)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + value = "after"; + const after = await inspectStorage(browser, true); + for (const [left, right] of [ + [before, after], + [after, before], + ] as const) + expect(compareStorage(left, right)).toMatchObject({ + status: "changed", + total_changes: 2, + changes: [ + expect.objectContaining({ change: "modified" }), + expect.objectContaining({ change: "modified" }), + ], + }); + for (const fingerprint of after.storage.content_fingerprints) + if (["local_storage", "session_storage"].includes(fingerprint.scope)) + fingerprint.complete = false; + expect(compareStorage(before, after)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); +}); + +it("reports complete storage key additions and removals without content fingerprints", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const before = await inspectStorage(browser); + expect(before.storage.fingerprints_complete).toBe(false); + const after = structuredClone(before); + after.storage.local_storage_keys.push("new-local"); + after.storage.session_storage_keys.push("new-session"); + after.storage.indexed_db_names.push("new-db"); + after.storage.cache_names.push("new-cache"); + const identities = [ + "storage:cache:new-cache", + "storage:indexed_db:new-db", + "storage:local_storage:new-local", + "storage:session_storage:new-session", + ]; + for (const [left, right, change] of [ + [before, after, "added"], + [after, before, "removed"], + ] as const) { + expect(compareStorage(left, right)).toMatchObject({ + status: "changed", + total_changes: 4, + changes: identities.map((identity) => ({ identity, change })), + }); + } + expect(compareStorage(after, after)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + const missingQuota = structuredClone(after); + missingQuota.storage.quota_bytes = null; + expect(compareStorage(before, missingQuota)).toMatchObject({ + status: "changed", + total_changes: 4, + }); + const partialInventory = structuredClone(after); + partialInventory.completeness.unavailable_sections.push("storage_keys"); + expect(compareStorage(before, partialInventory)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + expect(compareStorage(partialInventory, before)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); +}); + const expectInvalidDiff = (input: unknown): void => { expect(webCaptureDiffSchema.safeParse(input).success).toBe(false); }; From 39f96d7188a75e96af7875f16b02b6a0b7facfe3 Mon Sep 17 00:00:00 2001 From: Dragolone <160818051+Dragolone@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:05:34 +1100 Subject: [PATCH 14/15] fix(qlog): list a hidden stream's records without spreading call arguments (#2860) * fix(qlog): list a hidden stream's records without spreading call arguments When an explicit declaration matched a stream's derived total, the decoder pushed every contribution ordinal with a spread call. A stream with more than ~125k STREAM frames exceeded the engine's argument limit, so a valid capture within the input budget failed with RangeError instead of listing the records as incomplete. Push the ordinals in a loop. * fix(comparison): recheck historical calls-only changes (#2223) * fix(comparison): preserve incomplete function call evidence Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty. Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls. * fix(comparison): recheck historical calls-only changes --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> * fix(ci): isolate Linux Hopper process fixture inventory * fix(android): make DEX descriptor pattern portable --------- Co-authored-by: Lucien <164857476+dakjdakd@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/browser/history/QlogCapture.test.ts | 15 +++++++++++++++ src/browser/history/QlogCapture.ts | 6 +++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/browser/history/QlogCapture.test.ts b/src/browser/history/QlogCapture.test.ts index bd67d4e80..fb4e10072 100644 --- a/src/browser/history/QlogCapture.test.ts +++ b/src/browser/history/QlogCapture.test.ts @@ -186,6 +186,21 @@ it("excludes structural auth material and explicit normalized values without gue expect(traces[0]?.incomplete_fact_records).toContain(2); }); +it("omits a hidden derived total for a stream with more contributions than call arguments allow", () => { + const events = Array.from({ length: 130 }, (_, index) => + event( + Array.from({ length: 1_000 }, (_, offset) => + frame(index * 1_000 + offset, 1), + ), + ), + ); + const { traces } = facts(capture(events), ["130000"]); + expect(traces[0]?.streams).toEqual([]); + expect(traces[0]?.incomplete_fact_records).toEqual( + Array.from({ length: 130 }, (_, ordinal) => ordinal), + ); +}); + it("hides marked ancestor keys and safe derived sums while still validating hidden subtrees", () => { const text = capture([event([frame(0, 3), frame(3, 4)])]); expect(facts(text, ["7"]).traces[0]?.streams).toEqual([]); diff --git a/src/browser/history/QlogCapture.ts b/src/browser/history/QlogCapture.ts index 2e8a1f790..8d19fc64f 100644 --- a/src/browser/history/QlogCapture.ts +++ b/src/browser/history/QlogCapture.ts @@ -95,10 +95,10 @@ export const decodeQlogCapture = ( ...stream.reported_final_sizes, ...stream.covered_ranges.flatMap((r) => [r.start, r.end]), ]; + // A stream can have more contributions than call arguments allow. if (numericValues.some(excluded)) - fact.incomplete_fact_records.push( - ...stream.contributions.map((c) => c.record_ordinal), - ); + for (const contribution of stream.contributions) + fact.incomplete_fact_records.push(contribution.record_ordinal); else fact.streams.push(stream); } fact.incomplete_fact_records = [...new Set(fact.incomplete_fact_records)]; From 64041e3072e968002e2085445217ccde1f36e5d7 Mon Sep 17 00:00:00 2001 From: Aaditya Date: Wed, 14 Oct 2026 00:35:57 +0530 Subject: [PATCH 15/15] fix(session): reject an unknown configured provider before unbound routing (#2976) When every registered deep provider rejected a target, select() returned an intentionally unbound success for any concrete environment selector, including one that names no registered provider. A misconfigured REA_ANALYSIS_PROVIDER therefore opened the target unbound, and the later deep-operation diagnostic blamed target support instead of the unknown configuration. The same identifier requested explicitly already failed with unknown_provider. Take the unbound shortcut only for a registered environment preference. An unknown configured identifier now reaches concrete selection and fails with unknown_provider, matching the explicit request. Registered preferences and auto still leave artifact-only targets unbound. Fixes #2592 --- .../binary/AnalysisProviderRegistry.test.ts | 22 +++++++++++++++++++ .../binary/AnalysisProviderRegistry.ts | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/application/binary/AnalysisProviderRegistry.test.ts b/src/application/binary/AnalysisProviderRegistry.test.ts index 8eae6c9c9..fde2f6718 100644 --- a/src/application/binary/AnalysisProviderRegistry.test.ts +++ b/src/application/binary/AnalysisProviderRegistry.test.ts @@ -186,6 +186,28 @@ describe("analysis provider registry: binding failures and cancellation", () => expect(explicit.error.reason).toBe("target_unsupported"); }); + it("rejects an unknown environment preference even when every candidate rejects the target", async () => { + const alpha = candidate("alpha"); + const misconfigured = new AnalysisProviderRegistry( + [alpha.provider], + "missing", + ); + + for (const selected of [ + await misconfigured.select(ARTIFACT_TARGET), + await misconfigured.select(ARTIFACT_TARGET, "missing"), + ]) { + if (selected.ok) throw new Error("expected unknown provider rejection"); + expect(projectAnalysisError(selected.error)).toMatchObject({ + details: { + selection_reason: "unknown_provider", + requested_provider_id: "missing", + candidate_ids: ["alpha"], + }, + }); + } + }); + it("does not bind a profile with unresolved version or adapter failure", async () => { const unresolved = candidate("unresolved", { profile: "missing" }); const failed = candidate("failed", { profile: "error" }); diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index 5b7e78813..033677881 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -176,7 +176,7 @@ export class AnalysisProviderRegistry { return err(new AnalysisCancelledError("open_binary")); if ( request === undefined && - selector !== "auto" && + this.#byId.has(selector) && baseline.length > 0 && baseline.every( ({ targetSupport }) => targetSupport.status === "unsupported",