From a8b50ea147ab3aaa2870160481e09403c3310ea0 Mon Sep 17 00:00:00 2001 From: Shreyansh Jain <136733716+ShreyanshJain105@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:55:58 +0530 Subject: [PATCH 01/11] refactor(firmware): split verification into staged functions (#2713) Refactor firmware verifier into staged functions Restructure scripts/verify/firmware/analysis.mjs for readability without changing behavior. - Move prerequisite checks into checkPrerequisites() - Split the main flow into stageCli, stageMcp, stageExt4 and stageGhidra - Extract timeouts and size limits into named constants - Rename cli/call helpers to runCli/callTool Checks, order of operations, error messages and env vars are unchanged. Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- scripts/verify/firmware/analysis.mjs | 504 ++++++++++++++++----------- 1 file changed, 299 insertions(+), 205 deletions(-) diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index 1f84681c9..804e8fffa 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -1,15 +1,8 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { - access, - mkdir, - mkdtemp, - readFile, - rm, - symlink, -} from "node:fs/promises"; +import * as fs from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import * as nodePath from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna import { parseEvidence } from "../../../dist/domain/evidence.js"; import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js"; -const exec = promisify(execFile); -const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); -const entrypoint = resolve( +const execFileAsync = promisify(execFile); +const { join, resolve, dirname } = nodePath; + +const CALL_TIMEOUT_MS = 150_000; +const CONNECT_TIMEOUT_MS = 30_000; +const SEVENZIP_TIMEOUT_MS = 10_000; +const MAX_STDOUT_BYTES = 16 * 1024 * 1024; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const ENTRYPOINT = resolve( process.env.REA_FIRMWARE_TEST_ENTRYPOINT ?? - join(repository, "scripts/rea.mjs"), + join(REPO_ROOT, "scripts/rea.mjs"), ); -if (process.platform !== "linux") - throw new Error( - "verify:firmware requires Linux; other hosts are not verified", - ); -for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { - const path = process.env[variable]; - if (path === undefined) - throw new Error( - `verify:firmware requires ${variable}; see docs/firmware-analysis.md`, - ); - await access(path); -} -try { - await exec("7z", ["i"], { timeout: 10_000 }); -} catch (cause) { - throw new Error( - "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", - { cause }, - ); -} -const fixtureRoot = resolve( +const FIXTURE_ROOT = resolve( process.env.REA_FIRMWARE_FIXTURE_ROOT ?? - join(repository, "_reference/firmware-integration/generated"), + join(REPO_ROOT, "_reference/firmware-integration/generated"), ); -await access(join(fixtureRoot, "oracle.json")).catch((cause) => { - throw new Error( - "verify:firmware requires generated fixtures; run npm run fixtures:firmware", - { cause }, - ); -}); + +// --------------------------------------------------------------------------- +// Prerequisite checks +// --------------------------------------------------------------------------- + +async function checkPrerequisites() { + if (process.platform !== "linux") { + throw new Error( + "verify:firmware requires Linux; other hosts are not verified", + ); + } + + for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { + const value = process.env[name]; + if (value === undefined) { + throw new Error( + `verify:firmware requires ${name}; see docs/firmware-analysis.md`, + ); + } + await fs.access(value); + } + + try { + await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS }); + } catch (cause) { + throw new Error( + "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", + { cause }, + ); + } + + try { + await fs.access(join(FIXTURE_ROOT, "oracle.json")); + } catch (cause) { + throw new Error( + "verify:firmware requires generated fixtures; run npm run fixtures:firmware", + { cause }, + ); + } +} + +await checkPrerequisites(); + +// --------------------------------------------------------------------------- +// Shared state +// --------------------------------------------------------------------------- + const oracle = JSON.parse( - await readFile(join(fixtureRoot, "oracle.json"), "utf8"), + await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"), ); -const path = join(fixtureRoot, "firmware.bin"); -assert.equal(await hashFirmwareFile(path), oracle.sha256); -const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-")); -const environment = Object.fromEntries( - Object.entries(process.env).filter(([, value]) => value !== undefined), +const firmwarePath = join(FIXTURE_ROOT, "firmware.bin"); +assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256); + +const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-")); +const baseEnv = Object.fromEntries( + Object.entries(process.env).filter(([, v]) => v !== undefined), ); -const client = new Client({ name: "rea-real-firmware-verifier", version: "1" }); -const cli = async (args, env = environment) => { - const response = await exec( +const mcpClient = new Client({ + name: "rea-real-firmware-verifier", + version: "1", +}); + +// --------------------------------------------------------------------------- +// Invocation helpers +// --------------------------------------------------------------------------- + +async function runCli(args, env = baseEnv) { + const { stdout } = await execFileAsync( process.execPath, - [entrypoint, ...args, "--format", "json"], + [ENTRYPOINT, ...args, "--format", "json"], { env, - cwd: repository, - timeout: 150_000, - maxBuffer: 16 * 1024 * 1024, + cwd: REPO_ROOT, + timeout: CALL_TIMEOUT_MS, + maxBuffer: MAX_STDOUT_BYTES, }, ); - return parseEvidence(JSON.parse(response.stdout)); -}; -const call = async (name, args) => { - const response = await client.callTool( + return parseEvidence(JSON.parse(stdout)); +} + +async function callTool(name, args) { + const result = await mcpClient.callTool( { name, arguments: args }, - { timeout: 150_000 }, + { timeout: CALL_TIMEOUT_MS }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - assert.ok(response.structuredContent !== undefined); - return response.structuredContent; -}; -const verifyRegions = (evidence) => { + assert.notEqual(result.isError, true, JSON.stringify(result)); + assert.ok(result.structuredContent !== undefined); + return result.structuredContent; +} + +// --------------------------------------------------------------------------- +// Result verifiers +// --------------------------------------------------------------------------- + +function verifyRegions(evidence) { assert.equal(evidence.subject.digest.sha256, oracle.sha256); - const result = firmwareResultSchemas.inspect_firmware_regions.parse( + + const parsed = firmwareResultSchemas.inspect_firmware_regions.parse( evidence.normalized_result, ); - const gzip = result.regions.find( - (region) => region.signature === "gzip" && region.offset === oracle.offset, + + const gzipRegion = parsed.regions.find( + (r) => r.signature === "gzip" && r.offset === oracle.offset, ); assert.ok( - gzip !== undefined, + gzipRegion !== undefined, "Binwalk must find the independently generated gzip offset", ); - assert.equal(gzip.reported_size, oracle.length); - assert.equal(gzip.size_basis, "provider_reported_validation_unknown"); - return result.regions.map( - ({ provider_id: _provider_id, ...region }) => region, - ); -}; -const verifyFiles = async (evidence) => { - const result = firmwareResultSchemas.extract_firmware.parse( + assert.equal(gzipRegion.reported_size, oracle.length); + assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown"); + + // Strip provider-specific ids so CLI and MCP results compare equal. + return parsed.regions.map((region) => { + const { provider_id: _ignored, ...rest } = region; + return rest; + }); +} + +async function verifyFiles(evidence) { + const parsed = firmwareResultSchemas.extract_firmware.parse( evidence.normalized_result, ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip")); + + assert.ok(parsed.chunks.some((c) => c.handler === "gzip")); + for (const [suffix, expected] of Object.entries(oracle.files)) { - const file = result.files.find((candidate) => - candidate.relative_path.endsWith(`/${suffix}`), + const entry = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(file !== undefined, `Missing extracted ${suffix}`); - assert.equal(file.sha256, expected.sha256); - assert.equal(file.size, expected.size); - assert.equal(await hashFirmwareFile(file.path), expected.sha256); - assert.equal(file.original_file_range, null); - assert.equal(file.runtime_address, null); + assert.ok(entry !== undefined, `Missing extracted ${suffix}`); + assert.equal(entry.sha256, expected.sha256); + assert.equal(entry.size, expected.size); + assert.equal(await hashFirmwareFile(entry.path), expected.sha256); + assert.equal(entry.original_file_range, null); + assert.equal(entry.runtime_address, null); } - return result; -}; -try { - const inspected = await cli(["inspect-firmware-regions", path]); - const cliRegions = verifyRegions(inspected); - const extracted = await cli([ - "extract-firmware", - path, - join(root, "cli-output"), + + return parsed; +} + +const toContentIdentity = (parsed) => + parsed.files.map(({ relative_path, sha256, size }) => ({ + relative_path, + sha256, + size, + })); + +// --------------------------------------------------------------------------- +// Verification stages +// --------------------------------------------------------------------------- + +async function stageCli() { + const regionsEvidence = await runCli([ + "inspect-firmware-regions", + firmwarePath, ]); - const cliFiles = await verifyFiles(extracted); - assert.equal(cliFiles.coverage, "partial"); + const regions = verifyRegions(regionsEvidence); + + const extractEvidence = await runCli([ + "extract-firmware", + firmwarePath, + join(workDir, "cli-output"), + ]); + const files = await verifyFiles(extractEvidence); + + assert.equal(files.coverage, "partial"); assert.ok( - cliFiles.chunks.some( - (chunk) => - chunk.handler === null && - chunk.range.offset === 0 && - chunk.range.length === 64, + files.chunks.some( + (c) => + c.handler === null && c.range.offset === 0 && c.range.length === 64, ), ); - await client.connect( - new StdioClientTransport({ - command: process.execPath, - args: [entrypoint, "mcp"], - cwd: repository, - env: environment, - stderr: "pipe", - }), - { timeout: 30_000 }, - ); - const inspectedMcp = await call("inspect_firmware_regions", { path }); - const mcpEvidence = parseEvidence(inspectedMcp); - assert.deepEqual(verifyRegions(mcpEvidence), cliRegions); - const extractedMcp = await call("extract_firmware", { - path, - output_directory: join(root, "mcp-output"), + + return { regions, files }; +} + +async function connectMcp() { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ENTRYPOINT, "mcp"], + cwd: REPO_ROOT, + env: baseEnv, + stderr: "pipe", }); - const mcpFiles = await verifyFiles(parseEvidence(extractedMcp)); - const contentIdentity = (result) => - result.files.map(({ relative_path, sha256, size }) => ({ - relative_path, - sha256, - size, - })); - assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles)); - const selected = await call("extract_firmware", { - path, - output_directory: join(root, "selected-output"), + await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS }); +} + +async function stageMcp(cli) { + const regionsRaw = await callTool("inspect_firmware_regions", { + path: firmwarePath, + }); + assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions); + + const extractRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "mcp-output"), + }); + const mcpFiles = await verifyFiles(parseEvidence(extractRaw)); + assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files)); + + const rangedRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "selected-output"), range: { offset: oracle.offset, length: oracle.length }, }); - const selectedFiles = await verifyFiles(parseEvidence(selected)); - assert.equal(selectedFiles.selection.offset, oracle.offset); - assert.equal(selectedFiles.selection.length, oracle.length); - const depth = await call("extract_firmware", { - path, - output_directory: join(root, "depth-output"), + const ranged = await verifyFiles(parseEvidence(rangedRaw)); + assert.equal(ranged.selection.offset, oracle.offset); + assert.equal(ranged.selection.length, oracle.length); + + const depthRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "depth-output"), max_depth: 1, }); - const depthResult = firmwareResultSchemas.extract_firmware.parse( - depth.normalized_result, + const depthParsed = firmwareResultSchemas.extract_firmware.parse( + depthRaw.normalized_result, ); - assert.equal(depthResult.coverage, "partial"); - assert.ok(depthResult.depth_limited_paths.length > 0); + assert.equal(depthParsed.coverage, "partial"); + assert.ok(depthParsed.depth_limited_paths.length > 0); + console.log( "PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits", ); - if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") { - const ext4 = join(fixtureRoot, "ext4.bin"); - await access(ext4); - const response = await call("extract_firmware", { - path: ext4, - output_directory: join(root, "ext4-output"), - }); - const result = firmwareResultSchemas.extract_firmware.parse( - response.normalized_result, + + return mcpFiles; +} + +async function stageExt4() { + const ext4Path = join(FIXTURE_ROOT, "ext4.bin"); + await fs.access(ext4Path); + + const raw = await callTool("extract_firmware", { + path: ext4Path, + output_directory: join(workDir, "ext4-output"), + }); + const parsed = firmwareResultSchemas.extract_firmware.parse( + raw.normalized_result, + ); + assert.ok(parsed.chunks.some((c) => c.handler === "extfs")); + assert.equal(parsed.coverage, "complete"); + + for (const [suffix, expected] of Object.entries(oracle.files)) { + const match = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs")); - assert.equal(result.coverage, "complete"); - for (const [suffix, expected] of Object.entries(oracle.files)) - assert.equal( - result.files.find((file) => file.relative_path.endsWith(`/${suffix}`)) - ?.sha256, - expected.sha256, - ); - console.log("PASS real ext4 extraction with debugfs; no mounts"); - // Preserve the process-ownership utility while withholding all extractors. - const withoutExtractors = join(root, "without-extractors"); - await access("/bin/ps"); - await mkdir(withoutExtractors); - await symlink("/bin/ps", join(withoutExtractors, "ps")); - const missing = await cli( - ["extract-firmware", ext4, join(root, "missing-extractor-output")], - { ...environment, PATH: withoutExtractors }, - ); - const partial = firmwareResultSchemas.extract_firmware.parse( - missing.normalized_result, - ); - assert.equal(partial.coverage, "partial"); - assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs")); - console.log( - "PASS real missing-extractor diagnostics retained despite Unblob exit 1", - ); - } - if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") { - if (process.env.GHIDRA_INSTALL_DIR === undefined) - throw new Error( - "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", - ); - const child = mcpFiles.files.find((file) => - file.relative_path.endsWith("/usr/bin/probe"), - ); - assert.ok(child !== undefined); - await call("open_binary", { path: child.path, provider_id: "ghidra" }); - const dossier = await call("analyze_function", { - procedure: "firmware_probe", - }); - const evidence = parseEvidence(dossier); - assert.equal( - evidence.subject.digest.sha256, - oracle.files["usr/bin/probe"].sha256, - ); - const pseudocode = await call("procedure_pseudo_code", { - procedure: "firmware_probe", - }); - assert.equal(parseEvidence(pseudocode).provider.id, "ghidra"); - assert.ok( - JSON.stringify(evidence.normalized_result).includes("firmware_probe"), - ); - await call("close_binary", {}); - console.log( - "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + assert.equal(match?.sha256, expected.sha256); + } + console.log("PASS real ext4 extraction with debugfs; no mounts"); + + // Keep the process-ownership utility available but hide every extractor. + const emptyBin = join(workDir, "without-extractors"); + await fs.access("/bin/ps"); + await fs.mkdir(emptyBin); + await fs.symlink("/bin/ps", join(emptyBin, "ps")); + + const degraded = await runCli( + ["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")], + { ...baseEnv, PATH: emptyBin }, + ); + const degradedParsed = firmwareResultSchemas.extract_firmware.parse( + degraded.normalized_result, + ); + assert.equal(degradedParsed.coverage, "partial"); + assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs")); + console.log( + "PASS real missing-extractor diagnostics retained despite Unblob exit 1", + ); +} + +async function stageGhidra(mcpFiles) { + if (process.env.GHIDRA_INSTALL_DIR === undefined) { + throw new Error( + "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", ); } + + const probe = mcpFiles.files.find((f) => + f.relative_path.endsWith("/usr/bin/probe"), + ); + assert.ok(probe !== undefined); + + await callTool("open_binary", { path: probe.path, provider_id: "ghidra" }); + + const dossierRaw = await callTool("analyze_function", { + procedure: "firmware_probe", + }); + const dossier = parseEvidence(dossierRaw); assert.equal( - await hashFirmwareFile(path), + dossier.subject.digest.sha256, + oracle.files["usr/bin/probe"].sha256, + ); + + const pseudoRaw = await callTool("procedure_pseudo_code", { + procedure: "firmware_probe", + }); + assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra"); + assert.ok( + JSON.stringify(dossier.normalized_result).includes("firmware_probe"), + ); + + await callTool("close_binary", {}); + console.log( + "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + ); +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +try { + const cli = await stageCli(); + await connectMcp(); + const mcpFiles = await stageMcp(cli); + + if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4(); + if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") + await stageGhidra(mcpFiles); + + assert.equal( + await hashFirmwareFile(firmwarePath), oracle.sha256, "Original firmware bytes must remain unchanged", ); } finally { - await client.close(); - await rm(root, { recursive: true, force: true }); + await mcpClient.close(); + await fs.rm(workDir, { recursive: true, force: true }); } From fcfa5b7e88e2251359950d3fcca8a1ae61dc6814 Mon Sep 17 00:00:00 2001 From: Jeongwan Jeon Date: Tue, 13 Oct 2026 21:27:38 +0900 Subject: [PATCH 02/11] fix(artifacts): distinguish JVM classes from universal Mach-O files (#2327) * test(artifacts): reproduce JVM class inventory misclassification * fix(artifacts): validate FAT declarations before classifying executables --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- .../ArtifactGraphConstruction.test.ts | 103 ++++++++++++++++ src/domain/apple/machoContainer.ts | 5 + .../acceptance/artifactClassification.test.ts | 111 ++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 src/artifacts/inventory/ArtifactGraphConstruction.test.ts create mode 100644 tests/acceptance/artifactClassification.test.ts diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts new file mode 100644 index 000000000..925d3543a --- /dev/null +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; + +import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js"; +import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; + +const fat = (wide = false): Buffer => + Buffer.from(fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }], wide)); + +describe("bounded FAT inventory classification", () => { + it("requires the complete table but not slice bytes in the retained prefix", () => { + const bytes = fat(); + for (const length of [4, 7, 8, 27]) { + expect( + classifyArtifactContent( + "native", + bytes.subarray(0, length), + bytes.length, + ), + ).toEqual({ kind: "unknown", format: "unknown" }); + } + expect( + classifyArtifactContent( + "native.class", + bytes.subarray(0, 28), + bytes.length, + ), + ).toEqual({ kind: "executable", format: "mach-o-universal" }); + }); + + it.each([ + ["empty table", (bytes: Buffer) => bytes.writeUInt32BE(0, 4)], + ["slice overlapping table", (bytes: Buffer) => bytes.writeUInt32BE(0, 16)], + ["empty slice", (bytes: Buffer) => bytes.writeUInt32BE(0, 20)], + [ + "slice beyond file", + (bytes: Buffer) => bytes.writeUInt32BE(bytes.length, 20), + ], + ["unaligned slice", (bytes: Buffer) => bytes.writeUInt32BE(4097, 16)], + ["unsupported alignment", (bytes: Buffer) => bytes.writeUInt32BE(53, 24)], + ] as const)("keeps %s unknown", (_label, mutate) => { + const bytes = fat(); + mutate(bytes); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("does not infer Mach-O from an out-of-bounds architecture count", () => { + const bytes = fat(); + bytes.writeUInt32BE(0xffffffff, 4); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "resource", + format: "file", + }); + }); + + it("does not lose unsafe FAT64 range values to number conversion", () => { + const bytes = fat(true); + bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires valid ranges for every declared slice", () => { + const bytes = Buffer.from( + fatImage([ + { cpu: CPU.arm64, bytes: machoImage({}) }, + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + ]), + ); + bytes.writeUInt32BE(bytes.length, 40); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires observed slice magic for an unknown CPU type", () => { + const cpu = { type: 0x12345678, subtype: 0 }; + const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }])); + expect( + classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length), + ).toEqual({ kind: "unknown", format: "unknown" }); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "executable", + format: "mach-o-universal", + }); + }); + + it.each([NaN, Infinity, -1, 8.5, Number.MAX_SAFE_INTEGER + 1, 27])( + "keeps invalid or contradictory file size %s unknown", + (size) => { + const bytes = fat(); + expect(classifyArtifactContent("native", bytes, size)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }, + ); +}); diff --git a/src/domain/apple/machoContainer.ts b/src/domain/apple/machoContainer.ts index 7b552be64..c23cf7f64 100644 --- a/src/domain/apple/machoContainer.ts +++ b/src/domain/apple/machoContainer.ts @@ -63,6 +63,11 @@ export const readFatContainerHeader = ( const magic = view.getUint32(0, false); if (![FAT_MAGIC, FAT_MAGIC_64, FAT_CIGAM, FAT_CIGAM_64].includes(magic)) return { state: "not-fat" }; + if (!Number.isSafeInteger(fileSize) || fileSize < bytes.byteLength) + return { + state: "malformed", + reason: "FAT file size is invalid or contradicts retained bytes", + }; if (bytes.byteLength < 8) { return fileSize < 8 ? { state: "malformed", reason: "FAT header is truncated" } diff --git a/tests/acceptance/artifactClassification.test.ts b/tests/acceptance/artifactClassification.test.ts new file mode 100644 index 000000000..7436a4967 --- /dev/null +++ b/tests/acceptance/artifactClassification.test.ts @@ -0,0 +1,111 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect } from "vitest"; + +import { + CPU, + fatImage, + machoImage, +} from "../../src/artifacts/apple/MachoImage.fixture.js"; +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest.for(["jar", "zip"])( + "distinguishes JVM classes from universal Mach-O files in %s inventory", + async (extension, { cli }) => { + const root = await createTestTempDirectory("rea-class-inventory-"); + const path = join(root, `mixed.${extension}`); + // The large class can hold the entire 65-entry table suggested by its + // version bytes. Checking only the table length does not distinguish it. + const classes = [ + ["Hello.class", javaClass()], + ["Big.class", javaClass("x".repeat(4_000))], + ["renamed.bin", javaClass()], + ] as const; + const writer = new ZipWriter(new Uint8ArrayWriter()); + for (const [name, bytes] of classes) + await writer.add(name, new Uint8ArrayReader(bytes)); + const nativePaths: string[] = []; + for (const wide of [false, true]) { + for (const littleEndian of [false, true]) { + const name = `fat${wide ? 64 : 32}-${littleEndian ? "le" : "be"}`; + nativePaths.push(name); + await writer.add( + name, + new Uint8ArrayReader( + fatImage( + [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], + wide, + littleEndian, + ), + ), + ); + } + } + await writeFile(path, await writer.close()); + + const reply = await cli.run({ + arguments: ["inspect-artifact", path, "--json"], + }); + expect(reply.exitCode, reply.stdout + reply.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.json).normalized_result, + ); + const inventory = artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ); + for (const name of nativePaths) { + expect(inventory.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: name, + artifact_kind: "executable", + artifact_format: "mach-o-universal", + limitations: [], + }), + ); + } + for (const [name, bytes] of classes) { + const occurrence = inventory.occurrences.find( + (item) => item.logical_path === name, + ); + expect(occurrence).toBeDefined(); + expect.soft(occurrence?.artifact_kind, name).not.toBe("executable"); + expect.soft(occurrence?.artifact_format, name).not.toMatch(/^mach-o/u); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const node = inventory.nodes.find((item) => item.sha256 === sha256); + expect(node).toBeDefined(); + expect.soft(node?.format, name).not.toMatch(/^mach-o/u); + } + }, +); + +/** Minimal valid Java 21 class, with an optional unused UTF-8 constant. */ +const javaClass = (extra?: string): Buffer => { + const utf8 = (value: string): Buffer => { + const text = Buffer.from(value); + const header = Buffer.alloc(3); + header[0] = 1; + header.writeUInt16BE(text.length, 1); + return Buffer.concat([header, text]); + }; + const constants = [ + Buffer.from([7, 0, 2]), + utf8("Hello"), + Buffer.from([7, 0, 4]), + utf8("java/lang/Object"), + ...(extra === undefined ? [] : [utf8(extra)]), + ]; + const header = Buffer.from("cafebabe000000410000", "hex"); + header.writeUInt16BE(constants.length + 1, 8); + return Buffer.concat([ + header, + ...constants, + Buffer.from("0021000100030000000000000000", "hex"), + ]); +}; From d85374496965a8a997e1d13e98a27ff5e5ca1077 Mon Sep 17 00:00:00 2001 From: Jeongwan Jeon Date: Tue, 13 Oct 2026 21:28:31 +0900 Subject: [PATCH 03/11] fix: preserve process identities in pwntools cleanup errors (#2284) Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/native/pwntools/PwntoolsDecoder.ts | 23 ++++++++-------- src/process/OwnedCommand.ts | 19 +++++++++----- tests/boundary/process/ownedCommand.test.ts | 8 +++--- .../pwntoolsStageRetirement.test.ts | 26 ++++++++++++------- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/src/native/pwntools/PwntoolsDecoder.ts b/src/native/pwntools/PwntoolsDecoder.ts index 169f25601..6aa456631 100644 --- a/src/native/pwntools/PwntoolsDecoder.ts +++ b/src/native/pwntools/PwntoolsDecoder.ts @@ -32,7 +32,10 @@ import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorClean import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { runOwnedCommand } from "../../process/OwnedCommand.js"; +import { + ownedCommandResources, + runOwnedCommand, +} from "../../process/OwnedCommand.js"; import type { ProviderProcessSupervisor, OwnedProviderProcessSpawnOptions, @@ -562,20 +565,16 @@ export class PwntoolsDecoder< const pending = this.#pendingCleanup; if (pending?.process === undefined) return undefined; const stopped = await pending.process.stop(); + const ownership = pending.process.launch.ownership; if (stopped.status === "incomplete") return new ProviderCleanupError( this.identity.id, - [ - pending.process.launch.ownership?.runId ?? "owned-decoder", - ...(pending.root === undefined ? [] : [pending.root.path]), - // The retained group may still be live; name it as OwnedCommand does. - ...(pending.process.launch.ownership === undefined - ? [] - : [ - `pid:${pending.process.launch.ownership.leaderPid}`, - `process-group:${pending.process.launch.ownership.processGroupId}`, - ]), - ], + ownership === undefined + ? [ + "owned-decoder", + ...(pending.root === undefined ? [] : [pending.root.path]), + ] + : ownedCommandResources(ownership, pending.root?.path), { reason: stopped.reason, captured_output: { diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts index 02fb4defc..d375971ae 100644 --- a/src/process/OwnedCommand.ts +++ b/src/process/OwnedCommand.ts @@ -9,6 +9,7 @@ import { } from "./ProviderProcess.js"; import { cleanupOwnedProcessGroup, + type OwnedProcessGroup, type ProcessCleanupFailure, } from "./ProcessOwnership.js"; import { @@ -34,6 +35,17 @@ export class OwnedCommandFailure extends Error { } } +/** Report recorded launch identities; these labels do not authorize signaling. */ +export const ownedCommandResources = ( + ownership: OwnedProcessGroup, + directory?: string, +): readonly string[] => [ + ownership.runId, + ...(directory === undefined ? [] : [directory]), + `pid:${ownership.leaderPid}`, + `process-group:${ownership.processGroupId}`, +]; + /** Execute a single owned process with independent cleanup and bounded diagnostic retention. */ export const runOwnedCommand = async ( spawn: OwnedProviderProcessSpawnOptions, @@ -208,12 +220,7 @@ const collectOwnedCommand = async ( ? {} : { cleanupResult: stopped.cleanupResult }), }, - [ - spawn.runId, - ...(spawn.cwd === undefined ? [] : [spawn.cwd]), - `pid:${launched.ownership.leaderPid}`, - `process-group:${launched.ownership.processGroupId}`, - ], + ownedCommandResources(launched.ownership, spawn.cwd), supervisor, ); if (failure !== undefined) diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts index 34205af3d..ca0cb2db6 100644 --- a/tests/boundary/process/ownedCommand.test.ts +++ b/tests/boundary/process/ownedCommand.test.ts @@ -257,9 +257,11 @@ it.skipIf(process.platform === "win32").each(["cancelled", "timeout"] as const)( failures: [expect.objectContaining({ reason: "run-token-mismatch" })], }); expect(failure.cleanupResult).toEqual(cleanup); - expect(failure.resources).toEqual( - expect.arrayContaining([request.runId]), - ); + expect(failure.resources).toEqual([ + request.runId, + `pid:${launched.ownership.leaderPid}`, + `process-group:${launched.ownership.processGroupId}`, + ]); // An unresolved member does not grant authority to signal the shared group. const acquiredPid = launched.ownership.leaderPid; expect(() => process.kill(acquiredPid, 0)).not.toThrow(); diff --git a/tests/process-global/pwntoolsStageRetirement.test.ts b/tests/process-global/pwntoolsStageRetirement.test.ts index 6ec062535..7cdf0b48a 100644 --- a/tests/process-global/pwntoolsStageRetirement.test.ts +++ b/tests/process-global/pwntoolsStageRetirement.test.ts @@ -18,7 +18,10 @@ import { PWNTOOLS_MEMORY_FAILURE_EXIT, PWNTOOLS_PROVIDER_IDENTITY, } from "../../src/native/pwntools/PwntoolsRelease.js"; -import { cleanupOwnedProcessGroup } from "../../src/process/ProcessOwnership.js"; +import { + cleanupOwnedProcessGroup, + type OwnedProcessGroup, +} from "../../src/process/ProcessOwnership.js"; import { prepareProcessOwnershipInspection } from "../../src/process/ProcessOwnershipObservation.js"; import { spawnOwnedProviderProcess } from "../../src/process/ProviderProcess.js"; import { PrivateRuntimeRoot } from "../../src/process/PrivateRuntimeRoot.js"; @@ -232,12 +235,13 @@ it.runIf(supportedHost).each([ const { path, root, environment, basic } = await fixture(); let launches = 0; let cleanupAllowed = false; + let retainedOwnership: OwnedProcessGroup | undefined; const read = vi.spyOn(artifacts, "readStableArtifact"); const provider = new PwntoolsRecordedCrashProvider( environment, async (spawn) => { if (stage === "debugger" && ++launches === 1) return basic(spawn); - return changingWorker( + const launched = await changingWorker( spawn, join(spawn.cwd ?? "", "resource.failure"), { @@ -247,6 +251,8 @@ it.runIf(supportedHost).each([ }, () => cleanupAllowed, ); + retainedOwnership = launched.ownership; + return launched; }, async () => root, ); @@ -261,14 +267,14 @@ it.runIf(supportedHost).each([ if (result.ok) throw new Error("Expected unresolved process ownership"); const projected = projectAnalysisError(result.error); expect(projected.code).toBe("cleanup_incomplete"); - expect(projected).toMatchObject({ - details: { - resources: expect.arrayContaining([ - expect.stringMatching(/^pid:\d+$/u), - expect.stringMatching(/^process-group:\d+$/u), - ]), - }, - }); + if (retainedOwnership === undefined) + throw new Error("Expected retained process ownership"); + expect(projected.details?.resources).toEqual([ + retainedOwnership.runId, + root.path, + `pid:${retainedOwnership.leaderPid}`, + `process-group:${retainedOwnership.processGroupId}`, + ]); if (stage === "debugger") expect(projected).toMatchObject({ details: { From 2f3dee3fcf65813d6daf7c3209d921451c160ddb Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:37:48 +0800 Subject: [PATCH 04/11] test(javascript): verify Windows path case and load scope (#2635) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select the Windows curated lane for edits to the shared runtime path matcher and its workflow suite. Strengthen the existing POSIX/Windows mapping controls with matching basis, coherent target/script coordinates, caller spelling and static load-state assertions. Follow-up to #2422; thanks to @Aurantium-47 for identifying the recorded-path grammar issue. Verified with actual Windows 11 x64, official Node 24.18.0 and production CLI/MCP workflows using published npm 11.16.0 module Evidence. Four exact/mismatched implicit/explicit mapping cases pass with unchanged normalized hashes. The capture provider was Node Inspector; no IDA, Ghidra or Electron GUI verification is claimed. 230 focused tests, all eight CI static tasks, architecture guards and documentation checks pass after rebasing onto main fb28ec153. The unrelated release-checkpoint and dyld fixture corrections were integrated from upstream; no patches to those files are included here. Final Windows MCP replay at e6e86946d preserves all four normalized hashes, with no owned Node processes remaining. Resource probes ran serially at BelowNormal priority with one CPU and 512 MiB Node heaps; calls took 251–620 ms and the sampled process-family working-set sum peaked at 499,380,224 bytes. Source pins, provenance and measurement limits are recorded in experiments/analysis-efficiency/windows-posix-reconciliation.md. All 34 jobs in current-head CI run 38391921571 passed, including the Windows capability/package lane, all six portable test shards, JavaScript recovery and the required aggregate. Current-head Codex and Greptile reviews are clean; both prior scope-selection findings were fixed and resolved. --- .github/workflows/ci.yml | 1 + .../windows-posix-reconciliation.md | 128 +++++++++++++++ scripts/ci/scopes.mjs | 2 + .../boundary/filesystem/ciChangeScope.test.ts | 4 + .../javascriptApplicationWorkflows.test.ts | 154 ++++++++++-------- 5 files changed, 224 insertions(+), 65 deletions(-) create mode 100644 experiments/analysis-efficiency/windows-posix-reconciliation.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0516da85..d2cc59493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -597,6 +597,7 @@ jobs: tests/boundary/filesystem/setupUnavailableProvider.test.ts tests/boundary/filesystem/windowsNpxRegistration.test.ts tests/acceptance/setup/homeDirectoryFallback.test.ts + tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts tests/boundary/filesystem/webScriptManifestPaths.test.ts tests/boundary/process/ProcessEvidence.host.test.ts tests/acceptance/webModuleCli.test.ts diff --git a/experiments/analysis-efficiency/windows-posix-reconciliation.md b/experiments/analysis-efficiency/windows-posix-reconciliation.md new file mode 100644 index 000000000..0df06ac3e --- /dev/null +++ b/experiments/analysis-efficiency/windows-posix-reconciliation.md @@ -0,0 +1,128 @@ +# Windows reconciliation of recorded POSIX paths + +Follow-up to [#2422](https://github.com/morluto/rea/pull/2422), including +@Aurantium-47's diagnosis of the host-dependent relative-path operation. + +## Source pins and real data + +The initial reproduction and passive producer used main +fa4b8288655188426222773ead2ccbdaf76334fd. Before branching, main was updated +to 8b181716aca2cb467be499cba226abfb5ae19141. The owning matcher and its +affected reconciliation consumers were unchanged between those revisions. +The newer baseline again reproduced both errors on the real Windows host. + +Input: npm 11.16.0's published lib/utils/validate-lockfile.js, 1,023 bytes, +SHA-256 2295e2dac1c5816e1eae2d417804960ebbef17b64a0a44db4f3483ce6e6d9526. +The installed copy was independently checked against the exact module in the +[official registry package](https://registry.npmjs.org/npm/11.16.0). +The 2,945,850-byte tarball matched its published SHA-512 SRI before reading +that member; no archive extraction or installation occurred, and the tarball +was not retained. + +Two actual Linux filesystem directories, Runtime/App and Runtime/app, +held byte-identical module copies and had distinct directory identities. +A controlled Node 24.18.0 process loaded the public module and ran its real +function against 587 locked dependency records: compatible trees returned zero +errors, and one controlled version mismatch returned one error. A passive REA +Inspector capture recorded 114 loaded scripts, including the selected module. +The provider did not evaluate, pause, resume or read source. The owned process +exited normally. Function execution is established by the harness receipt; +REA's passive script metadata establishes loaded status. + +Production static analysis inspected both directories plus an independent +Static/public-module copy. The latter prevents implicit root matching from +hiding the explicit mapping result. Complete static/runtime Evidence was then +consumed by the actual **Windows** CLI and SDK 2.3.1 stdio MCP server, using +Windows 11 10.0.22631.0 and official Windows x64 Node 24.18.0. +No Electron GUI, IDA or Ghidra engine was used. + +## Correctness and provenance + +| Windows reconciliation | Baseline | Candidate | +| -------------------------------------- | ------------------------------ | ----------------------------------- | +| Exact POSIX input root | Matched, artifact path | Same complete normalized result | +| Case-mismatched POSIX input root | Incorrectly matched and loaded | Unknown; static load states unknown | +| Exact explicit POSIX mapping | Matched, operator file mapping | Same complete normalized result | +| Case-mismatched explicit POSIX mapping | Incorrectly matched and loaded | Unknown; static load states unknown | + +All four CLI and same-connection MCP normalized results agree. The input +Evidence, recorded runtime facts and caller-selected root spellings are +preserved. Rejected location membership does not establish that a module was +absent or unexecuted. The operation remains offline and performs no target or +filesystem inspection. + +The shared matcher now chooses posix.relative after identifying POSIX input; +Windows coordinates continue to use win32.relative. Both location matching +and active-layer scope use this owner. The obsolete host-default import is +removed; no downstream repair or new path abstraction is introduced. + +The related suite passed 102 tests: 15 workflow tests, 85 CI scope tests and +two curated-path checks. Four simulated Electron Evidence controls cover +POSIX mismatch/exact matching, Windows case-insensitive matching, the resulting +asset load state and retained mapping spelling. They are service regressions, +not real Electron-provider verification. The existing workflow test file is +registered in both the Windows execution lane and its package-change scope. + +## Delivery and resource observations + +Probes ran serially at BelowNormal priority with one CPU and 512 MiB Node heap +settings for CLI/server/client work. Heap settings do not bound aggregate memory. +Four corrected CLI exit-time observations reported 7.343–15.827 seconds CPU and +322,308–324,280 KiB peak working set, excluding the probe parent. Wall times +ranged from 7.785 to 17.601 seconds. There is no paired resource baseline. + +One complete MCP trial connected in 18,296 ms and discovered 189 tools in +875 ms (2,588,336 serialized bytes). The four subsequent same-connection calls +took 879, 320, 353 and 422 ms. Their serialized replies were 918,133 / 913,765 / +918,571 / 914,171 bytes; inputs totalled 681,312 bytes. Discovery plus these +inputs and replies totalled 6,934,288 bytes, excluding initialization, framing +and notifications. This is delivery volume, not model token usage. Actual +model context and total model task cost were not measured. + +A process-generation sampler recorded 233 observations over 25,671 ms: +15,797 ms aggregate observed CPU as a lower bound and a peak sampled working-set +sum of 491,597,824 bytes (about 469 MiB). Exited-process tails can be missed; +shared pages can be counted twice. This excludes the observer and is not an +exact physical-memory peak. The server wrote four structured completion records +to stderr; tool replies and normalized results passed the recorded checks. + +Cold/warm caches and observer differences were not normalized. No latency, +CPU, memory or model-token improvement is claimed. The fix adds no lookup I/O, +engine invocation, retry or cache. Complete default replies remain available; +only compact selected findings and hashes were sent to the investigating agent. + +## Main integration replay + +Runtime cb0d8a9fef2b907cff2ba27f1e017493e1173eb2, rebased onto main +73dd8d6ed, again passed the 101 related tests and all four real Windows CLI/MCP +cases using the unchanged older producer Evidence. Positive results and original +runtime/layer facts remain identical. Typecheck, lint, dependency and metadata +checks passed; all five changed files passed formatting independently. +The full format gate at that main revision failed on the unchanged upstream +GhidraDoctor.test.ts and GhidraSnapshotCache.test.ts files. The PR does not +repair those unrelated baseline files. Resource figures above refer to the +earlier candidate; the integration replay does not establish a speedup. + +Review found that owning only the Windows test path would still skip that lane +on a later implementation-only edit. The matcher implementation is now also +registered in package scope. A source-only classifier case verifies that it +selects both Windows package and JavaScript recovery lanes; 85 scope checks and +two curated-path checks pass. Runtime code and the real-data checks are unchanged. + +The original production fix merged as +[1a1b67c88](https://github.com/morluto/rea/commit/1a1b67c88) while this follow-up +was open. The follow-up was then rebased onto main fb28ec153 and contains only +the stronger workflow assertions, Windows lane ownership, scope classification +and this verification record. The updated 131 related tests, two representative +Ghidra boundary tests, seven IDA input-thinning tests, 24 semantic-structure +tests, 16 required-lane checks, 39 release-checkpoint tests and 11 dyld CLI +tests passed after the rebase (230 total). +Upstream a0317cf73 corrected the invalid FAT fixture and consolidated unrelated +duplicated helpers; e05d3399f aligned the older parameter-provenance assertion +with the upstream semantics. Those changes, the threshold change and the new +CI qualification tiers are part of main, outside this follow-up's diff. +Upstream 747e72991 restored release preflight coverage for reusable workflow +jobs. The independently reviewed, fully green #2755 fixture correction merged +as fb28ec153 and restored real x64 dyld CLI admission without altering the +original same-named arm64e assertions. +No production source is duplicated by the follow-up. diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index a8679726c..ffe15cb3d 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -56,6 +56,8 @@ export const SCOPES = { "tests/boundary/filesystem/setupUnavailableProvider.test.ts", "tests/boundary/filesystem/windowsNpxRegistration.test.ts", "tests/acceptance/setup/homeDirectoryFallback.test.ts", + "tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts", + "src/domain/javascript/javascriptRuntimeStaticCandidates.ts", "tests/boundary/filesystem/webScriptManifestPaths.test.ts", "tests/boundary/process/ProcessEvidence.host.test.ts", "tests/acceptance/webModuleCli.test.ts", diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 163219c39..321160a36 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -515,6 +515,10 @@ it.each([ "src/windows/Example.ts", { code: true, runtime: true, package: true, apple: false, evm: false }, ], + [ + "src/domain/javascript/javascriptRuntimeStaticCandidates.ts", + { code: true, runtime: true, package: true, javascript_recovery: true }, + ], ["install.sh", { code: true, runtime: true, package: true, evm: false }], [ ".agents/skills/reverse-engineer-anything/SKILL.md", diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index b79407821..8393bad54 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -444,73 +444,97 @@ describe("JavaScript application workflow error diagnostics", () => { }); }); +const RUNTIME_PATH_CASES = [ + { + name: "case-sensitive POSIX input root", + root: null, + path: "/applications/Example.app/Contents/Resources/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "case-sensitive POSIX mapping root", + root: "/runtime/App", + path: "/runtime/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "matching POSIX mapping root", + root: "/runtime/App", + path: "/runtime/App/renderer.js", + status: "matched", + basis: "operator-file-mapping", + }, + { + name: "case-insensitive Windows mapping root", + root: String.raw`C:\runtime\App`, + path: String.raw`c:\runtime\app\renderer.js`, + status: "matched", + basis: "operator-file-mapping", + }, +] as const; + describe("JavaScript runtime reconciliation local paths", () => { - it.each([ - { - name: "case-sensitive POSIX input root", - root: null, - path: "/applications/Example.app/Contents/Resources/app/renderer.js", - status: "unknown", - }, - { - name: "case-sensitive POSIX mapping root", - root: "/runtime/App", - path: "/runtime/app/renderer.js", - status: "unknown", - }, - { - name: "matching POSIX mapping root", - root: "/runtime/App", - path: "/runtime/App/renderer.js", - status: "matched", - }, - { - name: "case-insensitive Windows mapping root", - root: String.raw`C:\runtime\App`, - path: String.raw`c:\runtime\app\renderer.js`, - status: "matched", - }, - ])("uses $name on every host", ({ root, path, status }) => { - const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; - const original = base.runtime_observations[0]!; - const inspection = electronPageInspectionSchema.parse( - original.normalized_result, - ); - const runtime = createEvidence(undefined, original.provider, { - predicateType: original.predicate_type, - operation: original.operation, - parameters: { ...original.parameters, include_script_sources: false }, - result: { - ...inspection, - scripts: { - ...inspection.scripts, - items: inspection.scripts.items.map((script) => ({ - ...script, - file_path: path, - source: { included: false, reason: "not-requested" }, - })), + it.each(RUNTIME_PATH_CASES)( + "uses $name on every host", + ({ root, path, status, basis }) => { + const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; + const original = base.runtime_observations[0]!; + const inspection = electronPageInspectionSchema.parse( + original.normalized_result, + ); + // Keep target and script coordinates consistent so an unrelated target + // cannot activate the static layer after its script mapping was rejected. + const runtime = createEvidence(undefined, original.provider, { + predicateType: original.predicate_type, + operation: original.operation, + parameters: { ...original.parameters, include_script_sources: false }, + result: { + ...inspection, + target: { ...inspection.target, file_path: path }, + scripts: { + ...inspection.scripts, + items: inspection.scripts.items.map((script) => ({ + ...script, + file_path: path, + source: { included: false, reason: "not-requested" }, + })), + }, }, - }, - confidence: original.confidence, - authority: original.authority, - }); - const reconciled = reconcileJavaScriptRuntimeEvidence({ - static_layers: base.static_layers.map((layer) => ({ - ...layer, - runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], - })), - runtime_observations: [runtime], - }); - if (!reconciled.ok) throw reconciled.error; - const result = javascriptRuntimeReconciliationResultSchema.parse( - reconciled.value.normalized_result, - ); - expect( - result.reconciliations.find( - ({ entity_kind }) => entity_kind === "script", - ), - ).toMatchObject({ status }); - }); + confidence: original.confidence, + authority: original.authority, + }); + const reconciled = reconcileJavaScriptRuntimeEvidence({ + static_layers: base.static_layers.map((layer) => ({ + ...layer, + runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], + })), + runtime_observations: [runtime], + }); + if (!reconciled.ok) throw reconciled.error; + const result = javascriptRuntimeReconciliationResultSchema.parse( + reconciled.value.normalized_result, + ); + expect( + result.reconciliations.find( + ({ entity_kind }) => entity_kind === "script", + ), + ).toMatchObject({ status, basis }); + expect( + result.static_load_states.find( + ({ kind }) => kind === "javascript-asset", + ), + ).toMatchObject( + status === "matched" + ? { status: "loaded", reason: "runtime-script-correspondence" } + : { status: "unknown", reason: "layer-outside-runtime-scope" }, + ); + expect(result.static_layers[0]?.runtime_mappings).toEqual( + root === null ? [] : [{ kind: "file-root", root, artifact_prefix: "" }], + ); + }, + ); it("resolves relative file mapping roots before deriving result Evidence", () => { const absoluteRoot = resolve(process.cwd(), "runtime-cache-fixture"); From 8823c55b0ce325cf9edb7d2f9aadaf39db4e8f31 Mon Sep 17 00:00:00 2001 From: Omisakin Joshua <117924167+thefolahan@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:41:10 +0100 Subject: [PATCH 05/11] feat(ghidra): decode Swift String literals in function dossiers (#2049) * feat(ghidra): decode Swift String literals in function dossiers Swift passes a String as two words. Small literals keep their UTF-8 bytes inside those words, and immortal large literals keep a count plus a pointer to their bytes biased by 32. Ghidra creates no data reference for either, so analyze_function reported no strings for Swift code. On AArch64 programs that contain Swift metadata, the Ghidra bridge now runs constant propagation over the function and decodes register pairs where Swift consumes a String: consecutive argument registers at calls and tail calls, x0 and x1 at returns, and both registers of one STP. Large literals must resolve to terminated bytes inside one initialized block that is not executable. A pair whose registers both come from conditional selects is skipped, because Ghidra forks each select without correlating the condition and could otherwise pair words from different cases. Decoded values are a derivation from the ABI layout rather than observed references, so they appear in a new optional swift_string_literals field with their source instruction, registers and raw words, and in a matching inspect_analysis_view facet. referenced_strings is unchanged. Refs #2030 * fix(ghidra): bound Swift literal reads and compare decoded strings Large immortal literals are measured against the remaining heap before allocation, and a failed fit stays visible as a limitation. The canonical empty string is kept. Small and large literals now require matching address nullability, and function comparison treats decoded literals as string evidence without equating a missing record to an empty one. * fix(ghidra): budget Swift literal decoding and tighten use sites Large literals now draw on one heap budget per dossier, measured once before decoding, instead of a separate check per literal. A literal that does not fit is listed in a new swift_string_literal_omissions field with its address, byte length and reason, so consumers read omissions from structured data rather than from limitation text. strings_names comparison checks referenced strings and names first and reports a change there even when Swift literal coverage differs; it returns unknown only when those facts match and literal coverage is absent on one side or partial. A computed jump counts as a tail call only when it has no resolved target inside the function, so jump tables no longer qualify, and a return requires a RETURN p-code operation, so traps no longer qualify. Decoded literals keep control characters, which Swift strings may contain and JSON escapes. The Swift verifier builds the literal fixture for arm64 on any Mac host and now also covers the empty string and a literal containing a tab. * fix(ghidra): charge escaped JSON and follow copied selects The Swift literal budget now charges the worst case per UTF-8 byte: UTF-16 text, six escaped JSON characters held in Gson's growing buffer and in the returned String, and the UTF-8 bytes written to the socket. Literals made of control characters can no longer exceed the reserved heap. A jump counts as a tail call only when every resolved target lies outside the function. The conditional select check follows register copies back to their definition, so selects copied into argument registers are recognized like selects written there directly. strings_names reports a change when both dossiers decoded different literals at the same use site, even when either side has omissions; it returns unknown only when no such use site disagrees. * fix(ghidra): bound literal records and trace selects across branches Every Swift literal record now pays a fixed cost against the dossier budget in addition to its text, so many empty or small literals are bounded as well. The first record that does not fit is kept as an omission and decoding stops there, which keeps the omission record itself bounded; omissions now describe small or large literals. The conditional select check searches every predecessor path for the reaching definitions, following register copies, so selects made before a branch into the use block are recognized. A dossier records Swift literals and their omissions together. Coverage reads omissions first, and strings_names compares every use site that neither side omitted: a decoded literal is known where it was recorded, and an absence is known only in a dossier whose decoding examined every use site. * fix(ghidra): stop every flow at the literal budget and route omissions Ending one flow from the evaluator leaves other queued flows running, so the budget now records a shared stop that every flow checks before it examines another instruction. A record that exceeds the budget is routed by that capacity result rather than by whether text was decoded, so a small literal over budget becomes an omission instead of a literal. When coverage is partial, the per site comparison keeps each literal's representation as well as its value, matching the full strings_names projection. * fix(ghidra): keep unresolved Swift tail-call sites unknown --------- Co-authored-by: Cursor Agent Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- bridge/ghidra/ReaGhidraBridge.java | 380 ++++++++++++++++++ scripts/verify-real-ghidra-swift.mjs | 68 ++++ src/domain/analysisView/analysisView.test.ts | 20 +- src/domain/analysisView/analysisView.ts | 2 + src/domain/analysisView/nativeFunctionView.ts | 5 +- src/domain/functionComparison.test.ts | 230 +++++++++++ src/domain/functionComparisonDimensions.ts | 42 +- src/domain/functionComparisonNormalization.ts | 79 ++++ src/domain/hopperValues.ts | 76 ++++ src/ghidra/GhidraFunctionValues.test.ts | 98 +++++ src/ghidra/GhidraFunctionValues.ts | 17 + tests/conformance/swift/string-literals.swift | 23 ++ 12 files changed, 1035 insertions(+), 5 deletions(-) create mode 100644 tests/conformance/swift/string-literals.swift diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 02fc0f148..70132eeae 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -13,6 +13,8 @@ import java.net.UnixDomainSocketAddress; import java.nio.channels.Channels; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; @@ -22,7 +24,9 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermissions; import java.security.MessageDigest; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; import java.util.IdentityHashMap; @@ -82,6 +86,7 @@ import ghidra.program.model.address.SegmentedAddress; import ghidra.program.model.reloc.Relocation; import java.io.ByteArrayOutputStream; import ghidra.program.model.symbol.Namespace; +import ghidra.program.model.symbol.FlowType; import ghidra.program.model.symbol.RefType; import ghidra.program.model.symbol.Reference; import ghidra.program.model.symbol.ReferenceIterator; @@ -100,6 +105,9 @@ import ghidra.program.model.pcode.PcodeBlock; import ghidra.program.model.pcode.PcodeBlockBasic; import ghidra.program.model.pcode.Varnode; import ghidra.program.model.pcode.VarnodeAST; +import ghidra.program.util.ContextEvaluatorAdapter; +import ghidra.program.util.SymbolicPropogator; +import ghidra.program.util.VarnodeContext; public final class ReaGhidraBridge extends HeadlessScript { private static final int MAX_HIGH_PCODE_OPS = 3000; @@ -1319,6 +1327,13 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("outgoing_references", outgoing); result.add("referenced_strings", referencedStrings); result.add("referenced_names", referencedNames); + boolean swift = swiftProgram(); + boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:"); + SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null; + if (swiftLiterals != null) { + result.add("swift_string_literals", swiftLiterals.literals()); + result.add("swift_string_literal_omissions", swiftLiterals.omissions()); + } result.add("basic_blocks", blocks); result.add( "native_api", @@ -1338,6 +1353,20 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add( "Synthetic Ghidra entry-point references without actionable memory sources are omitted." ); + if (swift && aarch64) { + limitations.add( + "Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by conditional selects, passed through unresolved jumps, or built at runtime are not reported." + ); + if (swiftLiterals.omissions().size() > 0) { + limitations.add( + "Swift String literal records listed in swift_string_literal_omissions could not be retained or decoded within this dossier's remaining heap budget; later symbolic flows remain unexamined." + ); + } + } else if (swift) { + limitations.add( + "Swift String literal decoding is implemented for AArch64 register pairs only; no Swift literals were decoded for this architecture." + ); + } List boundaryLimitations = noReturnCallLimitations(scan.instructions); for (String limitation : boundaryLimitations) { limitations.add(limitation); @@ -3391,6 +3420,349 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } + private boolean swiftProgram() { + for (MemoryBlock block : currentProgram.getMemory().getBlocks()) { + String name = block.getName(); + if (name.startsWith("__swift5_") || name.startsWith("swift5_")) return true; + } + return false; + } + + /** + * Swift passes a String as two words: small literals carry their bytes in the words and + * immortal large literals carry their count and a biased pointer to the bytes. Ghidra keeps + * neither as a data reference, so decode constant pairs where Swift consumes them: argument + * and return registers at calls, tail calls and returns, and both registers of one STP. + * The canonical empty string is a small literal with a zero count. + */ + private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception { + Register[] registers = new Register[31]; + for (int index = 0; index < registers.length; index++) { + registers[index] = currentProgram.getRegister("x" + index); + } + TreeMap observed = new TreeMap<>(); + TreeMap omitted = new TreeMap<>(); + SwiftLiteralBudget budget = new SwiftLiteralBudget(); + Map selectedValues = new HashMap<>(); + SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false); + propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() { + @Override + public boolean evaluateContextBefore(VarnodeContext context, Instruction instruction) { + // Returning true ends only the current flow, so every queued flow checks the shared stop. + if (budget.stopped()) return true; + for (Register[] pair : swiftStringPairs(function, instruction, registers)) { + Long word = registerConstant(context, pair[0]); + Long object = registerConstant(context, pair[1]); + if (word == null || object == null) continue; + SwiftLiteralLayout layout = swiftLiteralLayout(word, object); + if (layout == null) continue; + // A selected word can combine with a path-dependent ordinary move. Cache the + // provenance question per site/register, charging retained cache state to the + // same dossier budget before allocating it. + boolean selected = false; + boolean provenanceOverCapacity = false; + for (Register register : pair) { + String provenanceKey = canonicalAddress(instruction.getAddress()) + "\u0000" + + register.getBaseRegister().getName(); + Boolean value = selectedValues.get(provenanceKey); + if (value == null) { + if (!budget.takeState()) { + provenanceOverCapacity = true; + break; + } + value = selectedValue(function, instruction, register); + selectedValues.put(provenanceKey, value); + } + if (value) { + selected = true; + break; + } + } + if (selected) continue; + // Forks can reach one site with different constants, so the words are part of the key. + String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + + "\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) + + "\u0000" + Long.toUnsignedString(object, 16); + if (observed.containsKey(key) || omitted.containsKey(key)) continue; + // Every record draws on the budget; the first one that does not fit is kept as an + // omission and decoding stops, so later sites are never examined. + String value = layout.smallValue(); + boolean overCapacity = provenanceOverCapacity || !budget.take(layout.byteLength()); + if (!overCapacity && value == null) { + try { + value = swiftLargeLiteralText(layout); + } catch (OutOfMemoryError exhausted) { + overCapacity = true; + } + } + if (value == null && !overCapacity) continue; + JsonObject item = overCapacity + ? swiftStringOmission(layout, word, object) + : swiftStringItem(value, layout, word, object); + item.addProperty("source_address", canonicalAddress(instruction.getAddress())); + JsonArray pairRegisters = new JsonArray(); + pairRegisters.add(pair[0].getName()); + pairRegisters.add(pair[1].getName()); + item.add("registers", pairRegisters); + (overCapacity ? omitted : observed).put(key, item); + if (overCapacity) { + budget.stop(); + return true; + } + } + return false; + } + }, false, monitor); + JsonArray literals = new JsonArray(); + observed.values().forEach(literals::add); + JsonArray omissions = new JsonArray(); + omitted.values().forEach(omissions::add); + return new SwiftStringLiteralDecode(literals, omissions); + } + + private static List swiftStringPairs( + Function function, + Instruction instruction, + Register[] registers + ) { + List pairs = new ArrayList<>(); + if (instruction.getMnemonicString().equals("stp")) { + Register first = instruction.getRegister(0); + Register second = instruction.getRegister(1); + if (first != null && second != null && first.getBitLength() == 64 && second.getBitLength() == 64) { + pairs.add(new Register[] {first, second}); + } + return pairs; + } + FlowType flow = instruction.getFlowType(); + boolean returns = false; + for (PcodeOp operation : instruction.getPcode()) { + if (operation.getOpcode() == PcodeOp.RETURN) returns = true; + } + // A jump is a proven tail call only when it has resolved targets and all leave the + // body. A targetless computed jump can still dispatch inside the function. + Address[] targets = instruction.getFlows(); + boolean tailCall = flow.isJump() && !flow.isConditional() && targets.length > 0 && + Arrays.stream(targets).allMatch(target -> !function.getBody().contains(target)); + if (!flow.isCall() && !returns && !tailCall) return pairs; + // A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned. + int lastFirstRegister = returns ? 0 : 6; + for (int index = 0; index <= lastFirstRegister; index++) { + pairs.add(new Register[] {registers[index], registers[index + 1]}); + } + return pairs; + } + + private static Long registerConstant(VarnodeContext context, Register register) { + Varnode value = context.getRegisterVarnodeValue(register); + return value != null && context.isConstant(value) ? value.getOffset() : null; + } + + // Search every predecessor path for the reaching definitions, following register copies, + // since a select copied into an argument register or defined before a branch forks just as + // independently as one written there directly. + private boolean selectedValue(Function function, Instruction use, Register register) { + ArrayDeque instructions = new ArrayDeque<>(); + ArrayDeque targets = new ArrayDeque<>(); + Set visited = new HashSet<>(); + for (Instruction predecessor : predecessors(function, use)) { + instructions.push(predecessor); + targets.push(register.getBaseRegister()); + } + while (!instructions.isEmpty()) { + Instruction instruction = instructions.pop(); + Register target = targets.pop(); + if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue; + boolean defines = false; + for (Object result : instruction.getResultObjects()) { + if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true; + } + Register next = target; + if (defines) { + PcodeOp[] operations = instruction.getPcode(); + for (PcodeOp operation : operations) { + if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true; + } + Register source = copiedRegister(operations); + if (source == null) continue; + next = source.getBaseRegister(); + } + for (Instruction predecessor : predecessors(function, instruction)) { + instructions.push(predecessor); + targets.push(next); + } + } + return false; + } + + private List predecessors(Function function, Instruction instruction) { + List result = new ArrayList<>(); + Address from = instruction.getFallFrom(); + if (from != null) { + Instruction previous = currentProgram.getListing().getInstructionAt(from); + if (previous != null) result.add(previous); + } + for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) { + if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue; + if (!function.getBody().contains(reference.getFromAddress())) continue; + Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress()); + if (source != null) result.add(source); + } + return result; + } + + private Register copiedRegister(PcodeOp[] operations) { + if (operations.length != 1) return null; + int opcode = operations[0].getOpcode(); + Varnode input = operations[0].getInput(0); + if ((opcode != PcodeOp.COPY && opcode != PcodeOp.INT_ZEXT) || input == null || !input.isRegister()) return null; + return currentProgram.getRegister(input); + } + + /** Validate the ABI layout of a pair; a small literal is decoded here, a large one only located. */ + private SwiftLiteralLayout swiftLiteralLayout(long word, long object) { + int discriminator = (int) (object >>> 60); + if (discriminator == 0xE || discriminator == 0xA) { + int count = (int) ((object >>> 56) & 0xF); + byte[] bytes = new byte[15]; + for (int index = 0; index < 8; index++) bytes[index] = (byte) (word >>> (8 * index)); + for (int index = 0; index < 7; index++) bytes[8 + index] = (byte) (object >>> (8 * index)); + for (int index = count; index < bytes.length; index++) { + if (bytes[index] != 0) return null; + } + // Count zero is the canonical empty string (object 0xE000000000000000, word 0). + String value = swiftStringText(bytes, count, discriminator == 0xE); + return value == null ? null : new SwiftLiteralLayout("small", null, count, true, value); + } + int flags = (int) (word >>> 60); + long count = word & 0xFFFFFFFFFFFFL; + // Immortal literals: ASCII and NFC or neither, tail allocated, with a pointer biased by 32. + if ((object >>> 56) != 0x80 || (flags != 0xD && flags != 0x1) || ((word >>> 48) & 0xFFF) != 0) { + return null; + } + try { + Address address = currentProgram.getAddressFactory().getDefaultAddressSpace() + .getAddress((object & 0x00FFFFFFFFFFFFFFL) + 0x20); + MemoryBlock block = currentProgram.getMemory().getBlock(address); + // The bytes and their terminator must lie in one initialized, non-executable block, + // and Java arrays bound one read. + if (block == null || !block.isInitialized() || block.isExecute() || + count >= block.getEnd().subtract(address) + 1 || count >= Integer.MAX_VALUE) { + return null; + } + return new SwiftLiteralLayout("large", address, count, flags == 0xD, null); + } catch (Exception unaddressable) { + return null; + } + } + + private String swiftLargeLiteralText(SwiftLiteralLayout layout) { + int count = (int) layout.byteLength(); + byte[] bytes = new byte[count + 1]; + try { + if (currentProgram.getMemory().getBytes(layout.address(), bytes) != bytes.length) return null; + } catch (Exception unreadable) { + return null; + } + return bytes[count] == 0 ? swiftStringText(bytes, count, layout.ascii()) : null; + } + + /** + * Every literal record of one dossier draws on half of the heap that was free when decoding + * began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up + * to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer + * (up to twice its length), again in the returned String, and once more as UTF-8 on the + * socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words + * and their serialized form, so many empty or small literals are bounded too. Retained + * provenance-cache entries draw on that same fixed-state allowance. + */ + private static final class SwiftLiteralBudget { + private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6; + private static final long BYTES_PER_RECORD = 4096; + private long remaining; + private boolean stopped; + + SwiftLiteralBudget() { + Runtime runtime = Runtime.getRuntime(); + remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2); + } + + boolean takeState() { + if (remaining < BYTES_PER_RECORD) return false; + remaining -= BYTES_PER_RECORD; + return true; + } + + boolean take(long bytes) { + if (remaining < BYTES_PER_RECORD || + bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) { + return false; + } + remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE; + return true; + } + + void stop() { + stopped = true; + } + + boolean stopped() { + return stopped; + } + } + + // Swift String values may contain any scalar, including controls; JSON escapes them. + private static String swiftStringText(byte[] bytes, int length, boolean ascii) { + if (length < 0 || length > bytes.length) return null; + for (int index = 0; index < length; index++) { + if (ascii && bytes[index] < 0) return null; + } + try { + return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes, 0, length)).toString(); + } catch (CharacterCodingException invalid) { + return null; + } + } + + private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { + JsonObject item = new JsonObject(); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.addProperty("byte_length", layout.byteLength()); + item.addProperty("reason", "capacity"); + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonObject swiftStringItem( + String value, + SwiftLiteralLayout layout, + long word, + long object + ) { + JsonObject item = new JsonObject(); + item.addProperty("value", value); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonArray swiftStringWords(long word, long object) { + JsonArray words = new JsonArray(); + words.add("0x" + Long.toUnsignedString(word, 16)); + words.add("0x" + Long.toUnsignedString(object, 16)); + return words; + } + private static String exclusiveAddress(Address inclusive) { Address next = inclusive.next(); if (next != null && next.getAddressSpace().equals(inclusive.getAddressSpace())) { @@ -3930,6 +4302,14 @@ public final class ReaGhidraBridge extends HeadlessScript { private record AddressNameRequest(String inputAddress, Address address, String name) {} private record FunctionEntry(Function function, InventoryItem item) {} private record FunctionAnalysisObservation(JsonObject dossier, List limitations) {} + private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {} + private record SwiftLiteralLayout( + String representation, + Address address, + long byteLength, + boolean ascii, + String smallValue + ) {} private record InstructionScan(List instructions, boolean truncated) {} private record SessionDescriptor( String transport, diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 0593408b0..0aa58e2f0 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -45,8 +45,13 @@ const transport = new StdioClientTransport({ const source = fileURLToPath( new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url), ); +const literalSource = fileURLToPath( + new URL("../tests/conformance/swift/string-literals.swift", import.meta.url), +); const binary = join(root, "swift-inventory"); +const literalBinary = join(root, "swift-literals"); const reports = []; +let literalReport; try { await execute("swiftc", ["-O", "-g", source, "-o", binary], { timeout: 60_000, @@ -54,6 +59,19 @@ try { await mkdir(join(root, "copied")); const copied = join(root, "copied", "swift-inventory"); await copyFile(binary, copied); + // Decoding covers AArch64 register pairs, so build that slice on any Mac host. + await execute( + "swiftc", + [ + "-O", + "-target", + "arm64-apple-macosx13.0", + literalSource, + "-o", + literalBinary, + ], + { timeout: 60_000 }, + ); await client.connect(transport); for (const [variant, path] of [ ["debug", binary], @@ -132,6 +150,7 @@ try { await call("close_binary", {}); } } + literalReport = await verifyStringLiterals(); } finally { await client.close(); await transport.close(); @@ -143,6 +162,7 @@ console.log( ok: true, provider: "ghidra", fixtures: reports, + swift_string_literals: literalReport, verifier_run: await completeVerifierRun(run), }, null, @@ -159,3 +179,51 @@ async function call(name, arguments_) { toolContract(name).outputSchema.parse(response.structuredContent); return requireMcpOperationResult(response, name); } + +// Literal text lives only in register constants, so it must come from the decoded pairs. +async function verifyStringLiterals() { + await call("open_binary", { path: literalBinary, provider_id: "ghidra" }); + try { + const procedures = await call("list_procedures", {}); + const expected = { + smallLiteral: [["SleepDisabled", "small"]], + largeLiteral: [["The power setting could not be changed.", "large"]], + unicodeLiterals: [ + ["café", "small"], + ["Résumé of the naïve décor, written twice", "large"], + ], + emptyAndControlLiterals: [ + ["", "small"], + ["tab\there", "small"], + ], + returnedLiteral: [["Holding awake", "small"]], + }; + const decoded = {}; + for (const [name, literals] of Object.entries(expected)) { + const procedure = procedures.find(({ value }) => value.includes(name)); + assert(procedure, `${name} is missing from the procedure inventory`); + const dossier = await call("analyze_function", { + procedure: procedure.address, + }); + const observed = dossier.swift_string_literals; + assert(Array.isArray(observed), `${name} recorded no Swift literals`); + assert.deepEqual(dossier.swift_string_literal_omissions, []); + assert.deepEqual( + observed.map(({ value, representation }) => [value, representation]), + literals, + ); + for (const literal of observed) { + assert.deepEqual(literal.registers, ["x0", "x1"]); + assert.equal( + literal.address === null, + literal.representation === "small", + `${literal.value} storage does not match its representation`, + ); + } + decoded[name] = observed.length; + } + return decoded; + } finally { + await call("close_binary", {}); + } +} diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index f1aa3e829..07e6909c3 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -169,7 +169,8 @@ it("pages every retained dossier collection and the native API record", () => { | "comments" | "unresolved_calls" | "referenced_strings" - | "referenced_names", + | "referenced_names" + | "swift_string_literals", result: unknown = original, ) => { const view = projectAnalysisView(nativeParent(result), { @@ -195,6 +196,23 @@ it("pages every retained dossier collection and the native API record", () => { }), ).toMatchObject({ item: [{ text: "entry" }] }); expect(page("referenced_names")).toMatchObject({ coverage: { total: 0 } }); + const literal = { + source_address: "0x401004", + value: "SleepDisabled", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7369447065656c53", "0xed000064656c6261"], + }; + expect( + page("swift_string_literals", { + ...original, + swift_string_literals: [literal], + }), + ).toMatchObject({ item: [literal], coverage: { total: 1 } }); + expect(page("swift_string_literals")).toMatchObject({ + item: { available: false, reason: "not recorded" }, + }); const { unresolved_calls: _omitted, ...withoutUnresolved } = original; expect( page("unresolved_calls", { diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 7f985f349..b08001e35 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -160,6 +160,8 @@ const nativeViewSchema = z.strictObject({ "unresolved_calls", "referenced_strings", "referenced_names", + "swift_string_literals", + "swift_string_literal_omissions", "native_api", "value_flow_summary", "value_flow_operations", diff --git a/src/domain/analysisView/nativeFunctionView.ts b/src/domain/analysisView/nativeFunctionView.ts index f5a29e0e1..b56de24a1 100644 --- a/src/domain/analysisView/nativeFunctionView.ts +++ b/src/domain/analysisView/nativeFunctionView.ts @@ -79,6 +79,8 @@ const nativeRows = ( unresolved_calls: dossier.unresolved_calls, referenced_strings: dossier.referenced_strings, referenced_names: dossier.referenced_names, + swift_string_literals: dossier.swift_string_literals, + swift_string_literal_omissions: dossier.swift_string_literal_omissions, value_flow_operations: flow?.operations, value_flow_def_use: flow?.def_use, value_flow_effects: flow?.effects, @@ -163,7 +165,8 @@ export const projectNativeFunctionView = ( singleton({ available: false, // Only value-flow facets inherit the value-flow failure reason; an - // older dossier simply has no unresolved_calls record. + // older dossier simply has no unresolved_calls record, and a target + // whose provider does not decode Swift literals has no such record. reason: view.facet.startsWith("value_flow_") && flow?.available === false ? flow.reason diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index e03a27748..677fdc0cc 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -716,6 +716,236 @@ describe("function comparison identity and malformed CFG", () => { }); }); +const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => + functionDossierSchema.parse({ + ...dossier("return 0;", base), + swift_string_literals: values.map((value) => ({ + source_address: base, + value, + representation: "small" as const, + address: null, + registers: ["x0", "x1"] as const, + words: ["0x0", "0xe"] as const, + })), + }); +const stringsDimension = ( + leftDigit: string, + left: ReturnType | ReturnType, + rightDigit: string, + right: ReturnType | ReturnType, +) => + compareFunctions( + observe(leftDigit, left), + observe(rightDigit, right), + ).dimensions.find(({ dimension }) => dimension === "strings_names"); + +describe("function comparison Swift string literals", () => { + it("treats differing decoded literals as a strings and names change", () => { + expect( + stringsDimension( + "1", + withLiterals("0x1000", ["SleepDisabled"]), + "2", + withLiterals("0x2000", ["Holding awake"]), + ), + ).toMatchObject({ status: "changed" }); + }); + + it("matches the same decoded literal after the function is relocated", () => { + expect( + stringsDimension( + "3", + withLiterals("0x1000", ["SleepDisabled"]), + "4", + withLiterals("0x2000", ["SleepDisabled"]), + ), + ).toMatchObject({ status: "unchanged" }); + }); + + it("keeps strings and names unknown when only one dossier recorded Swift literals", () => { + expect( + stringsDimension( + "5", + withLiterals("0x1000", []), + "6", + dossier("return 0;", "0x2000"), + ), + ).toMatchObject({ + status: "unknown", + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); + }); + + it("reports a referenced name change even when Swift literal coverage differs", () => { + const named = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + referenced_names: [ + { address: "0x3000", value: "helper", source_address: "0x2000" }, + ], + }); + expect( + stringsDimension("7", withLiterals("0x1000", []), "8", named), + ).toMatchObject({ status: "changed" }); + }); +}); + +describe("function comparison partial Swift literal coverage", () => { + it("does not equate a partial literal record with a complete one", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2000", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "9", + withLiterals("0x1000", ["SleepDisabled"]), + "a", + partial, + ), + ).toMatchObject({ status: "unknown" }); + }); + + it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("d", withLiterals("0x1000", []), "e", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("does not interpret an unexamined path as a missing literal at a retained site", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["A"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 0, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + ], + }); + const complete = withLiterals("0x1000", ["A", "B"]); + expect(stringsDimension("1", complete, "2", partial)).toMatchObject({ + status: "unknown", + }); + expect(stringsDimension("3", partial, "4", complete)).toMatchObject({ + status: "unknown", + }); + expect( + stringsDimension("5", withLiterals("0x1000", []), "6", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("treats an omission record without decoded literals as partial coverage", () => { + const omissionOnly = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly), + ).toMatchObject({ status: "unknown" }); + }); + + it("keeps different retained representations unknown when both dossiers are partial", () => { + const omission = { + source_address: "0x1004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + } as const; + const small = functionDossierSchema.parse({ + ...withLiterals("0x1000", ["SleepDisabled"]), + swift_string_literal_omissions: [omission], + }); + const large = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literals: [ + { + source_address: "0x2000", + value: "SleepDisabled", + representation: "large", + address: "0x3000", + registers: ["x0", "x1"], + words: ["0xd00000000000000d", "0x8000000000002fe0"], + }, + ], + swift_string_literal_omissions: [ + { ...omission, source_address: "0x2004" }, + ], + }); + expect(stringsDimension("1", small, "2", large)).toMatchObject({ + status: "unknown", + }); + }); + + it("keeps different retained literals unknown when both dossiers are partial", () => { + const partial = (base: "0x1000" | "0x2000", value: string) => + functionDossierSchema.parse({ + ...withLiterals(base, [value]), + swift_string_literal_omissions: [ + { + source_address: base === "0x1000" ? "0x1004" : "0x2004", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "b", + partial("0x1000", "SleepDisabled"), + "c", + partial("0x2000", "Holding awake"), + ), + ).toMatchObject({ status: "unknown" }); + }); +}); + const expectInlineComparisonEvidence = ( result: ReturnType, ): void => { diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 27bb7b176..c5a0f373f 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -11,6 +11,7 @@ import { referenceProjection, sorted, stringAndNameProjection, + swiftLiteralSitesConflict, } from "./functionComparisonNormalization.js"; import type { DimensionName, @@ -139,8 +140,12 @@ const compareStringsAndNames = ( right: FunctionSnapshot, context: ComparisonContext, ): FunctionDimension => { - const leftValues = stringAndNameProjection(left); - const rightValues = stringAndNameProjection(right); + const complete = + swiftLiteralCoverage(left) === swiftLiteralCoverage(right) && + swiftLiteralCoverage(left) !== "partial"; + // Facts both dossiers record decide a change before Swift literal coverage can. + const leftValues = stringAndNameProjection(left, complete); + const rightValues = stringAndNameProjection(right, complete); if (leftValues === null || rightValues === null) return unresolvedDimension({ dimension: "strings_names", @@ -151,9 +156,40 @@ const compareStringsAndNames = ( "Reference source locations could not be normalized to function offsets.", ], }); - return compareValues("strings_names", leftValues, rightValues, context); + const compared = compareValues( + "strings_names", + leftValues, + rightValues, + context, + ); + if (complete || compared.status === "changed") return compared; + // A retained literal differs only when the opposite dossier establishes its absence. + if (swiftLiteralSitesConflict(left, right)) { + const leftAll = stringAndNameProjection(left); + const rightAll = stringAndNameProjection(right); + if (leftAll !== null && rightAll !== null) + return compareValues("strings_names", leftAll, rightAll, context); + } + return unresolvedDimension({ + dimension: "strings_names", + links: context.links, + leftCount: null, + rightCount: null, + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); }; +const swiftLiteralCoverage = ( + snapshot: FunctionSnapshot, +): "absent" | "complete" | "partial" => + (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 + ? "partial" + : snapshot.dossier.swift_string_literals === undefined + ? "absent" + : "complete"; + const compareReferences = ( left: FunctionSnapshot, right: FunctionSnapshot, diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 5282272e1..f32220b0b 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -142,6 +142,7 @@ export const commentProjection = ( export const stringAndNameProjection = ( snapshot: FunctionSnapshot, + includeSwiftLiterals = true, ): readonly unknown[] | null => { const values = [ ...snapshot.dossier.referenced_strings.map( @@ -164,12 +165,90 @@ export const stringAndNameProjection = ( value, }), ), + ...(includeSwiftLiterals + ? (snapshot.dossier.swift_string_literals ?? []) + : [] + ).map( + ({ + source_address: sourceAddress, + value, + representation, + registers, + }) => ({ + kind: "swift_string", + source_offset: relativeAddress( + sourceAddress, + snapshot.dossier.procedure.address, + ), + value, + representation, + registers, + }), + ), ]; return values.some(({ source_offset: offset }) => offset === null) ? null : sorted(values); }; +/** + * A retained literal proves presence, not that every path through its site was examined. + * A differing retained fact proves a change only when the opposite dossier is complete + * and therefore establishes its absence. + */ +export const swiftLiteralSitesConflict = ( + left: FunctionSnapshot, + right: FunctionSnapshot, +): boolean => { + const leftSites = swiftLiteralSites(left); + const rightSites = swiftLiteralSites(right); + for (const site of new Set([ + ...leftSites.values.keys(), + ...rightSites.values.keys(), + ])) { + const leftValues = + leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined); + const rightValues = + rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined); + if (leftValues === undefined || rightValues === undefined) continue; + const leftFacts = new Set(leftValues); + const rightFacts = new Set(rightValues); + if ( + (rightSites.complete && + leftValues.some((value) => !rightFacts.has(value))) || + (leftSites.complete && rightValues.some((value) => !leftFacts.has(value))) + ) + return true; + } + return false; +}; + +const swiftLiteralSites = (snapshot: FunctionSnapshot) => { + const base = snapshot.dossier.procedure.address; + const omissions = snapshot.dossier.swift_string_literal_omissions ?? []; + const site = (sourceAddress: string, registers: readonly string[]) => { + const offset = relativeAddress(sourceAddress, base); + return offset === null ? null : canonicalJson([offset, registers]); + }; + // Keep the same literal facts the full strings_names projection compares. + const values = new Map(); + for (const literal of snapshot.dossier.swift_string_literals ?? []) { + const key = site(literal.source_address, literal.registers); + if (key === null) continue; + values.set( + key, + [ + ...(values.get(key) ?? []), + canonicalJson([literal.value, literal.representation]), + ].sort(compareUnicodeCodePoints), + ); + } + const complete = + snapshot.dossier.swift_string_literals !== undefined && + omissions.length === 0; + return { values, complete }; +}; + export const sorted = (values: readonly unknown[]): readonly unknown[] => values .map((value) => ({ diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 769d0f040..d6344ff3e 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -290,6 +290,70 @@ export const unresolvedCallSchema = z.strictObject({ reason: z.string(), }); +const swiftStringLiteralFields = { + source_address: z + .string() + .describe( + "Instruction that consumes both words: a call, tail call, return, or paired store.", + ), + value: z + .string() + .describe("Decoded UTF-8 text, including the canonical empty string."), + registers: z.tuple([z.string(), z.string()]), + words: z + .tuple([z.string(), z.string()]) + .describe("Observed constant register values, in register order."), +}; + +/** A Swift String literal decoded from the two words of its ABI value, not a data reference. */ +export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("small") + .describe("The UTF-8 bytes are inside the two words."), + address: z.null().describe("A small literal has no storage of its own."), + }), + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("large") + .describe( + "An immortal literal whose words hold its count and a pointer to its bytes.", + ), + address: z.string().describe("Address of a large literal's bytes."), + }), +]); + +const swiftStringLiteralOmissionFields = { + source_address: swiftStringLiteralFields.source_address, + byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + reason: z + .literal("capacity") + .describe( + "The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.", + ), + registers: swiftStringLiteralFields.registers, + words: swiftStringLiteralFields.words, +}; + +/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */ +export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( + "representation", + [ + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("small"), + address: z.null(), + }), + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), + }), + ], +); + /** Observed endpoints with complete or partial reference classification evidence. */ export const referenceEdgeSchema = z .object({ @@ -341,6 +405,18 @@ export const functionDossierSchema = z }) .strict(), ), + swift_string_literals: z + .array(swiftStringLiteralSchema) + .exactOptional() + .describe( + "Swift String literals decoded from constant register values. Absent when the provider does not decode them for this target.", + ), + swift_string_literal_omissions: z + .array(swiftStringLiteralOmissionSchema) + .exactOptional() + .describe( + "The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.", + ), basic_blocks: z.array( z .object({ diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 75d59c97f..333d2c913 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -195,6 +195,51 @@ describe("Ghidra function-analysis result values", () => { }); }); +describe("Ghidra Swift String literals", () => { + it("keeps a small Swift literal without storage apart from a large literal's bytes", () => { + const dossier = ghidraFunctionDossier(); + if ( + typeof dossier !== "object" || + dossier === null || + Array.isArray(dossier) + ) + throw new TypeError("Ghidra dossier fixture is invalid"); + const literals = [ + { + source_address: "0x401004", + value: "The user name ", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7265737520656854", "0xee0020656d616e20"], + }, + { + source_address: "0x401006", + value: "", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + { + source_address: "0x401008", + value: " cannot be written into a sudoers rule safely.", + representation: "large", + address: "0x402270", + registers: ["x0", "x1"], + words: ["0xd00000000000002e", "0x8000000000402250"], + }, + ]; + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + swift_string_literals: literals, + swift_string_literal_omissions: [], + }), + ).toMatchObject({ ok: true, value: { swift_string_literals: literals } }); + }); +}); + describe("Ghidra compact reference provenance", () => { it("distinguishes incoming references, examined emptiness and unavailable coverage at data addresses", () => { const instruction = { @@ -423,6 +468,34 @@ describe("Ghidra function-analysis malformed results", () => { ); }); +const malformedSwiftStringLiterals = ( + dossier: Record, +): Array<[string, GhidraFunctionOperation, JsonValue]> => + ( + [ + ["non-canonical Swift literal address", "large", "402240"], + ["small Swift literal with storage", "small", "0x402270"], + ["large Swift literal without storage", "large", null], + ] as const + ).map(([name, representation, address]) => [ + name, + "analyze_function", + { + ...dossier, + swift_string_literal_omissions: [], + swift_string_literals: [ + { + source_address: "0x401004", + value: "The power setting could not be changed.", + representation, + address, + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ]); + const malformedOutputs = (): Array< [string, GhidraFunctionOperation, JsonValue] > => { @@ -437,6 +510,11 @@ const malformedOutputs = (): Array< throw new TypeError("Ghidra native API fixture is invalid"); return [ ["non-canonical address", "procedure_callers", ["401000"]], + [ + "Swift literals without their omission record", + "analyze_function", + { ...dossier, swift_string_literals: [] }, + ], [ "missing classification", "procedure_info", @@ -467,6 +545,26 @@ const malformedOutputs = (): Array< Object.entries(dossier).filter(([key]) => key !== "native_api"), ), ], + ...malformedSwiftStringLiterals(dossier), + [ + "non-canonical Swift literal omission address", + "analyze_function", + { + ...dossier, + swift_string_literals: [], + swift_string_literal_omissions: [ + { + source_address: "0x401004", + representation: "large", + address: "402240", + byte_length: 39, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ], [ "non-canonical native API target", "analyze_function", diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index d45e41ba9..4e4c5054a 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -293,6 +293,15 @@ const ghidraFunctionDossier = functionDossierSchema }) .strict() .superRefine((value, context) => { + if ( + (value.swift_string_literals === undefined) !== + (value.swift_string_literal_omissions === undefined) + ) + context.addIssue({ + code: "custom", + message: + "Ghidra dossier must record Swift literals and their omissions together", + }); const addresses = [ value.procedure.address, ...value.comments.map(({ address }) => address), @@ -308,6 +317,14 @@ const ghidraFunctionDossier = functionDossierSchema address, source_address, ]), + ...(value.swift_string_literals ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), + ...(value.swift_string_literal_omissions ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), ...value.basic_blocks.flatMap(({ start, end, successors }) => [ start, end, diff --git a/tests/conformance/swift/string-literals.swift b/tests/conformance/swift/string-literals.swift new file mode 100644 index 000000000..345c4cf74 --- /dev/null +++ b/tests/conformance/swift/string-literals.swift @@ -0,0 +1,23 @@ +@inline(never) +func consume(_ value: String) { print(value) } +@inline(never) +func smallLiteral() { consume("SleepDisabled") } +@inline(never) +func largeLiteral() { consume("The power setting could not be changed.") } +@inline(never) +func unicodeLiterals() { + consume("café") + consume("Résumé of the naïve décor, written twice") +} +@inline(never) +func emptyAndControlLiterals() { + consume("") + consume("tab\there") +} +@inline(never) +func returnedLiteral() -> String { "Holding awake" } +smallLiteral() +largeLiteral() +unicodeLiterals() +emptyAndControlLiterals() +print(returnedLiteral(), CommandLine.arguments.count) From ec4f4ddc588a564b75d69249b4ebbe4d4051c792 Mon Sep 17 00:00:00 2001 From: yasshmittal Date: Tue, 13 Oct 2026 18:11:28 +0530 Subject: [PATCH 06/11] fix(ghidra): read UTF-8 signed language definitions (#2772) Buffer#toString kept a leading UTF-8 byte order mark as U+FEFF, so the strict XML parser rejected a valid .ldefs file as content outside the root element and aborted the whole language catalog read. XML 1.0 section 4.3.3 defines the mark as an encoding signature; decode with TextDecoder, which consumes it. Malformed XML is still rejected. Fixes #2533 Co-authored-by: Claude Opus 5.5 --- src/ghidra/GhidraLanguageCatalog.ts | 4 +++- src/ghidra/GhidraLanguageOverride.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/ghidra/GhidraLanguageCatalog.ts b/src/ghidra/GhidraLanguageCatalog.ts index 5da5943dc..6373f187f 100644 --- a/src/ghidra/GhidraLanguageCatalog.ts +++ b/src/ghidra/GhidraLanguageCatalog.ts @@ -75,8 +75,10 @@ export const readGhidraLanguageCatalog = async ( if (!entry.endsWith(".ldefs")) continue; const bytes = await readFile(join(directory, entry)); if (bytes.length > MAX_LDEFS_BYTES) continue; + // XML 1.0 section 4.3.3: a leading UTF-8 BOM is an encoding signature, + // not document content; TextDecoder consumes it, Buffer#toString does not. definitions.push( - ...parseGhidraLanguageDefinitions(bytes.toString("utf8")), + ...parseGhidraLanguageDefinitions(new TextDecoder().decode(bytes)), ); } } diff --git a/src/ghidra/GhidraLanguageOverride.test.ts b/src/ghidra/GhidraLanguageOverride.test.ts index 5f90a1454..02c5bf245 100644 --- a/src/ghidra/GhidraLanguageOverride.test.ts +++ b/src/ghidra/GhidraLanguageOverride.test.ts @@ -2,6 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { parseConfig } from "../config/parseConfig.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -190,6 +191,28 @@ describe("Ghidra language catalog", () => { ); }); + it("reads a UTF-8 signed definition beside an unsigned sibling", async () => { + const root = await createTestTempDirectory("rea-ghidra-bom-"); + const module = (name: string) => + join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(module("probe"), { recursive: true }); + await mkdir(module("x86"), { recursive: true }); + await writeFile( + join(module("probe"), "probe.ldefs"), + Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + Buffer.from( + '\n', + ), + ]), + ); + await writeFile(join(module("x86"), "x86.ldefs"), X86_LDEFS); + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "probe:LE:32:default", compilerSpecIds: [] }, + ...parseGhidraLanguageDefinitions(X86_LDEFS), + ]); + }); + it("admits only languages and compiler specs the installation defines", () => { const catalog = parseGhidraLanguageDefinitions(X86_LDEFS); expect( From 84cf7791a881e5b16a0288600fbc8d48d494ef01 Mon Sep 17 00:00:00 2001 From: Saim Shafique Date: Tue, 13 Oct 2026 17:42:40 +0500 Subject: [PATCH 07/11] fix(call-path): keep caller-only unknown coverage from blocking not_found (#2371) * fix(call-path): keep caller-only unknown coverage from blocking not_found build_call_path expands only callee edges, so relationship coverage that is unknown solely because callers are unobservable (the modern headless IDA profile) no longer marks the searched callee closure incomplete. The caller caveat stays visible in limitations, and callee-side unknowns still yield unknown. Fixes #2322. * test(call-path): cover IDA-capped callees beside unknown callers Greptile review noted the poll-timing coverage only exercised unknown callees, leaving the partial callees IDA returns at its 500-result cap untested. Add the combined unknown-callers/partial-callees case: isCallerOnlyUnknownCoverage must stay false and buildCallPath must keep search_scope.exhaustive false, so a future change cannot silently report not_found when IDA has omitted callees. --- src/domain/callPath.test.ts | 83 +++++++++++++++++++++++++ src/domain/callPath.ts | 7 ++- src/domain/functionCallCoverage.test.ts | 64 +++++++++++++++++++ src/domain/functionCallCoverage.ts | 22 ++++++- 4 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 src/domain/functionCallCoverage.test.ts diff --git a/src/domain/callPath.test.ts b/src/domain/callPath.test.ts index e0f343fca..e78a5f936 100644 --- a/src/domain/callPath.test.ts +++ b/src/domain/callPath.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildCallPath } from "./callPath.js"; +import { combineFunctionRelationshipCoverage } from "./functionCallCoverage.js"; import { callPathResultSchema, type CallPathInput } from "./callPathSchemas.js"; import { createEvidence, type Evidence } from "./evidence.js"; import type { JsonValue } from "./jsonValue.js"; @@ -396,6 +397,60 @@ describe("explicit call relationship coverage", () => { expect(result.limitations).toEqual([reason]); }); + it("establishes not_found when only caller coverage is unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "not_found", + search_scope: { exhaustive: true }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when callee coverage is unknown", () => { + const reason = + "Callees: Upstream callees capped results at 500; additional callees remain unknown."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when both caller and callee coverage are unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships. Callees: Upstream callees are not observed."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + it("ignores incomplete relationship coverage outside the reachable search closure", () => { const reason = "An unrelated function's callees were not observed."; const result = run([ @@ -415,6 +470,34 @@ describe("explicit call relationship coverage", () => { }); }); +describe("IDA-capped callee coverage beside unknown callers", () => { + it("keeps the search non-exhaustive for the combined coverage value", () => { + const combined = combineFunctionRelationshipCoverage( + { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", + }, + { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], + }, + ); + expect(combined.status).toBe("unknown"); + const result = run([ + observe("0x1000", [], "rea-workflow", { relationshipCoverage: combined }), + observe("0x4000", []), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + shortest_hops: null, + paths: [], + }); + }); +}); + describe("call path evidence admission", () => { it("normalizes hex addresses and rejects duplicates and mixed providers", () => { expect( diff --git a/src/domain/callPath.ts b/src/domain/callPath.ts index 2186c8010..0bd5e88e1 100644 --- a/src/domain/callPath.ts +++ b/src/domain/callPath.ts @@ -10,6 +10,7 @@ import { parseFunctionEvidence, type FunctionSnapshot, } from "./functionDossierEvidence.js"; +import { isCallerOnlyUnknownCoverage } from "./functionCallCoverage.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; /** Minimal directed caller-to-callee adjacency for call-path search. */ @@ -255,10 +256,14 @@ const inspectSearch = ({ // Explicit, Evidence-bound relationship coverage owns completeness; // generic provider/dossier limitations remain contextual. Legacy Evidence // without that metadata retains its conservative limitation-based rule. + // The search only expands callee edges, so coverage that is unknown + // solely because callers are unobservable does not make the searched + // callee closure incomplete; the caller caveat stays in limitations. if ( coverage === null ? snapshot.limitations.length > 0 - : coverage.status !== "complete" + : coverage.status !== "complete" && + !isCallerOnlyUnknownCoverage(coverage) ) incompleteExpandedCoverage = true; } else if (node === goal) { diff --git a/src/domain/functionCallCoverage.test.ts b/src/domain/functionCallCoverage.test.ts new file mode 100644 index 000000000..6fd3fb045 --- /dev/null +++ b/src/domain/functionCallCoverage.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { + combineFunctionRelationshipCoverage, + isCallerOnlyUnknownCoverage, +} from "./functionCallCoverage.js"; +import type { RelationshipCoverage } from "./relationshipCoverage.js"; + +const unknownCallers: RelationshipCoverage = { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", +}; +const unknownCallees: RelationshipCoverage = { + status: "unknown", + reason: "Upstream callees are not observed in this profile.", +}; +const cappedCallees: RelationshipCoverage = { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], +}; +const complete: RelationshipCoverage = { status: "complete" }; + +describe("isCallerOnlyUnknownCoverage", () => { + it("detects unknown callers beside complete callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + complete, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(true); + }); + + it("rejects unknown callees beside complete callers", () => { + const combined = combineFunctionRelationshipCoverage( + complete, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown coverage on both sides", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown callers beside IDA-capped callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + cappedCallees, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects complete coverage", () => { + const combined = combineFunctionRelationshipCoverage(complete, complete); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); +}); diff --git a/src/domain/functionCallCoverage.ts b/src/domain/functionCallCoverage.ts index eb2c05a13..b64e78cd6 100644 --- a/src/domain/functionCallCoverage.ts +++ b/src/domain/functionCallCoverage.ts @@ -2,6 +2,9 @@ import type { RelationshipCoverage } from "./relationshipCoverage.js"; import type { FunctionDossier } from "./hopperValues.js"; import { uniqueSorted } from "./canonicalOrdering.js"; +const callerCoveragePrefix = "Callers: "; +const calleeCoveragePrefix = "Callees: "; + /** Combine caller and callee read outcomes without losing either explanation. */ export const combineFunctionRelationshipCoverage = ( callers: RelationshipCoverage | undefined, @@ -20,7 +23,8 @@ export const combineFunctionRelationshipCoverage = ( ); const details = diagnostics.length > 0 ? { diagnostics } : {}; const reasons = reads.flatMap(({ direction, coverage }) => { - const label = direction === "callers" ? "Callers" : "Callees"; + const label = + direction === "callers" ? callerCoveragePrefix : calleeCoveragePrefix; const limitations = coverage === undefined ? ["Provider did not report relationship coverage."] @@ -29,7 +33,7 @@ export const combineFunctionRelationshipCoverage = ( : coverage.status === "partial" ? coverage.limitations : []; - return limitations.map((reason) => `${label}: ${reason}`); + return limitations.map((reason) => `${label}${reason}`); }); if ( reads.some( @@ -50,6 +54,20 @@ export const combineFunctionRelationshipCoverage = ( return { status: "complete", ...details }; }; +/** + * Whether combined relationship coverage is unknown only on the caller side. + * The combine helper above prefixes each direction's reasons, so a reason + * that carries the caller prefix and no callee prefix records observed + * callees with unobservable callers. Forward (callee-edge) searches stay + * exhaustive under that caveat; the caller reason remains a limitation. + */ +export const isCallerOnlyUnknownCoverage = ( + coverage: RelationshipCoverage, +): boolean => + coverage.status === "unknown" && + coverage.reason.startsWith(callerCoveragePrefix) && + !coverage.reason.includes(calleeCoveragePrefix); + /** Explain why two resolved call sets cannot establish call-set equality. */ export const functionCallCoverageLimitations = ( coverage: RelationshipCoverage | null, From 1ca5fd38b4833ac3fe6213a55e5abfd426228266 Mon Sep 17 00:00:00 2001 From: Jack Liang <130693642+Jackmeson1@users.noreply.github.com> Date: Tue, 13 Oct 2026 05:43:20 -0700 Subject: [PATCH 08/11] fix(cli)!: keep a selected JSON null in --filter-output output (#2333) createCliResultOutput rendered an explicit null selected by --filter-output as the empty document {}, the same output it uses when the selection has no value at all. On the streamed JSON/JSONL path, a consumer could not distinguish a selected null from a missing field. Write a selected null as null in streamed JSON and JSONL. This also preserves an unfiltered top-level null. The {} fallback stays for undefined, functions and symbols. The --full-output envelope already carried data: null for a selected null. The --token-count path still hands results that fit the formatted-string limit back to Incur and prints 0 for a selected null. Other formats and Incur-owned result paths are unchanged. Consumers must accept JSON null for a present null and distinguish it from a missing selection. Use --full-output if an object envelope is required. Keeping the old shared {} representation would continue hiding this distinction. Claude-Session: https://claude.ai/code/session_019TTGAv6fe859ebbCLLfLHP Co-authored-by: Claude Opus 5.5 --- src/cli/cliResultOutput.ts | 5 +++-- tests/boundary/cli/jsonOutput.test.ts | 32 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 67e745b0b..5efde8b33 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -216,8 +216,9 @@ export const createCliResultOutput = ( : filtered === undefined ? { ok: true, meta } : { ok: true, data: filtered, meta } - : filtered === undefined || - filtered === null || + : // A selected null is a value; only a selection with no JSON value + // falls back to the empty document. + filtered === undefined || typeof filtered === "function" || typeof filtered === "symbol" ? {} diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index 5e8e86e7f..0d1ee5a59 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -561,3 +561,35 @@ describe("CLI result output surface", () => { }); }, 60_000); }); + +describe("streamed CLI filtered selections", () => { + it.each([ + ["raw_result", "json", "null\n"], + ["raw_result", "jsonl", "null\n"], + ["nested.value", "json", '{"nested":{"value":null}}\n'], + ["missing", "json", "{}\n"], + ["count", "json", "0\n"], + ["enabled", "json", "false\n"], + ] as const)( + "keeps the selected value of --filter-output %s in %s", + async (filter, format, expected) => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + ["--format", format, "--filter-output", filter], + destination, + ); + if (output === undefined) throw new Error("Missing JSON output surface"); + await output.write( + { raw_result: null, nested: { value: null }, count: 0, enabled: false }, + { command: "fixture", duration: "1ms", format }, + ); + expect(Buffer.concat(chunks).toString("utf8")).toBe(expected); + }, + ); +}); From 36132d3e1770c8b69f86c86821d2c872234e370a Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 13 Oct 2026 08:43:39 -0400 Subject: [PATCH 09/11] fix(managed): classify 32-bit-preferred assemblies as platform neutral (#2342) The CLI header's 32BITREQUIRED and 32BITPREFERRED bits are one two-bit field. Reading 32BITREQUIRED on its own reported Prefer32Bit assemblies (flags 0x00020003) as x86-only with requires_32bit true, and left anycpu-prefer-32 reachable only through the reserved combination. Co-authored-by: Claude Opus 5.5 --- src/dotnet/ManagedArtifactInspector.test.ts | 24 +++++++++++++++++++++ src/dotnet/ManagedArtifactInspector.ts | 7 +++--- src/dotnet/ManagedNativeBoundaryHelpers.ts | 7 ++++-- src/dotnet/ManagedPeReader.ts | 14 ++++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/dotnet/ManagedArtifactInspector.test.ts b/src/dotnet/ManagedArtifactInspector.test.ts index 2d30bf580..3b734c176 100644 --- a/src/dotnet/ManagedArtifactInspector.test.ts +++ b/src/dotnet/ManagedArtifactInspector.test.ts @@ -292,6 +292,30 @@ describe("managed artifact inventory", () => { ]); expect(result.coverage).toMatchObject({ state: "complete", issues: [] }); }); + + // dotnet/runtime corhdr.h: 32BITREQUIRED and 32BITPREFERRED form one + // two-bit field, so neither bit can be read on its own. + it.each([ + ["neither bit", 0x0000_0001, "anycpu", false], + ["32BITREQUIRED alone", 0x0000_0003, "x86", true], + ["both bits", 0x0002_0003, "anycpu-prefer-32", false], + ["the reserved 32BITPREFERRED alone", 0x0002_0001, "anycpu", false], + ] as const)( + "reads %s as one 32-bit requirement", + (_label, cliFlags, architecture, requires32Bit) => { + const bytes = buildManagedPeFixture({ cliFlags }); + const target = managedPeFixtureTarget(bytes); + + expect( + inspectManagedArtifactBytes(bytes, target).classification + .managed_architecture, + ).toBe(architecture); + expect( + inspectManagedNativeBoundariesBytes(bytes, target).cli_native + ?.requires_32bit, + ).toBe(requires32Bit); + }, + ); }); describe("managed artifact coded indexes", () => { diff --git a/src/dotnet/ManagedArtifactInspector.ts b/src/dotnet/ManagedArtifactInspector.ts index 7df8aae83..7e7acb087 100644 --- a/src/dotnet/ManagedArtifactInspector.ts +++ b/src/dotnet/ManagedArtifactInspector.ts @@ -9,6 +9,7 @@ import { type ManagedParseIssue, } from "../domain/managed/managedArtifact.js"; import { + managedCli32BitRequirement, readManagedPeLayout, type ManagedCliHeader, type ManagedPeLayout, @@ -249,9 +250,9 @@ const managedArchitectureFor = ( ): ManagedArtifactInspection["classification"]["managed_architecture"] => { if (layout.architecture !== "x86" || (cli.flags & 1) === 0) return layout.architecture; - if ((cli.flags & 0x0000_0002) !== 0) return "x86"; - if ((cli.flags & 0x0002_0000) !== 0) return "anycpu-prefer-32"; - return "anycpu"; + const requirement = managedCli32BitRequirement(cli.flags); + if (requirement === "required") return "x86"; + return requirement === "preferred" ? "anycpu-prefer-32" : "anycpu"; }; const runtimeClassification = ({ diff --git a/src/dotnet/ManagedNativeBoundaryHelpers.ts b/src/dotnet/ManagedNativeBoundaryHelpers.ts index fb81e1d28..e4b1a5960 100644 --- a/src/dotnet/ManagedNativeBoundaryHelpers.ts +++ b/src/dotnet/ManagedNativeBoundaryHelpers.ts @@ -18,7 +18,10 @@ import { } from "./ManagedMetadataHeaps.js"; import { managedTableRowCounts } from "./ManagedMetadataInventory.js"; import type { ManagedMetadataInventory } from "./ManagedMetadataInventory.js"; -import type { ManagedPeLayout } from "./ManagedPeReader.js"; +import { + managedCli32BitRequirement, + type ManagedPeLayout, +} from "./ManagedPeReader.js"; import { readManagedValue } from "./ManagedReaderFailure.js"; type ModuleRef = ManagedNativeBoundaryInspection["module_refs"][number]; @@ -380,7 +383,7 @@ export const cliNative = ( if (pe.cli === null) return null; return { il_only: (pe.cli.flags & 0x0000_0001) !== 0, - requires_32bit: (pe.cli.flags & 0x0000_0002) !== 0, + requires_32bit: managedCli32BitRequirement(pe.cli.flags) === "required", strong_name_signed: (pe.cli.flags & 0x0000_0008) !== 0, native_entry_point: (pe.cli.flags & 0x0000_0010) !== 0, ready_to_run_signature: pe.cli.readyToRunSignature, diff --git a/src/dotnet/ManagedPeReader.ts b/src/dotnet/ManagedPeReader.ts index 2f9bc3151..9a30c007b 100644 --- a/src/dotnet/ManagedPeReader.ts +++ b/src/dotnet/ManagedPeReader.ts @@ -31,6 +31,20 @@ export interface ManagedCliHeader { readonly readyToRunSignature: boolean; } +/** + * How the CLI header constrains process bitness. 32BITREQUIRED and + * 32BITPREFERRED are one two-bit field: required alone is x86-only, both set + * is platform-neutral code that prefers a 32-bit process, and preferred alone + * is reserved and constrains nothing. + */ +export const managedCli32BitRequirement = ( + flags: number, +): "required" | "preferred" | "none" => { + const pair = flags & 0x0002_0002; + if (pair === 0x0000_0002) return "required"; + return pair === 0x0002_0002 ? "preferred" : "none"; +}; + export interface ManagedPeLayout { readonly machine: number; readonly architecture: "x86" | "x86_64" | "arm" | "arm64"; From d3a55720fc4aac8f9d2c5f5a4c9f20cdf5594d8e Mon Sep 17 00:00:00 2001 From: Chaoqi <136579807+Chaoqi31@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:44:17 -0400 Subject: [PATCH 10/11] fix(setup): keep Hermes config numbers as written (#2368) Setup and uninstall re-printed every number in an existing Hermes config.yaml from a JavaScript number. Hermes reads the file as YAML 1.1 with exact integers, so a Discord channel ID lost its low digits, an octal mode such as 0600 became 600, and 0.0000001 became 1e-7, which YAML 1.1 does not define as a float. Two IDs that round to the same double were also rejected as duplicate keys, so setup could not register at all. Hermes YAML parsing and editing move to HermesConfigurationYaml. A parsed number is written with its source spelling while that spelling still reads as the node's value, the rule yaml's own boolean tag follows, and two keys are duplicates only when both their value and their spelling match. Parsed values stay JavaScript numbers. --- .../ClientConfigurationDocument.ts | 26 ++++--- .../HermesConfigurationYaml.test.ts | 21 ++++++ src/application/HermesConfigurationYaml.ts | 74 +++++++++++++++++++ .../hermesClientConfiguration.test.ts | 32 ++++++++ 4 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 src/application/HermesConfigurationYaml.test.ts create mode 100644 src/application/HermesConfigurationYaml.ts diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index b32df3fd0..1b241a98b 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -1,9 +1,5 @@ import { parse as parseToml, stringify as stringifyToml } from "smol-toml"; -import { - parse as parseYaml, - parseDocument as parseYamlDocument, - stringify as stringifyYaml, -} from "yaml"; +import { stringify as stringifyYaml } from "yaml"; import { isDeepStrictEqual } from "node:util"; import { @@ -12,6 +8,10 @@ import { editDshDocument, dshClientRegistrationEntry, } from "./DshCordisPatch.js"; +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; import { applyEdits, findNodeAtLocation, @@ -118,7 +118,10 @@ const parseDocument = ( if (isEmptyClientConfigurationText(text)) return {}; // Hermes stores MCP servers in `mcp_servers` inside a YAML document. if (format === "hermes") - return requireObject(parseYaml(text), "The configuration root"); + return requireObject( + parseHermesConfigurationYaml(text), + "The configuration root", + ); // ZCode loads nonempty native files with strict JSON.parse. if (format === "zcode") return requireObject(JSON.parse(text), "The configuration root"); @@ -1419,12 +1422,11 @@ const serializeHermesConfiguration = ( typeof servers === "object" && servers !== null ? (servers as Record)[PRODUCT_IDENTITY.mcpServerKey] : undefined; - const yamlDocument = parseYamlDocument(originalText); - if (entry === undefined) - yamlDocument.deleteIn([serversKey, PRODUCT_IDENTITY.mcpServerKey]); - else yamlDocument.setIn([serversKey, PRODUCT_IDENTITY.mcpServerKey], entry); - // Editing REA must not reflow unrelated user prompts or other YAML scalars. - return yamlDocument.toString({ lineWidth: 0 }); + return editHermesConfigurationYaml( + originalText, + [serversKey, PRODUCT_IDENTITY.mcpServerKey], + entry, + ); }; /** Build the stdio entry shape expected by one client's configuration dialect. */ diff --git a/src/application/HermesConfigurationYaml.test.ts b/src/application/HermesConfigurationYaml.test.ts new file mode 100644 index 000000000..41b74fcc9 --- /dev/null +++ b/src/application/HermesConfigurationYaml.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; + +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; + +describe("Hermes configuration YAML", () => { + it("writes an edited number from its new value", () => { + expect( + editHermesConfigurationYaml("mode: 0600\nmask: 0x1F\n", ["mode"], 384), + ).toBe("mode: 384\nmask: 0x1F\n"); + }); + + it("accepts keys Hermes reads as distinct and rejects a repeated key", () => { + expect(() => + parseHermesConfigurationYaml("0600: octal\n600: decimal\n"), + ).not.toThrow(); + expect(() => parseHermesConfigurationYaml('a: 1\n"a": 2\n')).toThrow(); + }); +}); diff --git a/src/application/HermesConfigurationYaml.ts b/src/application/HermesConfigurationYaml.ts new file mode 100644 index 000000000..8d245da37 --- /dev/null +++ b/src/application/HermesConfigurationYaml.ts @@ -0,0 +1,74 @@ +import { + isScalar, + parse, + parseDocument, + type ParsedNode, + type ScalarTag, + type Tags, +} from "yaml"; + +const NUMBER_TAGS = new Set([ + "tag:yaml.org,2002:int", + "tag:yaml.org,2002:float", +]); + +/** + * Write a parsed number with its source spelling while that spelling still + * reads as the node's value. Hermes reads config.yaml as YAML 1.1 with exact + * integers, so re-printing a JavaScript number changes what it reads: a + * Discord channel ID loses its low digits, `0600` (octal) becomes `600`, and + * `0.0000001` becomes `1e-7`, which YAML 1.1 does not define as a float. + */ +const keepNumberSource = (tag: Tags[number]): Tags[number] => { + if (typeof tag === "string" || tag.collection !== undefined) return tag; + const { stringify } = tag; + if (!NUMBER_TAGS.has(tag.tag) || stringify === undefined) return tag; + return { + ...tag, + stringify: (item, context, ...rest) => { + const { source } = item; + if (source === undefined) return stringify(item, context, ...rest); + const reader = context.doc.schema.tags.find( + (candidate): candidate is ScalarTag => + candidate.default === true && candidate.test?.test(source) === true, + ); + const read = reader?.resolve( + source, + () => undefined, + context.doc.options, + ); + return Object.is(isScalar(read) ? read.value : read, item.value) + ? source + : stringify(item, context, ...rest); + }, + }; +}; + +const HERMES_YAML_OPTIONS = { + customTags: (tags: Tags) => tags.map(keepNumberSource), + // Hermes keeps integers exact, so IDs that round to one double, or `0600` + // and `600`, are distinct keys. + uniqueKeys: (a: ParsedNode, b: ParsedNode) => + a === b || + (isScalar(a) && + isScalar(b) && + a.value === b.value && + a.source === b.source), +}; + +/** Parse a Hermes `config.yaml`. */ +export const parseHermesConfigurationYaml = (text: string): unknown => + parse(text, HERMES_YAML_OPTIONS); + +/** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */ +export const editHermesConfigurationYaml = ( + text: string, + path: readonly string[], + value: unknown, +): string => { + const document = parseDocument(text, HERMES_YAML_OPTIONS); + if (value === undefined) document.deleteIn(path); + else document.setIn(path, value); + // Editing REA must not reflow unrelated user prompts or other YAML scalars. + return document.toString({ lineWidth: 0 }); +}; diff --git a/tests/boundary/filesystem/hermesClientConfiguration.test.ts b/tests/boundary/filesystem/hermesClientConfiguration.test.ts index 43a5f841c..218573d5c 100644 --- a/tests/boundary/filesystem/hermesClientConfiguration.test.ts +++ b/tests/boundary/filesystem/hermesClientConfiguration.test.ts @@ -237,6 +237,38 @@ mcp_servers: expect(await readFile(client.configPath, "utf8")).toBe(original); }); + it("keeps unrelated numbers as written, including Discord IDs", async () => { + const { home, client } = await hermesClient(); + // Hermes reads these exactly: two IDs one apart, an octal mode, a float. + const original = `discord: + free_response_channels: + - 1187654321098765432 + channel_prompts: + 1187654321098765433: Build questions. + 1187654321098765434: Release questions. + mode: 0600 + min_score: 0.0000001 +mcp_servers: + other: + command: other-server +`; + await writeFile(client.configPath, original); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text.startsWith(original)).toBe(true); + expect(parseYaml(text, { intAsBigInt: true })).toMatchObject({ + mcp_servers: { + rea: { command: command[0], args: command.slice(1), enabled: true }, + }, + }); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(await readFile(client.configPath, "utf8")).toBe(original); + }); + it("creates config.yaml when Hermes has no configuration yet", async () => { const { client } = await hermesClient(); expect( From f63674cac0329aa3ed76234b31cf8a3536ac07d0 Mon Sep 17 00:00:00 2001 From: Danish Rafie Ekaputra <155859586+0xRafie@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:44:41 +0700 Subject: [PATCH 11/11] docs(browser): clarify attach coverage and Chromium setup (#2287) Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/browser-observation.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 2f73f4261..dca1f994b 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -80,6 +80,27 @@ google-chrome \ REA does not launch, own, or terminate this browser. Use a dedicated profile and stop it yourself when the investigation is complete. Do not expose the debugging port on a non-loopback interface. +Other Chromium-family browsers, such as Chromium or Brave, work the same way +with the same two flags. On macOS, run the executable inside the application +bundle, for example +`"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"` or +`"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"`. Fedora installs +Chromium as `chromium-browser`. + +REA needs the browser's HTTP discovery endpoint on a loopback port. A browser +driven over `--remote-debugging-pipe` or a private channel, for example by an +automation SDK, exposes no such endpoint, so `list_browser_targets` has nothing +to attach to. Connection failures report `endpoint_unreachable`; a reachable +HTTP service that returns a non-200 or non-JSON discovery response reports +`invalid_endpoint_response`. +Reproduce that flow in a Chromium-family browser started with +`--remote-debugging-port`. + +In a minimal Linux container, headed Chromium under Xvfb can keep running while +its DevTools port never opens. Install the GLES, EGL, and Mesa DRI libraries +(on Fedora: `libglvnd-gles`, `mesa-libEGL`, and `mesa-dri-drivers`), then +restart the browser. + On Linux, a reachable but locked desktop keyring can stall Chrome before its first navigation. For a dedicated investigation profile, add `--password-store=basic` to bypass that keyring. Empty or placeholder frame URLs @@ -207,7 +228,7 @@ commands fail closed. A correlated reply's session ID must match the command's selected session. An optional method is unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. -Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. +Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Scripts are different: scripts the page parsed before REA attached are still reported under `scripts`. A page that loaded before attach can therefore list its scripts while `network.requests` lacks the requests that fetched them. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`.