From c95315d2ce365dd7a31aa67c8c13f4a99f209ced Mon Sep 17 00:00:00 2001 From: Liu Jie Date: Wed, 14 Oct 2026 01:04:33 +1300 Subject: [PATCH 01/28] fix(cli): reject unterminated output filter slices (#2315) --- src/cliOutput.ts | 32 +++++++++- tests/boundary/cli/cliOutput.test.ts | 89 ++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/src/cliOutput.ts b/src/cliOutput.ts index 5119eebc3..ccfaf6b62 100644 --- a/src/cliOutput.ts +++ b/src/cliOutput.ts @@ -9,7 +9,7 @@ export type CliOutputArgumentValidation = | { readonly ok: false; readonly format: StructuredOutputFormat; - readonly code: "UNSUPPORTED_OUTPUT_COMBINATION"; + readonly code: "UNSUPPORTED_OUTPUT_COMBINATION" | "INVALID_OUTPUT_FILTER"; readonly message: string; }; @@ -89,13 +89,39 @@ export const parseCliOutputArguments = ( return parsed(false); }; -/** Reject text-window controls that would corrupt a structured document. */ +/** Reject slices that cannot advance within Incur's comma-delimited paths. */ +const hasUnclosedFilterSlice = (expression: string): boolean => { + let depth = 0; + let unclosed = false; + for (const character of expression) { + if (character === "[") { + depth += 1; + unclosed = true; + } else if (character === "]") { + depth -= 1; + // Incur consumes everything through the next closing bracket, including + // nested opening brackets. Do not impose a different filter grammar. + unclosed = false; + } else if (character === "," && depth === 0 && unclosed) return true; + } + return unclosed; +}; + +/** Reject output controls that cannot safely produce a complete result. */ export const validateCliOutputArguments = ( arguments_: readonly string[], ): CliOutputArgumentValidation => { - const { format, parseError, tokenWindow } = + const { filterOutput, format, parseError, tokenWindow } = parseCliOutputArguments(arguments_); if (parseError) return { ok: true }; + if (filterOutput !== undefined && hasUnclosedFilterSlice(filterOutput)) + return { + ok: false, + format: format === "yaml" || format === "jsonl" ? format : "json", + code: "INVALID_OUTPUT_FILTER", + message: + "The --filter-output expression has a slice without a closing ]. Close the slice or remove --filter-output.", + }; if ( tokenWindow && (format === "json" || format === "jsonl" || format === "yaml") diff --git a/tests/boundary/cli/cliOutput.test.ts b/tests/boundary/cli/cliOutput.test.ts index 43d517e16..44b3ac234 100644 --- a/tests/boundary/cli/cliOutput.test.ts +++ b/tests/boundary/cli/cliOutput.test.ts @@ -58,6 +58,95 @@ describe("CLI output argument boundary", () => { }); }); +describe("unclosed CLI output filter slices", () => { + it.each([ + "providers[", + "providers[0,1].name[", + "name,providers[", + "][,]", + "providers],name[0,1]", + ])("rejects %s before the filter parser runs", (filter) => { + expect( + validateCliOutputArguments([ + "providers", + "--json", + "--filter-output", + filter, + ]), + ).toMatchObject({ ok: false, code: "INVALID_OUTPUT_FILTER" }); + }); + + it("preserves effective-filter and earlier flag-error precedence", () => { + for (const arguments_ of [ + [ + "--filter-output", + "providers[", + "--filter-output", + "capabilities[0,1].provider.id,open", + ], + ["--filter-output", "capabilities[-1,].provider.id"], + ["--filter-output", "capabilities[[0,1]"], + ["--format", "invalid", "--filter-output", "providers["], + ["--token-limit", "invalid", "--filter-output", "providers["], + ]) + expect(validateCliOutputArguments(arguments_)).toEqual({ ok: true }); + }); + + workspaceCliTest( + "fails promptly across output formats without invoking the command", + async ({ cli }) => { + for (const format of ["json", "jsonl", "yaml", "toon", "md"]) { + const result = await cli.run({ + arguments: [ + "analyze", + "/absent/target", + "--format", + format, + "--filter-output", + "result[0,1].name[", + ], + timeoutMs: 3000, + }); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("INVALID_OUTPUT_FILTER"); + expect(result.stdout).toContain("closing ]"); + expect(result.stderr).toBe(""); + } + const splitPath = await cli.run({ + arguments: ["providers", "--json", "--filter-output", "][,]"], + timeoutMs: 3000, + }); + expect(splitPath.exitCode).toBe(1); + expect(splitPath.json).toMatchObject({ + ok: false, + error: { code: "INVALID_OUTPUT_FILTER" }, + }); + }, + CLI_INTEGRATION_TIMEOUT_MS, + ); + + workspaceCliTest( + "retains closed slices and multiple filter paths", + async ({ cli }) => { + const result = await cli.run({ + arguments: [ + "providers", + "--json", + "--filter-output", + "capabilities[0,1].provider.id,open", + ], + timeoutMs: 3000, + }); + expect(result.exitCode).toBe(0); + expect(result.json).toEqual({ + capabilities: [{ provider: { id: expect.any(String) } }], + open: false, + }); + }, + CLI_INTEGRATION_TIMEOUT_MS, + ); +}); + describe("CLI output valued-flag parsing", () => { workspaceCliTest( "preserves complete JSON errors when native builtin flags follow unusual arguments", From 1dd6c2dff9252c27838589590e07f809a9b1ff42 Mon Sep 17 00:00:00 2001 From: Guangqi Zhang <2065586050@qq.com> Date: Tue, 13 Oct 2026 20:05:31 +0800 Subject: [PATCH 02/28] fix(setup): resolve DeepSeek Harness home overrides (#2461) --- docs/client-configuration.md | 11 +- src/application/SupportedClients.ts | 23 +++- tests/acceptance/setup/dshClientSetup.test.ts | 107 +++++++++++++++++- .../filesystem/dshClientConfiguration.test.ts | 60 +++++++++- 4 files changed, 190 insertions(+), 11 deletions(-) diff --git a/docs/client-configuration.md b/docs/client-configuration.md index 00e83e179..e72ca191f 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -209,8 +209,15 @@ replaces) the REA entry in the home patch `$DSH_HOME/cordis.patch.yml` patches and comments. `rea doctor --client dsh` checks that same entry id. `rea uninstall` removes it by scanning supported clients; uninstall takes no `--client` flag. The home patch applies to every Harness profile, so the -registration is written once and no per-profile edit is needed. The written -entry is: +registration is written once and no per-profile edit is needed. + +Setup, doctor and uninstall resolve `DSH_HOME` and `DSH_AGENTS_HOME` as Harness +does: unset, empty or whitespace-only values use `~/.dsh` and `~/.agents`, +respectively. Bare `~`, `~/` and `~\` expand to the user home; other relative +values resolve against the command's working directory. Nonblank values retain +their original spaces, and the agents root is independent of the Harness home. +Run REA and Harness from the same working directory when using relative overrides. +The written entry is: ```yaml - insert: diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 488dda49d..5839409d4 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -157,8 +157,27 @@ const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => { const grokDirectory = ({ home, env }: ClientPathContext): string => env.GROK_HOME || join(home, ".grok"); +/** Match Harness home-paths for both independent roots; trim only to detect blanks. */ +const dshDirectory = ( + home: string, + override: string | undefined, + fallback: string, +): string => { + const selected = + override !== undefined && override.trim().length > 0 + ? override + : join(home, fallback); + const expanded = + selected === "~" + ? home + : selected.startsWith("~/") || selected.startsWith("~\\") + ? join(home, selected.slice(2)) + : selected; + return resolve(expanded); +}; + const dshHomeDirectory = ({ home, env }: ClientPathContext): string => - env.DSH_HOME ?? join(home, ".dsh"); + dshDirectory(home, env.DSH_HOME, ".dsh"); const dshPatchDirectory = (context: ClientPathContext): string => join(dshHomeDirectory(context), "cordis.patch.yml"); @@ -555,7 +574,7 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ // The home Cordis patch applies to every profile; the skill directory is // shared agent storage Harness discovers. skillPath: ({ home, env }: ClientPathContext) => - join(env.DSH_AGENTS_HOME ?? join(home, ".agents"), "skills"), + join(dshDirectory(home, env.DSH_AGENTS_HOME, ".agents"), "skills"), configPath: dshPatchDirectory, markerPath: dshHomeDirectory, format: "dsh", diff --git a/tests/acceptance/setup/dshClientSetup.test.ts b/tests/acceptance/setup/dshClientSetup.test.ts index 4010369cd..b544c5816 100644 --- a/tests/acceptance/setup/dshClientSetup.test.ts +++ b/tests/acceptance/setup/dshClientSetup.test.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { expect } from "vitest"; @@ -8,6 +8,111 @@ import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +cliTest.for([ + { + label: "blank defaults", + harness: "", + agents: " \t ", + dshRoot: "home/.dsh", + agentsRoot: "home/.agents", + }, + { + label: "tilde roots", + harness: "~/selected harness", + agents: "~\\shared agents", + dshRoot: "home/selected harness", + agentsRoot: "home/shared agents", + }, + { + label: "relative roots", + harness: "./selected harness", + agents: "../shared agents", + dshRoot: "workspace/selected harness", + agentsRoot: "shared agents", + }, +])( + "DSH CLI uses Harness path semantics through setup, doctor and uninstall for $label", + async ({ harness, agents, dshRoot, agentsRoot }, { cli }) => { + const root = await createTestTempDirectory("rea-dsh-paths-"); + const home = join(root, "home"); + const cwd = join(root, "workspace"); + const config = join(root, dshRoot, "cordis.patch.yml"); + const skillRoot = join( + root, + agentsRoot, + "skills", + "reverse-engineer-anything", + ); + const skill = join(skillRoot, "SKILL.md"); + await mkdir(home); + await mkdir(cwd); + await mkdir(dirname(config), { recursive: true }); + const original = + "# caller patch\n- insert: [{id: caller, name: caller-plugin}]\n"; + await writeFile(config, original); + const run = (args: readonly string[]) => + cli.run({ + arguments: [...args, "--json"], + cwd, + environment: { + HOME: home, + USERPROFILE: home, + DSH_HOME: harness, + DSH_AGENTS_HOME: agents, + }, + }); + const planned = await run(["setup", "--client", "dsh", "--dry-run"]); + expect(planned.exitCode).toBe(0); + expect(planned.json).toMatchObject({ + status: "planned", + appliedActions: [], + plannedActions: expect.arrayContaining([ + expect.objectContaining({ id: "configure_client:dsh", target: config }), + expect.objectContaining({ kind: "install_skill", target: skillRoot }), + ]), + }); + expect(await readFile(config, "utf8")).toBe(original); + await expect(readFile(skill)).rejects.toMatchObject({ code: "ENOENT" }); + const configured = await run(["setup", "--client", "dsh", "--yes"]); + expect(configured.exitCode).toBe(0); + expect(configured.json).toMatchObject({ + status: "ready", + clients: { dsh: { status: "configured" } }, + }); + const text = await readFile(config, "utf8"); + expect(text).toContain("# caller patch"); + expect(text).toContain("rea-mcp"); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + expect(await readFile(skill, "utf8")).toContain( + "name: reverse-engineer-anything", + ); + const doctor = await run(["doctor", "--client", "dsh", "--skill"]); + expect(doctor.exitCode).toBe(0); + expect(doctor.json).toMatchObject({ healthy: true }); + const repeated = await run(["setup", "--client", "dsh", "--yes"]); + expect(repeated.exitCode).toBe(0); + expect(repeated.json).toMatchObject({ + status: "ready", + plannedActions: [], + appliedActions: [], + }); + expect(await readFile(config, "utf8")).toBe(text); + const removed = await run(["uninstall"]); + expect(removed.exitCode).toBe(0); + expect(removed.json).toMatchObject({ + status: "complete", + items: expect.arrayContaining([ + expect.objectContaining({ name: "dsh", status: "removed" }), + ]), + }); + expect(parse(await readFile(config, "utf8"))).toEqual([ + { insert: [{ id: "caller", name: "caller-plugin" }] }, + ]); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + await expect(readFile(skill)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + cliTest( "DSH CLI previews, approves, repairs, launches MCP, repeats and uninstalls while preserving caller YAML", async ({ cli }) => { diff --git a/tests/boundary/filesystem/dshClientConfiguration.test.ts b/tests/boundary/filesystem/dshClientConfiguration.test.ts index 72dbded9e..c0315f67b 100644 --- a/tests/boundary/filesystem/dshClientConfiguration.test.ts +++ b/tests/boundary/filesystem/dshClientConfiguration.test.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -37,28 +37,76 @@ const dshClient = ( env: Parameters[2], platform: NodeJS.Platform = "linux", ) => - supportedClients(join("/home/a"), platform, env).find( + supportedClients(resolve("/home/a"), platform, env).find( ({ name }) => name === "dsh", ); describe("DeepSeek Harness configuration paths", () => { + it.each([ + [undefined, ".dsh", ".agents", "shared"], + ["", ".dsh", ".agents", "shared"], + [" \t ", ".dsh", ".agents", "shared"], + ["~", "", "", "dsh"], + ["~/selected home", "selected home", "selected home", "dsh"], + ["~\\selected home", "selected home", "selected home", "dsh"], + ] as const)( + "resolves Harness and agents roots for %j", + (override, dshRoot, agentsRoot, client) => { + const home = resolve("/home/a"); + const environment = { DSH_HOME: override, DSH_AGENTS_HOME: override }; + expect( + supportedClients(home, process.platform, environment).find( + ({ name }) => name === "dsh", + ), + ).toMatchObject({ + configPath: join(home, dshRoot, "cordis.patch.yml"), + markerPath: join(home, dshRoot), + }); + expect(clientSkillDirectories(home, ["dsh"], environment)).toEqual([ + { client, directory: join(home, agentsRoot, "skills") }, + ]); + }, + ); + + it.each(["relative home", " literal home ", "~someone"])( + "resolves a literal root %j relative to cwd without trimming it", + (override) => { + const home = resolve("/home/a"); + const root = resolve(override); + const environment = { DSH_HOME: override, DSH_AGENTS_HOME: override }; + expect( + supportedClients(home, process.platform, environment).find( + ({ name }) => name === "dsh", + ), + ).toMatchObject({ + configPath: join(root, "cordis.patch.yml"), + markerPath: root, + }); + expect(clientSkillDirectories(home, ["dsh"], environment)).toEqual([ + { client: "dsh", directory: join(root, "skills") }, + ]); + }, + ); + it("resolves the home Cordis patch under the Harness home", () => { expect(dshClient({})).toEqual({ name: "dsh", displayName: "DeepSeek Harness", - configPath: join("/home/a", ".dsh", "cordis.patch.yml"), - markerPath: join("/home/a", ".dsh"), + configPath: resolve("/home/a", ".dsh", "cordis.patch.yml"), + markerPath: resolve("/home/a", ".dsh"), format: "dsh", }); // The home patch applies to every profile; DSH_HOME relocates it. expect(dshClient({ DSH_HOME: "/custom/harness" })?.configPath).toBe( - join("/custom/harness", "cordis.patch.yml"), + resolve("/custom/harness", "cordis.patch.yml"), ); expect( clientSkillDirectories(join("/home/a"), ["dsh"], { DSH_AGENTS_HOME: "/custom/agents", }), - ).toEqual([{ client: "dsh", directory: join("/custom/agents", "skills") }]); + ).toEqual([ + { client: "dsh", directory: resolve("/custom/agents", "skills") }, + ]); }); }); From 68ecdcd7a130806a03067fbfcce5ee90c70b968f Mon Sep 17 00:00:00 2001 From: Shenrui Ma <115921185+Shenrui-Ma@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:06:19 +0800 Subject: [PATCH 03/28] fix(reference): preserve directory intent when resolving source imports (#2770) Keep directory requests from becoming unresolved or pointing at sibling files when constructing reference-source dependency graphs. Canonicalize index candidates and preserve directory intent while retaining ordinary lookup rules. Constraint: Preserve current TypeScript candidate objects and exact-before-Unicode lookup Rejected: Reorder all file probes before index probes | Changes ordinary CommonJS candidate priority Confidence: high Scope-risk: narrow Tested: Public CLI regression fails on base; focused reference and resolver suites pass; npm run check Not-tested: Full suite on refreshed base; earlier full run had 12 baseline-reproduced failures --- .../ReferenceSourceImportEntries.ts | 18 +-- .../referenceSourceDirectoryImports.test.ts | 118 ++++++++++++++++++ 2 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 tests/boundary/cli/referenceSourceDirectoryImports.test.ts diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 36dae9949..49c6f82f5 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -97,18 +97,22 @@ const resolveInternalSpecifier = ( ); if (logicalPathEscapesRoot(normalized)) return { to: normalized, resolution: "unresolved" }; + // Preserve directory intent before normalization removes terminal dot segments. + const directoryReference = + specifier.endsWith("/") || [".", ".."].includes(posix.basename(specifier)); const suffixes = ["", ".ts", ".js", ".mjs", ".cjs", ".tsx", ".jsx"]; const candidates = suffixes.flatMap((suffix) => [ - `${normalized}${suffix}`, - `${normalized}/index${suffix}`, + ...(directoryReference ? [] : [`${normalized}${suffix}`]), + posix.join(normalized, `index${suffix}`), ]); // Retain present runtime files first; substitute only a missing counterpart. // Counterparts cover explicit filenames, not directory/index resolution. - candidates.push( - ...typescriptEmittedSourceCandidates(fromPath, normalized).map( - ({ path }) => path, - ), - ); + if (!directoryReference) + candidates.push( + ...typescriptEmittedSourceCandidates(fromPath, normalized).map( + ({ path }) => path, + ), + ); const match = lookupPath.resolveCandidates(candidates); return match === undefined ? { to: normalized, resolution: "unresolved" } diff --git a/tests/boundary/cli/referenceSourceDirectoryImports.test.ts b/tests/boundary/cli/referenceSourceDirectoryImports.test.ts new file mode 100644 index 000000000..6270855cb --- /dev/null +++ b/tests/boundary/cli/referenceSourceDirectoryImports.test.ts @@ -0,0 +1,118 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; + +import { describe, expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +// Reference-source reads require no-follow opens; Windows reports unsupported. +describe.skipIf(process.platform === "win32")( + "compiled reference-source directory imports", + () => { + cliTest( + "resolves directory spellings without selecting similarly named files", + async ({ cli }) => { + const workspace = await createTestTempDirectory( + "rea-reference-directory-imports-", + ); + const root = join(workspace, "source"); + const imports = [ + ["slash.cjs", "./lib/", "lib/index.js"], + ["dot.cjs", "./lib/.", "lib/index.js"], + ["self.cjs", ".", "index.js"], + ["self-slash.cjs", "./", "index.js"], + ["nested/parent.cjs", "..", "index.js"], + ["nested/parent-slash.cjs", "../", "index.js"], + ["direct.cjs", "./lib.js", "lib.js"], + ["fallback.cjs", "./fallback", "fallback/index.js"], + ["legacy-order.cjs", "./dep", "dep/index.js"], + ] as const; + const unresolved = [ + ["missing.cjs", "./missing/", "missing/"], + ["escape.cjs", "../outside/", "../outside/"], + ["main.ts", "./folder.js/.", "folder.js"], + ] as const; + const files = new Map([ + ["index.js", 'module.exports = "root index";\n'], + ["lib/index.js", 'module.exports = "library index";\n'], + ["lib.js", 'module.exports = "sibling JavaScript file";\n'], + ["lib.ts", 'export const value = "sibling TypeScript file";\n'], + ["fallback/index.js", 'module.exports = "fallback index";\n'], + ["dep/index.js", 'module.exports = "existing index priority";\n'], + ["dep.cjs", 'module.exports = "later CommonJS candidate";\n'], + [ + "folder.ts", + 'export const value = "emitted filename counterpart";\n', + ], + ["missing.js", 'module.exports = "not a directory";\n'], + ...[...imports, ...unresolved].map( + ([path, specifier]) => + [path, `require(${JSON.stringify(specifier)});\n`] as const, + ), + ]); + await Promise.all( + [...files].map(async ([path, contents]) => { + const target = join(root, path); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, contents); + }), + ); + const outside = join(workspace, "outside", "index.js"); + await mkdir(dirname(outside), { recursive: true }); + await writeFile(outside, 'module.exports = "outside import root";\n'); + + // Node's loader establishes each fixture's meaning independently. + for (const [path, specifier, target] of imports) + expect(createRequire(join(root, path)).resolve(specifier)).toBe( + join(root, target), + ); + expect(() => + createRequire(join(root, "missing.cjs")).resolve("./missing/"), + ).toThrow(); + expect(() => + createRequire(join(root, "main.ts")).resolve("./folder.js/."), + ).toThrow(); + expect( + createRequire(join(root, "escape.cjs")).resolve("../outside/"), + ).toBe(outside); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: workspace, + XDG_CONFIG_HOME: workspace, + XDG_CACHE_HOME: workspace, + }, + }); + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.parse_failures).toEqual([]); + expect(graph.relationships).toHaveLength( + imports.length + unresolved.length, + ); + for (const [from, , to] of imports) + expect.soft(graph.relationships, from).toContainEqual({ + from_path: from, + to, + kind: "requires", + resolution: "internal", + parse_state: "parsed", + }); + for (const [from, , to] of unresolved) + expect.soft(graph.relationships, from).toContainEqual({ + from_path: from, + to, + kind: "requires", + resolution: "unresolved", + parse_state: "parsed", + }); + expect( + graph.entries.some((entry) => entry.path.includes("outside")), + ).toBe(false); + }, + ); + }, +); From f90382756d4625d2a50b0cf6a4c30de038289b9a Mon Sep 17 00:00:00 2001 From: Iven Koo <73214171+IvenKooLab@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:06:50 +0800 Subject: [PATCH 04/28] fix(native): keep bounded trace edges and depth consistent (#2758) --- .../native/nativeInvestigationGraph.test.ts | 55 +++++++++++++++++++ src/domain/native/nativeInvestigationGraph.ts | 12 ++-- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/domain/native/nativeInvestigationGraph.test.ts b/src/domain/native/nativeInvestigationGraph.test.ts index 015458319..684b2a02e 100644 --- a/src/domain/native/nativeInvestigationGraph.test.ts +++ b/src/domain/native/nativeInvestigationGraph.test.ts @@ -83,6 +83,61 @@ const graph = nativeInvestigationGraphSchema.parse({ }); describe("native investigation graph traces", () => { + it.each(["forward", "backward"] as const)( + "keeps both endpoints of retained edges within the %s node budget", + (direction) => { + const trace = traceNativeInvestigationGraph(graph, { + start: direction === "forward" ? "a" : "c", + direction, + limits: { max_nodes: 1 }, + }); + expect(trace.nodes).toHaveLength(1); + expect(trace.edges).toEqual([]); + expect(trace).toMatchObject({ + truncated: true, + reason: "max_nodes_reached", + reached_depth: 0, + }); + }, + ); + + it.each([{ max_nodes: 2 }, { max_edges: 1 }])( + "reports retained node depth when a sibling hits the budget %j", + (limits) => { + const branching = { + ...graph, + edges: graph.edges + .filter((edge) => edge.resolution !== "unresolved") + .map((edge) => ({ ...edge, from: "a" })), + }; + const trace = traceNativeInvestigationGraph(branching, { + start: "a", + limits, + }); + expect(trace.nodes.map(({ id }) => id)).toEqual(["a", "b"]); + expect(trace.edges.map(({ id }) => id)).toEqual(["1"]); + expect(trace.reached_depth).toBe(1); + expect(trace.truncated).toBe(true); + }, + ); + + it("retains cycle edges to admitted nodes when the node budget is full", () => { + const cycle = { + ...graph, + nodes: graph.nodes.slice(0, 2), + edges: [ + graph.edges[0], + { ...graph.edges[0], id: "back", from: "b", to: "a" }, + ], + }; + const trace = traceNativeInvestigationGraph( + nativeInvestigationGraphSchema.parse(cycle), + { start: "a", limits: { max_nodes: 2 } }, + ); + expect(trace.edges.map(({ id }) => id)).toEqual(["1", "back"]); + expect(trace).toMatchObject({ reached_depth: 1, truncated: false }); + }); + it("retains observed and inferred route edges plus unresolved dispatch", () => { const trace = traceNativeInvestigationGraph(graph, { start: "a" }); expect(trace.nodes.map(({ id }) => id)).toEqual(["a", "b", "c"]); diff --git a/src/domain/native/nativeInvestigationGraph.ts b/src/domain/native/nativeInvestigationGraph.ts index 0dd0dbe8b..9b9efc412 100644 --- a/src/domain/native/nativeInvestigationGraph.ts +++ b/src/domain/native/nativeInvestigationGraph.ts @@ -207,7 +207,6 @@ export const traceNativeInvestigationGraph = ( while (queue.length > 0) { const current = queue.shift(); if (current === undefined) break; - reachedDepth = Math.max(reachedDepth, current.depth); const adjacent = adjacency.get(current.id) ?? []; for (const edge of adjacent) { @@ -227,19 +226,24 @@ export const traceNativeInvestigationGraph = ( reason ??= "max_depth_reached"; continue; } - selectedEdges.set(edge.id, edge); - if (alreadyVisited) continue; - if (visited.size >= limits.max_nodes) { + // Check destination capacity before retaining its edge. Otherwise the node + // bound leaves a resolved edge pointing outside the returned trace. + if (!alreadyVisited && visited.size >= limits.max_nodes) { truncated = true; reason = "max_nodes_reached"; break; } + selectedEdges.set(edge.id, edge); + if (alreadyVisited) continue; if (!nodeById.has(nextId)) { truncated = true; reason ??= "edge_target_missing_from_graph"; continue; } visited.add(nextId); + // A retained node has already been reached even if a later sibling + // exhausts a budget before this node is dequeued for expansion. + reachedDepth = Math.max(reachedDepth, current.depth + 1); queue.push({ id: nextId, depth: current.depth + 1 }); } if (reason === "max_edges_reached" || reason === "max_nodes_reached") break; From cd2ad1f24084e68910e4ceab4a33e1b88998ebe7 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 05:07:28 -0700 Subject: [PATCH 05/28] fix(artifacts): report malformed ZIP archives as format failures (#2683) * fix(artifacts): report malformed ZIP archives as format failures zip.js signals archive faults (missing end-of-central-directory record, unrecognized format, overlapping members, corrupt member data, unsupported compression) as plain Errors. ZipArtifactReader passed them through untyped, so the artifact provider's fallback reported every one as `io` with no detail and told the caller to "check file access". Type these faults at the ZIP reader boundary: structural and data corruption become `format` failures and unsupported archive features become `unavailable`, each carrying the zip.js reason as detail. This covers enumeration, the getData rejection, and the WritableStream abort path that delivers corrupt-data errors first. Co-Authored-By: Claude Opus 5.5 * test(artifacts): share the ZIP inventory failure assertion Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- src/artifacts/ZipArtifactReader.ts | 85 +++++++++++++++++-- .../filesystem/artifactZipIntegrity.test.ts | 66 +++++++++----- 2 files changed, 124 insertions(+), 27 deletions(-) diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index dfbee8b27..066b005b8 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -2,7 +2,29 @@ import type { FileHandle } from "node:fs/promises"; import { PassThrough, type Readable } from "node:stream"; import { once } from "node:events"; -import { Reader, ZipReader, type Entry, type FileEntry } from "@zip.js/zip.js"; +import { + ERR_AMBIGUOUS_ARCHIVE, + ERR_BAD_FORMAT, + ERR_CENTRAL_DIRECTORY_NOT_FOUND, + ERR_ENCRYPTED_CENTRAL_DIRECTORY, + ERR_ENTRY_DATA_OUT_OF_BOUNDS, + ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND, + ERR_EOCDR_NOT_FOUND, + ERR_EXTRAFIELD_ZIP64_NOT_FOUND, + ERR_INVALID_COMPRESSED_DATA, + ERR_INVALID_CRC32, + ERR_INVALID_UNCOMPRESSED_SIZE, + ERR_LOCAL_FILE_HEADER_NOT_FOUND, + ERR_OVERLAPPING_ENTRY, + ERR_SPLIT_ZIP_FILE, + ERR_UNSUPPORTED_COMPRESSION, + ERR_UNSUPPORTED_ENCRYPTION, + ERR_UNSUPPORTED_UINT64, + Reader, + ZipReader, + type Entry, + type FileEntry, +} from "@zip.js/zip.js"; import { ArtifactReaderFailure, @@ -159,9 +181,9 @@ export class ZipArtifactReader implements ArtifactReader { let index = 0; // A traversal owns its accounting, including overlapping or repeated passes. let declaredFileBytes = 0; - for await (const entry of this.#reader.getEntriesGenerator({ - decodeText: decodeZipEntryText, - })) { + for await (const entry of typedZipFailures( + this.#reader.getEntriesGenerator({ decodeText: decodeZipEntryText }), + )) { abortIfNeeded(signal); // ZIP member identity is its directory occurrence, not its filename. // Logical path uniqueness belongs to the artifact inventory boundary. @@ -327,7 +349,7 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => { output.end(); }, abort: (reason) => { - output.destroy(toError(reason)); + output.destroy(toError(typedZipFailure(reason))); }, }); void entry @@ -337,10 +359,61 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => { checkOverlappingEntry: true, onprogress: () => abortIfNeeded(signal), }) - .catch((cause: unknown) => output.destroy(toError(cause))); + .catch((cause: unknown) => output.destroy(toError(typedZipFailure(cause)))); return output; }; +/** zip.js reports archive faults as plain errors identified by message. */ +const ZIP_FAILURE_REASONS: ReadonlyMap = + new Map([ + ...[ + ERR_AMBIGUOUS_ARCHIVE, + ERR_BAD_FORMAT, + ERR_CENTRAL_DIRECTORY_NOT_FOUND, + ERR_ENTRY_DATA_OUT_OF_BOUNDS, + ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND, + ERR_EOCDR_NOT_FOUND, + ERR_EXTRAFIELD_ZIP64_NOT_FOUND, + ERR_INVALID_COMPRESSED_DATA, + ERR_INVALID_CRC32, + ERR_INVALID_UNCOMPRESSED_SIZE, + ERR_LOCAL_FILE_HEADER_NOT_FOUND, + ERR_OVERLAPPING_ENTRY, + ERR_UNSUPPORTED_UINT64, + ].map((message) => [message, "format"] as const), + ...[ + ERR_ENCRYPTED_CENTRAL_DIRECTORY, + ERR_SPLIT_ZIP_FILE, + ERR_UNSUPPORTED_COMPRESSION, + ERR_UNSUPPORTED_ENCRYPTION, + ].map((message) => [message, "unavailable"] as const), + ]); + +/** Type a zip.js archive fault so consumers do not report it as file I/O. */ +const typedZipFailure = (cause: unknown): unknown => { + if (!(cause instanceof Error) || cause instanceof ArtifactReaderFailure) + return cause; + const reason = ZIP_FAILURE_REASONS.get(cause.message); + if (reason === undefined) return cause; + return new ArtifactReaderFailure( + reason, + reason === "format" + ? `ZIP archive is malformed: ${cause.message}` + : `ZIP archive uses an unsupported feature: ${cause.message}`, + { cause }, + ); +}; + +async function* typedZipFailures( + source: AsyncIterable, +): AsyncIterable { + try { + yield* source; + } catch (cause: unknown) { + throw typedZipFailure(cause); + } +} + const isSymlink = (entry: Entry): boolean => entry.unixMode !== undefined && (entry.unixMode & 0o170000) === 0o120000; diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index 41397c520..cf7dcd595 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -366,16 +366,11 @@ it("rejects overlapping ZIP member data ranges through artifact inventory", asyn bytes.writeUInt32LE(firstLocalOffset, second + 42); await writeFile(path, bytes); - const target = await parseBinaryTarget(path); - if (!target.ok) throw target.error; - const result = await withArtifactClient(target.value, (client) => - client.execute("inventory_artifact", {}), + await expectInventoryFailure( + path, + "format", + "ZIP archive is malformed: Overlapping entry found", ); - - expect(result).toMatchObject({ - ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, - }); }); it("preserves encrypted-member and corrupt-CRC observations", async () => { @@ -438,16 +433,11 @@ it("preserves encrypted-member and corrupt-CRC observations", async () => { bytes[dataOffset] = firstByte ^ 1; await writeFile(corruptPath, bytes); - const corruptTarget = await parseBinaryTarget(corruptPath); - if (!corruptTarget.ok) throw corruptTarget.error; - const corruptResult = await withArtifactClient( - corruptTarget.value, - (client) => client.execute("inventory_artifact", {}), + await expectInventoryFailure( + corruptPath, + "format", + "ZIP archive is malformed: Invalid compressed data", ); - expect(corruptResult).toMatchObject({ - ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, - }); }); it("rejects a truncated ZIP through artifact inventory", async () => { @@ -455,17 +445,51 @@ it("rejects a truncated ZIP through artifact inventory", async () => { const path = join(root, "truncated.zip"); await writeFile(path, Buffer.from([0x50, 0x4b, 0x03, 0x04])); + await expectInventoryFailure( + path, + "format", + "ZIP archive is malformed: File format is not recognized", + ); +}); + +it("reports an unsupported ZIP compression method as unavailable", async () => { + const root = await createTestTempDirectory("rea-zip-method-"); + const path = join(root, "lzma.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("data.txt", new TextReader("member bytes"), { level: 0 }); + const bytes = Buffer.from(await writer.close()); + // Declare LZMA (14) in both headers; REA reads only stored and Deflate data. + const localHeader = bytes.indexOf(Buffer.from([0x50, 0x4b, 0x03, 0x04])); + const centralHeader = bytes.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + if (localHeader < 0 || centralHeader < 0) + throw new Error("Expected ZIP local and central headers"); + bytes.writeUInt16LE(14, localHeader + 8); + bytes.writeUInt16LE(14, centralHeader + 10); + await writeFile(path, bytes); + + await expectInventoryFailure( + path, + "unavailable", + "ZIP archive uses an unsupported feature: Compression method not supported", + ); +}); + +/** Inventory one archive through the provider and expect a typed failure. */ +const expectInventoryFailure = async ( + path: string, + reason: string, + detail: string, +): Promise => { const target = await parseBinaryTarget(path); if (!target.ok) throw target.error; const result = await withArtifactClient(target.value, (client) => client.execute("inventory_artifact", {}), ); - expect(result).toMatchObject({ ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, + error: { _tag: "ArtifactOperationError", reason, detail }, }); -}); +}; const writeSingleEntryZip = async ( path: string, From f974353c329ebfa58ba5f268078da30952a82ec0 Mon Sep 17 00:00:00 2001 From: Chaoqi <136579807+Chaoqi31@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:10:10 -0400 Subject: [PATCH 06/28] fix(javascript): keep paths after __dirname inside the source directory (#2366) Node's path.join does not reset at an absolute segment, so path.join(__dirname, '/preload.js') is /preload.js. The static path projection skipped the __dirname anchor and joined the remaining parts, which turned that common spelling into the root path /preload.js. In an app whose main script lives in src/, the BrowserWindow preload was then reported not-found, or resolved to an unrelated preload.js at the artifact root, and the utilityProcess.fork target was never resolved. String concatenation had the same defect: path.dirname(__filename) + '/preload.js' projected to /preload.js, and __dirname + '/preload.js' stayed unknown. A source directory now contributes "." to a join, so path.join(__dirname) projects to "." like path.resolve(__dirname). A concatenation that starts with __dirname or the source file's dirname keeps text that begins with a separator inside that directory; other text extends the directory's own name and stays unknown. The empty-string projection of path.dirname(__filename), which produced the wrong concatenation, is removed, so a bare path.dirname(__filename) is unknown like a bare __dirname. --- .../javascriptStaticAnalysisHelpers.ts | 60 ++++++++++++++----- .../javascriptStaticPathResolution.test.ts | 17 +++++- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts index 71562e6ca..f2045a1eb 100644 --- a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts +++ b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts @@ -181,18 +181,37 @@ export const staticPath = (node: t.Node): string | undefined => { if (t.isStringLiteral(node)) return node.value; if (t.isTemplateLiteral(node) && node.expressions.length === 0) return stringValue(node); - if (t.isBinaryExpression(node, { operator: "+" })) { - const left = staticPath(node.left); - const right = staticPath(node.right); - return left === undefined || right === undefined - ? undefined - : `${left}${right}`; - } + if (t.isBinaryExpression(node, { operator: "+" })) + return staticConcatenation(node); if (t.isCallExpression(node) || t.isNewExpression(node)) return staticCallPath(node); return undefined; }; +/** + * Concatenate inert text. A leading source directory contributes no text: + * `__dirname + "/preload.js"` names a file inside it, while text without a + * leading separator extends the directory's own name and stays unknown. + */ +const staticConcatenation = (node: t.BinaryExpression): string | undefined => { + const operands: t.Node[] = []; + let current: t.Node = node; + while (t.isBinaryExpression(current, { operator: "+" })) { + operands.push(current.right); + current = current.left; + } + const anchored = isDirectoryPath(current); + if (!anchored) operands.push(current); + let text = ""; + for (const operand of operands.reverse()) { + const value = staticPath(operand); + if (value === undefined) return undefined; + text += value; + } + if (!anchored) return text; + return text.startsWith("/") ? posix.join(".", text) : undefined; +}; + const staticCallPath = ( node: t.CallExpression | t.NewExpression, ): string | undefined => { @@ -204,11 +223,6 @@ const staticCallPath = ( if (name === "fileURLToPath" || name.endsWith(".fileURLToPath")) { return staticFileUrlPath(argumentNode(node.arguments[0])); } - if ( - (name === "dirname" || name.endsWith(".dirname")) && - isFileIdentity(argumentNode(node.arguments[0])) - ) - return ""; if (name.endsWith(".resolve")) return name === "win32.resolve" || name.endsWith(".win32.resolve") ? undefined @@ -216,7 +230,12 @@ const staticCallPath = ( if (!name.endsWith(".join") && name !== "join") return undefined; const parts: string[] = []; for (const argument of node.arguments) { - if (isDirectoryIdentity(argumentNode(argument))) continue; + if (isDirectoryIdentity(argumentNode(argument))) { + // join never resets at an absolute segment, so `/preload.js` after the + // source directory stays inside it. + parts.push("."); + continue; + } const value = t.isNode(argument) ? staticPath(argument) : undefined; if (value === undefined) return undefined; parts.push(value); @@ -331,11 +350,20 @@ const isFilesystemPathExpression = (node: t.Node): boolean => { }; const isDirectoryIdentity = (node: t.Node | undefined): boolean => { - if (t.isIdentifier(node, { name: "__dirname" })) return true; + if (isDirectoryPath(node)) return true; if (!t.isCallExpression(node) && !t.isNewExpression(node)) return false; const name = calleeName(node.callee); - if (name === "URL" || name.endsWith(".URL")) - return isFileIdentity(argumentNode(node.arguments[1])); + return ( + (name === "URL" || name.endsWith(".URL")) && + isFileIdentity(argumentNode(node.arguments[1])) + ); +}; + +/** `__dirname` or the dirname of the source file, as a string. */ +const isDirectoryPath = (node: t.Node | undefined): boolean => { + if (t.isIdentifier(node, { name: "__dirname" })) return true; + if (!t.isCallExpression(node)) return false; + const name = calleeName(node.callee); return ( (name === "dirname" || name.endsWith(".dirname")) && isFileIdentity(argumentNode(node.arguments[0])) diff --git a/src/domain/javascript/javascriptStaticPathResolution.test.ts b/src/domain/javascript/javascriptStaticPathResolution.test.ts index e54122440..30bbc317f 100644 --- a/src/domain/javascript/javascriptStaticPathResolution.test.ts +++ b/src/domain/javascript/javascriptStaticPathResolution.test.ts @@ -38,6 +38,9 @@ const anchoredExpressions = [ ["path.resolve(__dirname)", "."], ['path.posix.resolve(`/app`, `/shared`, "preload.js")', "/shared/preload.js"], ['path.posix.resolve("" + "/app", "preload.js")', "/app/preload.js"], + ['__dirname + "/preload.js"', "preload.js"], + ['__dirname + "/" + "preload.js"', "preload.js"], + ['path.dirname(__filename) + "/lib" + "/preload.js"', "lib/preload.js"], ] as const; const unknownExpressions = [ @@ -58,6 +61,8 @@ const unknownExpressions = [ 'path.win32.resolve(__dirname, "preload.js")', 'win32.resolve("/app", "preload.js")', 'resolve("/app", "preload.js")', + '__dirname + "preload.js"', + 'path.dirname(__filename) + "preload.js"', ]; describe("inert JavaScript path resolution", () => { @@ -100,6 +105,16 @@ describe("inert JavaScript path resolution", () => { "preload.js", "filesystem-expression", ], + [ + 'path.join(__dirname, "/preload.js")', + "preload.js", + "filesystem-expression", + ], + [ + 'path.join(dirname(fileURLToPath(import.meta.url)), "/lib", "/preload.js")', + "lib/preload.js", + "filesystem-expression", + ], [ 'path.join("ignored", __dirname, "preload.js")', "ignored/preload.js", @@ -111,6 +126,7 @@ describe("inert JavaScript path resolution", () => { "filesystem-expression", ], ['join("sub", "preload.js")', "sub/preload.js", "module-specifier"], + ["path.join(__dirname)", ".", "filesystem-expression"], ] as const)("preserves join behavior in %s", (expression, path, context) => { assertPathProjection(expression, path, context); }); @@ -182,7 +198,6 @@ describe("inert JavaScript path resolution", () => { it.each([ "path.join()", - "path.join(__dirname)", 'path.join("/app", runtimeFile)', "path.join(...parts)", ])("preserves unresolved join syntax in %s", (expression) => From eeeb4ca379a948109542e66a8d0adb3998f6f60d Mon Sep 17 00:00:00 2001 From: To3akaRin <131595336+To3akaRin@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:10:39 +0800 Subject: [PATCH 07/28] fix(javascript): preserve container ownership for getter mutations (#2765) --- docs/javascript-application-workflows.md | 7 ++ ...tSemanticAnalysis.getter_receivers.test.ts | 70 +++++++++++++++++- .../javascriptSemanticMemberMutations.ts | 49 +++++++++++-- .../javascriptSemanticObjectReferences.ts | 62 +++++++++++++--- .../getterReceiverMutationsCli.test.ts | 73 +++++++++++++++++-- 5 files changed, 233 insertions(+), 28 deletions(-) diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index 1e9e7fe19..041015b9c 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -208,6 +208,13 @@ arrays, and later explicit data properties and methods replace earlier object spread references. A known array prefix does not expose values from a following spread. Accessors, custom or uncertain iteration, unknown computed keys, and unknown spread lengths retain conservative uncertainty. +Writes through an object getter's returned receiver retain that uncertainty +when the object is an array element, nested inside mixed object/array +containers, or assigned to a previously declared identifier. Saving the getter +result in an alias does not make its old field values immutable. Known array +indices include holes; an element after a spread has only a minimum possible +index. Unrelated properties and known array prefixes remain precise, and a +getter read without a write does not invalidate the receiver's fields. Nested callable returns are not assigned to their parent callable. Projected graph observations carry source ranges but never source text. diff --git a/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts index 3b83bf482..c3b38842a 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts @@ -33,6 +33,38 @@ const getterReceivers = [ "nested owner", "const outer = { box: { x: 1, get self() { return this; } } }; outer.box.self.x = 2; return outer.box.x;", ], + [ + "array element", + "const arr = [{ x: 1, get self() { return this; } }]; arr[0].self.x = 2; return arr[0].x;", + ], + [ + "array element after a hole", + "const arr = [, { x: 1, get self() { return this; } }]; arr[1].self.x = 2; return arr[1].x;", + ], + [ + "mixed object and array owners", + "const outer = { rows: [{ box: { x: 1, get self() { return this; } } }] }; outer.rows[0].box.self.x = 2; return outer.rows[0].box.x;", + ], + [ + "assigned owner", + "let box; box = { x: 1, get self() { return this; } }; box.self.x = 2; return box.x;", + ], + [ + "saved array receiver alias", + "const arr = [{ x: 1, get self() { return this; } }]; const saved = arr[0].self; saved.x = 2; return arr[0].x;", + ], + [ + "old receiver captured inside its replacement initializer", + "const old = { x: 1 }; let box = old; box = { old: box, get self() { return this; } }; box.self.old.x = 2; return old.x;", + ], + [ + "array receiver after an unknown spread", + "const arr = [...items, { x: 1, get self() { return this; } }]; arr[index].self.x = 2; return arr[index].x;", + ], + [ + "saved receiver before its array alias is reassigned", + "const original = [{ x: 1, get self() { return this; } }]; let arr = original; const saved = arr[0].self; arr = [{ x: 1 }]; saved.x = 2; return original[0].x;", + ], [ "inherited receiver field", "const shared = { x: 1 }; class A { get v() { return this.s; } } class B extends A { s = shared; } new B().v.x = 2; return shared.x;", @@ -75,7 +107,7 @@ const getterReceivers = [ ], ] as const; -describe("getter receiver mutation aliases (#1665)", () => { +describe("getter receiver mutation aliases (#1665, #2654)", () => { it.each(getterReceivers)( "does not report a stale literal through %s", (_shape, body) => { @@ -103,6 +135,42 @@ describe("getter receiver mutation aliases (#1665)", () => { "unrelated sibling", "const box = { x: 1, y: 1, get self() { return this; } }; box.self.y = 2; return box.x;", ], + [ + "array receiver read without write", + "const arr = [{ x: 1, get self() { return this; } }]; const saved = arr[0].self; return arr[0].x;", + ], + [ + "assigned receiver read without write", + "let box; box = { x: 1, get self() { return this; } }; const saved = box.self; return box.x;", + ], + [ + "unrelated array receiver property", + "const arr = [{ x: 1, y: 1, get self() { return this; } }]; arr[0].self.y = 2; return arr[0].x;", + ], + [ + "unrelated array element", + "const arr = [{ x: 1, get self() { return this; } }, { x: 1 }]; arr[0].self.x = 2; return arr[1].x;", + ], + [ + "array replacement after a receiver was captured", + "const original = [{ x: 1, get self() { return this; } }]; const replacement = [{ x: 1 }]; let arr = original; const saved = arr[0].self; arr = replacement; saved.x = 2; return replacement[0].x;", + ], + [ + "known array prefix before a spread receiver", + "const arr = [{ x: 1 }, ...items, { x: 1, get self() { return this; } }]; arr[1].self.x = 2; return arr[0].x;", + ], + [ + "outer binding shadowed by an assigned receiver", + "let box; box = { x: 1 }; { let box; box = { x: 1, get self() { return this; } }; box.self.x = 2; } return box.x;", + ], + [ + "old object before an unconditional receiver assignment", + "const old = { x: 1 }; let box = old; box = { x: 1, get self() { return this; } }; box.self.x = 2; return old.x;", + ], + [ + "unmodified old receiver captured inside its replacement initializer", + "const old = { x: 1 }; let box = old; box = { x: 1, old: box, get self() { return this; } }; box.self.x = 2; return old.x;", + ], [ "replace getter", "const shared = { x: 1 }; const box = { s: shared, get v() { return this.s; } }; box.v = {}; return shared.x;", diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index c2f7efba3..35b63c815 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -18,6 +18,7 @@ import { semanticClassCandidates, semanticClassPropertyReferences, semanticObjectPropertyReferences, + type JavaScriptSemanticObjectReference, } from "./javascriptSemanticObjectReferences.js"; import { semanticIterationSources } from "./javascriptSemanticIterationSources.js"; import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js"; @@ -98,8 +99,20 @@ export function* collectSemanticMemberMutationsSteps( effect: ValueEffect = "write", mutation?: t.Node, originAt: t.Node | undefined = mutation, + ownerInitializer?: t.Node, ): void => { - const pending = [{ node, path, bindings, originAt }]; + const pending: (ReferencedValue & { + readonly bindings: ReadonlySet; + readonly originAt: t.Node | undefined; + })[] = [ + { + node, + path, + bindings, + originAt, + ...(ownerInitializer === undefined ? {} : { ownerInitializer }), + }, + ]; // Visit shallower alias traversals first so their exact prefixes suppress // longer loop paths before those paths enumerate every assignment order. for (let cursor = 0; cursor < pending.length; cursor++) { @@ -132,6 +145,8 @@ export function* collectSemanticMemberMutationsSteps( binding.bindingId, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]; const lifetime = JSON.stringify([...capture, current.path]); if (current.bindings.has(lifetime)) continue; @@ -165,6 +180,7 @@ export function* collectSemanticMemberMutationsSteps( "escape", mutation, current.originAt, + current.ownerInitializer, ); continue; } @@ -195,6 +211,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); const previous = selections.get(selectionIdentity); const excludedKeys = @@ -220,6 +238,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); let covering = coveringPaths.get(pathState); if (covering === undefined) { @@ -235,6 +255,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); if (recordedEffects.has(identity)) continue; // A recorded write invalidates the binding to a non-primitive value, so @@ -248,10 +270,23 @@ export function* collectSemanticMemberMutationsSteps( value?.status === "literal" || value?.status === "union"; recordedEffects.add(identity); const nested = new Set([...current.bindings, lifetime, captureKey]); + // Returning a literal's receiver reaches that allocation, not the + // binding's older values read while evaluating its initializer. const origins = - current.originAt === undefined - ? binding - : semanticMutationInitializers(binding, current.originAt, parents); + current.ownerInitializer !== undefined + ? { + initializers: binding.initializers.filter( + ({ node }) => node === current.ownerInitializer, + ), + referenceInitializers: [], + } + : current.originAt === undefined + ? binding + : semanticMutationInitializers( + binding, + current.originAt, + parents, + ); if (!primitiveWrite) { if (effect === "escape") binding.escapedPaths.push({ path, node: mutation }); @@ -367,6 +402,8 @@ export function* collectSemanticMemberMutationsSteps( binding === undefined ? null : binding.bindingId, binding === undefined ? node.name : null, pathKey, + value.ownerInitializer?.start, + value.ownerInitializer?.end, ]); if (leafIdentities.has(identity)) continue; leafIdentities.add(identity); @@ -551,9 +588,7 @@ export function* collectSemanticMemberMutationsSteps( clearSemanticValueCaches(state); } -interface ReferencedValue { - readonly node: t.Node; - readonly path: PropertyPath; +interface ReferencedValue extends JavaScriptSemanticObjectReference { readonly iterableFallbackPath?: PropertyPath; } diff --git a/src/domain/javascript/javascriptSemanticObjectReferences.ts b/src/domain/javascript/javascriptSemanticObjectReferences.ts index 387ba09d2..4d1bf3e76 100644 --- a/src/domain/javascript/javascriptSemanticObjectReferences.ts +++ b/src/domain/javascript/javascriptSemanticObjectReferences.ts @@ -14,9 +14,11 @@ import { } from "./javascriptSemanticPropertyPaths.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; -interface ReferencedValue { +export interface JavaScriptSemanticObjectReference { readonly node: t.Node; readonly path: PropertyPath; + /** The allocation owned by this binding, rather than a read of its prior value. */ + readonly ownerInitializer?: t.Node; } export const semanticClassCandidates = ( @@ -72,10 +74,10 @@ const instanceFieldReferences = ( path: PropertyPath, state: JavaScriptSemanticAnalysisState, ): { - readonly references: readonly ReferencedValue[]; + readonly references: readonly JavaScriptSemanticObjectReference[]; readonly definiteNames: ReadonlySet; } => { - const references: ReferencedValue[] = []; + const references: JavaScriptSemanticObjectReference[] = []; const pending = [{ klass: root, excluded: new Set() }]; const selections = new Map>(); let definiteNames: ReadonlySet | undefined; @@ -208,7 +210,7 @@ export const semanticClassPropertyReferences = ( selection: PropertyPath, state: JavaScriptSemanticAnalysisState, activeGetters: ReadonlyMap = new Map(), -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const root = lookup.owner; const receiver = lookup.receiver ?? root; const [key, ...path] = selection; @@ -216,7 +218,7 @@ export const semanticClassPropertyReferences = ( key == null || typeof key === "object" ? null : String(key); const pending: (t.ClassDeclaration | t.ClassExpression)[] = [root]; const seen = new Set(); - const possibleResults: ReferencedValue[] = []; + const possibleResults: JavaScriptSemanticObjectReference[] = []; // Instance fields are own properties, including fields initialized by base // constructors. They take precedence over every prototype descriptor. let instanceFields: ReadonlySet = new Set(); @@ -267,11 +269,11 @@ const objectReceiverReference = ( object: t.ObjectExpression, path: PropertyPath, state: JavaScriptSemanticAnalysisState, -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { // A literal's enclosing binding owns its slots. Returning the literal // itself would bypass invalidation of that binding's observed properties. let current: t.Node = object; - const projection: (string | number | null)[] = []; + const projection: PropertyPath[number][] = []; while (true) { const parent = state.parentsByNode.get(current); if ( @@ -279,7 +281,43 @@ const objectReceiverReference = ( parent.init === current && t.isIdentifier(parent.id) ) - return [{ node: parent.id, path: [...projection, ...path] }]; + return [ + { + node: parent.id, + path: [...projection, ...path], + ownerInitializer: current, + }, + ]; + if ( + t.isAssignmentExpression(parent, { operator: "=" }) && + parent.right === current && + t.isIdentifier(parent.left) + ) + return [ + { + node: parent.left, + path: [...projection, ...path], + ownerInitializer: current, + }, + ]; + if (t.isArrayExpression(parent)) { + let minimumIndex = 0; + let uncertainIndex = false; + for (const element of parent.elements) { + if (element === current) break; + if (t.isSpreadElement(element)) uncertainIndex = true; + else minimumIndex++; + } + // A spread contributes an unknown number of slots. Retain the known + // prefix without treating the element's syntax position as its index. + projection.unshift( + uncertainIndex + ? { excludedKeys: [], startIndex: minimumIndex } + : minimumIndex, + ); + current = parent; + continue; + } if (t.isObjectProperty(parent) && parent.value === current) { projection.unshift( semanticStaticPropertyKey(parent.key, parent.computed), @@ -312,7 +350,7 @@ const getterResultReferences = ( path: PropertyPath, state: JavaScriptSemanticAnalysisState, activeGetters: ReadonlyMap = new Map(), -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const { getter, owner, receiver = owner } = invocation; const previousLength = activeGetters.get(getter); if (previousLength !== undefined && previousLength <= path.length) return []; @@ -323,7 +361,7 @@ const getterResultReferences = ( const resolve = ( node: t.Node, remaining: PropertyPath, - ): readonly ReferencedValue[] => { + ): readonly JavaScriptSemanticObjectReference[] => { const expression = unwrapJavaScriptExpression(node).node; if (t.isThisExpression(expression)) { if (t.isClass(receiver)) { @@ -405,9 +443,9 @@ export const semanticObjectPropertyReferences = ( node: t.ObjectExpression, path: PropertyPath, state: JavaScriptSemanticAnalysisState, -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const [key, ...remaining] = path; - const references: ReferencedValue[] = []; + const references: JavaScriptSemanticObjectReference[] = []; const overwritten = new Set(); const dataOverwritten = new Set(); const accessorNames = new Set(); diff --git a/tests/acceptance/applications/getterReceiverMutationsCli.test.ts b/tests/acceptance/applications/getterReceiverMutationsCli.test.ts index a5ba0bfeb..95b6e4862 100644 --- a/tests/acceptance/applications/getterReceiverMutationsCli.test.ts +++ b/tests/acceptance/applications/getterReceiverMutationsCli.test.ts @@ -39,6 +39,47 @@ export function superclass() { new B().v.x = 2; return shared.x; } +export function arrayReceiver() { + const arr = [{ x: 1, get self() { return this; } }]; + arr[0].self.x = 2; + return arr[0].x; +} +export function nestedArrayReceiver() { + const outer = { arr: [{ x: 1, get self() { return this; } }] }; + outer.arr[0].self.x = 2; + return outer.arr[0].x; +} +export function assignedReceiver() { + let box; + box = { x: 1, get self() { return this; } }; + box.self.x = 2; + return box.x; +} +export function savedArrayReceiver() { + const arr = [{ x: 1, get self() { return this; } }]; + const saved = arr[0].self; + saved.x = 2; + return arr[0].x; +} +export function arrayReadOnly() { + const arr = [{ x: 1, get self() { return this; } }]; + const saved = arr[0].self; + return arr[0].x; +} +export function replacementKeepsOldObject() { + const old = { x: 1 }; + let box = old; + box = { x: 1, old: box, get self() { return this; } }; + box.self.x = 2; + return old.x; +} +export function replacementMutatesCapturedObject() { + const old = { x: 1 }; + let box = old; + box = { old: box, get self() { return this; } }; + box.self.old.x = 2; + return old.x; +} export function untouched() { const box = { x: 1, y: 1, get self() { return this; } }; box.self.y = 2; @@ -66,13 +107,20 @@ export function untouched() { const { graph } = z .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) .parse(document).normalized_result; - for (const name of [ - "self", - "property", - "privateField", - "superclass", - "untouched", - ]) { + for (const [name, status] of [ + ["self", "unknown"], + ["property", "unknown"], + ["privateField", "unknown"], + ["superclass", "unknown"], + ["arrayReceiver", "unknown"], + ["nestedArrayReceiver", "unknown"], + ["assignedReceiver", "unknown"], + ["savedArrayReceiver", "unknown"], + ["arrayReadOnly", "literal"], + ["replacementKeepsOldObject", "literal"], + ["replacementMutatesCapturedObject", "unknown"], + ["untouched", "literal"], + ] as const) { const projection = graph.nodes .flatMap(({ observations }) => observations) .find( @@ -82,7 +130,16 @@ export function untouched() { ); expect(projection, name).toBeDefined(); expect(projection?.properties.static_return_shapes).toMatchObject([ - { value_status: name === "untouched" ? "literal" : "unknown" }, + { + value_status: status, + fields: [ + { + path: "", + state: status, + ...(status === "literal" ? { value: 1 } : {}), + }, + ], + }, ]); } }, From a8b50ea147ab3aaa2870160481e09403c3310ea0 Mon Sep 17 00:00:00 2001 From: Shreyansh Jain <136733716+ShreyanshJain105@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:55:58 +0530 Subject: [PATCH 08/28] refactor(firmware): split verification into staged functions (#2713) Refactor firmware verifier into staged functions Restructure scripts/verify/firmware/analysis.mjs for readability without changing behavior. - Move prerequisite checks into checkPrerequisites() - Split the main flow into stageCli, stageMcp, stageExt4 and stageGhidra - Extract timeouts and size limits into named constants - Rename cli/call helpers to runCli/callTool Checks, order of operations, error messages and env vars are unchanged. Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- scripts/verify/firmware/analysis.mjs | 504 ++++++++++++++++----------- 1 file changed, 299 insertions(+), 205 deletions(-) diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index 1f84681c9..804e8fffa 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -1,15 +1,8 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { - access, - mkdir, - mkdtemp, - readFile, - rm, - symlink, -} from "node:fs/promises"; +import * as fs from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import * as nodePath from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna import { parseEvidence } from "../../../dist/domain/evidence.js"; import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js"; -const exec = promisify(execFile); -const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); -const entrypoint = resolve( +const execFileAsync = promisify(execFile); +const { join, resolve, dirname } = nodePath; + +const CALL_TIMEOUT_MS = 150_000; +const CONNECT_TIMEOUT_MS = 30_000; +const SEVENZIP_TIMEOUT_MS = 10_000; +const MAX_STDOUT_BYTES = 16 * 1024 * 1024; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const ENTRYPOINT = resolve( process.env.REA_FIRMWARE_TEST_ENTRYPOINT ?? - join(repository, "scripts/rea.mjs"), + join(REPO_ROOT, "scripts/rea.mjs"), ); -if (process.platform !== "linux") - throw new Error( - "verify:firmware requires Linux; other hosts are not verified", - ); -for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { - const path = process.env[variable]; - if (path === undefined) - throw new Error( - `verify:firmware requires ${variable}; see docs/firmware-analysis.md`, - ); - await access(path); -} -try { - await exec("7z", ["i"], { timeout: 10_000 }); -} catch (cause) { - throw new Error( - "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", - { cause }, - ); -} -const fixtureRoot = resolve( +const FIXTURE_ROOT = resolve( process.env.REA_FIRMWARE_FIXTURE_ROOT ?? - join(repository, "_reference/firmware-integration/generated"), + join(REPO_ROOT, "_reference/firmware-integration/generated"), ); -await access(join(fixtureRoot, "oracle.json")).catch((cause) => { - throw new Error( - "verify:firmware requires generated fixtures; run npm run fixtures:firmware", - { cause }, - ); -}); + +// --------------------------------------------------------------------------- +// Prerequisite checks +// --------------------------------------------------------------------------- + +async function checkPrerequisites() { + if (process.platform !== "linux") { + throw new Error( + "verify:firmware requires Linux; other hosts are not verified", + ); + } + + for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { + const value = process.env[name]; + if (value === undefined) { + throw new Error( + `verify:firmware requires ${name}; see docs/firmware-analysis.md`, + ); + } + await fs.access(value); + } + + try { + await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS }); + } catch (cause) { + throw new Error( + "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", + { cause }, + ); + } + + try { + await fs.access(join(FIXTURE_ROOT, "oracle.json")); + } catch (cause) { + throw new Error( + "verify:firmware requires generated fixtures; run npm run fixtures:firmware", + { cause }, + ); + } +} + +await checkPrerequisites(); + +// --------------------------------------------------------------------------- +// Shared state +// --------------------------------------------------------------------------- + const oracle = JSON.parse( - await readFile(join(fixtureRoot, "oracle.json"), "utf8"), + await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"), ); -const path = join(fixtureRoot, "firmware.bin"); -assert.equal(await hashFirmwareFile(path), oracle.sha256); -const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-")); -const environment = Object.fromEntries( - Object.entries(process.env).filter(([, value]) => value !== undefined), +const firmwarePath = join(FIXTURE_ROOT, "firmware.bin"); +assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256); + +const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-")); +const baseEnv = Object.fromEntries( + Object.entries(process.env).filter(([, v]) => v !== undefined), ); -const client = new Client({ name: "rea-real-firmware-verifier", version: "1" }); -const cli = async (args, env = environment) => { - const response = await exec( +const mcpClient = new Client({ + name: "rea-real-firmware-verifier", + version: "1", +}); + +// --------------------------------------------------------------------------- +// Invocation helpers +// --------------------------------------------------------------------------- + +async function runCli(args, env = baseEnv) { + const { stdout } = await execFileAsync( process.execPath, - [entrypoint, ...args, "--format", "json"], + [ENTRYPOINT, ...args, "--format", "json"], { env, - cwd: repository, - timeout: 150_000, - maxBuffer: 16 * 1024 * 1024, + cwd: REPO_ROOT, + timeout: CALL_TIMEOUT_MS, + maxBuffer: MAX_STDOUT_BYTES, }, ); - return parseEvidence(JSON.parse(response.stdout)); -}; -const call = async (name, args) => { - const response = await client.callTool( + return parseEvidence(JSON.parse(stdout)); +} + +async function callTool(name, args) { + const result = await mcpClient.callTool( { name, arguments: args }, - { timeout: 150_000 }, + { timeout: CALL_TIMEOUT_MS }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - assert.ok(response.structuredContent !== undefined); - return response.structuredContent; -}; -const verifyRegions = (evidence) => { + assert.notEqual(result.isError, true, JSON.stringify(result)); + assert.ok(result.structuredContent !== undefined); + return result.structuredContent; +} + +// --------------------------------------------------------------------------- +// Result verifiers +// --------------------------------------------------------------------------- + +function verifyRegions(evidence) { assert.equal(evidence.subject.digest.sha256, oracle.sha256); - const result = firmwareResultSchemas.inspect_firmware_regions.parse( + + const parsed = firmwareResultSchemas.inspect_firmware_regions.parse( evidence.normalized_result, ); - const gzip = result.regions.find( - (region) => region.signature === "gzip" && region.offset === oracle.offset, + + const gzipRegion = parsed.regions.find( + (r) => r.signature === "gzip" && r.offset === oracle.offset, ); assert.ok( - gzip !== undefined, + gzipRegion !== undefined, "Binwalk must find the independently generated gzip offset", ); - assert.equal(gzip.reported_size, oracle.length); - assert.equal(gzip.size_basis, "provider_reported_validation_unknown"); - return result.regions.map( - ({ provider_id: _provider_id, ...region }) => region, - ); -}; -const verifyFiles = async (evidence) => { - const result = firmwareResultSchemas.extract_firmware.parse( + assert.equal(gzipRegion.reported_size, oracle.length); + assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown"); + + // Strip provider-specific ids so CLI and MCP results compare equal. + return parsed.regions.map((region) => { + const { provider_id: _ignored, ...rest } = region; + return rest; + }); +} + +async function verifyFiles(evidence) { + const parsed = firmwareResultSchemas.extract_firmware.parse( evidence.normalized_result, ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip")); + + assert.ok(parsed.chunks.some((c) => c.handler === "gzip")); + for (const [suffix, expected] of Object.entries(oracle.files)) { - const file = result.files.find((candidate) => - candidate.relative_path.endsWith(`/${suffix}`), + const entry = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(file !== undefined, `Missing extracted ${suffix}`); - assert.equal(file.sha256, expected.sha256); - assert.equal(file.size, expected.size); - assert.equal(await hashFirmwareFile(file.path), expected.sha256); - assert.equal(file.original_file_range, null); - assert.equal(file.runtime_address, null); + assert.ok(entry !== undefined, `Missing extracted ${suffix}`); + assert.equal(entry.sha256, expected.sha256); + assert.equal(entry.size, expected.size); + assert.equal(await hashFirmwareFile(entry.path), expected.sha256); + assert.equal(entry.original_file_range, null); + assert.equal(entry.runtime_address, null); } - return result; -}; -try { - const inspected = await cli(["inspect-firmware-regions", path]); - const cliRegions = verifyRegions(inspected); - const extracted = await cli([ - "extract-firmware", - path, - join(root, "cli-output"), + + return parsed; +} + +const toContentIdentity = (parsed) => + parsed.files.map(({ relative_path, sha256, size }) => ({ + relative_path, + sha256, + size, + })); + +// --------------------------------------------------------------------------- +// Verification stages +// --------------------------------------------------------------------------- + +async function stageCli() { + const regionsEvidence = await runCli([ + "inspect-firmware-regions", + firmwarePath, ]); - const cliFiles = await verifyFiles(extracted); - assert.equal(cliFiles.coverage, "partial"); + const regions = verifyRegions(regionsEvidence); + + const extractEvidence = await runCli([ + "extract-firmware", + firmwarePath, + join(workDir, "cli-output"), + ]); + const files = await verifyFiles(extractEvidence); + + assert.equal(files.coverage, "partial"); assert.ok( - cliFiles.chunks.some( - (chunk) => - chunk.handler === null && - chunk.range.offset === 0 && - chunk.range.length === 64, + files.chunks.some( + (c) => + c.handler === null && c.range.offset === 0 && c.range.length === 64, ), ); - await client.connect( - new StdioClientTransport({ - command: process.execPath, - args: [entrypoint, "mcp"], - cwd: repository, - env: environment, - stderr: "pipe", - }), - { timeout: 30_000 }, - ); - const inspectedMcp = await call("inspect_firmware_regions", { path }); - const mcpEvidence = parseEvidence(inspectedMcp); - assert.deepEqual(verifyRegions(mcpEvidence), cliRegions); - const extractedMcp = await call("extract_firmware", { - path, - output_directory: join(root, "mcp-output"), + + return { regions, files }; +} + +async function connectMcp() { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ENTRYPOINT, "mcp"], + cwd: REPO_ROOT, + env: baseEnv, + stderr: "pipe", }); - const mcpFiles = await verifyFiles(parseEvidence(extractedMcp)); - const contentIdentity = (result) => - result.files.map(({ relative_path, sha256, size }) => ({ - relative_path, - sha256, - size, - })); - assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles)); - const selected = await call("extract_firmware", { - path, - output_directory: join(root, "selected-output"), + await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS }); +} + +async function stageMcp(cli) { + const regionsRaw = await callTool("inspect_firmware_regions", { + path: firmwarePath, + }); + assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions); + + const extractRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "mcp-output"), + }); + const mcpFiles = await verifyFiles(parseEvidence(extractRaw)); + assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files)); + + const rangedRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "selected-output"), range: { offset: oracle.offset, length: oracle.length }, }); - const selectedFiles = await verifyFiles(parseEvidence(selected)); - assert.equal(selectedFiles.selection.offset, oracle.offset); - assert.equal(selectedFiles.selection.length, oracle.length); - const depth = await call("extract_firmware", { - path, - output_directory: join(root, "depth-output"), + const ranged = await verifyFiles(parseEvidence(rangedRaw)); + assert.equal(ranged.selection.offset, oracle.offset); + assert.equal(ranged.selection.length, oracle.length); + + const depthRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "depth-output"), max_depth: 1, }); - const depthResult = firmwareResultSchemas.extract_firmware.parse( - depth.normalized_result, + const depthParsed = firmwareResultSchemas.extract_firmware.parse( + depthRaw.normalized_result, ); - assert.equal(depthResult.coverage, "partial"); - assert.ok(depthResult.depth_limited_paths.length > 0); + assert.equal(depthParsed.coverage, "partial"); + assert.ok(depthParsed.depth_limited_paths.length > 0); + console.log( "PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits", ); - if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") { - const ext4 = join(fixtureRoot, "ext4.bin"); - await access(ext4); - const response = await call("extract_firmware", { - path: ext4, - output_directory: join(root, "ext4-output"), - }); - const result = firmwareResultSchemas.extract_firmware.parse( - response.normalized_result, + + return mcpFiles; +} + +async function stageExt4() { + const ext4Path = join(FIXTURE_ROOT, "ext4.bin"); + await fs.access(ext4Path); + + const raw = await callTool("extract_firmware", { + path: ext4Path, + output_directory: join(workDir, "ext4-output"), + }); + const parsed = firmwareResultSchemas.extract_firmware.parse( + raw.normalized_result, + ); + assert.ok(parsed.chunks.some((c) => c.handler === "extfs")); + assert.equal(parsed.coverage, "complete"); + + for (const [suffix, expected] of Object.entries(oracle.files)) { + const match = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs")); - assert.equal(result.coverage, "complete"); - for (const [suffix, expected] of Object.entries(oracle.files)) - assert.equal( - result.files.find((file) => file.relative_path.endsWith(`/${suffix}`)) - ?.sha256, - expected.sha256, - ); - console.log("PASS real ext4 extraction with debugfs; no mounts"); - // Preserve the process-ownership utility while withholding all extractors. - const withoutExtractors = join(root, "without-extractors"); - await access("/bin/ps"); - await mkdir(withoutExtractors); - await symlink("/bin/ps", join(withoutExtractors, "ps")); - const missing = await cli( - ["extract-firmware", ext4, join(root, "missing-extractor-output")], - { ...environment, PATH: withoutExtractors }, - ); - const partial = firmwareResultSchemas.extract_firmware.parse( - missing.normalized_result, - ); - assert.equal(partial.coverage, "partial"); - assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs")); - console.log( - "PASS real missing-extractor diagnostics retained despite Unblob exit 1", - ); - } - if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") { - if (process.env.GHIDRA_INSTALL_DIR === undefined) - throw new Error( - "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", - ); - const child = mcpFiles.files.find((file) => - file.relative_path.endsWith("/usr/bin/probe"), - ); - assert.ok(child !== undefined); - await call("open_binary", { path: child.path, provider_id: "ghidra" }); - const dossier = await call("analyze_function", { - procedure: "firmware_probe", - }); - const evidence = parseEvidence(dossier); - assert.equal( - evidence.subject.digest.sha256, - oracle.files["usr/bin/probe"].sha256, - ); - const pseudocode = await call("procedure_pseudo_code", { - procedure: "firmware_probe", - }); - assert.equal(parseEvidence(pseudocode).provider.id, "ghidra"); - assert.ok( - JSON.stringify(evidence.normalized_result).includes("firmware_probe"), - ); - await call("close_binary", {}); - console.log( - "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + assert.equal(match?.sha256, expected.sha256); + } + console.log("PASS real ext4 extraction with debugfs; no mounts"); + + // Keep the process-ownership utility available but hide every extractor. + const emptyBin = join(workDir, "without-extractors"); + await fs.access("/bin/ps"); + await fs.mkdir(emptyBin); + await fs.symlink("/bin/ps", join(emptyBin, "ps")); + + const degraded = await runCli( + ["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")], + { ...baseEnv, PATH: emptyBin }, + ); + const degradedParsed = firmwareResultSchemas.extract_firmware.parse( + degraded.normalized_result, + ); + assert.equal(degradedParsed.coverage, "partial"); + assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs")); + console.log( + "PASS real missing-extractor diagnostics retained despite Unblob exit 1", + ); +} + +async function stageGhidra(mcpFiles) { + if (process.env.GHIDRA_INSTALL_DIR === undefined) { + throw new Error( + "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", ); } + + const probe = mcpFiles.files.find((f) => + f.relative_path.endsWith("/usr/bin/probe"), + ); + assert.ok(probe !== undefined); + + await callTool("open_binary", { path: probe.path, provider_id: "ghidra" }); + + const dossierRaw = await callTool("analyze_function", { + procedure: "firmware_probe", + }); + const dossier = parseEvidence(dossierRaw); assert.equal( - await hashFirmwareFile(path), + dossier.subject.digest.sha256, + oracle.files["usr/bin/probe"].sha256, + ); + + const pseudoRaw = await callTool("procedure_pseudo_code", { + procedure: "firmware_probe", + }); + assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra"); + assert.ok( + JSON.stringify(dossier.normalized_result).includes("firmware_probe"), + ); + + await callTool("close_binary", {}); + console.log( + "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + ); +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +try { + const cli = await stageCli(); + await connectMcp(); + const mcpFiles = await stageMcp(cli); + + if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4(); + if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") + await stageGhidra(mcpFiles); + + assert.equal( + await hashFirmwareFile(firmwarePath), oracle.sha256, "Original firmware bytes must remain unchanged", ); } finally { - await client.close(); - await rm(root, { recursive: true, force: true }); + await mcpClient.close(); + await fs.rm(workDir, { recursive: true, force: true }); } From fcfa5b7e88e2251359950d3fcca8a1ae61dc6814 Mon Sep 17 00:00:00 2001 From: Jeongwan Jeon Date: Tue, 13 Oct 2026 21:27:38 +0900 Subject: [PATCH 09/28] fix(artifacts): distinguish JVM classes from universal Mach-O files (#2327) * test(artifacts): reproduce JVM class inventory misclassification * fix(artifacts): validate FAT declarations before classifying executables --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- .../ArtifactGraphConstruction.test.ts | 103 ++++++++++++++++ src/domain/apple/machoContainer.ts | 5 + .../acceptance/artifactClassification.test.ts | 111 ++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 src/artifacts/inventory/ArtifactGraphConstruction.test.ts create mode 100644 tests/acceptance/artifactClassification.test.ts diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts new file mode 100644 index 000000000..925d3543a --- /dev/null +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; + +import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js"; +import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; + +const fat = (wide = false): Buffer => + Buffer.from(fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }], wide)); + +describe("bounded FAT inventory classification", () => { + it("requires the complete table but not slice bytes in the retained prefix", () => { + const bytes = fat(); + for (const length of [4, 7, 8, 27]) { + expect( + classifyArtifactContent( + "native", + bytes.subarray(0, length), + bytes.length, + ), + ).toEqual({ kind: "unknown", format: "unknown" }); + } + expect( + classifyArtifactContent( + "native.class", + bytes.subarray(0, 28), + bytes.length, + ), + ).toEqual({ kind: "executable", format: "mach-o-universal" }); + }); + + it.each([ + ["empty table", (bytes: Buffer) => bytes.writeUInt32BE(0, 4)], + ["slice overlapping table", (bytes: Buffer) => bytes.writeUInt32BE(0, 16)], + ["empty slice", (bytes: Buffer) => bytes.writeUInt32BE(0, 20)], + [ + "slice beyond file", + (bytes: Buffer) => bytes.writeUInt32BE(bytes.length, 20), + ], + ["unaligned slice", (bytes: Buffer) => bytes.writeUInt32BE(4097, 16)], + ["unsupported alignment", (bytes: Buffer) => bytes.writeUInt32BE(53, 24)], + ] as const)("keeps %s unknown", (_label, mutate) => { + const bytes = fat(); + mutate(bytes); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("does not infer Mach-O from an out-of-bounds architecture count", () => { + const bytes = fat(); + bytes.writeUInt32BE(0xffffffff, 4); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "resource", + format: "file", + }); + }); + + it("does not lose unsafe FAT64 range values to number conversion", () => { + const bytes = fat(true); + bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires valid ranges for every declared slice", () => { + const bytes = Buffer.from( + fatImage([ + { cpu: CPU.arm64, bytes: machoImage({}) }, + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + ]), + ); + bytes.writeUInt32BE(bytes.length, 40); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires observed slice magic for an unknown CPU type", () => { + const cpu = { type: 0x12345678, subtype: 0 }; + const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }])); + expect( + classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length), + ).toEqual({ kind: "unknown", format: "unknown" }); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "executable", + format: "mach-o-universal", + }); + }); + + it.each([NaN, Infinity, -1, 8.5, Number.MAX_SAFE_INTEGER + 1, 27])( + "keeps invalid or contradictory file size %s unknown", + (size) => { + const bytes = fat(); + expect(classifyArtifactContent("native", bytes, size)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }, + ); +}); diff --git a/src/domain/apple/machoContainer.ts b/src/domain/apple/machoContainer.ts index 7b552be64..c23cf7f64 100644 --- a/src/domain/apple/machoContainer.ts +++ b/src/domain/apple/machoContainer.ts @@ -63,6 +63,11 @@ export const readFatContainerHeader = ( const magic = view.getUint32(0, false); if (![FAT_MAGIC, FAT_MAGIC_64, FAT_CIGAM, FAT_CIGAM_64].includes(magic)) return { state: "not-fat" }; + if (!Number.isSafeInteger(fileSize) || fileSize < bytes.byteLength) + return { + state: "malformed", + reason: "FAT file size is invalid or contradicts retained bytes", + }; if (bytes.byteLength < 8) { return fileSize < 8 ? { state: "malformed", reason: "FAT header is truncated" } diff --git a/tests/acceptance/artifactClassification.test.ts b/tests/acceptance/artifactClassification.test.ts new file mode 100644 index 000000000..7436a4967 --- /dev/null +++ b/tests/acceptance/artifactClassification.test.ts @@ -0,0 +1,111 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect } from "vitest"; + +import { + CPU, + fatImage, + machoImage, +} from "../../src/artifacts/apple/MachoImage.fixture.js"; +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest.for(["jar", "zip"])( + "distinguishes JVM classes from universal Mach-O files in %s inventory", + async (extension, { cli }) => { + const root = await createTestTempDirectory("rea-class-inventory-"); + const path = join(root, `mixed.${extension}`); + // The large class can hold the entire 65-entry table suggested by its + // version bytes. Checking only the table length does not distinguish it. + const classes = [ + ["Hello.class", javaClass()], + ["Big.class", javaClass("x".repeat(4_000))], + ["renamed.bin", javaClass()], + ] as const; + const writer = new ZipWriter(new Uint8ArrayWriter()); + for (const [name, bytes] of classes) + await writer.add(name, new Uint8ArrayReader(bytes)); + const nativePaths: string[] = []; + for (const wide of [false, true]) { + for (const littleEndian of [false, true]) { + const name = `fat${wide ? 64 : 32}-${littleEndian ? "le" : "be"}`; + nativePaths.push(name); + await writer.add( + name, + new Uint8ArrayReader( + fatImage( + [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], + wide, + littleEndian, + ), + ), + ); + } + } + await writeFile(path, await writer.close()); + + const reply = await cli.run({ + arguments: ["inspect-artifact", path, "--json"], + }); + expect(reply.exitCode, reply.stdout + reply.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.json).normalized_result, + ); + const inventory = artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ); + for (const name of nativePaths) { + expect(inventory.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: name, + artifact_kind: "executable", + artifact_format: "mach-o-universal", + limitations: [], + }), + ); + } + for (const [name, bytes] of classes) { + const occurrence = inventory.occurrences.find( + (item) => item.logical_path === name, + ); + expect(occurrence).toBeDefined(); + expect.soft(occurrence?.artifact_kind, name).not.toBe("executable"); + expect.soft(occurrence?.artifact_format, name).not.toMatch(/^mach-o/u); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const node = inventory.nodes.find((item) => item.sha256 === sha256); + expect(node).toBeDefined(); + expect.soft(node?.format, name).not.toMatch(/^mach-o/u); + } + }, +); + +/** Minimal valid Java 21 class, with an optional unused UTF-8 constant. */ +const javaClass = (extra?: string): Buffer => { + const utf8 = (value: string): Buffer => { + const text = Buffer.from(value); + const header = Buffer.alloc(3); + header[0] = 1; + header.writeUInt16BE(text.length, 1); + return Buffer.concat([header, text]); + }; + const constants = [ + Buffer.from([7, 0, 2]), + utf8("Hello"), + Buffer.from([7, 0, 4]), + utf8("java/lang/Object"), + ...(extra === undefined ? [] : [utf8(extra)]), + ]; + const header = Buffer.from("cafebabe000000410000", "hex"); + header.writeUInt16BE(constants.length + 1, 8); + return Buffer.concat([ + header, + ...constants, + Buffer.from("0021000100030000000000000000", "hex"), + ]); +}; From d85374496965a8a997e1d13e98a27ff5e5ca1077 Mon Sep 17 00:00:00 2001 From: Jeongwan Jeon Date: Tue, 13 Oct 2026 21:28:31 +0900 Subject: [PATCH 10/28] fix: preserve process identities in pwntools cleanup errors (#2284) Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/native/pwntools/PwntoolsDecoder.ts | 23 ++++++++-------- src/process/OwnedCommand.ts | 19 +++++++++----- tests/boundary/process/ownedCommand.test.ts | 8 +++--- .../pwntoolsStageRetirement.test.ts | 26 ++++++++++++------- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/src/native/pwntools/PwntoolsDecoder.ts b/src/native/pwntools/PwntoolsDecoder.ts index 169f25601..6aa456631 100644 --- a/src/native/pwntools/PwntoolsDecoder.ts +++ b/src/native/pwntools/PwntoolsDecoder.ts @@ -32,7 +32,10 @@ import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorClean import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { runOwnedCommand } from "../../process/OwnedCommand.js"; +import { + ownedCommandResources, + runOwnedCommand, +} from "../../process/OwnedCommand.js"; import type { ProviderProcessSupervisor, OwnedProviderProcessSpawnOptions, @@ -562,20 +565,16 @@ export class PwntoolsDecoder< const pending = this.#pendingCleanup; if (pending?.process === undefined) return undefined; const stopped = await pending.process.stop(); + const ownership = pending.process.launch.ownership; if (stopped.status === "incomplete") return new ProviderCleanupError( this.identity.id, - [ - pending.process.launch.ownership?.runId ?? "owned-decoder", - ...(pending.root === undefined ? [] : [pending.root.path]), - // The retained group may still be live; name it as OwnedCommand does. - ...(pending.process.launch.ownership === undefined - ? [] - : [ - `pid:${pending.process.launch.ownership.leaderPid}`, - `process-group:${pending.process.launch.ownership.processGroupId}`, - ]), - ], + ownership === undefined + ? [ + "owned-decoder", + ...(pending.root === undefined ? [] : [pending.root.path]), + ] + : ownedCommandResources(ownership, pending.root?.path), { reason: stopped.reason, captured_output: { diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts index 02fb4defc..d375971ae 100644 --- a/src/process/OwnedCommand.ts +++ b/src/process/OwnedCommand.ts @@ -9,6 +9,7 @@ import { } from "./ProviderProcess.js"; import { cleanupOwnedProcessGroup, + type OwnedProcessGroup, type ProcessCleanupFailure, } from "./ProcessOwnership.js"; import { @@ -34,6 +35,17 @@ export class OwnedCommandFailure extends Error { } } +/** Report recorded launch identities; these labels do not authorize signaling. */ +export const ownedCommandResources = ( + ownership: OwnedProcessGroup, + directory?: string, +): readonly string[] => [ + ownership.runId, + ...(directory === undefined ? [] : [directory]), + `pid:${ownership.leaderPid}`, + `process-group:${ownership.processGroupId}`, +]; + /** Execute a single owned process with independent cleanup and bounded diagnostic retention. */ export const runOwnedCommand = async ( spawn: OwnedProviderProcessSpawnOptions, @@ -208,12 +220,7 @@ const collectOwnedCommand = async ( ? {} : { cleanupResult: stopped.cleanupResult }), }, - [ - spawn.runId, - ...(spawn.cwd === undefined ? [] : [spawn.cwd]), - `pid:${launched.ownership.leaderPid}`, - `process-group:${launched.ownership.processGroupId}`, - ], + ownedCommandResources(launched.ownership, spawn.cwd), supervisor, ); if (failure !== undefined) diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts index 34205af3d..ca0cb2db6 100644 --- a/tests/boundary/process/ownedCommand.test.ts +++ b/tests/boundary/process/ownedCommand.test.ts @@ -257,9 +257,11 @@ it.skipIf(process.platform === "win32").each(["cancelled", "timeout"] as const)( failures: [expect.objectContaining({ reason: "run-token-mismatch" })], }); expect(failure.cleanupResult).toEqual(cleanup); - expect(failure.resources).toEqual( - expect.arrayContaining([request.runId]), - ); + expect(failure.resources).toEqual([ + request.runId, + `pid:${launched.ownership.leaderPid}`, + `process-group:${launched.ownership.processGroupId}`, + ]); // An unresolved member does not grant authority to signal the shared group. const acquiredPid = launched.ownership.leaderPid; expect(() => process.kill(acquiredPid, 0)).not.toThrow(); diff --git a/tests/process-global/pwntoolsStageRetirement.test.ts b/tests/process-global/pwntoolsStageRetirement.test.ts index 6ec062535..7cdf0b48a 100644 --- a/tests/process-global/pwntoolsStageRetirement.test.ts +++ b/tests/process-global/pwntoolsStageRetirement.test.ts @@ -18,7 +18,10 @@ import { PWNTOOLS_MEMORY_FAILURE_EXIT, PWNTOOLS_PROVIDER_IDENTITY, } from "../../src/native/pwntools/PwntoolsRelease.js"; -import { cleanupOwnedProcessGroup } from "../../src/process/ProcessOwnership.js"; +import { + cleanupOwnedProcessGroup, + type OwnedProcessGroup, +} from "../../src/process/ProcessOwnership.js"; import { prepareProcessOwnershipInspection } from "../../src/process/ProcessOwnershipObservation.js"; import { spawnOwnedProviderProcess } from "../../src/process/ProviderProcess.js"; import { PrivateRuntimeRoot } from "../../src/process/PrivateRuntimeRoot.js"; @@ -232,12 +235,13 @@ it.runIf(supportedHost).each([ const { path, root, environment, basic } = await fixture(); let launches = 0; let cleanupAllowed = false; + let retainedOwnership: OwnedProcessGroup | undefined; const read = vi.spyOn(artifacts, "readStableArtifact"); const provider = new PwntoolsRecordedCrashProvider( environment, async (spawn) => { if (stage === "debugger" && ++launches === 1) return basic(spawn); - return changingWorker( + const launched = await changingWorker( spawn, join(spawn.cwd ?? "", "resource.failure"), { @@ -247,6 +251,8 @@ it.runIf(supportedHost).each([ }, () => cleanupAllowed, ); + retainedOwnership = launched.ownership; + return launched; }, async () => root, ); @@ -261,14 +267,14 @@ it.runIf(supportedHost).each([ if (result.ok) throw new Error("Expected unresolved process ownership"); const projected = projectAnalysisError(result.error); expect(projected.code).toBe("cleanup_incomplete"); - expect(projected).toMatchObject({ - details: { - resources: expect.arrayContaining([ - expect.stringMatching(/^pid:\d+$/u), - expect.stringMatching(/^process-group:\d+$/u), - ]), - }, - }); + if (retainedOwnership === undefined) + throw new Error("Expected retained process ownership"); + expect(projected.details?.resources).toEqual([ + retainedOwnership.runId, + root.path, + `pid:${retainedOwnership.leaderPid}`, + `process-group:${retainedOwnership.processGroupId}`, + ]); if (stage === "debugger") expect(projected).toMatchObject({ details: { From 2f3dee3fcf65813d6daf7c3209d921451c160ddb Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:37:48 +0800 Subject: [PATCH 11/28] test(javascript): verify Windows path case and load scope (#2635) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select the Windows curated lane for edits to the shared runtime path matcher and its workflow suite. Strengthen the existing POSIX/Windows mapping controls with matching basis, coherent target/script coordinates, caller spelling and static load-state assertions. Follow-up to #2422; thanks to @Aurantium-47 for identifying the recorded-path grammar issue. Verified with actual Windows 11 x64, official Node 24.18.0 and production CLI/MCP workflows using published npm 11.16.0 module Evidence. Four exact/mismatched implicit/explicit mapping cases pass with unchanged normalized hashes. The capture provider was Node Inspector; no IDA, Ghidra or Electron GUI verification is claimed. 230 focused tests, all eight CI static tasks, architecture guards and documentation checks pass after rebasing onto main fb28ec153. The unrelated release-checkpoint and dyld fixture corrections were integrated from upstream; no patches to those files are included here. Final Windows MCP replay at e6e86946d preserves all four normalized hashes, with no owned Node processes remaining. Resource probes ran serially at BelowNormal priority with one CPU and 512 MiB Node heaps; calls took 251–620 ms and the sampled process-family working-set sum peaked at 499,380,224 bytes. Source pins, provenance and measurement limits are recorded in experiments/analysis-efficiency/windows-posix-reconciliation.md. All 34 jobs in current-head CI run 38391921571 passed, including the Windows capability/package lane, all six portable test shards, JavaScript recovery and the required aggregate. Current-head Codex and Greptile reviews are clean; both prior scope-selection findings were fixed and resolved. --- .github/workflows/ci.yml | 1 + .../windows-posix-reconciliation.md | 128 +++++++++++++++ scripts/ci/scopes.mjs | 2 + .../boundary/filesystem/ciChangeScope.test.ts | 4 + .../javascriptApplicationWorkflows.test.ts | 154 ++++++++++-------- 5 files changed, 224 insertions(+), 65 deletions(-) create mode 100644 experiments/analysis-efficiency/windows-posix-reconciliation.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0516da85..d2cc59493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -597,6 +597,7 @@ jobs: tests/boundary/filesystem/setupUnavailableProvider.test.ts tests/boundary/filesystem/windowsNpxRegistration.test.ts tests/acceptance/setup/homeDirectoryFallback.test.ts + tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts tests/boundary/filesystem/webScriptManifestPaths.test.ts tests/boundary/process/ProcessEvidence.host.test.ts tests/acceptance/webModuleCli.test.ts diff --git a/experiments/analysis-efficiency/windows-posix-reconciliation.md b/experiments/analysis-efficiency/windows-posix-reconciliation.md new file mode 100644 index 000000000..0df06ac3e --- /dev/null +++ b/experiments/analysis-efficiency/windows-posix-reconciliation.md @@ -0,0 +1,128 @@ +# Windows reconciliation of recorded POSIX paths + +Follow-up to [#2422](https://github.com/morluto/rea/pull/2422), including +@Aurantium-47's diagnosis of the host-dependent relative-path operation. + +## Source pins and real data + +The initial reproduction and passive producer used main +fa4b8288655188426222773ead2ccbdaf76334fd. Before branching, main was updated +to 8b181716aca2cb467be499cba226abfb5ae19141. The owning matcher and its +affected reconciliation consumers were unchanged between those revisions. +The newer baseline again reproduced both errors on the real Windows host. + +Input: npm 11.16.0's published lib/utils/validate-lockfile.js, 1,023 bytes, +SHA-256 2295e2dac1c5816e1eae2d417804960ebbef17b64a0a44db4f3483ce6e6d9526. +The installed copy was independently checked against the exact module in the +[official registry package](https://registry.npmjs.org/npm/11.16.0). +The 2,945,850-byte tarball matched its published SHA-512 SRI before reading +that member; no archive extraction or installation occurred, and the tarball +was not retained. + +Two actual Linux filesystem directories, Runtime/App and Runtime/app, +held byte-identical module copies and had distinct directory identities. +A controlled Node 24.18.0 process loaded the public module and ran its real +function against 587 locked dependency records: compatible trees returned zero +errors, and one controlled version mismatch returned one error. A passive REA +Inspector capture recorded 114 loaded scripts, including the selected module. +The provider did not evaluate, pause, resume or read source. The owned process +exited normally. Function execution is established by the harness receipt; +REA's passive script metadata establishes loaded status. + +Production static analysis inspected both directories plus an independent +Static/public-module copy. The latter prevents implicit root matching from +hiding the explicit mapping result. Complete static/runtime Evidence was then +consumed by the actual **Windows** CLI and SDK 2.3.1 stdio MCP server, using +Windows 11 10.0.22631.0 and official Windows x64 Node 24.18.0. +No Electron GUI, IDA or Ghidra engine was used. + +## Correctness and provenance + +| Windows reconciliation | Baseline | Candidate | +| -------------------------------------- | ------------------------------ | ----------------------------------- | +| Exact POSIX input root | Matched, artifact path | Same complete normalized result | +| Case-mismatched POSIX input root | Incorrectly matched and loaded | Unknown; static load states unknown | +| Exact explicit POSIX mapping | Matched, operator file mapping | Same complete normalized result | +| Case-mismatched explicit POSIX mapping | Incorrectly matched and loaded | Unknown; static load states unknown | + +All four CLI and same-connection MCP normalized results agree. The input +Evidence, recorded runtime facts and caller-selected root spellings are +preserved. Rejected location membership does not establish that a module was +absent or unexecuted. The operation remains offline and performs no target or +filesystem inspection. + +The shared matcher now chooses posix.relative after identifying POSIX input; +Windows coordinates continue to use win32.relative. Both location matching +and active-layer scope use this owner. The obsolete host-default import is +removed; no downstream repair or new path abstraction is introduced. + +The related suite passed 102 tests: 15 workflow tests, 85 CI scope tests and +two curated-path checks. Four simulated Electron Evidence controls cover +POSIX mismatch/exact matching, Windows case-insensitive matching, the resulting +asset load state and retained mapping spelling. They are service regressions, +not real Electron-provider verification. The existing workflow test file is +registered in both the Windows execution lane and its package-change scope. + +## Delivery and resource observations + +Probes ran serially at BelowNormal priority with one CPU and 512 MiB Node heap +settings for CLI/server/client work. Heap settings do not bound aggregate memory. +Four corrected CLI exit-time observations reported 7.343–15.827 seconds CPU and +322,308–324,280 KiB peak working set, excluding the probe parent. Wall times +ranged from 7.785 to 17.601 seconds. There is no paired resource baseline. + +One complete MCP trial connected in 18,296 ms and discovered 189 tools in +875 ms (2,588,336 serialized bytes). The four subsequent same-connection calls +took 879, 320, 353 and 422 ms. Their serialized replies were 918,133 / 913,765 / +918,571 / 914,171 bytes; inputs totalled 681,312 bytes. Discovery plus these +inputs and replies totalled 6,934,288 bytes, excluding initialization, framing +and notifications. This is delivery volume, not model token usage. Actual +model context and total model task cost were not measured. + +A process-generation sampler recorded 233 observations over 25,671 ms: +15,797 ms aggregate observed CPU as a lower bound and a peak sampled working-set +sum of 491,597,824 bytes (about 469 MiB). Exited-process tails can be missed; +shared pages can be counted twice. This excludes the observer and is not an +exact physical-memory peak. The server wrote four structured completion records +to stderr; tool replies and normalized results passed the recorded checks. + +Cold/warm caches and observer differences were not normalized. No latency, +CPU, memory or model-token improvement is claimed. The fix adds no lookup I/O, +engine invocation, retry or cache. Complete default replies remain available; +only compact selected findings and hashes were sent to the investigating agent. + +## Main integration replay + +Runtime cb0d8a9fef2b907cff2ba27f1e017493e1173eb2, rebased onto main +73dd8d6ed, again passed the 101 related tests and all four real Windows CLI/MCP +cases using the unchanged older producer Evidence. Positive results and original +runtime/layer facts remain identical. Typecheck, lint, dependency and metadata +checks passed; all five changed files passed formatting independently. +The full format gate at that main revision failed on the unchanged upstream +GhidraDoctor.test.ts and GhidraSnapshotCache.test.ts files. The PR does not +repair those unrelated baseline files. Resource figures above refer to the +earlier candidate; the integration replay does not establish a speedup. + +Review found that owning only the Windows test path would still skip that lane +on a later implementation-only edit. The matcher implementation is now also +registered in package scope. A source-only classifier case verifies that it +selects both Windows package and JavaScript recovery lanes; 85 scope checks and +two curated-path checks pass. Runtime code and the real-data checks are unchanged. + +The original production fix merged as +[1a1b67c88](https://github.com/morluto/rea/commit/1a1b67c88) while this follow-up +was open. The follow-up was then rebased onto main fb28ec153 and contains only +the stronger workflow assertions, Windows lane ownership, scope classification +and this verification record. The updated 131 related tests, two representative +Ghidra boundary tests, seven IDA input-thinning tests, 24 semantic-structure +tests, 16 required-lane checks, 39 release-checkpoint tests and 11 dyld CLI +tests passed after the rebase (230 total). +Upstream a0317cf73 corrected the invalid FAT fixture and consolidated unrelated +duplicated helpers; e05d3399f aligned the older parameter-provenance assertion +with the upstream semantics. Those changes, the threshold change and the new +CI qualification tiers are part of main, outside this follow-up's diff. +Upstream 747e72991 restored release preflight coverage for reusable workflow +jobs. The independently reviewed, fully green #2755 fixture correction merged +as fb28ec153 and restored real x64 dyld CLI admission without altering the +original same-named arm64e assertions. +No production source is duplicated by the follow-up. diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index a8679726c..ffe15cb3d 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -56,6 +56,8 @@ export const SCOPES = { "tests/boundary/filesystem/setupUnavailableProvider.test.ts", "tests/boundary/filesystem/windowsNpxRegistration.test.ts", "tests/acceptance/setup/homeDirectoryFallback.test.ts", + "tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts", + "src/domain/javascript/javascriptRuntimeStaticCandidates.ts", "tests/boundary/filesystem/webScriptManifestPaths.test.ts", "tests/boundary/process/ProcessEvidence.host.test.ts", "tests/acceptance/webModuleCli.test.ts", diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 163219c39..321160a36 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -515,6 +515,10 @@ it.each([ "src/windows/Example.ts", { code: true, runtime: true, package: true, apple: false, evm: false }, ], + [ + "src/domain/javascript/javascriptRuntimeStaticCandidates.ts", + { code: true, runtime: true, package: true, javascript_recovery: true }, + ], ["install.sh", { code: true, runtime: true, package: true, evm: false }], [ ".agents/skills/reverse-engineer-anything/SKILL.md", diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index b79407821..8393bad54 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -444,73 +444,97 @@ describe("JavaScript application workflow error diagnostics", () => { }); }); +const RUNTIME_PATH_CASES = [ + { + name: "case-sensitive POSIX input root", + root: null, + path: "/applications/Example.app/Contents/Resources/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "case-sensitive POSIX mapping root", + root: "/runtime/App", + path: "/runtime/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "matching POSIX mapping root", + root: "/runtime/App", + path: "/runtime/App/renderer.js", + status: "matched", + basis: "operator-file-mapping", + }, + { + name: "case-insensitive Windows mapping root", + root: String.raw`C:\runtime\App`, + path: String.raw`c:\runtime\app\renderer.js`, + status: "matched", + basis: "operator-file-mapping", + }, +] as const; + describe("JavaScript runtime reconciliation local paths", () => { - it.each([ - { - name: "case-sensitive POSIX input root", - root: null, - path: "/applications/Example.app/Contents/Resources/app/renderer.js", - status: "unknown", - }, - { - name: "case-sensitive POSIX mapping root", - root: "/runtime/App", - path: "/runtime/app/renderer.js", - status: "unknown", - }, - { - name: "matching POSIX mapping root", - root: "/runtime/App", - path: "/runtime/App/renderer.js", - status: "matched", - }, - { - name: "case-insensitive Windows mapping root", - root: String.raw`C:\runtime\App`, - path: String.raw`c:\runtime\app\renderer.js`, - status: "matched", - }, - ])("uses $name on every host", ({ root, path, status }) => { - const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; - const original = base.runtime_observations[0]!; - const inspection = electronPageInspectionSchema.parse( - original.normalized_result, - ); - const runtime = createEvidence(undefined, original.provider, { - predicateType: original.predicate_type, - operation: original.operation, - parameters: { ...original.parameters, include_script_sources: false }, - result: { - ...inspection, - scripts: { - ...inspection.scripts, - items: inspection.scripts.items.map((script) => ({ - ...script, - file_path: path, - source: { included: false, reason: "not-requested" }, - })), + it.each(RUNTIME_PATH_CASES)( + "uses $name on every host", + ({ root, path, status, basis }) => { + const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; + const original = base.runtime_observations[0]!; + const inspection = electronPageInspectionSchema.parse( + original.normalized_result, + ); + // Keep target and script coordinates consistent so an unrelated target + // cannot activate the static layer after its script mapping was rejected. + const runtime = createEvidence(undefined, original.provider, { + predicateType: original.predicate_type, + operation: original.operation, + parameters: { ...original.parameters, include_script_sources: false }, + result: { + ...inspection, + target: { ...inspection.target, file_path: path }, + scripts: { + ...inspection.scripts, + items: inspection.scripts.items.map((script) => ({ + ...script, + file_path: path, + source: { included: false, reason: "not-requested" }, + })), + }, }, - }, - confidence: original.confidence, - authority: original.authority, - }); - const reconciled = reconcileJavaScriptRuntimeEvidence({ - static_layers: base.static_layers.map((layer) => ({ - ...layer, - runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], - })), - runtime_observations: [runtime], - }); - if (!reconciled.ok) throw reconciled.error; - const result = javascriptRuntimeReconciliationResultSchema.parse( - reconciled.value.normalized_result, - ); - expect( - result.reconciliations.find( - ({ entity_kind }) => entity_kind === "script", - ), - ).toMatchObject({ status }); - }); + confidence: original.confidence, + authority: original.authority, + }); + const reconciled = reconcileJavaScriptRuntimeEvidence({ + static_layers: base.static_layers.map((layer) => ({ + ...layer, + runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], + })), + runtime_observations: [runtime], + }); + if (!reconciled.ok) throw reconciled.error; + const result = javascriptRuntimeReconciliationResultSchema.parse( + reconciled.value.normalized_result, + ); + expect( + result.reconciliations.find( + ({ entity_kind }) => entity_kind === "script", + ), + ).toMatchObject({ status, basis }); + expect( + result.static_load_states.find( + ({ kind }) => kind === "javascript-asset", + ), + ).toMatchObject( + status === "matched" + ? { status: "loaded", reason: "runtime-script-correspondence" } + : { status: "unknown", reason: "layer-outside-runtime-scope" }, + ); + expect(result.static_layers[0]?.runtime_mappings).toEqual( + root === null ? [] : [{ kind: "file-root", root, artifact_prefix: "" }], + ); + }, + ); it("resolves relative file mapping roots before deriving result Evidence", () => { const absoluteRoot = resolve(process.cwd(), "runtime-cache-fixture"); From 8823c55b0ce325cf9edb7d2f9aadaf39db4e8f31 Mon Sep 17 00:00:00 2001 From: Omisakin Joshua <117924167+thefolahan@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:41:10 +0100 Subject: [PATCH 12/28] feat(ghidra): decode Swift String literals in function dossiers (#2049) * feat(ghidra): decode Swift String literals in function dossiers Swift passes a String as two words. Small literals keep their UTF-8 bytes inside those words, and immortal large literals keep a count plus a pointer to their bytes biased by 32. Ghidra creates no data reference for either, so analyze_function reported no strings for Swift code. On AArch64 programs that contain Swift metadata, the Ghidra bridge now runs constant propagation over the function and decodes register pairs where Swift consumes a String: consecutive argument registers at calls and tail calls, x0 and x1 at returns, and both registers of one STP. Large literals must resolve to terminated bytes inside one initialized block that is not executable. A pair whose registers both come from conditional selects is skipped, because Ghidra forks each select without correlating the condition and could otherwise pair words from different cases. Decoded values are a derivation from the ABI layout rather than observed references, so they appear in a new optional swift_string_literals field with their source instruction, registers and raw words, and in a matching inspect_analysis_view facet. referenced_strings is unchanged. Refs #2030 * fix(ghidra): bound Swift literal reads and compare decoded strings Large immortal literals are measured against the remaining heap before allocation, and a failed fit stays visible as a limitation. The canonical empty string is kept. Small and large literals now require matching address nullability, and function comparison treats decoded literals as string evidence without equating a missing record to an empty one. * fix(ghidra): budget Swift literal decoding and tighten use sites Large literals now draw on one heap budget per dossier, measured once before decoding, instead of a separate check per literal. A literal that does not fit is listed in a new swift_string_literal_omissions field with its address, byte length and reason, so consumers read omissions from structured data rather than from limitation text. strings_names comparison checks referenced strings and names first and reports a change there even when Swift literal coverage differs; it returns unknown only when those facts match and literal coverage is absent on one side or partial. A computed jump counts as a tail call only when it has no resolved target inside the function, so jump tables no longer qualify, and a return requires a RETURN p-code operation, so traps no longer qualify. Decoded literals keep control characters, which Swift strings may contain and JSON escapes. The Swift verifier builds the literal fixture for arm64 on any Mac host and now also covers the empty string and a literal containing a tab. * fix(ghidra): charge escaped JSON and follow copied selects The Swift literal budget now charges the worst case per UTF-8 byte: UTF-16 text, six escaped JSON characters held in Gson's growing buffer and in the returned String, and the UTF-8 bytes written to the socket. Literals made of control characters can no longer exceed the reserved heap. A jump counts as a tail call only when every resolved target lies outside the function. The conditional select check follows register copies back to their definition, so selects copied into argument registers are recognized like selects written there directly. strings_names reports a change when both dossiers decoded different literals at the same use site, even when either side has omissions; it returns unknown only when no such use site disagrees. * fix(ghidra): bound literal records and trace selects across branches Every Swift literal record now pays a fixed cost against the dossier budget in addition to its text, so many empty or small literals are bounded as well. The first record that does not fit is kept as an omission and decoding stops there, which keeps the omission record itself bounded; omissions now describe small or large literals. The conditional select check searches every predecessor path for the reaching definitions, following register copies, so selects made before a branch into the use block are recognized. A dossier records Swift literals and their omissions together. Coverage reads omissions first, and strings_names compares every use site that neither side omitted: a decoded literal is known where it was recorded, and an absence is known only in a dossier whose decoding examined every use site. * fix(ghidra): stop every flow at the literal budget and route omissions Ending one flow from the evaluator leaves other queued flows running, so the budget now records a shared stop that every flow checks before it examines another instruction. A record that exceeds the budget is routed by that capacity result rather than by whether text was decoded, so a small literal over budget becomes an omission instead of a literal. When coverage is partial, the per site comparison keeps each literal's representation as well as its value, matching the full strings_names projection. * fix(ghidra): keep unresolved Swift tail-call sites unknown --------- Co-authored-by: Cursor Agent Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- bridge/ghidra/ReaGhidraBridge.java | 380 ++++++++++++++++++ scripts/verify-real-ghidra-swift.mjs | 68 ++++ src/domain/analysisView/analysisView.test.ts | 20 +- src/domain/analysisView/analysisView.ts | 2 + src/domain/analysisView/nativeFunctionView.ts | 5 +- src/domain/functionComparison.test.ts | 230 +++++++++++ src/domain/functionComparisonDimensions.ts | 42 +- src/domain/functionComparisonNormalization.ts | 79 ++++ src/domain/hopperValues.ts | 76 ++++ src/ghidra/GhidraFunctionValues.test.ts | 98 +++++ src/ghidra/GhidraFunctionValues.ts | 17 + tests/conformance/swift/string-literals.swift | 23 ++ 12 files changed, 1035 insertions(+), 5 deletions(-) create mode 100644 tests/conformance/swift/string-literals.swift diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 02fc0f148..70132eeae 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -13,6 +13,8 @@ import java.net.UnixDomainSocketAddress; import java.nio.channels.Channels; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; @@ -22,7 +24,9 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermissions; import java.security.MessageDigest; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; import java.util.IdentityHashMap; @@ -82,6 +86,7 @@ import ghidra.program.model.address.SegmentedAddress; import ghidra.program.model.reloc.Relocation; import java.io.ByteArrayOutputStream; import ghidra.program.model.symbol.Namespace; +import ghidra.program.model.symbol.FlowType; import ghidra.program.model.symbol.RefType; import ghidra.program.model.symbol.Reference; import ghidra.program.model.symbol.ReferenceIterator; @@ -100,6 +105,9 @@ import ghidra.program.model.pcode.PcodeBlock; import ghidra.program.model.pcode.PcodeBlockBasic; import ghidra.program.model.pcode.Varnode; import ghidra.program.model.pcode.VarnodeAST; +import ghidra.program.util.ContextEvaluatorAdapter; +import ghidra.program.util.SymbolicPropogator; +import ghidra.program.util.VarnodeContext; public final class ReaGhidraBridge extends HeadlessScript { private static final int MAX_HIGH_PCODE_OPS = 3000; @@ -1319,6 +1327,13 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("outgoing_references", outgoing); result.add("referenced_strings", referencedStrings); result.add("referenced_names", referencedNames); + boolean swift = swiftProgram(); + boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:"); + SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null; + if (swiftLiterals != null) { + result.add("swift_string_literals", swiftLiterals.literals()); + result.add("swift_string_literal_omissions", swiftLiterals.omissions()); + } result.add("basic_blocks", blocks); result.add( "native_api", @@ -1338,6 +1353,20 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add( "Synthetic Ghidra entry-point references without actionable memory sources are omitted." ); + if (swift && aarch64) { + limitations.add( + "Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by conditional selects, passed through unresolved jumps, or built at runtime are not reported." + ); + if (swiftLiterals.omissions().size() > 0) { + limitations.add( + "Swift String literal records listed in swift_string_literal_omissions could not be retained or decoded within this dossier's remaining heap budget; later symbolic flows remain unexamined." + ); + } + } else if (swift) { + limitations.add( + "Swift String literal decoding is implemented for AArch64 register pairs only; no Swift literals were decoded for this architecture." + ); + } List boundaryLimitations = noReturnCallLimitations(scan.instructions); for (String limitation : boundaryLimitations) { limitations.add(limitation); @@ -3391,6 +3420,349 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } + private boolean swiftProgram() { + for (MemoryBlock block : currentProgram.getMemory().getBlocks()) { + String name = block.getName(); + if (name.startsWith("__swift5_") || name.startsWith("swift5_")) return true; + } + return false; + } + + /** + * Swift passes a String as two words: small literals carry their bytes in the words and + * immortal large literals carry their count and a biased pointer to the bytes. Ghidra keeps + * neither as a data reference, so decode constant pairs where Swift consumes them: argument + * and return registers at calls, tail calls and returns, and both registers of one STP. + * The canonical empty string is a small literal with a zero count. + */ + private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception { + Register[] registers = new Register[31]; + for (int index = 0; index < registers.length; index++) { + registers[index] = currentProgram.getRegister("x" + index); + } + TreeMap observed = new TreeMap<>(); + TreeMap omitted = new TreeMap<>(); + SwiftLiteralBudget budget = new SwiftLiteralBudget(); + Map selectedValues = new HashMap<>(); + SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false); + propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() { + @Override + public boolean evaluateContextBefore(VarnodeContext context, Instruction instruction) { + // Returning true ends only the current flow, so every queued flow checks the shared stop. + if (budget.stopped()) return true; + for (Register[] pair : swiftStringPairs(function, instruction, registers)) { + Long word = registerConstant(context, pair[0]); + Long object = registerConstant(context, pair[1]); + if (word == null || object == null) continue; + SwiftLiteralLayout layout = swiftLiteralLayout(word, object); + if (layout == null) continue; + // A selected word can combine with a path-dependent ordinary move. Cache the + // provenance question per site/register, charging retained cache state to the + // same dossier budget before allocating it. + boolean selected = false; + boolean provenanceOverCapacity = false; + for (Register register : pair) { + String provenanceKey = canonicalAddress(instruction.getAddress()) + "\u0000" + + register.getBaseRegister().getName(); + Boolean value = selectedValues.get(provenanceKey); + if (value == null) { + if (!budget.takeState()) { + provenanceOverCapacity = true; + break; + } + value = selectedValue(function, instruction, register); + selectedValues.put(provenanceKey, value); + } + if (value) { + selected = true; + break; + } + } + if (selected) continue; + // Forks can reach one site with different constants, so the words are part of the key. + String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + + "\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) + + "\u0000" + Long.toUnsignedString(object, 16); + if (observed.containsKey(key) || omitted.containsKey(key)) continue; + // Every record draws on the budget; the first one that does not fit is kept as an + // omission and decoding stops, so later sites are never examined. + String value = layout.smallValue(); + boolean overCapacity = provenanceOverCapacity || !budget.take(layout.byteLength()); + if (!overCapacity && value == null) { + try { + value = swiftLargeLiteralText(layout); + } catch (OutOfMemoryError exhausted) { + overCapacity = true; + } + } + if (value == null && !overCapacity) continue; + JsonObject item = overCapacity + ? swiftStringOmission(layout, word, object) + : swiftStringItem(value, layout, word, object); + item.addProperty("source_address", canonicalAddress(instruction.getAddress())); + JsonArray pairRegisters = new JsonArray(); + pairRegisters.add(pair[0].getName()); + pairRegisters.add(pair[1].getName()); + item.add("registers", pairRegisters); + (overCapacity ? omitted : observed).put(key, item); + if (overCapacity) { + budget.stop(); + return true; + } + } + return false; + } + }, false, monitor); + JsonArray literals = new JsonArray(); + observed.values().forEach(literals::add); + JsonArray omissions = new JsonArray(); + omitted.values().forEach(omissions::add); + return new SwiftStringLiteralDecode(literals, omissions); + } + + private static List swiftStringPairs( + Function function, + Instruction instruction, + Register[] registers + ) { + List pairs = new ArrayList<>(); + if (instruction.getMnemonicString().equals("stp")) { + Register first = instruction.getRegister(0); + Register second = instruction.getRegister(1); + if (first != null && second != null && first.getBitLength() == 64 && second.getBitLength() == 64) { + pairs.add(new Register[] {first, second}); + } + return pairs; + } + FlowType flow = instruction.getFlowType(); + boolean returns = false; + for (PcodeOp operation : instruction.getPcode()) { + if (operation.getOpcode() == PcodeOp.RETURN) returns = true; + } + // A jump is a proven tail call only when it has resolved targets and all leave the + // body. A targetless computed jump can still dispatch inside the function. + Address[] targets = instruction.getFlows(); + boolean tailCall = flow.isJump() && !flow.isConditional() && targets.length > 0 && + Arrays.stream(targets).allMatch(target -> !function.getBody().contains(target)); + if (!flow.isCall() && !returns && !tailCall) return pairs; + // A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned. + int lastFirstRegister = returns ? 0 : 6; + for (int index = 0; index <= lastFirstRegister; index++) { + pairs.add(new Register[] {registers[index], registers[index + 1]}); + } + return pairs; + } + + private static Long registerConstant(VarnodeContext context, Register register) { + Varnode value = context.getRegisterVarnodeValue(register); + return value != null && context.isConstant(value) ? value.getOffset() : null; + } + + // Search every predecessor path for the reaching definitions, following register copies, + // since a select copied into an argument register or defined before a branch forks just as + // independently as one written there directly. + private boolean selectedValue(Function function, Instruction use, Register register) { + ArrayDeque instructions = new ArrayDeque<>(); + ArrayDeque targets = new ArrayDeque<>(); + Set visited = new HashSet<>(); + for (Instruction predecessor : predecessors(function, use)) { + instructions.push(predecessor); + targets.push(register.getBaseRegister()); + } + while (!instructions.isEmpty()) { + Instruction instruction = instructions.pop(); + Register target = targets.pop(); + if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue; + boolean defines = false; + for (Object result : instruction.getResultObjects()) { + if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true; + } + Register next = target; + if (defines) { + PcodeOp[] operations = instruction.getPcode(); + for (PcodeOp operation : operations) { + if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true; + } + Register source = copiedRegister(operations); + if (source == null) continue; + next = source.getBaseRegister(); + } + for (Instruction predecessor : predecessors(function, instruction)) { + instructions.push(predecessor); + targets.push(next); + } + } + return false; + } + + private List predecessors(Function function, Instruction instruction) { + List result = new ArrayList<>(); + Address from = instruction.getFallFrom(); + if (from != null) { + Instruction previous = currentProgram.getListing().getInstructionAt(from); + if (previous != null) result.add(previous); + } + for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) { + if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue; + if (!function.getBody().contains(reference.getFromAddress())) continue; + Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress()); + if (source != null) result.add(source); + } + return result; + } + + private Register copiedRegister(PcodeOp[] operations) { + if (operations.length != 1) return null; + int opcode = operations[0].getOpcode(); + Varnode input = operations[0].getInput(0); + if ((opcode != PcodeOp.COPY && opcode != PcodeOp.INT_ZEXT) || input == null || !input.isRegister()) return null; + return currentProgram.getRegister(input); + } + + /** Validate the ABI layout of a pair; a small literal is decoded here, a large one only located. */ + private SwiftLiteralLayout swiftLiteralLayout(long word, long object) { + int discriminator = (int) (object >>> 60); + if (discriminator == 0xE || discriminator == 0xA) { + int count = (int) ((object >>> 56) & 0xF); + byte[] bytes = new byte[15]; + for (int index = 0; index < 8; index++) bytes[index] = (byte) (word >>> (8 * index)); + for (int index = 0; index < 7; index++) bytes[8 + index] = (byte) (object >>> (8 * index)); + for (int index = count; index < bytes.length; index++) { + if (bytes[index] != 0) return null; + } + // Count zero is the canonical empty string (object 0xE000000000000000, word 0). + String value = swiftStringText(bytes, count, discriminator == 0xE); + return value == null ? null : new SwiftLiteralLayout("small", null, count, true, value); + } + int flags = (int) (word >>> 60); + long count = word & 0xFFFFFFFFFFFFL; + // Immortal literals: ASCII and NFC or neither, tail allocated, with a pointer biased by 32. + if ((object >>> 56) != 0x80 || (flags != 0xD && flags != 0x1) || ((word >>> 48) & 0xFFF) != 0) { + return null; + } + try { + Address address = currentProgram.getAddressFactory().getDefaultAddressSpace() + .getAddress((object & 0x00FFFFFFFFFFFFFFL) + 0x20); + MemoryBlock block = currentProgram.getMemory().getBlock(address); + // The bytes and their terminator must lie in one initialized, non-executable block, + // and Java arrays bound one read. + if (block == null || !block.isInitialized() || block.isExecute() || + count >= block.getEnd().subtract(address) + 1 || count >= Integer.MAX_VALUE) { + return null; + } + return new SwiftLiteralLayout("large", address, count, flags == 0xD, null); + } catch (Exception unaddressable) { + return null; + } + } + + private String swiftLargeLiteralText(SwiftLiteralLayout layout) { + int count = (int) layout.byteLength(); + byte[] bytes = new byte[count + 1]; + try { + if (currentProgram.getMemory().getBytes(layout.address(), bytes) != bytes.length) return null; + } catch (Exception unreadable) { + return null; + } + return bytes[count] == 0 ? swiftStringText(bytes, count, layout.ascii()) : null; + } + + /** + * Every literal record of one dossier draws on half of the heap that was free when decoding + * began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up + * to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer + * (up to twice its length), again in the returned String, and once more as UTF-8 on the + * socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words + * and their serialized form, so many empty or small literals are bounded too. Retained + * provenance-cache entries draw on that same fixed-state allowance. + */ + private static final class SwiftLiteralBudget { + private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6; + private static final long BYTES_PER_RECORD = 4096; + private long remaining; + private boolean stopped; + + SwiftLiteralBudget() { + Runtime runtime = Runtime.getRuntime(); + remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2); + } + + boolean takeState() { + if (remaining < BYTES_PER_RECORD) return false; + remaining -= BYTES_PER_RECORD; + return true; + } + + boolean take(long bytes) { + if (remaining < BYTES_PER_RECORD || + bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) { + return false; + } + remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE; + return true; + } + + void stop() { + stopped = true; + } + + boolean stopped() { + return stopped; + } + } + + // Swift String values may contain any scalar, including controls; JSON escapes them. + private static String swiftStringText(byte[] bytes, int length, boolean ascii) { + if (length < 0 || length > bytes.length) return null; + for (int index = 0; index < length; index++) { + if (ascii && bytes[index] < 0) return null; + } + try { + return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes, 0, length)).toString(); + } catch (CharacterCodingException invalid) { + return null; + } + } + + private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { + JsonObject item = new JsonObject(); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.addProperty("byte_length", layout.byteLength()); + item.addProperty("reason", "capacity"); + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonObject swiftStringItem( + String value, + SwiftLiteralLayout layout, + long word, + long object + ) { + JsonObject item = new JsonObject(); + item.addProperty("value", value); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonArray swiftStringWords(long word, long object) { + JsonArray words = new JsonArray(); + words.add("0x" + Long.toUnsignedString(word, 16)); + words.add("0x" + Long.toUnsignedString(object, 16)); + return words; + } + private static String exclusiveAddress(Address inclusive) { Address next = inclusive.next(); if (next != null && next.getAddressSpace().equals(inclusive.getAddressSpace())) { @@ -3930,6 +4302,14 @@ public final class ReaGhidraBridge extends HeadlessScript { private record AddressNameRequest(String inputAddress, Address address, String name) {} private record FunctionEntry(Function function, InventoryItem item) {} private record FunctionAnalysisObservation(JsonObject dossier, List limitations) {} + private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {} + private record SwiftLiteralLayout( + String representation, + Address address, + long byteLength, + boolean ascii, + String smallValue + ) {} private record InstructionScan(List instructions, boolean truncated) {} private record SessionDescriptor( String transport, diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 0593408b0..0aa58e2f0 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -45,8 +45,13 @@ const transport = new StdioClientTransport({ const source = fileURLToPath( new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url), ); +const literalSource = fileURLToPath( + new URL("../tests/conformance/swift/string-literals.swift", import.meta.url), +); const binary = join(root, "swift-inventory"); +const literalBinary = join(root, "swift-literals"); const reports = []; +let literalReport; try { await execute("swiftc", ["-O", "-g", source, "-o", binary], { timeout: 60_000, @@ -54,6 +59,19 @@ try { await mkdir(join(root, "copied")); const copied = join(root, "copied", "swift-inventory"); await copyFile(binary, copied); + // Decoding covers AArch64 register pairs, so build that slice on any Mac host. + await execute( + "swiftc", + [ + "-O", + "-target", + "arm64-apple-macosx13.0", + literalSource, + "-o", + literalBinary, + ], + { timeout: 60_000 }, + ); await client.connect(transport); for (const [variant, path] of [ ["debug", binary], @@ -132,6 +150,7 @@ try { await call("close_binary", {}); } } + literalReport = await verifyStringLiterals(); } finally { await client.close(); await transport.close(); @@ -143,6 +162,7 @@ console.log( ok: true, provider: "ghidra", fixtures: reports, + swift_string_literals: literalReport, verifier_run: await completeVerifierRun(run), }, null, @@ -159,3 +179,51 @@ async function call(name, arguments_) { toolContract(name).outputSchema.parse(response.structuredContent); return requireMcpOperationResult(response, name); } + +// Literal text lives only in register constants, so it must come from the decoded pairs. +async function verifyStringLiterals() { + await call("open_binary", { path: literalBinary, provider_id: "ghidra" }); + try { + const procedures = await call("list_procedures", {}); + const expected = { + smallLiteral: [["SleepDisabled", "small"]], + largeLiteral: [["The power setting could not be changed.", "large"]], + unicodeLiterals: [ + ["café", "small"], + ["Résumé of the naïve décor, written twice", "large"], + ], + emptyAndControlLiterals: [ + ["", "small"], + ["tab\there", "small"], + ], + returnedLiteral: [["Holding awake", "small"]], + }; + const decoded = {}; + for (const [name, literals] of Object.entries(expected)) { + const procedure = procedures.find(({ value }) => value.includes(name)); + assert(procedure, `${name} is missing from the procedure inventory`); + const dossier = await call("analyze_function", { + procedure: procedure.address, + }); + const observed = dossier.swift_string_literals; + assert(Array.isArray(observed), `${name} recorded no Swift literals`); + assert.deepEqual(dossier.swift_string_literal_omissions, []); + assert.deepEqual( + observed.map(({ value, representation }) => [value, representation]), + literals, + ); + for (const literal of observed) { + assert.deepEqual(literal.registers, ["x0", "x1"]); + assert.equal( + literal.address === null, + literal.representation === "small", + `${literal.value} storage does not match its representation`, + ); + } + decoded[name] = observed.length; + } + return decoded; + } finally { + await call("close_binary", {}); + } +} diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index f1aa3e829..07e6909c3 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -169,7 +169,8 @@ it("pages every retained dossier collection and the native API record", () => { | "comments" | "unresolved_calls" | "referenced_strings" - | "referenced_names", + | "referenced_names" + | "swift_string_literals", result: unknown = original, ) => { const view = projectAnalysisView(nativeParent(result), { @@ -195,6 +196,23 @@ it("pages every retained dossier collection and the native API record", () => { }), ).toMatchObject({ item: [{ text: "entry" }] }); expect(page("referenced_names")).toMatchObject({ coverage: { total: 0 } }); + const literal = { + source_address: "0x401004", + value: "SleepDisabled", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7369447065656c53", "0xed000064656c6261"], + }; + expect( + page("swift_string_literals", { + ...original, + swift_string_literals: [literal], + }), + ).toMatchObject({ item: [literal], coverage: { total: 1 } }); + expect(page("swift_string_literals")).toMatchObject({ + item: { available: false, reason: "not recorded" }, + }); const { unresolved_calls: _omitted, ...withoutUnresolved } = original; expect( page("unresolved_calls", { diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 7f985f349..b08001e35 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -160,6 +160,8 @@ const nativeViewSchema = z.strictObject({ "unresolved_calls", "referenced_strings", "referenced_names", + "swift_string_literals", + "swift_string_literal_omissions", "native_api", "value_flow_summary", "value_flow_operations", diff --git a/src/domain/analysisView/nativeFunctionView.ts b/src/domain/analysisView/nativeFunctionView.ts index f5a29e0e1..b56de24a1 100644 --- a/src/domain/analysisView/nativeFunctionView.ts +++ b/src/domain/analysisView/nativeFunctionView.ts @@ -79,6 +79,8 @@ const nativeRows = ( unresolved_calls: dossier.unresolved_calls, referenced_strings: dossier.referenced_strings, referenced_names: dossier.referenced_names, + swift_string_literals: dossier.swift_string_literals, + swift_string_literal_omissions: dossier.swift_string_literal_omissions, value_flow_operations: flow?.operations, value_flow_def_use: flow?.def_use, value_flow_effects: flow?.effects, @@ -163,7 +165,8 @@ export const projectNativeFunctionView = ( singleton({ available: false, // Only value-flow facets inherit the value-flow failure reason; an - // older dossier simply has no unresolved_calls record. + // older dossier simply has no unresolved_calls record, and a target + // whose provider does not decode Swift literals has no such record. reason: view.facet.startsWith("value_flow_") && flow?.available === false ? flow.reason diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index e03a27748..677fdc0cc 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -716,6 +716,236 @@ describe("function comparison identity and malformed CFG", () => { }); }); +const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => + functionDossierSchema.parse({ + ...dossier("return 0;", base), + swift_string_literals: values.map((value) => ({ + source_address: base, + value, + representation: "small" as const, + address: null, + registers: ["x0", "x1"] as const, + words: ["0x0", "0xe"] as const, + })), + }); +const stringsDimension = ( + leftDigit: string, + left: ReturnType | ReturnType, + rightDigit: string, + right: ReturnType | ReturnType, +) => + compareFunctions( + observe(leftDigit, left), + observe(rightDigit, right), + ).dimensions.find(({ dimension }) => dimension === "strings_names"); + +describe("function comparison Swift string literals", () => { + it("treats differing decoded literals as a strings and names change", () => { + expect( + stringsDimension( + "1", + withLiterals("0x1000", ["SleepDisabled"]), + "2", + withLiterals("0x2000", ["Holding awake"]), + ), + ).toMatchObject({ status: "changed" }); + }); + + it("matches the same decoded literal after the function is relocated", () => { + expect( + stringsDimension( + "3", + withLiterals("0x1000", ["SleepDisabled"]), + "4", + withLiterals("0x2000", ["SleepDisabled"]), + ), + ).toMatchObject({ status: "unchanged" }); + }); + + it("keeps strings and names unknown when only one dossier recorded Swift literals", () => { + expect( + stringsDimension( + "5", + withLiterals("0x1000", []), + "6", + dossier("return 0;", "0x2000"), + ), + ).toMatchObject({ + status: "unknown", + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); + }); + + it("reports a referenced name change even when Swift literal coverage differs", () => { + const named = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + referenced_names: [ + { address: "0x3000", value: "helper", source_address: "0x2000" }, + ], + }); + expect( + stringsDimension("7", withLiterals("0x1000", []), "8", named), + ).toMatchObject({ status: "changed" }); + }); +}); + +describe("function comparison partial Swift literal coverage", () => { + it("does not equate a partial literal record with a complete one", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2000", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "9", + withLiterals("0x1000", ["SleepDisabled"]), + "a", + partial, + ), + ).toMatchObject({ status: "unknown" }); + }); + + it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("d", withLiterals("0x1000", []), "e", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("does not interpret an unexamined path as a missing literal at a retained site", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["A"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 0, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + ], + }); + const complete = withLiterals("0x1000", ["A", "B"]); + expect(stringsDimension("1", complete, "2", partial)).toMatchObject({ + status: "unknown", + }); + expect(stringsDimension("3", partial, "4", complete)).toMatchObject({ + status: "unknown", + }); + expect( + stringsDimension("5", withLiterals("0x1000", []), "6", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("treats an omission record without decoded literals as partial coverage", () => { + const omissionOnly = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly), + ).toMatchObject({ status: "unknown" }); + }); + + it("keeps different retained representations unknown when both dossiers are partial", () => { + const omission = { + source_address: "0x1004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + } as const; + const small = functionDossierSchema.parse({ + ...withLiterals("0x1000", ["SleepDisabled"]), + swift_string_literal_omissions: [omission], + }); + const large = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literals: [ + { + source_address: "0x2000", + value: "SleepDisabled", + representation: "large", + address: "0x3000", + registers: ["x0", "x1"], + words: ["0xd00000000000000d", "0x8000000000002fe0"], + }, + ], + swift_string_literal_omissions: [ + { ...omission, source_address: "0x2004" }, + ], + }); + expect(stringsDimension("1", small, "2", large)).toMatchObject({ + status: "unknown", + }); + }); + + it("keeps different retained literals unknown when both dossiers are partial", () => { + const partial = (base: "0x1000" | "0x2000", value: string) => + functionDossierSchema.parse({ + ...withLiterals(base, [value]), + swift_string_literal_omissions: [ + { + source_address: base === "0x1000" ? "0x1004" : "0x2004", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "b", + partial("0x1000", "SleepDisabled"), + "c", + partial("0x2000", "Holding awake"), + ), + ).toMatchObject({ status: "unknown" }); + }); +}); + const expectInlineComparisonEvidence = ( result: ReturnType, ): void => { diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 27bb7b176..c5a0f373f 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -11,6 +11,7 @@ import { referenceProjection, sorted, stringAndNameProjection, + swiftLiteralSitesConflict, } from "./functionComparisonNormalization.js"; import type { DimensionName, @@ -139,8 +140,12 @@ const compareStringsAndNames = ( right: FunctionSnapshot, context: ComparisonContext, ): FunctionDimension => { - const leftValues = stringAndNameProjection(left); - const rightValues = stringAndNameProjection(right); + const complete = + swiftLiteralCoverage(left) === swiftLiteralCoverage(right) && + swiftLiteralCoverage(left) !== "partial"; + // Facts both dossiers record decide a change before Swift literal coverage can. + const leftValues = stringAndNameProjection(left, complete); + const rightValues = stringAndNameProjection(right, complete); if (leftValues === null || rightValues === null) return unresolvedDimension({ dimension: "strings_names", @@ -151,9 +156,40 @@ const compareStringsAndNames = ( "Reference source locations could not be normalized to function offsets.", ], }); - return compareValues("strings_names", leftValues, rightValues, context); + const compared = compareValues( + "strings_names", + leftValues, + rightValues, + context, + ); + if (complete || compared.status === "changed") return compared; + // A retained literal differs only when the opposite dossier establishes its absence. + if (swiftLiteralSitesConflict(left, right)) { + const leftAll = stringAndNameProjection(left); + const rightAll = stringAndNameProjection(right); + if (leftAll !== null && rightAll !== null) + return compareValues("strings_names", leftAll, rightAll, context); + } + return unresolvedDimension({ + dimension: "strings_names", + links: context.links, + leftCount: null, + rightCount: null, + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); }; +const swiftLiteralCoverage = ( + snapshot: FunctionSnapshot, +): "absent" | "complete" | "partial" => + (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 + ? "partial" + : snapshot.dossier.swift_string_literals === undefined + ? "absent" + : "complete"; + const compareReferences = ( left: FunctionSnapshot, right: FunctionSnapshot, diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 5282272e1..f32220b0b 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -142,6 +142,7 @@ export const commentProjection = ( export const stringAndNameProjection = ( snapshot: FunctionSnapshot, + includeSwiftLiterals = true, ): readonly unknown[] | null => { const values = [ ...snapshot.dossier.referenced_strings.map( @@ -164,12 +165,90 @@ export const stringAndNameProjection = ( value, }), ), + ...(includeSwiftLiterals + ? (snapshot.dossier.swift_string_literals ?? []) + : [] + ).map( + ({ + source_address: sourceAddress, + value, + representation, + registers, + }) => ({ + kind: "swift_string", + source_offset: relativeAddress( + sourceAddress, + snapshot.dossier.procedure.address, + ), + value, + representation, + registers, + }), + ), ]; return values.some(({ source_offset: offset }) => offset === null) ? null : sorted(values); }; +/** + * A retained literal proves presence, not that every path through its site was examined. + * A differing retained fact proves a change only when the opposite dossier is complete + * and therefore establishes its absence. + */ +export const swiftLiteralSitesConflict = ( + left: FunctionSnapshot, + right: FunctionSnapshot, +): boolean => { + const leftSites = swiftLiteralSites(left); + const rightSites = swiftLiteralSites(right); + for (const site of new Set([ + ...leftSites.values.keys(), + ...rightSites.values.keys(), + ])) { + const leftValues = + leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined); + const rightValues = + rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined); + if (leftValues === undefined || rightValues === undefined) continue; + const leftFacts = new Set(leftValues); + const rightFacts = new Set(rightValues); + if ( + (rightSites.complete && + leftValues.some((value) => !rightFacts.has(value))) || + (leftSites.complete && rightValues.some((value) => !leftFacts.has(value))) + ) + return true; + } + return false; +}; + +const swiftLiteralSites = (snapshot: FunctionSnapshot) => { + const base = snapshot.dossier.procedure.address; + const omissions = snapshot.dossier.swift_string_literal_omissions ?? []; + const site = (sourceAddress: string, registers: readonly string[]) => { + const offset = relativeAddress(sourceAddress, base); + return offset === null ? null : canonicalJson([offset, registers]); + }; + // Keep the same literal facts the full strings_names projection compares. + const values = new Map(); + for (const literal of snapshot.dossier.swift_string_literals ?? []) { + const key = site(literal.source_address, literal.registers); + if (key === null) continue; + values.set( + key, + [ + ...(values.get(key) ?? []), + canonicalJson([literal.value, literal.representation]), + ].sort(compareUnicodeCodePoints), + ); + } + const complete = + snapshot.dossier.swift_string_literals !== undefined && + omissions.length === 0; + return { values, complete }; +}; + export const sorted = (values: readonly unknown[]): readonly unknown[] => values .map((value) => ({ diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 769d0f040..d6344ff3e 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -290,6 +290,70 @@ export const unresolvedCallSchema = z.strictObject({ reason: z.string(), }); +const swiftStringLiteralFields = { + source_address: z + .string() + .describe( + "Instruction that consumes both words: a call, tail call, return, or paired store.", + ), + value: z + .string() + .describe("Decoded UTF-8 text, including the canonical empty string."), + registers: z.tuple([z.string(), z.string()]), + words: z + .tuple([z.string(), z.string()]) + .describe("Observed constant register values, in register order."), +}; + +/** A Swift String literal decoded from the two words of its ABI value, not a data reference. */ +export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("small") + .describe("The UTF-8 bytes are inside the two words."), + address: z.null().describe("A small literal has no storage of its own."), + }), + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("large") + .describe( + "An immortal literal whose words hold its count and a pointer to its bytes.", + ), + address: z.string().describe("Address of a large literal's bytes."), + }), +]); + +const swiftStringLiteralOmissionFields = { + source_address: swiftStringLiteralFields.source_address, + byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + reason: z + .literal("capacity") + .describe( + "The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.", + ), + registers: swiftStringLiteralFields.registers, + words: swiftStringLiteralFields.words, +}; + +/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */ +export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( + "representation", + [ + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("small"), + address: z.null(), + }), + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), + }), + ], +); + /** Observed endpoints with complete or partial reference classification evidence. */ export const referenceEdgeSchema = z .object({ @@ -341,6 +405,18 @@ export const functionDossierSchema = z }) .strict(), ), + swift_string_literals: z + .array(swiftStringLiteralSchema) + .exactOptional() + .describe( + "Swift String literals decoded from constant register values. Absent when the provider does not decode them for this target.", + ), + swift_string_literal_omissions: z + .array(swiftStringLiteralOmissionSchema) + .exactOptional() + .describe( + "The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.", + ), basic_blocks: z.array( z .object({ diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 75d59c97f..333d2c913 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -195,6 +195,51 @@ describe("Ghidra function-analysis result values", () => { }); }); +describe("Ghidra Swift String literals", () => { + it("keeps a small Swift literal without storage apart from a large literal's bytes", () => { + const dossier = ghidraFunctionDossier(); + if ( + typeof dossier !== "object" || + dossier === null || + Array.isArray(dossier) + ) + throw new TypeError("Ghidra dossier fixture is invalid"); + const literals = [ + { + source_address: "0x401004", + value: "The user name ", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7265737520656854", "0xee0020656d616e20"], + }, + { + source_address: "0x401006", + value: "", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + { + source_address: "0x401008", + value: " cannot be written into a sudoers rule safely.", + representation: "large", + address: "0x402270", + registers: ["x0", "x1"], + words: ["0xd00000000000002e", "0x8000000000402250"], + }, + ]; + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + swift_string_literals: literals, + swift_string_literal_omissions: [], + }), + ).toMatchObject({ ok: true, value: { swift_string_literals: literals } }); + }); +}); + describe("Ghidra compact reference provenance", () => { it("distinguishes incoming references, examined emptiness and unavailable coverage at data addresses", () => { const instruction = { @@ -423,6 +468,34 @@ describe("Ghidra function-analysis malformed results", () => { ); }); +const malformedSwiftStringLiterals = ( + dossier: Record, +): Array<[string, GhidraFunctionOperation, JsonValue]> => + ( + [ + ["non-canonical Swift literal address", "large", "402240"], + ["small Swift literal with storage", "small", "0x402270"], + ["large Swift literal without storage", "large", null], + ] as const + ).map(([name, representation, address]) => [ + name, + "analyze_function", + { + ...dossier, + swift_string_literal_omissions: [], + swift_string_literals: [ + { + source_address: "0x401004", + value: "The power setting could not be changed.", + representation, + address, + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ]); + const malformedOutputs = (): Array< [string, GhidraFunctionOperation, JsonValue] > => { @@ -437,6 +510,11 @@ const malformedOutputs = (): Array< throw new TypeError("Ghidra native API fixture is invalid"); return [ ["non-canonical address", "procedure_callers", ["401000"]], + [ + "Swift literals without their omission record", + "analyze_function", + { ...dossier, swift_string_literals: [] }, + ], [ "missing classification", "procedure_info", @@ -467,6 +545,26 @@ const malformedOutputs = (): Array< Object.entries(dossier).filter(([key]) => key !== "native_api"), ), ], + ...malformedSwiftStringLiterals(dossier), + [ + "non-canonical Swift literal omission address", + "analyze_function", + { + ...dossier, + swift_string_literals: [], + swift_string_literal_omissions: [ + { + source_address: "0x401004", + representation: "large", + address: "402240", + byte_length: 39, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ], [ "non-canonical native API target", "analyze_function", diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index d45e41ba9..4e4c5054a 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -293,6 +293,15 @@ const ghidraFunctionDossier = functionDossierSchema }) .strict() .superRefine((value, context) => { + if ( + (value.swift_string_literals === undefined) !== + (value.swift_string_literal_omissions === undefined) + ) + context.addIssue({ + code: "custom", + message: + "Ghidra dossier must record Swift literals and their omissions together", + }); const addresses = [ value.procedure.address, ...value.comments.map(({ address }) => address), @@ -308,6 +317,14 @@ const ghidraFunctionDossier = functionDossierSchema address, source_address, ]), + ...(value.swift_string_literals ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), + ...(value.swift_string_literal_omissions ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), ...value.basic_blocks.flatMap(({ start, end, successors }) => [ start, end, diff --git a/tests/conformance/swift/string-literals.swift b/tests/conformance/swift/string-literals.swift new file mode 100644 index 000000000..345c4cf74 --- /dev/null +++ b/tests/conformance/swift/string-literals.swift @@ -0,0 +1,23 @@ +@inline(never) +func consume(_ value: String) { print(value) } +@inline(never) +func smallLiteral() { consume("SleepDisabled") } +@inline(never) +func largeLiteral() { consume("The power setting could not be changed.") } +@inline(never) +func unicodeLiterals() { + consume("café") + consume("Résumé of the naïve décor, written twice") +} +@inline(never) +func emptyAndControlLiterals() { + consume("") + consume("tab\there") +} +@inline(never) +func returnedLiteral() -> String { "Holding awake" } +smallLiteral() +largeLiteral() +unicodeLiterals() +emptyAndControlLiterals() +print(returnedLiteral(), CommandLine.arguments.count) From ec4f4ddc588a564b75d69249b4ebbe4d4051c792 Mon Sep 17 00:00:00 2001 From: yasshmittal Date: Tue, 13 Oct 2026 18:11:28 +0530 Subject: [PATCH 13/28] fix(ghidra): read UTF-8 signed language definitions (#2772) Buffer#toString kept a leading UTF-8 byte order mark as U+FEFF, so the strict XML parser rejected a valid .ldefs file as content outside the root element and aborted the whole language catalog read. XML 1.0 section 4.3.3 defines the mark as an encoding signature; decode with TextDecoder, which consumes it. Malformed XML is still rejected. Fixes #2533 Co-authored-by: Claude Opus 5.5 --- src/ghidra/GhidraLanguageCatalog.ts | 4 +++- src/ghidra/GhidraLanguageOverride.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/ghidra/GhidraLanguageCatalog.ts b/src/ghidra/GhidraLanguageCatalog.ts index 5da5943dc..6373f187f 100644 --- a/src/ghidra/GhidraLanguageCatalog.ts +++ b/src/ghidra/GhidraLanguageCatalog.ts @@ -75,8 +75,10 @@ export const readGhidraLanguageCatalog = async ( if (!entry.endsWith(".ldefs")) continue; const bytes = await readFile(join(directory, entry)); if (bytes.length > MAX_LDEFS_BYTES) continue; + // XML 1.0 section 4.3.3: a leading UTF-8 BOM is an encoding signature, + // not document content; TextDecoder consumes it, Buffer#toString does not. definitions.push( - ...parseGhidraLanguageDefinitions(bytes.toString("utf8")), + ...parseGhidraLanguageDefinitions(new TextDecoder().decode(bytes)), ); } } diff --git a/src/ghidra/GhidraLanguageOverride.test.ts b/src/ghidra/GhidraLanguageOverride.test.ts index 5f90a1454..02c5bf245 100644 --- a/src/ghidra/GhidraLanguageOverride.test.ts +++ b/src/ghidra/GhidraLanguageOverride.test.ts @@ -2,6 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { parseConfig } from "../config/parseConfig.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -190,6 +191,28 @@ describe("Ghidra language catalog", () => { ); }); + it("reads a UTF-8 signed definition beside an unsigned sibling", async () => { + const root = await createTestTempDirectory("rea-ghidra-bom-"); + const module = (name: string) => + join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(module("probe"), { recursive: true }); + await mkdir(module("x86"), { recursive: true }); + await writeFile( + join(module("probe"), "probe.ldefs"), + Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + Buffer.from( + '\n', + ), + ]), + ); + await writeFile(join(module("x86"), "x86.ldefs"), X86_LDEFS); + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "probe:LE:32:default", compilerSpecIds: [] }, + ...parseGhidraLanguageDefinitions(X86_LDEFS), + ]); + }); + it("admits only languages and compiler specs the installation defines", () => { const catalog = parseGhidraLanguageDefinitions(X86_LDEFS); expect( From 84cf7791a881e5b16a0288600fbc8d48d494ef01 Mon Sep 17 00:00:00 2001 From: Saim Shafique Date: Tue, 13 Oct 2026 17:42:40 +0500 Subject: [PATCH 14/28] fix(call-path): keep caller-only unknown coverage from blocking not_found (#2371) * fix(call-path): keep caller-only unknown coverage from blocking not_found build_call_path expands only callee edges, so relationship coverage that is unknown solely because callers are unobservable (the modern headless IDA profile) no longer marks the searched callee closure incomplete. The caller caveat stays visible in limitations, and callee-side unknowns still yield unknown. Fixes #2322. * test(call-path): cover IDA-capped callees beside unknown callers Greptile review noted the poll-timing coverage only exercised unknown callees, leaving the partial callees IDA returns at its 500-result cap untested. Add the combined unknown-callers/partial-callees case: isCallerOnlyUnknownCoverage must stay false and buildCallPath must keep search_scope.exhaustive false, so a future change cannot silently report not_found when IDA has omitted callees. --- src/domain/callPath.test.ts | 83 +++++++++++++++++++++++++ src/domain/callPath.ts | 7 ++- src/domain/functionCallCoverage.test.ts | 64 +++++++++++++++++++ src/domain/functionCallCoverage.ts | 22 ++++++- 4 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 src/domain/functionCallCoverage.test.ts diff --git a/src/domain/callPath.test.ts b/src/domain/callPath.test.ts index e0f343fca..e78a5f936 100644 --- a/src/domain/callPath.test.ts +++ b/src/domain/callPath.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildCallPath } from "./callPath.js"; +import { combineFunctionRelationshipCoverage } from "./functionCallCoverage.js"; import { callPathResultSchema, type CallPathInput } from "./callPathSchemas.js"; import { createEvidence, type Evidence } from "./evidence.js"; import type { JsonValue } from "./jsonValue.js"; @@ -396,6 +397,60 @@ describe("explicit call relationship coverage", () => { expect(result.limitations).toEqual([reason]); }); + it("establishes not_found when only caller coverage is unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "not_found", + search_scope: { exhaustive: true }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when callee coverage is unknown", () => { + const reason = + "Callees: Upstream callees capped results at 500; additional callees remain unknown."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when both caller and callee coverage are unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships. Callees: Upstream callees are not observed."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + it("ignores incomplete relationship coverage outside the reachable search closure", () => { const reason = "An unrelated function's callees were not observed."; const result = run([ @@ -415,6 +470,34 @@ describe("explicit call relationship coverage", () => { }); }); +describe("IDA-capped callee coverage beside unknown callers", () => { + it("keeps the search non-exhaustive for the combined coverage value", () => { + const combined = combineFunctionRelationshipCoverage( + { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", + }, + { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], + }, + ); + expect(combined.status).toBe("unknown"); + const result = run([ + observe("0x1000", [], "rea-workflow", { relationshipCoverage: combined }), + observe("0x4000", []), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + shortest_hops: null, + paths: [], + }); + }); +}); + describe("call path evidence admission", () => { it("normalizes hex addresses and rejects duplicates and mixed providers", () => { expect( diff --git a/src/domain/callPath.ts b/src/domain/callPath.ts index 2186c8010..0bd5e88e1 100644 --- a/src/domain/callPath.ts +++ b/src/domain/callPath.ts @@ -10,6 +10,7 @@ import { parseFunctionEvidence, type FunctionSnapshot, } from "./functionDossierEvidence.js"; +import { isCallerOnlyUnknownCoverage } from "./functionCallCoverage.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; /** Minimal directed caller-to-callee adjacency for call-path search. */ @@ -255,10 +256,14 @@ const inspectSearch = ({ // Explicit, Evidence-bound relationship coverage owns completeness; // generic provider/dossier limitations remain contextual. Legacy Evidence // without that metadata retains its conservative limitation-based rule. + // The search only expands callee edges, so coverage that is unknown + // solely because callers are unobservable does not make the searched + // callee closure incomplete; the caller caveat stays in limitations. if ( coverage === null ? snapshot.limitations.length > 0 - : coverage.status !== "complete" + : coverage.status !== "complete" && + !isCallerOnlyUnknownCoverage(coverage) ) incompleteExpandedCoverage = true; } else if (node === goal) { diff --git a/src/domain/functionCallCoverage.test.ts b/src/domain/functionCallCoverage.test.ts new file mode 100644 index 000000000..6fd3fb045 --- /dev/null +++ b/src/domain/functionCallCoverage.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { + combineFunctionRelationshipCoverage, + isCallerOnlyUnknownCoverage, +} from "./functionCallCoverage.js"; +import type { RelationshipCoverage } from "./relationshipCoverage.js"; + +const unknownCallers: RelationshipCoverage = { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", +}; +const unknownCallees: RelationshipCoverage = { + status: "unknown", + reason: "Upstream callees are not observed in this profile.", +}; +const cappedCallees: RelationshipCoverage = { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], +}; +const complete: RelationshipCoverage = { status: "complete" }; + +describe("isCallerOnlyUnknownCoverage", () => { + it("detects unknown callers beside complete callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + complete, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(true); + }); + + it("rejects unknown callees beside complete callers", () => { + const combined = combineFunctionRelationshipCoverage( + complete, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown coverage on both sides", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown callers beside IDA-capped callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + cappedCallees, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects complete coverage", () => { + const combined = combineFunctionRelationshipCoverage(complete, complete); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); +}); diff --git a/src/domain/functionCallCoverage.ts b/src/domain/functionCallCoverage.ts index eb2c05a13..b64e78cd6 100644 --- a/src/domain/functionCallCoverage.ts +++ b/src/domain/functionCallCoverage.ts @@ -2,6 +2,9 @@ import type { RelationshipCoverage } from "./relationshipCoverage.js"; import type { FunctionDossier } from "./hopperValues.js"; import { uniqueSorted } from "./canonicalOrdering.js"; +const callerCoveragePrefix = "Callers: "; +const calleeCoveragePrefix = "Callees: "; + /** Combine caller and callee read outcomes without losing either explanation. */ export const combineFunctionRelationshipCoverage = ( callers: RelationshipCoverage | undefined, @@ -20,7 +23,8 @@ export const combineFunctionRelationshipCoverage = ( ); const details = diagnostics.length > 0 ? { diagnostics } : {}; const reasons = reads.flatMap(({ direction, coverage }) => { - const label = direction === "callers" ? "Callers" : "Callees"; + const label = + direction === "callers" ? callerCoveragePrefix : calleeCoveragePrefix; const limitations = coverage === undefined ? ["Provider did not report relationship coverage."] @@ -29,7 +33,7 @@ export const combineFunctionRelationshipCoverage = ( : coverage.status === "partial" ? coverage.limitations : []; - return limitations.map((reason) => `${label}: ${reason}`); + return limitations.map((reason) => `${label}${reason}`); }); if ( reads.some( @@ -50,6 +54,20 @@ export const combineFunctionRelationshipCoverage = ( return { status: "complete", ...details }; }; +/** + * Whether combined relationship coverage is unknown only on the caller side. + * The combine helper above prefixes each direction's reasons, so a reason + * that carries the caller prefix and no callee prefix records observed + * callees with unobservable callers. Forward (callee-edge) searches stay + * exhaustive under that caveat; the caller reason remains a limitation. + */ +export const isCallerOnlyUnknownCoverage = ( + coverage: RelationshipCoverage, +): boolean => + coverage.status === "unknown" && + coverage.reason.startsWith(callerCoveragePrefix) && + !coverage.reason.includes(calleeCoveragePrefix); + /** Explain why two resolved call sets cannot establish call-set equality. */ export const functionCallCoverageLimitations = ( coverage: RelationshipCoverage | null, From 1ca5fd38b4833ac3fe6213a55e5abfd426228266 Mon Sep 17 00:00:00 2001 From: Jack Liang <130693642+Jackmeson1@users.noreply.github.com> Date: Tue, 13 Oct 2026 05:43:20 -0700 Subject: [PATCH 15/28] fix(cli)!: keep a selected JSON null in --filter-output output (#2333) createCliResultOutput rendered an explicit null selected by --filter-output as the empty document {}, the same output it uses when the selection has no value at all. On the streamed JSON/JSONL path, a consumer could not distinguish a selected null from a missing field. Write a selected null as null in streamed JSON and JSONL. This also preserves an unfiltered top-level null. The {} fallback stays for undefined, functions and symbols. The --full-output envelope already carried data: null for a selected null. The --token-count path still hands results that fit the formatted-string limit back to Incur and prints 0 for a selected null. Other formats and Incur-owned result paths are unchanged. Consumers must accept JSON null for a present null and distinguish it from a missing selection. Use --full-output if an object envelope is required. Keeping the old shared {} representation would continue hiding this distinction. Claude-Session: https://claude.ai/code/session_019TTGAv6fe859ebbCLLfLHP Co-authored-by: Claude Opus 5.5 --- src/cli/cliResultOutput.ts | 5 +++-- tests/boundary/cli/jsonOutput.test.ts | 32 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 67e745b0b..5efde8b33 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -216,8 +216,9 @@ export const createCliResultOutput = ( : filtered === undefined ? { ok: true, meta } : { ok: true, data: filtered, meta } - : filtered === undefined || - filtered === null || + : // A selected null is a value; only a selection with no JSON value + // falls back to the empty document. + filtered === undefined || typeof filtered === "function" || typeof filtered === "symbol" ? {} diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index 5e8e86e7f..0d1ee5a59 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -561,3 +561,35 @@ describe("CLI result output surface", () => { }); }, 60_000); }); + +describe("streamed CLI filtered selections", () => { + it.each([ + ["raw_result", "json", "null\n"], + ["raw_result", "jsonl", "null\n"], + ["nested.value", "json", '{"nested":{"value":null}}\n'], + ["missing", "json", "{}\n"], + ["count", "json", "0\n"], + ["enabled", "json", "false\n"], + ] as const)( + "keeps the selected value of --filter-output %s in %s", + async (filter, format, expected) => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + ["--format", format, "--filter-output", filter], + destination, + ); + if (output === undefined) throw new Error("Missing JSON output surface"); + await output.write( + { raw_result: null, nested: { value: null }, count: 0, enabled: false }, + { command: "fixture", duration: "1ms", format }, + ); + expect(Buffer.concat(chunks).toString("utf8")).toBe(expected); + }, + ); +}); From 36132d3e1770c8b69f86c86821d2c872234e370a Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 13 Oct 2026 08:43:39 -0400 Subject: [PATCH 16/28] fix(managed): classify 32-bit-preferred assemblies as platform neutral (#2342) The CLI header's 32BITREQUIRED and 32BITPREFERRED bits are one two-bit field. Reading 32BITREQUIRED on its own reported Prefer32Bit assemblies (flags 0x00020003) as x86-only with requires_32bit true, and left anycpu-prefer-32 reachable only through the reserved combination. Co-authored-by: Claude Opus 5.5 --- src/dotnet/ManagedArtifactInspector.test.ts | 24 +++++++++++++++++++++ src/dotnet/ManagedArtifactInspector.ts | 7 +++--- src/dotnet/ManagedNativeBoundaryHelpers.ts | 7 ++++-- src/dotnet/ManagedPeReader.ts | 14 ++++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/dotnet/ManagedArtifactInspector.test.ts b/src/dotnet/ManagedArtifactInspector.test.ts index 2d30bf580..3b734c176 100644 --- a/src/dotnet/ManagedArtifactInspector.test.ts +++ b/src/dotnet/ManagedArtifactInspector.test.ts @@ -292,6 +292,30 @@ describe("managed artifact inventory", () => { ]); expect(result.coverage).toMatchObject({ state: "complete", issues: [] }); }); + + // dotnet/runtime corhdr.h: 32BITREQUIRED and 32BITPREFERRED form one + // two-bit field, so neither bit can be read on its own. + it.each([ + ["neither bit", 0x0000_0001, "anycpu", false], + ["32BITREQUIRED alone", 0x0000_0003, "x86", true], + ["both bits", 0x0002_0003, "anycpu-prefer-32", false], + ["the reserved 32BITPREFERRED alone", 0x0002_0001, "anycpu", false], + ] as const)( + "reads %s as one 32-bit requirement", + (_label, cliFlags, architecture, requires32Bit) => { + const bytes = buildManagedPeFixture({ cliFlags }); + const target = managedPeFixtureTarget(bytes); + + expect( + inspectManagedArtifactBytes(bytes, target).classification + .managed_architecture, + ).toBe(architecture); + expect( + inspectManagedNativeBoundariesBytes(bytes, target).cli_native + ?.requires_32bit, + ).toBe(requires32Bit); + }, + ); }); describe("managed artifact coded indexes", () => { diff --git a/src/dotnet/ManagedArtifactInspector.ts b/src/dotnet/ManagedArtifactInspector.ts index 7df8aae83..7e7acb087 100644 --- a/src/dotnet/ManagedArtifactInspector.ts +++ b/src/dotnet/ManagedArtifactInspector.ts @@ -9,6 +9,7 @@ import { type ManagedParseIssue, } from "../domain/managed/managedArtifact.js"; import { + managedCli32BitRequirement, readManagedPeLayout, type ManagedCliHeader, type ManagedPeLayout, @@ -249,9 +250,9 @@ const managedArchitectureFor = ( ): ManagedArtifactInspection["classification"]["managed_architecture"] => { if (layout.architecture !== "x86" || (cli.flags & 1) === 0) return layout.architecture; - if ((cli.flags & 0x0000_0002) !== 0) return "x86"; - if ((cli.flags & 0x0002_0000) !== 0) return "anycpu-prefer-32"; - return "anycpu"; + const requirement = managedCli32BitRequirement(cli.flags); + if (requirement === "required") return "x86"; + return requirement === "preferred" ? "anycpu-prefer-32" : "anycpu"; }; const runtimeClassification = ({ diff --git a/src/dotnet/ManagedNativeBoundaryHelpers.ts b/src/dotnet/ManagedNativeBoundaryHelpers.ts index fb81e1d28..e4b1a5960 100644 --- a/src/dotnet/ManagedNativeBoundaryHelpers.ts +++ b/src/dotnet/ManagedNativeBoundaryHelpers.ts @@ -18,7 +18,10 @@ import { } from "./ManagedMetadataHeaps.js"; import { managedTableRowCounts } from "./ManagedMetadataInventory.js"; import type { ManagedMetadataInventory } from "./ManagedMetadataInventory.js"; -import type { ManagedPeLayout } from "./ManagedPeReader.js"; +import { + managedCli32BitRequirement, + type ManagedPeLayout, +} from "./ManagedPeReader.js"; import { readManagedValue } from "./ManagedReaderFailure.js"; type ModuleRef = ManagedNativeBoundaryInspection["module_refs"][number]; @@ -380,7 +383,7 @@ export const cliNative = ( if (pe.cli === null) return null; return { il_only: (pe.cli.flags & 0x0000_0001) !== 0, - requires_32bit: (pe.cli.flags & 0x0000_0002) !== 0, + requires_32bit: managedCli32BitRequirement(pe.cli.flags) === "required", strong_name_signed: (pe.cli.flags & 0x0000_0008) !== 0, native_entry_point: (pe.cli.flags & 0x0000_0010) !== 0, ready_to_run_signature: pe.cli.readyToRunSignature, diff --git a/src/dotnet/ManagedPeReader.ts b/src/dotnet/ManagedPeReader.ts index 2f9bc3151..9a30c007b 100644 --- a/src/dotnet/ManagedPeReader.ts +++ b/src/dotnet/ManagedPeReader.ts @@ -31,6 +31,20 @@ export interface ManagedCliHeader { readonly readyToRunSignature: boolean; } +/** + * How the CLI header constrains process bitness. 32BITREQUIRED and + * 32BITPREFERRED are one two-bit field: required alone is x86-only, both set + * is platform-neutral code that prefers a 32-bit process, and preferred alone + * is reserved and constrains nothing. + */ +export const managedCli32BitRequirement = ( + flags: number, +): "required" | "preferred" | "none" => { + const pair = flags & 0x0002_0002; + if (pair === 0x0000_0002) return "required"; + return pair === 0x0002_0002 ? "preferred" : "none"; +}; + export interface ManagedPeLayout { readonly machine: number; readonly architecture: "x86" | "x86_64" | "arm" | "arm64"; From d3a55720fc4aac8f9d2c5f5a4c9f20cdf5594d8e Mon Sep 17 00:00:00 2001 From: Chaoqi <136579807+Chaoqi31@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:44:17 -0400 Subject: [PATCH 17/28] fix(setup): keep Hermes config numbers as written (#2368) Setup and uninstall re-printed every number in an existing Hermes config.yaml from a JavaScript number. Hermes reads the file as YAML 1.1 with exact integers, so a Discord channel ID lost its low digits, an octal mode such as 0600 became 600, and 0.0000001 became 1e-7, which YAML 1.1 does not define as a float. Two IDs that round to the same double were also rejected as duplicate keys, so setup could not register at all. Hermes YAML parsing and editing move to HermesConfigurationYaml. A parsed number is written with its source spelling while that spelling still reads as the node's value, the rule yaml's own boolean tag follows, and two keys are duplicates only when both their value and their spelling match. Parsed values stay JavaScript numbers. --- .../ClientConfigurationDocument.ts | 26 ++++--- .../HermesConfigurationYaml.test.ts | 21 ++++++ src/application/HermesConfigurationYaml.ts | 74 +++++++++++++++++++ .../hermesClientConfiguration.test.ts | 32 ++++++++ 4 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 src/application/HermesConfigurationYaml.test.ts create mode 100644 src/application/HermesConfigurationYaml.ts diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index b32df3fd0..1b241a98b 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -1,9 +1,5 @@ import { parse as parseToml, stringify as stringifyToml } from "smol-toml"; -import { - parse as parseYaml, - parseDocument as parseYamlDocument, - stringify as stringifyYaml, -} from "yaml"; +import { stringify as stringifyYaml } from "yaml"; import { isDeepStrictEqual } from "node:util"; import { @@ -12,6 +8,10 @@ import { editDshDocument, dshClientRegistrationEntry, } from "./DshCordisPatch.js"; +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; import { applyEdits, findNodeAtLocation, @@ -118,7 +118,10 @@ const parseDocument = ( if (isEmptyClientConfigurationText(text)) return {}; // Hermes stores MCP servers in `mcp_servers` inside a YAML document. if (format === "hermes") - return requireObject(parseYaml(text), "The configuration root"); + return requireObject( + parseHermesConfigurationYaml(text), + "The configuration root", + ); // ZCode loads nonempty native files with strict JSON.parse. if (format === "zcode") return requireObject(JSON.parse(text), "The configuration root"); @@ -1419,12 +1422,11 @@ const serializeHermesConfiguration = ( typeof servers === "object" && servers !== null ? (servers as Record)[PRODUCT_IDENTITY.mcpServerKey] : undefined; - const yamlDocument = parseYamlDocument(originalText); - if (entry === undefined) - yamlDocument.deleteIn([serversKey, PRODUCT_IDENTITY.mcpServerKey]); - else yamlDocument.setIn([serversKey, PRODUCT_IDENTITY.mcpServerKey], entry); - // Editing REA must not reflow unrelated user prompts or other YAML scalars. - return yamlDocument.toString({ lineWidth: 0 }); + return editHermesConfigurationYaml( + originalText, + [serversKey, PRODUCT_IDENTITY.mcpServerKey], + entry, + ); }; /** Build the stdio entry shape expected by one client's configuration dialect. */ diff --git a/src/application/HermesConfigurationYaml.test.ts b/src/application/HermesConfigurationYaml.test.ts new file mode 100644 index 000000000..41b74fcc9 --- /dev/null +++ b/src/application/HermesConfigurationYaml.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; + +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; + +describe("Hermes configuration YAML", () => { + it("writes an edited number from its new value", () => { + expect( + editHermesConfigurationYaml("mode: 0600\nmask: 0x1F\n", ["mode"], 384), + ).toBe("mode: 384\nmask: 0x1F\n"); + }); + + it("accepts keys Hermes reads as distinct and rejects a repeated key", () => { + expect(() => + parseHermesConfigurationYaml("0600: octal\n600: decimal\n"), + ).not.toThrow(); + expect(() => parseHermesConfigurationYaml('a: 1\n"a": 2\n')).toThrow(); + }); +}); diff --git a/src/application/HermesConfigurationYaml.ts b/src/application/HermesConfigurationYaml.ts new file mode 100644 index 000000000..8d245da37 --- /dev/null +++ b/src/application/HermesConfigurationYaml.ts @@ -0,0 +1,74 @@ +import { + isScalar, + parse, + parseDocument, + type ParsedNode, + type ScalarTag, + type Tags, +} from "yaml"; + +const NUMBER_TAGS = new Set([ + "tag:yaml.org,2002:int", + "tag:yaml.org,2002:float", +]); + +/** + * Write a parsed number with its source spelling while that spelling still + * reads as the node's value. Hermes reads config.yaml as YAML 1.1 with exact + * integers, so re-printing a JavaScript number changes what it reads: a + * Discord channel ID loses its low digits, `0600` (octal) becomes `600`, and + * `0.0000001` becomes `1e-7`, which YAML 1.1 does not define as a float. + */ +const keepNumberSource = (tag: Tags[number]): Tags[number] => { + if (typeof tag === "string" || tag.collection !== undefined) return tag; + const { stringify } = tag; + if (!NUMBER_TAGS.has(tag.tag) || stringify === undefined) return tag; + return { + ...tag, + stringify: (item, context, ...rest) => { + const { source } = item; + if (source === undefined) return stringify(item, context, ...rest); + const reader = context.doc.schema.tags.find( + (candidate): candidate is ScalarTag => + candidate.default === true && candidate.test?.test(source) === true, + ); + const read = reader?.resolve( + source, + () => undefined, + context.doc.options, + ); + return Object.is(isScalar(read) ? read.value : read, item.value) + ? source + : stringify(item, context, ...rest); + }, + }; +}; + +const HERMES_YAML_OPTIONS = { + customTags: (tags: Tags) => tags.map(keepNumberSource), + // Hermes keeps integers exact, so IDs that round to one double, or `0600` + // and `600`, are distinct keys. + uniqueKeys: (a: ParsedNode, b: ParsedNode) => + a === b || + (isScalar(a) && + isScalar(b) && + a.value === b.value && + a.source === b.source), +}; + +/** Parse a Hermes `config.yaml`. */ +export const parseHermesConfigurationYaml = (text: string): unknown => + parse(text, HERMES_YAML_OPTIONS); + +/** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */ +export const editHermesConfigurationYaml = ( + text: string, + path: readonly string[], + value: unknown, +): string => { + const document = parseDocument(text, HERMES_YAML_OPTIONS); + if (value === undefined) document.deleteIn(path); + else document.setIn(path, value); + // Editing REA must not reflow unrelated user prompts or other YAML scalars. + return document.toString({ lineWidth: 0 }); +}; diff --git a/tests/boundary/filesystem/hermesClientConfiguration.test.ts b/tests/boundary/filesystem/hermesClientConfiguration.test.ts index 43a5f841c..218573d5c 100644 --- a/tests/boundary/filesystem/hermesClientConfiguration.test.ts +++ b/tests/boundary/filesystem/hermesClientConfiguration.test.ts @@ -237,6 +237,38 @@ mcp_servers: expect(await readFile(client.configPath, "utf8")).toBe(original); }); + it("keeps unrelated numbers as written, including Discord IDs", async () => { + const { home, client } = await hermesClient(); + // Hermes reads these exactly: two IDs one apart, an octal mode, a float. + const original = `discord: + free_response_channels: + - 1187654321098765432 + channel_prompts: + 1187654321098765433: Build questions. + 1187654321098765434: Release questions. + mode: 0600 + min_score: 0.0000001 +mcp_servers: + other: + command: other-server +`; + await writeFile(client.configPath, original); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text.startsWith(original)).toBe(true); + expect(parseYaml(text, { intAsBigInt: true })).toMatchObject({ + mcp_servers: { + rea: { command: command[0], args: command.slice(1), enabled: true }, + }, + }); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(await readFile(client.configPath, "utf8")).toBe(original); + }); + it("creates config.yaml when Hermes has no configuration yet", async () => { const { client } = await hermesClient(); expect( From f63674cac0329aa3ed76234b31cf8a3536ac07d0 Mon Sep 17 00:00:00 2001 From: Danish Rafie Ekaputra <155859586+0xRafie@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:44:41 +0700 Subject: [PATCH 18/28] docs(browser): clarify attach coverage and Chromium setup (#2287) Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/browser-observation.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 2f73f4261..dca1f994b 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -80,6 +80,27 @@ google-chrome \ REA does not launch, own, or terminate this browser. Use a dedicated profile and stop it yourself when the investigation is complete. Do not expose the debugging port on a non-loopback interface. +Other Chromium-family browsers, such as Chromium or Brave, work the same way +with the same two flags. On macOS, run the executable inside the application +bundle, for example +`"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"` or +`"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"`. Fedora installs +Chromium as `chromium-browser`. + +REA needs the browser's HTTP discovery endpoint on a loopback port. A browser +driven over `--remote-debugging-pipe` or a private channel, for example by an +automation SDK, exposes no such endpoint, so `list_browser_targets` has nothing +to attach to. Connection failures report `endpoint_unreachable`; a reachable +HTTP service that returns a non-200 or non-JSON discovery response reports +`invalid_endpoint_response`. +Reproduce that flow in a Chromium-family browser started with +`--remote-debugging-port`. + +In a minimal Linux container, headed Chromium under Xvfb can keep running while +its DevTools port never opens. Install the GLES, EGL, and Mesa DRI libraries +(on Fedora: `libglvnd-gles`, `mesa-libEGL`, and `mesa-dri-drivers`), then +restart the browser. + On Linux, a reachable but locked desktop keyring can stall Chrome before its first navigation. For a dedicated investigation profile, add `--password-store=basic` to bypass that keyring. Empty or placeholder frame URLs @@ -207,7 +228,7 @@ commands fail closed. A correlated reply's session ID must match the command's selected session. An optional method is unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. -Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. +Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Scripts are different: scripts the page parsed before REA attached are still reported under `scripts`. A page that loaded before attach can therefore list its scripts while `network.requests` lacks the requests that fetched them. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. From 0ca70e121c43db41d02c5afb6439e15cbf335289 Mon Sep 17 00:00:00 2001 From: AbdulAzizWang <95555196+AbdulAzizWang@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:51:18 +0800 Subject: [PATCH 19/28] fix(ghidra): pre-create the launch-support config directory (#2470) * fix(ghidra): pre-create the launch-support config directory Ghidra's launch support resolves its JDK before the headless run starts and persists that choice under `/ghidra/`. Inside REA's private runtime a child JVM cannot create those directories, so the first Windows Ghidra query aborted at ghidra.launch.AppConfig.saveJavaHome with: java.io.IOException: Failed to create directory: ...\config\ghidra\ghidra_12.1.4_PUBLIC at ghidra.launch.AppConfig.saveJavaHome(AppConfig.java:145) at LaunchSupport.findJavaHome(LaunchSupport.java:210) LaunchSupport then fell back to its interactive prompt, read EOF, and reported "JDK 21+ (64-bit) could not be found and must be manually chosen!", so the provider failed with "Ghidra exited before bridge startup" even though `rea doctor` reported a valid Ghidra installation and JDK. Create the release directory while this process still owns the runtime, leaving Ghidra only a file write inside it. The release coordinate comes from the installation's Ghidra/application.properties (application.version plus application.release.name) with the extraction directory name as a fallback. On Windows each level goes through the native private-runtime owner, which opens every existing component and creates only the final one. Verified on Windows x64 with Ghidra 12.1.4, a 64-bit Temurin JDK 21, and REA's packaged native addon: `rea inspect --provider ghidra` and the equivalent MCP `binary_overview` call both complete and report 21 segments, 1436 procedures, and 2761 strings. Windows Ghidra remains read-only P0; this change does not extend that boundary. * style(ghidra): format launch configuration regressions * fix(ghidra): reuse Java Properties parsing for launch release identity --------- Co-authored-by: AbdulAzizShanShi <441650742@QQ.COM> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/ghidra/GhidraLauncher.ts | 50 +++++++++++++- .../providers/ghidra/ghidraLauncher.test.ts | 67 ++++++++++++++++++- 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/src/ghidra/GhidraLauncher.ts b/src/ghidra/GhidraLauncher.ts index 59c555606..e1527ea81 100644 --- a/src/ghidra/GhidraLauncher.ts +++ b/src/ghidra/GhidraLauncher.ts @@ -1,3 +1,4 @@ +import { parseGhidraJavaProperties } from "./GhidraJavaProperties.js"; import { GHIDRA_SEED_SCRIPT, snapshotGhidraAnalysisSeeds, @@ -5,7 +6,7 @@ import { } from "./GhidraAnalysisSeeds.js"; import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; -import { mkdir, realpath } from "node:fs/promises"; +import { mkdir, readFile, realpath } from "node:fs/promises"; import { basename, dirname, join, posix, win32 } from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -219,7 +220,11 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { "Installed Allegrex content changed since profile resolution; reopen the target to resolve a fresh profile.", ); } - await createGhidraRuntimeDirectories(paths, platform); + await createGhidraRuntimeDirectories( + paths, + platform, + await ghidraConfigReleaseDirectory(this.options.analyzeHeadlessPath), + ); const seedPath = this.options.analysisSeeds === undefined ? undefined @@ -721,9 +726,44 @@ const ghidraRuntimePaths = (runtimeRoot: string) => ({ scriptLogPath: join(runtimeRoot, "script.log"), }); +/** + * Ghidra's launch support resolves its JDK before the headless run and then + * persists that choice under `/ghidra/`. Inside + * REA's Windows private runtime a child JVM cannot create those directories + * itself, so startup aborted with "Failed to create directory" and reported + * that no supported JDK was found. Create the release directory while this + * process still owns the runtime, leaving Ghidra only a file write inside it. + */ +const ghidraConfigReleaseDirectory = async ( + analyzeHeadlessPath: string, +): Promise => { + const installRoot = dirname(dirname(analyzeHeadlessPath)); + try { + const properties = await readFile( + join(installRoot, "Ghidra", "application.properties"), + "latin1", + ); + const parsed = parseGhidraJavaProperties(properties); + if (!parsed.ok) return basename(installRoot); + const version = parsed.value.get("application.version"); + const release = parsed.value.get("application.release.name"); + return version === undefined || + release === undefined || + release.length === 0 + ? basename(installRoot) + : `ghidra_${version}_${release}`; + } catch (cause: unknown) { + // An unreadable properties file leaves the release coordinate unknown; the + // conventional extraction directory carries the same name. + void cause; + return basename(installRoot); + } +}; + const createGhidraRuntimeDirectories = async ( paths: ReturnType, platform: NodeJS.Platform, + configReleaseDirectory: string, ): Promise => { if (platform === "win32") { const runtime = windowsPrivateRuntime(dirname(paths.projectRoot)); @@ -736,6 +776,11 @@ const createGhidraRuntimeDirectories = async ( paths.dataRoot, ]) runtime.mkdir(basename(path)); + // Each level needs its parent first: the native owner opens every existing + // component and creates only the final one. + const configRoot = basename(paths.configRoot); + runtime.mkdir(win32.join(configRoot, "ghidra")); + runtime.mkdir(win32.join(configRoot, "ghidra", configReleaseDirectory)); return; } await Promise.all( @@ -746,6 +791,7 @@ const createGhidraRuntimeDirectories = async ( paths.cacheRoot, paths.configRoot, paths.dataRoot, + join(paths.configRoot, "ghidra", configReleaseDirectory), ].map((path) => mkdir(path, { recursive: true, mode: 0o700 })), ); }; diff --git a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts index 270bf02b3..ea8fb964c 100644 --- a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts +++ b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts @@ -1,4 +1,11 @@ -import { access, chmod, readFile, stat } from "node:fs/promises"; +import { + access, + chmod, + mkdir, + readFile, + stat, + writeFile, +} from "node:fs/promises"; import { join, posix, win32 } from "node:path"; import { fileURLToPath } from "node:url"; @@ -480,6 +487,64 @@ describe("Ghidra Windows launcher", () => { ).toThrow(/metacharacters/u); }); + it.skipIf(process.platform !== "win32").each([ + { + properties: + "application.version=12.1.4\napplication.release.name=PUBLIC\n", + release: "ghidra_12.1.4_PUBLIC", + }, + { properties: undefined, release: "renamed-install" }, + { properties: "application.version=12.1.4\n", release: "renamed-install" }, + ])( + "prepares the release directory through the Windows runtime ($release)", + async ({ properties, release }) => { + const parent = await createTestTempDirectory("rea-launch-config-"); + const installRoot = join(parent, "renamed-install"); + if (properties !== undefined) { + await mkdir(join(installRoot, "Ghidra"), { recursive: true }); + await writeFile( + join(installRoot, "Ghidra", "application.properties"), + properties, + ); + } + const runtime = await createGhidraTestRuntime(parent); + runtimes.push(runtime); + const spawnProcess = vi.fn(async () => { + throw new Error("fixture stops before process creation"); + }); + const launcher = new GhidraHeadlessLauncher({ + environment: process.env, + platform: "win32", + analyzeHeadlessPath: join( + installRoot, + "support", + "analyzeHeadless.bat", + ), + javaHome: "C:\\Java\\jdk-21", + bridgeScriptPath: join(parent, "ReaGhidraBridge.java"), + spawnProcess, + }); + const launched = await launcher.launch({ + runtimeRoot: runtime.path, + transport: "unix-socket", + endpointPath: join(runtime.path, "bridge.sock"), + token: "fixture-token", + runId: "d6fcbb66-e829-4ff6-a535-0035aec63139", + targetPath: join(parent, "fixture.exe"), + targetSha256: "b".repeat(64), + providerVersion: "12.1.4", + profileDigest: "a".repeat(64), + }); + expect(launched.ok).toBe(false); + expect(spawnProcess).toHaveBeenCalledOnce(); + expect( + ( + await stat(join(runtime.path, "config", "ghidra", release)) + ).isDirectory(), + ).toBe(true); + }, + ); + it.skipIf(process.platform !== "win32")( "keeps authority in a private descriptor and isolates Ghidra state through the batch launcher", async () => { From 03ab521db3330493372dd47ab858fc142feea5b9 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:51:44 -0400 Subject: [PATCH 20/28] fix(process): resolve POSIX backslash basenames through PATH (#2471) Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- .../ProcessScenarioRuntimeValidation.test.ts | 22 ++++++++++++++ .../ProcessScenarioRuntimeValidation.ts | 6 ++-- .../processCaptureNormalization.test.ts | 29 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/process/capture/ProcessScenarioRuntimeValidation.test.ts b/src/process/capture/ProcessScenarioRuntimeValidation.test.ts index 1277b753f..7e5cb7e30 100644 --- a/src/process/capture/ProcessScenarioRuntimeValidation.test.ts +++ b/src/process/capture/ProcessScenarioRuntimeValidation.test.ts @@ -17,6 +17,28 @@ import { resolveProcessScenarioRuntimePaths, } from "./ProcessScenarioRuntimeValidation.js"; import { parseProcessScenario } from "../../domain/process/processScenario.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; + +it.skipIf(process.platform === "win32")( + "resolves a literal backslash in a POSIX executable basename through PATH", + async () => { + const root = await createTestTempDirectory("rea-process-posix-basename-"); + const bin = join(root, "bin"); + await mkdir(bin); + const command = "selected\\node"; + const selected = join(bin, command); + await symlink(process.execPath, selected); + await expect(resolveExecutable(command, root, bin)).resolves.toBe(selected); + const resolved = await resolveProcessScenarioRuntimePaths( + parseProcessScenario({ + executable: command, + working_directory: root, + environment: { PATH: bin }, + }), + ); + expect(resolved.executable).toBe(selected); + }, +); it("preserves the caller-selected executable symlink for process invocation", async () => { const root = await mkdtemp(join(tmpdir(), "rea-process-executable-alias-")); diff --git a/src/process/capture/ProcessScenarioRuntimeValidation.ts b/src/process/capture/ProcessScenarioRuntimeValidation.ts index 1ac9d2228..6d2978b5b 100644 --- a/src/process/capture/ProcessScenarioRuntimeValidation.ts +++ b/src/process/capture/ProcessScenarioRuntimeValidation.ts @@ -1,6 +1,6 @@ import { constants as fsConstants } from "node:fs"; import { access, realpath, stat } from "node:fs/promises"; -import { delimiter, isAbsolute, join, resolve } from "node:path"; +import { delimiter, isAbsolute, join, resolve, sep } from "node:path"; import type { ProcessScenario } from "../../domain/process/processScenario.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; @@ -71,7 +71,9 @@ export const resolveExecutable = async ( : resolve(workingDirectory, directory), ); const candidates = - isAbsolute(command) || command.includes("/") || command.includes("\\") + isAbsolute(command) || + command.includes("/") || + (sep === "\\" && command.includes("\\")) ? [resolve(workingDirectory, command)] : directories.map((directory) => join(directory, command)); const failures: string[] = []; diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 2837ccf40..159f84c93 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -28,6 +28,35 @@ const identitySchema = z.object({ const itWithLinuxCaptureCapability = process.platform === "linux" ? itWithCaptureCapability : it.skip; +itWithCaptureCapability( + "captures a POSIX PATH executable with a literal backslash in its basename", + async () => { + const root = await createTestTempDirectory("rea-process-backslash-name-"); + const bin = join(root, "bin"); + await mkdir(bin); + const command = "selected\\node"; + const selected = join(bin, command); + await symlink(process.execPath, selected); + const scenario = parseProcessScenario({ + executable: command, + working_directory: root, + environment: { PATH: bin }, + arguments: ["-e", 'process.stdout.write("selected executable")'], + settle_ms: 100, + }); + const result = await captureProcessScenario(scenario); + if (!result.ok) throw result.error; + expect(result.value.frames.map(({ data }) => data).join("")).toBe( + "selected executable", + ); + expect(result.value.exit).toMatchObject({ code: 0, reason: "exited" }); + expect(result.value.cleanup).toMatchObject({ + owned_process_group: "verified", + temporary_root: "removed", + }); + }, +); + itWithCaptureCapability( "preserves numeric reports and original normalized PTY text in Evidence", async () => { From 43738bec579fc69bab084bffa964916d43d43ed1 Mon Sep 17 00:00:00 2001 From: Aurantium <3550271410@qq.com> Date: Tue, 13 Oct 2026 05:52:17 -0700 Subject: [PATCH 21/28] fix(targets): retain unsupported PE machine in diagnostics (#2448) Assisted-by: OpenAI GPT-6 Co-authored-by: Aurantium-47 <280805072+Aurantium-47@users.noreply.github.com> --- src/domain/binaryTarget.ts | 8 ++++-- .../filesystem/targetDiagnostics.test.ts | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 48da65960..99bc6dd24 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -278,8 +278,12 @@ export const parsePeRecord = ( record.toString("binary", 0, 4) !== "PE\u0000\u0000" ) return err("invalid or truncated PE header"); - const machine = decodePeMachine(record.readUInt16LE(4)); - if (machine === undefined) return err("unsupported PE architecture"); + const machineType = record.readUInt16LE(4); + const machine = decodePeMachine(machineType); + if (machine === undefined) + return err( + `unsupported PE architecture (COFF machine 0x${machineType.toString(16).padStart(4, "0")})`, + ); const { architecture } = machine; const optionalHeaderSize = record.readUInt16LE(20); if (optionalHeaderSize > 4096 || record.length < 24 + optionalHeaderSize) diff --git a/tests/boundary/filesystem/targetDiagnostics.test.ts b/tests/boundary/filesystem/targetDiagnostics.test.ts index 5d1953760..dacd21687 100644 --- a/tests/boundary/filesystem/targetDiagnostics.test.ts +++ b/tests/boundary/filesystem/targetDiagnostics.test.ts @@ -4,9 +4,35 @@ import { describe, expect, it } from "vitest"; import { systemDoctorHost } from "../../../src/application/Doctor.js"; import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { pe } from "../../../src/domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("executable format refusal diagnostics", () => { + it.each([ + [0x01a6, "0x01a6"], + [0x0000, "0x0000"], + [0xffff, "0xffff"], + ] as const)( + "retains unsupported PE machine %i in the refusal", + async (machine, expectedMachine) => { + const directory = await createTestTempDirectory("rea-target-pe-machine-"); + const path = join(directory, "unsupported.exe"); + const bytes = pe(0x014c); + bytes.writeUInt16LE(machine, 0x44); + await writeFile(path, bytes); + + const result = await parseBinaryTarget(path); + if (result.ok) throw new Error("unsupported PE target was admitted"); + const reason = `unsupported PE architecture (COFF machine ${expectedMachine})`; + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + message: `Cannot open artifact: ${reason}.`, + details: { reason, constraint: "file_format_rejected" }, + }); + }, + ); + it.each(["NE", "LE", "LX"] as const)( "reports unsupported %s executables without path advice", async (signature) => { From 8e6f37e2a0873ac21ebcc889c27839bfcb7552ca Mon Sep 17 00:00:00 2001 From: Lucas LSMO Date: Tue, 13 Oct 2026 09:54:37 -0300 Subject: [PATCH 22/28] fix(formats): preserve high bits in remaining magic-byte checks (#2384) * fix(formats): preserve high bits in remaining magic-byte checks Node's "ascii" decoder masks the high bit, so byte-for-byte signature comparisons accepted lookalikes such as CD DA ("MZ" | 0x80) or a high-bit "bplist00"/"NIBArchive" header. ac57c88ef fixed this for the shared DOS/MZ reader; apply the same latin1 decoding to the remaining signature checks: NativeAOT and Windows native loader MZ/PE, binary plist and NIBArchive detection, and PNG IHDR checks. Display-name decodes (section and metadata stream names) are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) * style(windows): format signature admission regressions --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- scripts/verify-windows-native-artifact.mjs | 4 ++-- src/domain/native/nativeAotPe.test.ts | 19 +++++++++++++++++++ src/domain/native/nativeAotPe.ts | 4 ++-- src/windows/WindowsNativeLoader.test.ts | 16 ++++++++++++++++ src/windows/WindowsNativeLoader.ts | 4 ++-- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/scripts/verify-windows-native-artifact.mjs b/scripts/verify-windows-native-artifact.mjs index 1a6777322..4db66e0ee 100644 --- a/scripts/verify-windows-native-artifact.mjs +++ b/scripts/verify-windows-native-artifact.mjs @@ -27,9 +27,9 @@ if ( ); const offset = bytes.length >= 64 ? bytes.readUInt32LE(0x3c) : bytes.length; if ( - bytes.toString("ascii", 0, 2) !== "MZ" || + bytes.toString("latin1", 0, 2) !== "MZ" || offset > bytes.length - 26 || - bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" || + bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" || bytes.readUInt16LE(offset + 4) !== 0x8664 || bytes.readUInt16LE(offset + 24) !== 0x20b ) diff --git a/src/domain/native/nativeAotPe.test.ts b/src/domain/native/nativeAotPe.test.ts index 8ae624d44..ab031cc85 100644 --- a/src/domain/native/nativeAotPe.test.ts +++ b/src/domain/native/nativeAotPe.test.ts @@ -159,3 +159,22 @@ describe("read-only NativeAOT PE parser", () => { }); }); }); + +describe("NativeAOT PE signature admission", () => { + it.each([ + ["MZ", "Not a PE image"], + ["PE", "Invalid PE signature"], + ])( + "rejects a high-bit lookalike of the %s signature", + (signature, reason) => { + const bytes = nativeAotPeFixture(); + const offset = signature === "MZ" ? 0 : bytes.readUInt32LE(0x3c); + for (const position of [offset, offset + 1]) + bytes.writeUInt8(bytes.readUInt8(position) | 0x80, position); + expect( + parseNativeAotPe(bytes, nativeAotPeDigest(bytes), SUMMARY_BUDGET) + .summary, + ).toMatchObject({ status: "not_applicable", reason }); + }, + ); +}); diff --git a/src/domain/native/nativeAotPe.ts b/src/domain/native/nativeAotPe.ts index b67a58398..7f9c5eb55 100644 --- a/src/domain/native/nativeAotPe.ts +++ b/src/domain/native/nativeAotPe.ts @@ -384,11 +384,11 @@ class PeImage { constructor(bytes: Buffer) { this.#bytes = bytes; - if (bytes.length < 0x40 || bytes.toString("ascii", 0, 2) !== "MZ") + if (bytes.length < 0x40 || bytes.toString("latin1", 0, 2) !== "MZ") throw new Error("Not a PE image"); const peOffset = this.#u32(0x3c); this.#range(peOffset, 24); - if (bytes.toString("ascii", peOffset, peOffset + 4) !== "PE\0\0") + if (bytes.toString("latin1", peOffset, peOffset + 4) !== "PE\0\0") throw new Error("Invalid PE signature"); this.#machine = this.#u16(peOffset + 4); const count = this.#u16(peOffset + 6); diff --git a/src/windows/WindowsNativeLoader.test.ts b/src/windows/WindowsNativeLoader.test.ts index 1aa8d7375..c0fee8273 100644 --- a/src/windows/WindowsNativeLoader.test.ts +++ b/src/windows/WindowsNativeLoader.test.ts @@ -143,6 +143,22 @@ describe("packaged Windows native admission", () => { }, ); + it.each([ + { signature: "MZ", offset: 0 }, + { signature: "PE", offset: 64 }, + ])( + "rejects digest-valid high-bit $signature lookalikes before loading", + ({ offset }) => { + const value = fixture(); + value.bytes[offset] = (value.bytes[offset] ?? 0) | 0x80; + value.manifest.artifactSha256 = createHash("sha256") + .update(value.bytes) + .digest("hex"); + expect(loadWindowsNativeAuthority(value.host).available).toBe(false); + expect(value.loads()).toBe(0); + }, + ); + it("rejects a digest-valid artifact for another machine", () => { const value = fixture(); value.bytes.writeUInt16LE(0xaa64, 68); diff --git a/src/windows/WindowsNativeLoader.ts b/src/windows/WindowsNativeLoader.ts index 7719a0a85..2ab08e1ed 100644 --- a/src/windows/WindowsNativeLoader.ts +++ b/src/windows/WindowsNativeLoader.ts @@ -132,12 +132,12 @@ export const loadWindowsNativeAuthority = ( }; const assertWindowsX64Artifact = (bytes: Buffer): void => { - if (bytes.length < 64 || bytes.toString("ascii", 0, 2) !== "MZ") + if (bytes.length < 64 || bytes.toString("latin1", 0, 2) !== "MZ") throw new Error("Windows native artifact is not a PE image"); const offset = bytes.readUInt32LE(0x3c); if ( offset > bytes.length - 26 || - bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" || + bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" || bytes.readUInt16LE(offset + 4) !== 0x8664 || bytes.readUInt16LE(offset + 24) !== 0x20b ) From e0bd31c18c84fc406ad1c1fe412bc9018ede44ec Mon Sep 17 00:00:00 2001 From: Dery Ferdika <114888155+DeryFerd@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:55:40 +0700 Subject: [PATCH 23/28] fix(ghidra): bound the Java version probe output (#2361) The Java version probe captured its output with maxBuffer set to Infinity, so a chatty or hostile JVM selected as the Ghidra launcher could grow the captured stdout/stderr without limit and exhaust REA's memory during installation discovery. Bound the capture to 1 MiB, matching the budget already used for other Java listings. A probe that exceeds the budget yields no observation and is reported through the existing failed-probe path, the same one the Java check already uses whenever the probe cannot produce an observation. Co-authored-by: Claude Code --- src/ghidra/GhidraInstallation.ts | 11 ++++- .../filesystem/ghidraJavaProbe.test.ts | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index c6ca5212d..9675d133f 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -72,6 +72,15 @@ const NATIVE_ARCHITECTURES: readonly { /** Bound installation discovery reads; larger headers remain unknown. */ const EXECUTABLE_HEADER_PROBE_BYTES = 4096; +/** + * Bound the Java version probe so a chatty or hostile JVM cannot force an + * unbounded in-memory capture. Well-behaved JVMs print a few kilobytes for + * `-XshowSettings:properties`; the same 1 MiB budget used for other Java + * listings leaves ample headroom. A probe that exceeds the budget yields no + * observation and is reported through the existing failed-probe path. + */ +const JAVA_PROBE_MAX_OUTPUT_BYTES = 1024 * 1024; + /** Compatibility of one present native decompiler with the inspected host. */ type NativeDecompilerCompatibility = | { @@ -763,7 +772,7 @@ const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ encoding: "utf8", env: environment, timeout: 5_000, - maxBuffer: Number.POSITIVE_INFINITY, + maxBuffer: JAVA_PROBE_MAX_OUTPUT_BYTES, windowsHide: true, }, ); diff --git a/tests/boundary/filesystem/ghidraJavaProbe.test.ts b/tests/boundary/filesystem/ghidraJavaProbe.test.ts index c0a6974d5..415fbf3dd 100644 --- a/tests/boundary/filesystem/ghidraJavaProbe.test.ts +++ b/tests/boundary/filesystem/ghidraJavaProbe.test.ts @@ -87,3 +87,43 @@ it.each([ ); }, ); + +it("bounds the Java version probe and reports an over-budget probe as a failed Java check", async () => { + const root = await createTestTempDirectory("rea-java-probe-bound-"); + const configuredHome = join(root, "selected-jdk"); + const installDir = join(root, "ghidra"); + await mkdir(join(installDir, "Ghidra"), { recursive: true }); + await writeFile( + join(installDir, "Ghidra", "application.properties"), + `application.version=${SUPPORTED_GHIDRA_VERSION}\napplication.java.min=17\n`, + ); + // A chatty or hostile JVM that overruns the capture budget: spawnSync reports + // ENOBUFS, which the probe already treats as a failed observation. + mockSpawnSync.mockReturnValue({ + stdout: "", + stderr: "java.version = 21.0.11\n", + status: null, + error: Object.assign(new Error("spawnSync /java ENOBUFS"), { + code: "ENOBUFS", + }), + }); + + const inspection = inspectGhidraInstallation({ + environment: {}, + installDir, + javaHome: configuredHome, + }); + + const [, , options] = mockSpawnSync.mock.calls[0] as [ + string, + readonly string[], + { readonly maxBuffer?: number }, + ]; + expect(options.maxBuffer).toBe(1024 * 1024); + expect(inspection.javaVersion).toBeNull(); + expect(inspection.javaHome).toBe(configuredHome); + expect(inspection.checks.find(({ name }) => name === "java")).toMatchObject({ + status: "failed", + code: "runtime_missing", + }); +}); From 5f8af4ee5f615e97f343d70d91a74568d0ec9fbd Mon Sep 17 00:00:00 2001 From: Mykola Dementiev Date: Tue, 13 Oct 2026 14:56:20 +0200 Subject: [PATCH 24/28] feat(ida): resolve containing procedures and report the admitted program as the document (#2374) * test(mcp): expect trace_feature to report references the provider truncated A provider that reports a capped xrefs read makes the trace truncated with a residual unknown for that target. An explicit complete report and a provider that reports nothing leave the trace as before. * fix(mcp): report references a provider truncated in trace_feature Executions gain an optional flag for a capped xrefs read, set by the IDA adapter from the upstream cap. trace_feature marks the trace truncated with a residual unknown for that target only when the provider reports a truncation; a provider that reports nothing keeps its previous result. The trace_feature workflow profile is revised so earlier results are not replayed. * refactor(ida): report reference truncation whenever the runner observed an xrefs cap * test(mcp): cover session forwarding, per-target state, native reports and failures of the trace coverage * test(mcp): format the session forwarding test * test(ida): expect headless IDA to resolve containing procedures from lookups and segment facts Cover positive and negative containment, segment boundaries including addresses above 2^53, failures of the lookup and of the segment survey, cancellation, input validation and the admitted program as the only document. * test(ida): expect feature tracing and address context to work on headless IDA Run trace_feature, including a capped reference read, and inspect_address_context with and without an explicit document through the IDA session client. * test(mcp): expect trace_feature availability to follow the search operations it calls * fix(mcp): derive trace_feature availability from the search operations it calls * feat(ida): resolve containing procedures and report the admitted program as the document The modern headless profile resolves an address through the function lookup and, when no function contains it, tells a mapped address from an unmapped one using the database-scoped minimal survey. A survey that is missing, truncated or inconsistent fails the read. list_documents reports the single admitted program. The attached profiles do not declare either operation. * docs(ida): describe containing procedures, the admitted document and truncated feature traces * test(ida): verify containing procedures, the admitted document and feature tracing on real IDA * docs(testing): list the headless IDA workflows the real lane verifies * test(ida): make each survey failure case isolate a single defect * test(ida): pin large addresses, segment order, document identity, attached refusals and each trace_feature prerequisite * test(mcp): expect a repeated reference source to be resolved once in trace_feature * fix(mcp): resolve the containing procedure of a repeated reference source once * test(ida): trace the fixture procedure and derive probe addresses from its facts in the real lane * test(process): format processCaptureLifecycle.test.ts after the merge * test(ida): declare the alignment each synthetic universal slice offset satisfies * refactor(ida): move the real lane's feature trace assertions into a helper * test(ida): expect the real lane's feature trace check to accept references to every matched address * fix(ida): check each trace reference against its own matched address in the real lane * test(ida): pin that the new operations reject a document other than the admitted program * refactor(ida): move the real lane's mapped non-function probe into a helper * test(ida): expect the real lane's mapped data probe to skip strings inside functions * fix(ida): try the target's strings in order for the mapped non-function probe * test(ida): declare the real lane helper types for the verifier tests * test(ida): expect the real lane to reject a listed string that resolves as unmapped * fix(ida): fail the real lane when a listed string resolves as unmapped --- docs/ida-provider.md | 2 + docs/testing.md | 4 +- scripts/lib/ida-feature-trace.d.mts | 9 + scripts/lib/ida-feature-trace.mjs | 52 ++ scripts/verify-real-ida.mjs | 61 +- src/application/AnalysisProvider.ts | 11 + src/application/CapabilityInventory.test.ts | 68 ++ src/application/CapabilityInventory.ts | 4 +- src/application/EnhancedLiteralTracing.ts | 64 +- src/application/EnhancedTools.ts | 2 + src/application/InvestigationProviders.ts | 4 +- .../binary/AnalysisSnapshotCache.test.ts | 49 ++ .../binary/AnalysisSnapshotCache.ts | 3 + src/ida/IdaContainment.test.ts | 623 ++++++++++++++++++ src/ida/IdaNativeProtocol.test.ts | 30 + src/ida/IdaOperationRunner.ts | 106 ++- src/ida/IdaProtocolValues.ts | 50 ++ src/ida/IdaProviderCapabilities.ts | 79 ++- src/ida/IdaReferenceCoverage.test.ts | 96 +++ src/ida/IdaSessionClient.ts | 19 + ...enhancedTools.traceFeatureCoverage.test.ts | 249 +++++++ .../process/processCaptureLifecycle.test.ts | 6 +- .../providers/ida/containment.test.ts | 144 ++++ .../ida/verifierFeatureTrace.test.ts | 212 ++++++ .../analysis-sessions/binarySession.test.ts | 99 +++ .../conformance/ida/providerContracts.test.ts | 16 +- tests/fixtures/idaMcp.ts | 1 + 27 files changed, 1984 insertions(+), 79 deletions(-) create mode 100644 scripts/lib/ida-feature-trace.d.mts create mode 100644 scripts/lib/ida-feature-trace.mjs create mode 100644 src/ida/IdaContainment.test.ts create mode 100644 src/ida/IdaReferenceCoverage.test.ts create mode 100644 tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts create mode 100644 tests/boundary/providers/ida/containment.test.ts create mode 100644 tests/boundary/providers/ida/verifierFeatureTrace.test.ts diff --git a/docs/ida-provider.md b/docs/ida-provider.md index b22a35cce..50040425b 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -126,6 +126,8 @@ For guided registration, set `REA_IDA_MCP_CONFIG` before `rea setup --client Promise, +): Promise; diff --git a/scripts/lib/ida-feature-trace.mjs b/scripts/lib/ida-feature-trace.mjs new file mode 100644 index 000000000..3c4778d25 --- /dev/null +++ b/scripts/lib/ida-feature-trace.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; + +const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]); + +/** + * Assert the `trace_feature` result of the real IDA lane for one selected + * procedure. The query is a substring search, so the trace may also match other + * procedures and strings; each reference is checked against its own matched + * target, and a source outside every function (a data reference) is valid. + */ +export function assertFeatureTrace(trace, procedure) { + assert.ok( + trace.matches.some( + ({ type, address }) => + type === "procedure" && address === procedure.address, + ), + "The trace omitted the selected procedure's match.", + ); + assert.equal(typeof trace.truncated, "boolean"); + const matched = new Set(trace.matches.map(({ address }) => address)); + for (const reference of trace.references) { + assert.ok( + matched.has(reference.target_address), + `A reference targets ${reference.target_address}, which is not a match.`, + ); + const { containing_procedure: containing } = reference; + if (containing.found) + assert.equal(typeof containing.procedure.address, "string"); + else assert.ok(NEGATIVE_REASONS.has(containing.reason)); + } +} + +/** + * Find a string that IDA maps outside every function. A string inside a + * function chunk (inline data in code) is valid and skipped, so the strings are + * tried in order. Any other answer must be "not in a procedure": IDA lists only + * mapped strings, so an unmapped answer is a containment defect and fails the + * lane. Returns the address, or undefined when every string is inside a function. + */ +export async function findMappedNonFunction(strings, resolveAt) { + for (const { address } of strings) { + const resolved = await resolveAt(address); + if (resolved.found) continue; + assert.equal( + resolved.reason, + "not_in_procedure", + `The listed string at ${address} resolved as ${resolved.reason}.`, + ); + return address; + } + return undefined; +} diff --git a/scripts/verify-real-ida.mjs b/scripts/verify-real-ida.mjs index cedbf5076..bed0519a9 100644 --- a/scripts/verify-real-ida.mjs +++ b/scripts/verify-real-ida.mjs @@ -1,4 +1,8 @@ #!/usr/bin/env node +import { + assertFeatureTrace, + findMappedNonFunction, +} from "./lib/ida-feature-trace.mjs"; import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; @@ -6,7 +10,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { access, writeFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs, promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -261,6 +265,61 @@ const mcp = async (cliDossier) => { break; } } + if (config.mode === "headless") { + const documents = observe(await call("list_documents", {})); + assert.equal(documents.length, 1); + assert.equal(basename(documents[0]), basename(target)); + assert.ok( + !workspaces.has(dirname(documents[0])), + "list_documents reported the private workspace copy.", + ); + const resolveAt = async (address) => + observe(await call("resolve_containing_procedure", { address })); + const instructionAddresses = dossier.assembly + .map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1]) + .filter((address) => address !== undefined); + const queries = [ + dossier.procedure.address, + ...instructionAddresses.slice(1, 2), + ]; + for (const queried of queries) { + const containing = await resolveAt(queried); + assert.equal(containing.found, true); + assert.equal(containing.procedure.address, dossier.procedure.address); + assert.equal(containing.procedure.name, dossier.procedure.name); + } + const unmapped = await resolveAt("0xffffffffffffff00"); + assert.equal(unmapped.found, false); + assert.equal(unmapped.reason, "outside_segments"); + const strings = observe(await call("list_strings", {})); + const mappedString = await findMappedNonFunction(strings, resolveAt); + if (mappedString === undefined) + report.unverified.push( + "Mapped non-function containment: no string of the target is mapped outside every function.", + ); + const context = await client.callTool({ + name: "inspect_address_context", + arguments: { address: dossier.procedure.address }, + }); + assert.notEqual(context.isError, true); + const contextResult = context.structuredContent.result; + assert.equal(contextResult.document, documents[0]); + assert.equal(contextResult.procedure.state, "available"); + assert.equal( + contextResult.procedure.value.procedure.address, + dossier.procedure.address, + ); + const traced = await client.callTool({ + name: "trace_feature", + arguments: { query: dossier.procedure.name }, + }); + assert.notEqual(traced.isError, true); + const trace = traced.structuredContent.normalized_result; + assertFeatureTrace(trace, dossier.procedure); + report.checks.push( + "headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing", + ); + } if (values["expected-string"] === undefined) report.unverified.push( "String inventory/search correctness: no independent expected literal supplied.", diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index f3b51257a..376036fa1 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -50,6 +50,13 @@ export interface AnalysisExecution { readonly limitations: readonly string[]; /** Provider-owned completeness assessment for caller/callee observations. */ readonly relationshipCoverage?: RelationshipCoverage; + /** + * Whether the provider capped an `xrefs` read: true when it reported a + * truncation, false when it explicitly reported none. Absent when it reports + * nothing, which is not a completeness claim. Never part of Evidence or tool + * output. + */ + readonly referencesTruncated?: boolean; readonly locations: readonly EvidenceLocation[]; readonly subject: EvidenceSubjectTarget | null; } @@ -59,6 +66,7 @@ interface CreateAnalysisExecutionOptions { readonly analysisProfile?: AnalysisProfileCommitment; readonly limitations?: readonly string[]; readonly relationshipCoverage?: RelationshipCoverage; + readonly referencesTruncated?: boolean; readonly locations?: readonly EvidenceLocation[]; readonly subject?: EvidenceSubjectTarget; } @@ -103,6 +111,9 @@ export function createAnalysisExecution( options.relationshipCoverage, ), }), + ...(options.referencesTruncated === undefined + ? {} + : { referencesTruncated: options.referencesTruncated }), locations: [...(options.locations ?? [])], subject: options.subject ?? null, }; diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 1b8ec6411..4286ff1d0 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -497,3 +497,71 @@ it("reports host denial before target mismatch but keeps composed workflow prere ).reason, ).toBe("target_required"); }); + +describe("trace_feature requirements", () => { + const capabilities = (operations: readonly string[]) => + operations.map((operation) => ({ + operation, + available: true, + reason: null, + })); + + it("needs the search operations the workflow actually calls", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: true, reason: "available" }); + }); + + it.each(["search_strings", "search_procedures"])( + "is unavailable without %s", + (missing) => { + const operations = [ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ].filter((operation) => operation !== missing); + + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities(operations), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }, + ); + + it("is unavailable when only the list operations exist", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "list_strings", + "list_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }); +}); diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 6fefb8e20..4e364a1aa 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -121,8 +121,8 @@ const ENHANCED_REQUIREMENTS: Readonly> = { analyze_function: ["analyze_function"], inspect_native_api: ["analyze_function"], trace_feature: [ - "list_strings", - "list_procedures", + "search_strings", + "search_procedures", "xrefs", "resolve_containing_procedure", ], diff --git a/src/application/EnhancedLiteralTracing.ts b/src/application/EnhancedLiteralTracing.ts index 9c86781bc..79684ce0f 100644 --- a/src/application/EnhancedLiteralTracing.ts +++ b/src/application/EnhancedLiteralTracing.ts @@ -1,5 +1,8 @@ import { z } from "zod"; -import type { AnalysisOperation } from "./AnalysisProvider.js"; +import type { + AnalysisExecution, + AnalysisOperation, +} from "./AnalysisProvider.js"; import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -16,6 +19,12 @@ type AnalysisCall = ( signal?: AbortSignal, ) => Promise>; +type ObserveCall = ( + name: AnalysisOperation, + arguments_: Readonly>, + signal?: AbortSignal, +) => Promise>; + type LiteralSearchSource = readonly [ "search_strings" | "search_procedures", "string" | "procedure", @@ -39,6 +48,7 @@ export interface LiteralTraceInput { /** Trace matching strings and procedures through provider xrefs. */ export const traceLiteralFeature = async ( call: AnalysisCall, + observe: ObserveCall, input: LiteralTraceInput, signal?: AbortSignal, ): EnhancedResult => { @@ -49,7 +59,7 @@ export const traceLiteralFeature = async ( signal, ); if (!searched.ok) return searched; - const traced = await traceReferences(call, { + const traced = await traceReferences(call, observe, { matches: searched.value.matches, ...(signal === undefined ? {} : { signal }), }); @@ -59,8 +69,8 @@ export const traceLiteralFeature = async ( search_mode: "literal", matches: searched.value.matches, references: traced.value.references, - truncated: false, - residual_unknowns: [], + truncated: traced.value.residual.length > 0, + residual_unknowns: traced.value.residual, }); }; @@ -126,24 +136,39 @@ const addPageMatches = (input: { const traceReferences = async ( call: AnalysisCall, + observe: ObserveCall, request: ReferenceTraceRequest, -): Promise> => { +): Promise< + Result<{ references: TraceReference[]; residual: string[] }, AnalysisError> +> => { const references: TraceReference[] = []; + const residual: string[] = []; + // A source address is resolved once however many matches reference it. + const containing = new Map(); for (const match of request.matches) { - const xrefs = await call( + const observed = await observe( "xrefs", { address: match.address }, request.signal, ); - if (!xrefs.ok) return xrefs; - if (!Array.isArray(xrefs.value)) + if (!observed.ok) return observed; + // Only an explicit producer report of truncation is a coverage gap; a + // provider that reports nothing keeps its previous, unqualified result. + const unknown = `References to ${match.address} were truncated by the provider; additional references may exist.`; + if ( + observed.value.referencesTruncated === true && + !residual.includes(unknown) + ) + residual.push(unknown); + const sources = observed.value.result; + if (!Array.isArray(sources)) return err( new AnalysisOutputError( "xrefs", "provider returned a non-array result", ), ); - for (const source of xrefs.value) { + for (const source of sources) { if (typeof source !== "string") return err( new AnalysisOutputError( @@ -151,18 +176,23 @@ const traceReferences = async ( "provider returned a non-address value", ), ); - const resolved = await call( - "resolve_containing_procedure", - { address: source }, - request.signal, - ); - if (!resolved.ok) return resolved; + let containingProcedure = containing.get(source); + if (containingProcedure === undefined) { + const resolved = await call( + "resolve_containing_procedure", + { address: source }, + request.signal, + ); + if (!resolved.ok) return resolved; + containingProcedure = resolved.value; + containing.set(source, containingProcedure); + } references.push({ target_address: match.address, source_address: source, - containing_procedure: resolved.value, + containing_procedure: containingProcedure, }); } } - return ok({ references }); + return ok({ references, residual }); }; diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index 87d27fc5f..207186ccc 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -163,6 +163,8 @@ export class EnhancedTools { return traceLiteralFeature( (name, arguments_, operationSignal) => this.#call(name, arguments_, operationSignal), + (name, arguments_, operationSignal) => + this.#observe(name, arguments_, operationSignal), call.input, signal, ); diff --git a/src/application/InvestigationProviders.ts b/src/application/InvestigationProviders.ts index 5dcb18150..af23463f4 100644 --- a/src/application/InvestigationProviders.ts +++ b/src/application/InvestigationProviders.ts @@ -20,7 +20,9 @@ export const workflowAnalysisProfile = ( upstream_analysis_profile: upstream, ...(operation === "binary_overview" ? { workflow_revision: 3 } - : operation === "batch_decompile" || operation === "get_call_graph" + : operation === "batch_decompile" || + operation === "get_call_graph" || + operation === "trace_feature" ? { workflow_revision: 2 } : {}), }); diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 48c2a54bd..91b9f8807 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -661,3 +661,52 @@ it("does not replay pre-envelope call graphs under the revised workflow profile" }).success, ).toBe(false); }); + +describe("executions that report truncated references", () => { + const record = (referencesTruncated: boolean | undefined) => { + const cache = new AnalysisSnapshotCache(); + const parameters = { address: "0x1000", document: "fixture" }; + cache.record({ + target: ANALYSIS_SNAPSHOT_TARGET, + profile: ANALYSIS_SNAPSHOT_PROFILE, + operation: "xrefs", + parameters, + execution: createAnalysisExecution( + ["0x2000"], + ANALYSIS_SNAPSHOT_PROVIDER, + { + subject: ANALYSIS_SNAPSHOT_TARGET, + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + ...(referencesTruncated === undefined ? {} : { referencesTruncated }), + }, + ), + }); + return cache.lookup( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROFILE, + "xrefs", + parameters, + ); + }; + + it("are not replayed, because a replay would lose the truncation", () => { + expect(record(true)).toBeUndefined(); + }); + + it("do not stop an execution that is complete or silent from being replayed", () => { + expect(record(false)?.result).toEqual(["0x2000"]); + expect(record(undefined)?.result).toEqual(["0x2000"]); + }); +}); + +it("revises the trace_feature workflow profile so earlier results are not replayed", () => { + const profile = ANALYSIS_SNAPSHOT_PROFILE; + + expect(workflowAnalysisProfile(profile, "trace_feature")).not.toEqual( + workflowAnalysisProfile(profile), + ); + expect( + workflowAnalysisProfile(profile, "trace_call_path"), + "other workflows keep their profile", + ).toEqual(workflowAnalysisProfile(profile)); +}); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index bdaa9f68d..7e307baa0 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -337,6 +337,9 @@ export class AnalysisSnapshotCache { }): void { const { target, profile, operation, parameters, execution } = input; this.select(target, profile); + // The snapshot format does not carry reference truncation; a replay would + // present a capped read as complete, so it is not recorded. + if (execution.referencesTruncated === true) return; const queryId = analysisQueryId( snapshotTarget(target), snapshotBinding(profile), diff --git a/src/ida/IdaContainment.test.ts b/src/ida/IdaContainment.test.ts new file mode 100644 index 000000000..069388df8 --- /dev/null +++ b/src/ida/IdaContainment.test.ts @@ -0,0 +1,623 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { decodeIdaToolResult } from "./IdaMcpConnection.js"; +import { idaCapabilities } from "./IdaProviderCapabilities.js"; +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const segment = (name: string, start: string, end: string) => ({ + name, + start, + end, + size: `0x${(BigInt(end) - BigInt(start)).toString(16)}`, + permissions: "r--", +}); + +/** Adjacent .text and .data, a gap, then a segment above 2^53. */ +const SEGMENTS = [ + segment(".text", "0x1000", "0x2000"), + segment(".data", "0x2000", "0x3000"), + segment(".far", "0x5000", "0x6000"), + segment(".huge", "0x20000000000001", "0x20000000000002"), +]; + +/** The same segments in descending order, which a scan may not rely on. */ +const UNSORTED = [...SEGMENTS].reverse(); + +const survey = (segments: JsonValue = SEGMENTS) => ({ + metadata: { path: "/db/target.i64", module: "target", arch: "metapc" }, + statistics: { total_segments: Array.isArray(segments) ? segments.length : 0 }, + segments, + entrypoints: [], +}); + +const function_ = { addr: "0x1000", name: "main", size: "0x20" }; + +const MODERN_WITHOUT_SURVEY = [ + "idb_open", + "idb_list", + "idb_close", + "server_health", + "list_funcs", + "lookup_funcs", + "find_regex", + "decompile", + "disasm", + "callees", + "xrefs_to", +]; + +interface Upstream { + /** Answer for the lookup of the queried address; default is "Not a function". */ + readonly lookup?: (query: string) => JsonValue; + readonly survey?: JsonValue; + readonly tools?: readonly string[]; + readonly surveyFailure?: "error" | "preview"; + /** A distinct source alias of the admitted target, as for an app bundle. */ + readonly sourcePath?: string; + /** Report a different input from the second health read on. */ + readonly identityChangesAfterOperation?: boolean; +} + +const session = async ( + upstream: Upstream = {}, + mode: "headless" | "attached" = "headless", +) => { + const created = await createIdaTarget("x86_64"); + const { root } = created; + const target = + upstream.sourcePath === undefined + ? created.target + : { ...created.target, sourcePath: upstream.sourcePath }; + roots.push(root); + const producer = new RecordingIdaMcp(target, mode); + if (upstream.tools !== undefined) producer.tools = upstream.tools; + producer.beforeCall = async (name) => { + if (name !== "survey_binary") return; + if (upstream.surveyFailure === "error") + decodeIdaToolResult({ + content: [{ type: "text", text: "survey failed" }], + isError: true, + }); + if (upstream.surveyFailure === "preview") + decodeIdaToolResult({ + content: [{ type: "text", text: "preview" }], + _meta: { ida_mcp: { output_truncated: true } }, + }); + }; + let healthReads = 0; + producer.overrideCall = (name, args) => { + if (name === "server_health" && upstream.identityChangesAfterOperation) { + healthReads += 1; + if (healthReads === 2) + return { + status: "ok", + input_path: "/different/target.elf", + idb_path: "/different/target.elf.i64", + module: "target.elf", + imagebase: "0x1000", + auto_analysis_ready: true, + }; + } + if (name === "lookup_funcs") { + const queries = Array.isArray(args.queries) ? args.queries : []; + return queries.map((query) => + upstream.lookup === undefined + ? { query, fn: null, error: "Not a function" } + : upstream.lookup(String(query)), + ); + } + if (name === "survey_binary") return upstream.survey ?? survey(); + return undefined; + }; + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode, + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, producer, target }; +}; + +const containing = async ( + client: IdaSessionClient, + address: string, + options?: Parameters[2], +) => client.execute("resolve_containing_procedure", { address }, options); + +const found = (query: string, fn: typeof function_) => ({ + query, + fn, + error: null, +}); + +describe("IDA containment capabilities", () => { + it("declares both operations for the headless profile only", () => { + const declared = (mode: "attached" | "headless", native = false) => + idaCapabilities(mode, false, native) + .map(({ operation }) => operation) + .filter( + (operation) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ); + + expect( + idaCapabilities("headless", false) + .filter( + ({ operation }) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ) + .map(({ available }) => available), + "both are declared available, not merely listed", + ).toEqual([true, true]); + expect(declared("headless")).toEqual([ + "resolve_containing_procedure", + "list_documents", + ]); + expect(declared("attached")).toEqual([]); + expect(declared("attached", true)).toEqual([]); + expect( + idaCapabilities("attached", false).map(({ operation }) => operation), + "attached profiles do not even declare them", + ).not.toContain("list_documents"); + }); + + it.each(["resolve_containing_procedure", "list_documents"] as const)( + "refuses %s on the legacy attached profile without calling upstream", + async (operation) => { + const { client, producer } = await session({}, "attached"); + try { + const result = await client.execute( + operation, + operation === "list_documents" ? {} : { address: "0x1000" }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( + false, + ); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containing procedure", () => { + it("returns the canonical entry for an interior address without reading segments", async () => { + const { client, producer } = await session({ + lookup: (query) => found(query, function_), + }); + try { + const result = await containing(client, "1ffe"); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: "0x1ffe", + found: true, + procedure: { + address: "0x1000", + name: "main", + classification: null, + body: { + available: false, + reason: "IDA MCP does not report complete function body ranges.", + }, + }, + }); + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "the normalized address is the only query", + ).toMatchObject([{ args: { queries: ["0x1ffe"] } }]); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "a positive answer needs no segment facts", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it.each([ + ["inside the first segment", "0x1000", "not_in_procedure"], + ["inside an adjacent segment", "0x2000", "not_in_procedure"], + [ + "at the exclusive end of the last adjacent segment", + "0x3000", + "outside_segments", + ], + ["in the gap between segments", "0x4000", "outside_segments"], + ["one below a segment start", "0xfff", "outside_segments"], + ["the last byte of a segment", "0x5fff", "not_in_procedure"], + ["above every segment", "0x6000", "outside_segments"], + [ + "inside a one-byte segment above 2^53", + "0x20000000000001", + "not_in_procedure", + ], + [ + "one below that segment, which a rounded number cannot tell apart", + "0x20000000000000", + "outside_segments", + ], + [ + "at the exclusive end of that segment", + "0x20000000000002", + "outside_segments", + ], + ] as const)( + "reports an address that is not a function %s as %s", + async (_description, address, reason) => { + const { client, producer } = await session(); + try { + const result = await containing(client, address); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: address, + found: false, + procedure: null, + reason, + }); + expect( + producer.calls.find(({ name }) => name === "survey_binary")?.args, + "the minimal survey is the only segment source", + ).toMatchObject({ detail_level: "minimal" }); + expect( + producer.calls.filter(({ name }) => name === "survey_binary"), + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containment failures", () => { + it.each([ + ["no segments field", { metadata: {}, statistics: {} }], + ["segments that are null", survey(null)], + ["an empty segment list", survey([])], + [ + "a row with a malformed address", + survey([ + { + name: ".x", + start: "0xzz", + end: "0x2000", + size: "0x1000", + permissions: "r--", + }, + ]), + ], + [ + "a reversed segment", + survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + ], + [ + "a size that disagrees with the bounds", + survey([{ ...segment(".x", "0x1000", "0x2000"), size: "0x10" }]), + ], + [ + "a total that disagrees with the rows", + { ...survey(), statistics: { total_segments: 9 } }, + ], + ["a malformed row after valid ones", survey([...SEGMENTS, { name: ".y" }])], + ] as const)( + "fails instead of guessing a reason when the survey has %s", + async (_description, reply) => { + const { client } = await session({ survey: reply as JsonValue }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisOutputError"); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it.each(["error", "preview"] as const)( + "fails when the survey tool reports %s", + async (surveyFailure) => { + const { client } = await session({ surveyFailure }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it("fails without a segment source when the server does not advertise the survey", async () => { + const { client, producer } = await session({ + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const negative = await containing(client, "0x4000"); + + expect(negative.ok).toBe(false); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "an unadvertised tool is never requested", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("still answers a positive lookup when the server does not advertise the survey", async () => { + const { client } = await session({ + lookup: (query) => found(query, function_), + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const result = await containing(client, "0x1010"); + + expect(result.ok).toBe(true); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA containment lookup failures", () => { + it.each([ + ["an answer for another query", () => found("0x9999", function_)], + [ + "a function together with an error", + (query: string) => ({ query, fn: function_, error: "Not found" }), + ], + [ + "neither a function nor an error", + (query: string) => ({ query, fn: null, error: null }), + ], + [ + "an unknown error text", + (query: string) => ({ query, fn: null, error: "boom" }), + ], + [ + "a padded missing answer", + (query: string) => ({ query, fn: null, error: " Not a function" }), + ], + ] as const)( + "treats %s as a producer failure and reads no segments", + async (_description, lookup) => { + const { client, producer } = await session({ lookup }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisProtocolError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it('treats "Not found" for an address like any expected missing answer', async () => { + const { client } = await session({ + lookup: (query) => ({ query, fn: null, error: "Not found" }), + }); + try { + const result = await containing(client, "0x4000"); + + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + found: false, + reason: "outside_segments", + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops before the survey when the request is cancelled after the lookup", async () => { + const controller = new AbortController(); + const { client, producer } = await session(); + const inner = producer.beforeCall; + producer.beforeCall = async (name, args) => { + await inner?.(name, args); + if (name === "lookup_funcs") controller.abort(); + }; + try { + const result = await containing(client, "0x4000", { + signal: controller.signal, + }); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "nothing is dispatched after cancellation", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("does not depend on the order of the surveyed segments", async () => { + const { client } = await session({ survey: survey(UNSORTED) }); + try { + const inside = await containing(client, "0x1000"); + const gap = await containing(client, "0x4000"); + + if (!inside.ok) throw inside.error; + if (!gap.ok) throw gap.error; + expect(inside.value.result).toMatchObject({ reason: "not_in_procedure" }); + expect(gap.value.result).toMatchObject({ reason: "outside_segments" }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("retains the completed replies when the survey is rejected", async () => { + const { client } = await session({ + survey: survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error.partialObservation).toMatchObject({ + normalized_result: { + status: "partial", + observations: [ + expect.objectContaining({ tool: "lookup_funcs" }), + expect.objectContaining({ tool: "survey_binary" }), + ], + }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("rejects an address that is not hexadecimal and a document that is not the admitted target", async () => { + const { client, producer } = await session(); + try { + const badAddress = await containing(client, "main"); + const badDocument = await client.execute("resolve_containing_procedure", { + address: "0x1000", + document: "/somewhere/else", + }); + + for (const result of [badAddress, badDocument]) { + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "invalid input never reaches upstream", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA document identity", () => { + it("accepts an explicit admitted document and rejects an unknown one for the same operations", async () => { + const { client, target, producer } = await session(); + try { + const admitted = await client.execute("resolve_containing_procedure", { + address: "0x4000", + document: target.path, + }); + const unknownContainment = await client.execute( + "resolve_containing_procedure", + { address: "0x4000", document: "/somewhere/else" }, + ); + const unknownListing = await client.execute("list_documents", { + document: "/somewhere/else", + }); + + expect(admitted.ok, "the admitted program is accepted").toBe(true); + for (const rejected of [unknownContainment, unknownListing]) { + expect(rejected.ok).toBe(false); + if (!rejected.ok) + expect(rejected.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "only the admitted request reached upstream", + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists the admitted path, not its source alias or the private copy", async () => { + const { client, target, producer } = await session({ + sourcePath: "/Applications/Fixture.app", + }); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + expect( + producer.calls.filter(({ name }) => name === "server_health"), + "the target identity is checked before and after the local answer", + ).toHaveLength(2); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("fails when the target identity changes during the operation", async () => { + const { client } = await session({ identityChangesAfterOperation: true }); + try { + const result = await client.execute("list_documents", {}); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.message).toContain("identity changed"); + } + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops without dispatching when the request is already cancelled", async () => { + const controller = new AbortController(); + controller.abort(); + const { client, producer } = await session(); + try { + const result = await client.execute( + "list_documents", + {}, + { signal: controller.signal }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(producer.calls).toEqual([]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists exactly the admitted program", async () => { + const { client, target } = await session(); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts index d59f598dd..5da9d40f7 100644 --- a/src/ida/IdaNativeProtocol.test.ts +++ b/src/ida/IdaNativeProtocol.test.ts @@ -758,3 +758,33 @@ describe("native IDA malformed observations", () => { await client.close(); }); }); + +describe("native IDA attached operation admission", () => { + it("forwards the native producer's explicit reference cap flag", async () => { + const { client } = await fixture(); + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBe(false); + await client.close(); + }); + + it("does not admit the headless-only containment and document operations", async () => { + const { client, producer } = await fixture(); + for (const [operation, parameters] of [ + ["resolve_containing_procedure", { address: "0x1000" }], + ["list_documents", {}], + ] as const) { + const result = await client.execute(operation, parameters); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "nothing is dispatched upstream", + ).toBe(false); + await client.close(); + }); +}); diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index c30ec81bc..25a49c268 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -45,6 +45,7 @@ import { modernStringsSchema, modernXrefsSchema, nextOffset, + surveySegmentsSchema, type IdaDisassembly, type IdaFunction, } from "./IdaProtocolValues.js"; @@ -86,6 +87,8 @@ export class IdaOperationRunner { readonly locations: EvidenceLocation[] = []; readonly limitations: string[] = []; relationshipCoverage: RelationshipCoverage | undefined; + /** The upstream cap flag of the last `xrefs_to` read, when the producer reported one. */ + referencesTruncated: boolean | undefined; constructor( private readonly connection: IdaMcpConnection, private readonly protocol: "legacy" | "modern" | "native", @@ -97,6 +100,10 @@ export class IdaOperationRunner { readonly x86Only?: boolean; /** False when the upstream server does not advertise `callgraph`. */ readonly callGraphAvailable?: boolean; + /** False when the upstream server does not advertise `survey_binary`. */ + readonly surveyAvailable?: boolean; + /** The program identity REA admitted, reported by `list_documents`. */ + readonly admittedDocument?: string; } = {}, ) {} @@ -177,41 +184,89 @@ export class IdaOperationRunner { return entry; } - private lookupFunction( + /** Parse one lookup row: a function, an expected "missing" answer, or a producer failure. */ + private lookupOutcome( query: string, item: z.infer[number] | undefined, - requestedBy?: string, - ): IdaFunction { + ): { readonly fn: IdaFunction } | { readonly missing: string } { if (item?.query !== query) throw new AnalysisProtocolError( `IDA function lookup answered for a different query than ${JSON.stringify(query)}.`, ); if ( - requestedBy !== undefined && item.fn === null && item.error !== null && MISSING_PROCEDURE_ANSWERS.has(item.error) ) - throw new AnalysisInputError( - requestedBy, - { - cause: new AnalysisProtocolError( - `IDA function lookup failed for ${query}: ${item.error}`, - ), - }, - [ - { - path: [], - reason: "invalid_value", - message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${item.error}.`, - }, - ], - ); + return { missing: item.error }; if (item.fn === null || item.error !== null) throw new AnalysisProtocolError( `IDA function lookup failed for ${query}: ${item.error ?? "missing function"}`, ); - return item.fn; + return { fn: item.fn }; + } + + private lookupFunction( + query: string, + item: z.infer[number] | undefined, + requestedBy?: string, + ): IdaFunction { + const outcome = this.lookupOutcome(query, item); + if ("fn" in outcome) return outcome.fn; + if (requestedBy === undefined) + throw new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ); + throw new AnalysisInputError( + requestedBy, + { + cause: new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ), + }, + [ + { + path: [], + reason: "invalid_value", + message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${outcome.missing}.`, + }, + ], + ); + } + + /** The containing function of an address, or why none contains it. */ + private async containingProcedure(address: string): Promise { + const item = modernLookupSchema.parse( + await this.call("lookup_funcs", { queries: [address] }), + )[0]; + const outcome = this.lookupOutcome(address, item); + if ("fn" in outcome) + return { + query_address: address, + found: true, + procedure: identity(outcome.fn), + }; + return { + query_address: address, + found: false, + procedure: null, + reason: (await this.addressInSegment(address)) + ? "not_in_procedure" + : "outside_segments", + }; + } + + /** Segment membership from the database-scoped minimal survey; a missing fact is a failure. */ + private async addressInSegment(address: string): Promise { + if (this.options.surveyAvailable === false) + throw new AnalysisProtocolError( + "The upstream server does not advertise survey_binary, so segment membership cannot be established.", + ); + const { segments } = surveySegmentsSchema.parse( + await this.call("survey_binary", { detail_level: "minimal" }), + ); + const value = BigInt(address); + return segments.some(({ start, end }) => start <= value && value < end); } /** True only for modern callee reads on targets not proven x86-only; x86 reads never reach the graph code. */ @@ -390,6 +445,7 @@ export class IdaOperationRunner { throw new AnalysisProtocolError( `IDA xrefs failed: ${page?.error ?? "missing references"}`, ); + if (page.more !== undefined) this.referencesTruncated = page.more; if (page.more === true) this.limitations.push( "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", @@ -536,6 +592,16 @@ export class IdaOperationRunner { address === idaAddressSchema.parse(parameters.address), ); } + if (operation === "list_documents") { + if (this.options.admittedDocument === undefined) + throw new AnalysisProtocolError("The admitted IDA program is unknown."); + return [this.options.admittedDocument]; + } + if (operation === "resolve_containing_procedure") { + const queried = idaAddressSchema.parse(parameters.address); + this.locations.push({ kind: "address", address: queried }); + return this.containingProcedure(queried); + } if (operation === "xrefs") { const inspectedAddress = idaAddressSchema.parse(parameters.address); this.locations.push({ kind: "address", address: inspectedAddress }); diff --git a/src/ida/IdaProtocolValues.ts b/src/ida/IdaProtocolValues.ts index 93ddbef88..d4afb92db 100644 --- a/src/ida/IdaProtocolValues.ts +++ b/src/ida/IdaProtocolValues.ts @@ -247,3 +247,53 @@ export const absentDatabaseCloseSchema = (sessionId: string) => z .object({ error: z.literal(`Session not found: ${sessionId}`) }) .passthrough(); + +// Piped, so a malformed coordinate is a schema issue and never reaches BigInt. +const hexCoordinate = z + .string() + .regex(/^0x[0-9a-f]+$/iu) + .pipe(z.transform((value) => BigInt(value))); + +/** + * The minimal `survey_binary` reply, reduced to validated segment intervals + * (exclusive ends). Any malformed, reversed or inconsistent fact rejects the + * whole reply, because a membership answer needs every segment. + */ +export const surveySegmentsSchema = z + .object({ + statistics: z + .object({ total_segments: z.number().int().nonnegative().optional() }) + .passthrough() + .optional(), + segments: z + .array( + z + .object({ + start: hexCoordinate, + end: hexCoordinate, + size: hexCoordinate.optional(), + }) + .passthrough(), + ) + .min(1), + }) + .passthrough() + .superRefine(({ statistics, segments }, context) => { + if ( + statistics?.total_segments !== undefined && + statistics.total_segments !== segments.length + ) + context.addIssue({ + code: "custom", + message: "survey segment count disagrees with its statistics", + }); + for (const { start, end, size } of segments) { + // A malformed row has already reported its own issue; refinements still run. + if (typeof start !== "bigint" || typeof end !== "bigint") continue; + if (start >= end || (size !== undefined && size !== end - start)) + context.addIssue({ + code: "custom", + message: "survey segment bounds are reversed or inconsistent", + }); + } + }); diff --git a/src/ida/IdaProviderCapabilities.ts b/src/ida/IdaProviderCapabilities.ts index d52c949e0..0ccc016d2 100644 --- a/src/ida/IdaProviderCapabilities.ts +++ b/src/ida/IdaProviderCapabilities.ts @@ -24,9 +24,23 @@ export const IDA_OPERATIONS = Object.freeze([ "read_function_instructions", "xrefs", "analyze_function", + "resolve_containing_procedure", + "list_documents", ] as const satisfies readonly Exclude[]); export type IdaOperation = (typeof IDA_OPERATIONS)[number]; +/** Operations only the modern headless profile implements. */ +const HEADLESS_ONLY_OPERATIONS: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + +/** Whether the configured profile admits an operation. */ +export const idaProfileAdmits = ( + operation: IdaOperation, + mode: "attached" | "headless", +): boolean => mode === "headless" || !HEADLESS_ONLY_OPERATIONS.has(operation); + /** Shared limitations of observations from an externally mutable analysis database. */ export const IDA_LIMITATIONS = Object.freeze([ "Provider version identifies the REA IDA adapter. IDA engine and upstream distribution versions are unknown unless separately observed; MCP serverInfo is reported independently.", @@ -58,34 +72,37 @@ export const idaCapabilities = ( stdio: boolean, native = false, ): readonly CapabilityDescriptor[] => - IDA_OPERATIONS.map((operation) => ({ - provider: IDA_PROVIDER_IDENTITY, - operation, - ...((mode === "headless" || native) && operation === "procedure_callers" - ? ({ - available: false, - reason: native - ? "The native upstream profile reports code references without proving direct call edges." - : "The modern upstream profile reports code references without proving direct call edges.", - } as const) - : ({ available: true, reason: null } as const)), - cachePolicy: "live", - effects: { - mutatesArtifact: false, - launchesProcess: stdio || mode === "headless", - mayShowUi: - mode === "attached" && - (native - ? operation === "procedure_pseudo_code" || - operation === "analyze_function" - : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), - mayAccessNetwork: true, - mayWriteFilesystem: stdio || mode === "headless", - changesPermissions: false, - requiresRoot: false, - }, - limitations: - native && (operation === "list_strings" || operation === "search_strings") - ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] - : IDA_LIMITATIONS, - })); + IDA_OPERATIONS.filter((operation) => idaProfileAdmits(operation, mode)).map( + (operation) => ({ + provider: IDA_PROVIDER_IDENTITY, + operation, + ...((mode === "headless" || native) && operation === "procedure_callers" + ? ({ + available: false, + reason: native + ? "The native upstream profile reports code references without proving direct call edges." + : "The modern upstream profile reports code references without proving direct call edges.", + } as const) + : ({ available: true, reason: null } as const)), + cachePolicy: "live", + effects: { + mutatesArtifact: false, + launchesProcess: stdio || mode === "headless", + mayShowUi: + mode === "attached" && + (native + ? operation === "procedure_pseudo_code" || + operation === "analyze_function" + : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), + mayAccessNetwork: true, + mayWriteFilesystem: stdio || mode === "headless", + changesPermissions: false, + requiresRoot: false, + }, + limitations: + native && + (operation === "list_strings" || operation === "search_strings") + ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] + : IDA_LIMITATIONS, + }), + ); diff --git a/src/ida/IdaReferenceCoverage.test.ts b/src/ida/IdaReferenceCoverage.test.ts new file mode 100644 index 000000000..930c3a88a --- /dev/null +++ b/src/ida/IdaReferenceCoverage.test.ts @@ -0,0 +1,96 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { + callGraphSession as session, + removeCallGraphSessions, +} from "../../tests/fixtures/idaCallGraph.js"; + +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const attachedRoots: string[] = []; +afterEach(async () => { + await removeCallGraphSessions(); + for (const root of attachedRoots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const xrefs = async (more: boolean | undefined) => { + const { client, producer } = await session({ callees: [] }, "x86_64"); + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [{ xrefs: [], ...(more === undefined ? {} : { more }) }] + : inner?.(name, args); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + if (!result.ok) throw result.error; + return result.value; + } finally { + expect((await client.close()).ok).toBe(true); + } +}; + +describe("headless IDA reference coverage", () => { + it("reports a capped xrefs read as truncated", async () => { + const execution = await xrefs(true); + + expect(execution.referencesTruncated).toBe(true); + expect( + execution.limitations, + "the diagnostic limitation is still reported", + ).toContain( + "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", + ); + }); + + it("reports an explicitly uncapped xrefs read as not truncated", async () => { + expect((await xrefs(false)).referencesTruncated).toBe(false); + }); + + it("reports nothing when the producer does not say whether the read was capped", async () => { + expect((await xrefs(undefined)).referencesTruncated).toBeUndefined(); + }); + + it("does not report reference coverage for other operations", async () => { + const { client } = await session({ callees: [] }, "x86_64"); + try { + const result = await client.execute("procedure_callees", { + procedure: "main", + }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("reports nothing for the legacy profile, whose producer has no cap flag", async () => { + const { root, target } = await createIdaTarget("x86_64"); + attachedRoots.push(root); + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "attached", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + new RecordingIdaMcp(target, "attached"), + ); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaSessionClient.ts b/src/ida/IdaSessionClient.ts index 7d3ec066c..c940e4c6a 100644 --- a/src/ida/IdaSessionClient.ts +++ b/src/ida/IdaSessionClient.ts @@ -33,6 +33,7 @@ import { IDA_LIMITATIONS, IDA_OPERATIONS, IDA_PROVIDER_IDENTITY, + idaProfileAdmits, type IdaOperation, } from "./IdaProviderCapabilities.js"; import { @@ -122,6 +123,7 @@ export class IdaSessionClient implements AnalysisClient { #openAttempted = false; #openCompleted = false; #callGraphAdvertised = false; + #surveyAdvertised = false; #initialIdentity: string | undefined; #metadata: JsonValue = null; #lifecycle: JsonValue[] = []; @@ -194,6 +196,17 @@ export class IdaSessionClient implements AnalysisClient { "This integration admits read-only analysis operations only.", ), ); + if ( + operation !== "health" && + !idaProfileAdmits(operation, this.config.mode) + ) + return err( + new AnalysisCapabilityUnavailableError( + "ida", + operation, + "This operation is available for the modern headless profile only.", + ), + ); if ( operation === "procedure_callers" && idaProtocol(this.config) !== "legacy" @@ -269,6 +282,9 @@ export class IdaSessionClient implements AnalysisClient { ...(runner.relationshipCoverage === undefined ? {} : { relationshipCoverage: runner.relationshipCoverage }), + ...(runner.referencesTruncated !== undefined + ? { referencesTruncated: runner.referencesTruncated } + : {}), locations: [ { kind: "artifact-path", path: this.target.path }, ...runner.locations, @@ -293,6 +309,8 @@ export class IdaSessionClient implements AnalysisClient { selectorOrigin: options?.selectorOrigin, x86Only: isX86OnlyTarget(this.target), callGraphAvailable: this.#callGraphAdvertised, + surveyAvailable: this.#surveyAdvertised, + admittedDocument: this.target.path, }, ); } @@ -423,6 +441,7 @@ export class IdaSessionClient implements AnalysisClient { async #initialize(): Promise { const tools = new Set(await this.connection.connect()); this.#callGraphAdvertised = tools.has("callgraph"); + this.#surveyAdvertised = tools.has("survey_binary"); const expected = this.config.protocol === "native" ? nativeTools diff --git a/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts new file mode 100644 index 000000000..8c94d5de0 --- /dev/null +++ b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts @@ -0,0 +1,249 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import type { AnalysisOperationPort } from "../../../src/application/AnalysisProvider.js"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; +import { EnhancedTools } from "../../../src/application/EnhancedTools.js"; +import { AnalysisOutputError } from "../../../src/domain/analysisErrorCore.js"; +import { err, ok } from "../../../src/domain/result.js"; + +import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; + +afterEach(closeEnhancedToolResources); + +const PROVIDER = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", +} as const; + +type Coverage = boolean | undefined; + +interface Behavior { + /** What the provider reports for the references of each match address. */ + readonly coverage?: Readonly>; + readonly limitations?: readonly string[]; + /** Also report the first string match as a procedure match. */ + readonly duplicateProcedure?: boolean; + /** Fail the references read of this address. */ + readonly failXrefsOf?: string; + readonly calls?: string[]; +} + +const containing = (address: string) => ({ + query_address: address, + found: false, + procedure: null, + reason: "not_in_procedure", +}); + +/** + * A provider with two string matches (0x10 and 0x11), each referenced from + * one address (0x20 and 0x21). + */ +const provider = (behavior: Behavior = {}): AnalysisOperationPort => ({ + execute: async (name, arguments_) => { + const address = String(arguments_.address); + behavior.calls?.push(`${name}:${address}`); + switch (name) { + case "search_strings": + return ok( + createAnalysisExecution( + [ + { address: "0x10", value: "hello" }, + { address: "0x11", value: "hello again" }, + ], + PROVIDER, + ), + ); + case "search_procedures": + return ok( + createAnalysisExecution( + behavior.duplicateProcedure === true + ? [{ address: "0x10", value: "hello" }] + : [], + PROVIDER, + ), + ); + case "xrefs": { + if (address === behavior.failXrefsOf) + return err(new AnalysisOutputError("xrefs", "provider failed")); + const reported = behavior.coverage?.[address]; + return ok( + createAnalysisExecution( + [address === "0x10" ? "0x20" : "0x21"], + PROVIDER, + { + limitations: behavior.limitations ?? [], + ...(reported === undefined + ? {} + : { referencesTruncated: reported }), + }, + ), + ); + } + case "resolve_containing_procedure": + return ok(createAnalysisExecution(containing(address), PROVIDER)); + default: + throw new Error(`Unexpected operation: ${name}`); + } + }, +}); + +const expectedTrace = ( + residual: readonly string[], + extraMatches: readonly object[] = [], +) => ({ + query: "hello", + search_mode: "literal", + matches: [ + { type: "string", address: "0x10", value: "hello" }, + { type: "string", address: "0x11", value: "hello again" }, + ...extraMatches, + ], + references: [ + { + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + }, + { + target_address: "0x11", + source_address: "0x21", + containing_procedure: containing("0x21"), + }, + ...extraMatches.map(() => ({ + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + })), + ], + truncated: residual.length > 0, + residual_unknowns: residual, +}); + +const unknownFor = (address: string) => + `References to ${address} were truncated by the provider; additional references may exist.`; + +const trace = async (analysis: AnalysisOperationPort) => { + const client = await connect(analysis); + const result = await client.callTool({ + name: "trace_feature", + arguments: { query: "hello" }, + }); + expect(result.isError).not.toBe(true); + return result.structuredContent as { normalized_result: unknown }; +}; + +describe("trace_feature reference coverage", () => { + it("marks the trace truncated for the target whose references the provider capped", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect( + output.normalized_result, + "the received references are kept and only the capped target is a residual unknown", + ).toEqual(expectedTrace([unknownFor("0x10")])); + }); + + it("does not carry a truncation over to the next target that reports nothing", async () => { + const output = await trace(provider({ coverage: { "0x10": true } })); + + expect(output.normalized_result).toEqual( + expectedTrace([unknownFor("0x10")]), + ); + }); + + it("reports a capped target once when it is matched twice", async () => { + const output = await trace( + provider({ coverage: { "0x10": true }, duplicateProcedure: true }), + ); + + expect(output.normalized_result).toEqual( + expectedTrace( + [unknownFor("0x10")], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("resolves the containing procedure of a repeated reference source once", async () => { + const calls: string[] = []; + const output = await trace(provider({ duplicateProcedure: true, calls })); + + expect( + calls.filter((call) => call.startsWith("resolve_containing_procedure:")), + "the source 0x20 is referenced by two matches of the same address and resolved once", + ).toEqual([ + "resolve_containing_procedure:0x20", + "resolve_containing_procedure:0x21", + ]); + expect( + output.normalized_result, + "every reference is still reported with its containing procedure", + ).toEqual( + expectedTrace( + [], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("keeps the trace complete when the provider explicitly reports no cap", async () => { + const output = await trace( + provider({ coverage: { "0x10": false, "0x11": false } }), + ); + + expect(output.normalized_result).toEqual(expectedTrace([])); + }); + + it("does not change the trace of a provider that reports no coverage", async () => { + const output = await trace( + provider({ + limitations: ["Upstream xrefs_to capped results at 1000 references."], + }), + ); + + expect( + output.normalized_result, + "a limitation text alone never truncates, and an absent field keeps the previous result", + ).toEqual(expectedTrace([])); + }); + + it("never exposes the coverage field in the tool output", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect(JSON.stringify(output)).not.toMatch( + /referencesTruncated|references_truncated/, + ); + }); + + it("fails the trace when a references read fails and reads nothing further", async () => { + const calls: string[] = []; + const result = await new EnhancedTools( + provider({ failXrefsOf: "0x10", calls }), + ).execute("trace_feature", { query: "hello" }); + + expect(result.ok).toBe(false); + expect( + calls.filter((call) => call.startsWith("xrefs:")), + "the next target is not read after the failure", + ).toEqual(["xrefs:0x10"]); + }); + + it("stops without reading references when the trace is cancelled", async () => { + const calls: string[] = []; + const controller = new AbortController(); + controller.abort(); + const result = await new EnhancedTools(provider({ calls })).execute( + "trace_feature", + { query: "hello" }, + controller.signal, + ); + + expect(result.ok).toBe(false); + expect(calls.filter((call) => call.startsWith("xrefs:"))).toEqual([]); + }); +}); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index c3dc21542..01d8eba13 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -196,9 +196,9 @@ itWithCaptureCapability( result.value.frames.map(({ at_ms }) => at_ms), "capture-relative frame times stay ordered and nonnegative", ).toEqual( - [...result.value.frames.map(({ at_ms }) => at_ms)].sort( - (left, right) => left - right, - ), + result.value.frames + .map(({ at_ms }) => at_ms) + .sort((left, right) => left - right), ); expect(result.value.frames.every(({ at_ms }) => at_ms >= 0)).toBe(true); expect(result.value.exit.reason).toBe("exited"); diff --git a/tests/boundary/providers/ida/containment.test.ts b/tests/boundary/providers/ida/containment.test.ts new file mode 100644 index 000000000..75eb78f42 --- /dev/null +++ b/tests/boundary/providers/ida/containment.test.ts @@ -0,0 +1,144 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectAddressContext } from "../../../../src/application/AnalysisContextQueries.js"; +import { EnhancedTools } from "../../../../src/application/EnhancedTools.js"; +import { IdaSessionClient } from "../../../../src/ida/IdaSessionClient.js"; +import { createIdaTarget, RecordingIdaMcp } from "../../../fixtures/idaMcp.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const session = async (cappedReferences = false) => { + const { root, target } = await createIdaTarget("arm64"); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + if (cappedReferences) { + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [ + { + xrefs: [ + { + addr: "0x1004", + type: "code", + fn: { addr: "0x1000", name: "main", size: "0x20" }, + }, + ], + more: true, + }, + ] + : inner?.(name, args); + } + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "headless", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, target }; +}; + +const MAIN = { + found: true, + procedure: { address: "0x1000", name: "main" }, +}; + +describe("trace_feature on headless IDA", () => { + it("attributes a reference to its containing procedure", async () => { + const { client } = await session(); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + matches: [{ type: "string", address: "0x3000" }], + references: [ + { + target_address: "0x3000", + source_address: "0x1004", + containing_procedure: MAIN, + }, + ], + truncated: false, + residual_unknowns: [], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("keeps an upstream reference cap visible once containment makes the trace run", async () => { + const { client } = await session(true); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + references: [{ source_address: "0x1004", containing_procedure: MAIN }], + truncated: true, + residual_unknowns: [ + "References to 0x3000 were truncated by the provider; additional references may exist.", + ], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("inspect_address_context on headless IDA", () => { + it("resolves the admitted program as the document and reports the containing procedure", async () => { + const { client, target } = await session(); + try { + const result = await inspectAddressContext(client, { + address: "0x1004", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + address: "0x1004", + document: target.path, + procedure: { state: "available", value: MAIN }, + name: { state: "unavailable" }, + comment: { state: "unavailable" }, + inline_comment: { state: "unavailable" }, + bookmarks: { state: "unavailable" }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("answers the same way for an explicit admitted document", async () => { + const { client, target } = await session(); + try { + const implicit = await inspectAddressContext(client, { + address: "0x1004", + }); + const explicit = await inspectAddressContext(client, { + address: "0x1004", + document: target.path, + }); + + if (!implicit.ok) throw implicit.error; + if (!explicit.ok) throw explicit.error; + expect(explicit.value).toEqual(implicit.value); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/tests/boundary/providers/ida/verifierFeatureTrace.test.ts b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts new file mode 100644 index 000000000..77f654ffc --- /dev/null +++ b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { + assertFeatureTrace, + findMappedNonFunction, +} from "../../../../scripts/lib/ida-feature-trace.mjs"; + +const procedure = { address: "0x1000", name: "main" }; + +const containing = (found: boolean) => + found + ? { + query_address: "0x1004", + found: true, + procedure: { address: "0x1000", name: "main" }, + } + : { + query_address: "0x3010", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + +const trace = ( + matches: readonly { type: string; address: string }[], + references: readonly { + target_address: string; + containing_procedure: unknown; + }[], +) => ({ + query: "main", + search_mode: "literal", + matches, + references, + truncated: false, + residual_unknowns: [], +}); + +const SELECTED = { type: "procedure", address: "0x1000" }; +const SUBSTRING = { type: "procedure", address: "0x2000" }; +const STRING = { type: "string", address: "0x3000" }; + +describe("the real IDA lane's feature trace assertions", () => { + it("accepts references to every matched address, whoever contains the source", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED, SUBSTRING, STRING], + [ + { + target_address: "0x1000", + containing_procedure: containing(true), + }, + // A substring match such as main_helper, referenced from code. + { + target_address: "0x2000", + containing_procedure: containing(true), + }, + // A string containing the query, referenced from data outside functions. + { + target_address: "0x3000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("accepts a reference to the selected procedure from data outside functions", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("rejects a reference whose target is not one of the matches", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x9999", + containing_procedure: containing(true), + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace without the selected procedure's match", () => { + expect(() => assertFeatureTrace(trace([STRING], []), procedure)).toThrow(); + }); + + it("rejects a containment answer that is neither found nor a known reason", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: { + query_address: "0x1004", + found: false, + procedure: null, + reason: "unknown", + }, + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace that does not say whether it was truncated", () => { + expect(() => + assertFeatureTrace( + { ...trace([SELECTED], []), truncated: undefined }, + procedure, + ), + ).toThrow(); + }); +}); + +describe("the real IDA lane's mapped non-function probe", () => { + const inFunction = { + query_address: "0x1100", + found: true, + procedure: { address: "0x1000", name: "main" }, + }; + const data = { + query_address: "0x3000", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + const resolver = + (answers: Readonly>, asked: string[] = []) => + async (address: string) => { + asked.push(address); + return answers[address]; + }; + + it("skips a string inside a function chunk and uses the next one", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x3000" }], + resolver({ "0x1100": inFunction, "0x3000": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked, "strings are tried in order until one qualifies").toEqual([ + "0x1100", + "0x3000", + ]); + }); + + it("stops at the first string that qualifies", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x3000" }, { address: "0x3100" }], + resolver({ "0x3000": data, "0x3100": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked).toEqual(["0x3000"]); + }); + + it("reports no qualifying string when every string is inside a function", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x1200" }], + resolver({ "0x1100": inFunction, "0x1200": inFunction }), + ), + ).resolves.toBeUndefined(); + }); + + it("reports no qualifying string when the target has no strings", async () => { + await expect( + findMappedNonFunction([], resolver({})), + ).resolves.toBeUndefined(); + }); + + it("rejects a listed string that resolves as unmapped, since IDA cannot list an unmapped address", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x4000" }, { address: "0x3000" }], + resolver({ + "0x4000": { ...data, reason: "outside_segments" }, + "0x3000": data, + }), + ), + ).rejects.toThrow(); + }); +}); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index 813b90004..092f869e2 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -1278,3 +1278,102 @@ describe("selector origin forwarding", () => { ).toEqual(["provider", "provider", undefined]); }); }); + +describe("reference truncation forwarding", () => { + const referenceProvider = ( + referencesTruncated: boolean | undefined, + calls: string[], + ): AnalysisProvider => { + const identity = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + } as const; + return { + identity: () => identity, + resolveAnalysisProfile: () => + Promise.resolve( + resultOk({ + profile: createAnalysisProfile(identity, { fixture: true }), + }), + ), + capabilities: () => [ + { + provider: identity, + operation: "xrefs", + available: true, + reason: null, + effects: { + mutatesArtifact: false, + launchesProcess: false, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: false, + changesPermissions: false, + requiresRoot: false, + }, + limitations: [], + }, + ], + createClient: () => ({ + execute: (operation) => { + calls.push(operation); + return Promise.resolve( + resultOk( + createAnalysisExecution( + ["0x2000"], + identity, + referencesTruncated === undefined + ? {} + : { referencesTruncated }, + ), + ), + ); + }, + close: () => Promise.resolve(resultOk(null)), + }), + }; + }; + const parameters = { address: "0x1000", document: "fixture" }; + + it("forwards a truncation on both execution paths and never replays it from the snapshot", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(true, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const direct = await session.execute("xrefs", parameters); + const admitted = await session.withAdmittedAnalysis( + "trace_feature", + undefined, + (analysis) => analysis.execute("xrefs", parameters), + ); + + expect(direct.ok && direct.value.referencesTruncated).toBe(true); + if (!admitted.ok) throw admitted.error; + expect(admitted.value.ok && admitted.value.value.referencesTruncated).toBe( + true, + ); + expect( + calls, + "a truncated read is not recorded, so every call reaches the provider", + ).toEqual(["xrefs", "xrefs"]); + }); + + it("keeps an explicit complete report out of the way of snapshot replay", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(false, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const live = await session.execute("xrefs", parameters); + await session.execute("xrefs", parameters); + + expect(live.ok && live.value.referencesTruncated).toBe(false); + expect(calls, "the complete read is recorded and replayed").toEqual([ + "xrefs", + ]); + }); +}); diff --git a/tests/conformance/ida/providerContracts.test.ts b/tests/conformance/ida/providerContracts.test.ts index 39c524637..4e4b04c3b 100644 --- a/tests/conformance/ida/providerContracts.test.ts +++ b/tests/conformance/ida/providerContracts.test.ts @@ -29,15 +29,25 @@ const parametersFor = ( operation: string, ): Readonly> => { if (operation.startsWith("search_")) return { pattern: "a.b" }; - if (operation === "xrefs") return { address: "0x1000" }; + if (operation === "xrefs" || operation === "resolve_containing_procedure") + return { address: "0x1000" }; if (operation.startsWith("list_")) return {}; return { procedure: "main" }; }; +/** Operations only the headless profile declares. */ +const HEADLESS_ONLY: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + describe.each(["attached", "headless"] as const)( "IDA %s caller-visible contracts", (mode) => { - it.each(IDA_OPERATIONS)( + const operations = IDA_OPERATIONS.filter( + (operation) => mode === "headless" || !HEADLESS_ONLY.has(operation), + ); + it.each(operations)( "declares %s UI effects matching the upstream tools it reaches", async (operation) => { const { root, target } = await createIdaTarget(); @@ -69,7 +79,7 @@ describe.each(["attached", "headless"] as const)( }, ); - it.each(IDA_OPERATIONS)( + it.each(operations)( "serializes %s against its named advertised output contract", async (operation) => { const { root, target } = await createIdaTarget(); diff --git a/tests/fixtures/idaMcp.ts b/tests/fixtures/idaMcp.ts index 04bdb5f16..21e5e38f7 100644 --- a/tests/fixtures/idaMcp.ts +++ b/tests/fixtures/idaMcp.ts @@ -57,6 +57,7 @@ const modernNames = [ "callees", "xrefs_to", "callgraph", + "survey_binary", ]; /** Stateful producer fixture at the real connection seam; no module mocking. */ From 2a61deafc632079d10c572bc09e99233826516a3 Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 13 Oct 2026 08:56:41 -0400 Subject: [PATCH 25/28] fix(adb): report the device state when a push cannot probe the path (#2354) A push without overwrite read any unclassified failure of its existence probe as an existing path, so an offline or unauthorized device was reported as "the device path already exists". With overwrite it pushed after a probe that had not established anything. The probe's own refusal is now returned in both cases. Co-authored-by: Claude Opus 5.5 --- src/adb/AdbProvider.test.ts | 30 ++++++++++++++++++++++++++++++ src/adb/AdbProvider.ts | 18 ++++++++---------- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index ae81353c7..23645e456 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -810,6 +810,36 @@ exit 1`, expect(result.error.message).toContain("overwrite"); }); +it.each([ + ["offline", false, "error: device offline"], + ["unauthorized", false, "error: device unauthorized"], + ["offline", true, "error: device offline"], +] as const)( + "reports the %s device state when the push existence probe cannot run (overwrite %s)", + async (state, overwrite, stderr) => { + const { binary, root } = await writeAdbStub( + (stubRoot) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "-s" ]; then printf '%s\\n' "$3" >> "${stubRoot}/calls"; printf '%s\\n' "${stderr}" >&2; exit 1; fi`, + ); + const localFile = join(root, "payload.bin"); + await writeFile(localFile, "x"); + const projected = await refusedBy(providerOver(binary), { + operation: "push_adb_file", + input: { + serial: "emulator-5554", + local_path: localFile, + device_path: "/data/local/tmp/payload.bin", + overwrite, + }, + }); + // Existence is unknown, so nothing is claimed about the path and no push runs. + expect(projected).toMatchObject({ code: "unsupported_target" }); + expect(projected.message).toContain(state); + expect(projected.message).not.toContain("already exists"); + expect(await readFile(join(root, "calls"), "utf8")).toBe("shell\n"); + }, +); + it("reports a local push source that does not exist", async () => { const { binary } = await writeAdbStub( () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index b9af6d14f..d26345eba 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -1623,17 +1623,15 @@ export class AdbProvider { { timeoutMs: SETTINGS_TIMEOUT_MS, maxOutputBytes: 1024 }, signal, ); - let deviceExists: boolean; - if (existsProbe.ok) deviceExists = true; - else if (existsProbe.error instanceof AnalysisProtocolError) - deviceExists = false; - else if (existsProbe.error instanceof AnalysisTimeoutError) + // `test -e` exits nonzero with no adb diagnosis when the path is absent. + // Any other probe failure leaves existence unknown, so it is reported as + // is rather than read as an existing path or followed by a push. + if ( + !existsProbe.ok && + !(existsProbe.error instanceof AnalysisProtocolError) + ) return existsProbe; - else if (existsProbe.error instanceof AnalysisCancelledError) - return existsProbe; - else if (existsProbe.error instanceof AnalysisResourceConstraintError) - return existsProbe; - else deviceExists = true; // authorization/refusal states surface on push instead + const deviceExists = existsProbe.ok; if (deviceExists && !input.overwrite) return err( new AnalysisOutputError( From 4ea24f97670bebf8ca053179ca0dc5009e6f6a28 Mon Sep 17 00:00:00 2001 From: jhuang-tw <77732008+jhuang-tw@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:58:51 +0800 Subject: [PATCH 26/28] fix(ida): retain private input after ambiguous open (#1845) fix(ida): retain private input after unverified open --- docs/ida-provider.md | 2 + src/ida/IdaProtocolValues.ts | 8 +++ src/ida/IdaSessionClient.test.ts | 98 ++++++++++++++++++++++++++------ src/ida/IdaSessionClient.ts | 13 +++++ 4 files changed, 103 insertions(+), 18 deletions(-) diff --git a/docs/ida-provider.md b/docs/ida-provider.md index 50040425b..fbf67c8a2 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -88,6 +88,8 @@ REA copies the admitted binary into a private temporary directory, checks its di One REA session owns one database. Stdio registrations default `IDA_MCP_MAX_WORKERS` to `1`; the explicit upstream `--max-workers 1` also bounds a supervisor. Database size and engine memory depend on the target and upstream; REA does not impose an engine memory limit. The original binary is never executed or overwritten. Optional `workspaceRoot` selects an existing absolute temporary-directory parent on the REA host. Ensure it has space for the input copy and IDA working files. +When a completed `idb_open` response reports success or is otherwise ambiguous, but no active owned private worker can be verified, REA retains the private workspace even if `idb_list` appears empty. A later close can retry verification if the original owned worker becomes discoverable. A completed explicit `idb_open` rejection retains the existing no-worker cleanup path. This is a local admission/ownership safeguard and **does not** provide a process-termination receipt or change the legacy reaped-worker protocol. + Headless cleanup uses the upstream database lifecycle. Abruptly terminating REA, its proxy, or the supervisor does not establish worker release. Prefer `close_binary` for MCP sessions; after an interrupted process, inspect upstream session inventory and retained workspace diagnostics before removing working files. This adapter does not claim Windows Job Object containment for the upstream detached workers. ## Connect an agent once diff --git a/src/ida/IdaProtocolValues.ts b/src/ida/IdaProtocolValues.ts index d4afb92db..f18e85ee0 100644 --- a/src/ida/IdaProtocolValues.ts +++ b/src/ida/IdaProtocolValues.ts @@ -218,6 +218,14 @@ export const openDatabaseSchema = z .passthrough(), }) .passthrough(); +/** Only an explicit completed rejection preserves legacy no-worker cleanup. */ +export const rejectedOpenDatabaseSchema = z + .object({ + error: z.string().min(1), + success: z.never().optional(), + }) + .passthrough(); + export const databaseListSchema = z .object({ sessions: z.array( diff --git a/src/ida/IdaSessionClient.test.ts b/src/ida/IdaSessionClient.test.ts index e43f18cf7..b41a3c34a 100644 --- a/src/ida/IdaSessionClient.test.ts +++ b/src/ida/IdaSessionClient.test.ts @@ -515,7 +515,7 @@ describe("headless IDA transient busy health", () => { }); it.each([ - ["free-form text", "token=abc123 /private/home/user secret"], + ["free-form text", "token=abc123 /private secret"], ["an over-long name", "x".repeat(200)], ])( "reports %s from the worker as unknown instead of echoing it", @@ -899,6 +899,85 @@ describe("headless IDA reaped worker cleanup", () => { }); }); +/** A completed open response is not proof that its private Worker was ever released. */ +describe("headless IDA unresolved successful-open ownership", () => { + it.each(["matching", "foreign-input", "malformed"] as const)( + "retains the private input after %s success with incomplete initial discovery", + async (kind) => { + const { producer, client, root } = await fixture("headless"); + let hidden = true; + let ownedId: string | undefined; + let privateInput: string | undefined; + producer.beforeCall = async (name, args) => { + if (name !== "idb_open") return; + ownedId = z.string().parse(args.preferred_session_id); + privateInput = z.string().parse(args.input_path); + producer.database = ownedId; + producer.inputPath = privateInput; + }; + producer.overrideCall = (name) => { + if (name === "idb_open") { + const originalId = z.string().parse(ownedId); + const originalInput = z.string().parse(privateInput); + if (kind === "foreign-input") + return { + success: true, + session: { + session_id: "foreign-session", + input_path: "/foreign/input", + }, + }; + if (kind === "malformed") + return { success: true, session: { session_id: originalId } }; + return { + success: true, + session: { session_id: originalId, input_path: originalInput }, + }; + } + if (name === "idb_list" && hidden) return { sessions: [] }; + return undefined; + }; + + expect((await client.execute("health", {})).ok).toBe(false); + const incomplete = await client.close(); + expect(incomplete.ok).toBe(false); + if (!incomplete.ok) expect(incomplete.error.cleanupIncomplete).toBe(true); + expect(producer.closes).toBe(0); + expect(producer.database).toBe(ownedId); + expect( + (await readdir(root)).some((name) => name.startsWith("rea-ida-")), + ).toBe(true); + expect(producer.calls.some(({ name }) => name === "idb_close")).toBe( + false, + ); + + // A later positive private ownership record permits ordinary save=false + // cleanup, without ever adopting the foreign open-response identity. + hidden = false; + expect((await client.close()).ok).toBe(true); + expect(producer.calls.filter(({ name }) => name === "idb_close")).toEqual( + [{ name: "idb_close", args: { database: ownedId, save: false } }], + ); + expect(producer.closes).toBe(1); + expect(await readdir(root)).toEqual(["target.elf"]); + }, + ); + + it("preserves cleanup after a completed, explicit idb_open rejection", async () => { + const { producer, client, root } = await fixture("headless"); + producer.overrideCall = (name) => + name === "idb_open" + ? { error: "Input was rejected before creating a session" } + : undefined; + + expect((await client.execute("health", {})).ok).toBe(false); + expect((await client.close()).ok).toBe(true); + expect(producer.calls.some(({ name }) => name === "idb_close")).toBe(false); + expect(producer.closes).toBe(1); + expect(await readdir(root)).toEqual(["target.elf"]); + }); +}); + describe("headless IDA cleanup verification", () => { it("retries failed worker release through the retained connection before removing its workspace", async () => { const { producer, client, root } = await fixture("headless"); @@ -943,23 +1022,6 @@ describe("headless IDA cleanup verification", () => { ).toHaveLength(1); expect(await readdir(root)).toEqual(["target.elf"]); }); - it("never closes an unrelated database when startup returns a different input identity", async () => { - const { producer, client, root } = await fixture("headless"); - producer.overrideCall = (name) => - name === "idb_open" - ? { - success: true, - session: { - session_id: "user-database", - input_path: "/unrelated/input", - }, - } - : undefined; - expect((await client.execute("health", {})).ok).toBe(false); - expect((await client.close()).ok).toBe(true); - expect(producer.calls.some(({ name }) => name === "idb_close")).toBe(false); - expect(await readdir(root)).toEqual(["target.elf"]); - }); it("releases a privately owned worker after malformed open output using verified session inventory", async () => { const { producer, client, root } = await fixture("headless"); producer.beforeCall = async (name, args) => { diff --git a/src/ida/IdaSessionClient.ts b/src/ida/IdaSessionClient.ts index c940e4c6a..14b20b107 100644 --- a/src/ida/IdaSessionClient.ts +++ b/src/ida/IdaSessionClient.ts @@ -43,6 +43,7 @@ import { legacyMetadataSchema, modernHealthSchema, openDatabaseSchema, + rejectedOpenDatabaseSchema, } from "./IdaProtocolValues.js"; import { idaFileDigest, @@ -122,6 +123,7 @@ export class IdaSessionClient implements AnalysisClient { #databaseCloseCompleted = false; #openAttempted = false; #openCompleted = false; + #openRejected = false; #callGraphAdvertised = false; #surveyAdvertised = false; #initialIdentity: string | undefined; @@ -469,6 +471,9 @@ export class IdaSessionClient implements AnalysisClient { preferred_session_id: this.#workspace.sessionId, }); this.#openCompleted = true; + // A completed but malformed/positive open may still own a live Worker. + // Only an explicit producer rejection keeps the old no-worker branch. + this.#openRejected = rejectedOpenDatabaseSchema.safeParse(raw).success; this.#lifecycle.push(raw); const opened = openDatabaseSchema.parse(raw); if ( @@ -633,6 +638,14 @@ export class IdaSessionClient implements AnalysisClient { throw new AnalysisProtocolError( "An unowned worker still uses the private workspace; it was retained.", ); + if ( + this.#database === undefined && + this.#openCompleted && + !this.#openRejected + ) + throw new AnalysisProtocolError( + "IDA open returned without a verified private worker binding. An empty session inventory does not prove that its Worker exited; the private workspace was retained.", + ); } if (this.#database !== undefined) { if (!this.#databaseCloseCompleted) { From d2e16b2a4998818be5f29219467b97cc15ae8061 Mon Sep 17 00:00:00 2001 From: yasshmittal Date: Tue, 13 Oct 2026 18:33:14 +0530 Subject: [PATCH 28/28] fix(sqlite): attribute unknown column selections to columns (#2774) Every worker "selection" rejection was projected onto the table argument, including an exact column name that does not exist in an existing table. Give column rejections their own worker reason and map each input reason to the caller argument it requires correcting. Table rejections keep the table path. Fixes #2613 Co-authored-by: Claude Opus 5.5 --- src/sqlite/SqliteDatabaseInspection.ts | 2 +- src/sqlite/SqliteDatabaseLimits.ts | 1 + src/sqlite/SqliteDatabaseWorkerExecution.ts | 11 ++++- .../sqlite/sqliteDatabaseProvider.test.ts | 44 +++++++++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index f5f92bdf4..510fbaafc 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -394,7 +394,7 @@ const readRows = ( for (const name of columns) if (!available.has(name)) throw new SqliteInspectionFailure( - "selection", + "column-selection", `Selected column does not exist with that exact name in table ${input.table}: ${name}`, ); const projection = columns diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 99df7ed9d..408fc4bc3 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -12,6 +12,7 @@ export class SqliteInspectionFailure extends Error { readonly reason: | "format" | "selection" + | "column-selection" | "unsupported" | "unavailable" | "output-limit" diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index 9474a6d27..cba180399 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -36,6 +36,7 @@ const replySchema = z.discriminatedUnion("ok", [ reason: z.enum([ "format", "selection", + "column-selection", "unsupported", "unavailable", "output-limit", @@ -49,6 +50,14 @@ export type SqliteWorkerLauncher = NonNullable< Parameters[2] >["launcher"]; +// Each input rejection names the caller argument that requires correction. +const rejectedArgument = { + format: "path", + unsupported: "path", + selection: "table", + "column-selection": "columns", +} as const; + const rejectedReply = ( reply: z.output & { ok: false }, capturedOutput: AnalysisCapturedOutput, @@ -83,7 +92,7 @@ const rejectedReply = ( ); return new AnalysisInputError(operation, { capturedOutput }, [ { - path: [reply.reason === "selection" ? "table" : "path"], + path: [rejectedArgument[reply.reason]], reason: "invalid_format", message: reply.message, }, diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index 24dadc128..b5adf8297 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -5,6 +5,7 @@ import { DatabaseSync } from "node:sqlite"; import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; import { expect, it as nativeTest, onTestFinished } from "vitest"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; @@ -356,6 +357,49 @@ it("does not execute views when selecting rows", async () => { }); }); +it("attributes table and column selection failures to their own arguments", async () => { + const { database } = await createFixture(); + const provider = createProvider(); + onTestFinished(() => provider.close()); + const projection = async (selection: { + table: string; + columns?: string[]; + }) => { + const result = await provider.inspect({ + path: database.path, + row_limit: 1, + ...selection, + }); + if (result.ok) throw new Error("Expected a rejected selection"); + return projectAnalysisError(result.error); + }; + expect( + await projection({ table: "records", columns: ["nope"] }), + ).toMatchObject({ + details: { + issues: [ + { + path: ["columns"], + reason: "invalid_format", + message: + "Selected column does not exist with that exact name in table records: nope", + }, + ], + }, + }); + expect(await projection({ table: "absent" })).toMatchObject({ + details: { issues: [{ path: ["table"] }] }, + }); + const selected = await provider.inspect({ + path: database.path, + table: "records", + columns: ["id"], + row_limit: 1, + }); + if (!selected.ok) throw selected.error; + expect(selected.value.result).toMatchObject({ rows: { columns: ["id"] } }); +}); + it("rejects malformed databases without altering the selected file", async () => { const workspace = await createTestWorkspace("rea-sqlite-invalid-"); onTestFinished(() => removeTestWorkspace(workspace.root));