diff --git a/docs/ida-provider.md b/docs/ida-provider.md index b22a35cce..50040425b 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -126,6 +126,8 @@ For guided registration, set `REA_IDA_MCP_CONFIG` before `rea setup --client Promise, +): Promise; diff --git a/scripts/lib/ida-feature-trace.mjs b/scripts/lib/ida-feature-trace.mjs new file mode 100644 index 000000000..3c4778d25 --- /dev/null +++ b/scripts/lib/ida-feature-trace.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; + +const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]); + +/** + * Assert the `trace_feature` result of the real IDA lane for one selected + * procedure. The query is a substring search, so the trace may also match other + * procedures and strings; each reference is checked against its own matched + * target, and a source outside every function (a data reference) is valid. + */ +export function assertFeatureTrace(trace, procedure) { + assert.ok( + trace.matches.some( + ({ type, address }) => + type === "procedure" && address === procedure.address, + ), + "The trace omitted the selected procedure's match.", + ); + assert.equal(typeof trace.truncated, "boolean"); + const matched = new Set(trace.matches.map(({ address }) => address)); + for (const reference of trace.references) { + assert.ok( + matched.has(reference.target_address), + `A reference targets ${reference.target_address}, which is not a match.`, + ); + const { containing_procedure: containing } = reference; + if (containing.found) + assert.equal(typeof containing.procedure.address, "string"); + else assert.ok(NEGATIVE_REASONS.has(containing.reason)); + } +} + +/** + * Find a string that IDA maps outside every function. A string inside a + * function chunk (inline data in code) is valid and skipped, so the strings are + * tried in order. Any other answer must be "not in a procedure": IDA lists only + * mapped strings, so an unmapped answer is a containment defect and fails the + * lane. Returns the address, or undefined when every string is inside a function. + */ +export async function findMappedNonFunction(strings, resolveAt) { + for (const { address } of strings) { + const resolved = await resolveAt(address); + if (resolved.found) continue; + assert.equal( + resolved.reason, + "not_in_procedure", + `The listed string at ${address} resolved as ${resolved.reason}.`, + ); + return address; + } + return undefined; +} diff --git a/scripts/verify-real-ida.mjs b/scripts/verify-real-ida.mjs index cedbf5076..bed0519a9 100644 --- a/scripts/verify-real-ida.mjs +++ b/scripts/verify-real-ida.mjs @@ -1,4 +1,8 @@ #!/usr/bin/env node +import { + assertFeatureTrace, + findMappedNonFunction, +} from "./lib/ida-feature-trace.mjs"; import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; @@ -6,7 +10,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { access, writeFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs, promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -261,6 +265,61 @@ const mcp = async (cliDossier) => { break; } } + if (config.mode === "headless") { + const documents = observe(await call("list_documents", {})); + assert.equal(documents.length, 1); + assert.equal(basename(documents[0]), basename(target)); + assert.ok( + !workspaces.has(dirname(documents[0])), + "list_documents reported the private workspace copy.", + ); + const resolveAt = async (address) => + observe(await call("resolve_containing_procedure", { address })); + const instructionAddresses = dossier.assembly + .map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1]) + .filter((address) => address !== undefined); + const queries = [ + dossier.procedure.address, + ...instructionAddresses.slice(1, 2), + ]; + for (const queried of queries) { + const containing = await resolveAt(queried); + assert.equal(containing.found, true); + assert.equal(containing.procedure.address, dossier.procedure.address); + assert.equal(containing.procedure.name, dossier.procedure.name); + } + const unmapped = await resolveAt("0xffffffffffffff00"); + assert.equal(unmapped.found, false); + assert.equal(unmapped.reason, "outside_segments"); + const strings = observe(await call("list_strings", {})); + const mappedString = await findMappedNonFunction(strings, resolveAt); + if (mappedString === undefined) + report.unverified.push( + "Mapped non-function containment: no string of the target is mapped outside every function.", + ); + const context = await client.callTool({ + name: "inspect_address_context", + arguments: { address: dossier.procedure.address }, + }); + assert.notEqual(context.isError, true); + const contextResult = context.structuredContent.result; + assert.equal(contextResult.document, documents[0]); + assert.equal(contextResult.procedure.state, "available"); + assert.equal( + contextResult.procedure.value.procedure.address, + dossier.procedure.address, + ); + const traced = await client.callTool({ + name: "trace_feature", + arguments: { query: dossier.procedure.name }, + }); + assert.notEqual(traced.isError, true); + const trace = traced.structuredContent.normalized_result; + assertFeatureTrace(trace, dossier.procedure); + report.checks.push( + "headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing", + ); + } if (values["expected-string"] === undefined) report.unverified.push( "String inventory/search correctness: no independent expected literal supplied.", diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index f3b51257a..376036fa1 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -50,6 +50,13 @@ export interface AnalysisExecution { readonly limitations: readonly string[]; /** Provider-owned completeness assessment for caller/callee observations. */ readonly relationshipCoverage?: RelationshipCoverage; + /** + * Whether the provider capped an `xrefs` read: true when it reported a + * truncation, false when it explicitly reported none. Absent when it reports + * nothing, which is not a completeness claim. Never part of Evidence or tool + * output. + */ + readonly referencesTruncated?: boolean; readonly locations: readonly EvidenceLocation[]; readonly subject: EvidenceSubjectTarget | null; } @@ -59,6 +66,7 @@ interface CreateAnalysisExecutionOptions { readonly analysisProfile?: AnalysisProfileCommitment; readonly limitations?: readonly string[]; readonly relationshipCoverage?: RelationshipCoverage; + readonly referencesTruncated?: boolean; readonly locations?: readonly EvidenceLocation[]; readonly subject?: EvidenceSubjectTarget; } @@ -103,6 +111,9 @@ export function createAnalysisExecution( options.relationshipCoverage, ), }), + ...(options.referencesTruncated === undefined + ? {} + : { referencesTruncated: options.referencesTruncated }), locations: [...(options.locations ?? [])], subject: options.subject ?? null, }; diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 1b8ec6411..4286ff1d0 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -497,3 +497,71 @@ it("reports host denial before target mismatch but keeps composed workflow prere ).reason, ).toBe("target_required"); }); + +describe("trace_feature requirements", () => { + const capabilities = (operations: readonly string[]) => + operations.map((operation) => ({ + operation, + available: true, + reason: null, + })); + + it("needs the search operations the workflow actually calls", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: true, reason: "available" }); + }); + + it.each(["search_strings", "search_procedures"])( + "is unavailable without %s", + (missing) => { + const operations = [ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ].filter((operation) => operation !== missing); + + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities(operations), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }, + ); + + it("is unavailable when only the list operations exist", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "list_strings", + "list_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }); +}); diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 6fefb8e20..4e364a1aa 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -121,8 +121,8 @@ const ENHANCED_REQUIREMENTS: Readonly> = { analyze_function: ["analyze_function"], inspect_native_api: ["analyze_function"], trace_feature: [ - "list_strings", - "list_procedures", + "search_strings", + "search_procedures", "xrefs", "resolve_containing_procedure", ], diff --git a/src/application/EnhancedLiteralTracing.ts b/src/application/EnhancedLiteralTracing.ts index 9c86781bc..79684ce0f 100644 --- a/src/application/EnhancedLiteralTracing.ts +++ b/src/application/EnhancedLiteralTracing.ts @@ -1,5 +1,8 @@ import { z } from "zod"; -import type { AnalysisOperation } from "./AnalysisProvider.js"; +import type { + AnalysisExecution, + AnalysisOperation, +} from "./AnalysisProvider.js"; import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -16,6 +19,12 @@ type AnalysisCall = ( signal?: AbortSignal, ) => Promise>; +type ObserveCall = ( + name: AnalysisOperation, + arguments_: Readonly>, + signal?: AbortSignal, +) => Promise>; + type LiteralSearchSource = readonly [ "search_strings" | "search_procedures", "string" | "procedure", @@ -39,6 +48,7 @@ export interface LiteralTraceInput { /** Trace matching strings and procedures through provider xrefs. */ export const traceLiteralFeature = async ( call: AnalysisCall, + observe: ObserveCall, input: LiteralTraceInput, signal?: AbortSignal, ): EnhancedResult => { @@ -49,7 +59,7 @@ export const traceLiteralFeature = async ( signal, ); if (!searched.ok) return searched; - const traced = await traceReferences(call, { + const traced = await traceReferences(call, observe, { matches: searched.value.matches, ...(signal === undefined ? {} : { signal }), }); @@ -59,8 +69,8 @@ export const traceLiteralFeature = async ( search_mode: "literal", matches: searched.value.matches, references: traced.value.references, - truncated: false, - residual_unknowns: [], + truncated: traced.value.residual.length > 0, + residual_unknowns: traced.value.residual, }); }; @@ -126,24 +136,39 @@ const addPageMatches = (input: { const traceReferences = async ( call: AnalysisCall, + observe: ObserveCall, request: ReferenceTraceRequest, -): Promise> => { +): Promise< + Result<{ references: TraceReference[]; residual: string[] }, AnalysisError> +> => { const references: TraceReference[] = []; + const residual: string[] = []; + // A source address is resolved once however many matches reference it. + const containing = new Map(); for (const match of request.matches) { - const xrefs = await call( + const observed = await observe( "xrefs", { address: match.address }, request.signal, ); - if (!xrefs.ok) return xrefs; - if (!Array.isArray(xrefs.value)) + if (!observed.ok) return observed; + // Only an explicit producer report of truncation is a coverage gap; a + // provider that reports nothing keeps its previous, unqualified result. + const unknown = `References to ${match.address} were truncated by the provider; additional references may exist.`; + if ( + observed.value.referencesTruncated === true && + !residual.includes(unknown) + ) + residual.push(unknown); + const sources = observed.value.result; + if (!Array.isArray(sources)) return err( new AnalysisOutputError( "xrefs", "provider returned a non-array result", ), ); - for (const source of xrefs.value) { + for (const source of sources) { if (typeof source !== "string") return err( new AnalysisOutputError( @@ -151,18 +176,23 @@ const traceReferences = async ( "provider returned a non-address value", ), ); - const resolved = await call( - "resolve_containing_procedure", - { address: source }, - request.signal, - ); - if (!resolved.ok) return resolved; + let containingProcedure = containing.get(source); + if (containingProcedure === undefined) { + const resolved = await call( + "resolve_containing_procedure", + { address: source }, + request.signal, + ); + if (!resolved.ok) return resolved; + containingProcedure = resolved.value; + containing.set(source, containingProcedure); + } references.push({ target_address: match.address, source_address: source, - containing_procedure: resolved.value, + containing_procedure: containingProcedure, }); } } - return ok({ references }); + return ok({ references, residual }); }; diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index 87d27fc5f..207186ccc 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -163,6 +163,8 @@ export class EnhancedTools { return traceLiteralFeature( (name, arguments_, operationSignal) => this.#call(name, arguments_, operationSignal), + (name, arguments_, operationSignal) => + this.#observe(name, arguments_, operationSignal), call.input, signal, ); diff --git a/src/application/InvestigationProviders.ts b/src/application/InvestigationProviders.ts index 5dcb18150..af23463f4 100644 --- a/src/application/InvestigationProviders.ts +++ b/src/application/InvestigationProviders.ts @@ -20,7 +20,9 @@ export const workflowAnalysisProfile = ( upstream_analysis_profile: upstream, ...(operation === "binary_overview" ? { workflow_revision: 3 } - : operation === "batch_decompile" || operation === "get_call_graph" + : operation === "batch_decompile" || + operation === "get_call_graph" || + operation === "trace_feature" ? { workflow_revision: 2 } : {}), }); diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 48c2a54bd..91b9f8807 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -661,3 +661,52 @@ it("does not replay pre-envelope call graphs under the revised workflow profile" }).success, ).toBe(false); }); + +describe("executions that report truncated references", () => { + const record = (referencesTruncated: boolean | undefined) => { + const cache = new AnalysisSnapshotCache(); + const parameters = { address: "0x1000", document: "fixture" }; + cache.record({ + target: ANALYSIS_SNAPSHOT_TARGET, + profile: ANALYSIS_SNAPSHOT_PROFILE, + operation: "xrefs", + parameters, + execution: createAnalysisExecution( + ["0x2000"], + ANALYSIS_SNAPSHOT_PROVIDER, + { + subject: ANALYSIS_SNAPSHOT_TARGET, + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + ...(referencesTruncated === undefined ? {} : { referencesTruncated }), + }, + ), + }); + return cache.lookup( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROFILE, + "xrefs", + parameters, + ); + }; + + it("are not replayed, because a replay would lose the truncation", () => { + expect(record(true)).toBeUndefined(); + }); + + it("do not stop an execution that is complete or silent from being replayed", () => { + expect(record(false)?.result).toEqual(["0x2000"]); + expect(record(undefined)?.result).toEqual(["0x2000"]); + }); +}); + +it("revises the trace_feature workflow profile so earlier results are not replayed", () => { + const profile = ANALYSIS_SNAPSHOT_PROFILE; + + expect(workflowAnalysisProfile(profile, "trace_feature")).not.toEqual( + workflowAnalysisProfile(profile), + ); + expect( + workflowAnalysisProfile(profile, "trace_call_path"), + "other workflows keep their profile", + ).toEqual(workflowAnalysisProfile(profile)); +}); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index bdaa9f68d..7e307baa0 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -337,6 +337,9 @@ export class AnalysisSnapshotCache { }): void { const { target, profile, operation, parameters, execution } = input; this.select(target, profile); + // The snapshot format does not carry reference truncation; a replay would + // present a capped read as complete, so it is not recorded. + if (execution.referencesTruncated === true) return; const queryId = analysisQueryId( snapshotTarget(target), snapshotBinding(profile), diff --git a/src/ida/IdaContainment.test.ts b/src/ida/IdaContainment.test.ts new file mode 100644 index 000000000..069388df8 --- /dev/null +++ b/src/ida/IdaContainment.test.ts @@ -0,0 +1,623 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { decodeIdaToolResult } from "./IdaMcpConnection.js"; +import { idaCapabilities } from "./IdaProviderCapabilities.js"; +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const segment = (name: string, start: string, end: string) => ({ + name, + start, + end, + size: `0x${(BigInt(end) - BigInt(start)).toString(16)}`, + permissions: "r--", +}); + +/** Adjacent .text and .data, a gap, then a segment above 2^53. */ +const SEGMENTS = [ + segment(".text", "0x1000", "0x2000"), + segment(".data", "0x2000", "0x3000"), + segment(".far", "0x5000", "0x6000"), + segment(".huge", "0x20000000000001", "0x20000000000002"), +]; + +/** The same segments in descending order, which a scan may not rely on. */ +const UNSORTED = [...SEGMENTS].reverse(); + +const survey = (segments: JsonValue = SEGMENTS) => ({ + metadata: { path: "/db/target.i64", module: "target", arch: "metapc" }, + statistics: { total_segments: Array.isArray(segments) ? segments.length : 0 }, + segments, + entrypoints: [], +}); + +const function_ = { addr: "0x1000", name: "main", size: "0x20" }; + +const MODERN_WITHOUT_SURVEY = [ + "idb_open", + "idb_list", + "idb_close", + "server_health", + "list_funcs", + "lookup_funcs", + "find_regex", + "decompile", + "disasm", + "callees", + "xrefs_to", +]; + +interface Upstream { + /** Answer for the lookup of the queried address; default is "Not a function". */ + readonly lookup?: (query: string) => JsonValue; + readonly survey?: JsonValue; + readonly tools?: readonly string[]; + readonly surveyFailure?: "error" | "preview"; + /** A distinct source alias of the admitted target, as for an app bundle. */ + readonly sourcePath?: string; + /** Report a different input from the second health read on. */ + readonly identityChangesAfterOperation?: boolean; +} + +const session = async ( + upstream: Upstream = {}, + mode: "headless" | "attached" = "headless", +) => { + const created = await createIdaTarget("x86_64"); + const { root } = created; + const target = + upstream.sourcePath === undefined + ? created.target + : { ...created.target, sourcePath: upstream.sourcePath }; + roots.push(root); + const producer = new RecordingIdaMcp(target, mode); + if (upstream.tools !== undefined) producer.tools = upstream.tools; + producer.beforeCall = async (name) => { + if (name !== "survey_binary") return; + if (upstream.surveyFailure === "error") + decodeIdaToolResult({ + content: [{ type: "text", text: "survey failed" }], + isError: true, + }); + if (upstream.surveyFailure === "preview") + decodeIdaToolResult({ + content: [{ type: "text", text: "preview" }], + _meta: { ida_mcp: { output_truncated: true } }, + }); + }; + let healthReads = 0; + producer.overrideCall = (name, args) => { + if (name === "server_health" && upstream.identityChangesAfterOperation) { + healthReads += 1; + if (healthReads === 2) + return { + status: "ok", + input_path: "/different/target.elf", + idb_path: "/different/target.elf.i64", + module: "target.elf", + imagebase: "0x1000", + auto_analysis_ready: true, + }; + } + if (name === "lookup_funcs") { + const queries = Array.isArray(args.queries) ? args.queries : []; + return queries.map((query) => + upstream.lookup === undefined + ? { query, fn: null, error: "Not a function" } + : upstream.lookup(String(query)), + ); + } + if (name === "survey_binary") return upstream.survey ?? survey(); + return undefined; + }; + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode, + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, producer, target }; +}; + +const containing = async ( + client: IdaSessionClient, + address: string, + options?: Parameters[2], +) => client.execute("resolve_containing_procedure", { address }, options); + +const found = (query: string, fn: typeof function_) => ({ + query, + fn, + error: null, +}); + +describe("IDA containment capabilities", () => { + it("declares both operations for the headless profile only", () => { + const declared = (mode: "attached" | "headless", native = false) => + idaCapabilities(mode, false, native) + .map(({ operation }) => operation) + .filter( + (operation) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ); + + expect( + idaCapabilities("headless", false) + .filter( + ({ operation }) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ) + .map(({ available }) => available), + "both are declared available, not merely listed", + ).toEqual([true, true]); + expect(declared("headless")).toEqual([ + "resolve_containing_procedure", + "list_documents", + ]); + expect(declared("attached")).toEqual([]); + expect(declared("attached", true)).toEqual([]); + expect( + idaCapabilities("attached", false).map(({ operation }) => operation), + "attached profiles do not even declare them", + ).not.toContain("list_documents"); + }); + + it.each(["resolve_containing_procedure", "list_documents"] as const)( + "refuses %s on the legacy attached profile without calling upstream", + async (operation) => { + const { client, producer } = await session({}, "attached"); + try { + const result = await client.execute( + operation, + operation === "list_documents" ? {} : { address: "0x1000" }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( + false, + ); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containing procedure", () => { + it("returns the canonical entry for an interior address without reading segments", async () => { + const { client, producer } = await session({ + lookup: (query) => found(query, function_), + }); + try { + const result = await containing(client, "1ffe"); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: "0x1ffe", + found: true, + procedure: { + address: "0x1000", + name: "main", + classification: null, + body: { + available: false, + reason: "IDA MCP does not report complete function body ranges.", + }, + }, + }); + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "the normalized address is the only query", + ).toMatchObject([{ args: { queries: ["0x1ffe"] } }]); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "a positive answer needs no segment facts", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it.each([ + ["inside the first segment", "0x1000", "not_in_procedure"], + ["inside an adjacent segment", "0x2000", "not_in_procedure"], + [ + "at the exclusive end of the last adjacent segment", + "0x3000", + "outside_segments", + ], + ["in the gap between segments", "0x4000", "outside_segments"], + ["one below a segment start", "0xfff", "outside_segments"], + ["the last byte of a segment", "0x5fff", "not_in_procedure"], + ["above every segment", "0x6000", "outside_segments"], + [ + "inside a one-byte segment above 2^53", + "0x20000000000001", + "not_in_procedure", + ], + [ + "one below that segment, which a rounded number cannot tell apart", + "0x20000000000000", + "outside_segments", + ], + [ + "at the exclusive end of that segment", + "0x20000000000002", + "outside_segments", + ], + ] as const)( + "reports an address that is not a function %s as %s", + async (_description, address, reason) => { + const { client, producer } = await session(); + try { + const result = await containing(client, address); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: address, + found: false, + procedure: null, + reason, + }); + expect( + producer.calls.find(({ name }) => name === "survey_binary")?.args, + "the minimal survey is the only segment source", + ).toMatchObject({ detail_level: "minimal" }); + expect( + producer.calls.filter(({ name }) => name === "survey_binary"), + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containment failures", () => { + it.each([ + ["no segments field", { metadata: {}, statistics: {} }], + ["segments that are null", survey(null)], + ["an empty segment list", survey([])], + [ + "a row with a malformed address", + survey([ + { + name: ".x", + start: "0xzz", + end: "0x2000", + size: "0x1000", + permissions: "r--", + }, + ]), + ], + [ + "a reversed segment", + survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + ], + [ + "a size that disagrees with the bounds", + survey([{ ...segment(".x", "0x1000", "0x2000"), size: "0x10" }]), + ], + [ + "a total that disagrees with the rows", + { ...survey(), statistics: { total_segments: 9 } }, + ], + ["a malformed row after valid ones", survey([...SEGMENTS, { name: ".y" }])], + ] as const)( + "fails instead of guessing a reason when the survey has %s", + async (_description, reply) => { + const { client } = await session({ survey: reply as JsonValue }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisOutputError"); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it.each(["error", "preview"] as const)( + "fails when the survey tool reports %s", + async (surveyFailure) => { + const { client } = await session({ surveyFailure }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it("fails without a segment source when the server does not advertise the survey", async () => { + const { client, producer } = await session({ + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const negative = await containing(client, "0x4000"); + + expect(negative.ok).toBe(false); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "an unadvertised tool is never requested", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("still answers a positive lookup when the server does not advertise the survey", async () => { + const { client } = await session({ + lookup: (query) => found(query, function_), + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const result = await containing(client, "0x1010"); + + expect(result.ok).toBe(true); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA containment lookup failures", () => { + it.each([ + ["an answer for another query", () => found("0x9999", function_)], + [ + "a function together with an error", + (query: string) => ({ query, fn: function_, error: "Not found" }), + ], + [ + "neither a function nor an error", + (query: string) => ({ query, fn: null, error: null }), + ], + [ + "an unknown error text", + (query: string) => ({ query, fn: null, error: "boom" }), + ], + [ + "a padded missing answer", + (query: string) => ({ query, fn: null, error: " Not a function" }), + ], + ] as const)( + "treats %s as a producer failure and reads no segments", + async (_description, lookup) => { + const { client, producer } = await session({ lookup }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisProtocolError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it('treats "Not found" for an address like any expected missing answer', async () => { + const { client } = await session({ + lookup: (query) => ({ query, fn: null, error: "Not found" }), + }); + try { + const result = await containing(client, "0x4000"); + + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + found: false, + reason: "outside_segments", + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops before the survey when the request is cancelled after the lookup", async () => { + const controller = new AbortController(); + const { client, producer } = await session(); + const inner = producer.beforeCall; + producer.beforeCall = async (name, args) => { + await inner?.(name, args); + if (name === "lookup_funcs") controller.abort(); + }; + try { + const result = await containing(client, "0x4000", { + signal: controller.signal, + }); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "nothing is dispatched after cancellation", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("does not depend on the order of the surveyed segments", async () => { + const { client } = await session({ survey: survey(UNSORTED) }); + try { + const inside = await containing(client, "0x1000"); + const gap = await containing(client, "0x4000"); + + if (!inside.ok) throw inside.error; + if (!gap.ok) throw gap.error; + expect(inside.value.result).toMatchObject({ reason: "not_in_procedure" }); + expect(gap.value.result).toMatchObject({ reason: "outside_segments" }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("retains the completed replies when the survey is rejected", async () => { + const { client } = await session({ + survey: survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error.partialObservation).toMatchObject({ + normalized_result: { + status: "partial", + observations: [ + expect.objectContaining({ tool: "lookup_funcs" }), + expect.objectContaining({ tool: "survey_binary" }), + ], + }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("rejects an address that is not hexadecimal and a document that is not the admitted target", async () => { + const { client, producer } = await session(); + try { + const badAddress = await containing(client, "main"); + const badDocument = await client.execute("resolve_containing_procedure", { + address: "0x1000", + document: "/somewhere/else", + }); + + for (const result of [badAddress, badDocument]) { + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "invalid input never reaches upstream", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA document identity", () => { + it("accepts an explicit admitted document and rejects an unknown one for the same operations", async () => { + const { client, target, producer } = await session(); + try { + const admitted = await client.execute("resolve_containing_procedure", { + address: "0x4000", + document: target.path, + }); + const unknownContainment = await client.execute( + "resolve_containing_procedure", + { address: "0x4000", document: "/somewhere/else" }, + ); + const unknownListing = await client.execute("list_documents", { + document: "/somewhere/else", + }); + + expect(admitted.ok, "the admitted program is accepted").toBe(true); + for (const rejected of [unknownContainment, unknownListing]) { + expect(rejected.ok).toBe(false); + if (!rejected.ok) + expect(rejected.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "only the admitted request reached upstream", + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists the admitted path, not its source alias or the private copy", async () => { + const { client, target, producer } = await session({ + sourcePath: "/Applications/Fixture.app", + }); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + expect( + producer.calls.filter(({ name }) => name === "server_health"), + "the target identity is checked before and after the local answer", + ).toHaveLength(2); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("fails when the target identity changes during the operation", async () => { + const { client } = await session({ identityChangesAfterOperation: true }); + try { + const result = await client.execute("list_documents", {}); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.message).toContain("identity changed"); + } + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops without dispatching when the request is already cancelled", async () => { + const controller = new AbortController(); + controller.abort(); + const { client, producer } = await session(); + try { + const result = await client.execute( + "list_documents", + {}, + { signal: controller.signal }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(producer.calls).toEqual([]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists exactly the admitted program", async () => { + const { client, target } = await session(); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts index d59f598dd..5da9d40f7 100644 --- a/src/ida/IdaNativeProtocol.test.ts +++ b/src/ida/IdaNativeProtocol.test.ts @@ -758,3 +758,33 @@ describe("native IDA malformed observations", () => { await client.close(); }); }); + +describe("native IDA attached operation admission", () => { + it("forwards the native producer's explicit reference cap flag", async () => { + const { client } = await fixture(); + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBe(false); + await client.close(); + }); + + it("does not admit the headless-only containment and document operations", async () => { + const { client, producer } = await fixture(); + for (const [operation, parameters] of [ + ["resolve_containing_procedure", { address: "0x1000" }], + ["list_documents", {}], + ] as const) { + const result = await client.execute(operation, parameters); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "nothing is dispatched upstream", + ).toBe(false); + await client.close(); + }); +}); diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index c30ec81bc..25a49c268 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -45,6 +45,7 @@ import { modernStringsSchema, modernXrefsSchema, nextOffset, + surveySegmentsSchema, type IdaDisassembly, type IdaFunction, } from "./IdaProtocolValues.js"; @@ -86,6 +87,8 @@ export class IdaOperationRunner { readonly locations: EvidenceLocation[] = []; readonly limitations: string[] = []; relationshipCoverage: RelationshipCoverage | undefined; + /** The upstream cap flag of the last `xrefs_to` read, when the producer reported one. */ + referencesTruncated: boolean | undefined; constructor( private readonly connection: IdaMcpConnection, private readonly protocol: "legacy" | "modern" | "native", @@ -97,6 +100,10 @@ export class IdaOperationRunner { readonly x86Only?: boolean; /** False when the upstream server does not advertise `callgraph`. */ readonly callGraphAvailable?: boolean; + /** False when the upstream server does not advertise `survey_binary`. */ + readonly surveyAvailable?: boolean; + /** The program identity REA admitted, reported by `list_documents`. */ + readonly admittedDocument?: string; } = {}, ) {} @@ -177,41 +184,89 @@ export class IdaOperationRunner { return entry; } - private lookupFunction( + /** Parse one lookup row: a function, an expected "missing" answer, or a producer failure. */ + private lookupOutcome( query: string, item: z.infer[number] | undefined, - requestedBy?: string, - ): IdaFunction { + ): { readonly fn: IdaFunction } | { readonly missing: string } { if (item?.query !== query) throw new AnalysisProtocolError( `IDA function lookup answered for a different query than ${JSON.stringify(query)}.`, ); if ( - requestedBy !== undefined && item.fn === null && item.error !== null && MISSING_PROCEDURE_ANSWERS.has(item.error) ) - throw new AnalysisInputError( - requestedBy, - { - cause: new AnalysisProtocolError( - `IDA function lookup failed for ${query}: ${item.error}`, - ), - }, - [ - { - path: [], - reason: "invalid_value", - message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${item.error}.`, - }, - ], - ); + return { missing: item.error }; if (item.fn === null || item.error !== null) throw new AnalysisProtocolError( `IDA function lookup failed for ${query}: ${item.error ?? "missing function"}`, ); - return item.fn; + return { fn: item.fn }; + } + + private lookupFunction( + query: string, + item: z.infer[number] | undefined, + requestedBy?: string, + ): IdaFunction { + const outcome = this.lookupOutcome(query, item); + if ("fn" in outcome) return outcome.fn; + if (requestedBy === undefined) + throw new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ); + throw new AnalysisInputError( + requestedBy, + { + cause: new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ), + }, + [ + { + path: [], + reason: "invalid_value", + message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${outcome.missing}.`, + }, + ], + ); + } + + /** The containing function of an address, or why none contains it. */ + private async containingProcedure(address: string): Promise { + const item = modernLookupSchema.parse( + await this.call("lookup_funcs", { queries: [address] }), + )[0]; + const outcome = this.lookupOutcome(address, item); + if ("fn" in outcome) + return { + query_address: address, + found: true, + procedure: identity(outcome.fn), + }; + return { + query_address: address, + found: false, + procedure: null, + reason: (await this.addressInSegment(address)) + ? "not_in_procedure" + : "outside_segments", + }; + } + + /** Segment membership from the database-scoped minimal survey; a missing fact is a failure. */ + private async addressInSegment(address: string): Promise { + if (this.options.surveyAvailable === false) + throw new AnalysisProtocolError( + "The upstream server does not advertise survey_binary, so segment membership cannot be established.", + ); + const { segments } = surveySegmentsSchema.parse( + await this.call("survey_binary", { detail_level: "minimal" }), + ); + const value = BigInt(address); + return segments.some(({ start, end }) => start <= value && value < end); } /** True only for modern callee reads on targets not proven x86-only; x86 reads never reach the graph code. */ @@ -390,6 +445,7 @@ export class IdaOperationRunner { throw new AnalysisProtocolError( `IDA xrefs failed: ${page?.error ?? "missing references"}`, ); + if (page.more !== undefined) this.referencesTruncated = page.more; if (page.more === true) this.limitations.push( "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", @@ -536,6 +592,16 @@ export class IdaOperationRunner { address === idaAddressSchema.parse(parameters.address), ); } + if (operation === "list_documents") { + if (this.options.admittedDocument === undefined) + throw new AnalysisProtocolError("The admitted IDA program is unknown."); + return [this.options.admittedDocument]; + } + if (operation === "resolve_containing_procedure") { + const queried = idaAddressSchema.parse(parameters.address); + this.locations.push({ kind: "address", address: queried }); + return this.containingProcedure(queried); + } if (operation === "xrefs") { const inspectedAddress = idaAddressSchema.parse(parameters.address); this.locations.push({ kind: "address", address: inspectedAddress }); diff --git a/src/ida/IdaProtocolValues.ts b/src/ida/IdaProtocolValues.ts index 93ddbef88..d4afb92db 100644 --- a/src/ida/IdaProtocolValues.ts +++ b/src/ida/IdaProtocolValues.ts @@ -247,3 +247,53 @@ export const absentDatabaseCloseSchema = (sessionId: string) => z .object({ error: z.literal(`Session not found: ${sessionId}`) }) .passthrough(); + +// Piped, so a malformed coordinate is a schema issue and never reaches BigInt. +const hexCoordinate = z + .string() + .regex(/^0x[0-9a-f]+$/iu) + .pipe(z.transform((value) => BigInt(value))); + +/** + * The minimal `survey_binary` reply, reduced to validated segment intervals + * (exclusive ends). Any malformed, reversed or inconsistent fact rejects the + * whole reply, because a membership answer needs every segment. + */ +export const surveySegmentsSchema = z + .object({ + statistics: z + .object({ total_segments: z.number().int().nonnegative().optional() }) + .passthrough() + .optional(), + segments: z + .array( + z + .object({ + start: hexCoordinate, + end: hexCoordinate, + size: hexCoordinate.optional(), + }) + .passthrough(), + ) + .min(1), + }) + .passthrough() + .superRefine(({ statistics, segments }, context) => { + if ( + statistics?.total_segments !== undefined && + statistics.total_segments !== segments.length + ) + context.addIssue({ + code: "custom", + message: "survey segment count disagrees with its statistics", + }); + for (const { start, end, size } of segments) { + // A malformed row has already reported its own issue; refinements still run. + if (typeof start !== "bigint" || typeof end !== "bigint") continue; + if (start >= end || (size !== undefined && size !== end - start)) + context.addIssue({ + code: "custom", + message: "survey segment bounds are reversed or inconsistent", + }); + } + }); diff --git a/src/ida/IdaProviderCapabilities.ts b/src/ida/IdaProviderCapabilities.ts index d52c949e0..0ccc016d2 100644 --- a/src/ida/IdaProviderCapabilities.ts +++ b/src/ida/IdaProviderCapabilities.ts @@ -24,9 +24,23 @@ export const IDA_OPERATIONS = Object.freeze([ "read_function_instructions", "xrefs", "analyze_function", + "resolve_containing_procedure", + "list_documents", ] as const satisfies readonly Exclude[]); export type IdaOperation = (typeof IDA_OPERATIONS)[number]; +/** Operations only the modern headless profile implements. */ +const HEADLESS_ONLY_OPERATIONS: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + +/** Whether the configured profile admits an operation. */ +export const idaProfileAdmits = ( + operation: IdaOperation, + mode: "attached" | "headless", +): boolean => mode === "headless" || !HEADLESS_ONLY_OPERATIONS.has(operation); + /** Shared limitations of observations from an externally mutable analysis database. */ export const IDA_LIMITATIONS = Object.freeze([ "Provider version identifies the REA IDA adapter. IDA engine and upstream distribution versions are unknown unless separately observed; MCP serverInfo is reported independently.", @@ -58,34 +72,37 @@ export const idaCapabilities = ( stdio: boolean, native = false, ): readonly CapabilityDescriptor[] => - IDA_OPERATIONS.map((operation) => ({ - provider: IDA_PROVIDER_IDENTITY, - operation, - ...((mode === "headless" || native) && operation === "procedure_callers" - ? ({ - available: false, - reason: native - ? "The native upstream profile reports code references without proving direct call edges." - : "The modern upstream profile reports code references without proving direct call edges.", - } as const) - : ({ available: true, reason: null } as const)), - cachePolicy: "live", - effects: { - mutatesArtifact: false, - launchesProcess: stdio || mode === "headless", - mayShowUi: - mode === "attached" && - (native - ? operation === "procedure_pseudo_code" || - operation === "analyze_function" - : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), - mayAccessNetwork: true, - mayWriteFilesystem: stdio || mode === "headless", - changesPermissions: false, - requiresRoot: false, - }, - limitations: - native && (operation === "list_strings" || operation === "search_strings") - ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] - : IDA_LIMITATIONS, - })); + IDA_OPERATIONS.filter((operation) => idaProfileAdmits(operation, mode)).map( + (operation) => ({ + provider: IDA_PROVIDER_IDENTITY, + operation, + ...((mode === "headless" || native) && operation === "procedure_callers" + ? ({ + available: false, + reason: native + ? "The native upstream profile reports code references without proving direct call edges." + : "The modern upstream profile reports code references without proving direct call edges.", + } as const) + : ({ available: true, reason: null } as const)), + cachePolicy: "live", + effects: { + mutatesArtifact: false, + launchesProcess: stdio || mode === "headless", + mayShowUi: + mode === "attached" && + (native + ? operation === "procedure_pseudo_code" || + operation === "analyze_function" + : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), + mayAccessNetwork: true, + mayWriteFilesystem: stdio || mode === "headless", + changesPermissions: false, + requiresRoot: false, + }, + limitations: + native && + (operation === "list_strings" || operation === "search_strings") + ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] + : IDA_LIMITATIONS, + }), + ); diff --git a/src/ida/IdaReferenceCoverage.test.ts b/src/ida/IdaReferenceCoverage.test.ts new file mode 100644 index 000000000..930c3a88a --- /dev/null +++ b/src/ida/IdaReferenceCoverage.test.ts @@ -0,0 +1,96 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { + callGraphSession as session, + removeCallGraphSessions, +} from "../../tests/fixtures/idaCallGraph.js"; + +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const attachedRoots: string[] = []; +afterEach(async () => { + await removeCallGraphSessions(); + for (const root of attachedRoots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const xrefs = async (more: boolean | undefined) => { + const { client, producer } = await session({ callees: [] }, "x86_64"); + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [{ xrefs: [], ...(more === undefined ? {} : { more }) }] + : inner?.(name, args); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + if (!result.ok) throw result.error; + return result.value; + } finally { + expect((await client.close()).ok).toBe(true); + } +}; + +describe("headless IDA reference coverage", () => { + it("reports a capped xrefs read as truncated", async () => { + const execution = await xrefs(true); + + expect(execution.referencesTruncated).toBe(true); + expect( + execution.limitations, + "the diagnostic limitation is still reported", + ).toContain( + "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", + ); + }); + + it("reports an explicitly uncapped xrefs read as not truncated", async () => { + expect((await xrefs(false)).referencesTruncated).toBe(false); + }); + + it("reports nothing when the producer does not say whether the read was capped", async () => { + expect((await xrefs(undefined)).referencesTruncated).toBeUndefined(); + }); + + it("does not report reference coverage for other operations", async () => { + const { client } = await session({ callees: [] }, "x86_64"); + try { + const result = await client.execute("procedure_callees", { + procedure: "main", + }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("reports nothing for the legacy profile, whose producer has no cap flag", async () => { + const { root, target } = await createIdaTarget("x86_64"); + attachedRoots.push(root); + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "attached", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + new RecordingIdaMcp(target, "attached"), + ); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaSessionClient.ts b/src/ida/IdaSessionClient.ts index 7d3ec066c..c940e4c6a 100644 --- a/src/ida/IdaSessionClient.ts +++ b/src/ida/IdaSessionClient.ts @@ -33,6 +33,7 @@ import { IDA_LIMITATIONS, IDA_OPERATIONS, IDA_PROVIDER_IDENTITY, + idaProfileAdmits, type IdaOperation, } from "./IdaProviderCapabilities.js"; import { @@ -122,6 +123,7 @@ export class IdaSessionClient implements AnalysisClient { #openAttempted = false; #openCompleted = false; #callGraphAdvertised = false; + #surveyAdvertised = false; #initialIdentity: string | undefined; #metadata: JsonValue = null; #lifecycle: JsonValue[] = []; @@ -194,6 +196,17 @@ export class IdaSessionClient implements AnalysisClient { "This integration admits read-only analysis operations only.", ), ); + if ( + operation !== "health" && + !idaProfileAdmits(operation, this.config.mode) + ) + return err( + new AnalysisCapabilityUnavailableError( + "ida", + operation, + "This operation is available for the modern headless profile only.", + ), + ); if ( operation === "procedure_callers" && idaProtocol(this.config) !== "legacy" @@ -269,6 +282,9 @@ export class IdaSessionClient implements AnalysisClient { ...(runner.relationshipCoverage === undefined ? {} : { relationshipCoverage: runner.relationshipCoverage }), + ...(runner.referencesTruncated !== undefined + ? { referencesTruncated: runner.referencesTruncated } + : {}), locations: [ { kind: "artifact-path", path: this.target.path }, ...runner.locations, @@ -293,6 +309,8 @@ export class IdaSessionClient implements AnalysisClient { selectorOrigin: options?.selectorOrigin, x86Only: isX86OnlyTarget(this.target), callGraphAvailable: this.#callGraphAdvertised, + surveyAvailable: this.#surveyAdvertised, + admittedDocument: this.target.path, }, ); } @@ -423,6 +441,7 @@ export class IdaSessionClient implements AnalysisClient { async #initialize(): Promise { const tools = new Set(await this.connection.connect()); this.#callGraphAdvertised = tools.has("callgraph"); + this.#surveyAdvertised = tools.has("survey_binary"); const expected = this.config.protocol === "native" ? nativeTools diff --git a/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts new file mode 100644 index 000000000..8c94d5de0 --- /dev/null +++ b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts @@ -0,0 +1,249 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import type { AnalysisOperationPort } from "../../../src/application/AnalysisProvider.js"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; +import { EnhancedTools } from "../../../src/application/EnhancedTools.js"; +import { AnalysisOutputError } from "../../../src/domain/analysisErrorCore.js"; +import { err, ok } from "../../../src/domain/result.js"; + +import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; + +afterEach(closeEnhancedToolResources); + +const PROVIDER = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", +} as const; + +type Coverage = boolean | undefined; + +interface Behavior { + /** What the provider reports for the references of each match address. */ + readonly coverage?: Readonly>; + readonly limitations?: readonly string[]; + /** Also report the first string match as a procedure match. */ + readonly duplicateProcedure?: boolean; + /** Fail the references read of this address. */ + readonly failXrefsOf?: string; + readonly calls?: string[]; +} + +const containing = (address: string) => ({ + query_address: address, + found: false, + procedure: null, + reason: "not_in_procedure", +}); + +/** + * A provider with two string matches (0x10 and 0x11), each referenced from + * one address (0x20 and 0x21). + */ +const provider = (behavior: Behavior = {}): AnalysisOperationPort => ({ + execute: async (name, arguments_) => { + const address = String(arguments_.address); + behavior.calls?.push(`${name}:${address}`); + switch (name) { + case "search_strings": + return ok( + createAnalysisExecution( + [ + { address: "0x10", value: "hello" }, + { address: "0x11", value: "hello again" }, + ], + PROVIDER, + ), + ); + case "search_procedures": + return ok( + createAnalysisExecution( + behavior.duplicateProcedure === true + ? [{ address: "0x10", value: "hello" }] + : [], + PROVIDER, + ), + ); + case "xrefs": { + if (address === behavior.failXrefsOf) + return err(new AnalysisOutputError("xrefs", "provider failed")); + const reported = behavior.coverage?.[address]; + return ok( + createAnalysisExecution( + [address === "0x10" ? "0x20" : "0x21"], + PROVIDER, + { + limitations: behavior.limitations ?? [], + ...(reported === undefined + ? {} + : { referencesTruncated: reported }), + }, + ), + ); + } + case "resolve_containing_procedure": + return ok(createAnalysisExecution(containing(address), PROVIDER)); + default: + throw new Error(`Unexpected operation: ${name}`); + } + }, +}); + +const expectedTrace = ( + residual: readonly string[], + extraMatches: readonly object[] = [], +) => ({ + query: "hello", + search_mode: "literal", + matches: [ + { type: "string", address: "0x10", value: "hello" }, + { type: "string", address: "0x11", value: "hello again" }, + ...extraMatches, + ], + references: [ + { + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + }, + { + target_address: "0x11", + source_address: "0x21", + containing_procedure: containing("0x21"), + }, + ...extraMatches.map(() => ({ + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + })), + ], + truncated: residual.length > 0, + residual_unknowns: residual, +}); + +const unknownFor = (address: string) => + `References to ${address} were truncated by the provider; additional references may exist.`; + +const trace = async (analysis: AnalysisOperationPort) => { + const client = await connect(analysis); + const result = await client.callTool({ + name: "trace_feature", + arguments: { query: "hello" }, + }); + expect(result.isError).not.toBe(true); + return result.structuredContent as { normalized_result: unknown }; +}; + +describe("trace_feature reference coverage", () => { + it("marks the trace truncated for the target whose references the provider capped", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect( + output.normalized_result, + "the received references are kept and only the capped target is a residual unknown", + ).toEqual(expectedTrace([unknownFor("0x10")])); + }); + + it("does not carry a truncation over to the next target that reports nothing", async () => { + const output = await trace(provider({ coverage: { "0x10": true } })); + + expect(output.normalized_result).toEqual( + expectedTrace([unknownFor("0x10")]), + ); + }); + + it("reports a capped target once when it is matched twice", async () => { + const output = await trace( + provider({ coverage: { "0x10": true }, duplicateProcedure: true }), + ); + + expect(output.normalized_result).toEqual( + expectedTrace( + [unknownFor("0x10")], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("resolves the containing procedure of a repeated reference source once", async () => { + const calls: string[] = []; + const output = await trace(provider({ duplicateProcedure: true, calls })); + + expect( + calls.filter((call) => call.startsWith("resolve_containing_procedure:")), + "the source 0x20 is referenced by two matches of the same address and resolved once", + ).toEqual([ + "resolve_containing_procedure:0x20", + "resolve_containing_procedure:0x21", + ]); + expect( + output.normalized_result, + "every reference is still reported with its containing procedure", + ).toEqual( + expectedTrace( + [], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("keeps the trace complete when the provider explicitly reports no cap", async () => { + const output = await trace( + provider({ coverage: { "0x10": false, "0x11": false } }), + ); + + expect(output.normalized_result).toEqual(expectedTrace([])); + }); + + it("does not change the trace of a provider that reports no coverage", async () => { + const output = await trace( + provider({ + limitations: ["Upstream xrefs_to capped results at 1000 references."], + }), + ); + + expect( + output.normalized_result, + "a limitation text alone never truncates, and an absent field keeps the previous result", + ).toEqual(expectedTrace([])); + }); + + it("never exposes the coverage field in the tool output", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect(JSON.stringify(output)).not.toMatch( + /referencesTruncated|references_truncated/, + ); + }); + + it("fails the trace when a references read fails and reads nothing further", async () => { + const calls: string[] = []; + const result = await new EnhancedTools( + provider({ failXrefsOf: "0x10", calls }), + ).execute("trace_feature", { query: "hello" }); + + expect(result.ok).toBe(false); + expect( + calls.filter((call) => call.startsWith("xrefs:")), + "the next target is not read after the failure", + ).toEqual(["xrefs:0x10"]); + }); + + it("stops without reading references when the trace is cancelled", async () => { + const calls: string[] = []; + const controller = new AbortController(); + controller.abort(); + const result = await new EnhancedTools(provider({ calls })).execute( + "trace_feature", + { query: "hello" }, + controller.signal, + ); + + expect(result.ok).toBe(false); + expect(calls.filter((call) => call.startsWith("xrefs:"))).toEqual([]); + }); +}); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index c3dc21542..01d8eba13 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -196,9 +196,9 @@ itWithCaptureCapability( result.value.frames.map(({ at_ms }) => at_ms), "capture-relative frame times stay ordered and nonnegative", ).toEqual( - [...result.value.frames.map(({ at_ms }) => at_ms)].sort( - (left, right) => left - right, - ), + result.value.frames + .map(({ at_ms }) => at_ms) + .sort((left, right) => left - right), ); expect(result.value.frames.every(({ at_ms }) => at_ms >= 0)).toBe(true); expect(result.value.exit.reason).toBe("exited"); diff --git a/tests/boundary/providers/ida/containment.test.ts b/tests/boundary/providers/ida/containment.test.ts new file mode 100644 index 000000000..75eb78f42 --- /dev/null +++ b/tests/boundary/providers/ida/containment.test.ts @@ -0,0 +1,144 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectAddressContext } from "../../../../src/application/AnalysisContextQueries.js"; +import { EnhancedTools } from "../../../../src/application/EnhancedTools.js"; +import { IdaSessionClient } from "../../../../src/ida/IdaSessionClient.js"; +import { createIdaTarget, RecordingIdaMcp } from "../../../fixtures/idaMcp.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const session = async (cappedReferences = false) => { + const { root, target } = await createIdaTarget("arm64"); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + if (cappedReferences) { + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [ + { + xrefs: [ + { + addr: "0x1004", + type: "code", + fn: { addr: "0x1000", name: "main", size: "0x20" }, + }, + ], + more: true, + }, + ] + : inner?.(name, args); + } + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "headless", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, target }; +}; + +const MAIN = { + found: true, + procedure: { address: "0x1000", name: "main" }, +}; + +describe("trace_feature on headless IDA", () => { + it("attributes a reference to its containing procedure", async () => { + const { client } = await session(); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + matches: [{ type: "string", address: "0x3000" }], + references: [ + { + target_address: "0x3000", + source_address: "0x1004", + containing_procedure: MAIN, + }, + ], + truncated: false, + residual_unknowns: [], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("keeps an upstream reference cap visible once containment makes the trace run", async () => { + const { client } = await session(true); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + references: [{ source_address: "0x1004", containing_procedure: MAIN }], + truncated: true, + residual_unknowns: [ + "References to 0x3000 were truncated by the provider; additional references may exist.", + ], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("inspect_address_context on headless IDA", () => { + it("resolves the admitted program as the document and reports the containing procedure", async () => { + const { client, target } = await session(); + try { + const result = await inspectAddressContext(client, { + address: "0x1004", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + address: "0x1004", + document: target.path, + procedure: { state: "available", value: MAIN }, + name: { state: "unavailable" }, + comment: { state: "unavailable" }, + inline_comment: { state: "unavailable" }, + bookmarks: { state: "unavailable" }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("answers the same way for an explicit admitted document", async () => { + const { client, target } = await session(); + try { + const implicit = await inspectAddressContext(client, { + address: "0x1004", + }); + const explicit = await inspectAddressContext(client, { + address: "0x1004", + document: target.path, + }); + + if (!implicit.ok) throw implicit.error; + if (!explicit.ok) throw explicit.error; + expect(explicit.value).toEqual(implicit.value); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/tests/boundary/providers/ida/verifierFeatureTrace.test.ts b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts new file mode 100644 index 000000000..77f654ffc --- /dev/null +++ b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { + assertFeatureTrace, + findMappedNonFunction, +} from "../../../../scripts/lib/ida-feature-trace.mjs"; + +const procedure = { address: "0x1000", name: "main" }; + +const containing = (found: boolean) => + found + ? { + query_address: "0x1004", + found: true, + procedure: { address: "0x1000", name: "main" }, + } + : { + query_address: "0x3010", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + +const trace = ( + matches: readonly { type: string; address: string }[], + references: readonly { + target_address: string; + containing_procedure: unknown; + }[], +) => ({ + query: "main", + search_mode: "literal", + matches, + references, + truncated: false, + residual_unknowns: [], +}); + +const SELECTED = { type: "procedure", address: "0x1000" }; +const SUBSTRING = { type: "procedure", address: "0x2000" }; +const STRING = { type: "string", address: "0x3000" }; + +describe("the real IDA lane's feature trace assertions", () => { + it("accepts references to every matched address, whoever contains the source", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED, SUBSTRING, STRING], + [ + { + target_address: "0x1000", + containing_procedure: containing(true), + }, + // A substring match such as main_helper, referenced from code. + { + target_address: "0x2000", + containing_procedure: containing(true), + }, + // A string containing the query, referenced from data outside functions. + { + target_address: "0x3000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("accepts a reference to the selected procedure from data outside functions", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("rejects a reference whose target is not one of the matches", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x9999", + containing_procedure: containing(true), + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace without the selected procedure's match", () => { + expect(() => assertFeatureTrace(trace([STRING], []), procedure)).toThrow(); + }); + + it("rejects a containment answer that is neither found nor a known reason", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: { + query_address: "0x1004", + found: false, + procedure: null, + reason: "unknown", + }, + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace that does not say whether it was truncated", () => { + expect(() => + assertFeatureTrace( + { ...trace([SELECTED], []), truncated: undefined }, + procedure, + ), + ).toThrow(); + }); +}); + +describe("the real IDA lane's mapped non-function probe", () => { + const inFunction = { + query_address: "0x1100", + found: true, + procedure: { address: "0x1000", name: "main" }, + }; + const data = { + query_address: "0x3000", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + const resolver = + (answers: Readonly>, asked: string[] = []) => + async (address: string) => { + asked.push(address); + return answers[address]; + }; + + it("skips a string inside a function chunk and uses the next one", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x3000" }], + resolver({ "0x1100": inFunction, "0x3000": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked, "strings are tried in order until one qualifies").toEqual([ + "0x1100", + "0x3000", + ]); + }); + + it("stops at the first string that qualifies", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x3000" }, { address: "0x3100" }], + resolver({ "0x3000": data, "0x3100": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked).toEqual(["0x3000"]); + }); + + it("reports no qualifying string when every string is inside a function", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x1200" }], + resolver({ "0x1100": inFunction, "0x1200": inFunction }), + ), + ).resolves.toBeUndefined(); + }); + + it("reports no qualifying string when the target has no strings", async () => { + await expect( + findMappedNonFunction([], resolver({})), + ).resolves.toBeUndefined(); + }); + + it("rejects a listed string that resolves as unmapped, since IDA cannot list an unmapped address", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x4000" }, { address: "0x3000" }], + resolver({ + "0x4000": { ...data, reason: "outside_segments" }, + "0x3000": data, + }), + ), + ).rejects.toThrow(); + }); +}); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index 813b90004..092f869e2 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -1278,3 +1278,102 @@ describe("selector origin forwarding", () => { ).toEqual(["provider", "provider", undefined]); }); }); + +describe("reference truncation forwarding", () => { + const referenceProvider = ( + referencesTruncated: boolean | undefined, + calls: string[], + ): AnalysisProvider => { + const identity = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + } as const; + return { + identity: () => identity, + resolveAnalysisProfile: () => + Promise.resolve( + resultOk({ + profile: createAnalysisProfile(identity, { fixture: true }), + }), + ), + capabilities: () => [ + { + provider: identity, + operation: "xrefs", + available: true, + reason: null, + effects: { + mutatesArtifact: false, + launchesProcess: false, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: false, + changesPermissions: false, + requiresRoot: false, + }, + limitations: [], + }, + ], + createClient: () => ({ + execute: (operation) => { + calls.push(operation); + return Promise.resolve( + resultOk( + createAnalysisExecution( + ["0x2000"], + identity, + referencesTruncated === undefined + ? {} + : { referencesTruncated }, + ), + ), + ); + }, + close: () => Promise.resolve(resultOk(null)), + }), + }; + }; + const parameters = { address: "0x1000", document: "fixture" }; + + it("forwards a truncation on both execution paths and never replays it from the snapshot", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(true, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const direct = await session.execute("xrefs", parameters); + const admitted = await session.withAdmittedAnalysis( + "trace_feature", + undefined, + (analysis) => analysis.execute("xrefs", parameters), + ); + + expect(direct.ok && direct.value.referencesTruncated).toBe(true); + if (!admitted.ok) throw admitted.error; + expect(admitted.value.ok && admitted.value.value.referencesTruncated).toBe( + true, + ); + expect( + calls, + "a truncated read is not recorded, so every call reaches the provider", + ).toEqual(["xrefs", "xrefs"]); + }); + + it("keeps an explicit complete report out of the way of snapshot replay", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(false, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const live = await session.execute("xrefs", parameters); + await session.execute("xrefs", parameters); + + expect(live.ok && live.value.referencesTruncated).toBe(false); + expect(calls, "the complete read is recorded and replayed").toEqual([ + "xrefs", + ]); + }); +}); diff --git a/tests/conformance/ida/providerContracts.test.ts b/tests/conformance/ida/providerContracts.test.ts index 39c524637..4e4b04c3b 100644 --- a/tests/conformance/ida/providerContracts.test.ts +++ b/tests/conformance/ida/providerContracts.test.ts @@ -29,15 +29,25 @@ const parametersFor = ( operation: string, ): Readonly> => { if (operation.startsWith("search_")) return { pattern: "a.b" }; - if (operation === "xrefs") return { address: "0x1000" }; + if (operation === "xrefs" || operation === "resolve_containing_procedure") + return { address: "0x1000" }; if (operation.startsWith("list_")) return {}; return { procedure: "main" }; }; +/** Operations only the headless profile declares. */ +const HEADLESS_ONLY: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + describe.each(["attached", "headless"] as const)( "IDA %s caller-visible contracts", (mode) => { - it.each(IDA_OPERATIONS)( + const operations = IDA_OPERATIONS.filter( + (operation) => mode === "headless" || !HEADLESS_ONLY.has(operation), + ); + it.each(operations)( "declares %s UI effects matching the upstream tools it reaches", async (operation) => { const { root, target } = await createIdaTarget(); @@ -69,7 +79,7 @@ describe.each(["attached", "headless"] as const)( }, ); - it.each(IDA_OPERATIONS)( + it.each(operations)( "serializes %s against its named advertised output contract", async (operation) => { const { root, target } = await createIdaTarget(); diff --git a/tests/fixtures/idaMcp.ts b/tests/fixtures/idaMcp.ts index 04bdb5f16..21e5e38f7 100644 --- a/tests/fixtures/idaMcp.ts +++ b/tests/fixtures/idaMcp.ts @@ -57,6 +57,7 @@ const modernNames = [ "callees", "xrefs_to", "callgraph", + "survey_binary", ]; /** Stateful producer fixture at the real connection seam; no module mocking. */