From 7005166f5878737fb7b3c7ec4e3d2ee5f4dcce30 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:05:06 +0800 Subject: [PATCH] test(hopper): verify real boundaries and prune redundant happy paths --- docs/testing.md | 11 + scripts/lib/real-hopper-boundaries.mjs | 411 ++++++++++++++++++ scripts/lib/real-hopper-semantic.mjs | 18 +- scripts/verify-real-hopper.mjs | 98 +++-- .../hopper/hopperClient.health.test.ts | 13 - .../hopper/realHopperSemantic.test.ts | 16 - 6 files changed, 501 insertions(+), 66 deletions(-) create mode 100644 scripts/lib/real-hopper-boundaries.mjs diff --git a/docs/testing.md b/docs/testing.md index 3db5623b3..52e9f42e3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -151,6 +151,17 @@ packaging/install behavior and fake-provider integration; use the corresponding real-provider lanes for engine claims. Real Apple dispatch and Interface Builder verifiers currently prove format integration through production readers. +`verify:hopper` exercises an installed Hopper through the production stdio MCP +server and CLI. It checks source-owned call chains, CFG edges, references and +complete large inventories, then probes unnamed bookmarks, annotation clearing, +batch validation before mutation, malformed addresses and regexes, segment-end +partial reads, and synthetic file-offset rejection. Advertised schemas are +validated in their JSON Schema dialect and successful replies are checked against +their advertised output schemas. Disposable binary copies prove that switching +and closing actually removes the native document, and that CLI byte results and +invalid-address diagnostics agree with MCP. No provider is mocked in this lane. +The Linux demo lane remains a separate `verify:hopper:linux` command. + Golden tests use immutable captured text inputs with producer/source provenance under `tests/fixtures/golden/`. Expected results are reviewed for the semantic claim; capture commands do not automatically approve new expected outputs. diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs new file mode 100644 index 000000000..0b8279e4f --- /dev/null +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -0,0 +1,411 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { copyFile, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { promisify } from "node:util"; +import Ajv from "ajv/dist/2020.js"; +import addFormats from "ajv-formats"; +import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; + +/** Exercise real Hopper annotation and malformed-input boundaries over MCP. */ +export async function verifyHopperBoundaryContracts( + client, + options, + procedure, + targetPath, +) { + const catalog = await client.listTools(); + const ajv = new Ajv({ strict: false }); + addFormats(ajv); + const validators = new Map(); + for (const tool of catalog.tools) { + assert.ok( + ajv.validateSchema(tool.inputSchema), + `${tool.name} input schema`, + ); + assert.ok(tool.outputSchema, `${tool.name} omitted its output schema`); + assert.ok( + ajv.validateSchema(tool.outputSchema), + `${tool.name} output schema`, + ); + validators.set(tool.name, ajv.compile(tool.outputSchema)); + } + let successfulCalls = 0; + let rejectedCalls = 0; + const rejectedOperations = new Set(); + const call = async (name, args = {}) => { + const reply = await client.callTool({ name, arguments: args }, options); + const result = requireMcpResult(reply, name); + const validate = validators.get(name); + assert.ok(validate, `${name} is missing from tools/list`); + assert.ok( + validate(reply.structuredContent), + `${name}: ${JSON.stringify(validate.errors)}`, + ); + assert.deepEqual(reply.structuredContent, JSON.parse(mcpTextValue(reply))); + successfulCalls += 1; + return result; + }; + const invalid = async (name, args, reason) => { + const reply = await client.callTool({ name, arguments: args }, options); + assert.equal(reply.isError, true, `${name} accepted invalid input`); + const { error } = JSON.parse(mcpTextValue(reply)); + assert.equal(error.code, "invalid_request"); + assert.equal(error.category, "invalid_input"); + assert.equal(error.details.operation, name); + assert.match(error.message, reason); + rejectedCalls += 1; + rejectedOperations.add(name); + }; + const address = procedure.address; + const originalName = await call("address_name", { address }); + const originalComment = await call("comment", { address }); + const originalInlineComment = await call("inline_comment", { address }); + const activeDocument = await call("current_document"); + const foreignDocument = (await call("list_documents")).find( + (name) => name !== activeDocument, + ); + try { + if (foreignDocument !== undefined) { + await invalid( + "read_bytes", + { address, length: 8, document: foreignDocument }, + /belongs to another target/u, + ); + assert.equal(await call("current_document"), activeDocument); + } + for (const label of [undefined, "", "REA boundary bookmark 测试"]) { + assert.equal( + await call("set_bookmark", { + address, + ...(label === undefined ? {} : { name: label }), + }), + true, + ); + const bookmarks = await call("list_bookmarks"); + const bookmark = bookmarks.find((item) => item.address === address); + assert.ok(bookmark, "set_bookmark did not create a bookmark"); + assert.equal(bookmark.name, label || null); + const context = await call("inspect_address_context", { address }); + assert.equal(context.bookmarks.state, "available"); + assert.deepEqual(context.bookmarks.value, [bookmark]); + assert.equal(await call("unset_bookmark", { address }), true); + assert.ok( + !(await call("list_bookmarks")).some( + (item) => item.address === address, + ), + ); + } + + for (const [setter, getter] of [ + ["set_comment", "comment"], + ["set_inline_comment", "inline_comment"], + ]) { + const comment = "REA boundary comment 测试"; + assert.equal(await call(setter, { address, comment }), true); + assert.equal(await call(getter, { address }), comment); + assert.equal(await call(setter, { address, comment: "" }), true); + assert.equal(await call(getter, { address }), null); + } + const inline = "REA inline\nsecond line"; + const exact = await call("set_inline_comment", { + address, + comment: inline, + }); + const observed = await call("inline_comment", { address }); + assert.equal( + exact, + observed === inline, + "inline normalization must remain visible", + ); + + await invalid( + "set_addresses_names", + { + names: { + [address]: "rea_boundary_should_not_be_applied", + __rea_missing_batch_address__: "missing", + }, + }, + /Unknown Hopper address or name: __rea_missing_batch_address__/u, + ); + assert.equal(await call("address_name", { address }), originalName); + assert.equal( + (await call("procedure_info", { procedure: address })).name, + originalName, + ); + + const renamed = "rea_boundary_renamed"; + assert.equal( + await call("set_addresses_names", { names: { [address]: renamed } }).then( + (value) => value[address], + ), + true, + ); + assert.equal(await call("address_name", { address }), renamed); + const matches = await call("search_procedures", { + pattern: renamed, + case_sensitive: true, + }); + assert.ok( + matches.some( + (item) => item.address === address && item.value === renamed, + ), + ); + assert.equal( + (await call("analyze_function", { procedure: address })).procedure.name, + renamed, + ); + assert.equal( + await call("set_address_name", { address, name: "add" }), + true, + ); + assert.equal( + await call("procedure_address", { procedure: "add" }), + address, + ); + assert.equal( + (await call("procedure_info", { procedure: "add" })).entrypoint, + address, + ); + assert.equal( + (await call("read_bytes", { address: "add", length: 8 })).address, + address, + ); + assert.equal( + await call("procedure_address", { procedure: address.slice(2) }), + address, + ); + + for (const query of ["-1", "0x10000000000000000", "0xffffffffffffffff"]) { + for (const name of [ + "read_bytes", + "address_to_file_offset", + "resolve_containing_procedure", + "procedure_info", + ]) { + await invalid( + name, + name === "procedure_info" + ? { procedure: query } + : { + address: query, + ...(name === "read_bytes" ? { length: 8 } : {}), + }, + /usable unsigned 64-bit range/u, + ); + } + } + await invalid( + "read_bytes", + { address: "0x0", length: 8 }, + /Address is outside every segment/u, + ); + await invalid( + "procedure_info", + { procedure: address, document: "__rea_missing_document__" }, + /Unknown Hopper document/u, + ); + await invalid( + "search_procedures", + { pattern: "[", mode: "regex" }, + /Invalid regex pattern/u, + ); + // Invalid calls must leave the same authenticated bridge usable. + assert.equal( + (await call("read_bytes", { address, length: 8 })).returned_bytes, + 8, + ); + const mapping = await call("address_to_file_offset", { address }); + const bytes = await call("read_bytes", { address, length: 8 }); + const file = await readFile(targetPath); + assert.equal( + bytes.bytes_hex, + file + .subarray(mapping.file_offset, mapping.file_offset + 8) + .toString("hex"), + ); + const segments = await call("list_segments"); + const containing = segments.find( + (segment) => + BigInt(segment.start) <= BigInt(address) && + BigInt(address) < BigInt(segment.end), + ); + assert.ok(containing, "fixture procedure has no containing segment"); + const lastByte = `0x${(BigInt(containing.end) - 1n).toString(16)}`; + const prefix = await call("read_bytes", { address: lastByte, length: 8 }); + assert.equal(prefix.returned_bytes, 1); + assert.equal(prefix.bytes_hex.length, 2); + assert.equal(prefix.complete, false); + const synthetic = segments.find( + (segment) => segment.name === "External Symbols", + ); + assert.ok(synthetic, "fixture omitted synthetic external-symbol memory"); + await invalid( + "address_to_file_offset", + { address: synthetic.start }, + /no authoritative file-offset mapping/u, + ); + const health = (await call("binary_session")).provider_operation_health; + assert.equal( + health.state, + "idle", + "a live GUI bridge must remain healthy after its launcher helper exits", + ); + } finally { + await call("set_address_name", { address, name: originalName ?? "" }); + await call("set_comment", { address, comment: originalComment ?? "" }); + await call("set_inline_comment", { + address, + comment: originalInlineComment ?? "", + }); + await call("unset_bookmark", { address }); + } + return { + schemaCount: catalog.tools.length, + successfulCalls, + rejectedCalls, + rejectedOperations: [...rejectedOperations], + foreignDocumentVerified: foreignDocument !== undefined, + }; +} + +/** Prove actual document disposal and shared CLI/MCP behavior on disposable copies. */ +export async function verifyHopperLifecycleAndCli(client, options, targets) { + const directory = await realpath( + await mkdtemp(join(tmpdir(), "hopper-boundary-")), + ); + const suffix = basename(directory); + const primary = join(directory, `primary-${suffix}`); + const secondary = join(directory, `secondary-${suffix}`); + const call = async (name, args = {}) => + requireMcpResult( + await client.callTool({ name, arguments: args }, options), + name, + ); + const cli = promisify(execFile); + const runCli = (args) => { + const pending = cli(process.execPath, args, { + timeout: 180_000, + maxBuffer: 1_048_576, + }); + if (pending.child.pid !== undefined) + targets.ownedProcessIds.add(pending.child.pid); + return pending; + }; + try { + await copyFile(targets.primary, primary); + await copyFile(targets.secondary, secondary); + await call("open_binary", { path: primary }); + const firstDocument = await call("current_document"); + await call("open_binary", { path: secondary }); + const secondDocument = await call("current_document"); + assert.notEqual(firstDocument, secondDocument); + assert.ok( + !(await call("list_documents")).includes(firstDocument), + "target switching retained the old native document", + ); + const procedures = await call("list_procedures"); + await verifyBatchCancellation(client, options, procedures); + const address = procedures[0].address; + const expected = await call("read_bytes", { + address, + length: 8, + document: secondDocument, + }); + await call("close_binary"); + + const dispatcher = new URL("../rea.mjs", import.meta.url).pathname; + const common = [ + "--length", + "8", + "--provider", + "hopper", + "--format", + "json", + ]; + const { stdout } = await runCli([ + dispatcher, + "read-bytes", + secondary, + address, + ...common, + ]); + const evidence = JSON.parse(stdout); + assert.equal(evidence.provider.id, "hopper"); + assert.equal(evidence.subject.local_path, secondary); + assert.deepEqual(evidence.normalized_result, expected); + let failure; + try { + await runCli([ + dispatcher, + "read-bytes", + secondary, + "0x10000000000000000", + ...common, + ]); + } catch (error) { + assert.equal(error.code, 1); + failure = JSON.parse(error.stdout); + } + assert.ok(failure, "CLI accepted an overflowing address"); + assert.equal(failure.code, "invalid_request"); + assert.equal(failure.category, "invalid_input"); + assert.match(failure.message, /usable unsigned 64-bit range/u); + + await call("open_binary", { path: primary }); + assert.ok( + !(await call("list_documents")).includes(secondDocument), + "MCP or CLI cleanup retained the closed native document", + ); + await call("close_binary"); + return { + targetSwitchDisposedDocument: true, + cliByteParity: true, + cliInvalidAddress: true, + callerCancellationRecovered: true, + closedDocumentAbsent: true, + }; + } finally { + await call("close_binary"); + await rm(directory, { recursive: true, force: true }); + } +} + +const verifyBatchCancellation = async (client, options, procedures) => { + const controller = new AbortController(); + let cancelled = false; + await assert.rejects( + client.callTool( + { + name: "batch_decompile", + arguments: { addresses: procedures.map((item) => item.address) }, + }, + { + ...options, + signal: controller.signal, + onprogress: (update) => { + options.onprogress?.(update); + // Cancel between real decompilations, after Hopper served one item. + if (update.message?.includes("Hopper bridge completed request")) { + cancelled = true; + controller.abort(); + } + }, + }, + ), + /aborted/u, + ); + assert.ok(cancelled, "cancellation did not follow a real Hopper response"); + const recovered = requireMcpResult( + await client.callTool({ name: "list_procedures", arguments: {} }, options), + "list_procedures after cancellation", + ); + assert.deepEqual(recovered, procedures); + const status = requireMcpResult( + await client.callTool({ name: "binary_session", arguments: {} }, options), + "binary_session after cancellation", + ); + assert.equal(status.provider_operation_health.state, "idle"); +}; diff --git a/scripts/lib/real-hopper-semantic.mjs b/scripts/lib/real-hopper-semantic.mjs index 8bd20c440..6cf855f2c 100644 --- a/scripts/lib/real-hopper-semantic.mjs +++ b/scripts/lib/real-hopper-semantic.mjs @@ -57,7 +57,9 @@ export function requireBridgeDiagnostic(chunks) { (diagnostic) => diagnostic.type === "bridge-diagnostic" && diagnostic.category === "invalid_request" && - diagnostic.message === "Invalid Hopper bridge request", + diagnostic.message.includes( + "Unknown Hopper address or name: __rea_missing_procedure__", + ), ) ) throw new Error("The MCP runtime omitted the correlated Hopper diagnostic"); @@ -153,7 +155,13 @@ export async function verifyRealHopperFixture({ } /** Reject malformed, unsafe, or unexpected failure diagnostics from MCP. */ -export function requireSafeDiagnostics(chunks) { +export function requireSafeDiagnostics( + chunks, + expectedErrorTools = ["procedure_info"], + expectedErrorTags = ["HopperRemoteError"], +) { + const expected = new Set(expectedErrorTools); + const tags = new Set(expectedErrorTags); const lines = chunks .join("") .split("\n") @@ -173,10 +181,8 @@ export function requireSafeDiagnostics(chunks) { typeof diagnostic.level !== "number" || diagnostic.level >= 50 || (diagnostic.level >= 40 && - diagnostic.errorTag !== "HopperRemoteError" && - !( - diagnostic.tool === "procedure_info" && diagnostic.status === "error" - )) + !tags.has(diagnostic.errorTag) && + !(expected.has(diagnostic.tool) && diagnostic.status === "error")) ) throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`); } diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index 8877cc688..621cc2359 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -1,5 +1,7 @@ import { execFile } from "node:child_process"; -import { readdir } from "node:fs/promises"; +import { readFile, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -33,6 +35,10 @@ import { openAndVerifyLargeFixture } from "./lib/real-hopper-exhaustive-search.m import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; import { requireHopperSelection } from "./lib/real-hopper-selection.mjs"; import { verifyHopperInventories } from "./lib/real-hopper-inventory.mjs"; +import { + verifyHopperBoundaryContracts, + verifyHopperLifecycleAndCli, +} from "./lib/real-hopper-boundaries.mjs"; import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs"; const execFileAsync = promisify(execFile); const verifierRun = createVerifierRun(); @@ -47,9 +53,9 @@ const parseServerArgs = (encoded) => { return parsed; }; -const sessionsBefore = new Set( - (await readdir("/tmp")).filter((name) => name.startsWith("rea-")), -); +const runtimeParent = process.platform === "darwin" ? "/tmp" : tmpdir(); +const sessionsBefore = new Set(await readdir(runtimeParent)); +const ownedProcessIds = new Set(); const textValue = mcpTextValue; const requireSuccessfulTool = requireMcpResult; @@ -250,6 +256,9 @@ let summary; try { await client.connect(transport); + if (transport.pid === null) + throw new Error("The verifier did not capture its MCP process identity"); + ownedProcessIds.add(transport.pid); const progressUpdates = []; const options = { timeout, @@ -358,6 +367,12 @@ try { oracle: fixtureTargets.oracle, normalizedResult: requireSuccessfulTool, }); + const boundaryContracts = await verifyHopperBoundaryContracts( + client, + options, + fixtureAnalysis.procedures.entry, + targetA, + ); const switched = await client.callTool( { name: "open_binary", arguments: { path: targetB } }, options, @@ -396,15 +411,14 @@ try { throw new Error( "binary_overview retained another document after target switch", ); - const expectedDocumentsAfterTargetSwitch = - process.platform === "linux" - ? firstDocuments.length - : firstDocuments.length + 1; + const expectedDocumentsAfterTargetSwitch = firstDocuments.length; if ( !Array.isArray(documentsAfterTargetSwitch) || documentsAfterTargetSwitch.length !== expectedDocumentsAfterTargetSwitch ) - throw new Error("A distinct target did not receive one Hopper document"); + throw new Error( + "Target switching did not replace the previous Hopper document", + ); const reopenedTarget = await client.callTool( { name: "open_binary", arguments: { path: targetB } }, options, @@ -441,20 +455,32 @@ try { "Hopper's bundled MCP server was running during verification", ); } - const diagnosticCount = requireSafeDiagnostics(stderrChunks); - requireBridgeDiagnostic(stderrChunks); - const closed = await client.callTool( { name: "close_binary", arguments: {} }, options, ); if (closed.isError === true) throw new Error(textValue(closed)); + const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, { + primary: targetA, + secondary: targetB, + ownedProcessIds, + }); const closedSession = requireMcpResult( await fullSessionStatus(), "binary_session", ); if (closedSession.open !== false) throw new Error("The real session remained open after close_binary"); + const diagnosticCount = requireSafeDiagnostics( + stderrChunks, + [ + "procedure_info", + "batch_decompile", + ...boundaryContracts.rejectedOperations, + ], + ["HopperRemoteError", "HopperCancelledError", "AnalysisCancelledError"], + ); + requireBridgeDiagnostic(stderrChunks); summary = { toolCount: actualNames.length, @@ -463,6 +489,8 @@ try { segmentCount: firstOverview.segment_count, analyses: [firstAnalysis, secondAnalysis], fixtureAnalysis, + boundaryContracts, + lifecycleAndCli, inventoryCounts, largeInventory, fixtureManifest: fixtureTargets.manifestPath, @@ -491,29 +519,37 @@ try { } await new Promise((resolve) => setTimeout(resolve, 500)); -const sessionsAfter = (await readdir("/tmp")).filter( - (name) => name.startsWith("rea-") && !sessionsBefore.has(name), -); -if (sessionsAfter.length > 0) { - throw new Error("The MCP runtime leaked a bridge session directory"); -} -const afterCloseProcesses = await execFileAsync("ps", [ - "-ax", - "-o", - "command=", -]); -if ( - afterCloseProcesses.stdout - .split("\n") - .some((line) => line.includes("node dist/main.js")) -) { - throw new Error( - "The TypeScript MCP server remained alive after client close", - ); +for (const entry of await readdir(runtimeParent, { withFileTypes: true })) { + if ( + !entry.isDirectory() || + !entry.name.startsWith("rea-") || + sessionsBefore.has(entry.name) + ) + continue; + let ownership; + try { + ownership = JSON.parse( + await readFile(join(runtimeParent, entry.name, "ownership.json"), "utf8"), + ); + } catch (cause) { + if (cause.code === "ENOENT") continue; + throw cause; + } + if (ownedProcessIds.has(ownership.parent_pid)) + throw new Error( + `An owned Hopper runtime leaked its directory: ${entry.name}`, + ); } if (summary === undefined) throw new Error("Real-Hopper verification did not produce a summary"); const completedVerifierRun = await completeVerifierRun(verifierRun); +if ( + completedVerifierRun.process_lineage.status !== "verified" || + completedVerifierRun.process_lineage.descendants.length !== 0 +) + throw new Error( + "The verifier's process lineage did not prove owned child cleanup", + ); await new Promise((resolve, reject) => { process.stdout.write( `${JSON.stringify({ verifier_run: completedVerifierRun, ...summary, unrelatedHopperSurvived, cleanShutdown: true }, null, 2)}\n`, diff --git a/tests/boundary/providers/hopper/hopperClient.health.test.ts b/tests/boundary/providers/hopper/hopperClient.health.test.ts index 8e8bd1930..a21313c4c 100644 --- a/tests/boundary/providers/hopper/hopperClient.health.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.health.test.ts @@ -61,19 +61,6 @@ const start = async (owned: boolean) => { }; describe("Hopper provider lifecycle health", () => { - it("keeps a live bridge healthy after a non-owning launcher helper exits", async () => { - const { client, launcher } = await start(false); - launcher.observed.exit(0); - expect(client.operationHealth().state).toBe("idle"); - expect(await client.callTool("echo", { label: "GUI still alive" })).toEqual( - { - ok: true, - value: { label: "GUI still alive" }, - }, - ); - expect(client.operationHealth().state).toBe("idle"); - }); - it("settles disconnected requests without attributing the helper exit to the GUI", async () => { const { client, launcher } = await start(false); launcher.observed.exit(0); diff --git a/tests/conformance/providers/hopper/realHopperSemantic.test.ts b/tests/conformance/providers/hopper/realHopperSemantic.test.ts index da04a24ae..99b4f41bf 100644 --- a/tests/conformance/providers/hopper/realHopperSemantic.test.ts +++ b/tests/conformance/providers/hopper/realHopperSemantic.test.ts @@ -27,22 +27,6 @@ describe("real Hopper semantic fixture resolution", () => { ).rejects.toThrow(/current_document/u); }); - it("resolves the public search value and preserves its reported name", async () => { - const procedure = await resolveFixtureProcedure( - clientWithItems([ - { - address: "0x1000", - value: "_rea_entry", - }, - ]), - {}, - "rea_entry", - normalize, - ); - - expect(procedure).toEqual({ address: "0x1000", name: "_rea_entry" }); - }); - it.each([ ["empty", []], [