test(hopper): verify real boundaries and prune redundant happy paths

This commit is contained in:
morluto 2026-10-09 02:05:06 +08:00
parent 2f71375efc
commit 7005166f58
6 changed files with 501 additions and 66 deletions

View File

@ -151,6 +151,17 @@ packaging/install behavior and fake-provider integration; use the corresponding
real-provider lanes for engine claims. Real Apple dispatch and Interface
Builder verifiers currently prove format integration through production readers.
`verify:hopper` exercises an installed Hopper through the production stdio MCP
server and CLI. It checks source-owned call chains, CFG edges, references and
complete large inventories, then probes unnamed bookmarks, annotation clearing,
batch validation before mutation, malformed addresses and regexes, segment-end
partial reads, and synthetic file-offset rejection. Advertised schemas are
validated in their JSON Schema dialect and successful replies are checked against
their advertised output schemas. Disposable binary copies prove that switching
and closing actually removes the native document, and that CLI byte results and
invalid-address diagnostics agree with MCP. No provider is mocked in this lane.
The Linux demo lane remains a separate `verify:hopper:linux` command.
Golden tests use immutable captured text inputs with producer/source provenance
under `tests/fixtures/golden/`. Expected results are reviewed for the semantic
claim; capture commands do not automatically approve new expected outputs.

View File

@ -0,0 +1,411 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { copyFile, mkdtemp, readFile, realpath, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import { promisify } from "node:util";
import Ajv from "ajv/dist/2020.js";
import addFormats from "ajv-formats";
import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs";
/** Exercise real Hopper annotation and malformed-input boundaries over MCP. */
export async function verifyHopperBoundaryContracts(
client,
options,
procedure,
targetPath,
) {
const catalog = await client.listTools();
const ajv = new Ajv({ strict: false });
addFormats(ajv);
const validators = new Map();
for (const tool of catalog.tools) {
assert.ok(
ajv.validateSchema(tool.inputSchema),
`${tool.name} input schema`,
);
assert.ok(tool.outputSchema, `${tool.name} omitted its output schema`);
assert.ok(
ajv.validateSchema(tool.outputSchema),
`${tool.name} output schema`,
);
validators.set(tool.name, ajv.compile(tool.outputSchema));
}
let successfulCalls = 0;
let rejectedCalls = 0;
const rejectedOperations = new Set();
const call = async (name, args = {}) => {
const reply = await client.callTool({ name, arguments: args }, options);
const result = requireMcpResult(reply, name);
const validate = validators.get(name);
assert.ok(validate, `${name} is missing from tools/list`);
assert.ok(
validate(reply.structuredContent),
`${name}: ${JSON.stringify(validate.errors)}`,
);
assert.deepEqual(reply.structuredContent, JSON.parse(mcpTextValue(reply)));
successfulCalls += 1;
return result;
};
const invalid = async (name, args, reason) => {
const reply = await client.callTool({ name, arguments: args }, options);
assert.equal(reply.isError, true, `${name} accepted invalid input`);
const { error } = JSON.parse(mcpTextValue(reply));
assert.equal(error.code, "invalid_request");
assert.equal(error.category, "invalid_input");
assert.equal(error.details.operation, name);
assert.match(error.message, reason);
rejectedCalls += 1;
rejectedOperations.add(name);
};
const address = procedure.address;
const originalName = await call("address_name", { address });
const originalComment = await call("comment", { address });
const originalInlineComment = await call("inline_comment", { address });
const activeDocument = await call("current_document");
const foreignDocument = (await call("list_documents")).find(
(name) => name !== activeDocument,
);
try {
if (foreignDocument !== undefined) {
await invalid(
"read_bytes",
{ address, length: 8, document: foreignDocument },
/belongs to another target/u,
);
assert.equal(await call("current_document"), activeDocument);
}
for (const label of [undefined, "", "REA boundary bookmark 测试"]) {
assert.equal(
await call("set_bookmark", {
address,
...(label === undefined ? {} : { name: label }),
}),
true,
);
const bookmarks = await call("list_bookmarks");
const bookmark = bookmarks.find((item) => item.address === address);
assert.ok(bookmark, "set_bookmark did not create a bookmark");
assert.equal(bookmark.name, label || null);
const context = await call("inspect_address_context", { address });
assert.equal(context.bookmarks.state, "available");
assert.deepEqual(context.bookmarks.value, [bookmark]);
assert.equal(await call("unset_bookmark", { address }), true);
assert.ok(
!(await call("list_bookmarks")).some(
(item) => item.address === address,
),
);
}
for (const [setter, getter] of [
["set_comment", "comment"],
["set_inline_comment", "inline_comment"],
]) {
const comment = "REA boundary comment 测试";
assert.equal(await call(setter, { address, comment }), true);
assert.equal(await call(getter, { address }), comment);
assert.equal(await call(setter, { address, comment: "" }), true);
assert.equal(await call(getter, { address }), null);
}
const inline = "REA inline\nsecond line";
const exact = await call("set_inline_comment", {
address,
comment: inline,
});
const observed = await call("inline_comment", { address });
assert.equal(
exact,
observed === inline,
"inline normalization must remain visible",
);
await invalid(
"set_addresses_names",
{
names: {
[address]: "rea_boundary_should_not_be_applied",
__rea_missing_batch_address__: "missing",
},
},
/Unknown Hopper address or name: __rea_missing_batch_address__/u,
);
assert.equal(await call("address_name", { address }), originalName);
assert.equal(
(await call("procedure_info", { procedure: address })).name,
originalName,
);
const renamed = "rea_boundary_renamed";
assert.equal(
await call("set_addresses_names", { names: { [address]: renamed } }).then(
(value) => value[address],
),
true,
);
assert.equal(await call("address_name", { address }), renamed);
const matches = await call("search_procedures", {
pattern: renamed,
case_sensitive: true,
});
assert.ok(
matches.some(
(item) => item.address === address && item.value === renamed,
),
);
assert.equal(
(await call("analyze_function", { procedure: address })).procedure.name,
renamed,
);
assert.equal(
await call("set_address_name", { address, name: "add" }),
true,
);
assert.equal(
await call("procedure_address", { procedure: "add" }),
address,
);
assert.equal(
(await call("procedure_info", { procedure: "add" })).entrypoint,
address,
);
assert.equal(
(await call("read_bytes", { address: "add", length: 8 })).address,
address,
);
assert.equal(
await call("procedure_address", { procedure: address.slice(2) }),
address,
);
for (const query of ["-1", "0x10000000000000000", "0xffffffffffffffff"]) {
for (const name of [
"read_bytes",
"address_to_file_offset",
"resolve_containing_procedure",
"procedure_info",
]) {
await invalid(
name,
name === "procedure_info"
? { procedure: query }
: {
address: query,
...(name === "read_bytes" ? { length: 8 } : {}),
},
/usable unsigned 64-bit range/u,
);
}
}
await invalid(
"read_bytes",
{ address: "0x0", length: 8 },
/Address is outside every segment/u,
);
await invalid(
"procedure_info",
{ procedure: address, document: "__rea_missing_document__" },
/Unknown Hopper document/u,
);
await invalid(
"search_procedures",
{ pattern: "[", mode: "regex" },
/Invalid regex pattern/u,
);
// Invalid calls must leave the same authenticated bridge usable.
assert.equal(
(await call("read_bytes", { address, length: 8 })).returned_bytes,
8,
);
const mapping = await call("address_to_file_offset", { address });
const bytes = await call("read_bytes", { address, length: 8 });
const file = await readFile(targetPath);
assert.equal(
bytes.bytes_hex,
file
.subarray(mapping.file_offset, mapping.file_offset + 8)
.toString("hex"),
);
const segments = await call("list_segments");
const containing = segments.find(
(segment) =>
BigInt(segment.start) <= BigInt(address) &&
BigInt(address) < BigInt(segment.end),
);
assert.ok(containing, "fixture procedure has no containing segment");
const lastByte = `0x${(BigInt(containing.end) - 1n).toString(16)}`;
const prefix = await call("read_bytes", { address: lastByte, length: 8 });
assert.equal(prefix.returned_bytes, 1);
assert.equal(prefix.bytes_hex.length, 2);
assert.equal(prefix.complete, false);
const synthetic = segments.find(
(segment) => segment.name === "External Symbols",
);
assert.ok(synthetic, "fixture omitted synthetic external-symbol memory");
await invalid(
"address_to_file_offset",
{ address: synthetic.start },
/no authoritative file-offset mapping/u,
);
const health = (await call("binary_session")).provider_operation_health;
assert.equal(
health.state,
"idle",
"a live GUI bridge must remain healthy after its launcher helper exits",
);
} finally {
await call("set_address_name", { address, name: originalName ?? "" });
await call("set_comment", { address, comment: originalComment ?? "" });
await call("set_inline_comment", {
address,
comment: originalInlineComment ?? "",
});
await call("unset_bookmark", { address });
}
return {
schemaCount: catalog.tools.length,
successfulCalls,
rejectedCalls,
rejectedOperations: [...rejectedOperations],
foreignDocumentVerified: foreignDocument !== undefined,
};
}
/** Prove actual document disposal and shared CLI/MCP behavior on disposable copies. */
export async function verifyHopperLifecycleAndCli(client, options, targets) {
const directory = await realpath(
await mkdtemp(join(tmpdir(), "hopper-boundary-")),
);
const suffix = basename(directory);
const primary = join(directory, `primary-${suffix}`);
const secondary = join(directory, `secondary-${suffix}`);
const call = async (name, args = {}) =>
requireMcpResult(
await client.callTool({ name, arguments: args }, options),
name,
);
const cli = promisify(execFile);
const runCli = (args) => {
const pending = cli(process.execPath, args, {
timeout: 180_000,
maxBuffer: 1_048_576,
});
if (pending.child.pid !== undefined)
targets.ownedProcessIds.add(pending.child.pid);
return pending;
};
try {
await copyFile(targets.primary, primary);
await copyFile(targets.secondary, secondary);
await call("open_binary", { path: primary });
const firstDocument = await call("current_document");
await call("open_binary", { path: secondary });
const secondDocument = await call("current_document");
assert.notEqual(firstDocument, secondDocument);
assert.ok(
!(await call("list_documents")).includes(firstDocument),
"target switching retained the old native document",
);
const procedures = await call("list_procedures");
await verifyBatchCancellation(client, options, procedures);
const address = procedures[0].address;
const expected = await call("read_bytes", {
address,
length: 8,
document: secondDocument,
});
await call("close_binary");
const dispatcher = new URL("../rea.mjs", import.meta.url).pathname;
const common = [
"--length",
"8",
"--provider",
"hopper",
"--format",
"json",
];
const { stdout } = await runCli([
dispatcher,
"read-bytes",
secondary,
address,
...common,
]);
const evidence = JSON.parse(stdout);
assert.equal(evidence.provider.id, "hopper");
assert.equal(evidence.subject.local_path, secondary);
assert.deepEqual(evidence.normalized_result, expected);
let failure;
try {
await runCli([
dispatcher,
"read-bytes",
secondary,
"0x10000000000000000",
...common,
]);
} catch (error) {
assert.equal(error.code, 1);
failure = JSON.parse(error.stdout);
}
assert.ok(failure, "CLI accepted an overflowing address");
assert.equal(failure.code, "invalid_request");
assert.equal(failure.category, "invalid_input");
assert.match(failure.message, /usable unsigned 64-bit range/u);
await call("open_binary", { path: primary });
assert.ok(
!(await call("list_documents")).includes(secondDocument),
"MCP or CLI cleanup retained the closed native document",
);
await call("close_binary");
return {
targetSwitchDisposedDocument: true,
cliByteParity: true,
cliInvalidAddress: true,
callerCancellationRecovered: true,
closedDocumentAbsent: true,
};
} finally {
await call("close_binary");
await rm(directory, { recursive: true, force: true });
}
}
const verifyBatchCancellation = async (client, options, procedures) => {
const controller = new AbortController();
let cancelled = false;
await assert.rejects(
client.callTool(
{
name: "batch_decompile",
arguments: { addresses: procedures.map((item) => item.address) },
},
{
...options,
signal: controller.signal,
onprogress: (update) => {
options.onprogress?.(update);
// Cancel between real decompilations, after Hopper served one item.
if (update.message?.includes("Hopper bridge completed request")) {
cancelled = true;
controller.abort();
}
},
},
),
/aborted/u,
);
assert.ok(cancelled, "cancellation did not follow a real Hopper response");
const recovered = requireMcpResult(
await client.callTool({ name: "list_procedures", arguments: {} }, options),
"list_procedures after cancellation",
);
assert.deepEqual(recovered, procedures);
const status = requireMcpResult(
await client.callTool({ name: "binary_session", arguments: {} }, options),
"binary_session after cancellation",
);
assert.equal(status.provider_operation_health.state, "idle");
};

View File

@ -57,7 +57,9 @@ export function requireBridgeDiagnostic(chunks) {
(diagnostic) =>
diagnostic.type === "bridge-diagnostic" &&
diagnostic.category === "invalid_request" &&
diagnostic.message === "Invalid Hopper bridge request",
diagnostic.message.includes(
"Unknown Hopper address or name: __rea_missing_procedure__",
),
)
)
throw new Error("The MCP runtime omitted the correlated Hopper diagnostic");
@ -153,7 +155,13 @@ export async function verifyRealHopperFixture({
}
/** Reject malformed, unsafe, or unexpected failure diagnostics from MCP. */
export function requireSafeDiagnostics(chunks) {
export function requireSafeDiagnostics(
chunks,
expectedErrorTools = ["procedure_info"],
expectedErrorTags = ["HopperRemoteError"],
) {
const expected = new Set(expectedErrorTools);
const tags = new Set(expectedErrorTags);
const lines = chunks
.join("")
.split("\n")
@ -173,10 +181,8 @@ export function requireSafeDiagnostics(chunks) {
typeof diagnostic.level !== "number" ||
diagnostic.level >= 50 ||
(diagnostic.level >= 40 &&
diagnostic.errorTag !== "HopperRemoteError" &&
!(
diagnostic.tool === "procedure_info" && diagnostic.status === "error"
))
!tags.has(diagnostic.errorTag) &&
!(expected.has(diagnostic.tool) && diagnostic.status === "error"))
)
throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`);
}

View File

@ -1,5 +1,7 @@
import { execFile } from "node:child_process";
import { readdir } from "node:fs/promises";
import { readFile, readdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
@ -33,6 +35,10 @@ import { openAndVerifyLargeFixture } from "./lib/real-hopper-exhaustive-search.m
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { requireHopperSelection } from "./lib/real-hopper-selection.mjs";
import { verifyHopperInventories } from "./lib/real-hopper-inventory.mjs";
import {
verifyHopperBoundaryContracts,
verifyHopperLifecycleAndCli,
} from "./lib/real-hopper-boundaries.mjs";
import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs";
const execFileAsync = promisify(execFile);
const verifierRun = createVerifierRun();
@ -47,9 +53,9 @@ const parseServerArgs = (encoded) => {
return parsed;
};
const sessionsBefore = new Set(
(await readdir("/tmp")).filter((name) => name.startsWith("rea-")),
);
const runtimeParent = process.platform === "darwin" ? "/tmp" : tmpdir();
const sessionsBefore = new Set(await readdir(runtimeParent));
const ownedProcessIds = new Set();
const textValue = mcpTextValue;
const requireSuccessfulTool = requireMcpResult;
@ -250,6 +256,9 @@ let summary;
try {
await client.connect(transport);
if (transport.pid === null)
throw new Error("The verifier did not capture its MCP process identity");
ownedProcessIds.add(transport.pid);
const progressUpdates = [];
const options = {
timeout,
@ -358,6 +367,12 @@ try {
oracle: fixtureTargets.oracle,
normalizedResult: requireSuccessfulTool,
});
const boundaryContracts = await verifyHopperBoundaryContracts(
client,
options,
fixtureAnalysis.procedures.entry,
targetA,
);
const switched = await client.callTool(
{ name: "open_binary", arguments: { path: targetB } },
options,
@ -396,15 +411,14 @@ try {
throw new Error(
"binary_overview retained another document after target switch",
);
const expectedDocumentsAfterTargetSwitch =
process.platform === "linux"
? firstDocuments.length
: firstDocuments.length + 1;
const expectedDocumentsAfterTargetSwitch = firstDocuments.length;
if (
!Array.isArray(documentsAfterTargetSwitch) ||
documentsAfterTargetSwitch.length !== expectedDocumentsAfterTargetSwitch
)
throw new Error("A distinct target did not receive one Hopper document");
throw new Error(
"Target switching did not replace the previous Hopper document",
);
const reopenedTarget = await client.callTool(
{ name: "open_binary", arguments: { path: targetB } },
options,
@ -441,20 +455,32 @@ try {
"Hopper's bundled MCP server was running during verification",
);
}
const diagnosticCount = requireSafeDiagnostics(stderrChunks);
requireBridgeDiagnostic(stderrChunks);
const closed = await client.callTool(
{ name: "close_binary", arguments: {} },
options,
);
if (closed.isError === true) throw new Error(textValue(closed));
const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, {
primary: targetA,
secondary: targetB,
ownedProcessIds,
});
const closedSession = requireMcpResult(
await fullSessionStatus(),
"binary_session",
);
if (closedSession.open !== false)
throw new Error("The real session remained open after close_binary");
const diagnosticCount = requireSafeDiagnostics(
stderrChunks,
[
"procedure_info",
"batch_decompile",
...boundaryContracts.rejectedOperations,
],
["HopperRemoteError", "HopperCancelledError", "AnalysisCancelledError"],
);
requireBridgeDiagnostic(stderrChunks);
summary = {
toolCount: actualNames.length,
@ -463,6 +489,8 @@ try {
segmentCount: firstOverview.segment_count,
analyses: [firstAnalysis, secondAnalysis],
fixtureAnalysis,
boundaryContracts,
lifecycleAndCli,
inventoryCounts,
largeInventory,
fixtureManifest: fixtureTargets.manifestPath,
@ -491,29 +519,37 @@ try {
}
await new Promise((resolve) => setTimeout(resolve, 500));
const sessionsAfter = (await readdir("/tmp")).filter(
(name) => name.startsWith("rea-") && !sessionsBefore.has(name),
);
if (sessionsAfter.length > 0) {
throw new Error("The MCP runtime leaked a bridge session directory");
}
const afterCloseProcesses = await execFileAsync("ps", [
"-ax",
"-o",
"command=",
]);
if (
afterCloseProcesses.stdout
.split("\n")
.some((line) => line.includes("node dist/main.js"))
) {
throw new Error(
"The TypeScript MCP server remained alive after client close",
);
for (const entry of await readdir(runtimeParent, { withFileTypes: true })) {
if (
!entry.isDirectory() ||
!entry.name.startsWith("rea-") ||
sessionsBefore.has(entry.name)
)
continue;
let ownership;
try {
ownership = JSON.parse(
await readFile(join(runtimeParent, entry.name, "ownership.json"), "utf8"),
);
} catch (cause) {
if (cause.code === "ENOENT") continue;
throw cause;
}
if (ownedProcessIds.has(ownership.parent_pid))
throw new Error(
`An owned Hopper runtime leaked its directory: ${entry.name}`,
);
}
if (summary === undefined)
throw new Error("Real-Hopper verification did not produce a summary");
const completedVerifierRun = await completeVerifierRun(verifierRun);
if (
completedVerifierRun.process_lineage.status !== "verified" ||
completedVerifierRun.process_lineage.descendants.length !== 0
)
throw new Error(
"The verifier's process lineage did not prove owned child cleanup",
);
await new Promise((resolve, reject) => {
process.stdout.write(
`${JSON.stringify({ verifier_run: completedVerifierRun, ...summary, unrelatedHopperSurvived, cleanShutdown: true }, null, 2)}\n`,

View File

@ -61,19 +61,6 @@ const start = async (owned: boolean) => {
};
describe("Hopper provider lifecycle health", () => {
it("keeps a live bridge healthy after a non-owning launcher helper exits", async () => {
const { client, launcher } = await start(false);
launcher.observed.exit(0);
expect(client.operationHealth().state).toBe("idle");
expect(await client.callTool("echo", { label: "GUI still alive" })).toEqual(
{
ok: true,
value: { label: "GUI still alive" },
},
);
expect(client.operationHealth().state).toBe("idle");
});
it("settles disconnected requests without attributing the helper exit to the GUI", async () => {
const { client, launcher } = await start(false);
launcher.observed.exit(0);

View File

@ -27,22 +27,6 @@ describe("real Hopper semantic fixture resolution", () => {
).rejects.toThrow(/current_document/u);
});
it("resolves the public search value and preserves its reported name", async () => {
const procedure = await resolveFixtureProcedure(
clientWithItems([
{
address: "0x1000",
value: "_rea_entry",
},
]),
{},
"rea_entry",
normalize,
);
expect(procedure).toEqual({ address: "0x1000", name: "_rea_entry" });
});
it.each([
["empty", []],
[