From 8de25d02302e7a03c1a0c6ab19ccb459758973ca Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:49:48 +0800 Subject: [PATCH] test(mcp): cover strict compact wire contracts --- docs/mcp-schema-report.json | 515 ++++++++++++++++++++ tests/applicationWorkflowCli.test.ts | 52 +- tests/applicationWorkflowMcp.test.ts | 7 +- tests/artifactMcp.test.ts | 56 ++- tests/browserMcp.test.ts | 49 +- tests/bundleComparisonMcp.test.ts | 259 +++++----- tests/contractSnapshot.test.ts | 23 + tests/electronMcp.test.ts | 47 +- tests/enhancedTools.test.ts | 57 ++- tests/functionComparisonMcp.test.ts | 45 +- tests/integration/serverIntegration.test.ts | 95 +++- tests/investigationMcp.test.ts | 61 +-- tests/managedMcp.test.ts | 5 +- tests/officialTools.test.ts | 10 +- tests/providerSelectionMcp.test.ts | 5 +- tests/sessionMcp.test.ts | 19 +- tests/toolResult.test.ts | 32 +- 17 files changed, 1009 insertions(+), 328 deletions(-) create mode 100644 docs/mcp-schema-report.json diff --git a/docs/mcp-schema-report.json b/docs/mcp-schema-report.json new file mode 100644 index 000000000..dda37b59b --- /dev/null +++ b/docs/mcp-schema-report.json @@ -0,0 +1,515 @@ +{ + "generated_from": "@modelcontextprotocol/server 2.0.0-beta.4 wire registration", + "tool_count": 84, + "tools_list_bytes": 518457, + "budgets": { + "tools_list_bytes": 524288, + "per_tool_bytes": 65536 + }, + "tools": [ + { + "name": "address_name", + "bytes": 1133, + "input_schema_bytes": 401, + "output_schema_bytes": 313 + }, + { + "name": "analyze_function", + "bytes": 10206, + "input_schema_bytes": 2037, + "output_schema_bytes": 7692 + }, + { + "name": "analyze_javascript_application", + "bytes": 19871, + "input_schema_bytes": 2633, + "output_schema_bytes": 16597 + }, + { + "name": "analyze_swift_types", + "bytes": 1321, + "input_schema_bytes": 62, + "output_schema_bytes": 852 + }, + { + "name": "analyze_web_bundle", + "bytes": 14995, + "input_schema_bytes": 6550, + "output_schema_bytes": 7940 + }, + { + "name": "batch_decompile", + "bytes": 12102, + "input_schema_bytes": 226, + "output_schema_bytes": 11475 + }, + { + "name": "binary_overview", + "bytes": 1643, + "input_schema_bytes": 250, + "output_schema_bytes": 977 + }, + { + "name": "binary_session", + "bytes": 13235, + "input_schema_bytes": 367, + "output_schema_bytes": 12368 + }, + { + "name": "build_call_path", + "bytes": 11300, + "input_schema_bytes": 7493, + "output_schema_bytes": 3335 + }, + { + "name": "capture_process_scenario", + "bytes": 20991, + "input_schema_bytes": 10463, + "output_schema_bytes": 9774 + }, + { + "name": "capture_web_screenshot", + "bytes": 5661, + "input_schema_bytes": 771, + "output_schema_bytes": 4454 + }, + { + "name": "close_binary", + "bytes": 1073, + "input_schema_bytes": 214, + "output_schema_bytes": 379 + }, + { + "name": "comment", + "bytes": 1082, + "input_schema_bytes": 401, + "output_schema_bytes": 313 + }, + { + "name": "compare_application_versions", + "bytes": 27986, + "input_schema_bytes": 10250, + "output_schema_bytes": 17168 + }, + { + "name": "compare_artifacts", + "bytes": 4095, + "input_schema_bytes": 759, + "output_schema_bytes": 2851 + }, + { + "name": "compare_bundles", + "bytes": 4559, + "input_schema_bytes": 928, + "output_schema_bytes": 3132 + }, + { + "name": "compare_functions", + "bytes": 4828, + "input_schema_bytes": 725, + "output_schema_bytes": 3611 + }, + { + "name": "compare_process_captures", + "bytes": 3866, + "input_schema_bytes": 565, + "output_schema_bytes": 2716 + }, + { + "name": "compare_web_captures", + "bytes": 33869, + "input_schema_bytes": 31489, + "output_schema_bytes": 1940 + }, + { + "name": "compare_web_screenshots", + "bytes": 3428, + "input_schema_bytes": 1489, + "output_schema_bytes": 1546 + }, + { + "name": "correlate_static_and_runtime", + "bytes": 13332, + "input_schema_bytes": 8963, + "output_schema_bytes": 3876 + }, + { + "name": "current_address", + "bytes": 938, + "input_schema_bytes": 282, + "output_schema_bytes": 285 + }, + { + "name": "current_document", + "bytes": 842, + "input_schema_bytes": 219, + "output_schema_bytes": 285 + }, + { + "name": "current_procedure", + "bytes": 952, + "input_schema_bytes": 282, + "output_schema_bytes": 285 + }, + { + "name": "demangle_swift", + "bytes": 2276, + "input_schema_bytes": 221, + "output_schema_bytes": 1717 + }, + { + "name": "discover_webmcp_tools", + "bytes": 7256, + "input_schema_bytes": 1087, + "output_schema_bytes": 5765 + }, + { + "name": "export_evidence_bundle", + "bytes": 12103, + "input_schema_bytes": 196, + "output_schema_bytes": 11479 + }, + { + "name": "extract_artifact", + "bytes": 6414, + "input_schema_bytes": 1321, + "output_schema_bytes": 4701 + }, + { + "name": "find_changed_behavior", + "bytes": 12296, + "input_schema_bytes": 8586, + "output_schema_bytes": 3182 + }, + { + "name": "find_xrefs_to_name", + "bytes": 1346, + "input_schema_bytes": 128, + "output_schema_bytes": 763 + }, + { + "name": "get_call_graph", + "bytes": 12061, + "input_schema_bytes": 361, + "output_schema_bytes": 11228 + }, + { + "name": "get_objc_classes", + "bytes": 1141, + "input_schema_bytes": 127, + "output_schema_bytes": 606 + }, + { + "name": "get_objc_protocols", + "bytes": 1079, + "input_schema_bytes": 62, + "output_schema_bytes": 610 + }, + { + "name": "goto_address", + "bytes": 1094, + "input_schema_bytes": 424, + "output_schema_bytes": 285 + }, + { + "name": "import_evidence_bundle", + "bytes": 882, + "input_schema_bytes": 142, + "output_schema_bytes": 321 + }, + { + "name": "inline_comment", + "bytes": 1064, + "input_schema_bytes": 401, + "output_schema_bytes": 313 + }, + { + "name": "inspect_electron_page", + "bytes": 10081, + "input_schema_bytes": 1936, + "output_schema_bytes": 7685 + }, + { + "name": "inspect_macho", + "bytes": 9368, + "input_schema_bytes": 62, + "output_schema_bytes": 8933 + }, + { + "name": "inspect_managed_artifact", + "bytes": 12894, + "input_schema_bytes": 1251, + "output_schema_bytes": 11159 + }, + { + "name": "inspect_plist", + "bytes": 2883, + "input_schema_bytes": 189, + "output_schema_bytes": 2320 + }, + { + "name": "inspect_signature", + "bytes": 2946, + "input_schema_bytes": 62, + "output_schema_bytes": 2519 + }, + { + "name": "inspect_web_page", + "bytes": 23933, + "input_schema_bytes": 5042, + "output_schema_bytes": 18381 + }, + { + "name": "inventory_artifact", + "bytes": 10608, + "input_schema_bytes": 1885, + "output_schema_bytes": 8336 + }, + { + "name": "list_architectures", + "bytes": 2782, + "input_schema_bytes": 62, + "output_schema_bytes": 2387 + }, + { + "name": "list_bookmarks", + "bytes": 1091, + "input_schema_bytes": 282, + "output_schema_bytes": 437 + }, + { + "name": "list_browser_targets", + "bytes": 3072, + "input_schema_bytes": 658, + "output_schema_bytes": 1966 + }, + { + "name": "list_documents", + "bytes": 883, + "input_schema_bytes": 219, + "output_schema_bytes": 310 + }, + { + "name": "list_electron_targets", + "bytes": 2991, + "input_schema_bytes": 636, + "output_schema_bytes": 1932 + }, + { + "name": "list_names", + "bytes": 2842, + "input_schema_bytes": 594, + "output_schema_bytes": 1818 + }, + { + "name": "list_procedures", + "bytes": 2739, + "input_schema_bytes": 475, + "output_schema_bytes": 1818 + }, + { + "name": "list_segments", + "bytes": 2817, + "input_schema_bytes": 282, + "output_schema_bytes": 2128 + }, + { + "name": "list_strings", + "bytes": 2837, + "input_schema_bytes": 594, + "output_schema_bytes": 1818 + }, + { + "name": "list_unknowns", + "bytes": 4477, + "input_schema_bytes": 358, + "output_schema_bytes": 3710 + }, + { + "name": "next_address", + "bytes": 1039, + "input_schema_bytes": 401, + "output_schema_bytes": 285 + }, + { + "name": "observe_web_session", + "bytes": 6320, + "input_schema_bytes": 765, + "output_schema_bytes": 5090 + }, + { + "name": "open_binary", + "bytes": 1702, + "input_schema_bytes": 386, + "output_schema_bytes": 696 + }, + { + "name": "prev_address", + "bytes": 1045, + "input_schema_bytes": 401, + "output_schema_bytes": 285 + }, + { + "name": "procedure_address", + "bytes": 1082, + "input_schema_bytes": 383, + "output_schema_bytes": 285 + }, + { + "name": "procedure_assembly", + "bytes": 1096, + "input_schema_bytes": 383, + "output_schema_bytes": 285 + }, + { + "name": "procedure_callees", + "bytes": 1121, + "input_schema_bytes": 383, + "output_schema_bytes": 310 + }, + { + "name": "procedure_callers", + "bytes": 1085, + "input_schema_bytes": 383, + "output_schema_bytes": 310 + }, + { + "name": "procedure_info", + "bytes": 1982, + "input_schema_bytes": 383, + "output_schema_bytes": 1216 + }, + { + "name": "procedure_pseudo_code", + "bytes": 1172, + "input_schema_bytes": 383, + "output_schema_bytes": 313 + }, + { + "name": "procedure_references", + "bytes": 4319, + "input_schema_bytes": 798, + "output_schema_bytes": 3054 + }, + { + "name": "reconcile_javascript_runtime", + "bytes": 30201, + "input_schema_bytes": 9566, + "output_schema_bytes": 20110 + }, + { + "name": "record_unknown", + "bytes": 6725, + "input_schema_bytes": 2646, + "output_schema_bytes": 3685 + }, + { + "name": "resolve_containing_procedure", + "bytes": 2185, + "input_schema_bytes": 424, + "output_schema_bytes": 1280 + }, + { + "name": "run_controlled_replay", + "bytes": 21072, + "input_schema_bytes": 6098, + "output_schema_bytes": 14387 + }, + { + "name": "search_procedures", + "bytes": 2238, + "input_schema_bytes": 812, + "output_schema_bytes": 964 + }, + { + "name": "search_strings", + "bytes": 2247, + "input_schema_bytes": 812, + "output_schema_bytes": 964 + }, + { + "name": "set_address_name", + "bytes": 1141, + "input_schema_bytes": 478, + "output_schema_bytes": 286 + }, + { + "name": "set_addresses_names", + "bytes": 1175, + "input_schema_bytes": 427, + "output_schema_bytes": 361 + }, + { + "name": "set_bookmark", + "bytes": 1118, + "input_schema_bytes": 471, + "output_schema_bytes": 286 + }, + { + "name": "set_comment", + "bytes": 1148, + "input_schema_bytes": 487, + "output_schema_bytes": 286 + }, + { + "name": "set_current_document", + "bytes": 1003, + "input_schema_bytes": 298, + "output_schema_bytes": 285 + }, + { + "name": "set_inline_comment", + "bytes": 1142, + "input_schema_bytes": 487, + "output_schema_bytes": 286 + }, + { + "name": "swift_classes", + "bytes": 1135, + "input_schema_bytes": 127, + "output_schema_bytes": 606 + }, + { + "name": "trace_application_feature", + "bytes": 27741, + "input_schema_bytes": 9241, + "output_schema_bytes": 17969 + }, + { + "name": "trace_feature", + "bytes": 3340, + "input_schema_bytes": 555, + "output_schema_bytes": 2299 + }, + { + "name": "unset_bookmark", + "bytes": 1055, + "input_schema_bytes": 424, + "output_schema_bytes": 286 + }, + { + "name": "update_unknown", + "bytes": 7884, + "input_schema_bytes": 3810, + "output_schema_bytes": 3685 + }, + { + "name": "verify_reconstruction", + "bytes": 6692, + "input_schema_bytes": 2688, + "output_schema_bytes": 3510 + }, + { + "name": "verify_unknown_resolution", + "bytes": 4481, + "input_schema_bytes": 177, + "output_schema_bytes": 3866 + }, + { + "name": "xrefs", + "bytes": 1112, + "input_schema_bytes": 401, + "output_schema_bytes": 310 + } + ] +} diff --git a/tests/applicationWorkflowCli.test.ts b/tests/applicationWorkflowCli.test.ts index 2c51385f8..44220f044 100644 --- a/tests/applicationWorkflowCli.test.ts +++ b/tests/applicationWorkflowCli.test.ts @@ -55,13 +55,53 @@ describe("application workflow CLI parity", () => { }, }); }, 20_000); + + it("returns safe actionable JSON validation details", async () => { + const malformed = await runCli([ + "trace-application-feature", + "{not-json", + "--json", + ]); + expect(malformed).toMatchObject({ + code: "invalid_request", + retryable: true, + details: { + issues: [{ path: [], reason: "invalid_format", expected: "JSON" }], + }, + }); + expect(JSON.stringify(malformed)).not.toContain("not-json"); + + const missing = await runCli(["trace-application-feature", "{}", "--json"]); + expect(missing).toMatchObject({ + code: "invalid_request", + details: { + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["application"], + reason: "missing_argument", + }), + ]), + }, + }); + }); }); const runCli = async (arguments_: readonly string[]): Promise => { - const { stdout } = await execute( - process.execPath, - ["scripts/rea.mjs", ...arguments_], - { cwd: process.cwd(), env: process.env, maxBuffer: 16 * 1_024 * 1_024 }, - ); - return JSON.parse(stdout); + try { + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", ...arguments_], + { cwd: process.cwd(), env: process.env, maxBuffer: 16 * 1_024 * 1_024 }, + ); + return JSON.parse(stdout); + } catch (cause: unknown) { + if ( + typeof cause === "object" && + cause !== null && + "stdout" in cause && + typeof cause.stdout === "string" + ) + return JSON.parse(cause.stdout); + throw cause; + } }; diff --git a/tests/applicationWorkflowMcp.test.ts b/tests/applicationWorkflowMcp.test.ts index 7fcbecfde..913ec293e 100644 --- a/tests/applicationWorkflowMcp.test.ts +++ b/tests/applicationWorkflowMcp.test.ts @@ -37,9 +37,7 @@ describe("application workflow MCP parity", () => { }); expect(traced.isError).not.toBe(true); expect(traced.structuredContent).toMatchObject({ - operation: "trace_application_feature", - provider: { id: "rea-javascript-application-workflows" }, - normalized_result: { + result: { schema_version: 1, coverage: { status: expect.any(String) }, }, @@ -54,8 +52,7 @@ describe("application workflow MCP parity", () => { }); expect(compared.isError).not.toBe(true); expect(compared.structuredContent).toMatchObject({ - operation: "compare_application_versions", - normalized_result: { + result: { schema_version: 1, summary: { unknown: expect.any(Number) }, coverage: { status: expect.any(String) }, diff --git a/tests/artifactMcp.test.ts b/tests/artifactMcp.test.ts index 5259c95b2..bf48295bd 100644 --- a/tests/artifactMcp.test.ts +++ b/tests/artifactMcp.test.ts @@ -12,7 +12,6 @@ import { BinarySession } from "../src/application/BinarySession.js"; import { ArtifactProvider } from "../src/artifacts/ArtifactProvider.js"; import { ARTIFACT_COMPARISON_EXAMPLE } from "../src/contracts/artifactComparisonExample.js"; import { evidenceBundleSchema } from "../src/domain/evidenceBundle.js"; -import { parseEvidence } from "../src/domain/evidence.js"; import { createServer } from "../src/server/createServer.js"; import { observed } from "./fixtures/analysisExecution.js"; @@ -148,7 +147,9 @@ describe("artifact graph MCP integration", () => { }, }); expect(result.isError).not.toBe(true); - const evidence = parseEvidence(result.structuredContent); + const compact = compactResult(result.structuredContent); + const evidence = session.evidenceById(compact.evidence_id); + if (evidence === undefined) throw new Error("missing inventory Evidence"); const inventory = z .object({ occurrences: z.object({ @@ -165,7 +166,7 @@ describe("artifact graph MCP integration", () => { }), ), }) - .parse(evidence.normalized_result); + .parse(compact.result); expect(inventory.integrity_contradictions).toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -197,16 +198,12 @@ describe("artifact graph MCP integration", () => { const compared = await client.callTool({ name: "compare_artifacts", arguments: { - left: evidence, - right: evidence, + left_evidence_ids: [evidence.evidence_id], + right_evidence_ids: [evidence.evidence_id], }, }); expect(compared.isError).not.toBe(true); - const comparison = parseEvidence( - z.object({ result: z.unknown() }).parse(compared.structuredContent) - .result, - ); - expect(comparison.normalized_result).toMatchObject({ + expect(compactResult(compared.structuredContent).result).toMatchObject({ status: "contradiction", summary: { contradiction: 2 }, changes: { @@ -333,13 +330,15 @@ describe("artifact graph MCP integration", () => { arguments: {}, }); expect(inventory.isError).not.toBe(true); - const evidence = parseEvidence(inventory.structuredContent); + const inventoryResult = compactResult(inventory.structuredContent); + const evidence = session.evidenceById(inventoryResult.evidence_id); + if (evidence === undefined) throw new Error("missing inventory Evidence"); expect(evidence).toMatchObject({ provider: { id: "rea-artifact-graph" }, subject: { format: "ipa" }, - normalized_result: { - manifest: { root_format: "ipa" }, - }, + }); + expect(inventoryResult.result).toMatchObject({ + manifest: { root_format: "ipa" }, }); const openedChanged = await client.callTool({ name: "open_binary", @@ -350,24 +349,22 @@ describe("artifact graph MCP integration", () => { name: "inventory_artifact", arguments: {}, }); - const changedEvidence = parseEvidence(changedInventory.structuredContent); + const changedResult = compactResult(changedInventory.structuredContent); + const changedEvidence = session.evidenceById(changedResult.evidence_id); + if (changedEvidence === undefined) + throw new Error("missing changed Evidence"); const compared = await client.callTool({ name: "compare_artifacts", arguments: { - left: evidence, - right: changedEvidence, + left_evidence_ids: [evidence.evidence_id], + right_evidence_ids: [changedEvidence.evidence_id], unknown_registry_approved: true, }, }); expect(compared.isError).not.toBe(true); - const comparisonEvidence = parseEvidence( - z.object({ result: z.unknown() }).parse(compared.structuredContent) - .result, - ); - expect(comparisonEvidence).toMatchObject({ - provider: { id: "rea-artifact-comparison" }, - evidence_links: [evidence.evidence_id, changedEvidence.evidence_id], - normalized_result: { status: "changed" }, + expect(compactResult(compared.structuredContent)).toMatchObject({ + result: { status: "changed" }, + evidence_id: expect.stringMatching(/^ev_/u), }); const unknowns = await client.callTool({ name: "list_unknowns", @@ -398,3 +395,12 @@ describe("artifact graph MCP integration", () => { } }); }); + +const compactResult = (value: unknown) => + z + .object({ + result: z.unknown(), + evidence_id: z.string(), + evidence_uri: z.string(), + }) + .parse(value); diff --git a/tests/browserMcp.test.ts b/tests/browserMcp.test.ts index 769c71c92..41af7a0fc 100644 --- a/tests/browserMcp.test.ts +++ b/tests/browserMcp.test.ts @@ -77,12 +77,7 @@ describe("browser MCP tools", () => { }); expect(listed.isError).not.toBe(true); expect(listed.structuredContent).toMatchObject({ - operation: "list_browser_targets", - authority: "external-service", - confidence: "observed", - subject: null, - provider: { id: "rea-cdp-browser", version: "2" }, - normalized_result: { + result: { targets: { items: [{ target_id: "allowed-page" }] }, }, }); @@ -105,8 +100,7 @@ describe("browser MCP tools", () => { }); expect(inspected.isError).not.toBe(true); expect(inspected.structuredContent).toMatchObject({ - operation: "inspect_web_page", - normalized_result: { + result: { target: { target_id: "allowed-page" }, console: { prior_activity_available: false }, network: { @@ -124,14 +118,14 @@ describe("browser MCP tools", () => { arguments: { static_layers: JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE.static_layers, - runtime_observations: [inspected.structuredContent], + runtime_observations: [ + evidenceFor(connected.session, inspected.structuredContent), + ], }, }); expect(reconciled.isError).not.toBe(true); expect(reconciled.structuredContent).toMatchObject({ - operation: "reconcile_javascript_runtime", - provider: { id: "rea-javascript-runtime-reconciliation" }, - normalized_result: { summary: { runtime_scripts: 1 } }, + result: { summary: { runtime_scripts: 1 } }, }); expect(JSON.stringify(reconciled.structuredContent)).not.toContain( "source-secret", @@ -149,8 +143,7 @@ describe("browser MCP tools", () => { }); expect(analyzed.isError).not.toBe(true); expect(analyzed.structuredContent).toMatchObject({ - operation: "analyze_web_bundle", - normalized_result: { + result: { capture: { scripts_analyzed: 1 }, observations: { source_maps: { status: "not_requested" } }, }, @@ -167,8 +160,7 @@ describe("browser MCP tools", () => { }); expect(observedSession.isError).not.toBe(true); expect(observedSession.structuredContent).toMatchObject({ - operation: "observe_web_session", - normalized_result: { + result: { window: { end_reason: "window_elapsed" }, timeline: expect.arrayContaining([ expect.objectContaining({ type: "same_origin_reload" }), @@ -187,8 +179,7 @@ describe("browser MCP tools", () => { }); expect(webMcp.isError).not.toBe(true); expect(webMcp.structuredContent).toMatchObject({ - operation: "discover_webmcp_tools", - normalized_result: { + result: { tools: { items: [expect.objectContaining({ name: "search_orders" })], }, @@ -204,8 +195,7 @@ describe("browser MCP tools", () => { }); expect(compared.isError).not.toBe(true); expect(compared.structuredContent).toMatchObject({ - operation: "compare_web_captures", - normalized_result: { overall_status: "unknown" }, + result: { overall_status: "unknown" }, }); const screenshot = await connected.client.callTool({ name: "capture_web_screenshot", @@ -228,8 +218,7 @@ describe("browser MCP tools", () => { }); expect(visual.isError).not.toBe(true); expect(visual.structuredContent).toMatchObject({ - operation: "compare_web_screenshots", - normalized_result: { status: "identical", changed_pixels: 0 }, + result: { status: "identical", changed_pixels: 0 }, }); const evidenceId = evidenceIdOf(inspected.structuredContent); const evidence = await connected.client.readResource({ @@ -297,7 +286,7 @@ const connectBrowser = async (browser: FakeCdpBrowser) => { resources.push(client, server, session); await server.connect(serverTransport); await client.connect(clientTransport); - return { client }; + return { client, session }; }; const evidenceIdOf = (value: unknown): string => { @@ -312,13 +301,15 @@ const evidenceIdOf = (value: unknown): string => { }; const normalizedResultOf = (value: unknown): unknown => { - if ( - typeof value !== "object" || - value === null || - !("normalized_result" in value) - ) + if (typeof value !== "object" || value === null || !("result" in value)) throw new TypeError("Missing normalized browser result"); - return value.normalized_result; + return value.result; +}; + +const evidenceFor = (session: BinarySession, value: unknown) => { + const evidence = session.evidenceById(evidenceIdOf(value)); + if (evidence === undefined) throw new TypeError("Missing session Evidence"); + return evidence; }; const artifactOf = (value: unknown): unknown => { diff --git a/tests/bundleComparisonMcp.test.ts b/tests/bundleComparisonMcp.test.ts index 767d5210b..8506a7f24 100644 --- a/tests/bundleComparisonMcp.test.ts +++ b/tests/bundleComparisonMcp.test.ts @@ -1,14 +1,26 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { describe, expect, it } from "vitest"; -import { z } from "zod"; +import { afterEach, describe, expect, it } from "vitest"; import { BinarySession } from "../src/application/BinarySession.js"; -import { createEvidence, parseEvidence } from "../src/domain/evidence.js"; -import { createEvidenceBundle } from "../src/domain/evidenceBundle.js"; -import { createResidualUnknown } from "../src/domain/residualUnknown.js"; +import { createEvidence } from "../src/domain/evidence.js"; +import { + createEvidenceBundle, + serializeEvidenceBundle, +} from "../src/domain/evidenceBundle.js"; import { createServer } from "../src/server/createServer.js"; import { observed } from "./fixtures/analysisExecution.js"; +const roots: string[] = []; +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + const sourceEvidence = (label: string) => createEvidence( undefined, @@ -23,124 +35,30 @@ const sourceEvidence = (label: string) => ); describe("bundle comparison MCP integration", () => { - it("rejects altered records that reuse session Evidence IDs", async () => { - const session = new BinarySession(() => ({ - health: () => Promise.resolve(), - execute: () => Promise.resolve(observed(null)), - close: () => Promise.resolve(), - })); - const leftRecord = sourceEvidence("left-authority"); - const rightRecord = sourceEvidence("right-authority"); - session.recordEvidence(leftRecord); - session.recordEvidence(rightRecord); - const altered = { - ...leftRecord, - limitations: ["caller altered this record"], - }; - const server = createServer(session, session); - const client = new Client({ name: "bundle-authority-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const result = await client.callTool({ - name: "compare_bundles", - arguments: { - left: createEvidenceBundle([altered]), - right: createEvidenceBundle([rightRecord]), - }, - }); - expect(result.isError).toBe(true); - expect(session.exportEvidenceBundle().records).toHaveLength(2); - } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - } - }); - - it("rejects unknown histories that are not owned by the session", async () => { - const session = new BinarySession(() => ({ - health: () => Promise.resolve(), - execute: () => Promise.resolve(observed(null)), - close: () => Promise.resolve(), - })); - const leftRecord = sourceEvidence("left-unowned-unknown"); - const rightRecord = sourceEvidence("right-unowned-unknown"); - expect(session.recordEvidence(leftRecord).ok).toBe(true); - expect(session.recordEvidence(rightRecord).ok).toBe(true); - const unknown = createResidualUnknown( - { - approved: true, - question: "Unowned history?", - severity: "high", - domain: "bundle-comparison-test", - supporting_evidence_ids: [leftRecord.evidence_id], - contradicting_evidence_ids: [], - required_authority: "shipped-artifact", - required_confidence: "observed", - required_environment: null, - recommended_probes: [], - relationships: [], - }, - leftRecord.evidence_id, - null, - ); - const server = createServer(session, session); - const client = new Client({ name: "bundle-unknown-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const result = await client.callTool({ - name: "compare_bundles", - arguments: { - left: createEvidenceBundle([leftRecord], [unknown]), - right: createEvidenceBundle([rightRecord]), - }, - }); - expect(result.isError).toBe(true); - expect(session.exportEvidenceBundle()).toMatchObject({ - records: [leftRecord, rightRecord].sort((left, right) => - left.evidence_id.localeCompare(right.evidence_id), - ), - unknowns: [], - }); - } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - } - }); - - it("records a digest-anchored comparison linked to every source record", async () => { - const session = new BinarySession(() => ({ - health: () => Promise.resolve(), - execute: () => Promise.resolve(observed(null)), - close: () => Promise.resolve(), - })); + it("reads two approved bounded bundle files and records a compact result", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-bundle-mcp-")); + roots.push(root); + const leftPath = join(root, "left.json"); + const rightPath = join(root, "right.json"); const leftRecord = sourceEvidence("left"); const rightRecord = sourceEvidence("right"); - expect(session.recordEvidence(leftRecord).ok).toBe(true); - expect(session.recordEvidence(rightRecord).ok).toBe(true); - const server = createServer(session, session); - const client = new Client({ name: "bundle-comparison-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + await Promise.all([ + writeFile( + leftPath, + serializeEvidenceBundle(createEvidenceBundle([leftRecord])), + ), + writeFile( + rightPath, + serializeEvidenceBundle(createEvidenceBundle([rightRecord])), + ), + ]); + const connected = await connect(root); try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const result = await client.callTool({ + const result = await connected.client.callTool({ name: "compare_bundles", arguments: { - left: createEvidenceBundle([leftRecord]), - right: createEvidenceBundle([rightRecord]), + left_bundle_path: leftPath, + right_bundle_path: rightPath, record_pairs: [ { left_evidence_id: leftRecord.evidence_id, @@ -150,24 +68,101 @@ describe("bundle comparison MCP integration", () => { }, }); expect(result.isError).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(result.structuredContent) - .result, - ); - expect(evidence).toMatchObject({ - provider: { id: "rea-bundle-comparison" }, - evidence_links: [ - leftRecord.evidence_id, - rightRecord.evidence_id, - ].sort(), - normalized_result: { status: "changed" }, + expect(result.structuredContent).toMatchObject({ + result: { status: "changed" }, + evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u), + evidence_uri: expect.stringMatching(/^rea:\/\/evidence\/ev_/u), }); } finally { + await connected.close(); + } + }); + + it("rejects a bundle path outside the approved root", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-bundle-root-")); + const outside = await mkdtemp(join(tmpdir(), "rea-bundle-outside-")); + roots.push(root, outside); + const leftPath = join(root, "left.json"); + const rightPath = join(outside, "right.json"); + const encoded = serializeEvidenceBundle(createEvidenceBundle([])); + await Promise.all([ + writeFile(leftPath, encoded), + writeFile(rightPath, encoded), + ]); + const connected = await connect(root); + try { + const result = await connected.client.callTool({ + name: "compare_bundles", + arguments: { + left_bundle_path: leftPath, + right_bundle_path: rightPath, + }, + }); + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: "outside_approved_root" } }, + }); + } finally { + await connected.close(); + } + }); + + it("rejects an oversized approved bundle", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-bundle-size-")); + roots.push(root); + const leftPath = join(root, "left.json"); + const rightPath = join(root, "right.json"); + await Promise.all([ + writeFile(leftPath, " ".repeat(2_000)), + writeFile(rightPath, serializeEvidenceBundle(createEvidenceBundle([]))), + ]); + const connected = await connect(root, 1_024); + try { + const result = await connected.client.callTool({ + name: "compare_bundles", + arguments: { + left_bundle_path: leftPath, + right_bundle_path: rightPath, + }, + }); + expect(result).toMatchObject({ + isError: true, + structuredContent: { error: { code: "truncated" } }, + }); + } finally { + await connected.close(); + } + }); +}); + +const connect = async (root: string, maxBytes = 1024 * 1024) => { + const session = new BinarySession(() => ({ + health: () => Promise.resolve(), + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(), + })); + const server = createServer(session, session, { + evidenceFilePolicy: { + roots: [root], + maxBytes, + maxDepth: 68, + maxStringLength: 1024 * 1024, + maxNodes: 100_000, + }, + }); + const client = new Client({ name: "bundle-comparison-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { + client, + close: async () => { await Promise.allSettled([ client.close(), server.close(), session.close(), ]); - } - }); -}); + }, + }; +}; diff --git a/tests/contractSnapshot.test.ts b/tests/contractSnapshot.test.ts index 1127d8526..ce2adc529 100644 --- a/tests/contractSnapshot.test.ts +++ b/tests/contractSnapshot.test.ts @@ -20,6 +20,10 @@ import { officialOutputSchemas, sessionOutputSchemas, } from "../src/contracts/toolOutputSchemas.js"; +import { + annotationsFromEffects, + TOOL_EFFECTS, +} from "../src/contracts/toolEffects.js"; const convertContractJsonSchema = (schema: z.ZodType) => z.toJSONSchema(schema, { @@ -72,6 +76,10 @@ describe("tool contract surface", () => { const outputSchema = contractJsonSchema(contract.outputSchema); expect(inputSchema.type).toBe("object"); expect(outputSchema.type).toBe("object"); + expect(contract.title.length).toBeGreaterThan(2); + expect(contract.annotations).toEqual( + annotationsFromEffects(contract.effects), + ); expect(typeof contract.annotations.idempotentHint).toBe("boolean"); expect(typeof contract.annotations.openWorldHint).toBe("boolean"); expect(typeof contract.annotations.readOnlyHint).toBe("boolean"); @@ -86,6 +94,21 @@ describe("tool contract surface", () => { } }); + it("audits every tool effect explicitly with no heuristic fallback", () => { + const names = [ + ...OFFICIAL_TOOL_CONTRACTS, + ...ENHANCED_TOOL_CONTRACTS, + ...NATIVE_TOOL_CONTRACTS, + ...ARTIFACT_TOOL_CONTRACTS, + ...MANAGED_TOOL_CONTRACTS, + ...BROWSER_TOOL_CONTRACTS, + ...ELECTRON_TOOL_CONTRACTS, + ...APPLICATION_TOOL_CONTRACTS, + ...SESSION_TOOL_CONTRACTS, + ].map(({ name }) => name); + expect(Object.keys(TOOL_EFFECTS).sort()).toEqual(names.sort()); + }); + it("keeps exactly eighteen additive session contracts", () => { expect( SESSION_TOOL_CONTRACTS.map(({ name, kind }) => ({ name, kind })), diff --git a/tests/electronMcp.test.ts b/tests/electronMcp.test.ts index 9a2b52ff9..91864a7b8 100644 --- a/tests/electronMcp.test.ts +++ b/tests/electronMcp.test.ts @@ -91,9 +91,7 @@ describe("Electron MCP tools", () => { }); expect(listed.isError).not.toBe(true); expect(listed.structuredContent).toMatchObject({ - operation: "list_electron_targets", - provider: { id: "rea-cdp-electron" }, - normalized_result: { + result: { targets: { items: [{ target_id: "electron-page" }] }, }, }); @@ -118,30 +116,32 @@ describe("Electron MCP tools", () => { name: "reconcile_javascript_runtime", arguments: { static_layers: [ - { role: "application", analysis: analyzed.structuredContent }, + { + role: "application", + analysis: evidenceFor(session, analyzed.structuredContent), + }, + ], + runtime_observations: [ + evidenceFor(session, inspected.structuredContent), ], - runtime_observations: [inspected.structuredContent], }, }); expect(reconciled.isError).not.toBe(true); expect(reconciled.structuredContent).toMatchObject({ - operation: "reconcile_javascript_runtime", - provider: { id: "rea-javascript-runtime-reconciliation" }, - normalized_result: { + result: { summary: { runtime_scripts: 1, matched: expect.any(Number) }, source_map_authority: { used_for_primary_matching: false }, }, }); expect(inspected.isError).not.toBe(true); expect(inspected.structuredContent).toMatchObject({ - operation: "inspect_electron_page", - normalized_result: { - target: { file_path: join(root, "index.html") }, + result: { + target: { file_path: expect.stringMatching(/\/index\.html$/u) }, scripts: { items: [ expect.objectContaining({ frame_id: "frame-main", - file_path: join(root, "app.js"), + file_path: expect.stringMatching(/\/app\.js$/u), }), ], }, @@ -149,7 +149,7 @@ describe("Electron MCP tools", () => { expect.objectContaining({ target_id: "electron-worker", opener_target_id: "electron-page", - file_path: join(root, "worker.js"), + file_path: expect.stringMatching(/\/worker\.js$/u), }), ], }, @@ -170,7 +170,7 @@ describe("Electron MCP tools", () => { expect(denied.structuredContent).toMatchObject({ error: { code: "permission_required", - details: { missing: { roots: [outside] } }, + details: { missing: { roots: [expect.stringMatching(/outside-/u)] } }, }, }); expect(browser.commands).toHaveLength(commandsBeforeDenial); @@ -214,11 +214,9 @@ describe("Electron MCP tools", () => { expect(analyzed.isError).not.toBe(true); expect(analyzed.structuredContent).toMatchObject({ - operation: "analyze_javascript_application", - provider: { id: "rea-javascript-application" }, - normalized_result: { + result: { schema_version: 1, - input_path: root, + input_path: expect.stringMatching(/rea-electron-static-mcp-/u), summary: { browser_windows: 3, context_bridge_apis: 2, @@ -228,4 +226,17 @@ describe("Electron MCP tools", () => { }, }); }); + + const evidenceFor = (session: BinarySession, value: unknown) => { + if ( + typeof value !== "object" || + value === null || + !("evidence_id" in value) || + typeof value.evidence_id !== "string" + ) + throw new TypeError("Missing compact Evidence ID"); + const evidence = session.evidenceById(value.evidence_id); + if (evidence === undefined) throw new TypeError("Missing session Evidence"); + return evidence; + }; }); diff --git a/tests/enhancedTools.test.ts b/tests/enhancedTools.test.ts index 70ab4c67c..06dc20cf7 100644 --- a/tests/enhancedTools.test.ts +++ b/tests/enhancedTools.test.ts @@ -174,6 +174,14 @@ const jsonResult = (result: CallToolResult): JsonValue => { ) { return structured.data.normalized_result ?? null; } + if ( + typeof structured.data === "object" && + structured.data !== null && + !Array.isArray(structured.data) && + "evidence_id" in structured.data && + "result" in structured.data + ) + return structured.data.result ?? null; const text = result.content.find((item) => item.type === "text"); if (text?.type !== "text") throw new Error("Tool result omitted text content"); @@ -567,7 +575,7 @@ describe("enhanced MCP tools", () => { expect(result.isError).toBe(true); const text = result.content.find((item) => item.type === "text"); expect(text?.type === "text" ? text.text : "").toBe( - "Analysis returned an unreadable result. Retry once; if it continues, run `rea doctor`.", + JSON.stringify(result.structuredContent), ); }); @@ -603,6 +611,47 @@ describe("enhanced MCP tools", () => { expect(JSON.stringify(result)).toContain("operation budget"); }); + it("returns correction-only structured validation issues", async () => { + const client = await connect(); + const misspelled = await client.callTool({ + name: "trace_feature", + arguments: { query: "needle", max_operatons: 1 }, + }); + expect(misspelled).toMatchObject({ + isError: true, + structuredContent: { + error: { + code: "invalid_request", + retryable: true, + details: { + issues: [{ path: ["max_operatons"], reason: "unknown_argument" }], + }, + }, + }, + }); + const bounded = await client.callTool({ + name: "trace_feature", + arguments: { query: "needle", max_operations: 101 }, + }); + expect(bounded.structuredContent).toMatchObject({ + error: { + details: { + issues: [ + { + path: ["max_operations"], + reason: "out_of_range", + maximum: 100, + }, + ], + }, + }, + }); + expect(bounded.content[0]).toEqual({ + type: "text", + text: JSON.stringify(bounded.structuredContent), + }); + }); + it("returns a typed tool error for malformed Hopper boundary values", async () => { const client = await connect({ execute: () => Promise.resolve(ok(["not", "a", "procedure", "map"])), @@ -614,7 +663,7 @@ describe("enhanced MCP tools", () => { expect(result.isError).toBe(true); const text = result.content.find((item) => item.type === "text"); expect(text?.type === "text" ? text.text : "").toBe( - "Analysis returned an unreadable result. Retry once; if it continues, run `rea doctor`.", + JSON.stringify(result.structuredContent), ); }); @@ -635,7 +684,7 @@ describe("enhanced MCP tools", () => { expect(result.isError).toBe(true); const text = result.content.find((item) => item.type === "text"); expect(text?.type === "text" ? text.text : "").toBe( - "Analysis returned an unreadable result. Retry once; if it continues, run `rea doctor`.", + JSON.stringify(result.structuredContent), ); }); @@ -671,7 +720,7 @@ describe("enhanced MCP tools", () => { expect(result.isError).toBe(true); const text = result.content.find((item) => item.type === "text"); expect(text?.type === "text" ? text.text : "").toBe( - "Analysis returned an unreadable result. Retry once; if it continues, run `rea doctor`.", + JSON.stringify(result.structuredContent), ); }); diff --git a/tests/functionComparisonMcp.test.ts b/tests/functionComparisonMcp.test.ts index 4eec8b1f7..c84a67feb 100644 --- a/tests/functionComparisonMcp.test.ts +++ b/tests/functionComparisonMcp.test.ts @@ -1,10 +1,9 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { describe, expect, it } from "vitest"; -import { z } from "zod"; import { BinarySession } from "../src/application/BinarySession.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../src/contracts/functionComparisonExample.js"; -import { parseEvidence } from "../src/domain/evidence.js"; +import { createEvidence } from "../src/domain/evidence.js"; import { createServer } from "../src/server/createServer.js"; import { observed } from "./fixtures/analysisExecution.js"; @@ -34,22 +33,16 @@ describe("function comparison MCP integration", () => { const result = await client.callTool({ name: "compare_functions", arguments: { - left: FUNCTION_COMPARISON_EXAMPLE.left, - right: FUNCTION_COMPARISON_EXAMPLE.right, + left_evidence_ids: [FUNCTION_COMPARISON_EXAMPLE.left.evidence_id], + right_evidence_ids: [FUNCTION_COMPARISON_EXAMPLE.right.evidence_id], unknown_registry_approved: true, }, }); expect(result.isError).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(result.structuredContent) - .result, - ); - expect(evidence).toMatchObject({ - provider: { id: "rea-function-comparison" }, - evidence_links: [ - FUNCTION_COMPARISON_EXAMPLE.left.evidence_id, - FUNCTION_COMPARISON_EXAMPLE.right.evidence_id, - ], + expect(result.structuredContent).toMatchObject({ + result: { status: expect.any(String) }, + evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u), + evidence_uri: expect.stringMatching(/^rea:\/\/evidence\/ev_/u), }); const unknowns = await client.callTool({ name: "list_unknowns", @@ -93,8 +86,8 @@ describe("function comparison MCP integration", () => { const result = await client.callTool({ name: "compare_functions", arguments: { - left: FUNCTION_COMPARISON_EXAMPLE.left, - right: FUNCTION_COMPARISON_EXAMPLE.right, + left_evidence_ids: [FUNCTION_COMPARISON_EXAMPLE.left.evidence_id], + right_evidence_ids: [FUNCTION_COMPARISON_EXAMPLE.right.evidence_id], }, }); expect(result.isError).toBe(true); @@ -108,13 +101,22 @@ describe("function comparison MCP integration", () => { } }); - it("rejects altered payloads that reuse session Evidence IDs", async () => { + it("rejects Evidence IDs with the wrong operation", async () => { const session = new BinarySession(() => ({ health: () => Promise.resolve(), execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(), })); - session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.left); + const wrong = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "inventory_artifact", + parameters: {}, + result: {}, + }, + ); + session.recordEvidence(wrong); session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.right); const server = createServer(session, session); const client = new Client({ @@ -129,11 +131,8 @@ describe("function comparison MCP integration", () => { const result = await client.callTool({ name: "compare_functions", arguments: { - left: { - ...FUNCTION_COMPARISON_EXAMPLE.left, - limitations: ["caller altered this record"], - }, - right: FUNCTION_COMPARISON_EXAMPLE.right, + left_evidence_ids: [wrong.evidence_id], + right_evidence_ids: [FUNCTION_COMPARISON_EXAMPLE.right.evidence_id], }, }); expect(result.isError).toBe(true); diff --git a/tests/integration/serverIntegration.test.ts b/tests/integration/serverIntegration.test.ts index 21fb6dcf6..3c17e4ef4 100644 --- a/tests/integration/serverIntegration.test.ts +++ b/tests/integration/serverIntegration.test.ts @@ -61,6 +61,59 @@ const structured = (result: CallToolResult): Record => { return Object.fromEntries(Object.entries(result.structuredContent)); }; +const objectEntries = (value: unknown): readonly [string, unknown][] => + typeof value === "object" && value !== null && !Array.isArray(value) + ? Object.entries(value) + : []; + +const referencedSchema = (schema: unknown, root: unknown): unknown => { + const object = Object.fromEntries(objectEntries(schema)); + const reference = object.$ref; + if (typeof reference !== "string" || !reference.startsWith("#/$defs/")) + return schema; + return Object.fromEntries(objectEntries(root)).$defs instanceof Object + ? Object.fromEntries( + objectEntries(Object.fromEntries(objectEntries(root)).$defs), + )[reference.slice(8)] + : schema; +}; + +const assertDescribedStrictObjects = ( + schema: unknown, + root = schema, + path = "$", +): void => { + for (const [key, value] of objectEntries(schema)) { + if (key === "properties") { + const properties = objectEntries(value); + for (const [property, propertySchema] of properties) { + const resolved = referencedSchema(propertySchema, root); + const propertyObject = Object.fromEntries( + objectEntries(propertySchema), + ); + expect( + propertyObject.description ?? + Object.fromEntries(objectEntries(resolved)).description, + `${path}.properties.${property}`, + ).toEqual(expect.any(String)); + assertDescribedStrictObjects( + propertySchema, + root, + `${path}.properties.${property}`, + ); + } + expect( + Object.fromEntries(objectEntries(schema)).additionalProperties, + ).toBe(false); + continue; + } + assertDescribedStrictObjects(value, root, `${path}.${key}`); + } + if (Array.isArray(schema)) + for (const [index, value] of schema.entries()) + assertDescribedStrictObjects(value, root, `${path}[${String(index)}]`); +}; + describe("full MCP integration with multi-tool sequences", () => { it("executes a realistic workflow: list methods, decompile selected, get xrefs", async () => { const client = await connect({ @@ -98,10 +151,7 @@ describe("full MCP integration with multi-tool sequences", () => { }); expect(listResult.isError).not.toBe(true); expect(structured(listResult)).toMatchObject({ - subject: null, - operation: "list_procedures", - provider: { id: "fixture", version: "1" }, - normalized_result: { + result: { items: [ { address: "0x1000", value: "main" }, { address: "0x2000", value: "helper" }, @@ -116,9 +166,7 @@ describe("full MCP integration with multi-tool sequences", () => { arguments: { procedure: "0x1000" }, }); expect(structured(decompileResult)).toMatchObject({ - operation: "procedure_pseudo_code", - parameters: { document: null, procedure: "0x1000" }, - normalized_result: "pseudo for 0x1000", + result: "pseudo for 0x1000", }); const xrefResult = await client.callTool({ @@ -126,8 +174,7 @@ describe("full MCP integration with multi-tool sequences", () => { arguments: {}, }); expect(structured(xrefResult)).toMatchObject({ - operation: "xrefs", - normalized_result: ["0x1000"], + result: ["0x1000"], }); }); @@ -161,6 +208,30 @@ describe("full MCP integration with multi-tool sequences", () => { expect(names).toContain("binary_session"); expect(names).toContain("binary_overview"); expect(names).toContain("batch_decompile"); + + expect( + Buffer.byteLength(JSON.stringify(listed), "utf8"), + ).toBeLessThanOrEqual(512 * 1024); + const contracts = new Map( + TOOL_CONTRACTS.map((contract) => [contract.name, contract]), + ); + for (const tool of listed.tools) { + expect( + Buffer.byteLength(JSON.stringify(tool), "utf8"), + tool.name, + ).toBeLessThanOrEqual(64 * 1024); + const contract = contracts.get(tool.name); + expect(contract, tool.name).toBeDefined(); + expect(tool.title, tool.name).toBe(contract?.title); + expect(tool.description, tool.name).toBe(contract?.description); + expect(tool.annotations, tool.name).toEqual(contract?.annotations); + expect(tool.outputSchema, tool.name).toBeDefined(); + assertDescribedStrictObjects( + tool.inputSchema, + tool.inputSchema, + tool.name, + ); + } }); it("records approved trace truncation as a deduplicated residual unknown", async () => { @@ -355,9 +426,7 @@ describe("full MCP integration with multi-tool sequences", () => { name: "compare_process_captures", arguments: { left_evidence_id: leftEvidence.evidence_id, - left, right_evidence_id: rightEvidence.evidence_id, - right, unknown_registry_approved: true, }, }); @@ -408,9 +477,7 @@ describe("full MCP integration with multi-tool sequences", () => { category: "execution_failure", }, }); - expect(text(result)).toBe( - "Analysis could not complete. Retry once; if it continues, run `rea doctor`.", - ); + expect(text(result)).toBe(JSON.stringify(result.structuredContent)); }); it("handles concurrent tool calls without corruption", async () => { diff --git a/tests/investigationMcp.test.ts b/tests/investigationMcp.test.ts index 89cc31daf..7c8228bd9 100644 --- a/tests/investigationMcp.test.ts +++ b/tests/investigationMcp.test.ts @@ -15,7 +15,7 @@ import { PROCESS_COMPARISON_EVIDENCE, } from "../src/contracts/investigationExamples.js"; import { changedBehaviorResultSchema } from "../src/domain/changedBehavior.js"; -import { createEvidence, parseEvidence } from "../src/domain/evidence.js"; +import { createEvidence } from "../src/domain/evidence.js"; import { jsonObjectSchema, jsonValueSchema } from "../src/domain/jsonValue.js"; import { createServer } from "../src/server/createServer.js"; import { observed } from "./fixtures/analysisExecution.js"; @@ -96,9 +96,7 @@ describe("investigation MCP workflows", () => { arguments: arguments_, }); expect(first.isError).not.toBe(true); - const firstEvidence = parseEvidence( - z.object({ result: z.unknown() }).parse(first.structuredContent).result, - ); + const firstEvidence = sessionEvidence(session, first.structuredContent); expect(firstEvidence.normalized_result).toMatchObject({ behavior_status: "unknown", summary: { static_candidates: 2 }, @@ -136,10 +134,7 @@ describe("investigation MCP workflows", () => { arguments: arguments_, }); expect(second.isError, JSON.stringify(second)).not.toBe(true); - const secondEvidence = parseEvidence( - z.object({ result: z.unknown() }).parse(second.structuredContent) - .result, - ); + const secondEvidence = sessionEvidence(session, second.structuredContent); expect(secondEvidence.evidence_id).toBe(firstEvidence.evidence_id); expect( await readInvestigationWorkspace(workspace, filePolicy), @@ -245,9 +240,9 @@ describe("investigation MCP workflows", () => { arguments: { investigation_run: investigationRun }, }); expect(created.isError, JSON.stringify(created)).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(created.structuredContent) - .result, + const evidence = sessionEvidence( + initial.session, + created.structuredContent, ); const result = changedBehaviorResultSchema.parse( evidence.normalized_result, @@ -322,12 +317,10 @@ describe("investigation MCP workflows", () => { }, }); expect(response.isError).toBe(true); - expect(response.content).toEqual([ - { - type: "text", - text: "Artifact path is not allowed. Choose a path inside the artifact and try again.", - }, - ]); + expect(response.content[0]).toEqual({ + type: "text", + text: JSON.stringify(response.structuredContent), + }); } finally { await close(session, server, client); await rm(directory, { recursive: true, force: true }); @@ -354,10 +347,7 @@ describe("investigation MCP workflows", () => { }, }); expect(response.isError, JSON.stringify(response)).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(response.structuredContent) - .result, - ); + const evidence = sessionEvidence(session, response.structuredContent); expect(evidence).toMatchObject({ provider: { id: "rea-changed-behavior" }, normalized_result: { behavior_status: "unknown" }, @@ -386,10 +376,7 @@ describe("investigation MCP workflows", () => { arguments: INVESTIGATION_EXAMPLES.build_call_path, }); expect(response.isError).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(response.structuredContent) - .result, - ); + const evidence = sessionEvidence(session, response.structuredContent); expect(evidence).toMatchObject({ provider: { id: "rea-call-path" }, normalized_result: { @@ -481,10 +468,7 @@ describe("investigation MCP workflows", () => { arguments: INVESTIGATION_EXAMPLES.correlate_static_and_runtime, }); expect(response.isError).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(response.structuredContent) - .result, - ); + const evidence = sessionEvidence(session, response.structuredContent); expect(evidence).toMatchObject({ provider: { id: "rea-static-runtime-correlation" }, confidence: "inferred", @@ -560,10 +544,7 @@ describe("investigation MCP workflows", () => { }, }); expect(response.isError).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(response.structuredContent) - .result, - ); + const evidence = sessionEvidence(session, response.structuredContent); expect(evidence).toMatchObject({ provider: { id: "rea-reconstruction-verifier" }, normalized_result: { status: "pass", summary: { passed: 1 } }, @@ -613,10 +594,7 @@ describe("investigation MCP workflows", () => { }, }); expect(response.isError).not.toBe(true); - const evidence = parseEvidence( - z.object({ result: z.unknown() }).parse(response.structuredContent) - .result, - ); + const evidence = sessionEvidence(session, response.structuredContent); expect(evidence.normalized_result).toMatchObject({ status: "unknown", summary: { unknown: 1 }, @@ -669,6 +647,15 @@ const evidencePolicy = (root: string): EvidenceFilePolicy => ({ maxNodes: 1_000_000, }); +const sessionEvidence = (session: BinarySession, value: unknown) => { + const parsed = z + .object({ evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u) }) + .parse(value); + const evidence = session.evidenceById(parsed.evidence_id); + if (evidence === undefined) throw new TypeError("Missing session Evidence"); + return evidence; +}; + const investigationAuthority = async ( root: string, includeWrite: boolean, diff --git a/tests/managedMcp.test.ts b/tests/managedMcp.test.ts index 3b20a9a89..ef83b8c08 100644 --- a/tests/managedMcp.test.ts +++ b/tests/managedMcp.test.ts @@ -108,10 +108,7 @@ describe("managed artifact MCP tools", () => { ); expect(inspected).toMatchObject({ - operation: "inspect_managed_artifact", - provider: { id: "rea-dotnet-static" }, - subject: { local_path: path, format: "pe" }, - normalized_result: { + result: { classification: { status: "managed", runtime_family: "modern-dotnet", diff --git a/tests/officialTools.test.ts b/tests/officialTools.test.ts index 8f614655e..bcd8dcada 100644 --- a/tests/officialTools.test.ts +++ b/tests/officialTools.test.ts @@ -201,11 +201,9 @@ describe("official Hopper proxy tools", () => { arguments: {}, }); expect(result.isError).toBe(true); - expect(result.content).toEqual([ - { - type: "text", - text: "Analysis could not complete. Retry once; if it continues, run `rea doctor`.", - }, - ]); + expect(result.content[0]).toEqual({ + type: "text", + text: JSON.stringify(result.structuredContent), + }); }); }); diff --git a/tests/providerSelectionMcp.test.ts b/tests/providerSelectionMcp.test.ts index 74deb1948..36ccaf4b9 100644 --- a/tests/providerSelectionMcp.test.ts +++ b/tests/providerSelectionMcp.test.ts @@ -111,9 +111,8 @@ describe("provider selection over MCP", () => { }); expect(observed.isError).not.toBe(true); expect(structured(observed)).toMatchObject({ - provider: { id: "beta", version: "1" }, - analysis_profile: { provider: { id: "beta", version: "1" } }, - normalized_result: "beta:address_name", + result: "beta:address_name", + evidence_id: expect.stringMatching(/^ev_/u), }); expect(starts).toEqual(["beta"]); diff --git a/tests/sessionMcp.test.ts b/tests/sessionMcp.test.ts index 564b1ffa6..701e2dffe 100644 --- a/tests/sessionMcp.test.ts +++ b/tests/sessionMcp.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -73,12 +73,13 @@ describe("target-free MCP lifecycle", () => { }), ).result; - expect(first).toMatchObject({ path: targetPath }); - expect(second).toMatchObject({ path: targetPath }); + const canonicalTargetPath = await realpath(targetPath); + expect(first).toMatchObject({ path: canonicalTargetPath }); + expect(second).toMatchObject({ path: canonicalTargetPath }); expect(z.object({ sha256: z.string() }).parse(second).sha256).not.toBe( z.object({ sha256: z.string() }).parse(first).sha256, ); - expect(closed).toEqual([targetPath]); + expect(closed).toEqual([canonicalTargetPath]); expect( z .object({ result: z.object({ path: z.string(), sha256: z.string() }) }) @@ -88,7 +89,7 @@ describe("target-free MCP lifecycle", () => { ), ).result, ).toEqual({ - path: targetPath, + path: canonicalTargetPath, sha256: z.object({ sha256: z.string() }).parse(second).sha256, }); }); @@ -139,9 +140,7 @@ describe("target-free MCP lifecycle", () => { arguments: {}, }); expect(before.isError).toBe(true); - expect(text(before)).toBe( - "No app is open. Ask the user which app to investigate, then call open_binary with its local path.", - ); + expect(text(before)).toBe(JSON.stringify(before.structuredContent)); expect((await mcp.listTools()).tools).toHaveLength(89); const deniedCapture = await mcp.callTool({ name: "capture_process_scenario", @@ -158,7 +157,7 @@ describe("target-free MCP lifecycle", () => { }, }); expect(text(deniedCapture)).toBe( - "Process capture is disabled. Set `REA_PROCESS_CAPTURE_ENABLED=true`, configure approved roots, then restart REA.", + JSON.stringify(deniedCapture.structuredContent), ); expect( (await mcp.callTool({ name: "open_binary", arguments: { path: first } })) @@ -304,7 +303,7 @@ describe("target-free MCP lifecycle", () => { }); expect( text(await mcp.callTool({ name: "binary_session", arguments: {} })), - ).toContain('"open": false'); + ).toContain('"open":false'); expect( ( await mcp.callTool({ diff --git a/tests/toolResult.test.ts b/tests/toolResult.test.ts index 5a8f13ed3..1921403e5 100644 --- a/tests/toolResult.test.ts +++ b/tests/toolResult.test.ts @@ -6,17 +6,26 @@ import { ok } from "../src/domain/result.js"; import { err } from "../src/domain/result.js"; import { HopperProcessError } from "../src/domain/errors.js"; import { toCallToolResult } from "../src/server/toolResult.js"; -import { - createEvidence, - evidenceEnvelopeSchema, -} from "../src/domain/evidence.js"; +import { createEvidence } from "../src/domain/evidence.js"; +import { evidenceResultOf } from "../src/contracts/toolOutputSchemas.js"; const contract: ToolContract = { name: "provider_neutral_fixture", + title: "Provider Neutral Fixture", description: "Fixture contract for provider-neutral output validation.", kind: "enhanced", inputSchema: z.object({}), outputSchema: z.object({ value: z.string() }), + effects: { + mutatesTarget: false, + mutatesSession: false, + writesFilesystem: false, + launchesProcess: false, + accessesNetwork: false, + changesUiState: false, + mayDiscardData: false, + idempotent: true, + }, annotations: { readOnlyHint: true, destructiveHint: false, @@ -39,13 +48,8 @@ describe("tool result projection", () => { expect(JSON.stringify(result)).not.toContain("expected Hopper"); }); it("classifies output contract failures without naming a provider", () => { - expect(toCallToolResult(ok({ value: 42 }), contract)).toEqual({ - content: [ - { - type: "text", - text: "Analysis returned an unreadable result. Retry once; if it continues, run `rea doctor`.", - }, - ], + const result = toCallToolResult(ok({ value: 42 }), contract); + expect(result).toMatchObject({ structuredContent: { error: { code: "unreadable_output", @@ -62,6 +66,10 @@ describe("tool result projection", () => { }, isError: true, }); + expect(result.content[0]).toEqual({ + type: "text", + text: JSON.stringify(result.structuredContent), + }); }); it("links successful evidence only when its session resource exists", () => { @@ -77,7 +85,7 @@ describe("tool result projection", () => { ); const evidenceContract: ToolContract = { ...contract, - outputSchema: evidenceEnvelopeSchema, + outputSchema: evidenceResultOf(z.object({ value: z.string() })), }; expect(