From 5ac3dc96f064e691b357aeddc11fa76b7b4d66d1 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:00:00 +0800 Subject: [PATCH] test(acceptance): add application workflow MCP parity harness and scenarios Adds an owned in-memory MCP harness plus acceptance coverage for trace/compare/export-shape workflows over inline and recorded Evidence, including rejection of mismatched evidence references and spoofed providers. All 16 scenarios pass on the current base. --- .../applicationEvidenceMcp.test.ts | 234 +++++++ .../applicationExportShapeMcp.test.ts | 225 +++++++ .../applicationWorkflowVerifier.test.ts | 109 ++++ .../javascriptApplicationWorkflows.test.ts | 603 ++++++++++++++++++ tests/acceptance/applications/mcpHarness.ts | 39 ++ 5 files changed, 1210 insertions(+) create mode 100644 tests/acceptance/applications/applicationEvidenceMcp.test.ts create mode 100644 tests/acceptance/applications/applicationExportShapeMcp.test.ts create mode 100644 tests/acceptance/applications/applicationWorkflowVerifier.test.ts create mode 100644 tests/acceptance/applications/javascriptApplicationWorkflows.test.ts create mode 100644 tests/acceptance/applications/mcpHarness.ts diff --git a/tests/acceptance/applications/applicationEvidenceMcp.test.ts b/tests/acceptance/applications/applicationEvidenceMcp.test.ts new file mode 100644 index 000000000..6b21ac22c --- /dev/null +++ b/tests/acceptance/applications/applicationEvidenceMcp.test.ts @@ -0,0 +1,234 @@ +import type { Client } from "@modelcontextprotocol/client"; +import { describe, expect, it } from "vitest"; + +import type { BinarySession } from "../../../src/application/binary/BinarySession.js"; +import { + JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE, + JAVASCRIPT_FEATURE_TRACE_EXAMPLE, + JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE, + SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, +} from "../../../src/contracts/javascript/javascriptApplicationWorkflowExamples.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import { + createApplicationMcpHarness, + type ApplicationMcpHarness, +} from "./mcpHarness.js"; + +async function runInlineEvidenceScenarios( + client: Client, + session: BinarySession, +): Promise { + const traced = await client.callTool({ + name: "trace_application_feature", + arguments: JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE, + }); + expect(traced.isError).not.toBe(true); + expect(traced.structuredContent).toMatchObject({ + result: { + seed: { kind: "module", value: "renderer.js", match: "exact" }, + summary: { matched_seeds: 1, traced_nodes: 1, unknown_facts: 0 }, + coverage: { + status: "complete-within-source", + source_graph_status: "complete", + total_seed_matches: 1, + }, + }, + }); + + const compared = await client.callTool({ + name: "compare_application_versions", + arguments: { + ...JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE, + }, + }); + expect(compared.isError).not.toBe(true); + expect(compared.structuredContent).toMatchObject({ + result: { + summary: { unchanged: 0, added: 2, removed: 0, changed: 1, unknown: 0 }, + coverage: { + left_graph_status: "complete", + right_graph_status: "complete", + status: "complete-within-inputs", + }, + }, + }); + + const sourceCompared = await client.callTool({ + name: "compare_source_to_bundle", + arguments: { + reference: SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference, + application: JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE.application, + }, + }); + expect(sourceCompared.isError).not.toBe(true); + expect(sourceCompared.structuredContent).toMatchObject({ + result: { + reference: { + root_sha256: SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference.root_sha256, + }, + scoring: { algorithm: "rea-source-to-bundle-signals" }, + }, + }); + expect(session.exportEvidenceBundle().records.length).toBeGreaterThan(2); +} + +async function runInlineWorkflowScenarios( + harness: ApplicationMcpHarness, +): Promise { + const tracedInline = await harness.client.callTool({ + name: "trace_application_feature", + arguments: { + ...JAVASCRIPT_FEATURE_TRACE_EXAMPLE, + }, + }); + expect(tracedInline.isError).not.toBe(true); + expect(tracedInline.structuredContent).toMatchObject({ + result: { + seed: { kind: "module", value: "renderer.js", match: "exact" }, + summary: { matched_seeds: 1, traced_nodes: 1, unknown_facts: 0 }, + coverage: { + status: "complete-within-source", + source_graph_status: "complete", + total_seed_matches: 1, + }, + }, + }); + + const comparedInline = await harness.client.callTool({ + name: "compare_application_versions", + arguments: { + ...JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE, + }, + }); + expect(comparedInline).toMatchObject({ + structuredContent: { + result: { + evidence_links: expect.arrayContaining([ + JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE.left.evidence_id, + JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE.right.evidence_id, + ]), + }, + }, + }); + + const idsRejected = await harness.client.callTool({ + name: "compare_application_versions", + arguments: { + left_evidence_id: + JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE.left.evidence_id, + right_evidence_id: + JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE.right.evidence_id, + }, + }); + expect(idsRejected.isError).toBe(true); + + const sourceComparedById = await harness.client.callTool({ + name: "compare_source_to_bundle", + arguments: { + ...SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, + }, + }); + expect(sourceComparedById.isError).not.toBe(true); + expect(sourceComparedById.structuredContent).toMatchObject({ + result: { + reference: { + root_sha256: SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference.root_sha256, + inventory_state: "complete", + }, + scoring: { algorithm: "rea-source-to-bundle-signals" }, + summary: { unchanged: 0, modified: 0, removed: 1, unknown: 0 }, + items: [ + { + source_path: "src/main.ts", + status: "removed", + current_node_ids: [], + candidates: [], + }, + ], + }, + }); +} + +async function assertRejectedEvidenceReferences( + client: Client, + session: BinarySession, +): Promise { + const wrongOperation = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { operation: "inventory_artifact", parameters: {}, result: {} }, + ); + const wrongPredicate = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + predicateType: "fixture.application/v1", + operation: "analyze_javascript_application", + parameters: {}, + result: {}, + }, + ); + expect(session.recordEvidence(wrongOperation).ok).toBe(true); + expect(session.recordEvidence(wrongPredicate).ok).toBe(true); + for (const record of [wrongOperation, wrongPredicate]) { + const rejected = await client.callTool({ + name: "trace_application_feature", + arguments: { + ...JAVASCRIPT_FEATURE_TRACE_EXAMPLE, + application: record, + }, + }); + expect(rejected).toMatchObject({ + isError: true, + structuredContent: { + // A recorded Evidence with the wrong operation or predicate is an + // invalid caller input, not a provider failure. + error: { + code: "invalid_request", + category: "invalid_input", + retryable: true, + }, + }, + }); + } +} + +async function assertRejectedInlineEvidence(client: Client): Promise { + const duplicateNative = await client.callTool({ + name: "trace_application_feature", + arguments: { + ...JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE, + native_observations: [ + JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE.application, + JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE.application, + ], + }, + }); + expect(duplicateNative.isError).toBe(true); + + const spoofed = await client.callTool({ + name: "trace_application_feature", + arguments: { + ...JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE, + application: { + ...JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE.application, + provider: { id: "spoofed", name: "spoofed", version: "1" }, + }, + }, + }); + expect(spoofed.isError).toBe(true); +} + +describe("application workflow MCP parity", () => { + it("traces and compares authenticated graph Evidence in the session", async () => { + const harness = await createApplicationMcpHarness(); + try { + await runInlineEvidenceScenarios(harness.client, harness.session); + await runInlineWorkflowScenarios(harness); + await assertRejectedEvidenceReferences(harness.client, harness.session); + await assertRejectedInlineEvidence(harness.client); + } finally { + await harness.close(); + } + }); +}); diff --git a/tests/acceptance/applications/applicationExportShapeMcp.test.ts b/tests/acceptance/applications/applicationExportShapeMcp.test.ts new file mode 100644 index 000000000..4fad71615 --- /dev/null +++ b/tests/acceptance/applications/applicationExportShapeMcp.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from "vitest"; +import { copyFile, mkdir, rm, writeFile } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js"; +import { compareJavaScriptExportShapesEvidence } from "../../../src/application/javascript/JavaScriptApplicationWorkflowService.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { javaScriptExportShapeComparisonResultSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { createApplicationMcpHarness } from "./mcpHarness.js"; + +describe("application workflow MCP parity", () => { + it("compares exact parser export shapes with inline Evidence", async () => { + const root = await createTestTempDirectory("rea-export-shape-mcp-"); + const leftRoot = join(root, "left"); + const rightRoot = join(root, "right"); + await Promise.all([mkdir(leftRoot), mkdir(rightRoot)]); + await Promise.all([ + copyFile( + resolve("tests/fixtures/replay/parser.mjs"), + join(leftRoot, "parser.mjs"), + ), + copyFile( + resolve("tests/fixtures/replay/parser-v2.mjs"), + join(rightRoot, "parser.mjs"), + ), + ]); + const [left, right] = await Promise.all([ + analyzeJavaScriptApplication({ + input_path: leftRoot, + }), + analyzeJavaScriptApplication({ + input_path: rightRoot, + }), + ]); + if (!left.ok) throw left.error; + if (!right.ok) throw right.error; + const leftAnalysis = javascriptApplicationAnalysisResultSchema.parse( + left.value.normalized_result, + ); + const harness = await createApplicationMcpHarness(); + const { client, session } = harness; + const selectors = { + left_module_path: "parser.mjs", + left_export_name: "default", + right_module_path: "parser.mjs", + right_export_name: "default", + }; + try { + const relativeApplication = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: relative(process.cwd(), leftRoot) }, + }); + expect(relativeApplication.isError).not.toBe(true); + expect(relativeApplication.structuredContent).toMatchObject({ + evidence_id: left.value.evidence_id, + result: { input_path: leftAnalysis.input_path }, + evidence: { + subject: { local_path: left.value.subject?.local_path }, + }, + }); + const full = await client.callTool({ + name: "compare_javascript_export_shapes", + arguments: { left: left.value, right: right.value, ...selectors }, + }); + expect(full.isError).not.toBe(true); + expect(full.structuredContent).toMatchObject({ + result: { + summary: { added: 1, removed: 0, changed: 0, unknown: 0 }, + changes: [ + { + status: "added", + path: "/depth", + right: { availability: "literal", value: 1 }, + }, + ], + }, + }); + expect(session.exportEvidenceBundle().records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + operation: "compare_javascript_export_shapes", + predicate_type: "rea.javascript-export-shape-comparison", + }), + ]), + ); + + const analyzed = javascriptApplicationAnalysisResultSchema.parse( + left.value.normalized_result, + ); + const seed = analyzed.semantic_graph.relations[0]?.source_node_id; + if (seed === undefined) + throw new TypeError("Expected at least one semantic relation"); + const semantic = await client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application: left.value, + query: { + seed: { kind: "semantic-node", node_id: seed }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: true, + }, + }, + }); + expect(semantic.isError).not.toBe(true); + expect(semantic.structuredContent).toMatchObject({ + result: { + source_evidence_id: left.value.evidence_id, + source_graph_id: analyzed.semantic_graph.graph_id, + summary: { total_seed_matches: 1 }, + }, + }); + } finally { + await harness.close(); + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe("source-produced export comparison inventories", () => { + it("retains large source-produced export inventories through comparison schemas", async () => { + const fields = Array.from( + { length: 64 }, + (_, index) => `field${String(index)}: ${String(index)}`, + ).join(", "); + const returnSites = Array.from( + { length: 33 }, + (_, index) => + `if (value === ${String(index)}) return { type: "variant-${String(index)}", ${fields} };`, + ).join("\n"); + const extraExports = Array.from( + { length: 1_000 }, + (_, index) => + `export const candidate${String(index)} = ${String(index)};`, + ).join("\n"); + const inventorySource = `export default function parse(value) {\n${returnSites}\n throw new Error("no match");\n}\n${extraExports}`; + const pairedVariantsSource = `export default function parse(value) { +${returnSites} + throw new Error("no match"); +}`; + const [inventoryLeft, inventoryRight] = await Promise.all([ + analyzeSourceEvidence(inventorySource), + analyzeSourceEvidence(pairedVariantsSource), + ]); + const variants = compareEvidence(inventoryLeft, inventoryRight); + expect(variants.left).toMatchObject({ status: "selected" }); + expect(variants.coverage).toMatchObject({ + paired_variants: 33, + unpaired_left_variants: 0, + unpaired_right_variants: 0, + left_source_omitted_variants: 0, + right_source_omitted_variants: 0, + left_omitted_fields: 0, + right_omitted_fields: 0, + }); + + const missing = compareEvidence(inventoryLeft, inventoryRight, { + leftExportName: "missing", + }); + expect(missing.left).toMatchObject({ + status: "missing", + omitted_candidates: 0, + }); + expect(missing.left.candidates).toHaveLength(1_001); + + const changedFieldsLeft = Array.from( + { length: 10_001 }, + (_, index) => `field${String(index)}: ${String(index)}`, + ).join(", "); + const changedFieldsRight = Array.from( + { length: 10_001 }, + (_, index) => `field${String(index)}: ${String(index + 1)}`, + ).join(", "); + const [changeLeft, changeRight] = await Promise.all([ + analyzeSourceEvidence( + `export default () => ({ type: "record", ${changedFieldsLeft} });`, + ), + analyzeSourceEvidence( + `export default () => ({ type: "record", ${changedFieldsRight} });`, + ), + ]); + const changes = compareEvidence(changeLeft, changeRight); + expect(changes.changes).toHaveLength(10_001); + expect(changes.summary).toEqual({ + added: 0, + removed: 0, + changed: 10_001, + unknown: 0, + }); + expect(changes.coverage).toMatchObject({ + status: "complete-within-inputs", + omitted_changes: 0, + left_omitted_fields: 0, + right_omitted_fields: 0, + }); + }, 30_000); +}); + +const analyzeSourceEvidence = async (source: string) => { + const root = await createTestTempDirectory("rea-export-shape-inventory-"); + await writeFile(join(root, "parser.mjs"), source); + const analyzed = await analyzeJavaScriptApplication({ input_path: root }); + if (!analyzed.ok) throw analyzed.error; + return analyzed.value; +}; + +const compareEvidence = ( + left: Awaited>, + right: Awaited>, + selectors: { readonly leftExportName?: string } = {}, +) => { + const compared = compareJavaScriptExportShapesEvidence({ + left, + right, + left_module_path: "parser.mjs", + left_export_name: selectors.leftExportName ?? "default", + right_module_path: "parser.mjs", + right_export_name: "default", + }); + if (!compared.ok) throw compared.error; + return javaScriptExportShapeComparisonResultSchema.parse( + compared.value.normalized_result, + ); +}; diff --git a/tests/acceptance/applications/applicationWorkflowVerifier.test.ts b/tests/acceptance/applications/applicationWorkflowVerifier.test.ts new file mode 100644 index 000000000..f420e817c --- /dev/null +++ b/tests/acceptance/applications/applicationWorkflowVerifier.test.ts @@ -0,0 +1,109 @@ +import { execFile } from "node:child_process"; +import { copyFile, mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +import { writeVersionedJavaScriptApplicationFixtures } from "../../fixtures/javascriptArtifactApplication.js"; + +const execute = promisify(execFile); + +describe("local application workflow verifier", () => { + const temporary: string[] = []; + + afterEach(async () => { + await Promise.all( + temporary + .splice(0) + .map(async (path) => rm(path, { recursive: true, force: true })), + ); + }); + + it("reports two-version and trace results", async () => { + const root = await createTestTempDirectory("rea-application-verifier-"); + temporary.push(root); + const fixtures = await writeVersionedJavaScriptApplicationFixtures(root); + const { stdout } = await execute( + process.execPath, + [ + "scripts/verify/javascript/application-workflows.mjs", + "--left", + fixtures.left, + "--right", + fixtures.right, + "--seed-kind", + "module", + "--seed-value", + "stable", + ], + { cwd: process.cwd(), maxBuffer: 16 * 1_024 * 1_024 }, + ); + const result = JSON.parse(stdout); + expect(result).toMatchObject({ + verified: true, + summary: { + added: expect.any(Number), + removed: expect.any(Number), + changed: expect.any(Number), + unknown: expect.any(Number), + }, + matching: { + exact_module_source_digest: expect.any(Number), + structural_fingerprint: expect.any(Number), + }, + coverage: { status: expect.any(String) }, + trace: { summary: { matched_seeds: expect.any(Number) } }, + }); + }, 20_000); + + it("reports an added parser return field", async () => { + const root = await createTestTempDirectory("rea-shape-verifier-"); + temporary.push(root); + const left = join(root, "left"); + const right = join(root, "right"); + await Promise.all([mkdir(left), mkdir(right)]); + await Promise.all([ + copyFile( + join(process.cwd(), "tests/fixtures/replay/parser.mjs"), + join(left, "parser.mjs"), + ), + copyFile( + join(process.cwd(), "tests/fixtures/replay/parser-v2.mjs"), + join(right, "parser.mjs"), + ), + ]); + const { stdout } = await execute( + process.execPath, + [ + "scripts/verify/javascript/application-workflows.mjs", + "--left", + left, + "--right", + right, + "--left-module-path", + "parser.mjs", + "--left-export-name", + "default", + "--right-module-path", + "parser.mjs", + "--right-export-name", + "default", + ], + { cwd: process.cwd(), maxBuffer: 16 * 1_024 * 1_024 }, + ); + const result = JSON.parse(stdout); + expect(result.export_shape_comparison).toMatchObject({ + summary: { added: 1, removed: 0, changed: 0, unknown: 0 }, + changes: [ + { + status: "added", + path: "/depth", + right: { availability: "literal", value: 1 }, + }, + ], + }); + }, 20_000); +}); diff --git a/tests/acceptance/applications/javascriptApplicationWorkflows.test.ts b/tests/acceptance/applications/javascriptApplicationWorkflows.test.ts new file mode 100644 index 000000000..c3f72a195 --- /dev/null +++ b/tests/acceptance/applications/javascriptApplicationWorkflows.test.ts @@ -0,0 +1,603 @@ +import { rm } from "node:fs/promises"; +import { relative, resolve } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +import { compareApplicationVersionsRequestSchema } from "../../../src/contracts/javascript/applicationWorkflowInputContracts.js"; +import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js"; +import { + compareApplicationVersionsEvidence, + traceApplicationFeatureEvidence, +} from "../../../src/application/javascript/JavaScriptApplicationWorkflowService.js"; +import { reconcileJavaScriptRuntimeEvidence } from "../../../src/application/javascript/JavaScriptRuntimeReconciliationService.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { + applicationVersionComparisonResultSchema, + type ApplicationVersionComparisonResult, +} from "../../../src/domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; +import { javascriptRuntimeReconciliationResultSchema } from "../../../src/domain/javascript/javascriptRuntimeReconciliationSchemas.js"; +import { compareJavaScriptApplicationVersions } from "../../../src/domain/javascript/javascriptApplicationVersionComparison.js"; +import { createJavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; +import { traceApplicationFeature } from "../../../src/domain/javascript/javascriptFeatureTrace.js"; +import { applicationFeatureTraceResultSchema } from "../../../src/domain/javascript/javascriptFeatureTraceSchemas.js"; +import { + APPLICATION_GRAPH_DIGESTS, + buildSyntheticJavaScriptApplicationGraph, +} from "../../../src/domain/javascript/javascriptApplicationGraph.fixture.js"; +import { writeVersionedJavaScriptApplicationFixtures } from "../../fixtures/javascriptArtifactApplication.js"; +import { JAVASCRIPT_APPLICATION_PROVIDER } from "../../../src/application/InvestigationProviders.js"; +import { JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE } from "../../../src/contracts/javascript/javascriptRuntimeReconciliationExample.js"; + +const temporary: string[] = []; + +afterEach(async () => { + await Promise.all( + temporary + .splice(0) + .map(async (path) => rm(path, { recursive: true, force: true })), + ); +}); + +describe("JavaScript application workflows", () => { + it("traces a contextBridge API through IPC into linked native evidence", () => { + const graph = buildSyntheticJavaScriptApplicationGraph(); + const nativeEvidence = createEvidence( + { + path: "/tmp/synthetic.node", + sha256: APPLICATION_GRAPH_DIGESTS.nativeAddon, + format: "elf", + architecture: "x86_64", + }, + { id: "ghidra", name: "Ghidra", version: "fixture" }, + { + operation: "analyze_function", + parameters: { address: "openProject" }, + result: { name: "openProject", address: "0x1000" }, + limitations: ["Synthetic native provider observation."], + }, + ); + + const result = traceApplicationFeature({ + sourceEvidenceId: `ev_${"a".repeat(64)}`, + graph, + nativeEvidence: [nativeEvidence], + seed: { + kind: "api", + value: "desktopApi", + match: "exact", + case_sensitive: false, + }, + direction: "outgoing", + }); + + expect(() => + applicationFeatureTraceResultSchema.parse(result), + ).not.toThrow(); + expect(result.summary).toMatchObject({ + matched_seeds: 1, + native_handoffs: 1, + }); + expect(result.native_handoffs).toContainEqual( + expect.objectContaining({ + artifact_sha256: APPLICATION_GRAPH_DIGESTS.nativeAddon, + status: "evidence-linked", + evidence_ids: [nativeEvidence.evidence_id], + requested_exports: expect.arrayContaining(["openProject"]), + }), + ); + expect(result.paths).toEqual( + expect.arrayContaining([ + expect.objectContaining({ end_kind: "native-export" }), + ]), + ); + expect( + result.graph?.edges.some( + ({ evidence }) => + evidence.authority === "static-relationship-inference" && + evidence.state === "inferred", + ), + ).toBe(true); + }); + + it("keeps no-match explicit and traces the full reachable graph", () => { + const graph = buildSyntheticJavaScriptApplicationGraph(); + const noMatch = traceApplicationFeature({ + sourceEvidenceId: `ev_${"a".repeat(64)}`, + graph, + nativeEvidence: [], + seed: { + kind: "route", + value: "/missing", + match: "exact", + case_sensitive: false, + }, + direction: "both", + }); + expect(noMatch).toMatchObject({ + graph: null, + coverage: { status: "no-match" }, + summary: { matched_seeds: 0 }, + }); + + const complete = traceApplicationFeature({ + sourceEvidenceId: `ev_${"a".repeat(64)}`, + graph, + nativeEvidence: [], + seed: { + kind: "api", + value: "desktopApi", + match: "exact", + case_sensitive: false, + }, + direction: "outgoing", + }); + expect(complete.coverage.status).toBe("complete-within-source"); + expect(complete.coverage.total_seed_matches).toBe(1); + expect(complete.graph?.nodes.length).toBeGreaterThan(1); + expect(complete.coverage).not.toHaveProperty("omitted_nodes"); + }); +}); + +describe("JavaScript application comparison workflows", () => { + it("accepts complete inline Evidence and native observation arrays", () => { + const native = Array.from({ length: 65 }, (_, index) => + createEvidence( + { + path: `/tmp/native-${index}.node`, + sha256: index.toString(16).padStart(64, "0"), + format: "elf", + architecture: "x86_64", + }, + { id: "ghidra", name: "Ghidra", version: "fixture" }, + { + operation: "analyze_function", + parameters: { address: `0x${index.toString(16)}` }, + result: { address: `0x${index.toString(16)}` }, + }, + ), + ); + const applicationEvidence = (side: string) => + createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + predicateType: "rea.javascript-application-analysis", + operation: "analyze_javascript_application", + parameters: { side }, + result: {}, + }, + ); + const input = { + left: applicationEvidence("left"), + right: applicationEvidence("right"), + left_native_observations: native, + }; + expect( + compareApplicationVersionsRequestSchema.safeParse(input).success, + ).toBe(true); + }); +}); + +describe("complete comparison projections", () => { + it("matches rechunked modules by exact source and minified modules by structural fingerprint", async () => { + const root = await createTestTempDirectory("rea-application-versions-"); + temporary.push(root); + const fixtures = await writeVersionedJavaScriptApplicationFixtures(root); + const [left, right] = await Promise.all([ + analyzeFixture(fixtures.left), + analyzeFixture(fixtures.right), + ]); + const compared = compareApplicationVersionsEvidence({ left, right }); + if (!compared.ok) throw compared.error; + const result = applicationVersionComparisonResultSchema.parse( + compared.value.normalized_result, + ); + expectFullComparisonResultSchema(result); + const modules = result.items.filter( + ({ node_kind: kind }) => kind === "javascript-module", + ); + + expect(modules).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + match: expect.objectContaining({ + status: "matched", + basis: "exact-module-source-digest", + confidence: "exact", + }), + }), + expect.objectContaining({ + status: "changed", + match: expect.objectContaining({ + status: "matched", + basis: "structural-fingerprint", + confidence: "medium", + }), + }), + ]), + ); + expect(result.matching.ambiguous).toBeGreaterThanOrEqual(4); + expect(modules.filter(({ match }) => match.status === "ambiguous")).toEqual( + expect.arrayContaining([expect.objectContaining({ status: "unknown" })]), + ); + expect(result.summary.added).toBeGreaterThan(0); + expect(result.summary.removed).toBeGreaterThan(0); + expect(result.coverage).not.toHaveProperty("omitted_graph_nodes"); + expect(result.coverage).not.toHaveProperty("omitted_graph_edges"); + expect(result.coverage).not.toHaveProperty("omitted_graph_observations"); + expect(result.graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "changed_from", + evidence: expect.objectContaining({ + authority: "cross-version-comparison", + state: "inferred", + }), + }), + ]), + ); + expect(result.limitations.join(" ")).toMatch( + /module ordinals|fuzzy pairing/u, + ); + expectComparisonItemAlgebra(result); + }); + + it("pairs source-map originals without promoting the match to exact", async () => { + const root = await createTestTempDirectory("rea-source-map-versions-"); + temporary.push(root); + const fixtures = await writeVersionedJavaScriptApplicationFixtures(root); + const [left, right] = await Promise.all([ + analyzeFixture(fixtures.left), + analyzeFixture(fixtures.right), + ]); + const compared = compareApplicationVersionsEvidence({ left, right }); + if (!compared.ok) throw compared.error; + const result = applicationVersionComparisonResultSchema.parse( + compared.value.normalized_result, + ); + expect(result.items).toContainEqual( + expect.objectContaining({ + node_kind: "source-module", + status: "changed", + match: expect.objectContaining({ + basis: "source-map-identity", + confidence: "high", + }), + }), + ); + }); + + it("keeps incomplete one-sided absence unknown and reports source omissions", () => { + const complete = buildSyntheticJavaScriptApplicationGraph(); + const rootNode = complete.nodes.find(({ node_id: id }) => + complete.root_node_ids.includes(id), + ); + if (rootNode === undefined) throw new Error("Synthetic graph root missing"); + const partial = createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [rootNode.node_id], + nodes: [rootNode], + edges: [], + coverage: { + status: "partial", + truncated: true, + omitted_count: complete.nodes.length - 1, + limits: [{ name: "fixture_nodes", value: 1, unit: "items" }], + }, + limitations: ["Synthetic partial graph."], + }); + const input = { + left: { + evidenceId: `ev_${"a".repeat(64)}`, + rootArtifactSha256: APPLICATION_GRAPH_DIGESTS.package, + graph: complete, + }, + right: { + evidenceId: `ev_${"b".repeat(64)}`, + rootArtifactSha256: APPLICATION_GRAPH_DIGESTS.package, + graph: partial, + }, + leftNativeEvidence: [], + rightNativeEvidence: [], + }; + + const first = compareJavaScriptApplicationVersions(input); + const second = compareJavaScriptApplicationVersions(input); + expect(first.comparison_id).toBe(second.comparison_id); + expect(first.coverage).toMatchObject({ + status: "truncated", + right_graph_status: "partial", + right_graph_omitted_count: complete.nodes.length - 1, + }); + expect(first.summary.removed).toBe(0); + expect(first.summary.unknown).toBeGreaterThan(0); + }); +}); + +describe("JavaScript application workflow error diagnostics", () => { + it("returns safe explicit Evidence constraints from graph workflows", async () => { + const root = await createTestTempDirectory( + "rea-application-workflow-diagnostics-", + ); + temporary.push(root); + const evidence = await analyzeFixture(root); + const subject = evidence.subject; + if (subject === null) throw new Error("Analysis Evidence subject missing"); + const secretPath = "/private/token=do-not-return"; + const inconsistent = { + ...evidence, + subject: { ...subject, local_path: secretPath }, + }; + + const result = traceApplicationFeatureEvidence({ + application: inconsistent, + seed: { kind: "route", value: "/" }, + direction: "both", + }); + + if (result.ok) throw new Error("Expected inconsistent Evidence to fail"); + expect(result.error).toBeInstanceOf(AnalysisInputError); + if (!(result.error instanceof AnalysisInputError)) return; + expect(result.error.issues).toContainEqual( + expect.objectContaining({ + reason: "invalid_value", + message: + "JavaScript application Evidence subject does not match its result", + }), + ); + expect(JSON.stringify(result.error.issues)).not.toContain(secretPath); + }); + + it("preserves schema-authored Evidence validation messages", async () => { + const root = await createTestTempDirectory( + "rea-application-workflow-schema-diagnostics-", + ); + temporary.push(root); + const evidence = await analyzeFixture(root); + const subject = evidence.subject; + if (subject === null) throw new Error("Analysis Evidence subject missing"); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + const malformedResult = { + ...result, + semantic_graph: { + ...result.semantic_graph, + application_graph_id: `jag_${"f".repeat(64)}`, + }, + }; + const malformedEvidence = createEvidence( + { + path: subject.local_path, + sha256: subject.digest.sha256, + format: subject.format, + }, + JAVASCRIPT_APPLICATION_PROVIDER, + { + predicateType: evidence.predicate_type, + operation: evidence.operation, + parameters: evidence.parameters, + result: jsonValueSchema.parse(malformedResult), + confidence: evidence.confidence, + authority: evidence.authority, + limitations: evidence.limitations, + locations: evidence.locations, + evidenceLinks: evidence.evidence_links, + }, + ); + + const traced = traceApplicationFeatureEvidence({ + application: malformedEvidence, + seed: { kind: "route", value: "/" }, + direction: "both", + }); + const reconciled = reconcileJavaScriptRuntimeEvidence({ + static_layers: [{ role: "application", analysis: malformedEvidence }], + runtime_observations: [evidence], + }); + + expect(traced.ok).toBe(false); + expect(reconciled.ok).toBe(false); + if (!traced.ok && traced.error instanceof AnalysisInputError) + expect(traced.error.issues).toContainEqual( + expect.objectContaining({ + message: + "Semantic graph must commit the containing application graph", + }), + ); + if (!reconciled.ok && reconciled.error instanceof AnalysisInputError) + expect(reconciled.error.issues).toContainEqual( + expect.objectContaining({ + message: + "Semantic graph must commit the containing application graph", + }), + ); + }); + + it("returns supported-operation constraints from runtime reconciliation", async () => { + const root = await createTestTempDirectory( + "rea-runtime-workflow-diagnostics-", + ); + temporary.push(root); + const evidence = await analyzeFixture(root); + + const result = reconcileJavaScriptRuntimeEvidence({ + static_layers: [{ role: "application", analysis: evidence }], + runtime_observations: [evidence], + }); + + if (result.ok) throw new Error("Expected non-runtime Evidence to fail"); + expect(result.error).toBeInstanceOf(AnalysisInputError); + if (!(result.error instanceof AnalysisInputError)) return; + expect(result.error.issues).toContainEqual( + expect.objectContaining({ + reason: "invalid_value", + message: + "Runtime reconciliation requires inspect_web_page, inspect_electron_page, observe_javascript_runtime, or capture_electron_scenario Evidence", + }), + ); + }); +}); + +describe("JavaScript runtime reconciliation local paths", () => { + it("resolves relative file mapping roots before deriving result Evidence", () => { + const absoluteRoot = resolve(process.cwd(), "runtime-cache-fixture"); + const relativeRoot = relative(process.cwd(), absoluteRoot); + const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; + const [baseLayer] = base.static_layers; + if (baseLayer === undefined) throw new Error("Static layer missing"); + const input = { + ...base, + static_layers: [ + { + ...baseLayer, + runtime_mappings: [ + { kind: "file-root" as const, root: relativeRoot }, + ], + }, + ], + }; + const relativeResult = reconcileJavaScriptRuntimeEvidence(input); + const absoluteResult = reconcileJavaScriptRuntimeEvidence({ + ...input, + static_layers: [ + { + ...input.static_layers[0], + runtime_mappings: [{ kind: "file-root", root: absoluteRoot }], + }, + ], + }); + + expect(relativeResult.ok).toBe(true); + expect(absoluteResult.ok).toBe(true); + if (!relativeResult.ok || !absoluteResult.ok) return; + expect(relativeResult.value.evidence_id).toBe( + absoluteResult.value.evidence_id, + ); + expect(relativeResult.value.normalized_result).toEqual( + absoluteResult.value.normalized_result, + ); + const result = javascriptRuntimeReconciliationResultSchema.parse( + relativeResult.value.normalized_result, + ); + expect(result.static_layers[0]?.runtime_mappings[0]).toMatchObject({ + kind: "file-root", + root: absoluteRoot, + }); + }); + + it("preserves foreign absolute file roots without matching another path syntax", () => { + const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; + const [baseLayer] = base.static_layers; + if (baseLayer === undefined) throw new Error("Static layer missing"); + const remoteWindowsRoot = String.raw`C:\runtime\cache`; + const reconciled = reconcileJavaScriptRuntimeEvidence({ + ...base, + static_layers: [ + { + ...baseLayer, + runtime_mappings: [{ kind: "file-root", root: remoteWindowsRoot }], + }, + ], + }); + + expect(reconciled.ok).toBe(true); + if (!reconciled.ok) return; + const result = javascriptRuntimeReconciliationResultSchema.parse( + reconciled.value.normalized_result, + ); + expect(result.static_layers[0]?.runtime_mappings[0]).toMatchObject({ + kind: "file-root", + root: remoteWindowsRoot, + }); + expect( + result.reconciliations.some( + ({ basis }) => basis === "operator-file-mapping", + ), + ).toBe(false); + }); +}); + +const analyzeFixture = async (root: string) => { + const result = await analyzeJavaScriptApplication({ + input_path: root, + }); + if (!result.ok) throw result.error; + return result.value; +}; + +const expectComparisonItemAlgebra = ( + result: ApplicationVersionComparisonResult, +): void => { + const matched = result.items.find(({ match }) => match.status === "matched"); + expect(matched).toBeDefined(); + if (matched === undefined) return; + expect( + applicationVersionComparisonResultSchema.safeParse({ + ...result, + items: [{ ...matched, right_node_id: null }], + }).success, + ).toBe(false); + expect( + applicationVersionComparisonResultSchema.safeParse({ + ...result, + items: [ + { + ...matched, + match: { + ...matched.match, + basis: "none", + confidence: "unknown", + }, + }, + ], + }).success, + ).toBe(false); + expect( + applicationVersionComparisonResultSchema.safeParse({ + ...result, + coverage: { + ...result.coverage, + status: "complete-within-inputs", + left_graph_status: "partial", + }, + }).success, + ).toBe(false); +}; + +const expectFullComparisonResultSchema = ( + result: ApplicationVersionComparisonResult, +): void => { + const candidateNodeIds = Array.from( + { length: 1_001 }, + (_, index) => `jag_node_${index.toString(16).padStart(64, "0")}`, + ); + const expanded = { + ...result, + evidence_links: Array.from( + { length: 131 }, + (_, index) => `ev_${index.toString(16).padStart(64, "0")}`, + ), + items: result.items.map((item) => + item.match.status === "ambiguous" + ? { + ...item, + match: { + ...item.match, + ...(item.match.candidate_left_node_ids.length === 0 + ? { candidate_right_node_ids: candidateNodeIds } + : { candidate_left_node_ids: candidateNodeIds }), + }, + } + : item, + ), + }; + expect( + applicationVersionComparisonResultSchema.safeParse(expanded).success, + ).toBe(true); +}; diff --git a/tests/acceptance/applications/mcpHarness.ts b/tests/acceptance/applications/mcpHarness.ts new file mode 100644 index 000000000..5f971e477 --- /dev/null +++ b/tests/acceptance/applications/mcpHarness.ts @@ -0,0 +1,39 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; + +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import type { BinarySession } from "../../../src/application/binary/BinarySession.js"; +import { observed } from "../../fixtures/analysisExecution.js"; +import { createServer } from "../../../src/server/createServer.js"; + +/** Owned MCP SDK session shared by application workflow scenarios. */ +export interface ApplicationMcpHarness { + readonly client: Client; + readonly session: BinarySession; + readonly close: () => Promise; +} + +/** Connect application workflows through the production MCP registration. */ +export async function createApplicationMcpHarness(): Promise { + const session = createTestBinarySession(() => ({ + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(), + })); + const server = createServer(session, session); + const client = new Client({ + name: "application-workflow-test", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + + await server.connect(serverTransport); + await client.connect(clientTransport); + + return { + client, + session, + close: async () => { + await Promise.all([client.close(), server.close(), session.close()]); + }, + }; +}