From 173da755cd6b922f8310a1efa0fb8435be2e4d64 Mon Sep 17 00:00:00 2001 From: Liu Jie Date: Wed, 14 Oct 2026 03:01:41 +1300 Subject: [PATCH] fix(reconstruction): reject contradictory function comparison status (#2798) * fix(reconstruction): reject contradictory function comparison status * test(reconstruction): cover imported function match precedence --------- Co-authored-by: maxazure <405711+maxazure@users.noreply.github.com> --- src/domain/functionComparison.test.ts | 48 ++++++ src/domain/functionComparisonSchemas.ts | 42 ++++-- .../mcp/functionComparisonMcp.test.ts | 137 +++++++++++++++++- 3 files changed, 212 insertions(+), 15 deletions(-) diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index 677fdc0cc..954717bad 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -999,3 +999,51 @@ const expectFunctionDimensionAlgebra = ( }).success, ).toBe(false); }; + +it.each([ + { match: "mismatched", changedDimensions: false, expected: "changed" }, + { match: "ambiguous", changedDimensions: false, expected: "unknown" }, + { match: "ambiguous", changedDimensions: true, expected: "unknown" }, +] as const)( + "validates $match match precedence with changed dimensions=$changedDimensions", + ({ match, changedDimensions, expected }) => { + const result = compareFunctions( + FUNCTION_COMPARISON_EXAMPLE.left, + changedDimensions + ? FUNCTION_COMPARISON_EXAMPLE.right + : FUNCTION_COMPARISON_EXAMPLE.left, + ); + const candidate = { + ...result, + function_match: { + status: match, + method: "explicit", + left_name: "main", + right_name: "other", + }, + }; + expect( + functionComparisonResultSchema.safeParse({ + ...candidate, + status: expected, + }).success, + ).toBe(true); + for (const status of ["unchanged", "changed", "unknown"] as const) { + if (status === expected) continue; + const parsed = functionComparisonResultSchema.safeParse({ + ...candidate, + status, + }); + expect(parsed.success).toBe(false); + if (!parsed.success) + expect(parsed.error.issues).toMatchObject([ + { + path: ["status"], + code: "custom", + message: + "Function comparison status contradicts its dimensions or function match.", + }, + ]); + } + }, +); diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts index bafc3def1..844bdc9b9 100644 --- a/src/domain/functionComparisonSchemas.ts +++ b/src/domain/functionComparisonSchemas.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { overallStatus } from "./functionComparisonResults.js"; import { evidenceSchema } from "./evidence.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -79,20 +80,33 @@ const functionMatchSchema = z.union([ ]); /** Deterministic dimension-classified function comparison Evidence payload. */ -export const functionComparisonResultSchema = z.object({ - status: comparisonStatusSchema, - function_match: functionMatchSchema, - left_subject_sha256: digestSchema, - right_subject_sha256: digestSchema, - summary: z.object({ - unchanged: z.number().int().min(0), - changed: z.number().int().min(0), - unknown: z.number().int().min(0), - }), - dimensions: z.array(functionDimensionSchema).length(8), - changes: z.array(functionDimensionSchema), - limitations: z.array(z.string()), -}); +export const functionComparisonResultSchema = z + .object({ + status: comparisonStatusSchema, + function_match: functionMatchSchema, + left_subject_sha256: digestSchema, + right_subject_sha256: digestSchema, + summary: z.object({ + unchanged: z.number().int().min(0), + changed: z.number().int().min(0), + unknown: z.number().int().min(0), + }), + dimensions: z.array(functionDimensionSchema).length(8), + changes: z.array(functionDimensionSchema), + limitations: z.array(z.string()), + }) + .superRefine((result, context) => { + if ( + result.status !== + overallStatus(result.dimensions, result.function_match.status) + ) + context.addIssue({ + code: "custom", + path: ["status"], + message: + "Function comparison status contradicts its dimensions or function match.", + }); + }); export type FunctionComparisonResult = z.infer< typeof functionComparisonResultSchema diff --git a/tests/boundary/mcp/functionComparisonMcp.test.ts b/tests/boundary/mcp/functionComparisonMcp.test.ts index 8ec10f402..97f07dc1e 100644 --- a/tests/boundary/mcp/functionComparisonMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonMcp.test.ts @@ -1,6 +1,14 @@ import { ok as resultOk } from "../../../src/domain/result.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, onTestFinished } from "vitest"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { compareFunctions } from "../../../src/domain/functionComparison.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; @@ -271,3 +279,130 @@ describe("function comparison MCP integration", () => { } }); }); + +it.each([ + { status: "changed", expected: "fail" }, + { status: "unknown", expected: "unknown" }, + { status: "unchanged", expected: "pass" }, +] as const)( + "checks the aggregate status of imported $status function comparisons", + async ({ status, expected }) => { + const left = FUNCTION_COMPARISON_EXAMPLE.left; + const right = + status === "changed" + ? FUNCTION_COMPARISON_EXAMPLE.right + : createEvidence( + { + path: "/fixture/function-right", + sha256: "3".repeat(64), + format: "mach-o", + }, + status === "unknown" + ? { ...left.provider, id: "other-provider" } + : left.provider, + { + operation: left.operation, + parameters: left.parameters, + result: left.normalized_result, + confidence: left.confidence, + authority: left.authority, + relationshipCoverage: { status: "complete" }, + }, + ); + const result = compareFunctions(left, right); + expect(result.status).toBe(status); + const provider = { + id: "rea-function-comparison", + name: "REA function comparison", + version: "1", + }; + const observation = { + operation: "compare_functions", + predicateType: "rea.function-comparison", + parameters: { + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + }, + confidence: "derived" as const, + authority: "analyst-inference" as const, + evidenceLinks: [left.evidence_id, right.evidence_id], + }; + const consistent = createEvidence(undefined, provider, { + ...observation, + result: jsonValueSchema.parse(result), + }); + const inconsistent = + status === "unchanged" + ? null + : createEvidence(undefined, provider, { + ...observation, + result: jsonValueSchema.parse({ ...result, status: "unchanged" }), + }); + const directory = await createTestTempDirectory("rea-function-status-"); + const path = join(directory, "bundle.json"); + await writeFile( + path, + JSON.stringify( + createEvidenceBundle([ + left, + right, + consistent, + ...(inconsistent === null ? [] : [inconsistent]), + ]), + ), + ); + const session = createTestBinarySession(() => { + throw new Error("Imported comparison must not launch a provider"); + }); + const harness = await createApplicationMcpHarness({ session }); + onTestFinished(harness.close); + const imported = await harness.client.callTool({ + name: "import_evidence_bundle", + arguments: { path }, + }); + expect(imported.isError).not.toBe(true); + const verification = (evidence_id: string) => + harness.client.callTool({ + name: "verify_reconstruction", + arguments: { + specification: { + name: "Function aggregate", + claims: [ + { + kind: "structural-function", + claim_id: "overall", + title: "Function agrees", + comparison_evidence_id: evidence_id, + dimension: "overall", + }, + ], + }, + }, + }); + if (inconsistent !== null) { + const rejected = await verification(inconsistent.evidence_id); + expect(parseMcpToolError(rejected).error).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + reason: "invalid_value", + message: expect.stringContaining( + "Function comparison status contradicts its dimensions or function match", + ), + }, + ], + }, + }); + expect(harness.session.evidenceById(inconsistent.evidence_id)).toEqual( + inconsistent, + ); + } + const verified = await verification(consistent.evidence_id); + expect(verified.isError).not.toBe(true); + expect( + parseEvidence(verified.structuredContent).normalized_result, + ).toMatchObject({ status: expected }); + await harness.client.ping(); + }, +);