fix(reconstruction): reject contradictory function comparison status (#2798)
* fix(reconstruction): reject contradictory function comparison status * test(reconstruction): cover imported function match precedence --------- Co-authored-by: maxazure <405711+maxazure@users.noreply.github.com>
This commit is contained in:
Liu Jie
GitHub
parent
1eca600515
commit
173da755cd
|
|
@ -999,3 +999,51 @@ const expectFunctionDimensionAlgebra = (
|
|||
}).success,
|
||||
).toBe(false);
|
||||
};
|
||||
|
||||
it.each([
|
||||
{ match: "mismatched", changedDimensions: false, expected: "changed" },
|
||||
{ match: "ambiguous", changedDimensions: false, expected: "unknown" },
|
||||
{ match: "ambiguous", changedDimensions: true, expected: "unknown" },
|
||||
] as const)(
|
||||
"validates $match match precedence with changed dimensions=$changedDimensions",
|
||||
({ match, changedDimensions, expected }) => {
|
||||
const result = compareFunctions(
|
||||
FUNCTION_COMPARISON_EXAMPLE.left,
|
||||
changedDimensions
|
||||
? FUNCTION_COMPARISON_EXAMPLE.right
|
||||
: FUNCTION_COMPARISON_EXAMPLE.left,
|
||||
);
|
||||
const candidate = {
|
||||
...result,
|
||||
function_match: {
|
||||
status: match,
|
||||
method: "explicit",
|
||||
left_name: "main",
|
||||
right_name: "other",
|
||||
},
|
||||
};
|
||||
expect(
|
||||
functionComparisonResultSchema.safeParse({
|
||||
...candidate,
|
||||
status: expected,
|
||||
}).success,
|
||||
).toBe(true);
|
||||
for (const status of ["unchanged", "changed", "unknown"] as const) {
|
||||
if (status === expected) continue;
|
||||
const parsed = functionComparisonResultSchema.safeParse({
|
||||
...candidate,
|
||||
status,
|
||||
});
|
||||
expect(parsed.success).toBe(false);
|
||||
if (!parsed.success)
|
||||
expect(parsed.error.issues).toMatchObject([
|
||||
{
|
||||
path: ["status"],
|
||||
code: "custom",
|
||||
message:
|
||||
"Function comparison status contradicts its dimensions or function match.",
|
||||
},
|
||||
]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { z } from "zod";
|
||||
import { overallStatus } from "./functionComparisonResults.js";
|
||||
import { evidenceSchema } from "./evidence.js";
|
||||
import { digestSchema } from "./../domain/digests.js";
|
||||
import { prefixedDigestSchema } from "./../domain/digests.js";
|
||||
|
|
@ -79,20 +80,33 @@ const functionMatchSchema = z.union([
|
|||
]);
|
||||
|
||||
/** Deterministic dimension-classified function comparison Evidence payload. */
|
||||
export const functionComparisonResultSchema = z.object({
|
||||
status: comparisonStatusSchema,
|
||||
function_match: functionMatchSchema,
|
||||
left_subject_sha256: digestSchema,
|
||||
right_subject_sha256: digestSchema,
|
||||
summary: z.object({
|
||||
unchanged: z.number().int().min(0),
|
||||
changed: z.number().int().min(0),
|
||||
unknown: z.number().int().min(0),
|
||||
}),
|
||||
dimensions: z.array(functionDimensionSchema).length(8),
|
||||
changes: z.array(functionDimensionSchema),
|
||||
limitations: z.array(z.string()),
|
||||
});
|
||||
export const functionComparisonResultSchema = z
|
||||
.object({
|
||||
status: comparisonStatusSchema,
|
||||
function_match: functionMatchSchema,
|
||||
left_subject_sha256: digestSchema,
|
||||
right_subject_sha256: digestSchema,
|
||||
summary: z.object({
|
||||
unchanged: z.number().int().min(0),
|
||||
changed: z.number().int().min(0),
|
||||
unknown: z.number().int().min(0),
|
||||
}),
|
||||
dimensions: z.array(functionDimensionSchema).length(8),
|
||||
changes: z.array(functionDimensionSchema),
|
||||
limitations: z.array(z.string()),
|
||||
})
|
||||
.superRefine((result, context) => {
|
||||
if (
|
||||
result.status !==
|
||||
overallStatus(result.dimensions, result.function_match.status)
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["status"],
|
||||
message:
|
||||
"Function comparison status contradicts its dimensions or function match.",
|
||||
});
|
||||
});
|
||||
|
||||
export type FunctionComparisonResult = z.infer<
|
||||
typeof functionComparisonResultSchema
|
||||
|
|
|
|||
|
|
@ -1,6 +1,14 @@
|
|||
import { ok as resultOk } from "../../../src/domain/result.js";
|
||||
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, onTestFinished } from "vitest";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
|
||||
import { compareFunctions } from "../../../src/domain/functionComparison.js";
|
||||
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
||||
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
|
||||
|
|
@ -271,3 +279,130 @@ describe("function comparison MCP integration", () => {
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ status: "changed", expected: "fail" },
|
||||
{ status: "unknown", expected: "unknown" },
|
||||
{ status: "unchanged", expected: "pass" },
|
||||
] as const)(
|
||||
"checks the aggregate status of imported $status function comparisons",
|
||||
async ({ status, expected }) => {
|
||||
const left = FUNCTION_COMPARISON_EXAMPLE.left;
|
||||
const right =
|
||||
status === "changed"
|
||||
? FUNCTION_COMPARISON_EXAMPLE.right
|
||||
: createEvidence(
|
||||
{
|
||||
path: "/fixture/function-right",
|
||||
sha256: "3".repeat(64),
|
||||
format: "mach-o",
|
||||
},
|
||||
status === "unknown"
|
||||
? { ...left.provider, id: "other-provider" }
|
||||
: left.provider,
|
||||
{
|
||||
operation: left.operation,
|
||||
parameters: left.parameters,
|
||||
result: left.normalized_result,
|
||||
confidence: left.confidence,
|
||||
authority: left.authority,
|
||||
relationshipCoverage: { status: "complete" },
|
||||
},
|
||||
);
|
||||
const result = compareFunctions(left, right);
|
||||
expect(result.status).toBe(status);
|
||||
const provider = {
|
||||
id: "rea-function-comparison",
|
||||
name: "REA function comparison",
|
||||
version: "1",
|
||||
};
|
||||
const observation = {
|
||||
operation: "compare_functions",
|
||||
predicateType: "rea.function-comparison",
|
||||
parameters: {
|
||||
left_evidence_id: left.evidence_id,
|
||||
right_evidence_id: right.evidence_id,
|
||||
},
|
||||
confidence: "derived" as const,
|
||||
authority: "analyst-inference" as const,
|
||||
evidenceLinks: [left.evidence_id, right.evidence_id],
|
||||
};
|
||||
const consistent = createEvidence(undefined, provider, {
|
||||
...observation,
|
||||
result: jsonValueSchema.parse(result),
|
||||
});
|
||||
const inconsistent =
|
||||
status === "unchanged"
|
||||
? null
|
||||
: createEvidence(undefined, provider, {
|
||||
...observation,
|
||||
result: jsonValueSchema.parse({ ...result, status: "unchanged" }),
|
||||
});
|
||||
const directory = await createTestTempDirectory("rea-function-status-");
|
||||
const path = join(directory, "bundle.json");
|
||||
await writeFile(
|
||||
path,
|
||||
JSON.stringify(
|
||||
createEvidenceBundle([
|
||||
left,
|
||||
right,
|
||||
consistent,
|
||||
...(inconsistent === null ? [] : [inconsistent]),
|
||||
]),
|
||||
),
|
||||
);
|
||||
const session = createTestBinarySession(() => {
|
||||
throw new Error("Imported comparison must not launch a provider");
|
||||
});
|
||||
const harness = await createApplicationMcpHarness({ session });
|
||||
onTestFinished(harness.close);
|
||||
const imported = await harness.client.callTool({
|
||||
name: "import_evidence_bundle",
|
||||
arguments: { path },
|
||||
});
|
||||
expect(imported.isError).not.toBe(true);
|
||||
const verification = (evidence_id: string) =>
|
||||
harness.client.callTool({
|
||||
name: "verify_reconstruction",
|
||||
arguments: {
|
||||
specification: {
|
||||
name: "Function aggregate",
|
||||
claims: [
|
||||
{
|
||||
kind: "structural-function",
|
||||
claim_id: "overall",
|
||||
title: "Function agrees",
|
||||
comparison_evidence_id: evidence_id,
|
||||
dimension: "overall",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
if (inconsistent !== null) {
|
||||
const rejected = await verification(inconsistent.evidence_id);
|
||||
expect(parseMcpToolError(rejected).error).toMatchObject({
|
||||
code: "invalid_request",
|
||||
details: {
|
||||
issues: [
|
||||
{
|
||||
reason: "invalid_value",
|
||||
message: expect.stringContaining(
|
||||
"Function comparison status contradicts its dimensions or function match",
|
||||
),
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(harness.session.evidenceById(inconsistent.evidence_id)).toEqual(
|
||||
inconsistent,
|
||||
);
|
||||
}
|
||||
const verified = await verification(consistent.evidence_id);
|
||||
expect(verified.isError).not.toBe(true);
|
||||
expect(
|
||||
parseEvidence(verified.structuredContent).normalized_result,
|
||||
).toMatchObject({ status: expected });
|
||||
await harness.client.ping();
|
||||
},
|
||||
);
|
||||
|
|
|
|||
Loading…
Reference in New Issue