438 lines
15 KiB
TypeScript
438 lines
15 KiB
TypeScript
import { ok as resultOk } from "../../../src/domain/result.js";
|
|
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
|
import { describe, expect, it, onTestFinished } from "vitest";
|
|
import { writeFile } from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
|
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
|
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
|
import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
|
|
import { compareFunctions } from "../../../src/domain/functionComparison.js";
|
|
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
|
|
|
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
|
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
|
|
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
|
|
import { createServer } from "../../../src/server/createServer.js";
|
|
import { observed } from "../../fixtures/analysisExecution.js";
|
|
import { reconstructionVerificationResultSchema } from "../../../src/domain/reconstructionVerificationSchemas.js";
|
|
|
|
it.each([
|
|
{ comparisonStatus: "unchanged", verificationStatus: "pass" },
|
|
{ comparisonStatus: "changed", verificationStatus: "fail" },
|
|
] as const)(
|
|
"verifies public $comparisonStatus function comparison as $verificationStatus",
|
|
async ({ comparisonStatus, verificationStatus }) => {
|
|
let providerLaunches = 0;
|
|
const session = createTestBinarySession(() => {
|
|
providerLaunches += 1;
|
|
throw new Error("Function verification must not launch a provider");
|
|
});
|
|
const left = FUNCTION_COMPARISON_EXAMPLE.left;
|
|
const right =
|
|
comparisonStatus === "changed"
|
|
? FUNCTION_COMPARISON_EXAMPLE.right
|
|
: createEvidence(
|
|
{
|
|
path: "/tmp/function-unchanged",
|
|
sha256: "2".repeat(64),
|
|
format: "mach-o",
|
|
},
|
|
left.provider,
|
|
{
|
|
operation: left.operation,
|
|
parameters: left.parameters,
|
|
result: left.normalized_result,
|
|
confidence: left.confidence,
|
|
authority: left.authority,
|
|
...(left.relationship_coverage === null
|
|
? {}
|
|
: { relationshipCoverage: left.relationship_coverage }),
|
|
},
|
|
);
|
|
const server = createServer({ kind: "session", session });
|
|
const client = new Client({ name: "function-claims", version: "1" });
|
|
const [clientTransport, serverTransport] =
|
|
InMemoryTransport.createLinkedPair();
|
|
try {
|
|
await server.connect(serverTransport);
|
|
await client.connect(clientTransport);
|
|
const compared = await client.callTool({
|
|
name: "compare_functions",
|
|
arguments: { left, right },
|
|
});
|
|
expect(compared.isError).not.toBe(true);
|
|
const comparison = parseEvidence(compared.structuredContent);
|
|
expect(comparison.limitations).not.toContainEqual(
|
|
expect.stringContaining(
|
|
"without address or generated-name normalization",
|
|
),
|
|
);
|
|
expect(comparison.normalized_result).toMatchObject({
|
|
status: comparisonStatus,
|
|
dimensions: expect.arrayContaining(
|
|
["pseudocode", "assembly"].map((dimension) =>
|
|
expect.objectContaining({
|
|
dimension,
|
|
limitations: expect.arrayContaining([
|
|
expect.stringContaining(
|
|
"without address or generated-name normalization",
|
|
),
|
|
]),
|
|
}),
|
|
),
|
|
),
|
|
});
|
|
expect(comparison.parameters).toEqual({
|
|
left_evidence_id: left.evidence_id,
|
|
right_evidence_id: right.evidence_id,
|
|
});
|
|
expect(comparison.evidence_links).toEqual(
|
|
expect.arrayContaining([left.evidence_id, right.evidence_id]),
|
|
);
|
|
|
|
const verified = await client.callTool({
|
|
name: "verify_reconstruction",
|
|
arguments: {
|
|
specification: {
|
|
name: "Function pseudocode comparison",
|
|
claims: ["pseudocode", "assembly", "cfg", "overall"].map(
|
|
(dimension) => ({
|
|
kind: "structural-function",
|
|
claim_id: dimension,
|
|
title: `${dimension} agrees`,
|
|
comparison_evidence_id: comparison.evidence_id,
|
|
dimension,
|
|
}),
|
|
),
|
|
},
|
|
},
|
|
});
|
|
expect(
|
|
verified.isError,
|
|
JSON.stringify(verified.structuredContent),
|
|
).not.toBe(true);
|
|
const verification = parseEvidence(verified.structuredContent);
|
|
expect(verification.normalized_result).toMatchObject({
|
|
status: verificationStatus,
|
|
summary: {
|
|
total: 4,
|
|
passed: verificationStatus === "pass" ? 4 : 2,
|
|
failed: verificationStatus === "fail" ? 2 : 0,
|
|
unknown: 0,
|
|
},
|
|
claims: {
|
|
items: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
dimension: "pseudocode",
|
|
status: verificationStatus,
|
|
observed_status: comparisonStatus,
|
|
comparison_evidence_id: comparison.evidence_id,
|
|
left_evidence_ids: [left.evidence_id],
|
|
right_evidence_ids: [right.evidence_id],
|
|
limitations: expect.arrayContaining([
|
|
expect.stringContaining(
|
|
"without address or generated-name normalization",
|
|
),
|
|
]),
|
|
}),
|
|
]),
|
|
},
|
|
});
|
|
const claims = reconstructionVerificationResultSchema.parse(
|
|
verification.normalized_result,
|
|
).claims.items;
|
|
for (const claim of claims) {
|
|
const caveats = claim.limitations.join(" ");
|
|
if (claim.dimension === "cfg")
|
|
expect(caveats).not.toMatch(/generated-name normalization/u);
|
|
else expect(caveats).toMatch(/generated-name normalization/u);
|
|
if (claim.dimension === "assembly" || claim.dimension === "overall")
|
|
expect(caveats).toMatch(/Assembly is opaque/u);
|
|
else expect(caveats).not.toMatch(/Assembly is opaque/u);
|
|
}
|
|
expect(
|
|
session
|
|
.exportEvidenceBundle()
|
|
.records.find(({ evidence_id: id }) => id === comparison.evidence_id),
|
|
).toEqual(comparison);
|
|
} finally {
|
|
await Promise.allSettled([
|
|
client.close(),
|
|
server.close(),
|
|
session.close(),
|
|
]);
|
|
expect(providerLaunches).toBe(0);
|
|
}
|
|
},
|
|
);
|
|
|
|
describe("function comparison MCP integration", () => {
|
|
it("records linked comparison Evidence and its residual unknown", async () => {
|
|
const session = createTestBinarySession(() => ({
|
|
health: () => Promise.resolve(),
|
|
execute: () => Promise.resolve(observed(null)),
|
|
close: () => Promise.resolve(resultOk(null)),
|
|
}));
|
|
expect(session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.left).ok).toBe(
|
|
true,
|
|
);
|
|
expect(session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.right).ok).toBe(
|
|
true,
|
|
);
|
|
const server = createServer({ kind: "session", session });
|
|
const client = new Client({
|
|
name: "function-comparison-test",
|
|
version: "1",
|
|
});
|
|
const [clientTransport, serverTransport] =
|
|
InMemoryTransport.createLinkedPair();
|
|
try {
|
|
await server.connect(serverTransport);
|
|
await client.connect(clientTransport);
|
|
const result = await client.callTool({
|
|
name: "compare_functions",
|
|
arguments: {
|
|
left: FUNCTION_COMPARISON_EXAMPLE.left,
|
|
right: FUNCTION_COMPARISON_EXAMPLE.right,
|
|
},
|
|
});
|
|
expect(result.isError).not.toBe(true);
|
|
expect(result.structuredContent).toMatchObject({
|
|
normalized_result: { status: expect.any(String) },
|
|
evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u),
|
|
});
|
|
const unknowns = await client.callTool({
|
|
name: "list_unknowns",
|
|
arguments: { domain: "function-comparison" },
|
|
});
|
|
expect(unknowns.structuredContent).toMatchObject({
|
|
result: {
|
|
items: [
|
|
expect.objectContaining({
|
|
domain: "function-comparison",
|
|
recommended_probes: [
|
|
expect.objectContaining({
|
|
rationale:
|
|
"Capture complete dossiers for both functions under the same target context.",
|
|
}),
|
|
],
|
|
}),
|
|
],
|
|
},
|
|
});
|
|
} finally {
|
|
await Promise.allSettled([
|
|
client.close(),
|
|
server.close(),
|
|
session.close(),
|
|
]);
|
|
}
|
|
});
|
|
|
|
it("accepts inline Evidence without prior session registration", async () => {
|
|
const session = createTestBinarySession(() => ({
|
|
health: () => Promise.resolve(),
|
|
execute: () => Promise.resolve(observed(null)),
|
|
close: () => Promise.resolve(resultOk(null)),
|
|
}));
|
|
const server = createServer({ kind: "session", session });
|
|
const client = new Client({ name: "function-link-test", version: "1" });
|
|
const [clientTransport, serverTransport] =
|
|
InMemoryTransport.createLinkedPair();
|
|
try {
|
|
await server.connect(serverTransport);
|
|
await client.connect(clientTransport);
|
|
const result = await client.callTool({
|
|
name: "compare_functions",
|
|
arguments: {
|
|
left: FUNCTION_COMPARISON_EXAMPLE.left,
|
|
right: FUNCTION_COMPARISON_EXAMPLE.right,
|
|
},
|
|
});
|
|
expect(result.isError).not.toBe(true);
|
|
expect(result.structuredContent).toMatchObject({
|
|
normalized_result: { status: expect.any(String) },
|
|
});
|
|
} finally {
|
|
await Promise.allSettled([
|
|
client.close(),
|
|
server.close(),
|
|
session.close(),
|
|
]);
|
|
}
|
|
});
|
|
|
|
it("rejects inline Evidence with the wrong operation", async () => {
|
|
const session = createTestBinarySession(() => ({
|
|
health: () => Promise.resolve(),
|
|
execute: () => Promise.resolve(observed(null)),
|
|
close: () => Promise.resolve(resultOk(null)),
|
|
}));
|
|
const wrong = createEvidence(
|
|
undefined,
|
|
{ id: "fixture", name: "Fixture", version: "1" },
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: {},
|
|
result: {},
|
|
},
|
|
);
|
|
session.recordEvidence(wrong);
|
|
session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.right);
|
|
const server = createServer({ kind: "session", session });
|
|
const client = new Client({
|
|
name: "function-authority-test",
|
|
version: "1",
|
|
});
|
|
const [clientTransport, serverTransport] =
|
|
InMemoryTransport.createLinkedPair();
|
|
try {
|
|
await server.connect(serverTransport);
|
|
await client.connect(clientTransport);
|
|
const result = await client.callTool({
|
|
name: "compare_functions",
|
|
arguments: {
|
|
left: wrong,
|
|
right: FUNCTION_COMPARISON_EXAMPLE.right,
|
|
},
|
|
});
|
|
expect(result.isError).toBe(true);
|
|
expect(session.exportEvidenceBundle().records).toHaveLength(2);
|
|
} finally {
|
|
await Promise.allSettled([
|
|
client.close(),
|
|
server.close(),
|
|
session.close(),
|
|
]);
|
|
}
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
{ status: "changed", expected: "fail" },
|
|
{ status: "unknown", expected: "unknown" },
|
|
{ status: "unchanged", expected: "pass" },
|
|
] as const)(
|
|
"checks the aggregate status of imported $status function comparisons",
|
|
async ({ status, expected }) => {
|
|
const left = FUNCTION_COMPARISON_EXAMPLE.left;
|
|
const right =
|
|
status === "changed"
|
|
? FUNCTION_COMPARISON_EXAMPLE.right
|
|
: createEvidence(
|
|
{
|
|
path: "/fixture/function-right",
|
|
sha256: "3".repeat(64),
|
|
format: "mach-o",
|
|
},
|
|
status === "unknown"
|
|
? { ...left.provider, id: "other-provider" }
|
|
: left.provider,
|
|
{
|
|
operation: left.operation,
|
|
parameters: left.parameters,
|
|
result: left.normalized_result,
|
|
confidence: left.confidence,
|
|
authority: left.authority,
|
|
relationshipCoverage: { status: "complete" },
|
|
},
|
|
);
|
|
const result = compareFunctions(left, right);
|
|
expect(result.status).toBe(status);
|
|
const provider = {
|
|
id: "rea-function-comparison",
|
|
name: "REA function comparison",
|
|
version: "1",
|
|
};
|
|
const observation = {
|
|
operation: "compare_functions",
|
|
predicateType: "rea.function-comparison",
|
|
parameters: {
|
|
left_evidence_id: left.evidence_id,
|
|
right_evidence_id: right.evidence_id,
|
|
},
|
|
confidence: "derived" as const,
|
|
authority: "analyst-inference" as const,
|
|
evidenceLinks: [left.evidence_id, right.evidence_id],
|
|
};
|
|
const consistent = createEvidence(undefined, provider, {
|
|
...observation,
|
|
result: jsonValueSchema.parse(result),
|
|
});
|
|
const inconsistent =
|
|
status === "unchanged"
|
|
? null
|
|
: createEvidence(undefined, provider, {
|
|
...observation,
|
|
result: jsonValueSchema.parse({ ...result, status: "unchanged" }),
|
|
});
|
|
const directory = await createTestTempDirectory("rea-function-status-");
|
|
const path = join(directory, "bundle.json");
|
|
await writeFile(
|
|
path,
|
|
JSON.stringify(
|
|
createEvidenceBundle([
|
|
left,
|
|
right,
|
|
consistent,
|
|
...(inconsistent === null ? [] : [inconsistent]),
|
|
]),
|
|
),
|
|
);
|
|
const session = createTestBinarySession(() => {
|
|
throw new Error("Imported comparison must not launch a provider");
|
|
});
|
|
const harness = await createApplicationMcpHarness({ session });
|
|
onTestFinished(harness.close);
|
|
const imported = await harness.client.callTool({
|
|
name: "import_evidence_bundle",
|
|
arguments: { path },
|
|
});
|
|
expect(imported.isError).not.toBe(true);
|
|
const verification = (evidence_id: string) =>
|
|
harness.client.callTool({
|
|
name: "verify_reconstruction",
|
|
arguments: {
|
|
specification: {
|
|
name: "Function aggregate",
|
|
claims: [
|
|
{
|
|
kind: "structural-function",
|
|
claim_id: "overall",
|
|
title: "Function agrees",
|
|
comparison_evidence_id: evidence_id,
|
|
dimension: "overall",
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
if (inconsistent !== null) {
|
|
const rejected = await verification(inconsistent.evidence_id);
|
|
expect(parseMcpToolError(rejected).error).toMatchObject({
|
|
code: "invalid_request",
|
|
details: {
|
|
issues: [
|
|
{
|
|
reason: "invalid_value",
|
|
message: expect.stringContaining(
|
|
"Function comparison status contradicts its dimensions or function match",
|
|
),
|
|
},
|
|
],
|
|
},
|
|
});
|
|
expect(harness.session.evidenceById(inconsistent.evidence_id)).toEqual(
|
|
inconsistent,
|
|
);
|
|
}
|
|
const verified = await verification(consistent.evidence_id);
|
|
expect(verified.isError).not.toBe(true);
|
|
expect(
|
|
parseEvidence(verified.structuredContent).normalized_result,
|
|
).toMatchObject({ status: expected });
|
|
await harness.client.ping();
|
|
},
|
|
);
|