shifathosn
/
rea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(artifacts): compare leaf inventories using explicit coverage 

Classify coverage once at inventory production and require it in current outputs. Normalize older retained Evidence at the inventory parser, reject inconsistent page statuses, and remove comparison heuristics over limitation prose.

Validation: public CLI-to-MCP plist comparison, changed-behavior and unknowns workflow; boundary checks for legacy Evidence, partial coverage, page disagreement, content and executable changes; generated docs checks. Fixes #2173.
This commit is contained in:
morluto 2026-10-13 03:47:21 +08:00
parent 8d4d631606
commit ed485a3dc8
11 changed files with 379 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/artifacts/inventory/ArtifactInventory.ts
View File

@ -1,8 +1,8 @@
import { realpath } from "node:fs/promises";
import {
artifactInventoryResultSchema,
type ArtifactInventoryResult,
artifactInventoryOutputSchema,
type ArtifactInventoryOutput,
} from "../../domain/artifactGraph.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { scanCanonicalArtifactInventoryInScope } from "./scanCanonical.js";
@ -13,9 +13,9 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySn
export const inventoryArtifact = async (
inputPath: string,
options: ArtifactInventoryOptions,
): Promise<ArtifactInventoryResult> => {
): Promise<ArtifactInventoryOutput> => {
const snapshot = await scanArtifactInventory(inputPath, options);
return artifactInventoryResultSchema.parse(snapshot);
return artifactInventoryOutputSchema.parse(snapshot);
};
/** Scan an artifact once and retain the complete immutable graph for projection. */

54
src/artifacts/inventory/reader.ts
View File

@ -4,7 +4,11 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
import { ZipArtifactReader } from "../ZipArtifactReader.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
import type {
ArtifactInventoryCoverage,
ArtifactOccurrence,
IntegrityContradiction,
} from "../../domain/artifactGraph.js";
import {
isZipPackageFormat,
ZIP_NON_ENTRY_TAIL_LIMITATION,
@ -36,20 +40,42 @@ export const createReader = (
}
};
export const inventoryLimitations = (
/** Classify source coverage and preserve its user-facing explanation once. */
export const inventoryAssessment = (
format: ArtifactOccurrence["artifact_format"],
reader: ArtifactReader | undefined,
): string[] => {
if (reader !== undefined)
return isZipPackageFormat(reader.format)
? [ZIP_NON_ENTRY_TAIL_LIMITATION]
: [];
// dmg and pkg reach here only after classifyRootFormat matched their bytes.
if (format === "dmg" || format === "pkg")
return [
integrityContradictions: readonly IntegrityContradiction[],
caseCollisionLimitation: string | undefined,
): {
readonly coverage_status: ArtifactInventoryCoverage;
readonly limitations: string[];
} => {
let coverageStatus: ArtifactInventoryCoverage = "complete";
const limitations: string[] = [];
if (reader !== undefined) {
if (isZipPackageFormat(reader.format))
limitations.push(ZIP_NON_ENTRY_TAIL_LIMITATION);
} else if (format === "dmg" || format === "pkg") {
coverageStatus = "partial";
limitations.push(
`${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`,
];
if (format === "mach-o-universal" && process.platform !== "darwin")
return ["Universal Mach-O slices require the native macOS lipo adapter."];
return ["Artifact has no child container entries."];
);
} else if (format === "mach-o-universal" && process.platform !== "darwin") {
coverageStatus = "partial";
limitations.push(
"Universal Mach-O slices require the native macOS lipo adapter.",
);
} else {
// The graph's modeled scope is the root itself for recognized leaf formats.
limitations.push("Artifact has no child container entries.");
}
if (caseCollisionLimitation !== undefined) {
coverageStatus = "partial";
limitations.push(caseCollisionLimitation);
}
if (integrityContradictions.length > 0)
limitations.push(
`${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`,
);
return { coverage_status: coverageStatus, limitations };
};

31
src/artifacts/inventory/scanCanonical.ts
View File

@ -40,7 +40,7 @@ import {
hashStableRootArtifact,
hashStableRootArtifactHandle,
} from "./hashStableRootArtifact.js";
import { createReader, inventoryLimitations } from "./reader.js";
import { createReader, inventoryAssessment } from "./reader.js";
import {
scanReader,
type PendingIntegrityContradiction,
@ -268,6 +268,12 @@ const buildInventorySnapshot = async (
orderedOccurrences,
orderedEdges,
});
const assessment = inventoryAssessment(
rootFormat,
reader,
integrityContradictions,
input.caseCollisionLimitation,
);
return {
manifest,
nodes: orderedNodes,
@ -275,12 +281,8 @@ const buildInventorySnapshot = async (
edges: orderedEdges,
provenance: reader?.provenance() ?? [],
integrity_contradictions: integrityContradictions,
limitations: buildLimitations(
rootFormat,
reader,
integrityContradictions,
input.caseCollisionLimitation,
),
coverage_status: assessment.coverage_status,
limitations: assessment.limitations,
};
};
@ -393,18 +395,3 @@ const buildManifest = ({
occurrence_count: orderedOccurrences.length,
edge_count: orderedEdges.length,
});
const buildLimitations = (
rootFormat: ArtifactOccurrence["artifact_format"],
reader: ArtifactReader | undefined,
integrityContradictions: readonly IntegrityContradiction[],
caseCollisionLimitation: string | undefined,
): string[] => [
...inventoryLimitations(rootFormat, reader),
...(caseCollisionLimitation === undefined ? [] : [caseCollisionLimitation]),
...(integrityContradictions.length === 0
? []
: [
`${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`,
]),
];

12
src/domain/artifactComparison.ts
View File

@ -13,7 +13,6 @@ import {
} from "./artifactInventoryEvidence.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
import { canonicalJson } from "./comparisonSemantics.js";
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "./zipPackageFormat.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const comparisonStatusSchema = z.enum([
@ -105,9 +104,9 @@ export const compareArtifacts = (
const leftComplete = left.inventory.complete;
const rightComplete = right.inventory.complete;
const leftCovered =
leftComplete && !hasCoverageLimitation(left.inventory.limitations);
leftComplete && left.inventory.coverage_status === "complete";
const rightCovered =
rightComplete && !hasCoverageLimitation(right.inventory.limitations);
rightComplete && right.inventory.coverage_status === "complete";
const limitations = [
...left.inventory.limitations.map((item) => `Left: ${item}`),
...right.inventory.limitations.map((item) => `Right: ${item}`),
@ -433,13 +432,6 @@ const summarize = (
};
};
const hasCoverageLimitation = (limitations: readonly string[]): boolean =>
limitations.some(
(limitation) =>
limitation !== ZIP_NON_ENTRY_TAIL_LIMITATION &&
!/integrity contradiction\(s\) were recorded/u.test(limitation),
);
// Differing manifest root digests prove a root content change even when node
// details are unavailable. Fold that fact into an existing root row so one
// logical path keeps one classification.

17
src/domain/artifactGraph.ts
View File

@ -188,6 +188,12 @@ export const artifactGraphManifestSchema = z.object({
edge_count: z.number().int().min(0),
});
/** Whether the inventory producer covered its modeled child-entry scope. */
export const artifactInventoryCoverageSchema = z.enum(["complete", "partial"]);
export type ArtifactInventoryCoverage = z.infer<
typeof artifactInventoryCoverageSchema
>;
/** Complete inventory observation containing the stable artifact graph. */
export const artifactInventoryResultSchema = z.object({
manifest: artifactGraphManifestSchema,
@ -197,8 +203,19 @@ export const artifactInventoryResultSchema = z.object({
provenance: z.array(artifactCommandSchema),
integrity_contradictions: z.array(integrityContradictionSchema).default([]),
limitations: z.array(z.string()),
/** Optional only for persisted inventory Evidence written before this field. */
coverage_status: artifactInventoryCoverageSchema.optional(),
});
/** Current inventory producers always classify the coverage they examined. */
export const artifactInventoryOutputSchema =
artifactInventoryResultSchema.extend({
coverage_status: artifactInventoryCoverageSchema,
});
export type ArtifactInventoryOutput = z.infer<
typeof artifactInventoryOutputSchema
>;
/** One safely materialized artifact beneath caller-approved output root. */
const extractedArtifactSchema = z.object({
artifact_id: artifactIdSchema,

3
src/domain/artifactInspection.ts
View File

@ -1,6 +1,7 @@
import { z } from "zod";
import {
artifactInventoryOutputSchema,
artifactInventoryResultSchema,
type ArtifactInventoryResult,
} from "./artifactGraph.js";
@ -114,7 +115,7 @@ export const artifactInspectionOutputSchema =
.array(
artifactInspectionResultSchema.shape.substeps.element.safeExtend({
evidence: evidenceSchema.safeExtend({
normalized_result: artifactInventoryResultSchema,
normalized_result: artifactInventoryOutputSchema,
}),
}),
)

38
src/domain/artifactInventoryEvidence.ts
View File

@ -1,5 +1,6 @@
import {
artifactInventoryResultSchema,
type ArtifactInventoryCoverage,
type ArtifactInventoryResult,
type ArtifactNode,
type ArtifactOccurrence,
@ -17,6 +18,7 @@ import {
} from "./artifactIdentity.js";
import { parseEvidence, type Evidence } from "./evidence.js";
import { artifactInspectionResultSchema } from "./artifactInspection.js";
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "./zipPackageFormat.js";
/** Complete or partially assembled inventory used by comparison workflows. */
export interface InventorySet {
@ -27,6 +29,7 @@ export interface InventorySet {
readonly integrityContradictions: readonly IntegrityContradiction[];
readonly limitations: readonly string[];
readonly complete: boolean;
readonly coverage_status: ArtifactInventoryCoverage;
}
export interface ParsedInventorySet {
@ -123,6 +126,10 @@ const assembleInventoryResults = (
const first = pages[0];
if (first === undefined)
throw new TypeError("Artifact inventory requires Evidence pages");
const coverageStatuses = new Set(pages.map(resolveInventoryCoverageStatus));
if (coverageStatuses.size !== 1)
throw new TypeError("Artifact inventory pages disagree on coverage status");
const coverageStatus = [...coverageStatuses][0]!;
const manifestJson = canonicalJson(first.manifest, "Artifact inventory");
const nodes = new Map<string, ArtifactNode>();
const occurrences = new Map<string, ArtifactOccurrence>();
@ -152,11 +159,12 @@ const assembleInventoryResults = (
for (const edge of page.edges)
mergeExact(edges, edge.edge_id, edge, "edge");
}
const inventory = orderedInventory(first, pages, {
nodes,
occurrences,
edges,
});
const inventory = orderedInventory(
first,
pages,
{ nodes, occurrences, edges },
coverageStatus,
);
if (inventory.complete) validateCompleteInventory(inventory);
return inventory;
};
@ -172,6 +180,7 @@ const orderedInventory = (
ArtifactInventoryResult["edges"][number]
>;
},
coverageStatus: ArtifactInventoryCoverage,
): InventorySet => {
const nodes = [...values.nodes.values()].sort((left, right) =>
compareUnicodeCodePoints(left.artifact_id, right.artifact_id),
@ -195,9 +204,28 @@ const orderedInventory = (
nodes.length === first.manifest.node_count &&
occurrences.length === first.manifest.occurrence_count &&
edges.length === first.manifest.edge_count,
coverage_status: coverageStatus,
};
};
const resolveInventoryCoverageStatus = (
inventory: ArtifactInventoryResult,
): ArtifactInventoryCoverage => {
if (inventory.coverage_status !== undefined) return inventory.coverage_status;
// Older retained Evidence has no typed status. Keep only its known
// informational notices complete; unknown limitation text fails closed.
return inventory.limitations.every(isKnownNonCoverageNotice)
? "complete"
: "partial";
};
const isKnownNonCoverageNotice = (limitation: string): boolean =>
limitation === "Artifact has no child container entries." ||
limitation === ZIP_NON_ENTRY_TAIL_LIMITATION ||
/^\d+ integrity contradiction\(s\) were recorded; mismatched content is observed-untrusted\.$/u.test(
limitation,
);
const validateCompleteInventory = (inventory: InventorySet): void => {
const nodeIds = new Set(inventory.nodes.map(({ artifact_id: id }) => id));
const occurrenceIds = new Set(

2
src/domain/artifactInventorySnapshot.ts
View File

@ -1,4 +1,5 @@
import type {
ArtifactInventoryCoverage,
ArtifactInventoryResult,
ArtifactNode,
IntegrityContradiction,
@ -6,6 +7,7 @@ import type {
/** Immutable inventory produced by one complete artifact scan. */
export interface ArtifactInventorySnapshot {
readonly coverage_status: ArtifactInventoryCoverage;
readonly manifest: ArtifactInventoryResult["manifest"];
readonly nodes: readonly ArtifactNode[];
readonly occurrences: ArtifactInventoryResult["occurrences"];

84
tests/acceptance/artifactComparisonPlist.test.ts Normal file
View File

@ -0,0 +1,84 @@
import { execFile } from "node:child_process";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js";
import { parseEvidence } from "../../src/domain/evidence.js";
import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
const execute = promisify(execFile);
const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url));
const plist =
'<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>hello</string></dict></plist>';
it("compares identical complete plist inventories across CLI and MCP and creates no unknowns", async () => {
const root = await createTestTempDirectory("rea-plist-compare-acceptance-");
const leftPath = join(root, "left.plist");
const rightPath = join(root, "right.plist");
await Promise.all([writeFile(leftPath, plist), writeFile(rightPath, plist)]);
const cliResult = await execute(process.execPath, [
cli,
"inspect-artifact",
leftPath,
"--json",
]);
const cliInspection = artifactInspectionResultSchema.parse(
parseEvidence(JSON.parse(cliResult.stdout)).normalized_result,
);
const cliInventory = cliInspection.substeps[0]?.evidence;
if (cliInventory === undefined)
throw new Error("CLI inspection did not retain inventory Evidence");
expect(cliInventory.normalized_result).toMatchObject({
coverage_status: "complete",
limitations: ["Artifact has no child container entries."],
});
const { call } = await connectLocalToolsMcp();
const opened = await call("open_binary", { path: rightPath });
expect(opened.isError, JSON.stringify(opened)).not.toBe(true);
const inspected = await call("inspect_artifact", {});
expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true);
const mcpInspection = artifactInspectionResultSchema.parse(
parseEvidence(inspected.structuredContent).normalized_result,
);
const mcpInventory = mcpInspection.substeps[0]?.evidence;
if (mcpInventory === undefined)
throw new Error("MCP inspection did not retain inventory Evidence");
expect(mcpInventory.normalized_result).toMatchObject({
coverage_status: "complete",
limitations: ["Artifact has no child container entries."],
});
const compared = await call("compare_artifacts", {
left: cliInventory,
right: mcpInventory,
});
expect(compared.isError, JSON.stringify(compared)).not.toBe(true);
const comparison = parseEvidence(compared.structuredContent);
expect(comparison.normalized_result).toMatchObject({
status: "unchanged",
summary: { unchanged: 1, unknown: 0 },
changes: [],
});
const behavior = await call("find_changed_behavior", {
comparisons: [comparison],
});
expect(behavior.isError, JSON.stringify(behavior)).not.toBe(true);
expect(behavior.structuredContent).toMatchObject({
normalized_result: {
behavior_status: "unknown",
findings: { items: [] },
},
});
const unknowns = await call("list_unknowns", {
domain: "artifact-comparison",
});
expect(unknowns.isError, JSON.stringify(unknowns)).not.toBe(true);
expect(unknowns.structuredContent).toMatchObject({ result: { items: [] } });
});

181
tests/boundary/filesystem/artifactComparison.test.ts
View File

@ -1,4 +1,4 @@
import { mkdir, writeFile } from "node:fs/promises";
import { chmod, mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
@ -12,6 +12,7 @@ import {
artifactComparisonResultSchema,
compareArtifacts,
} from "../../../src/domain/artifactComparison.js";
import { parseArtifactInventoryEvidence } from "../../../src/domain/artifactInventoryEvidence.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
@ -61,6 +62,184 @@ const changedArtifactComparison = async () => {
};
};
describe("leaf artifact comparison", () => {
it("accepts persisted complete leaf inventory Evidence without coverage_status", async () => {
const root = await createTestTempDirectory("rea-artifact-plist-compare-");
const leftPath = join(root, "left.plist");
const rightPath = join(root, "right.plist");
const xml =
'<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>hello</string></dict></plist>';
await Promise.all([writeFile(leftPath, xml), writeFile(rightPath, xml)]);
const [left, right] = await Promise.all([
observe(leftPath),
observe(rightPath),
]);
// Persisted inventory Evidence predates coverage_status. The known leaf
// notice remains complete when normalized at the inventory boundary.
const parsed = artifactInventoryResultSchema.parse(left.normalized_result);
const { coverage_status: _coverageStatus, ...legacyInventory } = parsed;
const legacy = createEvidence(
{
path: left.subject!.local_path,
sha256: left.subject!.digest.sha256,
format: left.subject!.format,
},
PROVIDER,
{
operation: "inventory_artifact",
parameters: {},
result: jsonValueSchema.parse(legacyInventory),
confidence: "observed",
authority: "shipped-artifact",
},
);
expect(compareArtifacts(legacy, right)).toMatchObject({
status: "unchanged",
summary: { unchanged: 1, unknown: 0 },
changes: [],
});
});
it("keeps content and executable changes visible for complete plist roots", async () => {
const root = await createTestTempDirectory("rea-artifact-plist-deltas-");
const leftPath = join(root, "left.plist");
const rightPath = join(root, "right.plist");
const plist = (label: string) =>
`<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>${label}</string></dict></plist>`;
await Promise.all([
writeFile(leftPath, plist("hello"), { mode: 0o644 }),
writeFile(rightPath, plist("world"), { mode: 0o644 }),
]);
const [left, changed] = await Promise.all([
observe(leftPath),
observe(rightPath),
]);
expect(compareArtifacts(left, changed)).toMatchObject({
status: "changed",
summary: { changed: 1, unknown: 0 },
changes: [
expect.objectContaining({
logical_path: ".",
classification: "changed",
dimensions: expect.arrayContaining(["content"]),
}),
],
});
if (process.platform !== "win32") {
await writeFile(rightPath, plist("hello"), { mode: 0o644 });
await chmod(rightPath, 0o755);
const executable = await observe(rightPath);
expect(compareArtifacts(left, executable)).toMatchObject({
status: "changed",
summary: { changed: 1, unknown: 0 },
changes: [
expect.objectContaining({
logical_path: ".",
classification: "changed",
dimensions: expect.arrayContaining(["executable"]),
}),
],
});
}
});
});
describe("artifact inventory coverage", () => {
it("keeps explicit and legacy unknown coverage fail-closed", async () => {
const root = await createTestTempDirectory("rea-artifact-coverage-");
const path = join(root, "complete.plist");
await writeFile(
path,
'<?xml version="1.0"?><plist version="1.0"><dict/></plist>',
);
const complete = await observe(path);
const graph = artifactInventoryResultSchema.parse(
complete.normalized_result,
);
const partial = createEvidence(
{
path: complete.subject!.local_path,
sha256: complete.subject!.digest.sha256,
format: complete.subject!.format,
},
PROVIDER,
{
operation: "inventory_artifact",
parameters: {},
result: jsonValueSchema.parse({ ...graph, coverage_status: "partial" }),
confidence: "observed",
authority: "shipped-artifact",
},
);
expect(compareArtifacts(partial, partial)).toMatchObject({
status: "unknown",
summary: { unchanged: 0, unknown: 1 },
});
const { coverage_status: _coverageStatus, ...legacyGraph } = graph;
const unknownLegacy = createEvidence(
{
path: complete.subject!.local_path,
sha256: complete.subject!.digest.sha256,
format: complete.subject!.format,
},
PROVIDER,
{
operation: "inventory_artifact",
parameters: {},
result: jsonValueSchema.parse({
...legacyGraph,
limitations: ["Producer omitted unexamined child entries."],
}),
confidence: "observed",
authority: "shipped-artifact",
},
);
expect(compareArtifacts(unknownLegacy, unknownLegacy)).toMatchObject({
status: "unknown",
summary: { unchanged: 0, unknown: 1 },
});
});
it("rejects inventory fragments that disagree on producer coverage", async () => {
const parent = await createTestTempDirectory("rea-artifact-page-coverage-");
const path = join(parent, "fixture.app");
await mkdir(path);
await writeFile(join(path, "main.js"), "main();");
const source = await observe(path);
const fragments = fragmentInventoryEvidence(source, 2);
const firstFragment = fragments[0];
const fragment = fragments[1];
if (firstFragment === undefined || fragment === undefined)
throw new Error("Missing test fragment");
const fragmentGraph = artifactInventoryResultSchema.parse(
fragment.normalized_result,
);
const mixed = createEvidence(
{
path: fragment.subject!.local_path,
sha256: fragment.subject!.digest.sha256,
format: fragment.subject!.format,
},
PROVIDER,
{
operation: "inventory_artifact",
parameters: fragment.parameters,
result: jsonValueSchema.parse({
...fragmentGraph,
coverage_status: "partial",
}),
confidence: "derived",
authority: "shipped-artifact",
},
);
expect(() =>
parseArtifactInventoryEvidence([firstFragment, mixed]),
).toThrow("Artifact inventory pages disagree on coverage status");
});
});
describe("artifact comparison", () => {
it("classifies deterministic path changes and cites both inventories", async () => {
const firstObservation = await changedArtifactComparison();

8
tests/boundary/mcp/artifactMcp.inlineInventory.test.ts
View File

@ -26,6 +26,7 @@ import {
artifactInventoryResultSchema,
type ArtifactInventoryResult,
} from "../../../src/domain/artifactGraph.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
import {
analysisViewResultSchema,
type AnalysisViewResult,
@ -266,7 +267,7 @@ const assertHistoricalInventory = async (
{
operation: "inventory_artifact",
parameters: {},
result: legacyInventory,
result: jsonValueSchema.parse(legacyInventory),
},
);
const legacyInspection = createArtifactInspection(legacySource);
@ -490,7 +491,10 @@ it("keeps complete ZIP output and reads retained inventory views after the sourc
{
operation: "inventory_artifact",
parameters: {},
result: { ...graph, occurrences: graph.occurrences.slice(0, 1) },
result: jsonValueSchema.parse({
...graph,
occurrences: graph.occurrences.slice(0, 1),
}),
},
);
const partialReply = await client.callTool({