fix(verify): report final success only after resource cleanup

This commit is contained in:
morluto 2026-10-09 03:30:07 +08:00
parent 3eb7dbdc16
commit ee043e2cbf
17 changed files with 277 additions and 195 deletions

View File

@ -43,6 +43,9 @@ through the compiled public CLI or MCP runtime. Calling a compiled private
workflow or injecting a provider still establishes its narrower integration
claim, even if a directory name calls it acceptance.
Emit a verifier's final success report only after awaited cleanup. Observe final
process lineage after closing its owned resources.
Measure slow files before removing capacity regressions. Optimize repeated
process startup, fixture construction, and production algorithms when they
dominate runtime. Keep at least one input beyond each formerly failing size or

View File

@ -13,6 +13,7 @@ if (!executable)
const profile = await mkdtemp(join(tmpdir(), "rea-browser-dom-"));
const site = await startBrowserVerifierSite();
let context;
let report;
try {
context = await chromium.launchPersistentContext(profile, {
executablePath: executable,
@ -43,11 +44,15 @@ try {
process.argv[2],
page.url(),
);
process.stdout.write(
`${JSON.stringify({ ...result, browser: context.browser().version(), native_oracle: true, verified: true })}\n`,
);
report = {
...result,
browser: context.browser().version(),
native_oracle: true,
verified: true,
};
} finally {
await context?.close();
await site.close();
await rm(profile, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);

View File

@ -15,6 +15,7 @@ if (process.platform !== "darwin")
const root = await mkdtemp(join(tmpdir(), "rea-ui-fixture-"));
let child;
let fixturePid;
let report;
try {
const childRetrievalTest = join(root, "child-retrieval-test");
await promisify(execFile)("/usr/bin/xcrun", [
@ -187,9 +188,18 @@ try {
);
if (mismatch.ok || !mismatch.error.message.includes("target-mismatch"))
throw new Error("Real helper did not reject changed target bytes");
process.stdout.write(
`${JSON.stringify({ ok: true, verification_status: observation.ok ? "passed" : "permission-boundary-only", positive_e2e: observation.ok, observation: observation.ok ? "captured-selected-fixture-window" : "os-permission-denial-verified", scenario: scenarioStatus, permission_failure: observation.ok ? null : observation.error.message, mismatch_rejected: true, target_owned: true })}\n`,
);
report = {
ok: true,
verification_status: observation.ok ? "passed" : "permission-boundary-only",
positive_e2e: observation.ok,
observation: observation.ok
? "captured-selected-fixture-window"
: "os-permission-denial-verified",
scenario: scenarioStatus,
permission_failure: observation.ok ? null : observation.error.message,
mismatch_rejected: true,
target_owned: true,
};
} finally {
if (fixturePid !== undefined) {
try {
@ -205,3 +215,4 @@ try {
}
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);

View File

@ -34,6 +34,7 @@ let tarball;
let mcpStartup;
let mcpModuleLoading;
let update;
let report;
try {
({ tarball } = await verifyPackagePack({ root, workspace }));
@ -132,9 +133,44 @@ try {
evidenceRoot,
artifactArchive,
});
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), cli: true, analysisCli: true, artifactCli: true, managedCli: true, managedReconstructionCli: true, managedNativeVerificationCli: true, managedApplicationGraphCli: true, evidenceCli: true, incurMcpCommand: PRODUCT_IDENTITY.mcpCommand, lifecycleScriptsRequired: false, doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 4 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true, boundedRegexBridge: true, update, mcpStartup, mcpModuleLoading })}\n`,
);
report = {
cli: true,
analysisCli: true,
artifactCli: true,
managedCli: true,
managedReconstructionCli: true,
managedNativeVerificationCli: true,
managedApplicationGraphCli: true,
evidenceCli: true,
incurMcpCommand: PRODUCT_IDENTITY.mcpCommand,
lifecycleScriptsRequired: false,
doctor: "platform-appropriate",
setup: supportedSetupHost
? "planned-then-idempotent"
: "unsupported-host-rejected",
setupPlanReadOnly: supportedSetupHost,
existingHopperPreserved: supportedSetupHost,
clients: supportedSetupHost ? 4 : 0,
backupReadback: supportedSetupHost,
failureRecovery: supportedSetupHost,
configSymlinkLifecycle: supportedSetupHost,
skill: supportedSetupHost,
skillReferences: supportedSetupHost,
mcpTools: TOOL_CONTRACTS.length,
mcpPrompts: prompts.names.length,
promptCompletion: true,
promptCompletionLifecycle: true,
evidenceMcp: true,
targetFree: true,
targetLifecycle: true,
boundedRegexBridge: true,
update,
mcpStartup,
mcpModuleLoading,
};
} finally {
await rm(workspace, { recursive: true, force: true });
}
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
);

View File

@ -52,6 +52,7 @@ const profile = await mkdtemp(join(tmpdir(), "rea-real-browser-"));
const site = await startBrowserVerifierSite();
let browser;
let pageProxy;
let report;
try {
browser = spawn(
executable,
@ -338,35 +339,32 @@ try {
target_id: target,
});
const moduleTrace = await verifyBrowserModules(executable);
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
browser: observed.value.browser.product,
endpoint,
target,
domNodes: observed.value.dom.nodes.length,
accessibilityNodes: observed.value.accessibility.nodes.length,
scripts: observed.value.scripts.items.length,
networkRequests: observed.value.network.requests.length,
consoleEvents: observed.value.console.events.length,
websocketEvents: observed.value.network.websocket_events.length,
bundleScripts: bundle.value.capture.scripts_analyzed,
sourceMaps: bundle.value.observations.source_maps.processed,
sessionEvents: session.value.timeline.length,
pageScopedTransport: true,
screenshotBytes: screenshot.value.artifact.bytes,
largeScreenshot,
popupEvents,
networkEvidence,
scriptExport,
moduleTrace,
domDestinations,
browserScenarioCli: true,
browserScenarioAttachCleanup: "disconnected-external",
browserScenarioLaunchCleanup: "terminated-owned-process",
verified: true,
})}\n`,
);
report = {
browser: observed.value.browser.product,
endpoint,
target,
domNodes: observed.value.dom.nodes.length,
accessibilityNodes: observed.value.accessibility.nodes.length,
scripts: observed.value.scripts.items.length,
networkRequests: observed.value.network.requests.length,
consoleEvents: observed.value.console.events.length,
websocketEvents: observed.value.network.websocket_events.length,
bundleScripts: bundle.value.capture.scripts_analyzed,
sourceMaps: bundle.value.observations.source_maps.processed,
sessionEvents: session.value.timeline.length,
pageScopedTransport: true,
screenshotBytes: screenshot.value.artifact.bytes,
largeScreenshot,
popupEvents,
networkEvidence,
scriptExport,
moduleTrace,
domDestinations,
browserScenarioCli: true,
browserScenarioAttachCleanup: "disconnected-external",
browserScenarioLaunchCleanup: "terminated-owned-process",
verified: true,
};
} finally {
if (pageProxy !== undefined) await pageProxy.close();
if (browser !== undefined) await stopProcess(browser);
@ -378,6 +376,12 @@ try {
retryDelay: 100,
});
}
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
...report,
})}\n`,
);
async function verifyPageScopedTransport(provider, proxy, origin) {
const target = await pageTarget(provider, proxy.endpoint);

View File

@ -43,6 +43,7 @@ import {
} from "./verify-real-ghidra-function.mjs";
const exec = promisify(execFile);
const verifierRun = createVerifierRun();
let report;
const installDir = process.env.GHIDRA_INSTALL_DIR;
if (installDir === undefined || !isAbsolute(installDir))
throw new Error(
@ -265,33 +266,33 @@ try {
const customPath = process.env.GHIDRA_TARGET_PATH;
const custom =
customPath === undefined ? null : await verifyTarget(customPath, "custom");
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
ok: true,
provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION },
verification_lane: aarch64JumpTableOnly
? "aarch64-jump-table"
: crossFormat
? "cross-format"
: "host-native",
fixture_sources:
crossFormat || aarch64JumpTableOnly
? [sourcePath, crossFormatSourcePath]
: [sourcePath],
fixtures: [debug, stripped, layout, ...crossResults].map(summary),
malformed_target: "rejected-before-provider-start",
native_api_cli: aarch64JumpTableOnly
? (crossResults[0]?.native_api_cli ?? null)
: debug.native_api_cli,
native_value_e2e: debug.native_values,
custom_target: custom === null ? null : summary(custom),
cleanup: "complete",
})}\n`,
);
report = {
ok: true,
provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION },
verification_lane: aarch64JumpTableOnly
? "aarch64-jump-table"
: crossFormat
? "cross-format"
: "host-native",
fixture_sources:
crossFormat || aarch64JumpTableOnly
? [sourcePath, crossFormatSourcePath]
: [sourcePath],
fixtures: [debug, stripped, layout, ...crossResults].map(summary),
malformed_target: "rejected-before-provider-start",
native_api_cli: aarch64JumpTableOnly
? (crossResults[0]?.native_api_cli ?? null)
: debug.native_api_cli,
native_value_e2e: debug.native_values,
custom_target: custom === null ? null : summary(custom),
cleanup: "complete",
};
} finally {
await rm(fixtureRoot, { recursive: true, force: true });
}
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
);
function nativeFixtureTarget(platform, architecture) {
const supportedTargets = {

View File

@ -10,6 +10,7 @@ const execute = promisify(execFile);
const root = resolve(import.meta.dirname, "..");
const temporary = await mkdtemp(join(tmpdir(), "rea-test-discovery-"));
let discovered;
try {
const output = join(temporary, "discovery.json");
await execute(
@ -58,8 +59,9 @@ try {
) {
console.error(JSON.stringify(report, null, 2));
process.exitCode = 1;
} else
console.log(`Discovered all ${report.discovered} test files exactly once.`);
} else discovered = report.discovered;
} finally {
await rm(temporary, { recursive: true, force: true });
}
if (discovered !== undefined)
console.log(`Discovered all ${discovered} test files exactly once.`);

View File

@ -59,9 +59,6 @@ try {
}
if (failures.length > 0) throw new Error(failures.join("\n\n"));
process.stdout.write(
"Test temporary-directory hygiene verified: no owned residues remain.\n",
);
} finally {
await rm(isolatedTemporaryRoot, {
recursive: true,
@ -70,3 +67,6 @@ try {
retryDelay: 25,
});
}
process.stdout.write(
"Test temporary-directory hygiene verified: no owned residues remain.\n",
);

View File

@ -27,6 +27,7 @@ const verifierRun = createVerifierRun();
const root = process.cwd();
const workspace = await mkdtemp(join(tmpdir(), "rea-windows-package-"));
const prefix = join(workspace, "prefix");
let report;
try {
const packed = JSON.parse(
@ -212,23 +213,23 @@ try {
environment,
});
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
ok: true,
platform: process.platform,
package: packageResult.filename,
tools: toolCount,
ghidra_bridge: "present",
agent_setup: ["opencode", "vscode", "copilot_cli"],
setup_idempotent: true,
uninstall: true,
update,
})}\n`,
);
report = {
ok: true,
platform: process.platform,
package: packageResult.filename,
tools: toolCount,
ghidra_bridge: "present",
agent_setup: ["opencode", "vscode", "copilot_cli"],
setup_idempotent: true,
uninstall: true,
update,
};
} finally {
await rm(workspace, { recursive: true, force: true });
}
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
);
function npm(arguments_, cwd) {
const npmExecPath = process.env.npm_execpath;

View File

@ -21,6 +21,7 @@ await exec("/usr/bin/xcrun", ["--find", "actool"]);
await exec("/usr/bin/xcrun", ["--find", "clang"]);
await exec("/usr/bin/assetutil", ["--version"]);
const root = await mkdtemp(join(tmpdir(), "rea-asset-e2e-"));
let report;
try {
const app = join(root, "Fixture.app");
const resources = join(app, "Contents", "Resources");
@ -98,9 +99,16 @@ try {
});
assert.equal(invalid.isError, true);
});
process.stdout.write(
`${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, records: raw.length, pagination: true, malformed_input_rejected: true })}\n`,
);
report = {
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
records: raw.length,
pagination: true,
malformed_input_rejected: true,
};
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);

View File

@ -142,6 +142,7 @@ const checkSwift = (metadata, mode) => {
};
const root = await mkdtemp(join(tmpdir(), "rea-dispatch-"));
let report;
try {
const results = [];
for (const mode of LINK_MODES) {
@ -197,9 +198,8 @@ try {
});
}
}
process.stdout.write(
`${JSON.stringify({ ok: true, target_executed: false, fixtures: results })}\n`,
);
report = { ok: true, target_executed: false, fixtures: results };
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);

View File

@ -21,6 +21,7 @@ const sourceRoot = fileURLToPath(
new URL("../../../tests/conformance/interface-builder/", import.meta.url),
);
let report;
try {
await mkdir(join(appPath, "Contents", "MacOS"), { recursive: true });
await mkdir(resourcesPath, { recursive: true });
@ -83,21 +84,20 @@ try {
);
assert.equal(decoded.graph.truncated, false);
process.stdout.write(
`${JSON.stringify({
ok: true,
fixture: "compiled-appkit-nib",
provider: execution.value.provider,
target_sha256: decoded.target_sha256,
archive_sha256: decoded.documents[0].archive_sha256,
objects: decoded.documents[0].object_count,
connections: decoded.documents[0].connection_count,
recovered_control: button.name,
recovered_action: action.name,
hierarchy_complete: decoded.documents[0].hierarchy_complete,
truncated: decoded.graph.truncated,
})}\n`,
);
report = {
ok: true,
fixture: "compiled-appkit-nib",
provider: execution.value.provider,
target_sha256: decoded.target_sha256,
archive_sha256: decoded.documents[0].archive_sha256,
objects: decoded.documents[0].object_count,
connections: decoded.documents[0].connection_count,
recovered_control: button.name,
recovered_action: action.name,
hierarchy_complete: decoded.documents[0].hierarchy_complete,
truncated: decoded.graph.truncated,
};
} finally {
await rm(fixtureRoot, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);

View File

@ -17,6 +17,7 @@ if (process.platform !== "darwin")
"Real Foundation keyed archive verification requires macOS and Xcode Swift tools",
);
const root = await mkdtemp(join(tmpdir(), "rea-keyed-fixture-"));
let report;
try {
const archive = join(root, "model.plist");
await promisify(execFile)("/usr/bin/xcrun", [
@ -150,9 +151,21 @@ try {
[{ source: null, path: ["UID"], target: 1, status: "resolved" }],
);
}
process.stdout.write(
`${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, xml_golden: true, uid_named_root: true, format: graph.archive_format, objects: graph.total_objects, references: graph.total_references, shared_identity: true, cyclic_identity: true, target_classes_instantiated_by_reader: false })}\n`,
);
report = {
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
xml_golden: true,
uid_named_root: true,
format: graph.archive_format,
objects: graph.total_objects,
references: graph.total_references,
shared_identity: true,
cyclic_identity: true,
target_classes_instantiated_by_reader: false,
};
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);

View File

@ -65,6 +65,7 @@ const EXPECTED_BUNDLES = [
["Contents/XPCServices/Svc.xpc", "xpc-service", "macos-deep", "."],
];
let report;
try {
const { app } = await buildMacosBundleFixture(root);
const zip = join(root, "MacFixture.zip");
@ -155,19 +156,18 @@ try {
const dylibs = await verifyDylibResolution(app);
const dyldEnvironment = await verifyDyldEnvironment(root);
process.stdout.write(
`${JSON.stringify({
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
containers: containers.map(({ format }) => format),
bundles: EXPECTED_BUNDLES.length,
dmg_detached: true,
dylib_resolution: dylibs,
dyld_environment: dyldEnvironment,
})}\n`,
);
report = {
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
containers: containers.map(({ format }) => format),
bundles: EXPECTED_BUNDLES.length,
dmg_detached: true,
dylib_resolution: dylibs,
dyld_environment: dyldEnvironment,
};
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);

View File

@ -34,6 +34,7 @@ const sampleRss = () => {
};
const sampler = setInterval(sampleRss, 25);
sampler.unref();
let finalReport;
try {
const evidence = createEvidence(
@ -112,17 +113,15 @@ try {
assert.equal(digest, expectedHash.digest("hex"));
assert.equal((await stat(path)).size, expectedBytes);
sampleRss();
process.stdout.write(
`${JSON.stringify({
json_output_verified: true,
format,
output_bytes: expectedBytes,
engine_string_limit: constants.MAX_STRING_LENGTH,
output_sha256: digest,
evidence_id: evidence.evidence_id,
rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },
})}\n`,
);
finalReport = {
json_output_verified: true,
format,
output_bytes: expectedBytes,
engine_string_limit: constants.MAX_STRING_LENGTH,
output_sha256: digest,
evidence_id: evidence.evidence_id,
rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },
};
} finally {
clearInterval(sampler);
if (destination !== undefined && !destination.closed) {
@ -132,3 +131,4 @@ try {
}
await rm(workspace, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(finalReport)}\n`);

View File

@ -49,6 +49,7 @@ const call = async (name, args) => {
};
const fingerprint = (bytes) => createHash("sha256").update(bytes).digest("hex");
const counts = [];
let finalReport;
try {
const fixtures = await buildRecoveryFixtures(
root,
@ -128,19 +129,18 @@ try {
});
}
}
console.log(
JSON.stringify({
engine: "wakaru 1.13.0",
toolchains: fixtures.toolchains,
results: counts,
verified: true,
}),
);
finalReport = {
engine: "wakaru 1.13.0",
toolchains: fixtures.toolchains,
results: counts,
verified: true,
};
} finally {
await client.close();
await transport.close();
await rm(root, { recursive: true, force: true });
}
console.log(JSON.stringify(finalReport));
async function assertRecoveredArtifacts(evidence, result, fixture) {
assert.deepEqual(result.options, {

View File

@ -34,6 +34,8 @@ import {
const verifierRun = createVerifierRun();
const workspace = await mkdtemp(join(tmpdir(), "rea-managed-conformance-"));
let completionReportInput;
let finalReport;
const {
fixture,
fixtureBytes,
@ -378,65 +380,61 @@ try {
const manifestSelfTest = await runManagedAppManifestSelfTest();
const operatorManifest = await runOptionalManagedAppManifest();
const ilspyOracle = await runOptionalIlspyOracle();
const completionReport = createManagedCompletionReport(
{
modern,
framework,
nativeFunctionTarget,
readyToRun,
obfuscated,
left,
right,
nativeOnly,
malformed,
manifestSelfTest,
operatorManifest,
ilspyOracle,
completionReportInput = {
modern,
framework,
nativeFunctionTarget,
readyToRun,
obfuscated,
left,
right,
nativeOnly,
malformed,
manifestSelfTest,
operatorManifest,
ilspyOracle,
};
finalReport = {
verified:
11 + (operatorManifest === null ? 0 : 1) + (ilspyOracle === null ? 0 : 1),
managedSurfaces: [
"inspect_managed_artifact",
"inspect_managed_members",
"inspect_managed_native_boundaries",
"compare_managed_members",
"verify_managed_native_boundaries",
"import_managed_reconstruction",
"project_managed_application_graph",
],
coverage: [
"modern-dotnet-anycpu",
"dotnet-framework-x86-pinvoke",
"x64-ready-to-run-native-body",
"managed-native-verification",
"managed-application-graph-projection",
"unicode-obfuscated-identifiers",
"decompiler-reconstruction-import",
"mvid-and-token-drift",
"not-managed",
"malformed-metadata",
"operator-local-managed-manifest",
...(ilspyOracle === null ? [] : ["ilspy-reconstruction-oracle"]),
],
managedAppManifest: {
env: "REA_MANAGED_APP_MANIFEST_PATH",
selfTest: manifestSelfTest,
operator: operatorManifest ?? { configured: false },
},
await completeVerifierRun(verifierRun),
);
process.stdout.write(
`${JSON.stringify({
verified:
11 +
(operatorManifest === null ? 0 : 1) +
(ilspyOracle === null ? 0 : 1),
managedSurfaces: [
"inspect_managed_artifact",
"inspect_managed_members",
"inspect_managed_native_boundaries",
"compare_managed_members",
"verify_managed_native_boundaries",
"import_managed_reconstruction",
"project_managed_application_graph",
],
coverage: [
"modern-dotnet-anycpu",
"dotnet-framework-x86-pinvoke",
"x64-ready-to-run-native-body",
"managed-native-verification",
"managed-application-graph-projection",
"unicode-obfuscated-identifiers",
"decompiler-reconstruction-import",
"mvid-and-token-drift",
"not-managed",
"malformed-metadata",
"operator-local-managed-manifest",
...(ilspyOracle === null ? [] : ["ilspy-reconstruction-oracle"]),
],
managedAppManifest: {
env: "REA_MANAGED_APP_MANIFEST_PATH",
selfTest: manifestSelfTest,
operator: operatorManifest ?? { configured: false },
},
ilspyOracle: ilspyOracle ?? {
env: "REA_ILSPY_CMD_PATH",
configured: false,
},
completionReport,
})}\n`,
);
ilspyOracle: ilspyOracle ?? {
env: "REA_ILSPY_CMD_PATH",
configured: false,
},
};
} finally {
await rm(workspace, { recursive: true, force: true });
}
finalReport.completionReport = createManagedCompletionReport(
completionReportInput,
await completeVerifierRun(verifierRun),
);
process.stdout.write(`${JSON.stringify(finalReport)}\n`);