import { execFile } from "node:child_process"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { promisify } from "node:util"; import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const execute = promisify(execFile); // A recording port checks the experimental producer/consumer representation. // Real provider verification remains in the separate native experiment receipts. const generate = ` import {measureNativeEntry} from './experiments/analysis-efficiency/native-entry.mjs'; const sha='0'.repeat(64); const duplicate=process.argv[1]==='duplicate'; const stale=process.argv[1]==='stale-dos'; const collision=process.argv[1]; const dos=process.argv[1]==='dos'||stale||collision==='load-collision'; const dossier={procedure:{address:'0x1000'},pseudocode:'return;',callers:[],callees:[]}; class ToolFailure extends Error {} const build=async digit=>{ const parent='ev_'+digit.repeat(63)+'0'; let ordinal=0; const call=async(name,args)=>{ if(stale)process.stdout.write(name+'\\n'); if(name==='open_binary'||name==='close_binary')return {}; if(name==='inspect_native_load_image')return {evidence_id:'ev_'+digit.repeat(63)+'6',normalized_result:{status:'verified',target_sha256:stale?'f'.repeat(64):sha,entry:{linear_address:'0x1000'}}}; if(name==='resolve_containing_procedure')return {evidence_id:'ev_'+digit.repeat(63)+'7',normalized_result:{found:true}}; if(name==='analyze_function')return {evidence_id:parent,subject:{digest:{sha256:sha}},normalized_result:dossier}; const id='ev_'+digit.repeat(63)+(collision==='containing-collision'?7:collision==='load-collision'?6:duplicate?1:++ordinal); if(args.view.kind==='summary')return {evidence_id:id}; const facet=args.view.facet; const item=facet==='pseudocode'?{text:dossier.pseudocode,unit:'utf16-code-units'}:dossier[facet]; return {evidence_id:id,normalized_result:{parent_evidence_id:parent,artifact:{sha256:sha},item,coverage:{exhausted:true},limitations:[],unknowns:[]}}; }; return measureNativeEntry({seed:dos?undefined:'0x1000',values:{path:'/selected/input','native-question':'entry-views'},call,inputSha:sha,ToolFailure}); }; process.stdout.write(JSON.stringify([await build('1'),await build('2')])); `; it.each([ "observed", "missing", "errored", "wrong-target", "wrong-entry", "unreferenced", ])( "binds the DOS model grade to its observed load image: %s", async (scenario) => { // Synthetic transcripts exercise grading; they are not real-provider or // resource measurements. Real DOS transcripts are replayed separately. const root = await createTestTempDirectory("rea-dos-grader-"); const trial = join(root, "trial"); const referenceDirectory = join(root, "reference"); await mkdir(trial); await mkdir(referenceDirectory); const sha = "0".repeat(64); const imageId = "ev_" + "1".repeat(64); const procedureId = "ev_" + "2".repeat(64); const procedure = { address: "0x1000", name: "entry", classification: "observed", signature: null, locals: [], body: { ranges: [], total_bytes: 0 }, }; const loadImage = { status: "verified", target_sha256: sha, entry: { linear_address: "0x1000" }, }; const reference = { artifact_sha256: sha, entry_address: "0x1000", caller_count: 0, callee_count: 0, pseudocode_prefix: "return;", containing_procedure: { procedure }, procedure, load_image: loadImage, }; const answer = { ...reference, load_image_status: "verified", evidence_ids: [ procedureId, ...(["missing", "errored", "unreferenced"].includes(scenario) ? [] : [imageId]), ], }; const event = (tool: string, status: string, value: unknown) => ({ type: "item.completed", item: { type: "mcp_tool_call", tool, status, arguments: {}, result: { structured_content: value }, }, }); const events = [ event("analyze_function", "completed", { evidence_id: procedureId }), ]; if (scenario !== "missing") events.push( event( "inspect_native_load_image", scenario === "errored" ? "failed" : "completed", scenario === "errored" ? { error: { code: "unsupported_target" } } : { operation: "inspect_native_load_image", evidence_id: imageId, normalized_result: { ...loadImage, target_sha256: scenario === "wrong-target" ? "f".repeat(64) : sha, entry: { linear_address: scenario === "wrong-entry" ? "0x2000" : "0x1000", }, }, }, ), ); const resourceLog = join(root, "resource.log"); const output = join(root, "review.json"); for (const [path, value] of [ [ join(trial, "summary.json"), { completed_mcp_calls: [ { tool: "close_binary", status: "completed", arguments: {} }, ], }, ], [join(trial, "trial-settings.json"), {}], [join(trial, "cleanup.json"), { profile_removed: true }], [join(trial, "answer.txt"), answer], [join(referenceDirectory, "answer.json"), reference], [ resourceLog, { resource_scope: "synthetic-grader-fixture", abort: null, resource_counters_verified: true, resource_events_final: { "memory.events": { high: 0, max: 0, oom: 0, oom_kill: 0 }, "pids.events": { max: 0 }, }, }, ], ] as const) await writeFile(path, JSON.stringify(value)); await writeFile( join(trial, "events.jsonl"), events.map((value) => JSON.stringify(value)).join("\n"), ); const command = execute("python3", [ resolve("experiments/analysis-efficiency/review-model.py"), "--domain", "native", "--trial", trial, "--reference", referenceDirectory, "--resource-log", resourceLog, "--output", output, ]); if (scenario === "observed") await command; else await expect(command).rejects.toMatchObject({ code: 1 }); const report = JSON.parse(await readFile(output, "utf8")); expect(report.all_selected_checks_pass).toBe(scenario === "observed"); if (["missing", "errored", "unreferenced"].includes(scenario)) expect(report.checks.load_image_source_observed_and_referenced).toBe( false, ); if (scenario === "wrong-target") expect(report.checks["load_image.target_sha256"]).toBe(false); if (scenario === "wrong-entry") expect(report.checks["load_image.source_entry"]).toBe(false); }, ); it("retains the DOS load-image source for its facts and derived entry seed", async () => { const { stdout } = await execute(process.execPath, [ "--input-type=module", "-e", generate, "dos", ]); const [answer] = JSON.parse(stdout); expect(answer).toMatchObject({ entry_address: "0x1000", load_image: { status: "verified", entry: { linear_address: "0x1000" } }, evidence_ids: { native_load_image: "ev_" + "1".repeat(63) + "6" }, }); }); it("rejects a stale DOS load image before using its entry and closes the session", async () => { await expect( execute(process.execPath, [ "--input-type=module", "-e", generate, "stale-dos", ]), ).rejects.toMatchObject({ stdout: "open_binary\ninspect_native_load_image\nclose_binary\n", stderr: expect.stringContaining( "The DOS load image belongs to a different target", ), }); }); it.each(["containing-collision", "load-collision"])( "rejects a retained view that reuses an earlier producer identity: %s", async (collision) => { await expect( execute(process.execPath, [ "--input-type=module", "-e", generate, collision, ]), ).rejects.toMatchObject({ stderr: expect.stringContaining( "Retained native view reused an Evidence identity", ), }); }, ); it("rejects a producer that reuses a view identity across facets", async () => { await expect( execute(process.execPath, [ "--input-type=module", "-e", generate, "duplicate", ]), ).rejects.toMatchObject({ stderr: expect.stringContaining( "Retained native view reused an Evidence identity", ), }); }); it.each(["references", "item", "coverage", "limitations", "unknowns"])( "separates native references while comparing %s exactly", async (change) => { const { stdout: generated } = await execute(process.execPath, [ "--input-type=module", "-e", generate, ]); const [leftAnswer, rightAnswer] = JSON.parse(generated); if (change === "item") rightAnswer.retained_views[0].item.address = "0x1001"; if (change === "coverage") rightAnswer.retained_views[0].coverage.exhausted = false; if (change === "limitations") rightAnswer.retained_views[0].limitations.push("partial"); if (change === "unknowns") rightAnswer.retained_views[0].unknowns.push("unresolved"); const root = await createTestTempDirectory("rea-native-comparison-"); const left = join(root, "left"); const right = join(root, "right"); const summary = { artifact: "sharex.exe", native_question: "entry-views", provider_search_path_sha256: "0".repeat(64), native_workflow_source_sha256: "1".repeat(64), calls: [], failure: null, }; for (const [directory, answer] of [ [left, leftAnswer], [right, rightAnswer], ]) { await mkdir(directory); await writeFile(join(directory, "summary.json"), JSON.stringify(summary)); await writeFile(join(directory, "answer.json"), JSON.stringify(answer)); } const output = join(root, "comparison.json"); const command = execute(process.execPath, [ resolve("experiments/analysis-efficiency/compare.mjs"), "--left", left, "--right", right, "--output", output, ]); if (change !== "references") { await expect(command).rejects.toMatchObject({ code: 1, stdout: expect.stringContaining('"retained_views"'), }); return; } const { stdout } = await command; expect(JSON.parse(stdout)).toMatchObject({ settings_match: true, facts_match: true, preserved_run_specific_values: [ { side: "left", path: ["evidence_ids"], value: leftAnswer.evidence_ids, }, { side: "right", path: ["evidence_ids"], value: rightAnswer.evidence_ids, }, ], }); expect(leftAnswer.evidence_ids).not.toEqual(rightAnswer.evidence_ids); expect(leftAnswer.evidence_ids.retained_views[0]).toMatchObject({ evidence_id: "ev_" + "1".repeat(64), parent_evidence_id: "ev_" + "1".repeat(63) + "0", }); expect(leftAnswer.evidence_ids.native_summary).toBe( "ev_" + "1".repeat(63) + "5", ); }, );