diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 9a09e9d36..834d717cd 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -7,6 +7,10 @@ JSON measurements belong outside the checkout. This document keeps the measured findings, verification scope and revision references; the scripts and `corpus.json` keep the reproduction inputs. Generated `results/` is ignored. +Follow-up: [native PE/DOS entry views and real model verification](native-entry-views.md) +records the post-merge native continuation, including a small-function case where +four selected replies cost more bytes than the complete dossier. + The 12 detailed checkpoint receipts were archived byte-for-byte before removal from the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in the private experiment lab outside the checkout; its `archive-index.json` records diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjs index 8c386a4d9..b6ba1a552 100644 --- a/experiments/analysis-efficiency/compare.mjs +++ b/experiments/analysis-efficiency/compare.mjs @@ -30,6 +30,7 @@ const sameSettings = [ "provider_search_path_sha256", "provider_environment_policy_sha256", "measure_source_sha256", + "native_workflow_source_sha256", "package_lock_sha256", "node_version", "sdk_client_version", @@ -40,7 +41,9 @@ const sameSettings = [ const settingDifferences = sameSettings.filter( (key) => !isDeepStrictEqual(left[key], right[key]) || - (key === "provider_search_path_sha256" && + ((key === "provider_search_path_sha256" || + (key === "native_workflow_source_sha256" && + typeof left.native_question === "string")) && !/^[a-f0-9]{64}$/.test(left[key] ?? "")), ); // Producer/view IDs identify different delivery observations; a named recovery @@ -54,6 +57,17 @@ const facts = (answer) => const a = facts(leftAnswer); const b = facts(rightAnswer); const runSpecific = []; +for (const [side, answer] of [ + ["left", leftAnswer], + ["right", rightAnswer], +]) { + if (Object.hasOwn(answer, "evidence_ids")) + runSpecific.push({ + side, + path: ["evidence_ids"], + value: answer.evidence_ids, + }); +} // Binwalk assigns a new opaque region UUID in each invocation. Preserve both // original lookup values in this report; compare the observed region facts // separately. The product and original replies retain those IDs unchanged. diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json index 911fa88f5..5d2ad531d 100644 --- a/experiments/analysis-efficiency/corpus.json +++ b/experiments/analysis-efficiency/corpus.json @@ -116,6 +116,36 @@ "derivation": "selected regular ZIP member ShareX.HelpersLib.dll; no target execution" } }, + { + "id": "sharex.exe", + "file": "ShareX.exe", + "bytes": 198656, + "sha256": "46af1ccc514b395cb703633abb5a505a771942b7db9b1915f31aeea541b3685f", + "provenance": { + "project": "ShareX native executable component", + "version": "19.0.2", + "upstreamFile": "ShareX-19.0.2-portable.zip", + "url": "https://github.com/ShareX/ShareX/releases/download/v19.0.2/ShareX-19.0.2-portable.zip", + "transport_sha256": "967b5d838df5ce25fb5fd7cbd16f1bd797427e2df16907b022b74a4bc0fb9d54", + "transport_bytes": 168796487, + "derivation": "selected regular ZIP member ShareX.exe; PE32+ x86-64 with no CLR directory; no target execution" + } + }, + { + "id": "freedos.edlin.exe", + "file": "EDLIN.EXE", + "bytes": 31492, + "sha256": "96b2b6d543e2d8bdc83e313756444e103339eb5ee2c1229abe3607539b31ad6e", + "provenance": { + "project": "FreeDOS EDLIN", + "version": "2.24", + "upstreamFile": "edlin.zip", + "url": "https://www.ibiblio.org/pub/micro/pc-stuff/freedos/files/repositories/1.4/base/edlin.zip", + "transport_sha256": "9db8a0b50bc4be5c2358cd0fa1fb6fc77342c6d5175a6b1084547c370e428f8a", + "transport_bytes": 377009, + "derivation": "selected regular ZIP member BIN/EDLIN.EXE; original UPX-compressed DOS MZ; APPINFO/EDLIN.LSM reports version; no execution or unpacking" + } + }, { "id": "vscode.tree-sitter.wasm", "file": "tree-sitter.wasm", diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 348bf02c2..0bfd6da52 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -1,16 +1,14 @@ import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; -import { - mkdir, - open, - readFile, - readdir, - stat, - writeFile, -} from "node:fs/promises"; +import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; +import { + NATIVE_ARTIFACTS, + measureNativeEntry, + nativeEntrySeed, +} from "./native-entry.mjs"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; @@ -38,10 +36,15 @@ if ( !isAbsolute(values.path) || !isAbsolute(values.output) || !["complete", "summary"].includes(values.delivery) || - !["entry", "entry-and-main"].includes(values["native-question"]) || + !["entry", "entry-and-main", "entry-views"].includes( + values["native-question"], + ) || !["inventory", "selected"].includes(values["javascript-question"]) || !["default", "selected"].includes(values.schemas) || - (values.artifact !== "jq" && values["native-question"] !== "entry") || + (!NATIVE_ARTIFACTS.includes(values.artifact) && + values["native-question"] !== "entry") || + (values.artifact !== "jq" && + values["native-question"] === "entry-and-main") || (values.artifact !== "obsidian.asar" && values["javascript-question"] !== "inventory") ) @@ -118,6 +121,9 @@ await writeFile( ); const fingerprints = { measure_source_sha256: await fileSha(fileURLToPath(import.meta.url)), + native_workflow_source_sha256: await fileSha( + fileURLToPath(new URL("./native-entry.mjs", import.meta.url)), + ), runtime_files_sha256: sha(JSON.stringify(runtimeFiles)), runtime_dispatcher_sha256: await fileSha( resolve(values["runtime-root"], "scripts/rea.mjs"), @@ -545,111 +551,19 @@ const workflows = { evidence_ids: [info.evidence_id], }; }, - jq: async () => { - const file = await open(values.path, "r"); - let header; - try { - header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer; - } finally { - await file.close(); - } - if (header.subarray(0, 6).toString("hex") !== "7f454c460201") - throw new Error("Expected ELF64 little endian entry seed"); - const address = `0x${header.readBigUInt64LE(24).toString(16)}`; - await call("open_binary", { path: values.path, provider_id: "ghidra" }); - try { - const containing = await call("resolve_containing_procedure", { - address, - }); - const info = await call("analyze_function", { procedure: address }); - const n = normalized(info); - // This exact application entry is materialized in the pinned jq entry's - // observed MOV RDI, 0x401f00 instruction, not an original-source claim. - let main; - const mainEvidenceIds = []; - if (values["native-question"] === "entry-and-main") { - let sourceId; - try { - const completed = await call("analyze_function", { - procedure: "0x401f00", - }); - sourceId = completed.evidence_id; - } catch (error) { - const reference = - error instanceof ToolFailure && - error.value?.error?.code === "resource_constraint" - ? error.value.error.details?.reported_limits?.evidence_reference - : undefined; - if (reference?.kind !== "retained-evidence") throw error; - sourceId = reference.evidence_id; - } - mainEvidenceIds.push(sourceId); - main = {}; - for (const facet of [ - "procedure", - "pseudocode", - "referenced_strings", - "callees", - "callers", - "basic_blocks", - ]) { - let offset = 0; - const items = []; - const pages = []; - do { - const selected = await call("inspect_analysis_view", { - source: { kind: "retained-evidence", evidence_id: sourceId }, - view: { - kind: "native", - facet, - offset, - limit: - facet === "pseudocode" - ? 65536 - : facet === "basic_blocks" - ? 1 - : 64, - }, - }); - mainEvidenceIds.push(selected.evidence_id); - const v = normalized(selected); - items.push(v.item); - pages.push({ - coverage: v.coverage, - limitations: v.limitations, - unknowns: v.unknowns, - }); - if (v.coverage.exhausted || facet === "basic_blocks") break; - if ( - v.coverage.next_offset === null || - v.coverage.next_offset <= offset - ) - throw new Error("Native view did not advance"); - offset = v.coverage.next_offset; - } while (true); - main[facet] = { items, pages }; - } - } - return { - artifact_sha256: targetSha(info), - entry_address: address, - containing_procedure: normalized(containing), - procedure: n.procedure, - pseudocode_prefix: n.pseudocode.slice(0, 512), - caller_count: n.callers.length, - callee_count: n.callees.length, - ...(main === undefined ? {} : { main }), - analysis_profile: info.analysis_profile, - evidence_ids: [ - containing.evidence_id, - info.evidence_id, - ...mainEvidenceIds, - ], - }; - } finally { - await call("close_binary", {}); - } - }, + ...Object.fromEntries( + NATIVE_ARTIFACTS.map((artifact) => [ + artifact, + async () => + measureNativeEntry({ + seed: await nativeEntrySeed(artifact, values.path), + values, + call, + inputSha, + ToolFailure, + }), + ]), + ), }; try { @@ -740,7 +654,9 @@ const report = { artifact_sha256_after: finalInputSha, delivery: values.delivery, sdk_schema_mode: values.schemas, - native_question: values.artifact === "jq" ? values["native-question"] : null, + native_question: NATIVE_ARTIFACTS.includes(values.artifact) + ? values["native-question"] + : null, javascript_question: values.artifact === "obsidian.asar" ? values["javascript-question"] : null, provider_settings_sha256: sha(JSON.stringify(selectedProviders)), diff --git a/experiments/analysis-efficiency/native-entry-views.md b/experiments/analysis-efficiency/native-entry-views.md new file mode 100644 index 000000000..1c02fbfd1 --- /dev/null +++ b/experiments/analysis-efficiency/native-entry-views.md @@ -0,0 +1,178 @@ +# Native PE and DOS entry-view checkpoint + +Continuation of [#2016](https://github.com/morluto/rea/issues/2016) after +#2122 and #2193 merged. Real Ghidra observations and real Codex CLI answers +support the selected native facets on these shipped inputs. They do not justify +an automatic native summary policy: the four selected replies are larger than +the complete dossier for the small PE entry, and smaller for the DOS entry. +The initial complete analysis still runs and is delivered in both cases. + +This checkpoint changes experiment scripts, corpus inputs and grading only. +Existing production CLI/MCP contracts, defaults and session behavior are unchanged. + +## Inputs and runtime + +Production runtime: main `10400fdb57e4226c5ce15af17968a664f35498ce`, +including the merged WASM summary work and subsequent main changes. Node +24.18.0/npm 11.16.0, locked dependencies, actual Ghidra 12.1.4 on Linux. +Default auto-analysis remains enabled, with a 768 MiB JVM ceiling and one +active processor. The whole process family uses the existing one-CPU, +4 GiB hard/3 GiB early-stop, no-swap, bounded-task resource scope. + +| Shipped artifact | Selected bytes | SHA-256 | Static scope | +| ---------------------------------- | -------------: | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | +| ShareX 19.0.2 `ShareX.exe` | 198,656 | `46af1ccc514b395cb703633abb5a505a771942b7db9b1915f31aeea541b3685f` | Native PE32+ x86-64 component with no CLR directory; entry `0x1400125c0` derived from original image base/RVA | +| FreeDOS EDLIN 2.24 `BIN/EDLIN.EXE` | 31,492 | `96b2b6d543e2d8bdc83e313756444e103339eb5ee2c1229abe3607539b31ad6e` | Original UPX-compressed DOS MZ; admitted entry `0x10000`, independently verified static load image | + +Acquisition URLs, archive sizes/digests and exact regular ZIP members are in +[corpus.json](corpus.json). EDLIN's package `APPINFO/EDLIN.LSM` reports its +version and UPX compression. No binary was executed, unpacked or committed. +The independent PE header does not establish Windows runtime behavior. +DOS load-image verification and unpacking-stub pseudocode do not recover the +editor's application logic or establish DOS execution. DOS COM is still untested. + +The SDK and frozen model runtime share the exact `dist/` and `bridge/` file +table digest: +`e2b67eb0966072fd8b159edfab074cbaf1db9bfc0ac9c57d25ba1435cc0fdbcf`. +The model copy was independently checked against all 1,375 compiled files and +30 bridge files. Runtime copies and detailed receipts remain outside Git. + +## Public SDK observations + +The workflow opens the selected target with Ghidra, resolves the entry, +calls `analyze_function` once, reads retained procedure/pseudocode/callers/callees +facets, and closes the ephemeral session. DOS obtains its address from +`inspect_native_load_image` before resolving the procedure. Each facet is checked +against the complete producer facts, original parent ID, target digest and distinct +derived Evidence ID. The actual SDK validates named advertised output contracts. + +| Entry | Complete dossier JSON bytes | Four facet replies combined | Workflow seconds | First provider query seconds | Last-observed family CPU seconds | Sampled family peak MiB | +| ------------ | --------------------------: | --------------------------: | ---------------: | ---------------------------: | -------------------------------: | ----------------------: | +| ShareX PE | 50,936 | 68,257 | 63.123 | 57.490 | 63.660 | 1,197.85 | +| EDLIN DOS MZ | 224,670 | 70,539 | 26.436 | 21.366 | 26.899 | 939.25 | + +These are exploratory observations, not a paired implementation performance +cohort. JSON byte counts include the SDK tool-result representations, excluding +catalog discovery and wire framing; they are not model token counts. Four PE +facets cost about 34% more bytes than that dossier. The DOS facets cost about +69% fewer bytes **when reading already-retained Evidence**. Their first analysis +still returned the full dossier; summing both paths increases first-task traffic. +The first provider query includes the observable wait for the imported Program +and its analysis, not an independently measured engine-only stage. + +The observed PE entry contains 18 analyzed bytes, zero recorded callers and two +resolved callees. Ghidra's terminal-call caveat remains explicit. The DOS entry +contains 53 analyzed bytes and zero recorded callers/callees; overlapping +instruction and abnormal stack-frame warnings remain in the pseudocode. +Neither zero count establishes absence of unresolved runtime calls. + +An earlier PE probe completed production analysis and two facets, then failed +its harness assertion: a pseudocode page is `{text, unit}`, not a string. That +failed attempt remains archived and is excluded from successful workflow figures. +The corrected exploratory SDK driver fingerprints are +`4876c0ecaa84069ddc508e29649715c6c914bd3c4960a76fa9724b5992123699` +and native-helper `76023ad45cd1edab57dcd4a8ccf29a9b8a8378029ff2d9711ae35c73940e80d6`. +The published helper in `6d7f6885c` removes redundant scopes; replaying the actual +PE/DOS replies reproduces every answer field and the same 9/10 calls. Recorded +real jq entry and entry/main replies also preserve their selected facts, profile +and 4/12-call ordering. These replays are driver regression checks, not fresh +provider runs or timing measurements. + +## Real Codex CLI observations + +Three serial runs used Codex CLI 0.161.0, requested alias `gpt-5.6-terra`, high +reasoning, temporary isolated profiles and the frozen modified REA MCP runtime. +Resolved server-side model identity is not inferred. The same explicit seven-tool +subset was enabled: open, containing-procedure resolution, dossier, analysis view, +load image, retained Evidence discovery and close. This is not full-catalog tool +selection verification. Questions are [PE](questions/native-pe.txt) and +[DOS](questions/native-dos.txt). + +| Trial | Total input | Cached input | Uncached input | Output | Seconds | Calls / failed | +| ----- | ----------: | -----------: | -------------: | -----: | ------: | -------------: | +| PE 1 | 451,337 | 399,616 | 51,721 | 3,820 | 130.073 | 12 / 2 | +| DOS 1 | 730,578 | 658,944 | 71,634 | 3,310 | 134.435 | 9 / 0 | +| PE 2 | 498,322 | 429,056 | 69,266 | 3,038 | 192.373 | 9 / 1 | + +All three answers match the selected original digest, address, procedure +classification/signature/locals, inclusive body ranges, body byte count, +pseudocode prefix and relationship counts. Actual Evidence references were +observed in the responses; target bytes remained unchanged, close completed and +the copied profiles were removed. DOS additionally matches the verified load +status and every supplied load-image metadata field. Manual review preserves +static/packed-target and terminal-flow qualifications. + +The first grader expected only particular nested JSON shapes. It rejected PE's +equivalent text-page object/inclusive-range label and could not grade DOS's +explicit `entry @ 0x10000` display. The updated grader compares exact source +values for those representations without parsing displays into lookup identities. +It still rejects wrong text units, returned lengths, body endpoints and missing +facts. Original answers and first grading attempts remain archived; regrading +does not repeat model calls or change their tokens/times. + +The PE agent hit the advertised-but-unimplemented native summary recipe: both +trials attempted `view: {kind: "summary"}`; the first also tried the invalid +native facet `summary`. Those failed calls and all subsequent recovery/repeated +reads remain included. The public SDK independently reproduced `invalid_request` +for native summary on both inputs. This pre-existing contract-description mismatch +is recorded in `/tmp/rea-analysis-efficiency-bugs-20261012.md` for team repair, +as requested; this branch does not repair it. + +Model sampled peaks were 1,318.74 / 1,017.70 / 1,234.92 MiB, with last-observed +family CPU 72.956 / 71.534 / 74.089 seconds. SDK/model scopes did not abort and +their last-observed limit-event counters were zero. Final cgroup counters remained +unavailable after scope teardown: the strict resource-qualified overall grades +remain unverified. The selected fact/lifecycle checks pass; no universal latency, +OOM absence or CPU/memory saving is inferred from these samples. + +## Reproduction and next decision + +Use the pinned toolchain and build the selected production revision. Keep its +runtime copy, matching dependencies, provider settings, binaries and receipts +outside the checkout, as described in [README.md](README.md). + +```sh +python3 experiments/analysis-efficiency/resource-run.py -- \ + node experiments/analysis-efficiency/measure.mjs \ + --runtime-root /absolute/frozen-runtime --revision EXACT_PRODUCTION_SHA \ + --artifact sharex.exe --path /absolute/ShareX.exe \ + --providers /absolute/explicit-providers.json \ + --native-question entry-views --schemas selected \ + --output /absolute/new-native-measurement +``` + +For DOS select `--artifact freedos.edlin.exe` and the original extracted +`EDLIN.EXE`. `--native-question entry` retains the existing compact experimental +answer; `entry-views` performs additional verification reads and the explicit +summary refusal probe. It does not select summary delivery from `analyze_function`. +The helper fingerprint participates in comparison, and missing native helper +fingerprints are not treated as a comparable current driver. + +The reviewed `entry-views` answer separates selected facet facts from run-specific +references: `evidence_ids.retained_views` retains each facet's original view and +parent ID, while `retained_views` holds its item, coverage, limitations and unknowns. +DOS load-image facts retain their source ID and must match the selected carrier +digest before their entry address can be used; a stale result closes the session +without querying a procedure. +Derived view IDs are checked against every prior producer reference as well as +other derived views. DOS model grades require the actual completed load-image +response and its reported source ID; status, digest, entry and supplied metadata +are checked against that observation. Missing, failed or unreferenced responses +cannot establish load-image verification. +The comparator preserves both runs' reference maps and compares the facts. Existing +`entry`/`entry-and-main` answers keep their ID arrays. Historical SDK receipts above +retain their original pre-review representation; no replies or measurements were +rewritten, and no provider/model runs were repeated for this representation fix. + +Relevant validation includes the real workflows above, recorded-reply regression, +grader negative probes, JavaScript/Python syntax, formatting/diff checks and +`npm run check:fast`. The cold type/lint gate passed in 112.602 seconds with +sampled peak 3,220,914,176 bytes and five last-observed `memory.high` events; +this gate is not a performance-qualified experiment. Unrelated domain suites, +packaging and earlier model cohorts were not rerun for this experiment-only change. + +The next useful comparison is question-specific native primitives versus a +complete dossier, with identical requested facts and actual model trials. +Larger PE functions and another unpacked DOS/COM application can extend the +coverage progressively. Preserve existing complete defaults and solve the +summary-description bug separately before recommending a native summary route. diff --git a/experiments/analysis-efficiency/native-entry.mjs b/experiments/analysis-efficiency/native-entry.mjs new file mode 100644 index 000000000..84bc934ff --- /dev/null +++ b/experiments/analysis-efficiency/native-entry.mjs @@ -0,0 +1,272 @@ +import { open, readFile } from "node:fs/promises"; +import { isDeepStrictEqual } from "node:util"; + +export const NATIVE_ARTIFACTS = ["jq", "sharex.exe", "freedos.edlin.exe"]; +const normalized = (value) => value.normalized_result; + +const retainedEntryViews = async ( + call, + info, + inputSha, + ToolFailure, + sourceEvidenceIds, +) => { + const n = normalized(info); + const retainedViews = []; + const retainedViewEvidence = []; + const observedEvidenceIds = new Set([info.evidence_id, ...sourceEvidenceIds]); + let nativeSummaryOutcome, summaryEvidenceId; + for (const facet of ["procedure", "pseudocode", "callers", "callees"]) { + const selected = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: info.evidence_id }, + view: { + kind: "native", + facet, + offset: 0, + limit: facet === "pseudocode" ? 512 : 64, + }, + }); + const view = normalized(selected); + if (observedEvidenceIds.has(selected.evidence_id)) + throw new Error("Retained native view reused an Evidence identity"); + const expected = + facet === "procedure" + ? n.procedure + : facet === "pseudocode" + ? { text: n.pseudocode.slice(0, 512), unit: "utf16-code-units" } + : n[facet].slice(0, 64); + if ( + !isDeepStrictEqual(view.item, expected) || + view.parent_evidence_id !== info.evidence_id || + view.artifact.sha256 !== inputSha + ) + throw new Error( + "Retained native view changed the selected source facts or identity", + ); + observedEvidenceIds.add(selected.evidence_id); + retainedViews.push({ + facet, + item: view.item, + coverage: view.coverage, + limitations: view.limitations, + unknowns: view.unknowns, + }); + retainedViewEvidence.push({ + facet, + evidence_id: selected.evidence_id, + parent_evidence_id: view.parent_evidence_id, + }); + } + // Exercise the advertised summary recipe separately. A known refusal + // is recorded as such; it does not validate a nonexistent capability. + try { + const summary = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: info.evidence_id }, + view: { kind: "summary" }, + }); + if (observedEvidenceIds.has(summary.evidence_id)) + throw new Error("Native summary reused an Evidence identity"); + nativeSummaryOutcome = { status: "supported" }; + summaryEvidenceId = summary.evidence_id; + } catch (error) { + if ( + !(error instanceof ToolFailure) || + error.value?.error?.code !== "invalid_request" + ) + throw error; + nativeSummaryOutcome = { + status: "unsupported", + error: error.value.error, + }; + } + return { + retainedViews, + retainedViewEvidence, + nativeSummaryOutcome, + summaryEvidenceId, + }; +}; + +const retainedJqMain = async (call, ToolFailure) => { + // This exact application entry is materialized in the pinned jq entry's + // observed MOV RDI, 0x401f00 instruction, not an original-source claim. + let main; + const mainEvidenceIds = []; + let sourceId; + try { + const completed = await call("analyze_function", { + procedure: "0x401f00", + }); + sourceId = completed.evidence_id; + } catch (error) { + const reference = + error instanceof ToolFailure && + error.value?.error?.code === "resource_constraint" + ? error.value.error.details?.reported_limits?.evidence_reference + : undefined; + if (reference?.kind !== "retained-evidence") throw error; + sourceId = reference.evidence_id; + } + mainEvidenceIds.push(sourceId); + main = {}; + for (const facet of [ + "procedure", + "pseudocode", + "referenced_strings", + "callees", + "callers", + "basic_blocks", + ]) { + let offset = 0; + const items = []; + const pages = []; + do { + const selected = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: sourceId }, + view: { + kind: "native", + facet, + offset, + limit: + facet === "pseudocode" ? 65536 : facet === "basic_blocks" ? 1 : 64, + }, + }); + mainEvidenceIds.push(selected.evidence_id); + const v = normalized(selected); + items.push(v.item); + pages.push({ + coverage: v.coverage, + limitations: v.limitations, + unknowns: v.unknowns, + }); + if (v.coverage.exhausted || facet === "basic_blocks") break; + if (v.coverage.next_offset === null || v.coverage.next_offset <= offset) + throw new Error("Native view did not advance"); + offset = v.coverage.next_offset; + } while (true); + main[facet] = { items, pages }; + } + return { main, mainEvidenceIds }; +}; + +export const measureNativeEntry = async ({ + seed, + values, + call, + inputSha, + ToolFailure, +}) => { + await call("open_binary", { path: values.path, provider_id: "ghidra" }); + try { + // DOS addresses follow the actual admitted load image. A file header + // offset is not a provider address or proof of runtime execution. + let loadImage, loadImageEvidenceId; + if (seed === undefined) { + const imageEvidence = await call("inspect_native_load_image", {}); + loadImage = normalized(imageEvidence); + if (loadImage.status !== "verified") + throw new Error( + "The shipped DOS load image was not independently verified", + ); + if (loadImage.target_sha256 !== inputSha) + throw new Error("The DOS load image belongs to a different target"); + loadImageEvidenceId = imageEvidence.evidence_id; + seed = loadImage.entry.linear_address; + } + const address = seed; + const containing = await call("resolve_containing_procedure", { + address, + }); + const info = await call("analyze_function", { procedure: address }); + const n = normalized(info); + const selected = + values["native-question"] === "entry-views" + ? await retainedEntryViews(call, info, inputSha, ToolFailure, [ + containing.evidence_id, + ...(loadImageEvidenceId === undefined ? [] : [loadImageEvidenceId]), + ]) + : {}; + const { main, mainEvidenceIds = [] } = + values["native-question"] === "entry-and-main" + ? await retainedJqMain(call, ToolFailure) + : {}; + return { + artifact_sha256: info.subject?.digest.sha256, + entry_address: address, + containing_procedure: normalized(containing), + procedure: n.procedure, + pseudocode_prefix: n.pseudocode.slice(0, 512), + caller_count: n.callers.length, + callee_count: n.callees.length, + ...(main === undefined ? {} : { main }), + analysis_profile: info.analysis_profile, + ...(loadImage === undefined ? {} : { load_image: loadImage }), + ...(selected.retainedViews === undefined + ? {} + : { + retained_views: selected.retainedViews, + native_summary_outcome: selected.nativeSummaryOutcome, + }), + evidence_ids: + selected.retainedViews === undefined + ? [ + containing.evidence_id, + info.evidence_id, + ...mainEvidenceIds, + ...(loadImageEvidenceId === undefined + ? [] + : [loadImageEvidenceId]), + ] + : { + containing_procedure: containing.evidence_id, + function_dossier: info.evidence_id, + ...(loadImageEvidenceId === undefined + ? {} + : { native_load_image: loadImageEvidenceId }), + retained_views: selected.retainedViewEvidence, + ...(selected.summaryEvidenceId === undefined + ? {} + : { native_summary: selected.summaryEvidenceId }), + }, + }; + } finally { + await call("close_binary", {}); + } +}; + +export const nativeEntrySeed = async (artifact, path) => { + if (artifact === "jq") { + const file = await open(path, "r"); + let header; + try { + header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer; + } finally { + await file.close(); + } + if (header.subarray(0, 6).toString("hex") !== "7f454c460201") + throw new Error("Expected ELF64 little endian entry seed"); + return `0x${header.readBigUInt64LE(24).toString(16)}`; + } + if (artifact === "sharex.exe") { + const bytes = await readFile(path); + const pe = bytes.readUInt32LE(0x3c); + const optional = pe + 24; + if ( + bytes.subarray(pe, pe + 4).toString("hex") !== "50450000" || + bytes.readUInt16LE(pe + 4) !== 0x8664 || + bytes.readUInt16LE(optional) !== 0x20b || + bytes.readUInt32LE(optional + 108) <= 14 || + bytes.readUInt32LE(optional + 112 + 14 * 8) !== 0 || + bytes.readUInt32LE(optional + 116 + 14 * 8) !== 0 + ) + throw new Error( + "Expected the pinned native PE32+ entry seed without CLR metadata", + ); + const address = + bytes.readBigUInt64LE(optional + 24) + + BigInt(bytes.readUInt32LE(optional + 16)); + return `0x${address.toString(16)}`; + } + if (artifact === "freedos.edlin.exe") return undefined; + throw new Error("Select a pinned native artifact"); +}; diff --git a/experiments/analysis-efficiency/questions/native-dos.txt b/experiments/analysis-efficiency/questions/native-dos.txt new file mode 100644 index 000000000..48c52a9ec --- /dev/null +++ b/experiments/analysis-efficiency/questions/native-dos.txt @@ -0,0 +1 @@ +Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. This is the original UPX-compressed FreeDOS EDLIN 2.24 DOS MZ executable; inspect its admitted load image and entry stub, without running or unpacking it. Keep default auto-analysis enabled. Open the target with provider_id ghidra, call inspect_native_load_image, and use its verified entry.linear_address as the seed. Resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve missing/partial facts and terminal-flow caveats. Verified static import and unpacking-stub pseudocode do not prove DOS execution or recovery of the editor's application logic. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, load_image_status, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. Stop when these facts and cleanup are supported; report unsupported outcomes explicitly. diff --git a/experiments/analysis-efficiency/questions/native-pe.txt b/experiments/analysis-efficiency/questions/native-pe.txt new file mode 100644 index 000000000..749e96200 --- /dev/null +++ b/experiments/analysis-efficiency/questions/native-pe.txt @@ -0,0 +1 @@ +Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. The independently read, pinned PE32+ header reports image base 0x140000000 and entry RVA 0x125c0, so use address 0x1400125c0 as the analyst seed. This is a native x86-64 Windows PE component with no CLR directory, inspected on Linux; do not claim Windows runtime behavior. Keep default auto-analysis enabled. Open the target with provider_id ghidra, resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve terminal-flow caveats and unresolved static edges; recovered pseudocode is not original source. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. The seed applies only to the pinned ShareX.exe in corpus.json. Stop when these facts and cleanup are supported. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 22d6dd766..c31a01290 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -182,18 +182,61 @@ def main(): } for row in rows], key=lambda row: row['node_id']) check('all_selected_module_identities', select(answer.get('modules', [])), select(reference['modules'])) elif domain == 'native': - for key in ['entry_address', 'pseudocode_prefix', 'caller_count', 'callee_count']: + for key in ['entry_address', 'caller_count', 'callee_count']: check(key, answer.get(key), reference[key]) - containing = answer['containing_procedure'] + prefix = answer.get('pseudocode_prefix') + if isinstance(prefix, dict) and prefix.get('unit') == 'utf16-code-units': + text = prefix.get('text') + if 'requested' in prefix: + check('pseudocode_prefix.requested', prefix['requested'], 512) + if 'returned' in prefix: + check('pseudocode_prefix.returned', prefix['returned'], len(reference['pseudocode_prefix'].encode('utf-16-le')) // 2) + prefix = text + check('pseudocode_prefix', prefix, reference['pseudocode_prefix']) + containing = answer.get('containing_procedure') expected_containing = reference['containing_procedure']['procedure'] - for key in ['address', 'name']: - check('containing_procedure.' + key, containing.get(key, containing.get('procedure', {}).get(key)), expected_containing[key]) - procedure = answer['procedure'] + if isinstance(containing, str): + # Compare the explicitly reported display to the known source; + # never parse a display name into a canonical lookup identity. + check('containing_procedure.display', containing, + expected_containing['name'] + ' @ ' + expected_containing['address']) + else: + containing = containing if isinstance(containing, dict) else {} + nested = containing.get('procedure') + nested = nested if isinstance(nested, dict) else {} + for key in ['address', 'name']: + check('containing_procedure.' + key, containing.get(key, nested.get(key)), expected_containing[key]) + procedure = answer.get('procedure') + procedure = procedure if isinstance(procedure, dict) else {} expected = reference['procedure'] for key in ['address', 'name', 'classification', 'signature', 'locals']: check('procedure.' + key, procedure.get(key), expected[key]) - check('procedure.body_ranges', procedure.get('body_ranges', procedure.get('body', {}).get('ranges')), expected['body']['ranges']) - check('procedure.total_bytes', procedure.get('total_bytes', procedure.get('body', {}).get('total_bytes')), expected['body']['total_bytes']) + body = procedure.get('body') + body = body if isinstance(body, dict) else {} + check('procedure.body_ranges', procedure.get('body_ranges', body.get('ranges', body.get('ranges_inclusive'))), expected['body']['ranges']) + check('procedure.total_bytes', procedure.get('total_bytes', body.get('total_bytes')), expected['body']['total_bytes']) + if 'load_image' in reference: + sources = [source for source in producer_results.get('inspect_native_load_image', []) + if isinstance(source, dict) + and source.get('operation') == 'inspect_native_load_image' + and isinstance(source.get('evidence_id'), str) + and source.get('evidence_id') in reported_ids + and isinstance(source.get('normalized_result'), dict)] + checks['load_image_source_observed_and_referenced'] = bool(sources) + load = sources[-1]['normalized_result'] if sources else {} + entry = load.get('entry') + entry = entry if isinstance(entry, dict) else {} + check('load_image.target_sha256', load.get('target_sha256'), reference['artifact_sha256']) + check('load_image.source_status', load.get('status'), reference['load_image']['status']) + check('load_image.source_entry', entry.get('linear_address'), reference['entry_address']) + status = answer.get('load_image_status') + check('load_image_status', status.get('status') if isinstance(status, dict) else status, load.get('status')) + if isinstance(status, dict): + for key in ['format', 'load_segment', 'header_bytes', 'module_bytes', 'overlay_bytes']: + if key in status: + check('load_image.' + key, status[key], load.get(key)) + if 'entry_linear_address' in status: + check('load_image.entry_linear_address', status['entry_linear_address'], entry.get('linear_address')) calls = summary['completed_mcp_calls'] checks['default_analysis_not_disabled'] = all(not call['arguments'].get('skip_auto_analysis', False) for call in calls if call['tool'] == 'open_binary') checks['session_close_completed'] = any(call['tool'] == 'close_binary' and call['status'] == 'completed' for call in calls) @@ -248,6 +291,7 @@ def main(): and events['memory.events'].get('high') == 0 ) report = { + 'review_source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), 'domain': domain, 'checks': checks, 'all_selected_checks_pass': all(checks.values()), 'differences': differences, 'tool_errors': tool_errors, 'summary': summary, 'fingerprints': {key: value for key, value in settings.items() if 'sha256' in key or key in ['runtime_revision', 'enabled_tools', 'authority']}, diff --git a/tests/boundary/cli/nativeEfficiencyComparison.test.ts b/tests/boundary/cli/nativeEfficiencyComparison.test.ts new file mode 100644 index 000000000..65542f0ea --- /dev/null +++ b/tests/boundary/cli/nativeEfficiencyComparison.test.ts @@ -0,0 +1,336 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); + +// A recording port checks the experimental producer/consumer representation. +// Real provider verification remains in the separate native experiment receipts. +const generate = ` +import {measureNativeEntry} from './experiments/analysis-efficiency/native-entry.mjs'; +const sha='0'.repeat(64); +const duplicate=process.argv[1]==='duplicate'; +const stale=process.argv[1]==='stale-dos'; +const collision=process.argv[1]; +const dos=process.argv[1]==='dos'||stale||collision==='load-collision'; +const dossier={procedure:{address:'0x1000'},pseudocode:'return;',callers:[],callees:[]}; +class ToolFailure extends Error {} +const build=async digit=>{ + const parent='ev_'+digit.repeat(63)+'0'; + let ordinal=0; + const call=async(name,args)=>{ + if(stale)process.stdout.write(name+'\\n'); + if(name==='open_binary'||name==='close_binary')return {}; + if(name==='inspect_native_load_image')return {evidence_id:'ev_'+digit.repeat(63)+'6',normalized_result:{status:'verified',target_sha256:stale?'f'.repeat(64):sha,entry:{linear_address:'0x1000'}}}; + if(name==='resolve_containing_procedure')return {evidence_id:'ev_'+digit.repeat(63)+'7',normalized_result:{found:true}}; + if(name==='analyze_function')return {evidence_id:parent,subject:{digest:{sha256:sha}},normalized_result:dossier}; + const id='ev_'+digit.repeat(63)+(collision==='containing-collision'?7:collision==='load-collision'?6:duplicate?1:++ordinal); + if(args.view.kind==='summary')return {evidence_id:id}; + const facet=args.view.facet; + const item=facet==='pseudocode'?{text:dossier.pseudocode,unit:'utf16-code-units'}:dossier[facet]; + return {evidence_id:id,normalized_result:{parent_evidence_id:parent,artifact:{sha256:sha},item,coverage:{exhausted:true},limitations:[],unknowns:[]}}; + }; + return measureNativeEntry({seed:dos?undefined:'0x1000',values:{path:'/selected/input','native-question':'entry-views'},call,inputSha:sha,ToolFailure}); +}; +process.stdout.write(JSON.stringify([await build('1'),await build('2')])); +`; + +it.each([ + "observed", + "missing", + "errored", + "wrong-target", + "wrong-entry", + "unreferenced", +])( + "binds the DOS model grade to its observed load image: %s", + async (scenario) => { + // Synthetic transcripts exercise grading; they are not real-provider or + // resource measurements. Real DOS transcripts are replayed separately. + const root = await createTestTempDirectory("rea-dos-grader-"); + const trial = join(root, "trial"); + const referenceDirectory = join(root, "reference"); + await mkdir(trial); + await mkdir(referenceDirectory); + const sha = "0".repeat(64); + const imageId = "ev_" + "1".repeat(64); + const procedureId = "ev_" + "2".repeat(64); + const procedure = { + address: "0x1000", + name: "entry", + classification: "observed", + signature: null, + locals: [], + body: { ranges: [], total_bytes: 0 }, + }; + const loadImage = { + status: "verified", + target_sha256: sha, + entry: { linear_address: "0x1000" }, + }; + const reference = { + artifact_sha256: sha, + entry_address: "0x1000", + caller_count: 0, + callee_count: 0, + pseudocode_prefix: "return;", + containing_procedure: { procedure }, + procedure, + load_image: loadImage, + }; + const answer = { + ...reference, + load_image_status: "verified", + evidence_ids: [ + procedureId, + ...(["missing", "errored", "unreferenced"].includes(scenario) + ? [] + : [imageId]), + ], + }; + const event = (tool: string, status: string, value: unknown) => ({ + type: "item.completed", + item: { + type: "mcp_tool_call", + tool, + status, + arguments: {}, + result: { structured_content: value }, + }, + }); + const events = [ + event("analyze_function", "completed", { evidence_id: procedureId }), + ]; + if (scenario !== "missing") + events.push( + event( + "inspect_native_load_image", + scenario === "errored" ? "failed" : "completed", + scenario === "errored" + ? { error: { code: "unsupported_target" } } + : { + operation: "inspect_native_load_image", + evidence_id: imageId, + normalized_result: { + ...loadImage, + target_sha256: + scenario === "wrong-target" ? "f".repeat(64) : sha, + entry: { + linear_address: + scenario === "wrong-entry" ? "0x2000" : "0x1000", + }, + }, + }, + ), + ); + const resourceLog = join(root, "resource.log"); + const output = join(root, "review.json"); + for (const [path, value] of [ + [ + join(trial, "summary.json"), + { + completed_mcp_calls: [ + { tool: "close_binary", status: "completed", arguments: {} }, + ], + }, + ], + [join(trial, "trial-settings.json"), {}], + [join(trial, "cleanup.json"), { profile_removed: true }], + [join(trial, "answer.txt"), answer], + [join(referenceDirectory, "answer.json"), reference], + [ + resourceLog, + { + resource_scope: "synthetic-grader-fixture", + abort: null, + resource_counters_verified: true, + resource_events_final: { + "memory.events": { high: 0, max: 0, oom: 0, oom_kill: 0 }, + "pids.events": { max: 0 }, + }, + }, + ], + ] as const) + await writeFile(path, JSON.stringify(value)); + await writeFile( + join(trial, "events.jsonl"), + events.map((value) => JSON.stringify(value)).join("\n"), + ); + const command = execute("python3", [ + resolve("experiments/analysis-efficiency/review-model.py"), + "--domain", + "native", + "--trial", + trial, + "--reference", + referenceDirectory, + "--resource-log", + resourceLog, + "--output", + output, + ]); + if (scenario === "observed") await command; + else await expect(command).rejects.toMatchObject({ code: 1 }); + const report = JSON.parse(await readFile(output, "utf8")); + expect(report.all_selected_checks_pass).toBe(scenario === "observed"); + if (["missing", "errored", "unreferenced"].includes(scenario)) + expect(report.checks.load_image_source_observed_and_referenced).toBe( + false, + ); + if (scenario === "wrong-target") + expect(report.checks["load_image.target_sha256"]).toBe(false); + if (scenario === "wrong-entry") + expect(report.checks["load_image.source_entry"]).toBe(false); + }, +); + +it("retains the DOS load-image source for its facts and derived entry seed", async () => { + const { stdout } = await execute(process.execPath, [ + "--input-type=module", + "-e", + generate, + "dos", + ]); + const [answer] = JSON.parse(stdout); + expect(answer).toMatchObject({ + entry_address: "0x1000", + load_image: { status: "verified", entry: { linear_address: "0x1000" } }, + evidence_ids: { native_load_image: "ev_" + "1".repeat(63) + "6" }, + }); +}); + +it("rejects a stale DOS load image before using its entry and closes the session", async () => { + await expect( + execute(process.execPath, [ + "--input-type=module", + "-e", + generate, + "stale-dos", + ]), + ).rejects.toMatchObject({ + stdout: "open_binary\ninspect_native_load_image\nclose_binary\n", + stderr: expect.stringContaining( + "The DOS load image belongs to a different target", + ), + }); +}); + +it.each(["containing-collision", "load-collision"])( + "rejects a retained view that reuses an earlier producer identity: %s", + async (collision) => { + await expect( + execute(process.execPath, [ + "--input-type=module", + "-e", + generate, + collision, + ]), + ).rejects.toMatchObject({ + stderr: expect.stringContaining( + "Retained native view reused an Evidence identity", + ), + }); + }, +); + +it("rejects a producer that reuses a view identity across facets", async () => { + await expect( + execute(process.execPath, [ + "--input-type=module", + "-e", + generate, + "duplicate", + ]), + ).rejects.toMatchObject({ + stderr: expect.stringContaining( + "Retained native view reused an Evidence identity", + ), + }); +}); + +it.each(["references", "item", "coverage", "limitations", "unknowns"])( + "separates native references while comparing %s exactly", + async (change) => { + const { stdout: generated } = await execute(process.execPath, [ + "--input-type=module", + "-e", + generate, + ]); + const [leftAnswer, rightAnswer] = JSON.parse(generated); + if (change === "item") + rightAnswer.retained_views[0].item.address = "0x1001"; + if (change === "coverage") + rightAnswer.retained_views[0].coverage.exhausted = false; + if (change === "limitations") + rightAnswer.retained_views[0].limitations.push("partial"); + if (change === "unknowns") + rightAnswer.retained_views[0].unknowns.push("unresolved"); + + const root = await createTestTempDirectory("rea-native-comparison-"); + const left = join(root, "left"); + const right = join(root, "right"); + const summary = { + artifact: "sharex.exe", + native_question: "entry-views", + provider_search_path_sha256: "0".repeat(64), + native_workflow_source_sha256: "1".repeat(64), + calls: [], + failure: null, + }; + for (const [directory, answer] of [ + [left, leftAnswer], + [right, rightAnswer], + ]) { + await mkdir(directory); + await writeFile(join(directory, "summary.json"), JSON.stringify(summary)); + await writeFile(join(directory, "answer.json"), JSON.stringify(answer)); + } + const output = join(root, "comparison.json"); + const command = execute(process.execPath, [ + resolve("experiments/analysis-efficiency/compare.mjs"), + "--left", + left, + "--right", + right, + "--output", + output, + ]); + if (change !== "references") { + await expect(command).rejects.toMatchObject({ + code: 1, + stdout: expect.stringContaining('"retained_views"'), + }); + return; + } + const { stdout } = await command; + expect(JSON.parse(stdout)).toMatchObject({ + settings_match: true, + facts_match: true, + preserved_run_specific_values: [ + { + side: "left", + path: ["evidence_ids"], + value: leftAnswer.evidence_ids, + }, + { + side: "right", + path: ["evidence_ids"], + value: rightAnswer.evidence_ids, + }, + ], + }); + expect(leftAnswer.evidence_ids).not.toEqual(rightAnswer.evidence_ids); + expect(leftAnswer.evidence_ids.retained_views[0]).toMatchObject({ + evidence_id: "ev_" + "1".repeat(64), + parent_evidence_id: "ev_" + "1".repeat(63) + "0", + }); + expect(leftAnswer.evidence_ids.native_summary).toBe( + "ev_" + "1".repeat(63) + "5", + ); + }, +);