test(native): verify real PE and DOS entry views
Record real Ghidra/MCP and Codex CLI verification of shipped ShareX native PE and FreeDOS DOS MZ entry views. Preserve original Evidence provenance, separate run-specific references from facts, and reject stale/colliding producer identities or unsupported model claims. Sixteen focused experiment regression cases, original public-reply/transcript replays, and CI run 38273269072 pass for head 0667f7cb7c85f134311c7470ff4ee99ef9cc45f4. No production API/default change or general native speedup claim; final resource counters remain unknown. Continues #2016. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
Ysopwnable
GitHub
parent
428eee00fd
commit
1770f38cfc
|
|
@ -7,6 +7,10 @@ JSON measurements belong outside the checkout. This document keeps the measured
|
|||
findings, verification scope and revision references; the scripts and `corpus.json`
|
||||
keep the reproduction inputs. Generated `results/` is ignored.
|
||||
|
||||
Follow-up: [native PE/DOS entry views and real model verification](native-entry-views.md)
|
||||
records the post-merge native continuation, including a small-function case where
|
||||
four selected replies cost more bytes than the complete dossier.
|
||||
|
||||
The 12 detailed checkpoint receipts were archived byte-for-byte before removal
|
||||
from the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in the
|
||||
private experiment lab outside the checkout; its `archive-index.json` records
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ const sameSettings = [
|
|||
"provider_search_path_sha256",
|
||||
"provider_environment_policy_sha256",
|
||||
"measure_source_sha256",
|
||||
"native_workflow_source_sha256",
|
||||
"package_lock_sha256",
|
||||
"node_version",
|
||||
"sdk_client_version",
|
||||
|
|
@ -40,7 +41,9 @@ const sameSettings = [
|
|||
const settingDifferences = sameSettings.filter(
|
||||
(key) =>
|
||||
!isDeepStrictEqual(left[key], right[key]) ||
|
||||
(key === "provider_search_path_sha256" &&
|
||||
((key === "provider_search_path_sha256" ||
|
||||
(key === "native_workflow_source_sha256" &&
|
||||
typeof left.native_question === "string")) &&
|
||||
!/^[a-f0-9]{64}$/.test(left[key] ?? "")),
|
||||
);
|
||||
// Producer/view IDs identify different delivery observations; a named recovery
|
||||
|
|
@ -54,6 +57,17 @@ const facts = (answer) =>
|
|||
const a = facts(leftAnswer);
|
||||
const b = facts(rightAnswer);
|
||||
const runSpecific = [];
|
||||
for (const [side, answer] of [
|
||||
["left", leftAnswer],
|
||||
["right", rightAnswer],
|
||||
]) {
|
||||
if (Object.hasOwn(answer, "evidence_ids"))
|
||||
runSpecific.push({
|
||||
side,
|
||||
path: ["evidence_ids"],
|
||||
value: answer.evidence_ids,
|
||||
});
|
||||
}
|
||||
// Binwalk assigns a new opaque region UUID in each invocation. Preserve both
|
||||
// original lookup values in this report; compare the observed region facts
|
||||
// separately. The product and original replies retain those IDs unchanged.
|
||||
|
|
|
|||
|
|
@ -116,6 +116,36 @@
|
|||
"derivation": "selected regular ZIP member ShareX.HelpersLib.dll; no target execution"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "sharex.exe",
|
||||
"file": "ShareX.exe",
|
||||
"bytes": 198656,
|
||||
"sha256": "46af1ccc514b395cb703633abb5a505a771942b7db9b1915f31aeea541b3685f",
|
||||
"provenance": {
|
||||
"project": "ShareX native executable component",
|
||||
"version": "19.0.2",
|
||||
"upstreamFile": "ShareX-19.0.2-portable.zip",
|
||||
"url": "https://github.com/ShareX/ShareX/releases/download/v19.0.2/ShareX-19.0.2-portable.zip",
|
||||
"transport_sha256": "967b5d838df5ce25fb5fd7cbd16f1bd797427e2df16907b022b74a4bc0fb9d54",
|
||||
"transport_bytes": 168796487,
|
||||
"derivation": "selected regular ZIP member ShareX.exe; PE32+ x86-64 with no CLR directory; no target execution"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "freedos.edlin.exe",
|
||||
"file": "EDLIN.EXE",
|
||||
"bytes": 31492,
|
||||
"sha256": "96b2b6d543e2d8bdc83e313756444e103339eb5ee2c1229abe3607539b31ad6e",
|
||||
"provenance": {
|
||||
"project": "FreeDOS EDLIN",
|
||||
"version": "2.24",
|
||||
"upstreamFile": "edlin.zip",
|
||||
"url": "https://www.ibiblio.org/pub/micro/pc-stuff/freedos/files/repositories/1.4/base/edlin.zip",
|
||||
"transport_sha256": "9db8a0b50bc4be5c2358cd0fa1fb6fc77342c6d5175a6b1084547c370e428f8a",
|
||||
"transport_bytes": 377009,
|
||||
"derivation": "selected regular ZIP member BIN/EDLIN.EXE; original UPX-compressed DOS MZ; APPINFO/EDLIN.LSM reports version; no execution or unpacking"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "vscode.tree-sitter.wasm",
|
||||
"file": "tree-sitter.wasm",
|
||||
|
|
|
|||
|
|
@ -1,16 +1,14 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import {
|
||||
mkdir,
|
||||
open,
|
||||
readFile,
|
||||
readdir,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
||||
import { dirname, isAbsolute, resolve } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import {
|
||||
NATIVE_ARTIFACTS,
|
||||
measureNativeEntry,
|
||||
nativeEntrySeed,
|
||||
} from "./native-entry.mjs";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
|
|
@ -38,10 +36,15 @@ if (
|
|||
!isAbsolute(values.path) ||
|
||||
!isAbsolute(values.output) ||
|
||||
!["complete", "summary"].includes(values.delivery) ||
|
||||
!["entry", "entry-and-main"].includes(values["native-question"]) ||
|
||||
!["entry", "entry-and-main", "entry-views"].includes(
|
||||
values["native-question"],
|
||||
) ||
|
||||
!["inventory", "selected"].includes(values["javascript-question"]) ||
|
||||
!["default", "selected"].includes(values.schemas) ||
|
||||
(values.artifact !== "jq" && values["native-question"] !== "entry") ||
|
||||
(!NATIVE_ARTIFACTS.includes(values.artifact) &&
|
||||
values["native-question"] !== "entry") ||
|
||||
(values.artifact !== "jq" &&
|
||||
values["native-question"] === "entry-and-main") ||
|
||||
(values.artifact !== "obsidian.asar" &&
|
||||
values["javascript-question"] !== "inventory")
|
||||
)
|
||||
|
|
@ -118,6 +121,9 @@ await writeFile(
|
|||
);
|
||||
const fingerprints = {
|
||||
measure_source_sha256: await fileSha(fileURLToPath(import.meta.url)),
|
||||
native_workflow_source_sha256: await fileSha(
|
||||
fileURLToPath(new URL("./native-entry.mjs", import.meta.url)),
|
||||
),
|
||||
runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),
|
||||
runtime_dispatcher_sha256: await fileSha(
|
||||
resolve(values["runtime-root"], "scripts/rea.mjs"),
|
||||
|
|
@ -545,111 +551,19 @@ const workflows = {
|
|||
evidence_ids: [info.evidence_id],
|
||||
};
|
||||
},
|
||||
jq: async () => {
|
||||
const file = await open(values.path, "r");
|
||||
let header;
|
||||
try {
|
||||
header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer;
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
if (header.subarray(0, 6).toString("hex") !== "7f454c460201")
|
||||
throw new Error("Expected ELF64 little endian entry seed");
|
||||
const address = `0x${header.readBigUInt64LE(24).toString(16)}`;
|
||||
await call("open_binary", { path: values.path, provider_id: "ghidra" });
|
||||
try {
|
||||
const containing = await call("resolve_containing_procedure", {
|
||||
address,
|
||||
});
|
||||
const info = await call("analyze_function", { procedure: address });
|
||||
const n = normalized(info);
|
||||
// This exact application entry is materialized in the pinned jq entry's
|
||||
// observed MOV RDI, 0x401f00 instruction, not an original-source claim.
|
||||
let main;
|
||||
const mainEvidenceIds = [];
|
||||
if (values["native-question"] === "entry-and-main") {
|
||||
let sourceId;
|
||||
try {
|
||||
const completed = await call("analyze_function", {
|
||||
procedure: "0x401f00",
|
||||
});
|
||||
sourceId = completed.evidence_id;
|
||||
} catch (error) {
|
||||
const reference =
|
||||
error instanceof ToolFailure &&
|
||||
error.value?.error?.code === "resource_constraint"
|
||||
? error.value.error.details?.reported_limits?.evidence_reference
|
||||
: undefined;
|
||||
if (reference?.kind !== "retained-evidence") throw error;
|
||||
sourceId = reference.evidence_id;
|
||||
}
|
||||
mainEvidenceIds.push(sourceId);
|
||||
main = {};
|
||||
for (const facet of [
|
||||
"procedure",
|
||||
"pseudocode",
|
||||
"referenced_strings",
|
||||
"callees",
|
||||
"callers",
|
||||
"basic_blocks",
|
||||
]) {
|
||||
let offset = 0;
|
||||
const items = [];
|
||||
const pages = [];
|
||||
do {
|
||||
const selected = await call("inspect_analysis_view", {
|
||||
source: { kind: "retained-evidence", evidence_id: sourceId },
|
||||
view: {
|
||||
kind: "native",
|
||||
facet,
|
||||
offset,
|
||||
limit:
|
||||
facet === "pseudocode"
|
||||
? 65536
|
||||
: facet === "basic_blocks"
|
||||
? 1
|
||||
: 64,
|
||||
},
|
||||
});
|
||||
mainEvidenceIds.push(selected.evidence_id);
|
||||
const v = normalized(selected);
|
||||
items.push(v.item);
|
||||
pages.push({
|
||||
coverage: v.coverage,
|
||||
limitations: v.limitations,
|
||||
unknowns: v.unknowns,
|
||||
});
|
||||
if (v.coverage.exhausted || facet === "basic_blocks") break;
|
||||
if (
|
||||
v.coverage.next_offset === null ||
|
||||
v.coverage.next_offset <= offset
|
||||
)
|
||||
throw new Error("Native view did not advance");
|
||||
offset = v.coverage.next_offset;
|
||||
} while (true);
|
||||
main[facet] = { items, pages };
|
||||
}
|
||||
}
|
||||
return {
|
||||
artifact_sha256: targetSha(info),
|
||||
entry_address: address,
|
||||
containing_procedure: normalized(containing),
|
||||
procedure: n.procedure,
|
||||
pseudocode_prefix: n.pseudocode.slice(0, 512),
|
||||
caller_count: n.callers.length,
|
||||
callee_count: n.callees.length,
|
||||
...(main === undefined ? {} : { main }),
|
||||
analysis_profile: info.analysis_profile,
|
||||
evidence_ids: [
|
||||
containing.evidence_id,
|
||||
info.evidence_id,
|
||||
...mainEvidenceIds,
|
||||
],
|
||||
};
|
||||
} finally {
|
||||
await call("close_binary", {});
|
||||
}
|
||||
},
|
||||
...Object.fromEntries(
|
||||
NATIVE_ARTIFACTS.map((artifact) => [
|
||||
artifact,
|
||||
async () =>
|
||||
measureNativeEntry({
|
||||
seed: await nativeEntrySeed(artifact, values.path),
|
||||
values,
|
||||
call,
|
||||
inputSha,
|
||||
ToolFailure,
|
||||
}),
|
||||
]),
|
||||
),
|
||||
};
|
||||
|
||||
try {
|
||||
|
|
@ -740,7 +654,9 @@ const report = {
|
|||
artifact_sha256_after: finalInputSha,
|
||||
delivery: values.delivery,
|
||||
sdk_schema_mode: values.schemas,
|
||||
native_question: values.artifact === "jq" ? values["native-question"] : null,
|
||||
native_question: NATIVE_ARTIFACTS.includes(values.artifact)
|
||||
? values["native-question"]
|
||||
: null,
|
||||
javascript_question:
|
||||
values.artifact === "obsidian.asar" ? values["javascript-question"] : null,
|
||||
provider_settings_sha256: sha(JSON.stringify(selectedProviders)),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
# Native PE and DOS entry-view checkpoint
|
||||
|
||||
Continuation of [#2016](https://github.com/morluto/rea/issues/2016) after
|
||||
#2122 and #2193 merged. Real Ghidra observations and real Codex CLI answers
|
||||
support the selected native facets on these shipped inputs. They do not justify
|
||||
an automatic native summary policy: the four selected replies are larger than
|
||||
the complete dossier for the small PE entry, and smaller for the DOS entry.
|
||||
The initial complete analysis still runs and is delivered in both cases.
|
||||
|
||||
This checkpoint changes experiment scripts, corpus inputs and grading only.
|
||||
Existing production CLI/MCP contracts, defaults and session behavior are unchanged.
|
||||
|
||||
## Inputs and runtime
|
||||
|
||||
Production runtime: main `10400fdb57e4226c5ce15af17968a664f35498ce`,
|
||||
including the merged WASM summary work and subsequent main changes. Node
|
||||
24.18.0/npm 11.16.0, locked dependencies, actual Ghidra 12.1.4 on Linux.
|
||||
Default auto-analysis remains enabled, with a 768 MiB JVM ceiling and one
|
||||
active processor. The whole process family uses the existing one-CPU,
|
||||
4 GiB hard/3 GiB early-stop, no-swap, bounded-task resource scope.
|
||||
|
||||
| Shipped artifact | Selected bytes | SHA-256 | Static scope |
|
||||
| ---------------------------------- | -------------: | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- |
|
||||
| ShareX 19.0.2 `ShareX.exe` | 198,656 | `46af1ccc514b395cb703633abb5a505a771942b7db9b1915f31aeea541b3685f` | Native PE32+ x86-64 component with no CLR directory; entry `0x1400125c0` derived from original image base/RVA |
|
||||
| FreeDOS EDLIN 2.24 `BIN/EDLIN.EXE` | 31,492 | `96b2b6d543e2d8bdc83e313756444e103339eb5ee2c1229abe3607539b31ad6e` | Original UPX-compressed DOS MZ; admitted entry `0x10000`, independently verified static load image |
|
||||
|
||||
Acquisition URLs, archive sizes/digests and exact regular ZIP members are in
|
||||
[corpus.json](corpus.json). EDLIN's package `APPINFO/EDLIN.LSM` reports its
|
||||
version and UPX compression. No binary was executed, unpacked or committed.
|
||||
The independent PE header does not establish Windows runtime behavior.
|
||||
DOS load-image verification and unpacking-stub pseudocode do not recover the
|
||||
editor's application logic or establish DOS execution. DOS COM is still untested.
|
||||
|
||||
The SDK and frozen model runtime share the exact `dist/` and `bridge/` file
|
||||
table digest:
|
||||
`e2b67eb0966072fd8b159edfab074cbaf1db9bfc0ac9c57d25ba1435cc0fdbcf`.
|
||||
The model copy was independently checked against all 1,375 compiled files and
|
||||
30 bridge files. Runtime copies and detailed receipts remain outside Git.
|
||||
|
||||
## Public SDK observations
|
||||
|
||||
The workflow opens the selected target with Ghidra, resolves the entry,
|
||||
calls `analyze_function` once, reads retained procedure/pseudocode/callers/callees
|
||||
facets, and closes the ephemeral session. DOS obtains its address from
|
||||
`inspect_native_load_image` before resolving the procedure. Each facet is checked
|
||||
against the complete producer facts, original parent ID, target digest and distinct
|
||||
derived Evidence ID. The actual SDK validates named advertised output contracts.
|
||||
|
||||
| Entry | Complete dossier JSON bytes | Four facet replies combined | Workflow seconds | First provider query seconds | Last-observed family CPU seconds | Sampled family peak MiB |
|
||||
| ------------ | --------------------------: | --------------------------: | ---------------: | ---------------------------: | -------------------------------: | ----------------------: |
|
||||
| ShareX PE | 50,936 | 68,257 | 63.123 | 57.490 | 63.660 | 1,197.85 |
|
||||
| EDLIN DOS MZ | 224,670 | 70,539 | 26.436 | 21.366 | 26.899 | 939.25 |
|
||||
|
||||
These are exploratory observations, not a paired implementation performance
|
||||
cohort. JSON byte counts include the SDK tool-result representations, excluding
|
||||
catalog discovery and wire framing; they are not model token counts. Four PE
|
||||
facets cost about 34% more bytes than that dossier. The DOS facets cost about
|
||||
69% fewer bytes **when reading already-retained Evidence**. Their first analysis
|
||||
still returned the full dossier; summing both paths increases first-task traffic.
|
||||
The first provider query includes the observable wait for the imported Program
|
||||
and its analysis, not an independently measured engine-only stage.
|
||||
|
||||
The observed PE entry contains 18 analyzed bytes, zero recorded callers and two
|
||||
resolved callees. Ghidra's terminal-call caveat remains explicit. The DOS entry
|
||||
contains 53 analyzed bytes and zero recorded callers/callees; overlapping
|
||||
instruction and abnormal stack-frame warnings remain in the pseudocode.
|
||||
Neither zero count establishes absence of unresolved runtime calls.
|
||||
|
||||
An earlier PE probe completed production analysis and two facets, then failed
|
||||
its harness assertion: a pseudocode page is `{text, unit}`, not a string. That
|
||||
failed attempt remains archived and is excluded from successful workflow figures.
|
||||
The corrected exploratory SDK driver fingerprints are
|
||||
`4876c0ecaa84069ddc508e29649715c6c914bd3c4960a76fa9724b5992123699`
|
||||
and native-helper `76023ad45cd1edab57dcd4a8ccf29a9b8a8378029ff2d9711ae35c73940e80d6`.
|
||||
The published helper in `6d7f6885c` removes redundant scopes; replaying the actual
|
||||
PE/DOS replies reproduces every answer field and the same 9/10 calls. Recorded
|
||||
real jq entry and entry/main replies also preserve their selected facts, profile
|
||||
and 4/12-call ordering. These replays are driver regression checks, not fresh
|
||||
provider runs or timing measurements.
|
||||
|
||||
## Real Codex CLI observations
|
||||
|
||||
Three serial runs used Codex CLI 0.161.0, requested alias `gpt-5.6-terra`, high
|
||||
reasoning, temporary isolated profiles and the frozen modified REA MCP runtime.
|
||||
Resolved server-side model identity is not inferred. The same explicit seven-tool
|
||||
subset was enabled: open, containing-procedure resolution, dossier, analysis view,
|
||||
load image, retained Evidence discovery and close. This is not full-catalog tool
|
||||
selection verification. Questions are [PE](questions/native-pe.txt) and
|
||||
[DOS](questions/native-dos.txt).
|
||||
|
||||
| Trial | Total input | Cached input | Uncached input | Output | Seconds | Calls / failed |
|
||||
| ----- | ----------: | -----------: | -------------: | -----: | ------: | -------------: |
|
||||
| PE 1 | 451,337 | 399,616 | 51,721 | 3,820 | 130.073 | 12 / 2 |
|
||||
| DOS 1 | 730,578 | 658,944 | 71,634 | 3,310 | 134.435 | 9 / 0 |
|
||||
| PE 2 | 498,322 | 429,056 | 69,266 | 3,038 | 192.373 | 9 / 1 |
|
||||
|
||||
All three answers match the selected original digest, address, procedure
|
||||
classification/signature/locals, inclusive body ranges, body byte count,
|
||||
pseudocode prefix and relationship counts. Actual Evidence references were
|
||||
observed in the responses; target bytes remained unchanged, close completed and
|
||||
the copied profiles were removed. DOS additionally matches the verified load
|
||||
status and every supplied load-image metadata field. Manual review preserves
|
||||
static/packed-target and terminal-flow qualifications.
|
||||
|
||||
The first grader expected only particular nested JSON shapes. It rejected PE's
|
||||
equivalent text-page object/inclusive-range label and could not grade DOS's
|
||||
explicit `entry @ 0x10000` display. The updated grader compares exact source
|
||||
values for those representations without parsing displays into lookup identities.
|
||||
It still rejects wrong text units, returned lengths, body endpoints and missing
|
||||
facts. Original answers and first grading attempts remain archived; regrading
|
||||
does not repeat model calls or change their tokens/times.
|
||||
|
||||
The PE agent hit the advertised-but-unimplemented native summary recipe: both
|
||||
trials attempted `view: {kind: "summary"}`; the first also tried the invalid
|
||||
native facet `summary`. Those failed calls and all subsequent recovery/repeated
|
||||
reads remain included. The public SDK independently reproduced `invalid_request`
|
||||
for native summary on both inputs. This pre-existing contract-description mismatch
|
||||
is recorded in `/tmp/rea-analysis-efficiency-bugs-20261012.md` for team repair,
|
||||
as requested; this branch does not repair it.
|
||||
|
||||
Model sampled peaks were 1,318.74 / 1,017.70 / 1,234.92 MiB, with last-observed
|
||||
family CPU 72.956 / 71.534 / 74.089 seconds. SDK/model scopes did not abort and
|
||||
their last-observed limit-event counters were zero. Final cgroup counters remained
|
||||
unavailable after scope teardown: the strict resource-qualified overall grades
|
||||
remain unverified. The selected fact/lifecycle checks pass; no universal latency,
|
||||
OOM absence or CPU/memory saving is inferred from these samples.
|
||||
|
||||
## Reproduction and next decision
|
||||
|
||||
Use the pinned toolchain and build the selected production revision. Keep its
|
||||
runtime copy, matching dependencies, provider settings, binaries and receipts
|
||||
outside the checkout, as described in [README.md](README.md).
|
||||
|
||||
```sh
|
||||
python3 experiments/analysis-efficiency/resource-run.py -- \
|
||||
node experiments/analysis-efficiency/measure.mjs \
|
||||
--runtime-root /absolute/frozen-runtime --revision EXACT_PRODUCTION_SHA \
|
||||
--artifact sharex.exe --path /absolute/ShareX.exe \
|
||||
--providers /absolute/explicit-providers.json \
|
||||
--native-question entry-views --schemas selected \
|
||||
--output /absolute/new-native-measurement
|
||||
```
|
||||
|
||||
For DOS select `--artifact freedos.edlin.exe` and the original extracted
|
||||
`EDLIN.EXE`. `--native-question entry` retains the existing compact experimental
|
||||
answer; `entry-views` performs additional verification reads and the explicit
|
||||
summary refusal probe. It does not select summary delivery from `analyze_function`.
|
||||
The helper fingerprint participates in comparison, and missing native helper
|
||||
fingerprints are not treated as a comparable current driver.
|
||||
|
||||
The reviewed `entry-views` answer separates selected facet facts from run-specific
|
||||
references: `evidence_ids.retained_views` retains each facet's original view and
|
||||
parent ID, while `retained_views` holds its item, coverage, limitations and unknowns.
|
||||
DOS load-image facts retain their source ID and must match the selected carrier
|
||||
digest before their entry address can be used; a stale result closes the session
|
||||
without querying a procedure.
|
||||
Derived view IDs are checked against every prior producer reference as well as
|
||||
other derived views. DOS model grades require the actual completed load-image
|
||||
response and its reported source ID; status, digest, entry and supplied metadata
|
||||
are checked against that observation. Missing, failed or unreferenced responses
|
||||
cannot establish load-image verification.
|
||||
The comparator preserves both runs' reference maps and compares the facts. Existing
|
||||
`entry`/`entry-and-main` answers keep their ID arrays. Historical SDK receipts above
|
||||
retain their original pre-review representation; no replies or measurements were
|
||||
rewritten, and no provider/model runs were repeated for this representation fix.
|
||||
|
||||
Relevant validation includes the real workflows above, recorded-reply regression,
|
||||
grader negative probes, JavaScript/Python syntax, formatting/diff checks and
|
||||
`npm run check:fast`. The cold type/lint gate passed in 112.602 seconds with
|
||||
sampled peak 3,220,914,176 bytes and five last-observed `memory.high` events;
|
||||
this gate is not a performance-qualified experiment. Unrelated domain suites,
|
||||
packaging and earlier model cohorts were not rerun for this experiment-only change.
|
||||
|
||||
The next useful comparison is question-specific native primitives versus a
|
||||
complete dossier, with identical requested facts and actual model trials.
|
||||
Larger PE functions and another unpacked DOS/COM application can extend the
|
||||
coverage progressively. Preserve existing complete defaults and solve the
|
||||
summary-description bug separately before recommending a native summary route.
|
||||
|
|
@ -0,0 +1,272 @@
|
|||
import { open, readFile } from "node:fs/promises";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
|
||||
export const NATIVE_ARTIFACTS = ["jq", "sharex.exe", "freedos.edlin.exe"];
|
||||
const normalized = (value) => value.normalized_result;
|
||||
|
||||
const retainedEntryViews = async (
|
||||
call,
|
||||
info,
|
||||
inputSha,
|
||||
ToolFailure,
|
||||
sourceEvidenceIds,
|
||||
) => {
|
||||
const n = normalized(info);
|
||||
const retainedViews = [];
|
||||
const retainedViewEvidence = [];
|
||||
const observedEvidenceIds = new Set([info.evidence_id, ...sourceEvidenceIds]);
|
||||
let nativeSummaryOutcome, summaryEvidenceId;
|
||||
for (const facet of ["procedure", "pseudocode", "callers", "callees"]) {
|
||||
const selected = await call("inspect_analysis_view", {
|
||||
source: { kind: "retained-evidence", evidence_id: info.evidence_id },
|
||||
view: {
|
||||
kind: "native",
|
||||
facet,
|
||||
offset: 0,
|
||||
limit: facet === "pseudocode" ? 512 : 64,
|
||||
},
|
||||
});
|
||||
const view = normalized(selected);
|
||||
if (observedEvidenceIds.has(selected.evidence_id))
|
||||
throw new Error("Retained native view reused an Evidence identity");
|
||||
const expected =
|
||||
facet === "procedure"
|
||||
? n.procedure
|
||||
: facet === "pseudocode"
|
||||
? { text: n.pseudocode.slice(0, 512), unit: "utf16-code-units" }
|
||||
: n[facet].slice(0, 64);
|
||||
if (
|
||||
!isDeepStrictEqual(view.item, expected) ||
|
||||
view.parent_evidence_id !== info.evidence_id ||
|
||||
view.artifact.sha256 !== inputSha
|
||||
)
|
||||
throw new Error(
|
||||
"Retained native view changed the selected source facts or identity",
|
||||
);
|
||||
observedEvidenceIds.add(selected.evidence_id);
|
||||
retainedViews.push({
|
||||
facet,
|
||||
item: view.item,
|
||||
coverage: view.coverage,
|
||||
limitations: view.limitations,
|
||||
unknowns: view.unknowns,
|
||||
});
|
||||
retainedViewEvidence.push({
|
||||
facet,
|
||||
evidence_id: selected.evidence_id,
|
||||
parent_evidence_id: view.parent_evidence_id,
|
||||
});
|
||||
}
|
||||
// Exercise the advertised summary recipe separately. A known refusal
|
||||
// is recorded as such; it does not validate a nonexistent capability.
|
||||
try {
|
||||
const summary = await call("inspect_analysis_view", {
|
||||
source: { kind: "retained-evidence", evidence_id: info.evidence_id },
|
||||
view: { kind: "summary" },
|
||||
});
|
||||
if (observedEvidenceIds.has(summary.evidence_id))
|
||||
throw new Error("Native summary reused an Evidence identity");
|
||||
nativeSummaryOutcome = { status: "supported" };
|
||||
summaryEvidenceId = summary.evidence_id;
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof ToolFailure) ||
|
||||
error.value?.error?.code !== "invalid_request"
|
||||
)
|
||||
throw error;
|
||||
nativeSummaryOutcome = {
|
||||
status: "unsupported",
|
||||
error: error.value.error,
|
||||
};
|
||||
}
|
||||
return {
|
||||
retainedViews,
|
||||
retainedViewEvidence,
|
||||
nativeSummaryOutcome,
|
||||
summaryEvidenceId,
|
||||
};
|
||||
};
|
||||
|
||||
const retainedJqMain = async (call, ToolFailure) => {
|
||||
// This exact application entry is materialized in the pinned jq entry's
|
||||
// observed MOV RDI, 0x401f00 instruction, not an original-source claim.
|
||||
let main;
|
||||
const mainEvidenceIds = [];
|
||||
let sourceId;
|
||||
try {
|
||||
const completed = await call("analyze_function", {
|
||||
procedure: "0x401f00",
|
||||
});
|
||||
sourceId = completed.evidence_id;
|
||||
} catch (error) {
|
||||
const reference =
|
||||
error instanceof ToolFailure &&
|
||||
error.value?.error?.code === "resource_constraint"
|
||||
? error.value.error.details?.reported_limits?.evidence_reference
|
||||
: undefined;
|
||||
if (reference?.kind !== "retained-evidence") throw error;
|
||||
sourceId = reference.evidence_id;
|
||||
}
|
||||
mainEvidenceIds.push(sourceId);
|
||||
main = {};
|
||||
for (const facet of [
|
||||
"procedure",
|
||||
"pseudocode",
|
||||
"referenced_strings",
|
||||
"callees",
|
||||
"callers",
|
||||
"basic_blocks",
|
||||
]) {
|
||||
let offset = 0;
|
||||
const items = [];
|
||||
const pages = [];
|
||||
do {
|
||||
const selected = await call("inspect_analysis_view", {
|
||||
source: { kind: "retained-evidence", evidence_id: sourceId },
|
||||
view: {
|
||||
kind: "native",
|
||||
facet,
|
||||
offset,
|
||||
limit:
|
||||
facet === "pseudocode" ? 65536 : facet === "basic_blocks" ? 1 : 64,
|
||||
},
|
||||
});
|
||||
mainEvidenceIds.push(selected.evidence_id);
|
||||
const v = normalized(selected);
|
||||
items.push(v.item);
|
||||
pages.push({
|
||||
coverage: v.coverage,
|
||||
limitations: v.limitations,
|
||||
unknowns: v.unknowns,
|
||||
});
|
||||
if (v.coverage.exhausted || facet === "basic_blocks") break;
|
||||
if (v.coverage.next_offset === null || v.coverage.next_offset <= offset)
|
||||
throw new Error("Native view did not advance");
|
||||
offset = v.coverage.next_offset;
|
||||
} while (true);
|
||||
main[facet] = { items, pages };
|
||||
}
|
||||
return { main, mainEvidenceIds };
|
||||
};
|
||||
|
||||
export const measureNativeEntry = async ({
|
||||
seed,
|
||||
values,
|
||||
call,
|
||||
inputSha,
|
||||
ToolFailure,
|
||||
}) => {
|
||||
await call("open_binary", { path: values.path, provider_id: "ghidra" });
|
||||
try {
|
||||
// DOS addresses follow the actual admitted load image. A file header
|
||||
// offset is not a provider address or proof of runtime execution.
|
||||
let loadImage, loadImageEvidenceId;
|
||||
if (seed === undefined) {
|
||||
const imageEvidence = await call("inspect_native_load_image", {});
|
||||
loadImage = normalized(imageEvidence);
|
||||
if (loadImage.status !== "verified")
|
||||
throw new Error(
|
||||
"The shipped DOS load image was not independently verified",
|
||||
);
|
||||
if (loadImage.target_sha256 !== inputSha)
|
||||
throw new Error("The DOS load image belongs to a different target");
|
||||
loadImageEvidenceId = imageEvidence.evidence_id;
|
||||
seed = loadImage.entry.linear_address;
|
||||
}
|
||||
const address = seed;
|
||||
const containing = await call("resolve_containing_procedure", {
|
||||
address,
|
||||
});
|
||||
const info = await call("analyze_function", { procedure: address });
|
||||
const n = normalized(info);
|
||||
const selected =
|
||||
values["native-question"] === "entry-views"
|
||||
? await retainedEntryViews(call, info, inputSha, ToolFailure, [
|
||||
containing.evidence_id,
|
||||
...(loadImageEvidenceId === undefined ? [] : [loadImageEvidenceId]),
|
||||
])
|
||||
: {};
|
||||
const { main, mainEvidenceIds = [] } =
|
||||
values["native-question"] === "entry-and-main"
|
||||
? await retainedJqMain(call, ToolFailure)
|
||||
: {};
|
||||
return {
|
||||
artifact_sha256: info.subject?.digest.sha256,
|
||||
entry_address: address,
|
||||
containing_procedure: normalized(containing),
|
||||
procedure: n.procedure,
|
||||
pseudocode_prefix: n.pseudocode.slice(0, 512),
|
||||
caller_count: n.callers.length,
|
||||
callee_count: n.callees.length,
|
||||
...(main === undefined ? {} : { main }),
|
||||
analysis_profile: info.analysis_profile,
|
||||
...(loadImage === undefined ? {} : { load_image: loadImage }),
|
||||
...(selected.retainedViews === undefined
|
||||
? {}
|
||||
: {
|
||||
retained_views: selected.retainedViews,
|
||||
native_summary_outcome: selected.nativeSummaryOutcome,
|
||||
}),
|
||||
evidence_ids:
|
||||
selected.retainedViews === undefined
|
||||
? [
|
||||
containing.evidence_id,
|
||||
info.evidence_id,
|
||||
...mainEvidenceIds,
|
||||
...(loadImageEvidenceId === undefined
|
||||
? []
|
||||
: [loadImageEvidenceId]),
|
||||
]
|
||||
: {
|
||||
containing_procedure: containing.evidence_id,
|
||||
function_dossier: info.evidence_id,
|
||||
...(loadImageEvidenceId === undefined
|
||||
? {}
|
||||
: { native_load_image: loadImageEvidenceId }),
|
||||
retained_views: selected.retainedViewEvidence,
|
||||
...(selected.summaryEvidenceId === undefined
|
||||
? {}
|
||||
: { native_summary: selected.summaryEvidenceId }),
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await call("close_binary", {});
|
||||
}
|
||||
};
|
||||
|
||||
export const nativeEntrySeed = async (artifact, path) => {
|
||||
if (artifact === "jq") {
|
||||
const file = await open(path, "r");
|
||||
let header;
|
||||
try {
|
||||
header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer;
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
if (header.subarray(0, 6).toString("hex") !== "7f454c460201")
|
||||
throw new Error("Expected ELF64 little endian entry seed");
|
||||
return `0x${header.readBigUInt64LE(24).toString(16)}`;
|
||||
}
|
||||
if (artifact === "sharex.exe") {
|
||||
const bytes = await readFile(path);
|
||||
const pe = bytes.readUInt32LE(0x3c);
|
||||
const optional = pe + 24;
|
||||
if (
|
||||
bytes.subarray(pe, pe + 4).toString("hex") !== "50450000" ||
|
||||
bytes.readUInt16LE(pe + 4) !== 0x8664 ||
|
||||
bytes.readUInt16LE(optional) !== 0x20b ||
|
||||
bytes.readUInt32LE(optional + 108) <= 14 ||
|
||||
bytes.readUInt32LE(optional + 112 + 14 * 8) !== 0 ||
|
||||
bytes.readUInt32LE(optional + 116 + 14 * 8) !== 0
|
||||
)
|
||||
throw new Error(
|
||||
"Expected the pinned native PE32+ entry seed without CLR metadata",
|
||||
);
|
||||
const address =
|
||||
bytes.readBigUInt64LE(optional + 24) +
|
||||
BigInt(bytes.readUInt32LE(optional + 16));
|
||||
return `0x${address.toString(16)}`;
|
||||
}
|
||||
if (artifact === "freedos.edlin.exe") return undefined;
|
||||
throw new Error("Select a pinned native artifact");
|
||||
};
|
||||
|
|
@ -0,0 +1 @@
|
|||
Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. This is the original UPX-compressed FreeDOS EDLIN 2.24 DOS MZ executable; inspect its admitted load image and entry stub, without running or unpacking it. Keep default auto-analysis enabled. Open the target with provider_id ghidra, call inspect_native_load_image, and use its verified entry.linear_address as the seed. Resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve missing/partial facts and terminal-flow caveats. Verified static import and unpacking-stub pseudocode do not prove DOS execution or recovery of the editor's application logic. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, load_image_status, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. Stop when these facts and cleanup are supported; report unsupported outcomes explicitly.
|
||||
|
|
@ -0,0 +1 @@
|
|||
Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. The independently read, pinned PE32+ header reports image base 0x140000000 and entry RVA 0x125c0, so use address 0x1400125c0 as the analyst seed. This is a native x86-64 Windows PE component with no CLR directory, inspected on Linux; do not claim Windows runtime behavior. Keep default auto-analysis enabled. Open the target with provider_id ghidra, resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve terminal-flow caveats and unresolved static edges; recovered pseudocode is not original source. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. The seed applies only to the pinned ShareX.exe in corpus.json. Stop when these facts and cleanup are supported.
|
||||
|
|
@ -182,18 +182,61 @@ def main():
|
|||
} for row in rows], key=lambda row: row['node_id'])
|
||||
check('all_selected_module_identities', select(answer.get('modules', [])), select(reference['modules']))
|
||||
elif domain == 'native':
|
||||
for key in ['entry_address', 'pseudocode_prefix', 'caller_count', 'callee_count']:
|
||||
for key in ['entry_address', 'caller_count', 'callee_count']:
|
||||
check(key, answer.get(key), reference[key])
|
||||
containing = answer['containing_procedure']
|
||||
prefix = answer.get('pseudocode_prefix')
|
||||
if isinstance(prefix, dict) and prefix.get('unit') == 'utf16-code-units':
|
||||
text = prefix.get('text')
|
||||
if 'requested' in prefix:
|
||||
check('pseudocode_prefix.requested', prefix['requested'], 512)
|
||||
if 'returned' in prefix:
|
||||
check('pseudocode_prefix.returned', prefix['returned'], len(reference['pseudocode_prefix'].encode('utf-16-le')) // 2)
|
||||
prefix = text
|
||||
check('pseudocode_prefix', prefix, reference['pseudocode_prefix'])
|
||||
containing = answer.get('containing_procedure')
|
||||
expected_containing = reference['containing_procedure']['procedure']
|
||||
for key in ['address', 'name']:
|
||||
check('containing_procedure.' + key, containing.get(key, containing.get('procedure', {}).get(key)), expected_containing[key])
|
||||
procedure = answer['procedure']
|
||||
if isinstance(containing, str):
|
||||
# Compare the explicitly reported display to the known source;
|
||||
# never parse a display name into a canonical lookup identity.
|
||||
check('containing_procedure.display', containing,
|
||||
expected_containing['name'] + ' @ ' + expected_containing['address'])
|
||||
else:
|
||||
containing = containing if isinstance(containing, dict) else {}
|
||||
nested = containing.get('procedure')
|
||||
nested = nested if isinstance(nested, dict) else {}
|
||||
for key in ['address', 'name']:
|
||||
check('containing_procedure.' + key, containing.get(key, nested.get(key)), expected_containing[key])
|
||||
procedure = answer.get('procedure')
|
||||
procedure = procedure if isinstance(procedure, dict) else {}
|
||||
expected = reference['procedure']
|
||||
for key in ['address', 'name', 'classification', 'signature', 'locals']:
|
||||
check('procedure.' + key, procedure.get(key), expected[key])
|
||||
check('procedure.body_ranges', procedure.get('body_ranges', procedure.get('body', {}).get('ranges')), expected['body']['ranges'])
|
||||
check('procedure.total_bytes', procedure.get('total_bytes', procedure.get('body', {}).get('total_bytes')), expected['body']['total_bytes'])
|
||||
body = procedure.get('body')
|
||||
body = body if isinstance(body, dict) else {}
|
||||
check('procedure.body_ranges', procedure.get('body_ranges', body.get('ranges', body.get('ranges_inclusive'))), expected['body']['ranges'])
|
||||
check('procedure.total_bytes', procedure.get('total_bytes', body.get('total_bytes')), expected['body']['total_bytes'])
|
||||
if 'load_image' in reference:
|
||||
sources = [source for source in producer_results.get('inspect_native_load_image', [])
|
||||
if isinstance(source, dict)
|
||||
and source.get('operation') == 'inspect_native_load_image'
|
||||
and isinstance(source.get('evidence_id'), str)
|
||||
and source.get('evidence_id') in reported_ids
|
||||
and isinstance(source.get('normalized_result'), dict)]
|
||||
checks['load_image_source_observed_and_referenced'] = bool(sources)
|
||||
load = sources[-1]['normalized_result'] if sources else {}
|
||||
entry = load.get('entry')
|
||||
entry = entry if isinstance(entry, dict) else {}
|
||||
check('load_image.target_sha256', load.get('target_sha256'), reference['artifact_sha256'])
|
||||
check('load_image.source_status', load.get('status'), reference['load_image']['status'])
|
||||
check('load_image.source_entry', entry.get('linear_address'), reference['entry_address'])
|
||||
status = answer.get('load_image_status')
|
||||
check('load_image_status', status.get('status') if isinstance(status, dict) else status, load.get('status'))
|
||||
if isinstance(status, dict):
|
||||
for key in ['format', 'load_segment', 'header_bytes', 'module_bytes', 'overlay_bytes']:
|
||||
if key in status:
|
||||
check('load_image.' + key, status[key], load.get(key))
|
||||
if 'entry_linear_address' in status:
|
||||
check('load_image.entry_linear_address', status['entry_linear_address'], entry.get('linear_address'))
|
||||
calls = summary['completed_mcp_calls']
|
||||
checks['default_analysis_not_disabled'] = all(not call['arguments'].get('skip_auto_analysis', False) for call in calls if call['tool'] == 'open_binary')
|
||||
checks['session_close_completed'] = any(call['tool'] == 'close_binary' and call['status'] == 'completed' for call in calls)
|
||||
|
|
@ -248,6 +291,7 @@ def main():
|
|||
and events['memory.events'].get('high') == 0
|
||||
)
|
||||
report = {
|
||||
'review_source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
||||
'domain': domain, 'checks': checks, 'all_selected_checks_pass': all(checks.values()),
|
||||
'differences': differences, 'tool_errors': tool_errors, 'summary': summary,
|
||||
'fingerprints': {key: value for key, value in settings.items() if 'sha256' in key or key in ['runtime_revision', 'enabled_tools', 'authority']},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,336 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
|
||||
// A recording port checks the experimental producer/consumer representation.
|
||||
// Real provider verification remains in the separate native experiment receipts.
|
||||
const generate = `
|
||||
import {measureNativeEntry} from './experiments/analysis-efficiency/native-entry.mjs';
|
||||
const sha='0'.repeat(64);
|
||||
const duplicate=process.argv[1]==='duplicate';
|
||||
const stale=process.argv[1]==='stale-dos';
|
||||
const collision=process.argv[1];
|
||||
const dos=process.argv[1]==='dos'||stale||collision==='load-collision';
|
||||
const dossier={procedure:{address:'0x1000'},pseudocode:'return;',callers:[],callees:[]};
|
||||
class ToolFailure extends Error {}
|
||||
const build=async digit=>{
|
||||
const parent='ev_'+digit.repeat(63)+'0';
|
||||
let ordinal=0;
|
||||
const call=async(name,args)=>{
|
||||
if(stale)process.stdout.write(name+'\\n');
|
||||
if(name==='open_binary'||name==='close_binary')return {};
|
||||
if(name==='inspect_native_load_image')return {evidence_id:'ev_'+digit.repeat(63)+'6',normalized_result:{status:'verified',target_sha256:stale?'f'.repeat(64):sha,entry:{linear_address:'0x1000'}}};
|
||||
if(name==='resolve_containing_procedure')return {evidence_id:'ev_'+digit.repeat(63)+'7',normalized_result:{found:true}};
|
||||
if(name==='analyze_function')return {evidence_id:parent,subject:{digest:{sha256:sha}},normalized_result:dossier};
|
||||
const id='ev_'+digit.repeat(63)+(collision==='containing-collision'?7:collision==='load-collision'?6:duplicate?1:++ordinal);
|
||||
if(args.view.kind==='summary')return {evidence_id:id};
|
||||
const facet=args.view.facet;
|
||||
const item=facet==='pseudocode'?{text:dossier.pseudocode,unit:'utf16-code-units'}:dossier[facet];
|
||||
return {evidence_id:id,normalized_result:{parent_evidence_id:parent,artifact:{sha256:sha},item,coverage:{exhausted:true},limitations:[],unknowns:[]}};
|
||||
};
|
||||
return measureNativeEntry({seed:dos?undefined:'0x1000',values:{path:'/selected/input','native-question':'entry-views'},call,inputSha:sha,ToolFailure});
|
||||
};
|
||||
process.stdout.write(JSON.stringify([await build('1'),await build('2')]));
|
||||
`;
|
||||
|
||||
it.each([
|
||||
"observed",
|
||||
"missing",
|
||||
"errored",
|
||||
"wrong-target",
|
||||
"wrong-entry",
|
||||
"unreferenced",
|
||||
])(
|
||||
"binds the DOS model grade to its observed load image: %s",
|
||||
async (scenario) => {
|
||||
// Synthetic transcripts exercise grading; they are not real-provider or
|
||||
// resource measurements. Real DOS transcripts are replayed separately.
|
||||
const root = await createTestTempDirectory("rea-dos-grader-");
|
||||
const trial = join(root, "trial");
|
||||
const referenceDirectory = join(root, "reference");
|
||||
await mkdir(trial);
|
||||
await mkdir(referenceDirectory);
|
||||
const sha = "0".repeat(64);
|
||||
const imageId = "ev_" + "1".repeat(64);
|
||||
const procedureId = "ev_" + "2".repeat(64);
|
||||
const procedure = {
|
||||
address: "0x1000",
|
||||
name: "entry",
|
||||
classification: "observed",
|
||||
signature: null,
|
||||
locals: [],
|
||||
body: { ranges: [], total_bytes: 0 },
|
||||
};
|
||||
const loadImage = {
|
||||
status: "verified",
|
||||
target_sha256: sha,
|
||||
entry: { linear_address: "0x1000" },
|
||||
};
|
||||
const reference = {
|
||||
artifact_sha256: sha,
|
||||
entry_address: "0x1000",
|
||||
caller_count: 0,
|
||||
callee_count: 0,
|
||||
pseudocode_prefix: "return;",
|
||||
containing_procedure: { procedure },
|
||||
procedure,
|
||||
load_image: loadImage,
|
||||
};
|
||||
const answer = {
|
||||
...reference,
|
||||
load_image_status: "verified",
|
||||
evidence_ids: [
|
||||
procedureId,
|
||||
...(["missing", "errored", "unreferenced"].includes(scenario)
|
||||
? []
|
||||
: [imageId]),
|
||||
],
|
||||
};
|
||||
const event = (tool: string, status: string, value: unknown) => ({
|
||||
type: "item.completed",
|
||||
item: {
|
||||
type: "mcp_tool_call",
|
||||
tool,
|
||||
status,
|
||||
arguments: {},
|
||||
result: { structured_content: value },
|
||||
},
|
||||
});
|
||||
const events = [
|
||||
event("analyze_function", "completed", { evidence_id: procedureId }),
|
||||
];
|
||||
if (scenario !== "missing")
|
||||
events.push(
|
||||
event(
|
||||
"inspect_native_load_image",
|
||||
scenario === "errored" ? "failed" : "completed",
|
||||
scenario === "errored"
|
||||
? { error: { code: "unsupported_target" } }
|
||||
: {
|
||||
operation: "inspect_native_load_image",
|
||||
evidence_id: imageId,
|
||||
normalized_result: {
|
||||
...loadImage,
|
||||
target_sha256:
|
||||
scenario === "wrong-target" ? "f".repeat(64) : sha,
|
||||
entry: {
|
||||
linear_address:
|
||||
scenario === "wrong-entry" ? "0x2000" : "0x1000",
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
const resourceLog = join(root, "resource.log");
|
||||
const output = join(root, "review.json");
|
||||
for (const [path, value] of [
|
||||
[
|
||||
join(trial, "summary.json"),
|
||||
{
|
||||
completed_mcp_calls: [
|
||||
{ tool: "close_binary", status: "completed", arguments: {} },
|
||||
],
|
||||
},
|
||||
],
|
||||
[join(trial, "trial-settings.json"), {}],
|
||||
[join(trial, "cleanup.json"), { profile_removed: true }],
|
||||
[join(trial, "answer.txt"), answer],
|
||||
[join(referenceDirectory, "answer.json"), reference],
|
||||
[
|
||||
resourceLog,
|
||||
{
|
||||
resource_scope: "synthetic-grader-fixture",
|
||||
abort: null,
|
||||
resource_counters_verified: true,
|
||||
resource_events_final: {
|
||||
"memory.events": { high: 0, max: 0, oom: 0, oom_kill: 0 },
|
||||
"pids.events": { max: 0 },
|
||||
},
|
||||
},
|
||||
],
|
||||
] as const)
|
||||
await writeFile(path, JSON.stringify(value));
|
||||
await writeFile(
|
||||
join(trial, "events.jsonl"),
|
||||
events.map((value) => JSON.stringify(value)).join("\n"),
|
||||
);
|
||||
const command = execute("python3", [
|
||||
resolve("experiments/analysis-efficiency/review-model.py"),
|
||||
"--domain",
|
||||
"native",
|
||||
"--trial",
|
||||
trial,
|
||||
"--reference",
|
||||
referenceDirectory,
|
||||
"--resource-log",
|
||||
resourceLog,
|
||||
"--output",
|
||||
output,
|
||||
]);
|
||||
if (scenario === "observed") await command;
|
||||
else await expect(command).rejects.toMatchObject({ code: 1 });
|
||||
const report = JSON.parse(await readFile(output, "utf8"));
|
||||
expect(report.all_selected_checks_pass).toBe(scenario === "observed");
|
||||
if (["missing", "errored", "unreferenced"].includes(scenario))
|
||||
expect(report.checks.load_image_source_observed_and_referenced).toBe(
|
||||
false,
|
||||
);
|
||||
if (scenario === "wrong-target")
|
||||
expect(report.checks["load_image.target_sha256"]).toBe(false);
|
||||
if (scenario === "wrong-entry")
|
||||
expect(report.checks["load_image.source_entry"]).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("retains the DOS load-image source for its facts and derived entry seed", async () => {
|
||||
const { stdout } = await execute(process.execPath, [
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
generate,
|
||||
"dos",
|
||||
]);
|
||||
const [answer] = JSON.parse(stdout);
|
||||
expect(answer).toMatchObject({
|
||||
entry_address: "0x1000",
|
||||
load_image: { status: "verified", entry: { linear_address: "0x1000" } },
|
||||
evidence_ids: { native_load_image: "ev_" + "1".repeat(63) + "6" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a stale DOS load image before using its entry and closes the session", async () => {
|
||||
await expect(
|
||||
execute(process.execPath, [
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
generate,
|
||||
"stale-dos",
|
||||
]),
|
||||
).rejects.toMatchObject({
|
||||
stdout: "open_binary\ninspect_native_load_image\nclose_binary\n",
|
||||
stderr: expect.stringContaining(
|
||||
"The DOS load image belongs to a different target",
|
||||
),
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["containing-collision", "load-collision"])(
|
||||
"rejects a retained view that reuses an earlier producer identity: %s",
|
||||
async (collision) => {
|
||||
await expect(
|
||||
execute(process.execPath, [
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
generate,
|
||||
collision,
|
||||
]),
|
||||
).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining(
|
||||
"Retained native view reused an Evidence identity",
|
||||
),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects a producer that reuses a view identity across facets", async () => {
|
||||
await expect(
|
||||
execute(process.execPath, [
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
generate,
|
||||
"duplicate",
|
||||
]),
|
||||
).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining(
|
||||
"Retained native view reused an Evidence identity",
|
||||
),
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["references", "item", "coverage", "limitations", "unknowns"])(
|
||||
"separates native references while comparing %s exactly",
|
||||
async (change) => {
|
||||
const { stdout: generated } = await execute(process.execPath, [
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
generate,
|
||||
]);
|
||||
const [leftAnswer, rightAnswer] = JSON.parse(generated);
|
||||
if (change === "item")
|
||||
rightAnswer.retained_views[0].item.address = "0x1001";
|
||||
if (change === "coverage")
|
||||
rightAnswer.retained_views[0].coverage.exhausted = false;
|
||||
if (change === "limitations")
|
||||
rightAnswer.retained_views[0].limitations.push("partial");
|
||||
if (change === "unknowns")
|
||||
rightAnswer.retained_views[0].unknowns.push("unresolved");
|
||||
|
||||
const root = await createTestTempDirectory("rea-native-comparison-");
|
||||
const left = join(root, "left");
|
||||
const right = join(root, "right");
|
||||
const summary = {
|
||||
artifact: "sharex.exe",
|
||||
native_question: "entry-views",
|
||||
provider_search_path_sha256: "0".repeat(64),
|
||||
native_workflow_source_sha256: "1".repeat(64),
|
||||
calls: [],
|
||||
failure: null,
|
||||
};
|
||||
for (const [directory, answer] of [
|
||||
[left, leftAnswer],
|
||||
[right, rightAnswer],
|
||||
]) {
|
||||
await mkdir(directory);
|
||||
await writeFile(join(directory, "summary.json"), JSON.stringify(summary));
|
||||
await writeFile(join(directory, "answer.json"), JSON.stringify(answer));
|
||||
}
|
||||
const output = join(root, "comparison.json");
|
||||
const command = execute(process.execPath, [
|
||||
resolve("experiments/analysis-efficiency/compare.mjs"),
|
||||
"--left",
|
||||
left,
|
||||
"--right",
|
||||
right,
|
||||
"--output",
|
||||
output,
|
||||
]);
|
||||
if (change !== "references") {
|
||||
await expect(command).rejects.toMatchObject({
|
||||
code: 1,
|
||||
stdout: expect.stringContaining('"retained_views"'),
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { stdout } = await command;
|
||||
expect(JSON.parse(stdout)).toMatchObject({
|
||||
settings_match: true,
|
||||
facts_match: true,
|
||||
preserved_run_specific_values: [
|
||||
{
|
||||
side: "left",
|
||||
path: ["evidence_ids"],
|
||||
value: leftAnswer.evidence_ids,
|
||||
},
|
||||
{
|
||||
side: "right",
|
||||
path: ["evidence_ids"],
|
||||
value: rightAnswer.evidence_ids,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(leftAnswer.evidence_ids).not.toEqual(rightAnswer.evidence_ids);
|
||||
expect(leftAnswer.evidence_ids.retained_views[0]).toMatchObject({
|
||||
evidence_id: "ev_" + "1".repeat(64),
|
||||
parent_evidence_id: "ev_" + "1".repeat(63) + "0",
|
||||
});
|
||||
expect(leftAnswer.evidence_ids.native_summary).toBe(
|
||||
"ev_" + "1".repeat(63) + "5",
|
||||
);
|
||||
},
|
||||
);
|
||||
Loading…
Reference in New Issue