From 1dd20ce1ff4ab66c74350929577e753aa17a9006 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 05:08:42 +0800 Subject: [PATCH] test(analysis): verify real preset snapshot and cleanup boundaries (#2349) Verify existing caller-selected Ghidra profiles and immutable JSON snapshot boundaries through real CLI/MCP workflows on GNU true and shipped jq. Preserve exact task facts, profile/source identity, original snapshot records and existing defaults; require actual stored query entries and scoped headless launch evidence. Exercise pending startup cancellation with production token/native start-identity inspection, recovery under the selected target, controlled real permission cleanup failure, retained ownership and restored-permission close retry. Independently confirm recovered socket/root cleanup. Preserve failed observer assumptions, source fingerprints, measured scope and final-resource counter unknowns. No general fast preset, product API change or new performance claim. Refs #2016 Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- experiments/analysis-efficiency/README.md | 4 + .../analysis-efficiency/preset-lifecycle.md | 208 +++++++++ .../verify-preset-lifecycle.mjs | 395 ++++++++++++++++++ .../verify-preset-snapshot.mjs | 387 +++++++++++++++++ 4 files changed, 994 insertions(+) create mode 100644 experiments/analysis-efficiency/preset-lifecycle.md create mode 100644 experiments/analysis-efficiency/verify-preset-lifecycle.mjs create mode 100644 experiments/analysis-efficiency/verify-preset-snapshot.mjs diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 852e28ccd..894eb0347 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -19,6 +19,10 @@ Follow-up: [EVM interface summary delivery](evm-interface-summary.md) records the opt-in retained view, old-client/Evidence migration checks and repeated SDK and real-model pairs, including mixed token/time results and strict-answer limits. +Follow-up: [task-scoped preset lifecycle verification](preset-lifecycle.md) checks +real Ghidra profile/snapshot boundaries, startup cancellation and retained cleanup +retry while preserving defaults and distinguishing verification from performance. + The 12 detailed checkpoint receipts were archived byte-for-byte before removal from the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in the private experiment lab outside the checkout; its `archive-index.json` records diff --git a/experiments/analysis-efficiency/preset-lifecycle.md b/experiments/analysis-efficiency/preset-lifecycle.md new file mode 100644 index 000000000..ee46c4914 --- /dev/null +++ b/experiments/analysis-efficiency/preset-lifecycle.md @@ -0,0 +1,208 @@ +# Task-scoped preset lifecycle verification + +Continues [#2016](https://github.com/morluto/rea/issues/2016) after +[#2308](https://github.com/morluto/rea/pull/2308) merged. This increment verifies +existing caller-selected Ghidra settings and immutable JSON snapshots through +actual public CLI/MCP workflows. Production defaults, provider selection and +public contracts remain unchanged; an engine database cache is still deferred. + +Runtime baseline is main `f76163c2467db623f9de0bda757c9d8610ca9565`, built with +Node 24.18.0 and matching locked dependencies. Real Ghidra 12.1.4 uses the +existing caller-supplied JDK 21 and 768 MiB Java heap under one CPU. No target +execution, annotation or attached GUI database operation is requested. + +## Small shipped input and profile boundaries + +The first input is the installed `/usr/bin/true`, 35,664 bytes, SHA-256 +`c79bf44242829108e323378531f4ac839513ca1fba45efd6583643526e1e9fd2`. +The local package database reports Debian `coreutils` `9.1-1` `amd64`; these are +installed-artifact observations, not independent archive authenticity proof. +The target itself is never invoked, including for version discovery. REA's +actual `list_procedures` chooses an internal, non-thunk procedure at `0x102000`. + +The default profile reports `analyzer_preset: "ghidra-default"`. The selected +alternative reports `ghidra-default+disabled:Decompiler Switch Analysis` and +the exact disabled name in its analysis options. Their profile digests differ. +The selected initializer's complete normalized facts agree, while the profile +and provenance remain distinct. This does not establish equivalence for other +procedures, call graphs or applications. The earlier jq experiment found a +callee body difference; the existing default remains the recommendation. + +The public [snapshot verifier](verify-preset-snapshot.mjs) checks: + +- The default `analyze_function` actually produces a stored query entry and an + authenticated original Evidence record in the exported snapshot. +- The same profile returns identical task facts, subject, parameters and profile + through a fresh MCP connection. Its limitations explicitly mark snapshot reuse; + its distinct response identity does not masquerade as the original record. +- A disabled profile refuses the default snapshot before headless Ghidra starts, + then successfully analyzes the same caller-selected target with its own profile. +- Restoring the default refuses the disabled snapshot and accepts the original + default snapshot. Original snapshot/source bytes stay unchanged. +- The existing CLI `function --provider ghidra --snapshot ... --json` returns + the original cached Evidence identity and exact task facts without modifying + the source snapshot. + +Actual `strace` records the known headless launch command once for fresh default +and once for fresh disabled analysis, and zero times for either rejection or +accepted reuse. The audit covers that known command, not an arbitrary execution +or network-absence guarantee. Source bytes and snapshot bytes remain unchanged. + +## Real cancellation and cleanup retry + +The [lifecycle verifier](verify-preset-lifecycle.mjs) observes actual pending +startup before the bridge endpoint exists. It verifies the JVM's run token, +launcher relationship and native process start identity with production process +inspection primitives, then cancels through the SDK's normal request signal. +Read-only token sweeps confirm cleanup, including processes outside the old +parent chain; owned runtime/socket directories are removed. The selected target +remains open and a fresh provider query succeeds. + +REA preserves the logical run ID when reimporting that selected target. A new +native process start identity proves the new generation; requiring a new logical +ID would test a different contract. No PID/path pair alone establishes ownership. + +After recovery, the verifier removes write permission only from the verified, +user-owned temporary Ghidra project directory. The actual public `close_binary` +reports `cleanup_incomplete` with the retained resource and `EACCES` reason. +Permissions are restored, a second close succeeds, and token sweeps/root removal +confirm cleanup. The original executable is unchanged. This is a controlled real +filesystem failure, not a recording provider simulation. The error does not +authorize automatic retry before fixing its cause. + +These observations pass for both the default and disabled-switch profiles on the +small input. The source verifier preserves verification and restoration/close +failures together, always attempts cleanup, validates the manifest at its read +boundary and publishes success only after its cleanup callbacks complete. + +## Failed harness assumptions and measurement limits + +Keep the initial probes and their corrections outside Git: + +- A primitive-only `procedure_info` snapshot contained no replay entries. Equal + content-addressed output after another analysis was not proof of cache reuse. + The corrected probe requires a stored `analyze_function` entry, a cache-reuse + marker and actual headless launch traces. +- After cancellation, activity was `not_observed`, rather than `idle`, because + no provider activity snapshot remained. Preserve that unknown; successful fresh + work proves recovery without relabeling absent metadata. +- Reimport preserved the logical run ID. Native start identity, not that logical + ID, establishes the new process generation. + +These were verification assumptions, not newly demonstrated REA defects. No +unrelated production repair is made. Existing defects remain in the requested +private `/tmp` record. + +The initial complete snapshot/profile/CLI source run takes 91.838 seconds and +samples 1,180,246,016 peak family bytes. The real default/disabled cancellation +and permission-retry prototype takes 66.977 seconds and samples 1,040,846,848 +bytes. Their last-observed CPU totals are 90.486 and 65.542 seconds respectively. +Both use one CPU, 4 GiB hard memory, 3 GiB early-stop, no swap and 192 tasks. +No scope abort occurs, but final counters disappear at teardown and remain +unverified. Sampled values are not final OOM/limit-event proof. + +These are lifecycle verification costs. Discovery, profile inspection, rejection, +cleanup and syscall tracing make their call sequences different. They are not +paired completed-question performance trials, and contain no model token counts. +Explicit snapshot reuse is existing REA behavior, not a new speedup introduced by +these verifiers. Broader native inputs and failure scenarios remain follow-ups. + +## Shipped jq continuation + +The same source snapshot verifier additionally passes on the pinned public jq +binary, SHA-256 +`b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f`. +Procedure `0x403a50` comes from the earlier actual REA entry observation, rather +than a newly guessed file offset. Default and disabled profiles each perform a +fresh headless analysis. Both profile mismatch directions reject before a +headless launch, both valid default reuses return exact task facts with zero +headless launches, and CLI reuse retains the original Evidence ID and snapshot +bytes. The selected entry agrees across profiles; the earlier differing callee +still prevents a general equivalence or fast-preset recommendation. + +Default/disabled fresh `analyze_function` calls take 294.525/298.621 seconds in +this verification; default snapshot calls take 0.222 and 0.168 seconds. The CLI +reuse takes 2.899 seconds. These are calls within different verification sequences, +not repeated completed-question performance cohorts. They demonstrate the existing +explicit snapshot mechanism without an engine database cache; they do not measure +model tokens or introduce a new optimization. + +The complete jq verification takes 624.537 seconds, samples 1,218,355,200 peak +family bytes and observes 567.198 CPU seconds before teardown. The final resource +counters remain unknown. The source verifier SHA-256 is +`42060686f9633da06bb5f257fa5c70f588e66569cf0befebe6a1a13f43bf0e88`; +the initial lifecycle verifier hash is +`b1680f4f506ef9328d1beabb4b1fbe97c911b3f06986857f0a59a4b97a3b61d3`. +Both hashes also match completed small-input source runs, including the final +cleanup/error-preservation implementation. Runtime production remains the pinned +main baseline; the follow-up changes only experiment sources and documentation. + +## Invalid analyzer failure and healthy peers + +Real MCP and CLI requests with `REA_GHIDRA_DISABLED_ANALYZERS` set to +`REA Nonexistent Analyzer 2016` both report `execution_failure`. Their diagnostics +preserve the exact unavailable analyzer name, selected provider, operation, +target identity and Ghidra pre-script failure. MCP still answers a bundled EVM +peer request and closes the selected native target; original target bytes stay +unchanged and the owned Ghidra runtime directory is removed. + +The first healthy-peer probe assumed ELF layout support was built in. This main +requires an explicitly supplied pwntools interpreter for that format and correctly +reports `provider_unavailable`; none was configured. The corrected probe uses +the already verified bundled EVM peer. No installer or provider fallback is added. + +The first CLI observer also looked inside `error` as if it had MCP's shape. The +actual CLI envelope has an error string and sibling `code`, `message`, `details` +and other fields. Regrading its saved complete reply proves the named diagnostics +are preserved. The original incorrect assertion/report remains archived; no +provider rerun or production repair is needed for that correction. + +The invalid-name behavior is reproducible with the existing CLI: + +```sh +REA_GHIDRA_DISABLED_ANALYZERS='REA Nonexistent Analyzer 2016' \ + node ABS_RUNTIME/scripts/rea.mjs function ABS_TRUE_BINARY 0x102000 \ + --provider ghidra --json +``` + +Use the address actually observed by REA for a different target. The deliberately +invalid name is expected to fail; fixing the setting is required before retrying. + +## Reproduction + +Build the pinned runtime and supply an explicit string-valued provider settings +JSON, real Ghidra/JDK, a caller-supplied Linux strace executable and a fresh private +output directory. The snapshot verifier streams target hashes and stores actual +SDK/CLI replies, profile commitments and exec traces outside the checkout: + +```sh +node experiments/analysis-efficiency/verify-preset-snapshot.mjs \ + --runtime-root ABS_RUNTIME --revision SHA --path ABS_NATIVE_TARGET \ + --providers ABS_SETTINGS_JSON --strace ABS_STRACE --output NEW_ABS_DIR +``` + +Select `--procedure` from actual REA evidence when the question already identifies +one; otherwise the verifier uses the actual procedure list. `--timeout-ms` is an +explicit observation window; `--disabled-analyzers` uses the existing setting. + +Run cancellation/permission retry as a non-root Linux user: + +```sh +node experiments/analysis-efficiency/verify-preset-lifecycle.mjs \ + --runtime-root ABS_RUNTIME --revision SHA --path ABS_NATIVE_TARGET \ + --providers ABS_SETTINGS_JSON --output NEW_ABS_DIR +``` + +Runtime revision is caller-supplied; keep the built/frozen runtime receipt alongside +the verifier source fingerprint. Target bytes, provider projects, raw JSON, traces +and profiles are not committed. The scripts acquire no dependencies or change +the user's installed configuration. + +## Final cleanup review + +The final review adds an independent filesystem check that the recovered +bridge socket directory is absent after permission restoration and close retry, +and preserves its ownership manifest in the receipt. Both default and disabled +profiles pass this related real workflow again. Its source SHA-256 is +`f5d29efdb056032072b99ac1e6fe1d6d2099bbe84c324c75032bd0178bb747c5`. The previous observations and source hashes stay archived; +the jq/model/provider matrices were not repeated for this observer change. diff --git a/experiments/analysis-efficiency/verify-preset-lifecycle.mjs b/experiments/analysis-efficiency/verify-preset-lifecycle.mjs new file mode 100644 index 000000000..8675c8590 --- /dev/null +++ b/experiments/analysis-efficiency/verify-preset-lifecycle.mjs @@ -0,0 +1,395 @@ +import assert from "node:assert/strict"; +import { createHash, randomUUID } from "node:crypto"; +import { + access, + chmod, + lstat, + mkdir, + readFile, + readdir, + writeFile, +} from "node:fs/promises"; +import { dirname, join, isAbsolute } from "node:path"; +import { parseArgs } from "node:util"; +import { createReadStream } from "node:fs"; +import { setTimeout as delay } from "node:timers/promises"; +import { pathToFileURL } from "node:url"; +import { z } from "zod"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +const { values } = parseArgs({ + options: { + "runtime-root": { type: "string" }, + revision: { type: "string" }, + path: { type: "string" }, + providers: { type: "string" }, + output: { type: "string" }, + "disabled-analyzers": { + type: "string", + default: "Decompiler Switch Analysis", + }, + "timeout-ms": { type: "string", default: "900000" }, + }, +}); +for (const key of ["runtime-root", "path", "providers", "output"]) + assert.ok(values[key] && isAbsolute(values[key]), "Supply --" + key + " ABS"); +assert.match(values.revision ?? "", /^[a-f0-9]{40}$/u); +assert.equal(process.platform, "linux"); +assert.notEqual( + process.getuid(), + 0, + "Permission fault injection requires a non-root user", +); +const timeout = Number(values["timeout-ms"]); +assert.ok(Number.isSafeInteger(timeout) && timeout > 0); +const lab = values.output, + runtime = values["runtime-root"]; +const { ghidraDisabledAnalyzersSchema } = await import( + pathToFileURL(runtime + "/dist/config/ghidraDisabledAnalyzers.js") +); +assert.ok( + ghidraDisabledAnalyzersSchema.parse(values["disabled-analyzers"]) !== + undefined, + "Select at least one analyzer to disable", +); +await mkdir(lab, { mode: 0o700 }); +const providerBytes = await readFile(values.providers); +const providers = JSON.parse(providerBytes.toString("utf8")); +assert.ok( + providers !== null && + !Array.isArray(providers) && + typeof providers === "object", +); +assert.ok(Object.values(providers).every((x) => typeof x === "string")); +const verifierSource = await readFile(new URL(import.meta.url)); +const { + observeOwnedProcessLineage, + observeProcessStartIdentity, + systemProcessOwnershipHost: host, +} = await import( + pathToFileURL(runtime + "/dist/process/ProcessOwnershipObservation.js") +); +const { verifyNoTokenOwnedProcesses } = await import( + pathToFileURL(runtime + "/dist/process/ProcessOwnership.js") +); +const target = values.path; +const sha = (x) => createHash("sha256").update(x).digest("hex"); +const fileSha = async (path) => { + const hash = createHash("sha256"); + for await (const part of createReadStream(path)) hash.update(part); + return hash.digest("hex"); +}; +const before = await fileSha(target); +const rows = []; +const roots = async (work) => + (await readdir(work)) + .filter((x) => x.startsWith("rea-ghidra-")) + .map((x) => join(work, x)); +const ownershipSchema = z + .object({ + run_id: z.string().uuid(), + pid: z.number().int().positive(), + process_group_id: z.number().int().positive(), + parent_pid: z.number().int().positive(), + launcher: z.string().min(1), + endpoint_path: z.string().min(1), + ownership_kind: z.literal("posix-process-group"), + }) + .passthrough(); +const manifest = async (root) => + ownershipSchema.parse( + JSON.parse(await readFile(join(root, "ownership.json"), "utf8")), + ); +const verifyWithCleanup = async (verify, cleanups) => { + const failures = []; + let result; + try { + result = await verify(); + } catch (error) { + failures.push(error); + } + for (const cleanup of cleanups) { + try { + await cleanup(); + } catch (error) { + failures.push(error); + } + } + if (failures.length > 0) + throw new AggregateError( + failures, + "Verification or resource cleanup failed", + ); + return result; +}; +const restoreProject = async (project, mode) => { + if (project === undefined || mode === undefined) return; + try { + await chmod(project, mode); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } +}; +const unreadyLaunch = async (work) => { + for (const root of await roots(work)) { + let record; + try { + record = await manifest(root); + } catch (error) { + if (error.code === "ENOENT") continue; + throw error; + } + const lineage = await observeOwnedProcessLineage(owned(record)); + if (lineage.status !== "verified") continue; + const identity = await observeProcessStartIdentity( + record.pid, + undefined, + record.run_id, + ); + if (identity?.state !== "readable") continue; + try { + await access(record.endpoint_path); + } catch (error) { + if (error.code !== "ENOENT") throw error; + return { root, manifest: record, lineage, start_identity: identity }; + } + } +}; +const owned = (m) => ({ + runId: m.run_id, + leaderPid: m.pid, + processGroupId: m.process_group_id, + expectedParentPid: m.parent_pid, + expectedCommand: m.launcher, +}); +const cancelStartup = async ({ client, defs, work, call }) => { + const baseline = await host.captureBaseline(); + const controller = new AbortController(); + const pattern = "rea-lifecycle-" + randomUUID(); + let settled = false; + const pending = client + .callTool( + { name: "search_procedures", arguments: { pattern } }, + { + toolDefinition: defs.get("search_procedures"), + timeout, + signal: controller.signal, + }, + ) + .then( + (reply) => { + settled = true; + return { reply }; + }, + (error) => { + settled = true; + return { + error: { + name: error.name, + message: error.message, + code: error.code, + }, + }; + }, + ); + let launch; + try { + const deadline = Date.now() + 30000; + while (Date.now() < deadline && !settled) { + launch = await unreadyLaunch(work); + if (launch) break; + await delay(50); + } + assert.ok(launch, "No token-verified live JVM before endpoint readiness"); + assert.equal(settled, false); + } finally { + controller.abort(new Error("Preset startup cancellation")); + await pending; + } + const canceled = await pending; + assert.ok(canceled.error, "Pending SDK request did not cancel"); + await writeFile(work + "/cancellation.json", JSON.stringify(canceled), { + flag: "wx", + mode: 0o600, + }); + let cleanup; + const deadline = Date.now() + 30000; + while (Date.now() < deadline) { + cleanup = await verifyNoTokenOwnedProcesses( + launch.manifest.run_id, + undefined, + baseline, + { + leaderPid: launch.manifest.pid, + processGroupId: launch.manifest.process_group_id, + }, + ); + const liveRoots = await roots(work); + if (cleanup.cleaned && liveRoots.length === 0) break; + await delay(100); + } + assert.equal(cleanup.cleaned, true, JSON.stringify(cleanup)); + assert.deepEqual(await roots(work), []); + await assert.rejects(access(dirname(launch.manifest.endpoint_path)), { + code: "ENOENT", + }); + const afterCancel = (await call("binary_session")).result; + assert.equal(afterCancel.open, true); + assert.notEqual(afterCancel.analysis_activity.status, "busy"); + return { launch, cleanup, afterCancel, pattern }; +}; +for (const profile of ["default", "disabled"]) { + const work = lab + "/" + profile; + await mkdir(work, { mode: 0o700 }); + const client = new Client({ + name: "rea-real-preset-lifecycle", + version: "1", + }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [runtime + "/scripts/rea.mjs", "mcp"], + env: { + ...process.env, + ...providers, + NODE_OPTIONS: "--max-old-space-size=512", + TMPDIR: work, + REA_GHIDRA_DISABLED_ANALYZERS: + profile === "default" ? "" : values["disabled-analyzers"], + }, + stderr: "pipe", + }); + const defs = new Map(); + let count = 0; + const call = async (name, args = {}, error = false) => { + const definition = defs.get(name); + assert.ok(definition, "Missing named contract " + name); + const reply = await client.callTool( + { name, arguments: args }, + { toolDefinition: definition, timeout }, + ); + await writeFile( + work + "/call-" + ++count + ".json", + JSON.stringify(reply), + { flag: "wx", mode: 0o600 }, + ); + const value = + reply.structuredContent ?? + JSON.parse(reply.content.find((x) => x.type === "text").text); + assert.equal(reply.isError === true, error, JSON.stringify(value)); + return value; + }; + let project, mode; + const row = await verifyWithCleanup(async () => { + await client.connect(transport); + transport.stderr?.resume(); + for (const t of (await client.listTools()).tools) defs.set(t.name, t); + await call("open_binary", { path: target, provider_id: "ghidra" }); + const selectedProfile = (await call("binary_session")).result + .analysis_provider_binding.analysis_profile; + const { launch, cleanup, afterCancel, pattern } = await cancelStartup({ + client, + defs, + work, + call, + }); + const recovered = await call("search_procedures", { pattern }); + assert.deepEqual(recovered.normalized_result, []); + const activeRoots = await roots(work); + assert.equal(activeRoots.length, 1); + const active = await manifest(activeRoots[0]); + const activeIdentity = await observeProcessStartIdentity( + active.pid, + undefined, + active.run_id, + ); + assert.equal(activeIdentity?.state, "readable"); + assert.notEqual(activeIdentity.identity, launch.start_identity.identity); + const activeLineage = await observeOwnedProcessLineage(owned(active)); + assert.equal(activeLineage.status, "verified"); + const activeBaseline = await host.captureBaseline(); + project = join(activeRoots[0], "project"); + const info = await lstat(project); + assert.ok(info.isDirectory() && !info.isSymbolicLink()); + assert.equal( + info.uid, + process.getuid(), + "Fault must affect only this user-owned project", + ); + mode = info.mode & 0o777; + await chmod(project, 0o500); + const refusal = await verifyWithCleanup( + () => call("close_binary", {}, true), + [() => restoreProject(project, mode)], + ); + assert.equal(refusal.error.code, "cleanup_incomplete"); + assert.ok(refusal.error.details.resources.includes(activeRoots[0])); + await writeFile(work + "/cleanup-refusal.json", JSON.stringify(refusal), { + flag: "wx", + mode: 0o600, + }); + assert.ok( + (await roots(work)).includes(activeRoots[0]), + "Failed cleanup resource ownership disappeared", + ); + await call("close_binary"); + assert.deepEqual(await roots(work), []); + await assert.rejects(access(dirname(active.endpoint_path)), { + code: "ENOENT", + }); + const retried = await verifyNoTokenOwnedProcesses( + active.run_id, + undefined, + activeBaseline, + { leaderPid: active.pid, processGroupId: active.process_group_id }, + ); + assert.equal(retried.cleaned, true, JSON.stringify(retried)); + assert.equal((await call("binary_session")).result.open, false); + assert.equal(await fileSha(target), before); + return { + profile, + selected_profile: selectedProfile, + source_sha256: before, + actual_pending_startup_canceled: true, + verified_launch: launch, + token_cleanup: cleanup, + post_cancellation_activity: afterCancel.analysis_activity, + recovery_preserves_selected_target: true, + logical_run_id_after_recovery: active.run_id, + recovered_ownership: active, + new_verified_process_generation_after_recovery: activeIdentity, + real_permission_cleanup_failure: refusal, + retry_cleanup: retried, + source_bytes_unchanged: true, + }; + }, [() => restoreProject(project, mode), () => client.close()]); + rows.push(row); + await writeFile( + lab + "/progress.json", + JSON.stringify(rows, null, 2) + "\n", + { mode: 0o600 }, + ); + process.stdout.write(JSON.stringify({ profile, status: "passed" }) + "\n"); +} +assert.deepEqual( + await readFile(new URL(import.meta.url)), + verifierSource, + "Verifier source changed during execution", +); +await writeFile( + lab + "/summary.json", + JSON.stringify( + { + caller_supplied_runtime_revision: values.revision, + verifier_source_sha256: sha(verifierSource), + provider_settings_sha256: sha(providerBytes), + source_path: target, + source_sha256: before, + rows, + scope: + "Actual public MCP on Linux: token/start-identity verified pending Ghidra startup cancellation, fresh work under the selected target, controlled permission failure in an owned temporary project, retained-resource close retry and read-only token sweeps. Original input is unchanged. Runtime revision is caller-supplied; this is lifecycle verification, not paired performance or general OS coverage.", + }, + null, + 2, + ) + "\n", + { flag: "wx", mode: 0o600 }, +); diff --git a/experiments/analysis-efficiency/verify-preset-snapshot.mjs b/experiments/analysis-efficiency/verify-preset-snapshot.mjs new file mode 100644 index 000000000..ff8acca4b --- /dev/null +++ b/experiments/analysis-efficiency/verify-preset-snapshot.mjs @@ -0,0 +1,387 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { isAbsolute, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { parseArgs, promisify, isDeepStrictEqual } from "node:util"; +import { pathToFileURL } from "node:url"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; + +const { values } = parseArgs({ + options: { + "runtime-root": { type: "string" }, + revision: { type: "string" }, + path: { type: "string" }, + providers: { type: "string" }, + strace: { type: "string" }, + output: { type: "string" }, + procedure: { type: "string" }, + "timeout-ms": { type: "string", default: "900000" }, + "disabled-analyzers": { + type: "string", + default: "Decompiler Switch Analysis", + }, + }, +}); +for (const key of ["runtime-root", "path", "providers", "strace", "output"]) + assert.ok(values[key] && isAbsolute(values[key]), `Supply --${key} ABS`); +assert.match(values.revision ?? "", /^[a-f0-9]{40}$/u); +assert.equal(process.platform, "linux", "This verifier requires Linux strace"); +const timeout = Number(values["timeout-ms"]); +assert.ok(Number.isSafeInteger(timeout) && timeout > 0); +assert.ok(values["disabled-analyzers"].trim().length > 0); +const verifierSource = await readFile(new URL(import.meta.url)); +await mkdir(values.output, { mode: 0o700 }); +const fileSha = async (path) => { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +}; +const before = await fileSha(values.path); +const providerBytes = await readFile(values.providers); +const providers = JSON.parse(providerBytes.toString("utf8")); +assert.ok( + providers !== null && + !Array.isArray(providers) && + typeof providers === "object", +); +assert.ok(Object.values(providers).every((value) => typeof value === "string")); +const entrypoint = resolve(values["runtime-root"], "scripts/rea.mjs"); +const { parseAnalysisSnapshot } = await import( + pathToFileURL( + resolve(values["runtime-root"], "dist/domain/analysisSnapshot.js"), + ) +); +const { parseEvidence } = await import( + pathToFileURL(resolve(values["runtime-root"], "dist/domain/evidence.js")) +); +const { createVerifierRun, completeVerifierRun } = await import( + pathToFileURL(resolve(values["runtime-root"], "scripts/lib/verifier-run.mjs")) +); +const run = createVerifierRun(); +const snapshotPath = resolve(values.output, "default.snapshot.json"); +const changedSnapshotPath = resolve(values.output, "changed.snapshot.json"); +const tracedArgs = (work, args) => [ + "-ff", + "-s", + "4096", + "-e", + "trace=execve,execveat", + "-o", + resolve(work, "exec.trace"), + process.execPath, + entrypoint, + ...args, +]; +const headlessLaunches = async (work) => { + const files = (await readdir(work)).filter((name) => + name.startsWith("exec.trace."), + ); + assert.ok(files.length > 0, "No actual syscall trace was recorded"); + const lines = ( + await Promise.all( + files.map((name) => readFile(resolve(work, name), "utf8")), + ) + ).flatMap((text) => text.split("\n")); + // This checks the complete known Ghidra launch argument, not arbitrary target execution. + return lines.filter( + (line) => + line.includes("ghidra.app.util.headless.AnalyzeHeadless") && + line.endsWith("= 0"), + ).length; +}; +const environmentFor = (work, disabled) => ({ + ...process.env, + ...providers, + NODE_OPTIONS: "--max-old-space-size=512", + TMPDIR: work, + REA_PROCESS_RUN_ID: run.run_id, + REA_GHIDRA_DISABLED_ANALYZERS: disabled, +}); +const exactTaskFacts = (actual, expected) => { + assert.deepEqual(actual.normalized_result, expected.normalized_result); + assert.deepEqual(actual.subject, expected.subject); + assert.deepEqual(actual.analysis_profile, expected.analysis_profile); + assert.deepEqual(actual.parameters, expected.parameters); +}; +let selected = values.procedure, + original, + profile, + snapshotBytes; +const rows = []; +for (const scenario of [ + "default", + "same-profile", + "changed-profile", + "restored-default", +]) { + const work = resolve(values.output, scenario); + await mkdir(work, { mode: 0o700 }); + const environment = environmentFor( + work, + scenario === "changed-profile" ? values["disabled-analyzers"] : "", + ); + const client = new Client({ + name: "rea-preset-snapshot-verifier", + version: "1", + }); + const transport = new StdioClientTransport({ + command: values.strace, + args: tracedArgs(work, ["mcp"]), + env: environment, + stderr: "pipe", + }); + const definitions = new Map(); + const calls = []; + const call = async (name, args = {}, expectedError = false) => { + const definition = definitions.get(name); + assert.ok(definition, `Missing named contract ${name}`); + const start = performance.now(); + const reply = await client.callTool( + { name, arguments: args }, + { toolDefinition: definition, timeout }, + ); + await writeFile( + resolve(work, `call-${calls.length + 1}.json`), + JSON.stringify(reply), + { flag: "wx", mode: 0o600 }, + ); + const value = + reply.structuredContent ?? + JSON.parse(reply.content.find((item) => item.type === "text").text); + calls.push({ + name, + arguments: args, + elapsed_ms: performance.now() - start, + is_error: reply.isError === true, + evidence_id: value.evidence_id ?? null, + }); + assert.equal(reply.isError === true, expectedError, JSON.stringify(value)); + return value; + }; + try { + await client.connect(transport); + transport.stderr?.resume(); + for (const definition of (await client.listTools()).tools) + definitions.set(definition.name, definition); + if (scenario === "changed-profile") { + const rejected = await call( + "open_binary", + { + path: values.path, + provider_id: "ghidra", + snapshot_path: snapshotPath, + }, + true, + ); + assert.equal(rejected.error.code, "evidence_integrity_mismatch"); + assert.match(JSON.stringify(rejected.error), /profile_mismatch/u); + assert.equal((await call("binary_session")).result.open, false); + assert.equal(await headlessLaunches(work), 0); + await client.ping(); + await call("open_binary", { path: values.path, provider_id: "ghidra" }); + const changed = (await call("binary_session")).result + .analysis_provider_binding.analysis_profile; + assert.notEqual(changed.digest, profile.digest); + assert.notEqual( + changed.parameters.analyzer_preset, + profile.parameters.analyzer_preset, + ); + const fact = parseEvidence( + await call("analyze_function", { procedure: selected }), + ); + assert.deepEqual(fact.analysis_profile, changed); + await call("close_binary", { snapshot_path: changedSnapshotPath }); + rows.push({ + scenario, + wrong_snapshot_rejected_before_headless_launch: true, + selected_profile: changed, + source_snapshot_unchanged: true, + selected_function_facts_match_default: isDeepStrictEqual( + fact.normalized_result, + original.normalized_result, + ), + limitations: fact.limitations, + calls, + }); + } else { + if (scenario === "restored-default") { + const rejected = await call( + "open_binary", + { + path: values.path, + provider_id: "ghidra", + snapshot_path: changedSnapshotPath, + }, + true, + ); + assert.equal(rejected.error.code, "evidence_integrity_mismatch"); + assert.match(JSON.stringify(rejected.error), /profile_mismatch/u); + assert.equal((await call("binary_session")).result.open, false); + assert.equal(await headlessLaunches(work), 0); + } + await call("open_binary", { + path: values.path, + provider_id: "ghidra", + ...(scenario === "default" ? {} : { snapshot_path: snapshotPath }), + }); + if (selected === undefined) { + const procedures = (await call("list_procedures")).normalized_result; + const observed = procedures.find( + (row) => + row.procedure?.external === false && row.procedure.thunk === false, + ); + assert.ok( + observed, + "No provider-observed internal procedure; select --procedure explicitly", + ); + selected = observed.address; + } + const fact = parseEvidence( + await call("analyze_function", { procedure: selected }), + ); + const actualProfile = (await call("binary_session")).result + .analysis_provider_binding.analysis_profile; + assert.equal(actualProfile.parameters.analyzer_preset, "ghidra-default"); + if (scenario === "default") { + original = fact; + profile = actualProfile; + await call("close_binary", { snapshot_path: snapshotPath }); + snapshotBytes = await readFile(snapshotPath); + const snapshot = parseAnalysisSnapshot( + JSON.parse(snapshotBytes.toString("utf8")), + ); + assert.ok( + snapshot.entries.some( + (entry) => entry.operation === "analyze_function", + ), + "The actual query must be stored, not merely equal after re-analysis", + ); + assert.ok( + snapshot.evidence_bundle.records.some( + (record) => record.evidence_id === original.evidence_id, + ), + ); + } else { + assert.deepEqual(actualProfile, profile); + exactTaskFacts(fact, original); + assert.ok( + fact.limitations.some((limitation) => + limitation.includes("local REA analysis snapshot"), + ), + "No producer cache-reuse marker", + ); + await call("close_binary"); + } + rows.push({ + scenario, + selected_profile: actualProfile, + original_evidence_id: original.evidence_id, + returned_evidence_id: fact.evidence_id, + limitations: fact.limitations, + calls, + }); + } + assert.equal((await call("binary_session")).result.open, false); + } finally { + await client.close(); + } + const launches = await headlessLaunches(work); + assert.equal( + launches, + scenario === "default" || scenario === "changed-profile" ? 1 : 0, + ); + rows.at(-1).headless_launches = launches; + assert.equal(await fileSha(values.path), before); + assert.deepEqual(await readFile(snapshotPath), snapshotBytes); + await writeFile( + resolve(values.output, "progress.json"), + JSON.stringify(rows, null, 2), + { mode: 0o600 }, + ); + process.stdout.write( + JSON.stringify({ + scenario, + status: "passed", + headless_launches: launches, + }) + "\n", + ); +} +const cliWork = resolve(values.output, "cli"); +await mkdir(cliWork, { mode: 0o700 }); +const cliStart = performance.now(); +const { stdout, stderr } = await promisify(execFile)( + values.strace, + tracedArgs(cliWork, [ + "function", + values.path, + selected, + "--provider", + "ghidra", + "--snapshot", + snapshotPath, + "--json", + ]), + { + env: environmentFor(cliWork, ""), + timeout, + maxBuffer: 16 * 1024 * 1024, + }, +); +await writeFile(resolve(cliWork, "reply.json"), stdout, { + flag: "wx", + mode: 0o600, +}); +await writeFile(resolve(cliWork, "stderr.log"), stderr, { + flag: "wx", + mode: 0o600, +}); +const cli = parseEvidence(JSON.parse(stdout)); +exactTaskFacts(cli, original); +assert.equal(cli.evidence_id, original.evidence_id); +assert.deepEqual(await readFile(snapshotPath), snapshotBytes); +assert.equal(await headlessLaunches(cliWork), 0); +assert.equal(await fileSha(values.path), before); +const lineage = await completeVerifierRun(run); +assert.equal(lineage.process_lineage.status, "verified"); +assert.deepEqual(lineage.process_lineage.descendants, []); +const report = { + caller_supplied_runtime_revision: values.revision, + verifier_source_sha256: createHash("sha256") + .update(verifierSource) + .digest("hex"), + source_sha256: before, + source_path: values.path, + provider_settings_sha256: createHash("sha256") + .update(providerBytes) + .digest("hex"), + original_evidence_id: original.evidence_id, + source_snapshot_sha256: createHash("sha256") + .update(snapshotBytes) + .digest("hex"), + selected_procedure: selected, + original_record_and_source_bytes_unchanged: true, + rows, + cli: { + elapsed_ms: performance.now() - cliStart, + returned_evidence_id: cli.evidence_id, + exact_original_task_facts: true, + headless_launches: 0, + }, + lineage, + scope: + "Actual Ghidra public SDK and CLI, explicit default/disabled profiles and immutable JSON snapshot reuse. Revision is caller-supplied. Selection/discovery and all verification calls are included; timings are not paired performance evidence. No engine database cache, target execution or annotation is requested. Final descendant observation is point-in-time; syscall audit covers the known headless launch command.", +}; +assert.deepEqual( + await readFile(new URL(import.meta.url)), + verifierSource, + "Verifier source changed during execution", +); +await writeFile( + resolve(values.output, "summary.json"), + JSON.stringify(report, null, 2), + { flag: "wx", mode: 0o600 }, +);