shifathosn
/
rea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: reconcile final main for PR #2791

This commit is contained in:
morluto 2026-10-13 20:54:03 +08:00
parent 324afce053 f63674cac0
commit ce6bce4389
42 changed files with 2266 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/ci.yml vendored
View File

@ -597,6 +597,7 @@ jobs:
tests/boundary/filesystem/setupUnavailableProvider.test.ts
tests/boundary/filesystem/windowsNpxRegistration.test.ts
tests/acceptance/setup/homeDirectoryFallback.test.ts
tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts
tests/boundary/filesystem/webScriptManifestPaths.test.ts
tests/boundary/process/ProcessEvidence.host.test.ts
tests/acceptance/webModuleCli.test.ts

380
bridge/ghidra/ReaGhidraBridge.java
View File

@ -13,6 +13,8 @@ import java.net.UnixDomainSocketAddress;
import java.nio.channels.Channels;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
@ -22,7 +24,9 @@ import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.MessageDigest;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashSet;
import java.util.IdentityHashMap;
@ -82,6 +86,7 @@ import ghidra.program.model.address.SegmentedAddress;
import ghidra.program.model.reloc.Relocation;
import java.io.ByteArrayOutputStream;
import ghidra.program.model.symbol.Namespace;
import ghidra.program.model.symbol.FlowType;
import ghidra.program.model.symbol.RefType;
import ghidra.program.model.symbol.Reference;
import ghidra.program.model.symbol.ReferenceIterator;
@ -100,6 +105,9 @@ import ghidra.program.model.pcode.PcodeBlock;
import ghidra.program.model.pcode.PcodeBlockBasic;
import ghidra.program.model.pcode.Varnode;
import ghidra.program.model.pcode.VarnodeAST;
import ghidra.program.util.ContextEvaluatorAdapter;
import ghidra.program.util.SymbolicPropogator;
import ghidra.program.util.VarnodeContext;
public final class ReaGhidraBridge extends HeadlessScript {
private static final int MAX_HIGH_PCODE_OPS = 3000;
@ -1319,6 +1327,13 @@ public final class ReaGhidraBridge extends HeadlessScript {
result.add("outgoing_references", outgoing);
result.add("referenced_strings", referencedStrings);
result.add("referenced_names", referencedNames);
boolean swift = swiftProgram();
boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:");
SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null;
if (swiftLiterals != null) {
result.add("swift_string_literals", swiftLiterals.literals());
result.add("swift_string_literal_omissions", swiftLiterals.omissions());
}
result.add("basic_blocks", blocks);
result.add(
"native_api",
@ -1338,6 +1353,20 @@ public final class ReaGhidraBridge extends HeadlessScript {
limitations.add(
"Synthetic Ghidra entry-point references without actionable memory sources are omitted."
);
if (swift && aarch64) {
limitations.add(
"Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by conditional selects, passed through unresolved jumps, or built at runtime are not reported."
);
if (swiftLiterals.omissions().size() > 0) {
limitations.add(
"Swift String literal records listed in swift_string_literal_omissions could not be retained or decoded within this dossier's remaining heap budget; later symbolic flows remain unexamined."
);
}
} else if (swift) {
limitations.add(
"Swift String literal decoding is implemented for AArch64 register pairs only; no Swift literals were decoded for this architecture."
);
}
List<String> boundaryLimitations = noReturnCallLimitations(scan.instructions);
for (String limitation : boundaryLimitations) {
limitations.add(limitation);
@ -3391,6 +3420,349 @@ public final class ReaGhidraBridge extends HeadlessScript {
return result;
}
private boolean swiftProgram() {
for (MemoryBlock block : currentProgram.getMemory().getBlocks()) {
String name = block.getName();
if (name.startsWith("__swift5_") || name.startsWith("swift5_")) return true;
}
return false;
}
/**
* Swift passes a String as two words: small literals carry their bytes in the words and
* immortal large literals carry their count and a biased pointer to the bytes. Ghidra keeps
* neither as a data reference, so decode constant pairs where Swift consumes them: argument
* and return registers at calls, tail calls and returns, and both registers of one STP.
* The canonical empty string is a small literal with a zero count.
*/
private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception {
Register[] registers = new Register[31];
for (int index = 0; index < registers.length; index++) {
registers[index] = currentProgram.getRegister("x" + index);
}
TreeMap<String, JsonObject> observed = new TreeMap<>();
TreeMap<String, JsonObject> omitted = new TreeMap<>();
SwiftLiteralBudget budget = new SwiftLiteralBudget();
Map<String, Boolean> selectedValues = new HashMap<>();
SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false);
propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() {
@Override
public boolean evaluateContextBefore(VarnodeContext context, Instruction instruction) {
// Returning true ends only the current flow, so every queued flow checks the shared stop.
if (budget.stopped()) return true;
for (Register[] pair : swiftStringPairs(function, instruction, registers)) {
Long word = registerConstant(context, pair[0]);
Long object = registerConstant(context, pair[1]);
if (word == null || object == null) continue;
SwiftLiteralLayout layout = swiftLiteralLayout(word, object);
if (layout == null) continue;
// A selected word can combine with a path-dependent ordinary move. Cache the
// provenance question per site/register, charging retained cache state to the
// same dossier budget before allocating it.
boolean selected = false;
boolean provenanceOverCapacity = false;
for (Register register : pair) {
String provenanceKey = canonicalAddress(instruction.getAddress()) + "\u0000" +
register.getBaseRegister().getName();
Boolean value = selectedValues.get(provenanceKey);
if (value == null) {
if (!budget.takeState()) {
provenanceOverCapacity = true;
break;
}
value = selectedValue(function, instruction, register);
selectedValues.put(provenanceKey, value);
}
if (value) {
selected = true;
break;
}
}
if (selected) continue;
// Forks can reach one site with different constants, so the words are part of the key.
String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() +
"\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) +
"\u0000" + Long.toUnsignedString(object, 16);
if (observed.containsKey(key) || omitted.containsKey(key)) continue;
// Every record draws on the budget; the first one that does not fit is kept as an
// omission and decoding stops, so later sites are never examined.
String value = layout.smallValue();
boolean overCapacity = provenanceOverCapacity || !budget.take(layout.byteLength());
if (!overCapacity && value == null) {
try {
value = swiftLargeLiteralText(layout);
} catch (OutOfMemoryError exhausted) {
overCapacity = true;
}
}
if (value == null && !overCapacity) continue;
JsonObject item = overCapacity
? swiftStringOmission(layout, word, object)
: swiftStringItem(value, layout, word, object);
item.addProperty("source_address", canonicalAddress(instruction.getAddress()));
JsonArray pairRegisters = new JsonArray();
pairRegisters.add(pair[0].getName());
pairRegisters.add(pair[1].getName());
item.add("registers", pairRegisters);
(overCapacity ? omitted : observed).put(key, item);
if (overCapacity) {
budget.stop();
return true;
}
}
return false;
}
}, false, monitor);
JsonArray literals = new JsonArray();
observed.values().forEach(literals::add);
JsonArray omissions = new JsonArray();
omitted.values().forEach(omissions::add);
return new SwiftStringLiteralDecode(literals, omissions);
}
private static List<Register[]> swiftStringPairs(
Function function,
Instruction instruction,
Register[] registers
) {
List<Register[]> pairs = new ArrayList<>();
if (instruction.getMnemonicString().equals("stp")) {
Register first = instruction.getRegister(0);
Register second = instruction.getRegister(1);
if (first != null && second != null && first.getBitLength() == 64 && second.getBitLength() == 64) {
pairs.add(new Register[] {first, second});
}
return pairs;
}
FlowType flow = instruction.getFlowType();
boolean returns = false;
for (PcodeOp operation : instruction.getPcode()) {
if (operation.getOpcode() == PcodeOp.RETURN) returns = true;
}
// A jump is a proven tail call only when it has resolved targets and all leave the
// body. A targetless computed jump can still dispatch inside the function.
Address[] targets = instruction.getFlows();
boolean tailCall = flow.isJump() && !flow.isConditional() && targets.length > 0 &&
Arrays.stream(targets).allMatch(target -> !function.getBody().contains(target));
if (!flow.isCall() && !returns && !tailCall) return pairs;
// A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned.
int lastFirstRegister = returns ? 0 : 6;
for (int index = 0; index <= lastFirstRegister; index++) {
pairs.add(new Register[] {registers[index], registers[index + 1]});
}
return pairs;
}
private static Long registerConstant(VarnodeContext context, Register register) {
Varnode value = context.getRegisterVarnodeValue(register);
return value != null && context.isConstant(value) ? value.getOffset() : null;
}
// Search every predecessor path for the reaching definitions, following register copies,
// since a select copied into an argument register or defined before a branch forks just as
// independently as one written there directly.
private boolean selectedValue(Function function, Instruction use, Register register) {
ArrayDeque<Instruction> instructions = new ArrayDeque<>();
ArrayDeque<Register> targets = new ArrayDeque<>();
Set<String> visited = new HashSet<>();
for (Instruction predecessor : predecessors(function, use)) {
instructions.push(predecessor);
targets.push(register.getBaseRegister());
}
while (!instructions.isEmpty()) {
Instruction instruction = instructions.pop();
Register target = targets.pop();
if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue;
boolean defines = false;
for (Object result : instruction.getResultObjects()) {
if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true;
}
Register next = target;
if (defines) {
PcodeOp[] operations = instruction.getPcode();
for (PcodeOp operation : operations) {
if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true;
}
Register source = copiedRegister(operations);
if (source == null) continue;
next = source.getBaseRegister();
}
for (Instruction predecessor : predecessors(function, instruction)) {
instructions.push(predecessor);
targets.push(next);
}
}
return false;
}
private List<Instruction> predecessors(Function function, Instruction instruction) {
List<Instruction> result = new ArrayList<>();
Address from = instruction.getFallFrom();
if (from != null) {
Instruction previous = currentProgram.getListing().getInstructionAt(from);
if (previous != null) result.add(previous);
}
for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) {
if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue;
if (!function.getBody().contains(reference.getFromAddress())) continue;
Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress());
if (source != null) result.add(source);
}
return result;
}
private Register copiedRegister(PcodeOp[] operations) {
if (operations.length != 1) return null;
int opcode = operations[0].getOpcode();
Varnode input = operations[0].getInput(0);
if ((opcode != PcodeOp.COPY && opcode != PcodeOp.INT_ZEXT) || input == null || !input.isRegister()) return null;
return currentProgram.getRegister(input);
}
/** Validate the ABI layout of a pair; a small literal is decoded here, a large one only located. */
private SwiftLiteralLayout swiftLiteralLayout(long word, long object) {
int discriminator = (int) (object >>> 60);
if (discriminator == 0xE || discriminator == 0xA) {
int count = (int) ((object >>> 56) & 0xF);
byte[] bytes = new byte[15];
for (int index = 0; index < 8; index++) bytes[index] = (byte) (word >>> (8 * index));
for (int index = 0; index < 7; index++) bytes[8 + index] = (byte) (object >>> (8 * index));
for (int index = count; index < bytes.length; index++) {
if (bytes[index] != 0) return null;
}
// Count zero is the canonical empty string (object 0xE000000000000000, word 0).
String value = swiftStringText(bytes, count, discriminator == 0xE);
return value == null ? null : new SwiftLiteralLayout("small", null, count, true, value);
}
int flags = (int) (word >>> 60);
long count = word & 0xFFFFFFFFFFFFL;
// Immortal literals: ASCII and NFC or neither, tail allocated, with a pointer biased by 32.
if ((object >>> 56) != 0x80 || (flags != 0xD && flags != 0x1) || ((word >>> 48) & 0xFFF) != 0) {
return null;
}
try {
Address address = currentProgram.getAddressFactory().getDefaultAddressSpace()
.getAddress((object & 0x00FFFFFFFFFFFFFFL) + 0x20);
MemoryBlock block = currentProgram.getMemory().getBlock(address);
// The bytes and their terminator must lie in one initialized, non-executable block,
// and Java arrays bound one read.
if (block == null || !block.isInitialized() || block.isExecute() ||
count >= block.getEnd().subtract(address) + 1 || count >= Integer.MAX_VALUE) {
return null;
}
return new SwiftLiteralLayout("large", address, count, flags == 0xD, null);
} catch (Exception unaddressable) {
return null;
}
}
private String swiftLargeLiteralText(SwiftLiteralLayout layout) {
int count = (int) layout.byteLength();
byte[] bytes = new byte[count + 1];
try {
if (currentProgram.getMemory().getBytes(layout.address(), bytes) != bytes.length) return null;
} catch (Exception unreadable) {
return null;
}
return bytes[count] == 0 ? swiftStringText(bytes, count, layout.ascii()) : null;
}
/**
* Every literal record of one dossier draws on half of the heap that was free when decoding
* began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up
* to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer
* (up to twice its length), again in the returned String, and once more as UTF-8 on the
* socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words
* and their serialized form, so many empty or small literals are bounded too. Retained
* provenance-cache entries draw on that same fixed-state allowance.
*/
private static final class SwiftLiteralBudget {
private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6;
private static final long BYTES_PER_RECORD = 4096;
private long remaining;
private boolean stopped;
SwiftLiteralBudget() {
Runtime runtime = Runtime.getRuntime();
remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2);
}
boolean takeState() {
if (remaining < BYTES_PER_RECORD) return false;
remaining -= BYTES_PER_RECORD;
return true;
}
boolean take(long bytes) {
if (remaining < BYTES_PER_RECORD ||
bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) {
return false;
}
remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE;
return true;
}
void stop() {
stopped = true;
}
boolean stopped() {
return stopped;
}
}
// Swift String values may contain any scalar, including controls; JSON escapes them.
private static String swiftStringText(byte[] bytes, int length, boolean ascii) {
if (length < 0 || length > bytes.length) return null;
for (int index = 0; index < length; index++) {
if (ascii && bytes[index] < 0) return null;
}
try {
return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes, 0, length)).toString();
} catch (CharacterCodingException invalid) {
return null;
}
}
private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) {
JsonObject item = new JsonObject();
item.addProperty("representation", layout.representation());
if (layout.address() == null) {
item.add("address", JsonNull.INSTANCE);
} else {
item.addProperty("address", canonicalAddress(layout.address()));
}
item.addProperty("byte_length", layout.byteLength());
item.addProperty("reason", "capacity");
item.add("words", swiftStringWords(word, object));
return item;
}
private static JsonObject swiftStringItem(
String value,
SwiftLiteralLayout layout,
long word,
long object
) {
JsonObject item = new JsonObject();
item.addProperty("value", value);
item.addProperty("representation", layout.representation());
if (layout.address() == null) {
item.add("address", JsonNull.INSTANCE);
} else {
item.addProperty("address", canonicalAddress(layout.address()));
}
item.add("words", swiftStringWords(word, object));
return item;
}
private static JsonArray swiftStringWords(long word, long object) {
JsonArray words = new JsonArray();
words.add("0x" + Long.toUnsignedString(word, 16));
words.add("0x" + Long.toUnsignedString(object, 16));
return words;
}
private static String exclusiveAddress(Address inclusive) {
Address next = inclusive.next();
if (next != null && next.getAddressSpace().equals(inclusive.getAddressSpace())) {
@ -3930,6 +4302,14 @@ public final class ReaGhidraBridge extends HeadlessScript {
private record AddressNameRequest(String inputAddress, Address address, String name) {}
private record FunctionEntry(Function function, InventoryItem item) {}
private record FunctionAnalysisObservation(JsonObject dossier, List<String> limitations) {}
private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {}
private record SwiftLiteralLayout(
String representation,
Address address,
long byteLength,
boolean ascii,
String smallValue
) {}
private record InstructionScan(List<Instruction> instructions, boolean truncated) {}
private record SessionDescriptor(
String transport,

23
docs/guides/web/browser-observation.md
View File

@ -80,6 +80,27 @@ google-chrome \
REA does not launch, own, or terminate this browser. Use a dedicated profile and stop it yourself when the investigation is complete. Do not expose the debugging port on a non-loopback interface.
Other Chromium-family browsers, such as Chromium or Brave, work the same way
with the same two flags. On macOS, run the executable inside the application
bundle, for example
`"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"` or
`"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"`. Fedora installs
Chromium as `chromium-browser`.
REA needs the browser's HTTP discovery endpoint on a loopback port. A browser
driven over `--remote-debugging-pipe` or a private channel, for example by an
automation SDK, exposes no such endpoint, so `list_browser_targets` has nothing
to attach to. Connection failures report `endpoint_unreachable`; a reachable
HTTP service that returns a non-200 or non-JSON discovery response reports
`invalid_endpoint_response`.
Reproduce that flow in a Chromium-family browser started with
`--remote-debugging-port`.
In a minimal Linux container, headed Chromium under Xvfb can keep running while
its DevTools port never opens. Install the GLES, EGL, and Mesa DRI libraries
(on Fedora: `libglvnd-gles`, `mesa-libEGL`, and `mesa-dri-drivers`), then
restart the browser.
On Linux, a reachable but locked desktop keyring can stall Chrome before its
first navigation. For a dedicated investigation profile, add
`--password-store=basic` to bypass that keyring. Empty or placeholder frame URLs
@ -207,7 +228,7 @@ commands fail closed. A correlated reply's session ID must match the command's
selected session. An optional method is unavailable only when the browser
rejects it as unknown; other command rejections retain their reported failure.
Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete.
Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Scripts are different: scripts the page parsed before REA attached are still reported under `scripts`. A page that loaded before attach can therefore list its scripts while `network.requests` lacks the requests that fetched them. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete.
Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`.

128
experiments/analysis-efficiency/windows-posix-reconciliation.md Normal file
View File

@ -0,0 +1,128 @@
# Windows reconciliation of recorded POSIX paths
Follow-up to [#2422](https://github.com/morluto/rea/pull/2422), including
@Aurantium-47's diagnosis of the host-dependent relative-path operation.
## Source pins and real data
The initial reproduction and passive producer used main
fa4b8288655188426222773ead2ccbdaf76334fd. Before branching, main was updated
to 8b181716aca2cb467be499cba226abfb5ae19141. The owning matcher and its
affected reconciliation consumers were unchanged between those revisions.
The newer baseline again reproduced both errors on the real Windows host.
Input: npm 11.16.0's published lib/utils/validate-lockfile.js, 1,023 bytes,
SHA-256 2295e2dac1c5816e1eae2d417804960ebbef17b64a0a44db4f3483ce6e6d9526.
The installed copy was independently checked against the exact module in the
[official registry package](https://registry.npmjs.org/npm/11.16.0).
The 2,945,850-byte tarball matched its published SHA-512 SRI before reading
that member; no archive extraction or installation occurred, and the tarball
was not retained.
Two actual Linux filesystem directories, Runtime/App and Runtime/app,
held byte-identical module copies and had distinct directory identities.
A controlled Node 24.18.0 process loaded the public module and ran its real
function against 587 locked dependency records: compatible trees returned zero
errors, and one controlled version mismatch returned one error. A passive REA
Inspector capture recorded 114 loaded scripts, including the selected module.
The provider did not evaluate, pause, resume or read source. The owned process
exited normally. Function execution is established by the harness receipt;
REA's passive script metadata establishes loaded status.
Production static analysis inspected both directories plus an independent
Static/public-module copy. The latter prevents implicit root matching from
hiding the explicit mapping result. Complete static/runtime Evidence was then
consumed by the actual **Windows** CLI and SDK 2.3.1 stdio MCP server, using
Windows 11 10.0.22631.0 and official Windows x64 Node 24.18.0.
No Electron GUI, IDA or Ghidra engine was used.
## Correctness and provenance
| Windows reconciliation | Baseline | Candidate |
| -------------------------------------- | ------------------------------ | ----------------------------------- |
| Exact POSIX input root | Matched, artifact path | Same complete normalized result |
| Case-mismatched POSIX input root | Incorrectly matched and loaded | Unknown; static load states unknown |
| Exact explicit POSIX mapping | Matched, operator file mapping | Same complete normalized result |
| Case-mismatched explicit POSIX mapping | Incorrectly matched and loaded | Unknown; static load states unknown |
All four CLI and same-connection MCP normalized results agree. The input
Evidence, recorded runtime facts and caller-selected root spellings are
preserved. Rejected location membership does not establish that a module was
absent or unexecuted. The operation remains offline and performs no target or
filesystem inspection.
The shared matcher now chooses posix.relative after identifying POSIX input;
Windows coordinates continue to use win32.relative. Both location matching
and active-layer scope use this owner. The obsolete host-default import is
removed; no downstream repair or new path abstraction is introduced.
The related suite passed 102 tests: 15 workflow tests, 85 CI scope tests and
two curated-path checks. Four simulated Electron Evidence controls cover
POSIX mismatch/exact matching, Windows case-insensitive matching, the resulting
asset load state and retained mapping spelling. They are service regressions,
not real Electron-provider verification. The existing workflow test file is
registered in both the Windows execution lane and its package-change scope.
## Delivery and resource observations
Probes ran serially at BelowNormal priority with one CPU and 512 MiB Node heap
settings for CLI/server/client work. Heap settings do not bound aggregate memory.
Four corrected CLI exit-time observations reported 7.343–15.827 seconds CPU and
322,308–324,280 KiB peak working set, excluding the probe parent. Wall times
ranged from 7.785 to 17.601 seconds. There is no paired resource baseline.
One complete MCP trial connected in 18,296 ms and discovered 189 tools in
875 ms (2,588,336 serialized bytes). The four subsequent same-connection calls
took 879, 320, 353 and 422 ms. Their serialized replies were 918,133 / 913,765 /
918,571 / 914,171 bytes; inputs totalled 681,312 bytes. Discovery plus these
inputs and replies totalled 6,934,288 bytes, excluding initialization, framing
and notifications. This is delivery volume, not model token usage. Actual
model context and total model task cost were not measured.
A process-generation sampler recorded 233 observations over 25,671 ms:
15,797 ms aggregate observed CPU as a lower bound and a peak sampled working-set
sum of 491,597,824 bytes (about 469 MiB). Exited-process tails can be missed;
shared pages can be counted twice. This excludes the observer and is not an
exact physical-memory peak. The server wrote four structured completion records
to stderr; tool replies and normalized results passed the recorded checks.
Cold/warm caches and observer differences were not normalized. No latency,
CPU, memory or model-token improvement is claimed. The fix adds no lookup I/O,
engine invocation, retry or cache. Complete default replies remain available;
only compact selected findings and hashes were sent to the investigating agent.
## Main integration replay
Runtime cb0d8a9fef2b907cff2ba27f1e017493e1173eb2, rebased onto main
73dd8d6ed, again passed the 101 related tests and all four real Windows CLI/MCP
cases using the unchanged older producer Evidence. Positive results and original
runtime/layer facts remain identical. Typecheck, lint, dependency and metadata
checks passed; all five changed files passed formatting independently.
The full format gate at that main revision failed on the unchanged upstream
GhidraDoctor.test.ts and GhidraSnapshotCache.test.ts files. The PR does not
repair those unrelated baseline files. Resource figures above refer to the
earlier candidate; the integration replay does not establish a speedup.
Review found that owning only the Windows test path would still skip that lane
on a later implementation-only edit. The matcher implementation is now also
registered in package scope. A source-only classifier case verifies that it
selects both Windows package and JavaScript recovery lanes; 85 scope checks and
two curated-path checks pass. Runtime code and the real-data checks are unchanged.
The original production fix merged as
[1a1b67c88](https://github.com/morluto/rea/commit/1a1b67c88) while this follow-up
was open. The follow-up was then rebased onto main fb28ec153 and contains only
the stronger workflow assertions, Windows lane ownership, scope classification
and this verification record. The updated 131 related tests, two representative
Ghidra boundary tests, seven IDA input-thinning tests, 24 semantic-structure
tests, 16 required-lane checks, 39 release-checkpoint tests and 11 dyld CLI
tests passed after the rebase (230 total).
Upstream a0317cf73 corrected the invalid FAT fixture and consolidated unrelated
duplicated helpers; e05d3399f aligned the older parameter-provenance assertion
with the upstream semantics. Those changes, the threshold change and the new
CI qualification tiers are part of main, outside this follow-up's diff.
Upstream 747e72991 restored release preflight coverage for reusable workflow
jobs. The independently reviewed, fully green #2755 fixture correction merged
as fb28ec153 and restored real x64 dyld CLI admission without altering the
original same-named arm64e assertions.
No production source is duplicated by the follow-up.

2
scripts/ci/scopes.mjs
View File

@ -56,6 +56,8 @@ export const SCOPES = {
"tests/boundary/filesystem/setupUnavailableProvider.test.ts",
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
"tests/acceptance/setup/homeDirectoryFallback.test.ts",
"tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts",
"src/domain/javascript/javascriptRuntimeStaticCandidates.ts",
"tests/boundary/filesystem/webScriptManifestPaths.test.ts",
"tests/boundary/process/ProcessEvidence.host.test.ts",
"tests/acceptance/webModuleCli.test.ts",

68
scripts/verify-real-ghidra-swift.mjs
View File

@ -45,8 +45,13 @@ const transport = new StdioClientTransport({
const source = fileURLToPath(
new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url),
);
const literalSource = fileURLToPath(
new URL("../tests/conformance/swift/string-literals.swift", import.meta.url),
);
const binary = join(root, "swift-inventory");
const literalBinary = join(root, "swift-literals");
const reports = [];
let literalReport;
try {
await execute("swiftc", ["-O", "-g", source, "-o", binary], {
timeout: 60_000,
@ -54,6 +59,19 @@ try {
await mkdir(join(root, "copied"));
const copied = join(root, "copied", "swift-inventory");
await copyFile(binary, copied);
// Decoding covers AArch64 register pairs, so build that slice on any Mac host.
await execute(
"swiftc",
[
"-O",
"-target",
"arm64-apple-macosx13.0",
literalSource,
"-o",
literalBinary,
],
{ timeout: 60_000 },
);
await client.connect(transport);
for (const [variant, path] of [
["debug", binary],
@ -132,6 +150,7 @@ try {
await call("close_binary", {});
}
}
literalReport = await verifyStringLiterals();
} finally {
await client.close();
await transport.close();
@ -143,6 +162,7 @@ console.log(
ok: true,
provider: "ghidra",
fixtures: reports,
swift_string_literals: literalReport,
verifier_run: await completeVerifierRun(run),
},
null,
@ -159,3 +179,51 @@ async function call(name, arguments_) {
toolContract(name).outputSchema.parse(response.structuredContent);
return requireMcpOperationResult(response, name);
}
// Literal text lives only in register constants, so it must come from the decoded pairs.
async function verifyStringLiterals() {
await call("open_binary", { path: literalBinary, provider_id: "ghidra" });
try {
const procedures = await call("list_procedures", {});
const expected = {
smallLiteral: [["SleepDisabled", "small"]],
largeLiteral: [["The power setting could not be changed.", "large"]],
unicodeLiterals: [
["café", "small"],
["Résumé of the naïve décor, written twice", "large"],
],
emptyAndControlLiterals: [
["", "small"],
["tab\there", "small"],
],
returnedLiteral: [["Holding awake", "small"]],
};
const decoded = {};
for (const [name, literals] of Object.entries(expected)) {
const procedure = procedures.find(({ value }) => value.includes(name));
assert(procedure, `${name} is missing from the procedure inventory`);
const dossier = await call("analyze_function", {
procedure: procedure.address,
});
const observed = dossier.swift_string_literals;
assert(Array.isArray(observed), `${name} recorded no Swift literals`);
assert.deepEqual(dossier.swift_string_literal_omissions, []);
assert.deepEqual(
observed.map(({ value, representation }) => [value, representation]),
literals,
);
for (const literal of observed) {
assert.deepEqual(literal.registers, ["x0", "x1"]);
assert.equal(
literal.address === null,
literal.representation === "small",
`${literal.value} storage does not match its representation`,
);
}
decoded[name] = observed.length;
}
return decoded;
} finally {
await call("close_binary", {});
}
}

504
scripts/verify/firmware/analysis.mjs
View File

@ -1,15 +1,8 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import {
access,
mkdir,
mkdtemp,
readFile,
rm,
symlink,
} from "node:fs/promises";
import * as fs from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import * as nodePath from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna
import { parseEvidence } from "../../../dist/domain/evidence.js";
import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js";
const exec = promisify(execFile);
const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
const entrypoint = resolve(
const execFileAsync = promisify(execFile);
const { join, resolve, dirname } = nodePath;
const CALL_TIMEOUT_MS = 150_000;
const CONNECT_TIMEOUT_MS = 30_000;
const SEVENZIP_TIMEOUT_MS = 10_000;
const MAX_STDOUT_BYTES = 16 * 1024 * 1024;
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
const ENTRYPOINT = resolve(
process.env.REA_FIRMWARE_TEST_ENTRYPOINT ??
join(repository, "scripts/rea.mjs"),
join(REPO_ROOT, "scripts/rea.mjs"),
);
if (process.platform !== "linux")
throw new Error(
"verify:firmware requires Linux; other hosts are not verified",
);
for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
const path = process.env[variable];
if (path === undefined)
throw new Error(
`verify:firmware requires ${variable}; see docs/guides/firmware/firmware-analysis.md`,
);
await access(path);
}
try {
await exec("7z", ["i"], { timeout: 10_000 });
} catch (cause) {
throw new Error(
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
{ cause },
);
}
const fixtureRoot = resolve(
const FIXTURE_ROOT = resolve(
process.env.REA_FIRMWARE_FIXTURE_ROOT ??
join(repository, "_reference/firmware-integration/generated"),
join(REPO_ROOT, "_reference/firmware-integration/generated"),
);
await access(join(fixtureRoot, "oracle.json")).catch((cause) => {
throw new Error(
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
{ cause },
);
});
// ---------------------------------------------------------------------------
// Prerequisite checks
// ---------------------------------------------------------------------------
async function checkPrerequisites() {
if (process.platform !== "linux") {
throw new Error(
"verify:firmware requires Linux; other hosts are not verified",
);
}
for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
const value = process.env[name];
if (value === undefined) {
throw new Error(
`verify:firmware requires ${name}; see docs/guides/firmware/firmware-analysis.md`,
);
}
await fs.access(value);
}
try {
await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS });
} catch (cause) {
throw new Error(
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
{ cause },
);
}
try {
await fs.access(join(FIXTURE_ROOT, "oracle.json"));
} catch (cause) {
throw new Error(
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
{ cause },
);
}
}
await checkPrerequisites();
// ---------------------------------------------------------------------------
// Shared state
// ---------------------------------------------------------------------------
const oracle = JSON.parse(
await readFile(join(fixtureRoot, "oracle.json"), "utf8"),
await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"),
);
const path = join(fixtureRoot, "firmware.bin");
assert.equal(await hashFirmwareFile(path), oracle.sha256);
const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-"));
const environment = Object.fromEntries(
Object.entries(process.env).filter(([, value]) => value !== undefined),
const firmwarePath = join(FIXTURE_ROOT, "firmware.bin");
assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256);
const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-"));
const baseEnv = Object.fromEntries(
Object.entries(process.env).filter(([, v]) => v !== undefined),
);
const client = new Client({ name: "rea-real-firmware-verifier", version: "1" });
const cli = async (args, env = environment) => {
const response = await exec(
const mcpClient = new Client({
name: "rea-real-firmware-verifier",
version: "1",
});
// ---------------------------------------------------------------------------
// Invocation helpers
// ---------------------------------------------------------------------------
async function runCli(args, env = baseEnv) {
const { stdout } = await execFileAsync(
process.execPath,
[entrypoint, ...args, "--format", "json"],
[ENTRYPOINT, ...args, "--format", "json"],
{
env,
cwd: repository,
timeout: 150_000,
maxBuffer: 16 * 1024 * 1024,
cwd: REPO_ROOT,
timeout: CALL_TIMEOUT_MS,
maxBuffer: MAX_STDOUT_BYTES,
},
);
return parseEvidence(JSON.parse(response.stdout));
};
const call = async (name, args) => {
const response = await client.callTool(
return parseEvidence(JSON.parse(stdout));
}
async function callTool(name, args) {
const result = await mcpClient.callTool(
{ name, arguments: args },
{ timeout: 150_000 },
{ timeout: CALL_TIMEOUT_MS },
);
assert.notEqual(response.isError, true, JSON.stringify(response));
assert.ok(response.structuredContent !== undefined);
return response.structuredContent;
};
const verifyRegions = (evidence) => {
assert.notEqual(result.isError, true, JSON.stringify(result));
assert.ok(result.structuredContent !== undefined);
return result.structuredContent;
}
// ---------------------------------------------------------------------------
// Result verifiers
// ---------------------------------------------------------------------------
function verifyRegions(evidence) {
assert.equal(evidence.subject.digest.sha256, oracle.sha256);
const result = firmwareResultSchemas.inspect_firmware_regions.parse(
const parsed = firmwareResultSchemas.inspect_firmware_regions.parse(
evidence.normalized_result,
);
const gzip = result.regions.find(
(region) => region.signature === "gzip" && region.offset === oracle.offset,
const gzipRegion = parsed.regions.find(
(r) => r.signature === "gzip" && r.offset === oracle.offset,
);
assert.ok(
gzip !== undefined,
gzipRegion !== undefined,
"Binwalk must find the independently generated gzip offset",
);
assert.equal(gzip.reported_size, oracle.length);
assert.equal(gzip.size_basis, "provider_reported_validation_unknown");
return result.regions.map(
({ provider_id: _provider_id, ...region }) => region,
);
};
const verifyFiles = async (evidence) => {
const result = firmwareResultSchemas.extract_firmware.parse(
assert.equal(gzipRegion.reported_size, oracle.length);
assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown");
// Strip provider-specific ids so CLI and MCP results compare equal.
return parsed.regions.map((region) => {
const { provider_id: _ignored, ...rest } = region;
return rest;
});
}
async function verifyFiles(evidence) {
const parsed = firmwareResultSchemas.extract_firmware.parse(
evidence.normalized_result,
);
assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip"));
assert.ok(parsed.chunks.some((c) => c.handler === "gzip"));
for (const [suffix, expected] of Object.entries(oracle.files)) {
const file = result.files.find((candidate) =>
candidate.relative_path.endsWith(`/${suffix}`),
const entry = parsed.files.find((f) =>
f.relative_path.endsWith(`/${suffix}`),
);
assert.ok(file !== undefined, `Missing extracted ${suffix}`);
assert.equal(file.sha256, expected.sha256);
assert.equal(file.size, expected.size);
assert.equal(await hashFirmwareFile(file.path), expected.sha256);
assert.equal(file.original_file_range, null);
assert.equal(file.runtime_address, null);
assert.ok(entry !== undefined, `Missing extracted ${suffix}`);
assert.equal(entry.sha256, expected.sha256);
assert.equal(entry.size, expected.size);
assert.equal(await hashFirmwareFile(entry.path), expected.sha256);
assert.equal(entry.original_file_range, null);
assert.equal(entry.runtime_address, null);
}
return result;
};
try {
const inspected = await cli(["inspect-firmware-regions", path]);
const cliRegions = verifyRegions(inspected);
const extracted = await cli([
"extract-firmware",
path,
join(root, "cli-output"),
return parsed;
}
const toContentIdentity = (parsed) =>
parsed.files.map(({ relative_path, sha256, size }) => ({
relative_path,
sha256,
size,
}));
// ---------------------------------------------------------------------------
// Verification stages
// ---------------------------------------------------------------------------
async function stageCli() {
const regionsEvidence = await runCli([
"inspect-firmware-regions",
firmwarePath,
]);
const cliFiles = await verifyFiles(extracted);
assert.equal(cliFiles.coverage, "partial");
const regions = verifyRegions(regionsEvidence);
const extractEvidence = await runCli([
"extract-firmware",
firmwarePath,
join(workDir, "cli-output"),
]);
const files = await verifyFiles(extractEvidence);
assert.equal(files.coverage, "partial");
assert.ok(
cliFiles.chunks.some(
(chunk) =>
chunk.handler === null &&
chunk.range.offset === 0 &&
chunk.range.length === 64,
files.chunks.some(
(c) =>
c.handler === null && c.range.offset === 0 && c.range.length === 64,
),
);
await client.connect(
new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
cwd: repository,
env: environment,
stderr: "pipe",
}),
{ timeout: 30_000 },
);
const inspectedMcp = await call("inspect_firmware_regions", { path });
const mcpEvidence = parseEvidence(inspectedMcp);
assert.deepEqual(verifyRegions(mcpEvidence), cliRegions);
const extractedMcp = await call("extract_firmware", {
path,
output_directory: join(root, "mcp-output"),
return { regions, files };
}
async function connectMcp() {
const transport = new StdioClientTransport({
command: process.execPath,
args: [ENTRYPOINT, "mcp"],
cwd: REPO_ROOT,
env: baseEnv,
stderr: "pipe",
});
const mcpFiles = await verifyFiles(parseEvidence(extractedMcp));
const contentIdentity = (result) =>
result.files.map(({ relative_path, sha256, size }) => ({
relative_path,
sha256,
size,
}));
assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles));
const selected = await call("extract_firmware", {
path,
output_directory: join(root, "selected-output"),
await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS });
}
async function stageMcp(cli) {
const regionsRaw = await callTool("inspect_firmware_regions", {
path: firmwarePath,
});
assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions);
const extractRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "mcp-output"),
});
const mcpFiles = await verifyFiles(parseEvidence(extractRaw));
assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files));
const rangedRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "selected-output"),
range: { offset: oracle.offset, length: oracle.length },
});
const selectedFiles = await verifyFiles(parseEvidence(selected));
assert.equal(selectedFiles.selection.offset, oracle.offset);
assert.equal(selectedFiles.selection.length, oracle.length);
const depth = await call("extract_firmware", {
path,
output_directory: join(root, "depth-output"),
const ranged = await verifyFiles(parseEvidence(rangedRaw));
assert.equal(ranged.selection.offset, oracle.offset);
assert.equal(ranged.selection.length, oracle.length);
const depthRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "depth-output"),
max_depth: 1,
});
const depthResult = firmwareResultSchemas.extract_firmware.parse(
depth.normalized_result,
const depthParsed = firmwareResultSchemas.extract_firmware.parse(
depthRaw.normalized_result,
);
assert.equal(depthResult.coverage, "partial");
assert.ok(depthResult.depth_limited_paths.length > 0);
assert.equal(depthParsed.coverage, "partial");
assert.ok(depthParsed.depth_limited_paths.length > 0);
console.log(
"PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits",
);
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") {
const ext4 = join(fixtureRoot, "ext4.bin");
await access(ext4);
const response = await call("extract_firmware", {
path: ext4,
output_directory: join(root, "ext4-output"),
});
const result = firmwareResultSchemas.extract_firmware.parse(
response.normalized_result,
return mcpFiles;
}
async function stageExt4() {
const ext4Path = join(FIXTURE_ROOT, "ext4.bin");
await fs.access(ext4Path);
const raw = await callTool("extract_firmware", {
path: ext4Path,
output_directory: join(workDir, "ext4-output"),
});
const parsed = firmwareResultSchemas.extract_firmware.parse(
raw.normalized_result,
);
assert.ok(parsed.chunks.some((c) => c.handler === "extfs"));
assert.equal(parsed.coverage, "complete");
for (const [suffix, expected] of Object.entries(oracle.files)) {
const match = parsed.files.find((f) =>
f.relative_path.endsWith(`/${suffix}`),
);
assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs"));
assert.equal(result.coverage, "complete");
for (const [suffix, expected] of Object.entries(oracle.files))
assert.equal(
result.files.find((file) => file.relative_path.endsWith(`/${suffix}`))
?.sha256,
expected.sha256,
);
console.log("PASS real ext4 extraction with debugfs; no mounts");
// Preserve the process-ownership utility while withholding all extractors.
const withoutExtractors = join(root, "without-extractors");
await access("/bin/ps");
await mkdir(withoutExtractors);
await symlink("/bin/ps", join(withoutExtractors, "ps"));
const missing = await cli(
["extract-firmware", ext4, join(root, "missing-extractor-output")],
{ ...environment, PATH: withoutExtractors },
);
const partial = firmwareResultSchemas.extract_firmware.parse(
missing.normalized_result,
);
assert.equal(partial.coverage, "partial");
assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs"));
console.log(
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
);
}
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") {
if (process.env.GHIDRA_INSTALL_DIR === undefined)
throw new Error(
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
);
const child = mcpFiles.files.find((file) =>
file.relative_path.endsWith("/usr/bin/probe"),
);
assert.ok(child !== undefined);
await call("open_binary", { path: child.path, provider_id: "ghidra" });
const dossier = await call("analyze_function", {
procedure: "firmware_probe",
});
const evidence = parseEvidence(dossier);
assert.equal(
evidence.subject.digest.sha256,
oracle.files["usr/bin/probe"].sha256,
);
const pseudocode = await call("procedure_pseudo_code", {
procedure: "firmware_probe",
});
assert.equal(parseEvidence(pseudocode).provider.id, "ghidra");
assert.ok(
JSON.stringify(evidence.normalized_result).includes("firmware_probe"),
);
await call("close_binary", {});
console.log(
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
assert.equal(match?.sha256, expected.sha256);
}
console.log("PASS real ext4 extraction with debugfs; no mounts");
// Keep the process-ownership utility available but hide every extractor.
const emptyBin = join(workDir, "without-extractors");
await fs.access("/bin/ps");
await fs.mkdir(emptyBin);
await fs.symlink("/bin/ps", join(emptyBin, "ps"));
const degraded = await runCli(
["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")],
{ ...baseEnv, PATH: emptyBin },
);
const degradedParsed = firmwareResultSchemas.extract_firmware.parse(
degraded.normalized_result,
);
assert.equal(degradedParsed.coverage, "partial");
assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs"));
console.log(
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
);
}
async function stageGhidra(mcpFiles) {
if (process.env.GHIDRA_INSTALL_DIR === undefined) {
throw new Error(
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/providers/ghidra-provider.md",
);
}
const probe = mcpFiles.files.find((f) =>
f.relative_path.endsWith("/usr/bin/probe"),
);
assert.ok(probe !== undefined);
await callTool("open_binary", { path: probe.path, provider_id: "ghidra" });
const dossierRaw = await callTool("analyze_function", {
procedure: "firmware_probe",
});
const dossier = parseEvidence(dossierRaw);
assert.equal(
await hashFirmwareFile(path),
dossier.subject.digest.sha256,
oracle.files["usr/bin/probe"].sha256,
);
const pseudoRaw = await callTool("procedure_pseudo_code", {
procedure: "firmware_probe",
});
assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra");
assert.ok(
JSON.stringify(dossier.normalized_result).includes("firmware_probe"),
);
await callTool("close_binary", {});
console.log(
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
);
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
try {
const cli = await stageCli();
await connectMcp();
const mcpFiles = await stageMcp(cli);
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4();
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1")
await stageGhidra(mcpFiles);
assert.equal(
await hashFirmwareFile(firmwarePath),
oracle.sha256,
"Original firmware bytes must remain unchanged",
);
} finally {
await client.close();
await rm(root, { recursive: true, force: true });
await mcpClient.close();
await fs.rm(workDir, { recursive: true, force: true });
}

26
src/application/ClientConfigurationDocument.ts
View File

@ -1,9 +1,5 @@
import { parse as parseToml, stringify as stringifyToml } from "smol-toml";
import {
parse as parseYaml,
parseDocument as parseYamlDocument,
stringify as stringifyYaml,
} from "yaml";
import { stringify as stringifyYaml } from "yaml";
import { isDeepStrictEqual } from "node:util";
import {
@ -12,6 +8,10 @@ import {
editDshDocument,
dshClientRegistrationEntry,
} from "./DshCordisPatch.js";
import {
editHermesConfigurationYaml,
parseHermesConfigurationYaml,
} from "./HermesConfigurationYaml.js";
import {
applyEdits,
findNodeAtLocation,
@ -118,7 +118,10 @@ const parseDocument = (
if (isEmptyClientConfigurationText(text)) return {};
// Hermes stores MCP servers in `mcp_servers` inside a YAML document.
if (format === "hermes")
return requireObject(parseYaml(text), "The configuration root");
return requireObject(
parseHermesConfigurationYaml(text),
"The configuration root",
);
// ZCode loads nonempty native files with strict JSON.parse.
if (format === "zcode")
return requireObject(JSON.parse(text), "The configuration root");
@ -1419,12 +1422,11 @@ const serializeHermesConfiguration = (
typeof servers === "object" && servers !== null
? (servers as Record<string, unknown>)[PRODUCT_IDENTITY.mcpServerKey]
: undefined;
const yamlDocument = parseYamlDocument(originalText);
if (entry === undefined)
yamlDocument.deleteIn([serversKey, PRODUCT_IDENTITY.mcpServerKey]);
else yamlDocument.setIn([serversKey, PRODUCT_IDENTITY.mcpServerKey], entry);
// Editing REA must not reflow unrelated user prompts or other YAML scalars.
return yamlDocument.toString({ lineWidth: 0 });
return editHermesConfigurationYaml(
originalText,
[serversKey, PRODUCT_IDENTITY.mcpServerKey],
entry,
);
};
/** Build the stdio entry shape expected by one client's configuration dialect. */

21
src/application/HermesConfigurationYaml.test.ts Normal file
View File

@ -0,0 +1,21 @@
import { describe, expect, it } from "vitest";
import {
editHermesConfigurationYaml,
parseHermesConfigurationYaml,
} from "./HermesConfigurationYaml.js";
describe("Hermes configuration YAML", () => {
it("writes an edited number from its new value", () => {
expect(
editHermesConfigurationYaml("mode: 0600\nmask: 0x1F\n", ["mode"], 384),
).toBe("mode: 384\nmask: 0x1F\n");
});
it("accepts keys Hermes reads as distinct and rejects a repeated key", () => {
expect(() =>
parseHermesConfigurationYaml("0600: octal\n600: decimal\n"),
).not.toThrow();
expect(() => parseHermesConfigurationYaml('a: 1\n"a": 2\n')).toThrow();
});
});

74
src/application/HermesConfigurationYaml.ts Normal file
View File

@ -0,0 +1,74 @@
import {
isScalar,
parse,
parseDocument,
type ParsedNode,
type ScalarTag,
type Tags,
} from "yaml";
const NUMBER_TAGS = new Set([
"tag:yaml.org,2002:int",
"tag:yaml.org,2002:float",
]);
/**
* Write a parsed number with its source spelling while that spelling still
* reads as the node's value. Hermes reads config.yaml as YAML 1.1 with exact
* integers, so re-printing a JavaScript number changes what it reads: a
* Discord channel ID loses its low digits, `0600` (octal) becomes `600`, and
* `0.0000001` becomes `1e-7`, which YAML 1.1 does not define as a float.
*/
const keepNumberSource = (tag: Tags[number]): Tags[number] => {
if (typeof tag === "string" || tag.collection !== undefined) return tag;
const { stringify } = tag;
if (!NUMBER_TAGS.has(tag.tag) || stringify === undefined) return tag;
return {
...tag,
stringify: (item, context, ...rest) => {
const { source } = item;
if (source === undefined) return stringify(item, context, ...rest);
const reader = context.doc.schema.tags.find(
(candidate): candidate is ScalarTag =>
candidate.default === true && candidate.test?.test(source) === true,
);
const read = reader?.resolve(
source,
() => undefined,
context.doc.options,
);
return Object.is(isScalar(read) ? read.value : read, item.value)
? source
: stringify(item, context, ...rest);
},
};
};
const HERMES_YAML_OPTIONS = {
customTags: (tags: Tags) => tags.map(keepNumberSource),
// Hermes keeps integers exact, so IDs that round to one double, or `0600`
// and `600`, are distinct keys.
uniqueKeys: (a: ParsedNode, b: ParsedNode) =>
a === b ||
(isScalar(a) &&
isScalar(b) &&
a.value === b.value &&
a.source === b.source),
};
/** Parse a Hermes `config.yaml`. */
export const parseHermesConfigurationYaml = (text: string): unknown =>
parse(text, HERMES_YAML_OPTIONS);
/** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */
export const editHermesConfigurationYaml = (
text: string,
path: readonly string[],
value: unknown,
): string => {
const document = parseDocument(text, HERMES_YAML_OPTIONS);
if (value === undefined) document.deleteIn(path);
else document.setIn(path, value);
// Editing REA must not reflow unrelated user prompts or other YAML scalars.
return document.toString({ lineWidth: 0 });
};

103
src/artifacts/inventory/ArtifactGraphConstruction.test.ts Normal file
View File

@ -0,0 +1,103 @@
import { describe, expect, it } from "vitest";
import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
const fat = (wide = false): Buffer =>
Buffer.from(fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }], wide));
describe("bounded FAT inventory classification", () => {
it("requires the complete table but not slice bytes in the retained prefix", () => {
const bytes = fat();
for (const length of [4, 7, 8, 27]) {
expect(
classifyArtifactContent(
"native",
bytes.subarray(0, length),
bytes.length,
),
).toEqual({ kind: "unknown", format: "unknown" });
}
expect(
classifyArtifactContent(
"native.class",
bytes.subarray(0, 28),
bytes.length,
),
).toEqual({ kind: "executable", format: "mach-o-universal" });
});
it.each([
["empty table", (bytes: Buffer) => bytes.writeUInt32BE(0, 4)],
["slice overlapping table", (bytes: Buffer) => bytes.writeUInt32BE(0, 16)],
["empty slice", (bytes: Buffer) => bytes.writeUInt32BE(0, 20)],
[
"slice beyond file",
(bytes: Buffer) => bytes.writeUInt32BE(bytes.length, 20),
],
["unaligned slice", (bytes: Buffer) => bytes.writeUInt32BE(4097, 16)],
["unsupported alignment", (bytes: Buffer) => bytes.writeUInt32BE(53, 24)],
] as const)("keeps %s unknown", (_label, mutate) => {
const bytes = fat();
mutate(bytes);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "unknown",
format: "unknown",
});
});
it("does not infer Mach-O from an out-of-bounds architecture count", () => {
const bytes = fat();
bytes.writeUInt32BE(0xffffffff, 4);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "resource",
format: "file",
});
});
it("does not lose unsafe FAT64 range values to number conversion", () => {
const bytes = fat(true);
bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "unknown",
format: "unknown",
});
});
it("requires valid ranges for every declared slice", () => {
const bytes = Buffer.from(
fatImage([
{ cpu: CPU.arm64, bytes: machoImage({}) },
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
]),
);
bytes.writeUInt32BE(bytes.length, 40);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "unknown",
format: "unknown",
});
});
it("requires observed slice magic for an unknown CPU type", () => {
const cpu = { type: 0x12345678, subtype: 0 };
const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }]));
expect(
classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length),
).toEqual({ kind: "unknown", format: "unknown" });
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "executable",
format: "mach-o-universal",
});
});
it.each([NaN, Infinity, -1, 8.5, Number.MAX_SAFE_INTEGER + 1, 27])(
"keeps invalid or contradictory file size %s unknown",
(size) => {
const bytes = fat();
expect(classifyArtifactContent("native", bytes, size)).toEqual({
kind: "unknown",
format: "unknown",
});
},
);
});

5
src/cli/cliResultOutput.ts
View File

@ -216,8 +216,9 @@ export const createCliResultOutput = (
: filtered === undefined
? { ok: true, meta }
: { ok: true, data: filtered, meta }
: filtered === undefined ||
filtered === null ||
: // A selected null is a value; only a selection with no JSON value
// falls back to the empty document.
filtered === undefined ||
typeof filtered === "function" ||
typeof filtered === "symbol"
? {}

20
src/domain/analysisView/analysisView.test.ts
View File

@ -169,7 +169,8 @@ it("pages every retained dossier collection and the native API record", () => {
| "comments"
| "unresolved_calls"
| "referenced_strings"
| "referenced_names",
| "referenced_names"
| "swift_string_literals",
result: unknown = original,
) => {
const view = projectAnalysisView(nativeParent(result), {
@ -195,6 +196,23 @@ it("pages every retained dossier collection and the native API record", () => {
}),
).toMatchObject({ item: [{ text: "entry" }] });
expect(page("referenced_names")).toMatchObject({ coverage: { total: 0 } });
const literal = {
source_address: "0x401004",
value: "SleepDisabled",
representation: "small",
address: null,
registers: ["x0", "x1"],
words: ["0x7369447065656c53", "0xed000064656c6261"],
};
expect(
page("swift_string_literals", {
...original,
swift_string_literals: [literal],
}),
).toMatchObject({ item: [literal], coverage: { total: 1 } });
expect(page("swift_string_literals")).toMatchObject({
item: { available: false, reason: "not recorded" },
});
const { unresolved_calls: _omitted, ...withoutUnresolved } = original;
expect(
page("unresolved_calls", {

2
src/domain/analysisView/analysisView.ts
View File

@ -160,6 +160,8 @@ const nativeViewSchema = z.strictObject({
"unresolved_calls",
"referenced_strings",
"referenced_names",
"swift_string_literals",
"swift_string_literal_omissions",
"native_api",
"value_flow_summary",
"value_flow_operations",

5
src/domain/analysisView/nativeFunctionView.ts
View File

@ -79,6 +79,8 @@ const nativeRows = (
unresolved_calls: dossier.unresolved_calls,
referenced_strings: dossier.referenced_strings,
referenced_names: dossier.referenced_names,
swift_string_literals: dossier.swift_string_literals,
swift_string_literal_omissions: dossier.swift_string_literal_omissions,
value_flow_operations: flow?.operations,
value_flow_def_use: flow?.def_use,
value_flow_effects: flow?.effects,
@ -163,7 +165,8 @@ export const projectNativeFunctionView = (
singleton({
available: false,
// Only value-flow facets inherit the value-flow failure reason; an
// older dossier simply has no unresolved_calls record.
// older dossier simply has no unresolved_calls record, and a target
// whose provider does not decode Swift literals has no such record.
reason:
view.facet.startsWith("value_flow_") && flow?.available === false
? flow.reason

5
src/domain/apple/machoContainer.ts
View File

@ -63,6 +63,11 @@ export const readFatContainerHeader = (
const magic = view.getUint32(0, false);
if (![FAT_MAGIC, FAT_MAGIC_64, FAT_CIGAM, FAT_CIGAM_64].includes(magic))
return { state: "not-fat" };
if (!Number.isSafeInteger(fileSize) || fileSize < bytes.byteLength)
return {
state: "malformed",
reason: "FAT file size is invalid or contradicts retained bytes",
};
if (bytes.byteLength < 8) {
return fileSize < 8
? { state: "malformed", reason: "FAT header is truncated" }

83
src/domain/callPath.test.ts
View File

@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import { buildCallPath } from "./callPath.js";
import { combineFunctionRelationshipCoverage } from "./functionCallCoverage.js";
import { callPathResultSchema, type CallPathInput } from "./callPathSchemas.js";
import { createEvidence, type Evidence } from "./evidence.js";
import type { JsonValue } from "./jsonValue.js";
@ -396,6 +397,60 @@ describe("explicit call relationship coverage", () => {
expect(result.limitations).toEqual([reason]);
});
it("establishes not_found when only caller coverage is unknown", () => {
const reason =
"Callers: Upstream xrefs do not establish direct caller relationships.";
const result = run([
observe("0x1000", ["0x2000"], "rea-workflow", {
relationshipCoverage: { status: "unknown", reason },
}),
observe("0x2000", [], "rea-workflow", {
relationshipCoverage: { status: "unknown", reason },
}),
]);
expect(result).toMatchObject({
status: "not_found",
search_scope: { exhaustive: true },
});
expect(result.limitations).toContain(reason);
});
it("stays unknown when callee coverage is unknown", () => {
const reason =
"Callees: Upstream callees capped results at 500; additional callees remain unknown.";
const result = run([
observe("0x1000", ["0x2000"], "rea-workflow", {
relationshipCoverage: { status: "unknown", reason },
}),
observe("0x2000", [], "rea-workflow", {
relationshipCoverage: { status: "unknown", reason },
}),
]);
expect(result).toMatchObject({
status: "unknown",
search_scope: { exhaustive: false },
});
expect(result.limitations).toContain(reason);
});
it("stays unknown when both caller and callee coverage are unknown", () => {
const reason =
"Callers: Upstream xrefs do not establish direct caller relationships. Callees: Upstream callees are not observed.";
const result = run([
observe("0x1000", ["0x2000"], "rea-workflow", {
relationshipCoverage: { status: "unknown", reason },
}),
observe("0x2000", [], "rea-workflow", {
relationshipCoverage: { status: "unknown", reason },
}),
]);
expect(result).toMatchObject({
status: "unknown",
search_scope: { exhaustive: false },
});
expect(result.limitations).toContain(reason);
});
it("ignores incomplete relationship coverage outside the reachable search closure", () => {
const reason = "An unrelated function's callees were not observed.";
const result = run([
@ -415,6 +470,34 @@ describe("explicit call relationship coverage", () => {
});
});
describe("IDA-capped callee coverage beside unknown callers", () => {
it("keeps the search non-exhaustive for the combined coverage value", () => {
const combined = combineFunctionRelationshipCoverage(
{
status: "unknown",
reason: "Upstream xrefs do not establish direct caller relationships.",
},
{
status: "partial",
limitations: [
"Upstream callees capped results at 500; additional callees remain unknown.",
],
},
);
expect(combined.status).toBe("unknown");
const result = run([
observe("0x1000", [], "rea-workflow", { relationshipCoverage: combined }),
observe("0x4000", []),
]);
expect(result).toMatchObject({
status: "unknown",
search_scope: { exhaustive: false },
shortest_hops: null,
paths: [],
});
});
});
describe("call path evidence admission", () => {
it("normalizes hex addresses and rejects duplicates and mixed providers", () => {
expect(

7
src/domain/callPath.ts
View File

@ -10,6 +10,7 @@ import {
parseFunctionEvidence,
type FunctionSnapshot,
} from "./functionDossierEvidence.js";
import { isCallerOnlyUnknownCoverage } from "./functionCallCoverage.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
/** Minimal directed caller-to-callee adjacency for call-path search. */
@ -255,10 +256,14 @@ const inspectSearch = ({
// Explicit, Evidence-bound relationship coverage owns completeness;
// generic provider/dossier limitations remain contextual. Legacy Evidence
// without that metadata retains its conservative limitation-based rule.
// The search only expands callee edges, so coverage that is unknown
// solely because callers are unobservable does not make the searched
// callee closure incomplete; the caller caveat stays in limitations.
if (
coverage === null
? snapshot.limitations.length > 0
: coverage.status !== "complete"
: coverage.status !== "complete" &&
!isCallerOnlyUnknownCoverage(coverage)
)
incompleteExpandedCoverage = true;
} else if (node === goal) {

64
src/domain/functionCallCoverage.test.ts Normal file
View File

@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import {
combineFunctionRelationshipCoverage,
isCallerOnlyUnknownCoverage,
} from "./functionCallCoverage.js";
import type { RelationshipCoverage } from "./relationshipCoverage.js";
const unknownCallers: RelationshipCoverage = {
status: "unknown",
reason: "Upstream xrefs do not establish direct caller relationships.",
};
const unknownCallees: RelationshipCoverage = {
status: "unknown",
reason: "Upstream callees are not observed in this profile.",
};
const cappedCallees: RelationshipCoverage = {
status: "partial",
limitations: [
"Upstream callees capped results at 500; additional callees remain unknown.",
],
};
const complete: RelationshipCoverage = { status: "complete" };
describe("isCallerOnlyUnknownCoverage", () => {
it("detects unknown callers beside complete callees", () => {
const combined = combineFunctionRelationshipCoverage(
unknownCallers,
complete,
);
expect(combined.status).toBe("unknown");
expect(isCallerOnlyUnknownCoverage(combined)).toBe(true);
});
it("rejects unknown callees beside complete callers", () => {
const combined = combineFunctionRelationshipCoverage(
complete,
unknownCallees,
);
expect(isCallerOnlyUnknownCoverage(combined)).toBe(false);
});
it("rejects unknown coverage on both sides", () => {
const combined = combineFunctionRelationshipCoverage(
unknownCallers,
unknownCallees,
);
expect(isCallerOnlyUnknownCoverage(combined)).toBe(false);
});
it("rejects unknown callers beside IDA-capped callees", () => {
const combined = combineFunctionRelationshipCoverage(
unknownCallers,
cappedCallees,
);
expect(combined.status).toBe("unknown");
expect(isCallerOnlyUnknownCoverage(combined)).toBe(false);
});
it("rejects complete coverage", () => {
const combined = combineFunctionRelationshipCoverage(complete, complete);
expect(isCallerOnlyUnknownCoverage(combined)).toBe(false);
});
});

22
src/domain/functionCallCoverage.ts
View File

@ -2,6 +2,9 @@ import type { RelationshipCoverage } from "./relationshipCoverage.js";
import type { FunctionDossier } from "./hopperValues.js";
import { uniqueSorted } from "./canonicalOrdering.js";
const callerCoveragePrefix = "Callers: ";
const calleeCoveragePrefix = "Callees: ";
/** Combine caller and callee read outcomes without losing either explanation. */
export const combineFunctionRelationshipCoverage = (
callers: RelationshipCoverage | undefined,
@ -20,7 +23,8 @@ export const combineFunctionRelationshipCoverage = (
);
const details = diagnostics.length > 0 ? { diagnostics } : {};
const reasons = reads.flatMap(({ direction, coverage }) => {
const label = direction === "callers" ? "Callers" : "Callees";
const label =
direction === "callers" ? callerCoveragePrefix : calleeCoveragePrefix;
const limitations =
coverage === undefined
? ["Provider did not report relationship coverage."]
@ -29,7 +33,7 @@ export const combineFunctionRelationshipCoverage = (
: coverage.status === "partial"
? coverage.limitations
: [];
return limitations.map((reason) => `${label}: ${reason}`);
return limitations.map((reason) => `${label}${reason}`);
});
if (
reads.some(
@ -50,6 +54,20 @@ export const combineFunctionRelationshipCoverage = (
return { status: "complete", ...details };
};
/**
* Whether combined relationship coverage is unknown only on the caller side.
* The combine helper above prefixes each direction's reasons, so a reason
* that carries the caller prefix and no callee prefix records observed
* callees with unobservable callers. Forward (callee-edge) searches stay
* exhaustive under that caveat; the caller reason remains a limitation.
*/
export const isCallerOnlyUnknownCoverage = (
coverage: RelationshipCoverage,
): boolean =>
coverage.status === "unknown" &&
coverage.reason.startsWith(callerCoveragePrefix) &&
!coverage.reason.includes(calleeCoveragePrefix);
/** Explain why two resolved call sets cannot establish call-set equality. */
export const functionCallCoverageLimitations = (
coverage: RelationshipCoverage | null,

230
src/domain/functionComparison.test.ts
View File

@ -716,6 +716,236 @@ describe("function comparison identity and malformed CFG", () => {
});
});
const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) =>
functionDossierSchema.parse({
...dossier("return 0;", base),
swift_string_literals: values.map((value) => ({
source_address: base,
value,
representation: "small" as const,
address: null,
registers: ["x0", "x1"] as const,
words: ["0x0", "0xe"] as const,
})),
});
const stringsDimension = (
leftDigit: string,
left: ReturnType<typeof withLiterals> | ReturnType<typeof dossier>,
rightDigit: string,
right: ReturnType<typeof withLiterals> | ReturnType<typeof dossier>,
) =>
compareFunctions(
observe(leftDigit, left),
observe(rightDigit, right),
).dimensions.find(({ dimension }) => dimension === "strings_names");
describe("function comparison Swift string literals", () => {
it("treats differing decoded literals as a strings and names change", () => {
expect(
stringsDimension(
"1",
withLiterals("0x1000", ["SleepDisabled"]),
"2",
withLiterals("0x2000", ["Holding awake"]),
),
).toMatchObject({ status: "changed" });
});
it("matches the same decoded literal after the function is relocated", () => {
expect(
stringsDimension(
"3",
withLiterals("0x1000", ["SleepDisabled"]),
"4",
withLiterals("0x2000", ["SleepDisabled"]),
),
).toMatchObject({ status: "unchanged" });
});
it("keeps strings and names unknown when only one dossier recorded Swift literals", () => {
expect(
stringsDimension(
"5",
withLiterals("0x1000", []),
"6",
dossier("return 0;", "0x2000"),
),
).toMatchObject({
status: "unknown",
limitations: [
"Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.",
],
});
});
it("reports a referenced name change even when Swift literal coverage differs", () => {
const named = functionDossierSchema.parse({
...dossier("return 0;", "0x2000"),
referenced_names: [
{ address: "0x3000", value: "helper", source_address: "0x2000" },
],
});
expect(
stringsDimension("7", withLiterals("0x1000", []), "8", named),
).toMatchObject({ status: "changed" });
});
});
describe("function comparison partial Swift literal coverage", () => {
it("does not equate a partial literal record with a complete one", () => {
const partial = functionDossierSchema.parse({
...withLiterals("0x2000", ["SleepDisabled"]),
swift_string_literal_omissions: [
{
source_address: "0x2000",
representation: "large",
address: "0x3000",
byte_length: 64,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0xd000000000000040", "0x8000000000002fe0"],
},
],
});
expect(
stringsDimension(
"9",
withLiterals("0x1000", ["SleepDisabled"]),
"a",
partial,
),
).toMatchObject({ status: "unknown" });
});
it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => {
const partial = functionDossierSchema.parse({
...withLiterals("0x2000", ["SleepDisabled"]),
swift_string_literal_omissions: [
{
source_address: "0x2004",
representation: "small",
address: null,
byte_length: 4,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0x732f4220", "0xe400000000000000"],
},
],
});
expect(
stringsDimension("d", withLiterals("0x1000", []), "e", partial),
).toMatchObject({ status: "changed" });
});
it("does not interpret an unexamined path as a missing literal at a retained site", () => {
const partial = functionDossierSchema.parse({
...withLiterals("0x2000", ["A"]),
swift_string_literal_omissions: [
{
source_address: "0x2004",
representation: "small",
address: null,
byte_length: 0,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0x0", "0xe000000000000000"],
},
],
});
const complete = withLiterals("0x1000", ["A", "B"]);
expect(stringsDimension("1", complete, "2", partial)).toMatchObject({
status: "unknown",
});
expect(stringsDimension("3", partial, "4", complete)).toMatchObject({
status: "unknown",
});
expect(
stringsDimension("5", withLiterals("0x1000", []), "6", partial),
).toMatchObject({ status: "changed" });
});
it("treats an omission record without decoded literals as partial coverage", () => {
const omissionOnly = functionDossierSchema.parse({
...dossier("return 0;", "0x2000"),
swift_string_literal_omissions: [
{
source_address: "0x2004",
representation: "small",
address: null,
byte_length: 4,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0x732f4220", "0xe400000000000000"],
},
],
});
expect(
stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly),
).toMatchObject({ status: "unknown" });
});
it("keeps different retained representations unknown when both dossiers are partial", () => {
const omission = {
source_address: "0x1004",
representation: "small",
address: null,
byte_length: 4,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0x732f4220", "0xe400000000000000"],
} as const;
const small = functionDossierSchema.parse({
...withLiterals("0x1000", ["SleepDisabled"]),
swift_string_literal_omissions: [omission],
});
const large = functionDossierSchema.parse({
...dossier("return 0;", "0x2000"),
swift_string_literals: [
{
source_address: "0x2000",
value: "SleepDisabled",
representation: "large",
address: "0x3000",
registers: ["x0", "x1"],
words: ["0xd00000000000000d", "0x8000000000002fe0"],
},
],
swift_string_literal_omissions: [
{ ...omission, source_address: "0x2004" },
],
});
expect(stringsDimension("1", small, "2", large)).toMatchObject({
status: "unknown",
});
});
it("keeps different retained literals unknown when both dossiers are partial", () => {
const partial = (base: "0x1000" | "0x2000", value: string) =>
functionDossierSchema.parse({
...withLiterals(base, [value]),
swift_string_literal_omissions: [
{
source_address: base === "0x1000" ? "0x1004" : "0x2004",
representation: "large",
address: "0x3000",
byte_length: 64,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0xd000000000000040", "0x8000000000002fe0"],
},
],
});
expect(
stringsDimension(
"b",
partial("0x1000", "SleepDisabled"),
"c",
partial("0x2000", "Holding awake"),
),
).toMatchObject({ status: "unknown" });
});
});
const expectInlineComparisonEvidence = (
result: ReturnType<typeof compareFunctions>,
): void => {

42
src/domain/functionComparisonDimensions.ts
View File

@ -11,6 +11,7 @@ import {
referenceProjection,
sorted,
stringAndNameProjection,
swiftLiteralSitesConflict,
} from "./functionComparisonNormalization.js";
import type {
DimensionName,
@ -139,8 +140,12 @@ const compareStringsAndNames = (
right: FunctionSnapshot,
context: ComparisonContext,
): FunctionDimension => {
const leftValues = stringAndNameProjection(left);
const rightValues = stringAndNameProjection(right);
const complete =
swiftLiteralCoverage(left) === swiftLiteralCoverage(right) &&
swiftLiteralCoverage(left) !== "partial";
// Facts both dossiers record decide a change before Swift literal coverage can.
const leftValues = stringAndNameProjection(left, complete);
const rightValues = stringAndNameProjection(right, complete);
if (leftValues === null || rightValues === null)
return unresolvedDimension({
dimension: "strings_names",
@ -151,9 +156,40 @@ const compareStringsAndNames = (
"Reference source locations could not be normalized to function offsets.",
],
});
return compareValues("strings_names", leftValues, rightValues, context);
const compared = compareValues(
"strings_names",
leftValues,
rightValues,
context,
);
if (complete || compared.status === "changed") return compared;
// A retained literal differs only when the opposite dossier establishes its absence.
if (swiftLiteralSitesConflict(left, right)) {
const leftAll = stringAndNameProjection(left);
const rightAll = stringAndNameProjection(right);
if (leftAll !== null && rightAll !== null)
return compareValues("strings_names", leftAll, rightAll, context);
}
return unresolvedDimension({
dimension: "strings_names",
links: context.links,
leftCount: null,
rightCount: null,
limitations: [
"Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.",
],
});
};
const swiftLiteralCoverage = (
snapshot: FunctionSnapshot,
): "absent" | "complete" | "partial" =>
(snapshot.dossier.swift_string_literal_omissions ?? []).length > 0
? "partial"
: snapshot.dossier.swift_string_literals === undefined
? "absent"
: "complete";
const compareReferences = (
left: FunctionSnapshot,
right: FunctionSnapshot,

79
src/domain/functionComparisonNormalization.ts
View File

@ -142,6 +142,7 @@ export const commentProjection = (
export const stringAndNameProjection = (
snapshot: FunctionSnapshot,
includeSwiftLiterals = true,
): readonly unknown[] | null => {
const values = [
...snapshot.dossier.referenced_strings.map(
@ -164,12 +165,90 @@ export const stringAndNameProjection = (
value,
}),
),
...(includeSwiftLiterals
? (snapshot.dossier.swift_string_literals ?? [])
: []
).map(
({
source_address: sourceAddress,
value,
representation,
registers,
}) => ({
kind: "swift_string",
source_offset: relativeAddress(
sourceAddress,
snapshot.dossier.procedure.address,
),
value,
representation,
registers,
}),
),
];
return values.some(({ source_offset: offset }) => offset === null)
? null
: sorted(values);
};
/**
* A retained literal proves presence, not that every path through its site was examined.
* A differing retained fact proves a change only when the opposite dossier is complete
* and therefore establishes its absence.
*/
export const swiftLiteralSitesConflict = (
left: FunctionSnapshot,
right: FunctionSnapshot,
): boolean => {
const leftSites = swiftLiteralSites(left);
const rightSites = swiftLiteralSites(right);
for (const site of new Set([
...leftSites.values.keys(),
...rightSites.values.keys(),
])) {
const leftValues =
leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined);
const rightValues =
rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined);
if (leftValues === undefined || rightValues === undefined) continue;
const leftFacts = new Set(leftValues);
const rightFacts = new Set(rightValues);
if (
(rightSites.complete &&
leftValues.some((value) => !rightFacts.has(value))) ||
(leftSites.complete && rightValues.some((value) => !leftFacts.has(value)))
)
return true;
}
return false;
};
const swiftLiteralSites = (snapshot: FunctionSnapshot) => {
const base = snapshot.dossier.procedure.address;
const omissions = snapshot.dossier.swift_string_literal_omissions ?? [];
const site = (sourceAddress: string, registers: readonly string[]) => {
const offset = relativeAddress(sourceAddress, base);
return offset === null ? null : canonicalJson([offset, registers]);
};
// Keep the same literal facts the full strings_names projection compares.
const values = new Map<string, string[]>();
for (const literal of snapshot.dossier.swift_string_literals ?? []) {
const key = site(literal.source_address, literal.registers);
if (key === null) continue;
values.set(
key,
[
...(values.get(key) ?? []),
canonicalJson([literal.value, literal.representation]),
].sort(compareUnicodeCodePoints),
);
}
const complete =
snapshot.dossier.swift_string_literals !== undefined &&
omissions.length === 0;
return { values, complete };
};
export const sorted = (values: readonly unknown[]): readonly unknown[] =>
values
.map((value) => ({

76
src/domain/hopperValues.ts
View File

@ -290,6 +290,70 @@ export const unresolvedCallSchema = z.strictObject({
reason: z.string(),
});
const swiftStringLiteralFields = {
source_address: z
.string()
.describe(
"Instruction that consumes both words: a call, tail call, return, or paired store.",
),
value: z
.string()
.describe("Decoded UTF-8 text, including the canonical empty string."),
registers: z.tuple([z.string(), z.string()]),
words: z
.tuple([z.string(), z.string()])
.describe("Observed constant register values, in register order."),
};
/** A Swift String literal decoded from the two words of its ABI value, not a data reference. */
export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [
z.strictObject({
...swiftStringLiteralFields,
representation: z
.literal("small")
.describe("The UTF-8 bytes are inside the two words."),
address: z.null().describe("A small literal has no storage of its own."),
}),
z.strictObject({
...swiftStringLiteralFields,
representation: z
.literal("large")
.describe(
"An immortal literal whose words hold its count and a pointer to its bytes.",
),
address: z.string().describe("Address of a large literal's bytes."),
}),
]);
const swiftStringLiteralOmissionFields = {
source_address: swiftStringLiteralFields.source_address,
byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
reason: z
.literal("capacity")
.describe(
"The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.",
),
registers: swiftStringLiteralFields.registers,
words: swiftStringLiteralFields.words,
};
/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */
export const swiftStringLiteralOmissionSchema = z.discriminatedUnion(
"representation",
[
z.strictObject({
...swiftStringLiteralOmissionFields,
representation: z.literal("small"),
address: z.null(),
}),
z.strictObject({
...swiftStringLiteralOmissionFields,
representation: z.literal("large"),
address: z.string().describe("Address of the literal's bytes."),
}),
],
);
/** Observed endpoints with complete or partial reference classification evidence. */
export const referenceEdgeSchema = z
.object({
@ -341,6 +405,18 @@ export const functionDossierSchema = z
})
.strict(),
),
swift_string_literals: z
.array(swiftStringLiteralSchema)
.exactOptional()
.describe(
"Swift String literals decoded from constant register values. Absent when the provider does not decode them for this target.",
),
swift_string_literal_omissions: z
.array(swiftStringLiteralOmissionSchema)
.exactOptional()
.describe(
"The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.",
),
basic_blocks: z.array(
z
.object({

24
src/dotnet/ManagedArtifactInspector.test.ts
View File

@ -292,6 +292,30 @@ describe("managed artifact inventory", () => {
]);
expect(result.coverage).toMatchObject({ state: "complete", issues: [] });
});
// dotnet/runtime corhdr.h: 32BITREQUIRED and 32BITPREFERRED form one
// two-bit field, so neither bit can be read on its own.
it.each([
["neither bit", 0x0000_0001, "anycpu", false],
["32BITREQUIRED alone", 0x0000_0003, "x86", true],
["both bits", 0x0002_0003, "anycpu-prefer-32", false],
["the reserved 32BITPREFERRED alone", 0x0002_0001, "anycpu", false],
] as const)(
"reads %s as one 32-bit requirement",
(_label, cliFlags, architecture, requires32Bit) => {
const bytes = buildManagedPeFixture({ cliFlags });
const target = managedPeFixtureTarget(bytes);
expect(
inspectManagedArtifactBytes(bytes, target).classification
.managed_architecture,
).toBe(architecture);
expect(
inspectManagedNativeBoundariesBytes(bytes, target).cli_native
?.requires_32bit,
).toBe(requires32Bit);
},
);
});
describe("managed artifact coded indexes", () => {

7
src/dotnet/ManagedArtifactInspector.ts
View File

@ -9,6 +9,7 @@ import {
type ManagedParseIssue,
} from "../domain/managed/managedArtifact.js";
import {
managedCli32BitRequirement,
readManagedPeLayout,
type ManagedCliHeader,
type ManagedPeLayout,
@ -249,9 +250,9 @@ const managedArchitectureFor = (
): ManagedArtifactInspection["classification"]["managed_architecture"] => {
if (layout.architecture !== "x86" || (cli.flags & 1) === 0)
return layout.architecture;
if ((cli.flags & 0x0000_0002) !== 0) return "x86";
if ((cli.flags & 0x0002_0000) !== 0) return "anycpu-prefer-32";
return "anycpu";
const requirement = managedCli32BitRequirement(cli.flags);
if (requirement === "required") return "x86";
return requirement === "preferred" ? "anycpu-prefer-32" : "anycpu";
};
const runtimeClassification = ({

7
src/dotnet/ManagedNativeBoundaryHelpers.ts
View File

@ -18,7 +18,10 @@ import {
} from "./ManagedMetadataHeaps.js";
import { managedTableRowCounts } from "./ManagedMetadataInventory.js";
import type { ManagedMetadataInventory } from "./ManagedMetadataInventory.js";
import type { ManagedPeLayout } from "./ManagedPeReader.js";
import {
managedCli32BitRequirement,
type ManagedPeLayout,
} from "./ManagedPeReader.js";
import { readManagedValue } from "./ManagedReaderFailure.js";
type ModuleRef = ManagedNativeBoundaryInspection["module_refs"][number];
@ -380,7 +383,7 @@ export const cliNative = (
if (pe.cli === null) return null;
return {
il_only: (pe.cli.flags & 0x0000_0001) !== 0,
requires_32bit: (pe.cli.flags & 0x0000_0002) !== 0,
requires_32bit: managedCli32BitRequirement(pe.cli.flags) === "required",
strong_name_signed: (pe.cli.flags & 0x0000_0008) !== 0,
native_entry_point: (pe.cli.flags & 0x0000_0010) !== 0,
ready_to_run_signature: pe.cli.readyToRunSignature,

14
src/dotnet/ManagedPeReader.ts
View File

@ -31,6 +31,20 @@ export interface ManagedCliHeader {
readonly readyToRunSignature: boolean;
}
/**
* How the CLI header constrains process bitness. 32BITREQUIRED and
* 32BITPREFERRED are one two-bit field: required alone is x86-only, both set
* is platform-neutral code that prefers a 32-bit process, and preferred alone
* is reserved and constrains nothing.
*/
export const managedCli32BitRequirement = (
flags: number,
): "required" | "preferred" | "none" => {
const pair = flags & 0x0002_0002;
if (pair === 0x0000_0002) return "required";
return pair === 0x0002_0002 ? "preferred" : "none";
};
export interface ManagedPeLayout {
readonly machine: number;
readonly architecture: "x86" | "x86_64" | "arm" | "arm64";

98
src/ghidra/GhidraFunctionValues.test.ts
View File

@ -195,6 +195,51 @@ describe("Ghidra function-analysis result values", () => {
});
});
describe("Ghidra Swift String literals", () => {
it("keeps a small Swift literal without storage apart from a large literal's bytes", () => {
const dossier = ghidraFunctionDossier();
if (
typeof dossier !== "object" ||
dossier === null ||
Array.isArray(dossier)
)
throw new TypeError("Ghidra dossier fixture is invalid");
const literals = [
{
source_address: "0x401004",
value: "The user name ",
representation: "small",
address: null,
registers: ["x0", "x1"],
words: ["0x7265737520656854", "0xee0020656d616e20"],
},
{
source_address: "0x401006",
value: "",
representation: "small",
address: null,
registers: ["x0", "x1"],
words: ["0x0", "0xe000000000000000"],
},
{
source_address: "0x401008",
value: " cannot be written into a sudoers rule safely.",
representation: "large",
address: "0x402270",
registers: ["x0", "x1"],
words: ["0xd00000000000002e", "0x8000000000402250"],
},
];
expect(
parseFunctionValueFixture("analyze_function", {
...dossier,
swift_string_literals: literals,
swift_string_literal_omissions: [],
}),
).toMatchObject({ ok: true, value: { swift_string_literals: literals } });
});
});
describe("Ghidra compact reference provenance", () => {
it("distinguishes incoming references, examined emptiness and unavailable coverage at data addresses", () => {
const instruction = {
@ -423,6 +468,34 @@ describe("Ghidra function-analysis malformed results", () => {
);
});
const malformedSwiftStringLiterals = (
dossier: Record<string, JsonValue>,
): Array<[string, GhidraFunctionOperation, JsonValue]> =>
(
[
["non-canonical Swift literal address", "large", "402240"],
["small Swift literal with storage", "small", "0x402270"],
["large Swift literal without storage", "large", null],
] as const
).map(([name, representation, address]) => [
name,
"analyze_function",
{
...dossier,
swift_string_literal_omissions: [],
swift_string_literals: [
{
source_address: "0x401004",
value: "The power setting could not be changed.",
representation,
address,
registers: ["x0", "x1"],
words: ["0xd000000000000027", "0x8000000000402220"],
},
],
},
]);
const malformedOutputs = (): Array<
[string, GhidraFunctionOperation, JsonValue]
> => {
@ -437,6 +510,11 @@ const malformedOutputs = (): Array<
throw new TypeError("Ghidra native API fixture is invalid");
return [
["non-canonical address", "procedure_callers", ["401000"]],
[
"Swift literals without their omission record",
"analyze_function",
{ ...dossier, swift_string_literals: [] },
],
[
"missing classification",
"procedure_info",
@ -467,6 +545,26 @@ const malformedOutputs = (): Array<
Object.entries(dossier).filter(([key]) => key !== "native_api"),
),
],
...malformedSwiftStringLiterals(dossier),
[
"non-canonical Swift literal omission address",
"analyze_function",
{
...dossier,
swift_string_literals: [],
swift_string_literal_omissions: [
{
source_address: "0x401004",
representation: "large",
address: "402240",
byte_length: 39,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0xd000000000000027", "0x8000000000402220"],
},
],
},
],
[
"non-canonical native API target",
"analyze_function",

17
src/ghidra/GhidraFunctionValues.ts
View File

@ -293,6 +293,15 @@ const ghidraFunctionDossier = functionDossierSchema
})
.strict()
.superRefine((value, context) => {
if (
(value.swift_string_literals === undefined) !==
(value.swift_string_literal_omissions === undefined)
)
context.addIssue({
code: "custom",
message:
"Ghidra dossier must record Swift literals and their omissions together",
});
const addresses = [
value.procedure.address,
...value.comments.map(({ address }) => address),
@ -308,6 +317,14 @@ const ghidraFunctionDossier = functionDossierSchema
address,
source_address,
]),
...(value.swift_string_literals ?? []).flatMap(
({ address, source_address }) =>
address === null ? [source_address] : [address, source_address],
),
...(value.swift_string_literal_omissions ?? []).flatMap(
({ address, source_address }) =>
address === null ? [source_address] : [address, source_address],
),
...value.basic_blocks.flatMap(({ start, end, successors }) => [
start,
end,

4
src/ghidra/GhidraLanguageCatalog.ts
View File

@ -75,8 +75,10 @@ export const readGhidraLanguageCatalog = async (
if (!entry.endsWith(".ldefs")) continue;
const bytes = await readFile(join(directory, entry));
if (bytes.length > MAX_LDEFS_BYTES) continue;
// XML 1.0 section 4.3.3: a leading UTF-8 BOM is an encoding signature,
// not document content; TextDecoder consumes it, Buffer#toString does not.
definitions.push(
...parseGhidraLanguageDefinitions(bytes.toString("utf8")),
...parseGhidraLanguageDefinitions(new TextDecoder().decode(bytes)),
);
}
}

23
src/ghidra/GhidraLanguageOverride.test.ts
View File

@ -2,6 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { parseConfig } from "../config/parseConfig.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
@ -190,6 +191,28 @@ describe("Ghidra language catalog", () => {
);
});
it("reads a UTF-8 signed definition beside an unsigned sibling", async () => {
const root = await createTestTempDirectory("rea-ghidra-bom-");
const module = (name: string) =>
join(root, "Ghidra/Processors", name, "data/languages");
await mkdir(module("probe"), { recursive: true });
await mkdir(module("x86"), { recursive: true });
await writeFile(
join(module("probe"), "probe.ldefs"),
Buffer.concat([
Buffer.from([0xef, 0xbb, 0xbf]),
Buffer.from(
'<?xml version="1.0" encoding="UTF-8"?>\n<language_definitions><language id="probe:LE:32:default"/></language_definitions>',
),
]),
);
await writeFile(join(module("x86"), "x86.ldefs"), X86_LDEFS);
expect(await readGhidraLanguageCatalog(root)).toEqual([
{ id: "probe:LE:32:default", compilerSpecIds: [] },
...parseGhidraLanguageDefinitions(X86_LDEFS),
]);
});
it("admits only languages and compiler specs the installation defines", () => {
const catalog = parseGhidraLanguageDefinitions(X86_LDEFS);
expect(

23
src/native/pwntools/PwntoolsDecoder.ts
View File

@ -32,7 +32,10 @@ import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorClean
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
import { err, ok, type Result } from "../../domain/result.js";
import { runOwnedCommand } from "../../process/OwnedCommand.js";
import {
ownedCommandResources,
runOwnedCommand,
} from "../../process/OwnedCommand.js";
import type {
ProviderProcessSupervisor,
OwnedProviderProcessSpawnOptions,
@ -562,20 +565,16 @@ export class PwntoolsDecoder<
const pending = this.#pendingCleanup;
if (pending?.process === undefined) return undefined;
const stopped = await pending.process.stop();
const ownership = pending.process.launch.ownership;
if (stopped.status === "incomplete")
return new ProviderCleanupError(
this.identity.id,
[
pending.process.launch.ownership?.runId ?? "owned-decoder",
...(pending.root === undefined ? [] : [pending.root.path]),
// The retained group may still be live; name it as OwnedCommand does.
...(pending.process.launch.ownership === undefined
? []
: [
`pid:${pending.process.launch.ownership.leaderPid}`,
`process-group:${pending.process.launch.ownership.processGroupId}`,
]),
],
ownership === undefined
? [
"owned-decoder",
...(pending.root === undefined ? [] : [pending.root.path]),
]
: ownedCommandResources(ownership, pending.root?.path),
{
reason: stopped.reason,
captured_output: {

19
src/process/OwnedCommand.ts
View File

@ -9,6 +9,7 @@ import {
} from "./ProviderProcess.js";
import {
cleanupOwnedProcessGroup,
type OwnedProcessGroup,
type ProcessCleanupFailure,
} from "./ProcessOwnership.js";
import {
@ -34,6 +35,17 @@ export class OwnedCommandFailure extends Error {
}
}
/** Report recorded launch identities; these labels do not authorize signaling. */
export const ownedCommandResources = (
ownership: OwnedProcessGroup,
directory?: string,
): readonly string[] => [
ownership.runId,
...(directory === undefined ? [] : [directory]),
`pid:${ownership.leaderPid}`,
`process-group:${ownership.processGroupId}`,
];
/** Execute a single owned process with independent cleanup and bounded diagnostic retention. */
export const runOwnedCommand = async (
spawn: OwnedProviderProcessSpawnOptions,
@ -208,12 +220,7 @@ const collectOwnedCommand = async (
? {}
: { cleanupResult: stopped.cleanupResult }),
},
[
spawn.runId,
...(spawn.cwd === undefined ? [] : [spawn.cwd]),
`pid:${launched.ownership.leaderPid}`,
`process-group:${launched.ownership.processGroupId}`,
],
ownedCommandResources(launched.ownership, spawn.cwd),
supervisor,
);
if (failure !== undefined)

111
tests/acceptance/artifactClassification.test.ts Normal file
View File

@ -0,0 +1,111 @@
import { createHash } from "node:crypto";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js";
import { expect } from "vitest";
import {
CPU,
fatImage,
machoImage,
} from "../../src/artifacts/apple/MachoImage.fixture.js";
import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js";
import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js";
import { parseEvidence } from "../../src/domain/evidence.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { cliTest } from "../support/cli/cliFixture.js";
cliTest.for(["jar", "zip"])(
"distinguishes JVM classes from universal Mach-O files in %s inventory",
async (extension, { cli }) => {
const root = await createTestTempDirectory("rea-class-inventory-");
const path = join(root, `mixed.${extension}`);
// The large class can hold the entire 65-entry table suggested by its
// version bytes. Checking only the table length does not distinguish it.
const classes = [
["Hello.class", javaClass()],
["Big.class", javaClass("x".repeat(4_000))],
["renamed.bin", javaClass()],
] as const;
const writer = new ZipWriter(new Uint8ArrayWriter());
for (const [name, bytes] of classes)
await writer.add(name, new Uint8ArrayReader(bytes));
const nativePaths: string[] = [];
for (const wide of [false, true]) {
for (const littleEndian of [false, true]) {
const name = `fat${wide ? 64 : 32}-${littleEndian ? "le" : "be"}`;
nativePaths.push(name);
await writer.add(
name,
new Uint8ArrayReader(
fatImage(
[{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }],
wide,
littleEndian,
),
),
);
}
}
await writeFile(path, await writer.close());
const reply = await cli.run({
arguments: ["inspect-artifact", path, "--json"],
});
expect(reply.exitCode, reply.stdout + reply.stderr).toBe(0);
const inspection = artifactInspectionResultSchema.parse(
parseEvidence(reply.json).normalized_result,
);
const inventory = artifactInventoryResultSchema.parse(
inspection.substeps[0]?.evidence.normalized_result,
);
for (const name of nativePaths) {
expect(inventory.occurrences).toContainEqual(
expect.objectContaining({
logical_path: name,
artifact_kind: "executable",
artifact_format: "mach-o-universal",
limitations: [],
}),
);
}
for (const [name, bytes] of classes) {
const occurrence = inventory.occurrences.find(
(item) => item.logical_path === name,
);
expect(occurrence).toBeDefined();
expect.soft(occurrence?.artifact_kind, name).not.toBe("executable");
expect.soft(occurrence?.artifact_format, name).not.toMatch(/^mach-o/u);
const sha256 = createHash("sha256").update(bytes).digest("hex");
const node = inventory.nodes.find((item) => item.sha256 === sha256);
expect(node).toBeDefined();
expect.soft(node?.format, name).not.toMatch(/^mach-o/u);
}
},
);
/** Minimal valid Java 21 class, with an optional unused UTF-8 constant. */
const javaClass = (extra?: string): Buffer => {
const utf8 = (value: string): Buffer => {
const text = Buffer.from(value);
const header = Buffer.alloc(3);
header[0] = 1;
header.writeUInt16BE(text.length, 1);
return Buffer.concat([header, text]);
};
const constants = [
Buffer.from([7, 0, 2]),
utf8("Hello"),
Buffer.from([7, 0, 4]),
utf8("java/lang/Object"),
...(extra === undefined ? [] : [utf8(extra)]),
];
const header = Buffer.from("cafebabe000000410000", "hex");
header.writeUInt16BE(constants.length + 1, 8);
return Buffer.concat([
header,
...constants,
Buffer.from("0021000100030000000000000000", "hex"),
]);
};

32
tests/boundary/cli/jsonOutput.test.ts
View File

@ -561,3 +561,35 @@ describe("CLI result output surface", () => {
});
}, 60_000);
});
describe("streamed CLI filtered selections", () => {
it.each([
["raw_result", "json", "null\n"],
["raw_result", "jsonl", "null\n"],
["nested.value", "json", '{"nested":{"value":null}}\n'],
["missing", "json", "{}\n"],
["count", "json", "0\n"],
["enabled", "json", "false\n"],
] as const)(
"keeps the selected value of --filter-output %s in %s",
async (filter, format, expected) => {
const chunks: Buffer[] = [];
const destination = new Writable({
write(chunk: Buffer, _encoding, callback) {
chunks.push(chunk);
callback();
},
});
const output = createCliResultOutput(
["--format", format, "--filter-output", filter],
destination,
);
if (output === undefined) throw new Error("Missing JSON output surface");
await output.write(
{ raw_result: null, nested: { value: null }, count: 0, enabled: false },
{ command: "fixture", duration: "1ms", format },
);
expect(Buffer.concat(chunks).toString("utf8")).toBe(expected);
},
);
});

4
tests/boundary/filesystem/ciChangeScope.test.ts
View File

@ -525,6 +525,10 @@ it.each([
"src/windows/Example.ts",
{ code: true, runtime: true, package: true, apple: false, evm: false },
],
[
"src/domain/javascript/javascriptRuntimeStaticCandidates.ts",
{ code: true, runtime: true, package: true, javascript_recovery: true },
],
["install.sh", { code: true, runtime: true, package: true, evm: false }],
[
".agents/skills/reverse-engineer-anything/SKILL.md",

32
tests/boundary/filesystem/hermesClientConfiguration.test.ts
View File

@ -237,6 +237,38 @@ mcp_servers:
expect(await readFile(client.configPath, "utf8")).toBe(original);
});
it("keeps unrelated numbers as written, including Discord IDs", async () => {
const { home, client } = await hermesClient();
// Hermes reads these exactly: two IDs one apart, an octal mode, a float.
const original = `discord:
free_response_channels:
- 1187654321098765432
channel_prompts:
1187654321098765433: Build questions.
1187654321098765434: Release questions.
mode: 0600
min_score: 0.0000001
mcp_servers:
other:
command: other-server
`;
await writeFile(client.configPath, original);
expect(
(await configureClientConfiguration(client, {}, command)).status,
).toBe("configured");
const text = await readFile(client.configPath, "utf8");
expect(text.startsWith(original)).toBe(true);
expect(parseYaml(text, { intAsBigInt: true })).toMatchObject({
mcp_servers: {
rea: { command: command[0], args: command.slice(1), enabled: true },
},
});
expect((await systemUninstallHost(home).removeClient(client)).status).toBe(
"removed",
);
expect(await readFile(client.configPath, "utf8")).toBe(original);
});
it("creates config.yaml when Hermes has no configuration yet", async () => {
const { client } = await hermesClient();
expect(

154
tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts
View File

@ -444,73 +444,97 @@ describe("JavaScript application workflow error diagnostics", () => {
});
});
const RUNTIME_PATH_CASES = [
{
name: "case-sensitive POSIX input root",
root: null,
path: "/applications/Example.app/Contents/Resources/app/renderer.js",
status: "unknown",
basis: "none",
},
{
name: "case-sensitive POSIX mapping root",
root: "/runtime/App",
path: "/runtime/app/renderer.js",
status: "unknown",
basis: "none",
},
{
name: "matching POSIX mapping root",
root: "/runtime/App",
path: "/runtime/App/renderer.js",
status: "matched",
basis: "operator-file-mapping",
},
{
name: "case-insensitive Windows mapping root",
root: String.raw`C:\runtime\App`,
path: String.raw`c:\runtime\app\renderer.js`,
status: "matched",
basis: "operator-file-mapping",
},
] as const;
describe("JavaScript runtime reconciliation local paths", () => {
it.each([
{
name: "case-sensitive POSIX input root",
root: null,
path: "/applications/Example.app/Contents/Resources/app/renderer.js",
status: "unknown",
},
{
name: "case-sensitive POSIX mapping root",
root: "/runtime/App",
path: "/runtime/app/renderer.js",
status: "unknown",
},
{
name: "matching POSIX mapping root",
root: "/runtime/App",
path: "/runtime/App/renderer.js",
status: "matched",
},
{
name: "case-insensitive Windows mapping root",
root: String.raw`C:\runtime\App`,
path: String.raw`c:\runtime\app\renderer.js`,
status: "matched",
},
])("uses $name on every host", ({ root, path, status }) => {
const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE;
const original = base.runtime_observations[0]!;
const inspection = electronPageInspectionSchema.parse(
original.normalized_result,
);
const runtime = createEvidence(undefined, original.provider, {
predicateType: original.predicate_type,
operation: original.operation,
parameters: { ...original.parameters, include_script_sources: false },
result: {
...inspection,
scripts: {
...inspection.scripts,
items: inspection.scripts.items.map((script) => ({
...script,
file_path: path,
source: { included: false, reason: "not-requested" },
})),
it.each(RUNTIME_PATH_CASES)(
"uses $name on every host",
({ root, path, status, basis }) => {
const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE;
const original = base.runtime_observations[0]!;
const inspection = electronPageInspectionSchema.parse(
original.normalized_result,
);
// Keep target and script coordinates consistent so an unrelated target
// cannot activate the static layer after its script mapping was rejected.
const runtime = createEvidence(undefined, original.provider, {
predicateType: original.predicate_type,
operation: original.operation,
parameters: { ...original.parameters, include_script_sources: false },
result: {
...inspection,
target: { ...inspection.target, file_path: path },
scripts: {
...inspection.scripts,
items: inspection.scripts.items.map((script) => ({
...script,
file_path: path,
source: { included: false, reason: "not-requested" },
})),
},
},
},
confidence: original.confidence,
authority: original.authority,
});
const reconciled = reconcileJavaScriptRuntimeEvidence({
static_layers: base.static_layers.map((layer) => ({
...layer,
runtime_mappings: root === null ? [] : [{ kind: "file-root", root }],
})),
runtime_observations: [runtime],
});
if (!reconciled.ok) throw reconciled.error;
const result = javascriptRuntimeReconciliationResultSchema.parse(
reconciled.value.normalized_result,
);
expect(
result.reconciliations.find(
({ entity_kind }) => entity_kind === "script",
),
).toMatchObject({ status });
});
confidence: original.confidence,
authority: original.authority,
});
const reconciled = reconcileJavaScriptRuntimeEvidence({
static_layers: base.static_layers.map((layer) => ({
...layer,
runtime_mappings: root === null ? [] : [{ kind: "file-root", root }],
})),
runtime_observations: [runtime],
});
if (!reconciled.ok) throw reconciled.error;
const result = javascriptRuntimeReconciliationResultSchema.parse(
reconciled.value.normalized_result,
);
expect(
result.reconciliations.find(
({ entity_kind }) => entity_kind === "script",
),
).toMatchObject({ status, basis });
expect(
result.static_load_states.find(
({ kind }) => kind === "javascript-asset",
),
).toMatchObject(
status === "matched"
? { status: "loaded", reason: "runtime-script-correspondence" }
: { status: "unknown", reason: "layer-outside-runtime-scope" },
);
expect(result.static_layers[0]?.runtime_mappings).toEqual(
root === null ? [] : [{ kind: "file-root", root, artifact_prefix: "" }],
);
},
);
it("resolves relative file mapping roots before deriving result Evidence", () => {
const absoluteRoot = resolve(process.cwd(), "runtime-cache-fixture");

8
tests/boundary/process/ownedCommand.test.ts
View File

@ -257,9 +257,11 @@ it.skipIf(process.platform === "win32").each(["cancelled", "timeout"] as const)(
failures: [expect.objectContaining({ reason: "run-token-mismatch" })],
});
expect(failure.cleanupResult).toEqual(cleanup);
expect(failure.resources).toEqual(
expect.arrayContaining([request.runId]),
);
expect(failure.resources).toEqual([
request.runId,
`pid:${launched.ownership.leaderPid}`,
`process-group:${launched.ownership.processGroupId}`,
]);
// An unresolved member does not grant authority to signal the shared group.
const acquiredPid = launched.ownership.leaderPid;
expect(() => process.kill(acquiredPid, 0)).not.toThrow();

23
tests/conformance/swift/string-literals.swift Normal file
View File

@ -0,0 +1,23 @@
@inline(never)
func consume(_ value: String) { print(value) }
@inline(never)
func smallLiteral() { consume("SleepDisabled") }
@inline(never)
func largeLiteral() { consume("The power setting could not be changed.") }
@inline(never)
func unicodeLiterals() {
consume("café")
consume("Résumé of the naïve décor, written twice")
}
@inline(never)
func emptyAndControlLiterals() {
consume("")
consume("tab\there")
}
@inline(never)
func returnedLiteral() -> String { "Holding awake" }
smallLiteral()
largeLiteral()
unicodeLiterals()
emptyAndControlLiterals()
print(returnedLiteral(), CommandLine.arguments.count)

26
tests/process-global/pwntoolsStageRetirement.test.ts
View File

@ -18,7 +18,10 @@ import {
PWNTOOLS_MEMORY_FAILURE_EXIT,
PWNTOOLS_PROVIDER_IDENTITY,
} from "../../src/native/pwntools/PwntoolsRelease.js";
import { cleanupOwnedProcessGroup } from "../../src/process/ProcessOwnership.js";
import {
cleanupOwnedProcessGroup,
type OwnedProcessGroup,
} from "../../src/process/ProcessOwnership.js";
import { prepareProcessOwnershipInspection } from "../../src/process/ProcessOwnershipObservation.js";
import { spawnOwnedProviderProcess } from "../../src/process/ProviderProcess.js";
import { PrivateRuntimeRoot } from "../../src/process/PrivateRuntimeRoot.js";
@ -232,12 +235,13 @@ it.runIf(supportedHost).each([
const { path, root, environment, basic } = await fixture();
let launches = 0;
let cleanupAllowed = false;
let retainedOwnership: OwnedProcessGroup | undefined;
const read = vi.spyOn(artifacts, "readStableArtifact");
const provider = new PwntoolsRecordedCrashProvider(
environment,
async (spawn) => {
if (stage === "debugger" && ++launches === 1) return basic(spawn);
return changingWorker(
const launched = await changingWorker(
spawn,
join(spawn.cwd ?? "", "resource.failure"),
{
@ -247,6 +251,8 @@ it.runIf(supportedHost).each([
},
() => cleanupAllowed,
);
retainedOwnership = launched.ownership;
return launched;
},
async () => root,
);
@ -261,14 +267,14 @@ it.runIf(supportedHost).each([
if (result.ok) throw new Error("Expected unresolved process ownership");
const projected = projectAnalysisError(result.error);
expect(projected.code).toBe("cleanup_incomplete");
expect(projected).toMatchObject({
details: {
resources: expect.arrayContaining([
expect.stringMatching(/^pid:\d+$/u),
expect.stringMatching(/^process-group:\d+$/u),
]),
},
});
if (retainedOwnership === undefined)
throw new Error("Expected retained process ownership");
expect(projected.details?.resources).toEqual([
retainedOwnership.runId,
root.path,
`pid:${retainedOwnership.leaderPid}`,
`process-group:${retainedOwnership.processGroupId}`,
]);
if (stage === "debugger")
expect(projected).toMatchObject({
details: {