diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0516da85..d2cc59493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -597,6 +597,7 @@ jobs: tests/boundary/filesystem/setupUnavailableProvider.test.ts tests/boundary/filesystem/windowsNpxRegistration.test.ts tests/acceptance/setup/homeDirectoryFallback.test.ts + tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts tests/boundary/filesystem/webScriptManifestPaths.test.ts tests/boundary/process/ProcessEvidence.host.test.ts tests/acceptance/webModuleCli.test.ts diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 02fc0f148..70132eeae 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -13,6 +13,8 @@ import java.net.UnixDomainSocketAddress; import java.nio.channels.Channels; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; @@ -22,7 +24,9 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermissions; import java.security.MessageDigest; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; import java.util.IdentityHashMap; @@ -82,6 +86,7 @@ import ghidra.program.model.address.SegmentedAddress; import ghidra.program.model.reloc.Relocation; import java.io.ByteArrayOutputStream; import ghidra.program.model.symbol.Namespace; +import ghidra.program.model.symbol.FlowType; import ghidra.program.model.symbol.RefType; import ghidra.program.model.symbol.Reference; import ghidra.program.model.symbol.ReferenceIterator; @@ -100,6 +105,9 @@ import ghidra.program.model.pcode.PcodeBlock; import ghidra.program.model.pcode.PcodeBlockBasic; import ghidra.program.model.pcode.Varnode; import ghidra.program.model.pcode.VarnodeAST; +import ghidra.program.util.ContextEvaluatorAdapter; +import ghidra.program.util.SymbolicPropogator; +import ghidra.program.util.VarnodeContext; public final class ReaGhidraBridge extends HeadlessScript { private static final int MAX_HIGH_PCODE_OPS = 3000; @@ -1319,6 +1327,13 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("outgoing_references", outgoing); result.add("referenced_strings", referencedStrings); result.add("referenced_names", referencedNames); + boolean swift = swiftProgram(); + boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:"); + SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null; + if (swiftLiterals != null) { + result.add("swift_string_literals", swiftLiterals.literals()); + result.add("swift_string_literal_omissions", swiftLiterals.omissions()); + } result.add("basic_blocks", blocks); result.add( "native_api", @@ -1338,6 +1353,20 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add( "Synthetic Ghidra entry-point references without actionable memory sources are omitted." ); + if (swift && aarch64) { + limitations.add( + "Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by conditional selects, passed through unresolved jumps, or built at runtime are not reported." + ); + if (swiftLiterals.omissions().size() > 0) { + limitations.add( + "Swift String literal records listed in swift_string_literal_omissions could not be retained or decoded within this dossier's remaining heap budget; later symbolic flows remain unexamined." + ); + } + } else if (swift) { + limitations.add( + "Swift String literal decoding is implemented for AArch64 register pairs only; no Swift literals were decoded for this architecture." + ); + } List boundaryLimitations = noReturnCallLimitations(scan.instructions); for (String limitation : boundaryLimitations) { limitations.add(limitation); @@ -3391,6 +3420,349 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } + private boolean swiftProgram() { + for (MemoryBlock block : currentProgram.getMemory().getBlocks()) { + String name = block.getName(); + if (name.startsWith("__swift5_") || name.startsWith("swift5_")) return true; + } + return false; + } + + /** + * Swift passes a String as two words: small literals carry their bytes in the words and + * immortal large literals carry their count and a biased pointer to the bytes. Ghidra keeps + * neither as a data reference, so decode constant pairs where Swift consumes them: argument + * and return registers at calls, tail calls and returns, and both registers of one STP. + * The canonical empty string is a small literal with a zero count. + */ + private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception { + Register[] registers = new Register[31]; + for (int index = 0; index < registers.length; index++) { + registers[index] = currentProgram.getRegister("x" + index); + } + TreeMap observed = new TreeMap<>(); + TreeMap omitted = new TreeMap<>(); + SwiftLiteralBudget budget = new SwiftLiteralBudget(); + Map selectedValues = new HashMap<>(); + SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false); + propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() { + @Override + public boolean evaluateContextBefore(VarnodeContext context, Instruction instruction) { + // Returning true ends only the current flow, so every queued flow checks the shared stop. + if (budget.stopped()) return true; + for (Register[] pair : swiftStringPairs(function, instruction, registers)) { + Long word = registerConstant(context, pair[0]); + Long object = registerConstant(context, pair[1]); + if (word == null || object == null) continue; + SwiftLiteralLayout layout = swiftLiteralLayout(word, object); + if (layout == null) continue; + // A selected word can combine with a path-dependent ordinary move. Cache the + // provenance question per site/register, charging retained cache state to the + // same dossier budget before allocating it. + boolean selected = false; + boolean provenanceOverCapacity = false; + for (Register register : pair) { + String provenanceKey = canonicalAddress(instruction.getAddress()) + "\u0000" + + register.getBaseRegister().getName(); + Boolean value = selectedValues.get(provenanceKey); + if (value == null) { + if (!budget.takeState()) { + provenanceOverCapacity = true; + break; + } + value = selectedValue(function, instruction, register); + selectedValues.put(provenanceKey, value); + } + if (value) { + selected = true; + break; + } + } + if (selected) continue; + // Forks can reach one site with different constants, so the words are part of the key. + String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + + "\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) + + "\u0000" + Long.toUnsignedString(object, 16); + if (observed.containsKey(key) || omitted.containsKey(key)) continue; + // Every record draws on the budget; the first one that does not fit is kept as an + // omission and decoding stops, so later sites are never examined. + String value = layout.smallValue(); + boolean overCapacity = provenanceOverCapacity || !budget.take(layout.byteLength()); + if (!overCapacity && value == null) { + try { + value = swiftLargeLiteralText(layout); + } catch (OutOfMemoryError exhausted) { + overCapacity = true; + } + } + if (value == null && !overCapacity) continue; + JsonObject item = overCapacity + ? swiftStringOmission(layout, word, object) + : swiftStringItem(value, layout, word, object); + item.addProperty("source_address", canonicalAddress(instruction.getAddress())); + JsonArray pairRegisters = new JsonArray(); + pairRegisters.add(pair[0].getName()); + pairRegisters.add(pair[1].getName()); + item.add("registers", pairRegisters); + (overCapacity ? omitted : observed).put(key, item); + if (overCapacity) { + budget.stop(); + return true; + } + } + return false; + } + }, false, monitor); + JsonArray literals = new JsonArray(); + observed.values().forEach(literals::add); + JsonArray omissions = new JsonArray(); + omitted.values().forEach(omissions::add); + return new SwiftStringLiteralDecode(literals, omissions); + } + + private static List swiftStringPairs( + Function function, + Instruction instruction, + Register[] registers + ) { + List pairs = new ArrayList<>(); + if (instruction.getMnemonicString().equals("stp")) { + Register first = instruction.getRegister(0); + Register second = instruction.getRegister(1); + if (first != null && second != null && first.getBitLength() == 64 && second.getBitLength() == 64) { + pairs.add(new Register[] {first, second}); + } + return pairs; + } + FlowType flow = instruction.getFlowType(); + boolean returns = false; + for (PcodeOp operation : instruction.getPcode()) { + if (operation.getOpcode() == PcodeOp.RETURN) returns = true; + } + // A jump is a proven tail call only when it has resolved targets and all leave the + // body. A targetless computed jump can still dispatch inside the function. + Address[] targets = instruction.getFlows(); + boolean tailCall = flow.isJump() && !flow.isConditional() && targets.length > 0 && + Arrays.stream(targets).allMatch(target -> !function.getBody().contains(target)); + if (!flow.isCall() && !returns && !tailCall) return pairs; + // A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned. + int lastFirstRegister = returns ? 0 : 6; + for (int index = 0; index <= lastFirstRegister; index++) { + pairs.add(new Register[] {registers[index], registers[index + 1]}); + } + return pairs; + } + + private static Long registerConstant(VarnodeContext context, Register register) { + Varnode value = context.getRegisterVarnodeValue(register); + return value != null && context.isConstant(value) ? value.getOffset() : null; + } + + // Search every predecessor path for the reaching definitions, following register copies, + // since a select copied into an argument register or defined before a branch forks just as + // independently as one written there directly. + private boolean selectedValue(Function function, Instruction use, Register register) { + ArrayDeque instructions = new ArrayDeque<>(); + ArrayDeque targets = new ArrayDeque<>(); + Set visited = new HashSet<>(); + for (Instruction predecessor : predecessors(function, use)) { + instructions.push(predecessor); + targets.push(register.getBaseRegister()); + } + while (!instructions.isEmpty()) { + Instruction instruction = instructions.pop(); + Register target = targets.pop(); + if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue; + boolean defines = false; + for (Object result : instruction.getResultObjects()) { + if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true; + } + Register next = target; + if (defines) { + PcodeOp[] operations = instruction.getPcode(); + for (PcodeOp operation : operations) { + if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true; + } + Register source = copiedRegister(operations); + if (source == null) continue; + next = source.getBaseRegister(); + } + for (Instruction predecessor : predecessors(function, instruction)) { + instructions.push(predecessor); + targets.push(next); + } + } + return false; + } + + private List predecessors(Function function, Instruction instruction) { + List result = new ArrayList<>(); + Address from = instruction.getFallFrom(); + if (from != null) { + Instruction previous = currentProgram.getListing().getInstructionAt(from); + if (previous != null) result.add(previous); + } + for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) { + if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue; + if (!function.getBody().contains(reference.getFromAddress())) continue; + Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress()); + if (source != null) result.add(source); + } + return result; + } + + private Register copiedRegister(PcodeOp[] operations) { + if (operations.length != 1) return null; + int opcode = operations[0].getOpcode(); + Varnode input = operations[0].getInput(0); + if ((opcode != PcodeOp.COPY && opcode != PcodeOp.INT_ZEXT) || input == null || !input.isRegister()) return null; + return currentProgram.getRegister(input); + } + + /** Validate the ABI layout of a pair; a small literal is decoded here, a large one only located. */ + private SwiftLiteralLayout swiftLiteralLayout(long word, long object) { + int discriminator = (int) (object >>> 60); + if (discriminator == 0xE || discriminator == 0xA) { + int count = (int) ((object >>> 56) & 0xF); + byte[] bytes = new byte[15]; + for (int index = 0; index < 8; index++) bytes[index] = (byte) (word >>> (8 * index)); + for (int index = 0; index < 7; index++) bytes[8 + index] = (byte) (object >>> (8 * index)); + for (int index = count; index < bytes.length; index++) { + if (bytes[index] != 0) return null; + } + // Count zero is the canonical empty string (object 0xE000000000000000, word 0). + String value = swiftStringText(bytes, count, discriminator == 0xE); + return value == null ? null : new SwiftLiteralLayout("small", null, count, true, value); + } + int flags = (int) (word >>> 60); + long count = word & 0xFFFFFFFFFFFFL; + // Immortal literals: ASCII and NFC or neither, tail allocated, with a pointer biased by 32. + if ((object >>> 56) != 0x80 || (flags != 0xD && flags != 0x1) || ((word >>> 48) & 0xFFF) != 0) { + return null; + } + try { + Address address = currentProgram.getAddressFactory().getDefaultAddressSpace() + .getAddress((object & 0x00FFFFFFFFFFFFFFL) + 0x20); + MemoryBlock block = currentProgram.getMemory().getBlock(address); + // The bytes and their terminator must lie in one initialized, non-executable block, + // and Java arrays bound one read. + if (block == null || !block.isInitialized() || block.isExecute() || + count >= block.getEnd().subtract(address) + 1 || count >= Integer.MAX_VALUE) { + return null; + } + return new SwiftLiteralLayout("large", address, count, flags == 0xD, null); + } catch (Exception unaddressable) { + return null; + } + } + + private String swiftLargeLiteralText(SwiftLiteralLayout layout) { + int count = (int) layout.byteLength(); + byte[] bytes = new byte[count + 1]; + try { + if (currentProgram.getMemory().getBytes(layout.address(), bytes) != bytes.length) return null; + } catch (Exception unreadable) { + return null; + } + return bytes[count] == 0 ? swiftStringText(bytes, count, layout.ascii()) : null; + } + + /** + * Every literal record of one dossier draws on half of the heap that was free when decoding + * began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up + * to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer + * (up to twice its length), again in the returned String, and once more as UTF-8 on the + * socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words + * and their serialized form, so many empty or small literals are bounded too. Retained + * provenance-cache entries draw on that same fixed-state allowance. + */ + private static final class SwiftLiteralBudget { + private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6; + private static final long BYTES_PER_RECORD = 4096; + private long remaining; + private boolean stopped; + + SwiftLiteralBudget() { + Runtime runtime = Runtime.getRuntime(); + remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2); + } + + boolean takeState() { + if (remaining < BYTES_PER_RECORD) return false; + remaining -= BYTES_PER_RECORD; + return true; + } + + boolean take(long bytes) { + if (remaining < BYTES_PER_RECORD || + bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) { + return false; + } + remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE; + return true; + } + + void stop() { + stopped = true; + } + + boolean stopped() { + return stopped; + } + } + + // Swift String values may contain any scalar, including controls; JSON escapes them. + private static String swiftStringText(byte[] bytes, int length, boolean ascii) { + if (length < 0 || length > bytes.length) return null; + for (int index = 0; index < length; index++) { + if (ascii && bytes[index] < 0) return null; + } + try { + return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes, 0, length)).toString(); + } catch (CharacterCodingException invalid) { + return null; + } + } + + private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { + JsonObject item = new JsonObject(); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.addProperty("byte_length", layout.byteLength()); + item.addProperty("reason", "capacity"); + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonObject swiftStringItem( + String value, + SwiftLiteralLayout layout, + long word, + long object + ) { + JsonObject item = new JsonObject(); + item.addProperty("value", value); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonArray swiftStringWords(long word, long object) { + JsonArray words = new JsonArray(); + words.add("0x" + Long.toUnsignedString(word, 16)); + words.add("0x" + Long.toUnsignedString(object, 16)); + return words; + } + private static String exclusiveAddress(Address inclusive) { Address next = inclusive.next(); if (next != null && next.getAddressSpace().equals(inclusive.getAddressSpace())) { @@ -3930,6 +4302,14 @@ public final class ReaGhidraBridge extends HeadlessScript { private record AddressNameRequest(String inputAddress, Address address, String name) {} private record FunctionEntry(Function function, InventoryItem item) {} private record FunctionAnalysisObservation(JsonObject dossier, List limitations) {} + private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {} + private record SwiftLiteralLayout( + String representation, + Address address, + long byteLength, + boolean ascii, + String smallValue + ) {} private record InstructionScan(List instructions, boolean truncated) {} private record SessionDescriptor( String transport, diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 2f73f4261..dca1f994b 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -80,6 +80,27 @@ google-chrome \ REA does not launch, own, or terminate this browser. Use a dedicated profile and stop it yourself when the investigation is complete. Do not expose the debugging port on a non-loopback interface. +Other Chromium-family browsers, such as Chromium or Brave, work the same way +with the same two flags. On macOS, run the executable inside the application +bundle, for example +`"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"` or +`"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"`. Fedora installs +Chromium as `chromium-browser`. + +REA needs the browser's HTTP discovery endpoint on a loopback port. A browser +driven over `--remote-debugging-pipe` or a private channel, for example by an +automation SDK, exposes no such endpoint, so `list_browser_targets` has nothing +to attach to. Connection failures report `endpoint_unreachable`; a reachable +HTTP service that returns a non-200 or non-JSON discovery response reports +`invalid_endpoint_response`. +Reproduce that flow in a Chromium-family browser started with +`--remote-debugging-port`. + +In a minimal Linux container, headed Chromium under Xvfb can keep running while +its DevTools port never opens. Install the GLES, EGL, and Mesa DRI libraries +(on Fedora: `libglvnd-gles`, `mesa-libEGL`, and `mesa-dri-drivers`), then +restart the browser. + On Linux, a reachable but locked desktop keyring can stall Chrome before its first navigation. For a dedicated investigation profile, add `--password-store=basic` to bypass that keyring. Empty or placeholder frame URLs @@ -207,7 +228,7 @@ commands fail closed. A correlated reply's session ID must match the command's selected session. An optional method is unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. -Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. +Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Scripts are different: scripts the page parsed before REA attached are still reported under `scripts`. A page that loaded before attach can therefore list its scripts while `network.requests` lacks the requests that fetched them. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. diff --git a/docs/client-configuration.md b/docs/client-configuration.md index 00e83e179..e72ca191f 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -209,8 +209,15 @@ replaces) the REA entry in the home patch `$DSH_HOME/cordis.patch.yml` patches and comments. `rea doctor --client dsh` checks that same entry id. `rea uninstall` removes it by scanning supported clients; uninstall takes no `--client` flag. The home patch applies to every Harness profile, so the -registration is written once and no per-profile edit is needed. The written -entry is: +registration is written once and no per-profile edit is needed. + +Setup, doctor and uninstall resolve `DSH_HOME` and `DSH_AGENTS_HOME` as Harness +does: unset, empty or whitespace-only values use `~/.dsh` and `~/.agents`, +respectively. Bare `~`, `~/` and `~\` expand to the user home; other relative +values resolve against the command's working directory. Nonblank values retain +their original spaces, and the agents root is independent of the Harness home. +Run REA and Harness from the same working directory when using relative overrides. +The written entry is: ```yaml - insert: diff --git a/docs/ida-provider.md b/docs/ida-provider.md index b22a35cce..fbf67c8a2 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -88,6 +88,8 @@ REA copies the admitted binary into a private temporary directory, checks its di One REA session owns one database. Stdio registrations default `IDA_MCP_MAX_WORKERS` to `1`; the explicit upstream `--max-workers 1` also bounds a supervisor. Database size and engine memory depend on the target and upstream; REA does not impose an engine memory limit. The original binary is never executed or overwritten. Optional `workspaceRoot` selects an existing absolute temporary-directory parent on the REA host. Ensure it has space for the input copy and IDA working files. +When a completed `idb_open` response reports success or is otherwise ambiguous, but no active owned private worker can be verified, REA retains the private workspace even if `idb_list` appears empty. A later close can retry verification if the original owned worker becomes discoverable. A completed explicit `idb_open` rejection retains the existing no-worker cleanup path. This is a local admission/ownership safeguard and **does not** provide a process-termination receipt or change the legacy reaped-worker protocol. + Headless cleanup uses the upstream database lifecycle. Abruptly terminating REA, its proxy, or the supervisor does not establish worker release. Prefer `close_binary` for MCP sessions; after an interrupted process, inspect upstream session inventory and retained workspace diagnostics before removing working files. This adapter does not claim Windows Job Object containment for the upstream detached workers. ## Connect an agent once @@ -126,6 +128,8 @@ For guided registration, set `REA_IDA_MCP_CONFIG` before `rea setup --client Promise, +): Promise; diff --git a/scripts/lib/ida-feature-trace.mjs b/scripts/lib/ida-feature-trace.mjs new file mode 100644 index 000000000..3c4778d25 --- /dev/null +++ b/scripts/lib/ida-feature-trace.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; + +const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]); + +/** + * Assert the `trace_feature` result of the real IDA lane for one selected + * procedure. The query is a substring search, so the trace may also match other + * procedures and strings; each reference is checked against its own matched + * target, and a source outside every function (a data reference) is valid. + */ +export function assertFeatureTrace(trace, procedure) { + assert.ok( + trace.matches.some( + ({ type, address }) => + type === "procedure" && address === procedure.address, + ), + "The trace omitted the selected procedure's match.", + ); + assert.equal(typeof trace.truncated, "boolean"); + const matched = new Set(trace.matches.map(({ address }) => address)); + for (const reference of trace.references) { + assert.ok( + matched.has(reference.target_address), + `A reference targets ${reference.target_address}, which is not a match.`, + ); + const { containing_procedure: containing } = reference; + if (containing.found) + assert.equal(typeof containing.procedure.address, "string"); + else assert.ok(NEGATIVE_REASONS.has(containing.reason)); + } +} + +/** + * Find a string that IDA maps outside every function. A string inside a + * function chunk (inline data in code) is valid and skipped, so the strings are + * tried in order. Any other answer must be "not in a procedure": IDA lists only + * mapped strings, so an unmapped answer is a containment defect and fails the + * lane. Returns the address, or undefined when every string is inside a function. + */ +export async function findMappedNonFunction(strings, resolveAt) { + for (const { address } of strings) { + const resolved = await resolveAt(address); + if (resolved.found) continue; + assert.equal( + resolved.reason, + "not_in_procedure", + `The listed string at ${address} resolved as ${resolved.reason}.`, + ); + return address; + } + return undefined; +} diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 0593408b0..0aa58e2f0 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -45,8 +45,13 @@ const transport = new StdioClientTransport({ const source = fileURLToPath( new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url), ); +const literalSource = fileURLToPath( + new URL("../tests/conformance/swift/string-literals.swift", import.meta.url), +); const binary = join(root, "swift-inventory"); +const literalBinary = join(root, "swift-literals"); const reports = []; +let literalReport; try { await execute("swiftc", ["-O", "-g", source, "-o", binary], { timeout: 60_000, @@ -54,6 +59,19 @@ try { await mkdir(join(root, "copied")); const copied = join(root, "copied", "swift-inventory"); await copyFile(binary, copied); + // Decoding covers AArch64 register pairs, so build that slice on any Mac host. + await execute( + "swiftc", + [ + "-O", + "-target", + "arm64-apple-macosx13.0", + literalSource, + "-o", + literalBinary, + ], + { timeout: 60_000 }, + ); await client.connect(transport); for (const [variant, path] of [ ["debug", binary], @@ -132,6 +150,7 @@ try { await call("close_binary", {}); } } + literalReport = await verifyStringLiterals(); } finally { await client.close(); await transport.close(); @@ -143,6 +162,7 @@ console.log( ok: true, provider: "ghidra", fixtures: reports, + swift_string_literals: literalReport, verifier_run: await completeVerifierRun(run), }, null, @@ -159,3 +179,51 @@ async function call(name, arguments_) { toolContract(name).outputSchema.parse(response.structuredContent); return requireMcpOperationResult(response, name); } + +// Literal text lives only in register constants, so it must come from the decoded pairs. +async function verifyStringLiterals() { + await call("open_binary", { path: literalBinary, provider_id: "ghidra" }); + try { + const procedures = await call("list_procedures", {}); + const expected = { + smallLiteral: [["SleepDisabled", "small"]], + largeLiteral: [["The power setting could not be changed.", "large"]], + unicodeLiterals: [ + ["café", "small"], + ["Résumé of the naïve décor, written twice", "large"], + ], + emptyAndControlLiterals: [ + ["", "small"], + ["tab\there", "small"], + ], + returnedLiteral: [["Holding awake", "small"]], + }; + const decoded = {}; + for (const [name, literals] of Object.entries(expected)) { + const procedure = procedures.find(({ value }) => value.includes(name)); + assert(procedure, `${name} is missing from the procedure inventory`); + const dossier = await call("analyze_function", { + procedure: procedure.address, + }); + const observed = dossier.swift_string_literals; + assert(Array.isArray(observed), `${name} recorded no Swift literals`); + assert.deepEqual(dossier.swift_string_literal_omissions, []); + assert.deepEqual( + observed.map(({ value, representation }) => [value, representation]), + literals, + ); + for (const literal of observed) { + assert.deepEqual(literal.registers, ["x0", "x1"]); + assert.equal( + literal.address === null, + literal.representation === "small", + `${literal.value} storage does not match its representation`, + ); + } + decoded[name] = observed.length; + } + return decoded; + } finally { + await call("close_binary", {}); + } +} diff --git a/scripts/verify-real-ida.mjs b/scripts/verify-real-ida.mjs index cedbf5076..bed0519a9 100644 --- a/scripts/verify-real-ida.mjs +++ b/scripts/verify-real-ida.mjs @@ -1,4 +1,8 @@ #!/usr/bin/env node +import { + assertFeatureTrace, + findMappedNonFunction, +} from "./lib/ida-feature-trace.mjs"; import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; @@ -6,7 +10,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { access, writeFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs, promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -261,6 +265,61 @@ const mcp = async (cliDossier) => { break; } } + if (config.mode === "headless") { + const documents = observe(await call("list_documents", {})); + assert.equal(documents.length, 1); + assert.equal(basename(documents[0]), basename(target)); + assert.ok( + !workspaces.has(dirname(documents[0])), + "list_documents reported the private workspace copy.", + ); + const resolveAt = async (address) => + observe(await call("resolve_containing_procedure", { address })); + const instructionAddresses = dossier.assembly + .map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1]) + .filter((address) => address !== undefined); + const queries = [ + dossier.procedure.address, + ...instructionAddresses.slice(1, 2), + ]; + for (const queried of queries) { + const containing = await resolveAt(queried); + assert.equal(containing.found, true); + assert.equal(containing.procedure.address, dossier.procedure.address); + assert.equal(containing.procedure.name, dossier.procedure.name); + } + const unmapped = await resolveAt("0xffffffffffffff00"); + assert.equal(unmapped.found, false); + assert.equal(unmapped.reason, "outside_segments"); + const strings = observe(await call("list_strings", {})); + const mappedString = await findMappedNonFunction(strings, resolveAt); + if (mappedString === undefined) + report.unverified.push( + "Mapped non-function containment: no string of the target is mapped outside every function.", + ); + const context = await client.callTool({ + name: "inspect_address_context", + arguments: { address: dossier.procedure.address }, + }); + assert.notEqual(context.isError, true); + const contextResult = context.structuredContent.result; + assert.equal(contextResult.document, documents[0]); + assert.equal(contextResult.procedure.state, "available"); + assert.equal( + contextResult.procedure.value.procedure.address, + dossier.procedure.address, + ); + const traced = await client.callTool({ + name: "trace_feature", + arguments: { query: dossier.procedure.name }, + }); + assert.notEqual(traced.isError, true); + const trace = traced.structuredContent.normalized_result; + assertFeatureTrace(trace, dossier.procedure); + report.checks.push( + "headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing", + ); + } if (values["expected-string"] === undefined) report.unverified.push( "String inventory/search correctness: no independent expected literal supplied.", diff --git a/scripts/verify-windows-native-artifact.mjs b/scripts/verify-windows-native-artifact.mjs index 1a6777322..4db66e0ee 100644 --- a/scripts/verify-windows-native-artifact.mjs +++ b/scripts/verify-windows-native-artifact.mjs @@ -27,9 +27,9 @@ if ( ); const offset = bytes.length >= 64 ? bytes.readUInt32LE(0x3c) : bytes.length; if ( - bytes.toString("ascii", 0, 2) !== "MZ" || + bytes.toString("latin1", 0, 2) !== "MZ" || offset > bytes.length - 26 || - bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" || + bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" || bytes.readUInt16LE(offset + 4) !== 0x8664 || bytes.readUInt16LE(offset + 24) !== 0x20b ) diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index 1f84681c9..804e8fffa 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -1,15 +1,8 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { - access, - mkdir, - mkdtemp, - readFile, - rm, - symlink, -} from "node:fs/promises"; +import * as fs from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import * as nodePath from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna import { parseEvidence } from "../../../dist/domain/evidence.js"; import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js"; -const exec = promisify(execFile); -const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); -const entrypoint = resolve( +const execFileAsync = promisify(execFile); +const { join, resolve, dirname } = nodePath; + +const CALL_TIMEOUT_MS = 150_000; +const CONNECT_TIMEOUT_MS = 30_000; +const SEVENZIP_TIMEOUT_MS = 10_000; +const MAX_STDOUT_BYTES = 16 * 1024 * 1024; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const ENTRYPOINT = resolve( process.env.REA_FIRMWARE_TEST_ENTRYPOINT ?? - join(repository, "scripts/rea.mjs"), + join(REPO_ROOT, "scripts/rea.mjs"), ); -if (process.platform !== "linux") - throw new Error( - "verify:firmware requires Linux; other hosts are not verified", - ); -for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { - const path = process.env[variable]; - if (path === undefined) - throw new Error( - `verify:firmware requires ${variable}; see docs/firmware-analysis.md`, - ); - await access(path); -} -try { - await exec("7z", ["i"], { timeout: 10_000 }); -} catch (cause) { - throw new Error( - "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", - { cause }, - ); -} -const fixtureRoot = resolve( +const FIXTURE_ROOT = resolve( process.env.REA_FIRMWARE_FIXTURE_ROOT ?? - join(repository, "_reference/firmware-integration/generated"), + join(REPO_ROOT, "_reference/firmware-integration/generated"), ); -await access(join(fixtureRoot, "oracle.json")).catch((cause) => { - throw new Error( - "verify:firmware requires generated fixtures; run npm run fixtures:firmware", - { cause }, - ); -}); + +// --------------------------------------------------------------------------- +// Prerequisite checks +// --------------------------------------------------------------------------- + +async function checkPrerequisites() { + if (process.platform !== "linux") { + throw new Error( + "verify:firmware requires Linux; other hosts are not verified", + ); + } + + for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { + const value = process.env[name]; + if (value === undefined) { + throw new Error( + `verify:firmware requires ${name}; see docs/firmware-analysis.md`, + ); + } + await fs.access(value); + } + + try { + await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS }); + } catch (cause) { + throw new Error( + "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", + { cause }, + ); + } + + try { + await fs.access(join(FIXTURE_ROOT, "oracle.json")); + } catch (cause) { + throw new Error( + "verify:firmware requires generated fixtures; run npm run fixtures:firmware", + { cause }, + ); + } +} + +await checkPrerequisites(); + +// --------------------------------------------------------------------------- +// Shared state +// --------------------------------------------------------------------------- + const oracle = JSON.parse( - await readFile(join(fixtureRoot, "oracle.json"), "utf8"), + await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"), ); -const path = join(fixtureRoot, "firmware.bin"); -assert.equal(await hashFirmwareFile(path), oracle.sha256); -const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-")); -const environment = Object.fromEntries( - Object.entries(process.env).filter(([, value]) => value !== undefined), +const firmwarePath = join(FIXTURE_ROOT, "firmware.bin"); +assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256); + +const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-")); +const baseEnv = Object.fromEntries( + Object.entries(process.env).filter(([, v]) => v !== undefined), ); -const client = new Client({ name: "rea-real-firmware-verifier", version: "1" }); -const cli = async (args, env = environment) => { - const response = await exec( +const mcpClient = new Client({ + name: "rea-real-firmware-verifier", + version: "1", +}); + +// --------------------------------------------------------------------------- +// Invocation helpers +// --------------------------------------------------------------------------- + +async function runCli(args, env = baseEnv) { + const { stdout } = await execFileAsync( process.execPath, - [entrypoint, ...args, "--format", "json"], + [ENTRYPOINT, ...args, "--format", "json"], { env, - cwd: repository, - timeout: 150_000, - maxBuffer: 16 * 1024 * 1024, + cwd: REPO_ROOT, + timeout: CALL_TIMEOUT_MS, + maxBuffer: MAX_STDOUT_BYTES, }, ); - return parseEvidence(JSON.parse(response.stdout)); -}; -const call = async (name, args) => { - const response = await client.callTool( + return parseEvidence(JSON.parse(stdout)); +} + +async function callTool(name, args) { + const result = await mcpClient.callTool( { name, arguments: args }, - { timeout: 150_000 }, + { timeout: CALL_TIMEOUT_MS }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - assert.ok(response.structuredContent !== undefined); - return response.structuredContent; -}; -const verifyRegions = (evidence) => { + assert.notEqual(result.isError, true, JSON.stringify(result)); + assert.ok(result.structuredContent !== undefined); + return result.structuredContent; +} + +// --------------------------------------------------------------------------- +// Result verifiers +// --------------------------------------------------------------------------- + +function verifyRegions(evidence) { assert.equal(evidence.subject.digest.sha256, oracle.sha256); - const result = firmwareResultSchemas.inspect_firmware_regions.parse( + + const parsed = firmwareResultSchemas.inspect_firmware_regions.parse( evidence.normalized_result, ); - const gzip = result.regions.find( - (region) => region.signature === "gzip" && region.offset === oracle.offset, + + const gzipRegion = parsed.regions.find( + (r) => r.signature === "gzip" && r.offset === oracle.offset, ); assert.ok( - gzip !== undefined, + gzipRegion !== undefined, "Binwalk must find the independently generated gzip offset", ); - assert.equal(gzip.reported_size, oracle.length); - assert.equal(gzip.size_basis, "provider_reported_validation_unknown"); - return result.regions.map( - ({ provider_id: _provider_id, ...region }) => region, - ); -}; -const verifyFiles = async (evidence) => { - const result = firmwareResultSchemas.extract_firmware.parse( + assert.equal(gzipRegion.reported_size, oracle.length); + assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown"); + + // Strip provider-specific ids so CLI and MCP results compare equal. + return parsed.regions.map((region) => { + const { provider_id: _ignored, ...rest } = region; + return rest; + }); +} + +async function verifyFiles(evidence) { + const parsed = firmwareResultSchemas.extract_firmware.parse( evidence.normalized_result, ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip")); + + assert.ok(parsed.chunks.some((c) => c.handler === "gzip")); + for (const [suffix, expected] of Object.entries(oracle.files)) { - const file = result.files.find((candidate) => - candidate.relative_path.endsWith(`/${suffix}`), + const entry = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(file !== undefined, `Missing extracted ${suffix}`); - assert.equal(file.sha256, expected.sha256); - assert.equal(file.size, expected.size); - assert.equal(await hashFirmwareFile(file.path), expected.sha256); - assert.equal(file.original_file_range, null); - assert.equal(file.runtime_address, null); + assert.ok(entry !== undefined, `Missing extracted ${suffix}`); + assert.equal(entry.sha256, expected.sha256); + assert.equal(entry.size, expected.size); + assert.equal(await hashFirmwareFile(entry.path), expected.sha256); + assert.equal(entry.original_file_range, null); + assert.equal(entry.runtime_address, null); } - return result; -}; -try { - const inspected = await cli(["inspect-firmware-regions", path]); - const cliRegions = verifyRegions(inspected); - const extracted = await cli([ - "extract-firmware", - path, - join(root, "cli-output"), + + return parsed; +} + +const toContentIdentity = (parsed) => + parsed.files.map(({ relative_path, sha256, size }) => ({ + relative_path, + sha256, + size, + })); + +// --------------------------------------------------------------------------- +// Verification stages +// --------------------------------------------------------------------------- + +async function stageCli() { + const regionsEvidence = await runCli([ + "inspect-firmware-regions", + firmwarePath, ]); - const cliFiles = await verifyFiles(extracted); - assert.equal(cliFiles.coverage, "partial"); + const regions = verifyRegions(regionsEvidence); + + const extractEvidence = await runCli([ + "extract-firmware", + firmwarePath, + join(workDir, "cli-output"), + ]); + const files = await verifyFiles(extractEvidence); + + assert.equal(files.coverage, "partial"); assert.ok( - cliFiles.chunks.some( - (chunk) => - chunk.handler === null && - chunk.range.offset === 0 && - chunk.range.length === 64, + files.chunks.some( + (c) => + c.handler === null && c.range.offset === 0 && c.range.length === 64, ), ); - await client.connect( - new StdioClientTransport({ - command: process.execPath, - args: [entrypoint, "mcp"], - cwd: repository, - env: environment, - stderr: "pipe", - }), - { timeout: 30_000 }, - ); - const inspectedMcp = await call("inspect_firmware_regions", { path }); - const mcpEvidence = parseEvidence(inspectedMcp); - assert.deepEqual(verifyRegions(mcpEvidence), cliRegions); - const extractedMcp = await call("extract_firmware", { - path, - output_directory: join(root, "mcp-output"), + + return { regions, files }; +} + +async function connectMcp() { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ENTRYPOINT, "mcp"], + cwd: REPO_ROOT, + env: baseEnv, + stderr: "pipe", }); - const mcpFiles = await verifyFiles(parseEvidence(extractedMcp)); - const contentIdentity = (result) => - result.files.map(({ relative_path, sha256, size }) => ({ - relative_path, - sha256, - size, - })); - assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles)); - const selected = await call("extract_firmware", { - path, - output_directory: join(root, "selected-output"), + await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS }); +} + +async function stageMcp(cli) { + const regionsRaw = await callTool("inspect_firmware_regions", { + path: firmwarePath, + }); + assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions); + + const extractRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "mcp-output"), + }); + const mcpFiles = await verifyFiles(parseEvidence(extractRaw)); + assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files)); + + const rangedRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "selected-output"), range: { offset: oracle.offset, length: oracle.length }, }); - const selectedFiles = await verifyFiles(parseEvidence(selected)); - assert.equal(selectedFiles.selection.offset, oracle.offset); - assert.equal(selectedFiles.selection.length, oracle.length); - const depth = await call("extract_firmware", { - path, - output_directory: join(root, "depth-output"), + const ranged = await verifyFiles(parseEvidence(rangedRaw)); + assert.equal(ranged.selection.offset, oracle.offset); + assert.equal(ranged.selection.length, oracle.length); + + const depthRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "depth-output"), max_depth: 1, }); - const depthResult = firmwareResultSchemas.extract_firmware.parse( - depth.normalized_result, + const depthParsed = firmwareResultSchemas.extract_firmware.parse( + depthRaw.normalized_result, ); - assert.equal(depthResult.coverage, "partial"); - assert.ok(depthResult.depth_limited_paths.length > 0); + assert.equal(depthParsed.coverage, "partial"); + assert.ok(depthParsed.depth_limited_paths.length > 0); + console.log( "PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits", ); - if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") { - const ext4 = join(fixtureRoot, "ext4.bin"); - await access(ext4); - const response = await call("extract_firmware", { - path: ext4, - output_directory: join(root, "ext4-output"), - }); - const result = firmwareResultSchemas.extract_firmware.parse( - response.normalized_result, + + return mcpFiles; +} + +async function stageExt4() { + const ext4Path = join(FIXTURE_ROOT, "ext4.bin"); + await fs.access(ext4Path); + + const raw = await callTool("extract_firmware", { + path: ext4Path, + output_directory: join(workDir, "ext4-output"), + }); + const parsed = firmwareResultSchemas.extract_firmware.parse( + raw.normalized_result, + ); + assert.ok(parsed.chunks.some((c) => c.handler === "extfs")); + assert.equal(parsed.coverage, "complete"); + + for (const [suffix, expected] of Object.entries(oracle.files)) { + const match = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs")); - assert.equal(result.coverage, "complete"); - for (const [suffix, expected] of Object.entries(oracle.files)) - assert.equal( - result.files.find((file) => file.relative_path.endsWith(`/${suffix}`)) - ?.sha256, - expected.sha256, - ); - console.log("PASS real ext4 extraction with debugfs; no mounts"); - // Preserve the process-ownership utility while withholding all extractors. - const withoutExtractors = join(root, "without-extractors"); - await access("/bin/ps"); - await mkdir(withoutExtractors); - await symlink("/bin/ps", join(withoutExtractors, "ps")); - const missing = await cli( - ["extract-firmware", ext4, join(root, "missing-extractor-output")], - { ...environment, PATH: withoutExtractors }, - ); - const partial = firmwareResultSchemas.extract_firmware.parse( - missing.normalized_result, - ); - assert.equal(partial.coverage, "partial"); - assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs")); - console.log( - "PASS real missing-extractor diagnostics retained despite Unblob exit 1", - ); - } - if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") { - if (process.env.GHIDRA_INSTALL_DIR === undefined) - throw new Error( - "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", - ); - const child = mcpFiles.files.find((file) => - file.relative_path.endsWith("/usr/bin/probe"), - ); - assert.ok(child !== undefined); - await call("open_binary", { path: child.path, provider_id: "ghidra" }); - const dossier = await call("analyze_function", { - procedure: "firmware_probe", - }); - const evidence = parseEvidence(dossier); - assert.equal( - evidence.subject.digest.sha256, - oracle.files["usr/bin/probe"].sha256, - ); - const pseudocode = await call("procedure_pseudo_code", { - procedure: "firmware_probe", - }); - assert.equal(parseEvidence(pseudocode).provider.id, "ghidra"); - assert.ok( - JSON.stringify(evidence.normalized_result).includes("firmware_probe"), - ); - await call("close_binary", {}); - console.log( - "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + assert.equal(match?.sha256, expected.sha256); + } + console.log("PASS real ext4 extraction with debugfs; no mounts"); + + // Keep the process-ownership utility available but hide every extractor. + const emptyBin = join(workDir, "without-extractors"); + await fs.access("/bin/ps"); + await fs.mkdir(emptyBin); + await fs.symlink("/bin/ps", join(emptyBin, "ps")); + + const degraded = await runCli( + ["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")], + { ...baseEnv, PATH: emptyBin }, + ); + const degradedParsed = firmwareResultSchemas.extract_firmware.parse( + degraded.normalized_result, + ); + assert.equal(degradedParsed.coverage, "partial"); + assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs")); + console.log( + "PASS real missing-extractor diagnostics retained despite Unblob exit 1", + ); +} + +async function stageGhidra(mcpFiles) { + if (process.env.GHIDRA_INSTALL_DIR === undefined) { + throw new Error( + "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", ); } + + const probe = mcpFiles.files.find((f) => + f.relative_path.endsWith("/usr/bin/probe"), + ); + assert.ok(probe !== undefined); + + await callTool("open_binary", { path: probe.path, provider_id: "ghidra" }); + + const dossierRaw = await callTool("analyze_function", { + procedure: "firmware_probe", + }); + const dossier = parseEvidence(dossierRaw); assert.equal( - await hashFirmwareFile(path), + dossier.subject.digest.sha256, + oracle.files["usr/bin/probe"].sha256, + ); + + const pseudoRaw = await callTool("procedure_pseudo_code", { + procedure: "firmware_probe", + }); + assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra"); + assert.ok( + JSON.stringify(dossier.normalized_result).includes("firmware_probe"), + ); + + await callTool("close_binary", {}); + console.log( + "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + ); +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +try { + const cli = await stageCli(); + await connectMcp(); + const mcpFiles = await stageMcp(cli); + + if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4(); + if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") + await stageGhidra(mcpFiles); + + assert.equal( + await hashFirmwareFile(firmwarePath), oracle.sha256, "Original firmware bytes must remain unchanged", ); } finally { - await client.close(); - await rm(root, { recursive: true, force: true }); + await mcpClient.close(); + await fs.rm(workDir, { recursive: true, force: true }); } diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index ae81353c7..23645e456 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -810,6 +810,36 @@ exit 1`, expect(result.error.message).toContain("overwrite"); }); +it.each([ + ["offline", false, "error: device offline"], + ["unauthorized", false, "error: device unauthorized"], + ["offline", true, "error: device offline"], +] as const)( + "reports the %s device state when the push existence probe cannot run (overwrite %s)", + async (state, overwrite, stderr) => { + const { binary, root } = await writeAdbStub( + (stubRoot) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "-s" ]; then printf '%s\\n' "$3" >> "${stubRoot}/calls"; printf '%s\\n' "${stderr}" >&2; exit 1; fi`, + ); + const localFile = join(root, "payload.bin"); + await writeFile(localFile, "x"); + const projected = await refusedBy(providerOver(binary), { + operation: "push_adb_file", + input: { + serial: "emulator-5554", + local_path: localFile, + device_path: "/data/local/tmp/payload.bin", + overwrite, + }, + }); + // Existence is unknown, so nothing is claimed about the path and no push runs. + expect(projected).toMatchObject({ code: "unsupported_target" }); + expect(projected.message).toContain(state); + expect(projected.message).not.toContain("already exists"); + expect(await readFile(join(root, "calls"), "utf8")).toBe("shell\n"); + }, +); + it("reports a local push source that does not exist", async () => { const { binary } = await writeAdbStub( () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index b9af6d14f..d26345eba 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -1623,17 +1623,15 @@ export class AdbProvider { { timeoutMs: SETTINGS_TIMEOUT_MS, maxOutputBytes: 1024 }, signal, ); - let deviceExists: boolean; - if (existsProbe.ok) deviceExists = true; - else if (existsProbe.error instanceof AnalysisProtocolError) - deviceExists = false; - else if (existsProbe.error instanceof AnalysisTimeoutError) + // `test -e` exits nonzero with no adb diagnosis when the path is absent. + // Any other probe failure leaves existence unknown, so it is reported as + // is rather than read as an existing path or followed by a push. + if ( + !existsProbe.ok && + !(existsProbe.error instanceof AnalysisProtocolError) + ) return existsProbe; - else if (existsProbe.error instanceof AnalysisCancelledError) - return existsProbe; - else if (existsProbe.error instanceof AnalysisResourceConstraintError) - return existsProbe; - else deviceExists = true; // authorization/refusal states surface on push instead + const deviceExists = existsProbe.ok; if (deviceExists && !input.overwrite) return err( new AnalysisOutputError( diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index f3b51257a..376036fa1 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -50,6 +50,13 @@ export interface AnalysisExecution { readonly limitations: readonly string[]; /** Provider-owned completeness assessment for caller/callee observations. */ readonly relationshipCoverage?: RelationshipCoverage; + /** + * Whether the provider capped an `xrefs` read: true when it reported a + * truncation, false when it explicitly reported none. Absent when it reports + * nothing, which is not a completeness claim. Never part of Evidence or tool + * output. + */ + readonly referencesTruncated?: boolean; readonly locations: readonly EvidenceLocation[]; readonly subject: EvidenceSubjectTarget | null; } @@ -59,6 +66,7 @@ interface CreateAnalysisExecutionOptions { readonly analysisProfile?: AnalysisProfileCommitment; readonly limitations?: readonly string[]; readonly relationshipCoverage?: RelationshipCoverage; + readonly referencesTruncated?: boolean; readonly locations?: readonly EvidenceLocation[]; readonly subject?: EvidenceSubjectTarget; } @@ -103,6 +111,9 @@ export function createAnalysisExecution( options.relationshipCoverage, ), }), + ...(options.referencesTruncated === undefined + ? {} + : { referencesTruncated: options.referencesTruncated }), locations: [...(options.locations ?? [])], subject: options.subject ?? null, }; diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 1b8ec6411..4286ff1d0 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -497,3 +497,71 @@ it("reports host denial before target mismatch but keeps composed workflow prere ).reason, ).toBe("target_required"); }); + +describe("trace_feature requirements", () => { + const capabilities = (operations: readonly string[]) => + operations.map((operation) => ({ + operation, + available: true, + reason: null, + })); + + it("needs the search operations the workflow actually calls", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: true, reason: "available" }); + }); + + it.each(["search_strings", "search_procedures"])( + "is unavailable without %s", + (missing) => { + const operations = [ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ].filter((operation) => operation !== missing); + + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities(operations), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }, + ); + + it("is unavailable when only the list operations exist", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "list_strings", + "list_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }); +}); diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 6fefb8e20..4e364a1aa 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -121,8 +121,8 @@ const ENHANCED_REQUIREMENTS: Readonly> = { analyze_function: ["analyze_function"], inspect_native_api: ["analyze_function"], trace_feature: [ - "list_strings", - "list_procedures", + "search_strings", + "search_procedures", "xrefs", "resolve_containing_procedure", ], diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index b32df3fd0..1b241a98b 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -1,9 +1,5 @@ import { parse as parseToml, stringify as stringifyToml } from "smol-toml"; -import { - parse as parseYaml, - parseDocument as parseYamlDocument, - stringify as stringifyYaml, -} from "yaml"; +import { stringify as stringifyYaml } from "yaml"; import { isDeepStrictEqual } from "node:util"; import { @@ -12,6 +8,10 @@ import { editDshDocument, dshClientRegistrationEntry, } from "./DshCordisPatch.js"; +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; import { applyEdits, findNodeAtLocation, @@ -118,7 +118,10 @@ const parseDocument = ( if (isEmptyClientConfigurationText(text)) return {}; // Hermes stores MCP servers in `mcp_servers` inside a YAML document. if (format === "hermes") - return requireObject(parseYaml(text), "The configuration root"); + return requireObject( + parseHermesConfigurationYaml(text), + "The configuration root", + ); // ZCode loads nonempty native files with strict JSON.parse. if (format === "zcode") return requireObject(JSON.parse(text), "The configuration root"); @@ -1419,12 +1422,11 @@ const serializeHermesConfiguration = ( typeof servers === "object" && servers !== null ? (servers as Record)[PRODUCT_IDENTITY.mcpServerKey] : undefined; - const yamlDocument = parseYamlDocument(originalText); - if (entry === undefined) - yamlDocument.deleteIn([serversKey, PRODUCT_IDENTITY.mcpServerKey]); - else yamlDocument.setIn([serversKey, PRODUCT_IDENTITY.mcpServerKey], entry); - // Editing REA must not reflow unrelated user prompts or other YAML scalars. - return yamlDocument.toString({ lineWidth: 0 }); + return editHermesConfigurationYaml( + originalText, + [serversKey, PRODUCT_IDENTITY.mcpServerKey], + entry, + ); }; /** Build the stdio entry shape expected by one client's configuration dialect. */ diff --git a/src/application/EnhancedLiteralTracing.ts b/src/application/EnhancedLiteralTracing.ts index 9c86781bc..79684ce0f 100644 --- a/src/application/EnhancedLiteralTracing.ts +++ b/src/application/EnhancedLiteralTracing.ts @@ -1,5 +1,8 @@ import { z } from "zod"; -import type { AnalysisOperation } from "./AnalysisProvider.js"; +import type { + AnalysisExecution, + AnalysisOperation, +} from "./AnalysisProvider.js"; import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -16,6 +19,12 @@ type AnalysisCall = ( signal?: AbortSignal, ) => Promise>; +type ObserveCall = ( + name: AnalysisOperation, + arguments_: Readonly>, + signal?: AbortSignal, +) => Promise>; + type LiteralSearchSource = readonly [ "search_strings" | "search_procedures", "string" | "procedure", @@ -39,6 +48,7 @@ export interface LiteralTraceInput { /** Trace matching strings and procedures through provider xrefs. */ export const traceLiteralFeature = async ( call: AnalysisCall, + observe: ObserveCall, input: LiteralTraceInput, signal?: AbortSignal, ): EnhancedResult => { @@ -49,7 +59,7 @@ export const traceLiteralFeature = async ( signal, ); if (!searched.ok) return searched; - const traced = await traceReferences(call, { + const traced = await traceReferences(call, observe, { matches: searched.value.matches, ...(signal === undefined ? {} : { signal }), }); @@ -59,8 +69,8 @@ export const traceLiteralFeature = async ( search_mode: "literal", matches: searched.value.matches, references: traced.value.references, - truncated: false, - residual_unknowns: [], + truncated: traced.value.residual.length > 0, + residual_unknowns: traced.value.residual, }); }; @@ -126,24 +136,39 @@ const addPageMatches = (input: { const traceReferences = async ( call: AnalysisCall, + observe: ObserveCall, request: ReferenceTraceRequest, -): Promise> => { +): Promise< + Result<{ references: TraceReference[]; residual: string[] }, AnalysisError> +> => { const references: TraceReference[] = []; + const residual: string[] = []; + // A source address is resolved once however many matches reference it. + const containing = new Map(); for (const match of request.matches) { - const xrefs = await call( + const observed = await observe( "xrefs", { address: match.address }, request.signal, ); - if (!xrefs.ok) return xrefs; - if (!Array.isArray(xrefs.value)) + if (!observed.ok) return observed; + // Only an explicit producer report of truncation is a coverage gap; a + // provider that reports nothing keeps its previous, unqualified result. + const unknown = `References to ${match.address} were truncated by the provider; additional references may exist.`; + if ( + observed.value.referencesTruncated === true && + !residual.includes(unknown) + ) + residual.push(unknown); + const sources = observed.value.result; + if (!Array.isArray(sources)) return err( new AnalysisOutputError( "xrefs", "provider returned a non-array result", ), ); - for (const source of xrefs.value) { + for (const source of sources) { if (typeof source !== "string") return err( new AnalysisOutputError( @@ -151,18 +176,23 @@ const traceReferences = async ( "provider returned a non-address value", ), ); - const resolved = await call( - "resolve_containing_procedure", - { address: source }, - request.signal, - ); - if (!resolved.ok) return resolved; + let containingProcedure = containing.get(source); + if (containingProcedure === undefined) { + const resolved = await call( + "resolve_containing_procedure", + { address: source }, + request.signal, + ); + if (!resolved.ok) return resolved; + containingProcedure = resolved.value; + containing.set(source, containingProcedure); + } references.push({ target_address: match.address, source_address: source, - containing_procedure: resolved.value, + containing_procedure: containingProcedure, }); } } - return ok({ references }); + return ok({ references, residual }); }; diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index 87d27fc5f..207186ccc 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -163,6 +163,8 @@ export class EnhancedTools { return traceLiteralFeature( (name, arguments_, operationSignal) => this.#call(name, arguments_, operationSignal), + (name, arguments_, operationSignal) => + this.#observe(name, arguments_, operationSignal), call.input, signal, ); diff --git a/src/application/HermesConfigurationYaml.test.ts b/src/application/HermesConfigurationYaml.test.ts new file mode 100644 index 000000000..41b74fcc9 --- /dev/null +++ b/src/application/HermesConfigurationYaml.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; + +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; + +describe("Hermes configuration YAML", () => { + it("writes an edited number from its new value", () => { + expect( + editHermesConfigurationYaml("mode: 0600\nmask: 0x1F\n", ["mode"], 384), + ).toBe("mode: 384\nmask: 0x1F\n"); + }); + + it("accepts keys Hermes reads as distinct and rejects a repeated key", () => { + expect(() => + parseHermesConfigurationYaml("0600: octal\n600: decimal\n"), + ).not.toThrow(); + expect(() => parseHermesConfigurationYaml('a: 1\n"a": 2\n')).toThrow(); + }); +}); diff --git a/src/application/HermesConfigurationYaml.ts b/src/application/HermesConfigurationYaml.ts new file mode 100644 index 000000000..8d245da37 --- /dev/null +++ b/src/application/HermesConfigurationYaml.ts @@ -0,0 +1,74 @@ +import { + isScalar, + parse, + parseDocument, + type ParsedNode, + type ScalarTag, + type Tags, +} from "yaml"; + +const NUMBER_TAGS = new Set([ + "tag:yaml.org,2002:int", + "tag:yaml.org,2002:float", +]); + +/** + * Write a parsed number with its source spelling while that spelling still + * reads as the node's value. Hermes reads config.yaml as YAML 1.1 with exact + * integers, so re-printing a JavaScript number changes what it reads: a + * Discord channel ID loses its low digits, `0600` (octal) becomes `600`, and + * `0.0000001` becomes `1e-7`, which YAML 1.1 does not define as a float. + */ +const keepNumberSource = (tag: Tags[number]): Tags[number] => { + if (typeof tag === "string" || tag.collection !== undefined) return tag; + const { stringify } = tag; + if (!NUMBER_TAGS.has(tag.tag) || stringify === undefined) return tag; + return { + ...tag, + stringify: (item, context, ...rest) => { + const { source } = item; + if (source === undefined) return stringify(item, context, ...rest); + const reader = context.doc.schema.tags.find( + (candidate): candidate is ScalarTag => + candidate.default === true && candidate.test?.test(source) === true, + ); + const read = reader?.resolve( + source, + () => undefined, + context.doc.options, + ); + return Object.is(isScalar(read) ? read.value : read, item.value) + ? source + : stringify(item, context, ...rest); + }, + }; +}; + +const HERMES_YAML_OPTIONS = { + customTags: (tags: Tags) => tags.map(keepNumberSource), + // Hermes keeps integers exact, so IDs that round to one double, or `0600` + // and `600`, are distinct keys. + uniqueKeys: (a: ParsedNode, b: ParsedNode) => + a === b || + (isScalar(a) && + isScalar(b) && + a.value === b.value && + a.source === b.source), +}; + +/** Parse a Hermes `config.yaml`. */ +export const parseHermesConfigurationYaml = (text: string): unknown => + parse(text, HERMES_YAML_OPTIONS); + +/** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */ +export const editHermesConfigurationYaml = ( + text: string, + path: readonly string[], + value: unknown, +): string => { + const document = parseDocument(text, HERMES_YAML_OPTIONS); + if (value === undefined) document.deleteIn(path); + else document.setIn(path, value); + // Editing REA must not reflow unrelated user prompts or other YAML scalars. + return document.toString({ lineWidth: 0 }); +}; diff --git a/src/application/InvestigationProviders.ts b/src/application/InvestigationProviders.ts index 5dcb18150..af23463f4 100644 --- a/src/application/InvestigationProviders.ts +++ b/src/application/InvestigationProviders.ts @@ -20,7 +20,9 @@ export const workflowAnalysisProfile = ( upstream_analysis_profile: upstream, ...(operation === "binary_overview" ? { workflow_revision: 3 } - : operation === "batch_decompile" || operation === "get_call_graph" + : operation === "batch_decompile" || + operation === "get_call_graph" || + operation === "trace_feature" ? { workflow_revision: 2 } : {}), }); diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 36dae9949..49c6f82f5 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -97,18 +97,22 @@ const resolveInternalSpecifier = ( ); if (logicalPathEscapesRoot(normalized)) return { to: normalized, resolution: "unresolved" }; + // Preserve directory intent before normalization removes terminal dot segments. + const directoryReference = + specifier.endsWith("/") || [".", ".."].includes(posix.basename(specifier)); const suffixes = ["", ".ts", ".js", ".mjs", ".cjs", ".tsx", ".jsx"]; const candidates = suffixes.flatMap((suffix) => [ - `${normalized}${suffix}`, - `${normalized}/index${suffix}`, + ...(directoryReference ? [] : [`${normalized}${suffix}`]), + posix.join(normalized, `index${suffix}`), ]); // Retain present runtime files first; substitute only a missing counterpart. // Counterparts cover explicit filenames, not directory/index resolution. - candidates.push( - ...typescriptEmittedSourceCandidates(fromPath, normalized).map( - ({ path }) => path, - ), - ); + if (!directoryReference) + candidates.push( + ...typescriptEmittedSourceCandidates(fromPath, normalized).map( + ({ path }) => path, + ), + ); const match = lookupPath.resolveCandidates(candidates); return match === undefined ? { to: normalized, resolution: "unresolved" } diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 488dda49d..5839409d4 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -157,8 +157,27 @@ const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => { const grokDirectory = ({ home, env }: ClientPathContext): string => env.GROK_HOME || join(home, ".grok"); +/** Match Harness home-paths for both independent roots; trim only to detect blanks. */ +const dshDirectory = ( + home: string, + override: string | undefined, + fallback: string, +): string => { + const selected = + override !== undefined && override.trim().length > 0 + ? override + : join(home, fallback); + const expanded = + selected === "~" + ? home + : selected.startsWith("~/") || selected.startsWith("~\\") + ? join(home, selected.slice(2)) + : selected; + return resolve(expanded); +}; + const dshHomeDirectory = ({ home, env }: ClientPathContext): string => - env.DSH_HOME ?? join(home, ".dsh"); + dshDirectory(home, env.DSH_HOME, ".dsh"); const dshPatchDirectory = (context: ClientPathContext): string => join(dshHomeDirectory(context), "cordis.patch.yml"); @@ -555,7 +574,7 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ // The home Cordis patch applies to every profile; the skill directory is // shared agent storage Harness discovers. skillPath: ({ home, env }: ClientPathContext) => - join(env.DSH_AGENTS_HOME ?? join(home, ".agents"), "skills"), + join(dshDirectory(home, env.DSH_AGENTS_HOME, ".agents"), "skills"), configPath: dshPatchDirectory, markerPath: dshHomeDirectory, format: "dsh", diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 48c2a54bd..91b9f8807 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -661,3 +661,52 @@ it("does not replay pre-envelope call graphs under the revised workflow profile" }).success, ).toBe(false); }); + +describe("executions that report truncated references", () => { + const record = (referencesTruncated: boolean | undefined) => { + const cache = new AnalysisSnapshotCache(); + const parameters = { address: "0x1000", document: "fixture" }; + cache.record({ + target: ANALYSIS_SNAPSHOT_TARGET, + profile: ANALYSIS_SNAPSHOT_PROFILE, + operation: "xrefs", + parameters, + execution: createAnalysisExecution( + ["0x2000"], + ANALYSIS_SNAPSHOT_PROVIDER, + { + subject: ANALYSIS_SNAPSHOT_TARGET, + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + ...(referencesTruncated === undefined ? {} : { referencesTruncated }), + }, + ), + }); + return cache.lookup( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROFILE, + "xrefs", + parameters, + ); + }; + + it("are not replayed, because a replay would lose the truncation", () => { + expect(record(true)).toBeUndefined(); + }); + + it("do not stop an execution that is complete or silent from being replayed", () => { + expect(record(false)?.result).toEqual(["0x2000"]); + expect(record(undefined)?.result).toEqual(["0x2000"]); + }); +}); + +it("revises the trace_feature workflow profile so earlier results are not replayed", () => { + const profile = ANALYSIS_SNAPSHOT_PROFILE; + + expect(workflowAnalysisProfile(profile, "trace_feature")).not.toEqual( + workflowAnalysisProfile(profile), + ); + expect( + workflowAnalysisProfile(profile, "trace_call_path"), + "other workflows keep their profile", + ).toEqual(workflowAnalysisProfile(profile)); +}); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index bdaa9f68d..7e307baa0 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -337,6 +337,9 @@ export class AnalysisSnapshotCache { }): void { const { target, profile, operation, parameters, execution } = input; this.select(target, profile); + // The snapshot format does not carry reference truncation; a replay would + // present a capped read as complete, so it is not recorded. + if (execution.referencesTruncated === true) return; const queryId = analysisQueryId( snapshotTarget(target), snapshotBinding(profile), diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index dfbee8b27..066b005b8 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -2,7 +2,29 @@ import type { FileHandle } from "node:fs/promises"; import { PassThrough, type Readable } from "node:stream"; import { once } from "node:events"; -import { Reader, ZipReader, type Entry, type FileEntry } from "@zip.js/zip.js"; +import { + ERR_AMBIGUOUS_ARCHIVE, + ERR_BAD_FORMAT, + ERR_CENTRAL_DIRECTORY_NOT_FOUND, + ERR_ENCRYPTED_CENTRAL_DIRECTORY, + ERR_ENTRY_DATA_OUT_OF_BOUNDS, + ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND, + ERR_EOCDR_NOT_FOUND, + ERR_EXTRAFIELD_ZIP64_NOT_FOUND, + ERR_INVALID_COMPRESSED_DATA, + ERR_INVALID_CRC32, + ERR_INVALID_UNCOMPRESSED_SIZE, + ERR_LOCAL_FILE_HEADER_NOT_FOUND, + ERR_OVERLAPPING_ENTRY, + ERR_SPLIT_ZIP_FILE, + ERR_UNSUPPORTED_COMPRESSION, + ERR_UNSUPPORTED_ENCRYPTION, + ERR_UNSUPPORTED_UINT64, + Reader, + ZipReader, + type Entry, + type FileEntry, +} from "@zip.js/zip.js"; import { ArtifactReaderFailure, @@ -159,9 +181,9 @@ export class ZipArtifactReader implements ArtifactReader { let index = 0; // A traversal owns its accounting, including overlapping or repeated passes. let declaredFileBytes = 0; - for await (const entry of this.#reader.getEntriesGenerator({ - decodeText: decodeZipEntryText, - })) { + for await (const entry of typedZipFailures( + this.#reader.getEntriesGenerator({ decodeText: decodeZipEntryText }), + )) { abortIfNeeded(signal); // ZIP member identity is its directory occurrence, not its filename. // Logical path uniqueness belongs to the artifact inventory boundary. @@ -327,7 +349,7 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => { output.end(); }, abort: (reason) => { - output.destroy(toError(reason)); + output.destroy(toError(typedZipFailure(reason))); }, }); void entry @@ -337,10 +359,61 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => { checkOverlappingEntry: true, onprogress: () => abortIfNeeded(signal), }) - .catch((cause: unknown) => output.destroy(toError(cause))); + .catch((cause: unknown) => output.destroy(toError(typedZipFailure(cause)))); return output; }; +/** zip.js reports archive faults as plain errors identified by message. */ +const ZIP_FAILURE_REASONS: ReadonlyMap = + new Map([ + ...[ + ERR_AMBIGUOUS_ARCHIVE, + ERR_BAD_FORMAT, + ERR_CENTRAL_DIRECTORY_NOT_FOUND, + ERR_ENTRY_DATA_OUT_OF_BOUNDS, + ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND, + ERR_EOCDR_NOT_FOUND, + ERR_EXTRAFIELD_ZIP64_NOT_FOUND, + ERR_INVALID_COMPRESSED_DATA, + ERR_INVALID_CRC32, + ERR_INVALID_UNCOMPRESSED_SIZE, + ERR_LOCAL_FILE_HEADER_NOT_FOUND, + ERR_OVERLAPPING_ENTRY, + ERR_UNSUPPORTED_UINT64, + ].map((message) => [message, "format"] as const), + ...[ + ERR_ENCRYPTED_CENTRAL_DIRECTORY, + ERR_SPLIT_ZIP_FILE, + ERR_UNSUPPORTED_COMPRESSION, + ERR_UNSUPPORTED_ENCRYPTION, + ].map((message) => [message, "unavailable"] as const), + ]); + +/** Type a zip.js archive fault so consumers do not report it as file I/O. */ +const typedZipFailure = (cause: unknown): unknown => { + if (!(cause instanceof Error) || cause instanceof ArtifactReaderFailure) + return cause; + const reason = ZIP_FAILURE_REASONS.get(cause.message); + if (reason === undefined) return cause; + return new ArtifactReaderFailure( + reason, + reason === "format" + ? `ZIP archive is malformed: ${cause.message}` + : `ZIP archive uses an unsupported feature: ${cause.message}`, + { cause }, + ); +}; + +async function* typedZipFailures( + source: AsyncIterable, +): AsyncIterable { + try { + yield* source; + } catch (cause: unknown) { + throw typedZipFailure(cause); + } +} + const isSymlink = (entry: Entry): boolean => entry.unixMode !== undefined && (entry.unixMode & 0o170000) === 0o120000; diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts new file mode 100644 index 000000000..925d3543a --- /dev/null +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; + +import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js"; +import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; + +const fat = (wide = false): Buffer => + Buffer.from(fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }], wide)); + +describe("bounded FAT inventory classification", () => { + it("requires the complete table but not slice bytes in the retained prefix", () => { + const bytes = fat(); + for (const length of [4, 7, 8, 27]) { + expect( + classifyArtifactContent( + "native", + bytes.subarray(0, length), + bytes.length, + ), + ).toEqual({ kind: "unknown", format: "unknown" }); + } + expect( + classifyArtifactContent( + "native.class", + bytes.subarray(0, 28), + bytes.length, + ), + ).toEqual({ kind: "executable", format: "mach-o-universal" }); + }); + + it.each([ + ["empty table", (bytes: Buffer) => bytes.writeUInt32BE(0, 4)], + ["slice overlapping table", (bytes: Buffer) => bytes.writeUInt32BE(0, 16)], + ["empty slice", (bytes: Buffer) => bytes.writeUInt32BE(0, 20)], + [ + "slice beyond file", + (bytes: Buffer) => bytes.writeUInt32BE(bytes.length, 20), + ], + ["unaligned slice", (bytes: Buffer) => bytes.writeUInt32BE(4097, 16)], + ["unsupported alignment", (bytes: Buffer) => bytes.writeUInt32BE(53, 24)], + ] as const)("keeps %s unknown", (_label, mutate) => { + const bytes = fat(); + mutate(bytes); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("does not infer Mach-O from an out-of-bounds architecture count", () => { + const bytes = fat(); + bytes.writeUInt32BE(0xffffffff, 4); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "resource", + format: "file", + }); + }); + + it("does not lose unsafe FAT64 range values to number conversion", () => { + const bytes = fat(true); + bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires valid ranges for every declared slice", () => { + const bytes = Buffer.from( + fatImage([ + { cpu: CPU.arm64, bytes: machoImage({}) }, + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + ]), + ); + bytes.writeUInt32BE(bytes.length, 40); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires observed slice magic for an unknown CPU type", () => { + const cpu = { type: 0x12345678, subtype: 0 }; + const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }])); + expect( + classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length), + ).toEqual({ kind: "unknown", format: "unknown" }); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "executable", + format: "mach-o-universal", + }); + }); + + it.each([NaN, Infinity, -1, 8.5, Number.MAX_SAFE_INTEGER + 1, 27])( + "keeps invalid or contradictory file size %s unknown", + (size) => { + const bytes = fat(); + expect(classifyArtifactContent("native", bytes, size)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }, + ); +}); diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 67e745b0b..5efde8b33 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -216,8 +216,9 @@ export const createCliResultOutput = ( : filtered === undefined ? { ok: true, meta } : { ok: true, data: filtered, meta } - : filtered === undefined || - filtered === null || + : // A selected null is a value; only a selection with no JSON value + // falls back to the empty document. + filtered === undefined || typeof filtered === "function" || typeof filtered === "symbol" ? {} diff --git a/src/cliOutput.ts b/src/cliOutput.ts index 5119eebc3..ccfaf6b62 100644 --- a/src/cliOutput.ts +++ b/src/cliOutput.ts @@ -9,7 +9,7 @@ export type CliOutputArgumentValidation = | { readonly ok: false; readonly format: StructuredOutputFormat; - readonly code: "UNSUPPORTED_OUTPUT_COMBINATION"; + readonly code: "UNSUPPORTED_OUTPUT_COMBINATION" | "INVALID_OUTPUT_FILTER"; readonly message: string; }; @@ -89,13 +89,39 @@ export const parseCliOutputArguments = ( return parsed(false); }; -/** Reject text-window controls that would corrupt a structured document. */ +/** Reject slices that cannot advance within Incur's comma-delimited paths. */ +const hasUnclosedFilterSlice = (expression: string): boolean => { + let depth = 0; + let unclosed = false; + for (const character of expression) { + if (character === "[") { + depth += 1; + unclosed = true; + } else if (character === "]") { + depth -= 1; + // Incur consumes everything through the next closing bracket, including + // nested opening brackets. Do not impose a different filter grammar. + unclosed = false; + } else if (character === "," && depth === 0 && unclosed) return true; + } + return unclosed; +}; + +/** Reject output controls that cannot safely produce a complete result. */ export const validateCliOutputArguments = ( arguments_: readonly string[], ): CliOutputArgumentValidation => { - const { format, parseError, tokenWindow } = + const { filterOutput, format, parseError, tokenWindow } = parseCliOutputArguments(arguments_); if (parseError) return { ok: true }; + if (filterOutput !== undefined && hasUnclosedFilterSlice(filterOutput)) + return { + ok: false, + format: format === "yaml" || format === "jsonl" ? format : "json", + code: "INVALID_OUTPUT_FILTER", + message: + "The --filter-output expression has a slice without a closing ]. Close the slice or remove --filter-output.", + }; if ( tokenWindow && (format === "json" || format === "jsonl" || format === "yaml") diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index f1aa3e829..07e6909c3 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -169,7 +169,8 @@ it("pages every retained dossier collection and the native API record", () => { | "comments" | "unresolved_calls" | "referenced_strings" - | "referenced_names", + | "referenced_names" + | "swift_string_literals", result: unknown = original, ) => { const view = projectAnalysisView(nativeParent(result), { @@ -195,6 +196,23 @@ it("pages every retained dossier collection and the native API record", () => { }), ).toMatchObject({ item: [{ text: "entry" }] }); expect(page("referenced_names")).toMatchObject({ coverage: { total: 0 } }); + const literal = { + source_address: "0x401004", + value: "SleepDisabled", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7369447065656c53", "0xed000064656c6261"], + }; + expect( + page("swift_string_literals", { + ...original, + swift_string_literals: [literal], + }), + ).toMatchObject({ item: [literal], coverage: { total: 1 } }); + expect(page("swift_string_literals")).toMatchObject({ + item: { available: false, reason: "not recorded" }, + }); const { unresolved_calls: _omitted, ...withoutUnresolved } = original; expect( page("unresolved_calls", { diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 7f985f349..b08001e35 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -160,6 +160,8 @@ const nativeViewSchema = z.strictObject({ "unresolved_calls", "referenced_strings", "referenced_names", + "swift_string_literals", + "swift_string_literal_omissions", "native_api", "value_flow_summary", "value_flow_operations", diff --git a/src/domain/analysisView/nativeFunctionView.ts b/src/domain/analysisView/nativeFunctionView.ts index f5a29e0e1..b56de24a1 100644 --- a/src/domain/analysisView/nativeFunctionView.ts +++ b/src/domain/analysisView/nativeFunctionView.ts @@ -79,6 +79,8 @@ const nativeRows = ( unresolved_calls: dossier.unresolved_calls, referenced_strings: dossier.referenced_strings, referenced_names: dossier.referenced_names, + swift_string_literals: dossier.swift_string_literals, + swift_string_literal_omissions: dossier.swift_string_literal_omissions, value_flow_operations: flow?.operations, value_flow_def_use: flow?.def_use, value_flow_effects: flow?.effects, @@ -163,7 +165,8 @@ export const projectNativeFunctionView = ( singleton({ available: false, // Only value-flow facets inherit the value-flow failure reason; an - // older dossier simply has no unresolved_calls record. + // older dossier simply has no unresolved_calls record, and a target + // whose provider does not decode Swift literals has no such record. reason: view.facet.startsWith("value_flow_") && flow?.available === false ? flow.reason diff --git a/src/domain/apple/machoContainer.ts b/src/domain/apple/machoContainer.ts index 7b552be64..c23cf7f64 100644 --- a/src/domain/apple/machoContainer.ts +++ b/src/domain/apple/machoContainer.ts @@ -63,6 +63,11 @@ export const readFatContainerHeader = ( const magic = view.getUint32(0, false); if (![FAT_MAGIC, FAT_MAGIC_64, FAT_CIGAM, FAT_CIGAM_64].includes(magic)) return { state: "not-fat" }; + if (!Number.isSafeInteger(fileSize) || fileSize < bytes.byteLength) + return { + state: "malformed", + reason: "FAT file size is invalid or contradicts retained bytes", + }; if (bytes.byteLength < 8) { return fileSize < 8 ? { state: "malformed", reason: "FAT header is truncated" } diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 48da65960..99bc6dd24 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -278,8 +278,12 @@ export const parsePeRecord = ( record.toString("binary", 0, 4) !== "PE\u0000\u0000" ) return err("invalid or truncated PE header"); - const machine = decodePeMachine(record.readUInt16LE(4)); - if (machine === undefined) return err("unsupported PE architecture"); + const machineType = record.readUInt16LE(4); + const machine = decodePeMachine(machineType); + if (machine === undefined) + return err( + `unsupported PE architecture (COFF machine 0x${machineType.toString(16).padStart(4, "0")})`, + ); const { architecture } = machine; const optionalHeaderSize = record.readUInt16LE(20); if (optionalHeaderSize > 4096 || record.length < 24 + optionalHeaderSize) diff --git a/src/domain/callPath.test.ts b/src/domain/callPath.test.ts index e0f343fca..e78a5f936 100644 --- a/src/domain/callPath.test.ts +++ b/src/domain/callPath.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildCallPath } from "./callPath.js"; +import { combineFunctionRelationshipCoverage } from "./functionCallCoverage.js"; import { callPathResultSchema, type CallPathInput } from "./callPathSchemas.js"; import { createEvidence, type Evidence } from "./evidence.js"; import type { JsonValue } from "./jsonValue.js"; @@ -396,6 +397,60 @@ describe("explicit call relationship coverage", () => { expect(result.limitations).toEqual([reason]); }); + it("establishes not_found when only caller coverage is unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "not_found", + search_scope: { exhaustive: true }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when callee coverage is unknown", () => { + const reason = + "Callees: Upstream callees capped results at 500; additional callees remain unknown."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when both caller and callee coverage are unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships. Callees: Upstream callees are not observed."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + it("ignores incomplete relationship coverage outside the reachable search closure", () => { const reason = "An unrelated function's callees were not observed."; const result = run([ @@ -415,6 +470,34 @@ describe("explicit call relationship coverage", () => { }); }); +describe("IDA-capped callee coverage beside unknown callers", () => { + it("keeps the search non-exhaustive for the combined coverage value", () => { + const combined = combineFunctionRelationshipCoverage( + { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", + }, + { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], + }, + ); + expect(combined.status).toBe("unknown"); + const result = run([ + observe("0x1000", [], "rea-workflow", { relationshipCoverage: combined }), + observe("0x4000", []), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + shortest_hops: null, + paths: [], + }); + }); +}); + describe("call path evidence admission", () => { it("normalizes hex addresses and rejects duplicates and mixed providers", () => { expect( diff --git a/src/domain/callPath.ts b/src/domain/callPath.ts index 2186c8010..0bd5e88e1 100644 --- a/src/domain/callPath.ts +++ b/src/domain/callPath.ts @@ -10,6 +10,7 @@ import { parseFunctionEvidence, type FunctionSnapshot, } from "./functionDossierEvidence.js"; +import { isCallerOnlyUnknownCoverage } from "./functionCallCoverage.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; /** Minimal directed caller-to-callee adjacency for call-path search. */ @@ -255,10 +256,14 @@ const inspectSearch = ({ // Explicit, Evidence-bound relationship coverage owns completeness; // generic provider/dossier limitations remain contextual. Legacy Evidence // without that metadata retains its conservative limitation-based rule. + // The search only expands callee edges, so coverage that is unknown + // solely because callers are unobservable does not make the searched + // callee closure incomplete; the caller caveat stays in limitations. if ( coverage === null ? snapshot.limitations.length > 0 - : coverage.status !== "complete" + : coverage.status !== "complete" && + !isCallerOnlyUnknownCoverage(coverage) ) incompleteExpandedCoverage = true; } else if (node === goal) { diff --git a/src/domain/functionCallCoverage.test.ts b/src/domain/functionCallCoverage.test.ts new file mode 100644 index 000000000..6fd3fb045 --- /dev/null +++ b/src/domain/functionCallCoverage.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { + combineFunctionRelationshipCoverage, + isCallerOnlyUnknownCoverage, +} from "./functionCallCoverage.js"; +import type { RelationshipCoverage } from "./relationshipCoverage.js"; + +const unknownCallers: RelationshipCoverage = { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", +}; +const unknownCallees: RelationshipCoverage = { + status: "unknown", + reason: "Upstream callees are not observed in this profile.", +}; +const cappedCallees: RelationshipCoverage = { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], +}; +const complete: RelationshipCoverage = { status: "complete" }; + +describe("isCallerOnlyUnknownCoverage", () => { + it("detects unknown callers beside complete callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + complete, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(true); + }); + + it("rejects unknown callees beside complete callers", () => { + const combined = combineFunctionRelationshipCoverage( + complete, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown coverage on both sides", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown callers beside IDA-capped callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + cappedCallees, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects complete coverage", () => { + const combined = combineFunctionRelationshipCoverage(complete, complete); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); +}); diff --git a/src/domain/functionCallCoverage.ts b/src/domain/functionCallCoverage.ts index eb2c05a13..b64e78cd6 100644 --- a/src/domain/functionCallCoverage.ts +++ b/src/domain/functionCallCoverage.ts @@ -2,6 +2,9 @@ import type { RelationshipCoverage } from "./relationshipCoverage.js"; import type { FunctionDossier } from "./hopperValues.js"; import { uniqueSorted } from "./canonicalOrdering.js"; +const callerCoveragePrefix = "Callers: "; +const calleeCoveragePrefix = "Callees: "; + /** Combine caller and callee read outcomes without losing either explanation. */ export const combineFunctionRelationshipCoverage = ( callers: RelationshipCoverage | undefined, @@ -20,7 +23,8 @@ export const combineFunctionRelationshipCoverage = ( ); const details = diagnostics.length > 0 ? { diagnostics } : {}; const reasons = reads.flatMap(({ direction, coverage }) => { - const label = direction === "callers" ? "Callers" : "Callees"; + const label = + direction === "callers" ? callerCoveragePrefix : calleeCoveragePrefix; const limitations = coverage === undefined ? ["Provider did not report relationship coverage."] @@ -29,7 +33,7 @@ export const combineFunctionRelationshipCoverage = ( : coverage.status === "partial" ? coverage.limitations : []; - return limitations.map((reason) => `${label}: ${reason}`); + return limitations.map((reason) => `${label}${reason}`); }); if ( reads.some( @@ -50,6 +54,20 @@ export const combineFunctionRelationshipCoverage = ( return { status: "complete", ...details }; }; +/** + * Whether combined relationship coverage is unknown only on the caller side. + * The combine helper above prefixes each direction's reasons, so a reason + * that carries the caller prefix and no callee prefix records observed + * callees with unobservable callers. Forward (callee-edge) searches stay + * exhaustive under that caveat; the caller reason remains a limitation. + */ +export const isCallerOnlyUnknownCoverage = ( + coverage: RelationshipCoverage, +): boolean => + coverage.status === "unknown" && + coverage.reason.startsWith(callerCoveragePrefix) && + !coverage.reason.includes(calleeCoveragePrefix); + /** Explain why two resolved call sets cannot establish call-set equality. */ export const functionCallCoverageLimitations = ( coverage: RelationshipCoverage | null, diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index e03a27748..677fdc0cc 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -716,6 +716,236 @@ describe("function comparison identity and malformed CFG", () => { }); }); +const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => + functionDossierSchema.parse({ + ...dossier("return 0;", base), + swift_string_literals: values.map((value) => ({ + source_address: base, + value, + representation: "small" as const, + address: null, + registers: ["x0", "x1"] as const, + words: ["0x0", "0xe"] as const, + })), + }); +const stringsDimension = ( + leftDigit: string, + left: ReturnType | ReturnType, + rightDigit: string, + right: ReturnType | ReturnType, +) => + compareFunctions( + observe(leftDigit, left), + observe(rightDigit, right), + ).dimensions.find(({ dimension }) => dimension === "strings_names"); + +describe("function comparison Swift string literals", () => { + it("treats differing decoded literals as a strings and names change", () => { + expect( + stringsDimension( + "1", + withLiterals("0x1000", ["SleepDisabled"]), + "2", + withLiterals("0x2000", ["Holding awake"]), + ), + ).toMatchObject({ status: "changed" }); + }); + + it("matches the same decoded literal after the function is relocated", () => { + expect( + stringsDimension( + "3", + withLiterals("0x1000", ["SleepDisabled"]), + "4", + withLiterals("0x2000", ["SleepDisabled"]), + ), + ).toMatchObject({ status: "unchanged" }); + }); + + it("keeps strings and names unknown when only one dossier recorded Swift literals", () => { + expect( + stringsDimension( + "5", + withLiterals("0x1000", []), + "6", + dossier("return 0;", "0x2000"), + ), + ).toMatchObject({ + status: "unknown", + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); + }); + + it("reports a referenced name change even when Swift literal coverage differs", () => { + const named = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + referenced_names: [ + { address: "0x3000", value: "helper", source_address: "0x2000" }, + ], + }); + expect( + stringsDimension("7", withLiterals("0x1000", []), "8", named), + ).toMatchObject({ status: "changed" }); + }); +}); + +describe("function comparison partial Swift literal coverage", () => { + it("does not equate a partial literal record with a complete one", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2000", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "9", + withLiterals("0x1000", ["SleepDisabled"]), + "a", + partial, + ), + ).toMatchObject({ status: "unknown" }); + }); + + it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("d", withLiterals("0x1000", []), "e", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("does not interpret an unexamined path as a missing literal at a retained site", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["A"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 0, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + ], + }); + const complete = withLiterals("0x1000", ["A", "B"]); + expect(stringsDimension("1", complete, "2", partial)).toMatchObject({ + status: "unknown", + }); + expect(stringsDimension("3", partial, "4", complete)).toMatchObject({ + status: "unknown", + }); + expect( + stringsDimension("5", withLiterals("0x1000", []), "6", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("treats an omission record without decoded literals as partial coverage", () => { + const omissionOnly = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly), + ).toMatchObject({ status: "unknown" }); + }); + + it("keeps different retained representations unknown when both dossiers are partial", () => { + const omission = { + source_address: "0x1004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + } as const; + const small = functionDossierSchema.parse({ + ...withLiterals("0x1000", ["SleepDisabled"]), + swift_string_literal_omissions: [omission], + }); + const large = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literals: [ + { + source_address: "0x2000", + value: "SleepDisabled", + representation: "large", + address: "0x3000", + registers: ["x0", "x1"], + words: ["0xd00000000000000d", "0x8000000000002fe0"], + }, + ], + swift_string_literal_omissions: [ + { ...omission, source_address: "0x2004" }, + ], + }); + expect(stringsDimension("1", small, "2", large)).toMatchObject({ + status: "unknown", + }); + }); + + it("keeps different retained literals unknown when both dossiers are partial", () => { + const partial = (base: "0x1000" | "0x2000", value: string) => + functionDossierSchema.parse({ + ...withLiterals(base, [value]), + swift_string_literal_omissions: [ + { + source_address: base === "0x1000" ? "0x1004" : "0x2004", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "b", + partial("0x1000", "SleepDisabled"), + "c", + partial("0x2000", "Holding awake"), + ), + ).toMatchObject({ status: "unknown" }); + }); +}); + const expectInlineComparisonEvidence = ( result: ReturnType, ): void => { diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 27bb7b176..c5a0f373f 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -11,6 +11,7 @@ import { referenceProjection, sorted, stringAndNameProjection, + swiftLiteralSitesConflict, } from "./functionComparisonNormalization.js"; import type { DimensionName, @@ -139,8 +140,12 @@ const compareStringsAndNames = ( right: FunctionSnapshot, context: ComparisonContext, ): FunctionDimension => { - const leftValues = stringAndNameProjection(left); - const rightValues = stringAndNameProjection(right); + const complete = + swiftLiteralCoverage(left) === swiftLiteralCoverage(right) && + swiftLiteralCoverage(left) !== "partial"; + // Facts both dossiers record decide a change before Swift literal coverage can. + const leftValues = stringAndNameProjection(left, complete); + const rightValues = stringAndNameProjection(right, complete); if (leftValues === null || rightValues === null) return unresolvedDimension({ dimension: "strings_names", @@ -151,9 +156,40 @@ const compareStringsAndNames = ( "Reference source locations could not be normalized to function offsets.", ], }); - return compareValues("strings_names", leftValues, rightValues, context); + const compared = compareValues( + "strings_names", + leftValues, + rightValues, + context, + ); + if (complete || compared.status === "changed") return compared; + // A retained literal differs only when the opposite dossier establishes its absence. + if (swiftLiteralSitesConflict(left, right)) { + const leftAll = stringAndNameProjection(left); + const rightAll = stringAndNameProjection(right); + if (leftAll !== null && rightAll !== null) + return compareValues("strings_names", leftAll, rightAll, context); + } + return unresolvedDimension({ + dimension: "strings_names", + links: context.links, + leftCount: null, + rightCount: null, + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); }; +const swiftLiteralCoverage = ( + snapshot: FunctionSnapshot, +): "absent" | "complete" | "partial" => + (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 + ? "partial" + : snapshot.dossier.swift_string_literals === undefined + ? "absent" + : "complete"; + const compareReferences = ( left: FunctionSnapshot, right: FunctionSnapshot, diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 5282272e1..f32220b0b 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -142,6 +142,7 @@ export const commentProjection = ( export const stringAndNameProjection = ( snapshot: FunctionSnapshot, + includeSwiftLiterals = true, ): readonly unknown[] | null => { const values = [ ...snapshot.dossier.referenced_strings.map( @@ -164,12 +165,90 @@ export const stringAndNameProjection = ( value, }), ), + ...(includeSwiftLiterals + ? (snapshot.dossier.swift_string_literals ?? []) + : [] + ).map( + ({ + source_address: sourceAddress, + value, + representation, + registers, + }) => ({ + kind: "swift_string", + source_offset: relativeAddress( + sourceAddress, + snapshot.dossier.procedure.address, + ), + value, + representation, + registers, + }), + ), ]; return values.some(({ source_offset: offset }) => offset === null) ? null : sorted(values); }; +/** + * A retained literal proves presence, not that every path through its site was examined. + * A differing retained fact proves a change only when the opposite dossier is complete + * and therefore establishes its absence. + */ +export const swiftLiteralSitesConflict = ( + left: FunctionSnapshot, + right: FunctionSnapshot, +): boolean => { + const leftSites = swiftLiteralSites(left); + const rightSites = swiftLiteralSites(right); + for (const site of new Set([ + ...leftSites.values.keys(), + ...rightSites.values.keys(), + ])) { + const leftValues = + leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined); + const rightValues = + rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined); + if (leftValues === undefined || rightValues === undefined) continue; + const leftFacts = new Set(leftValues); + const rightFacts = new Set(rightValues); + if ( + (rightSites.complete && + leftValues.some((value) => !rightFacts.has(value))) || + (leftSites.complete && rightValues.some((value) => !leftFacts.has(value))) + ) + return true; + } + return false; +}; + +const swiftLiteralSites = (snapshot: FunctionSnapshot) => { + const base = snapshot.dossier.procedure.address; + const omissions = snapshot.dossier.swift_string_literal_omissions ?? []; + const site = (sourceAddress: string, registers: readonly string[]) => { + const offset = relativeAddress(sourceAddress, base); + return offset === null ? null : canonicalJson([offset, registers]); + }; + // Keep the same literal facts the full strings_names projection compares. + const values = new Map(); + for (const literal of snapshot.dossier.swift_string_literals ?? []) { + const key = site(literal.source_address, literal.registers); + if (key === null) continue; + values.set( + key, + [ + ...(values.get(key) ?? []), + canonicalJson([literal.value, literal.representation]), + ].sort(compareUnicodeCodePoints), + ); + } + const complete = + snapshot.dossier.swift_string_literals !== undefined && + omissions.length === 0; + return { values, complete }; +}; + export const sorted = (values: readonly unknown[]): readonly unknown[] => values .map((value) => ({ diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 769d0f040..d6344ff3e 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -290,6 +290,70 @@ export const unresolvedCallSchema = z.strictObject({ reason: z.string(), }); +const swiftStringLiteralFields = { + source_address: z + .string() + .describe( + "Instruction that consumes both words: a call, tail call, return, or paired store.", + ), + value: z + .string() + .describe("Decoded UTF-8 text, including the canonical empty string."), + registers: z.tuple([z.string(), z.string()]), + words: z + .tuple([z.string(), z.string()]) + .describe("Observed constant register values, in register order."), +}; + +/** A Swift String literal decoded from the two words of its ABI value, not a data reference. */ +export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("small") + .describe("The UTF-8 bytes are inside the two words."), + address: z.null().describe("A small literal has no storage of its own."), + }), + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("large") + .describe( + "An immortal literal whose words hold its count and a pointer to its bytes.", + ), + address: z.string().describe("Address of a large literal's bytes."), + }), +]); + +const swiftStringLiteralOmissionFields = { + source_address: swiftStringLiteralFields.source_address, + byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + reason: z + .literal("capacity") + .describe( + "The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.", + ), + registers: swiftStringLiteralFields.registers, + words: swiftStringLiteralFields.words, +}; + +/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */ +export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( + "representation", + [ + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("small"), + address: z.null(), + }), + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), + }), + ], +); + /** Observed endpoints with complete or partial reference classification evidence. */ export const referenceEdgeSchema = z .object({ @@ -341,6 +405,18 @@ export const functionDossierSchema = z }) .strict(), ), + swift_string_literals: z + .array(swiftStringLiteralSchema) + .exactOptional() + .describe( + "Swift String literals decoded from constant register values. Absent when the provider does not decode them for this target.", + ), + swift_string_literal_omissions: z + .array(swiftStringLiteralOmissionSchema) + .exactOptional() + .describe( + "The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.", + ), basic_blocks: z.array( z .object({ diff --git a/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts index 3b83bf482..c3b38842a 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts @@ -33,6 +33,38 @@ const getterReceivers = [ "nested owner", "const outer = { box: { x: 1, get self() { return this; } } }; outer.box.self.x = 2; return outer.box.x;", ], + [ + "array element", + "const arr = [{ x: 1, get self() { return this; } }]; arr[0].self.x = 2; return arr[0].x;", + ], + [ + "array element after a hole", + "const arr = [, { x: 1, get self() { return this; } }]; arr[1].self.x = 2; return arr[1].x;", + ], + [ + "mixed object and array owners", + "const outer = { rows: [{ box: { x: 1, get self() { return this; } } }] }; outer.rows[0].box.self.x = 2; return outer.rows[0].box.x;", + ], + [ + "assigned owner", + "let box; box = { x: 1, get self() { return this; } }; box.self.x = 2; return box.x;", + ], + [ + "saved array receiver alias", + "const arr = [{ x: 1, get self() { return this; } }]; const saved = arr[0].self; saved.x = 2; return arr[0].x;", + ], + [ + "old receiver captured inside its replacement initializer", + "const old = { x: 1 }; let box = old; box = { old: box, get self() { return this; } }; box.self.old.x = 2; return old.x;", + ], + [ + "array receiver after an unknown spread", + "const arr = [...items, { x: 1, get self() { return this; } }]; arr[index].self.x = 2; return arr[index].x;", + ], + [ + "saved receiver before its array alias is reassigned", + "const original = [{ x: 1, get self() { return this; } }]; let arr = original; const saved = arr[0].self; arr = [{ x: 1 }]; saved.x = 2; return original[0].x;", + ], [ "inherited receiver field", "const shared = { x: 1 }; class A { get v() { return this.s; } } class B extends A { s = shared; } new B().v.x = 2; return shared.x;", @@ -75,7 +107,7 @@ const getterReceivers = [ ], ] as const; -describe("getter receiver mutation aliases (#1665)", () => { +describe("getter receiver mutation aliases (#1665, #2654)", () => { it.each(getterReceivers)( "does not report a stale literal through %s", (_shape, body) => { @@ -103,6 +135,42 @@ describe("getter receiver mutation aliases (#1665)", () => { "unrelated sibling", "const box = { x: 1, y: 1, get self() { return this; } }; box.self.y = 2; return box.x;", ], + [ + "array receiver read without write", + "const arr = [{ x: 1, get self() { return this; } }]; const saved = arr[0].self; return arr[0].x;", + ], + [ + "assigned receiver read without write", + "let box; box = { x: 1, get self() { return this; } }; const saved = box.self; return box.x;", + ], + [ + "unrelated array receiver property", + "const arr = [{ x: 1, y: 1, get self() { return this; } }]; arr[0].self.y = 2; return arr[0].x;", + ], + [ + "unrelated array element", + "const arr = [{ x: 1, get self() { return this; } }, { x: 1 }]; arr[0].self.x = 2; return arr[1].x;", + ], + [ + "array replacement after a receiver was captured", + "const original = [{ x: 1, get self() { return this; } }]; const replacement = [{ x: 1 }]; let arr = original; const saved = arr[0].self; arr = replacement; saved.x = 2; return replacement[0].x;", + ], + [ + "known array prefix before a spread receiver", + "const arr = [{ x: 1 }, ...items, { x: 1, get self() { return this; } }]; arr[1].self.x = 2; return arr[0].x;", + ], + [ + "outer binding shadowed by an assigned receiver", + "let box; box = { x: 1 }; { let box; box = { x: 1, get self() { return this; } }; box.self.x = 2; } return box.x;", + ], + [ + "old object before an unconditional receiver assignment", + "const old = { x: 1 }; let box = old; box = { x: 1, get self() { return this; } }; box.self.x = 2; return old.x;", + ], + [ + "unmodified old receiver captured inside its replacement initializer", + "const old = { x: 1 }; let box = old; box = { x: 1, old: box, get self() { return this; } }; box.self.x = 2; return old.x;", + ], [ "replace getter", "const shared = { x: 1 }; const box = { s: shared, get v() { return this.s; } }; box.v = {}; return shared.x;", diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index c2f7efba3..35b63c815 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -18,6 +18,7 @@ import { semanticClassCandidates, semanticClassPropertyReferences, semanticObjectPropertyReferences, + type JavaScriptSemanticObjectReference, } from "./javascriptSemanticObjectReferences.js"; import { semanticIterationSources } from "./javascriptSemanticIterationSources.js"; import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js"; @@ -98,8 +99,20 @@ export function* collectSemanticMemberMutationsSteps( effect: ValueEffect = "write", mutation?: t.Node, originAt: t.Node | undefined = mutation, + ownerInitializer?: t.Node, ): void => { - const pending = [{ node, path, bindings, originAt }]; + const pending: (ReferencedValue & { + readonly bindings: ReadonlySet; + readonly originAt: t.Node | undefined; + })[] = [ + { + node, + path, + bindings, + originAt, + ...(ownerInitializer === undefined ? {} : { ownerInitializer }), + }, + ]; // Visit shallower alias traversals first so their exact prefixes suppress // longer loop paths before those paths enumerate every assignment order. for (let cursor = 0; cursor < pending.length; cursor++) { @@ -132,6 +145,8 @@ export function* collectSemanticMemberMutationsSteps( binding.bindingId, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]; const lifetime = JSON.stringify([...capture, current.path]); if (current.bindings.has(lifetime)) continue; @@ -165,6 +180,7 @@ export function* collectSemanticMemberMutationsSteps( "escape", mutation, current.originAt, + current.ownerInitializer, ); continue; } @@ -195,6 +211,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); const previous = selections.get(selectionIdentity); const excludedKeys = @@ -220,6 +238,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); let covering = coveringPaths.get(pathState); if (covering === undefined) { @@ -235,6 +255,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); if (recordedEffects.has(identity)) continue; // A recorded write invalidates the binding to a non-primitive value, so @@ -248,10 +270,23 @@ export function* collectSemanticMemberMutationsSteps( value?.status === "literal" || value?.status === "union"; recordedEffects.add(identity); const nested = new Set([...current.bindings, lifetime, captureKey]); + // Returning a literal's receiver reaches that allocation, not the + // binding's older values read while evaluating its initializer. const origins = - current.originAt === undefined - ? binding - : semanticMutationInitializers(binding, current.originAt, parents); + current.ownerInitializer !== undefined + ? { + initializers: binding.initializers.filter( + ({ node }) => node === current.ownerInitializer, + ), + referenceInitializers: [], + } + : current.originAt === undefined + ? binding + : semanticMutationInitializers( + binding, + current.originAt, + parents, + ); if (!primitiveWrite) { if (effect === "escape") binding.escapedPaths.push({ path, node: mutation }); @@ -367,6 +402,8 @@ export function* collectSemanticMemberMutationsSteps( binding === undefined ? null : binding.bindingId, binding === undefined ? node.name : null, pathKey, + value.ownerInitializer?.start, + value.ownerInitializer?.end, ]); if (leafIdentities.has(identity)) continue; leafIdentities.add(identity); @@ -551,9 +588,7 @@ export function* collectSemanticMemberMutationsSteps( clearSemanticValueCaches(state); } -interface ReferencedValue { - readonly node: t.Node; - readonly path: PropertyPath; +interface ReferencedValue extends JavaScriptSemanticObjectReference { readonly iterableFallbackPath?: PropertyPath; } diff --git a/src/domain/javascript/javascriptSemanticObjectReferences.ts b/src/domain/javascript/javascriptSemanticObjectReferences.ts index 387ba09d2..4d1bf3e76 100644 --- a/src/domain/javascript/javascriptSemanticObjectReferences.ts +++ b/src/domain/javascript/javascriptSemanticObjectReferences.ts @@ -14,9 +14,11 @@ import { } from "./javascriptSemanticPropertyPaths.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; -interface ReferencedValue { +export interface JavaScriptSemanticObjectReference { readonly node: t.Node; readonly path: PropertyPath; + /** The allocation owned by this binding, rather than a read of its prior value. */ + readonly ownerInitializer?: t.Node; } export const semanticClassCandidates = ( @@ -72,10 +74,10 @@ const instanceFieldReferences = ( path: PropertyPath, state: JavaScriptSemanticAnalysisState, ): { - readonly references: readonly ReferencedValue[]; + readonly references: readonly JavaScriptSemanticObjectReference[]; readonly definiteNames: ReadonlySet; } => { - const references: ReferencedValue[] = []; + const references: JavaScriptSemanticObjectReference[] = []; const pending = [{ klass: root, excluded: new Set() }]; const selections = new Map>(); let definiteNames: ReadonlySet | undefined; @@ -208,7 +210,7 @@ export const semanticClassPropertyReferences = ( selection: PropertyPath, state: JavaScriptSemanticAnalysisState, activeGetters: ReadonlyMap = new Map(), -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const root = lookup.owner; const receiver = lookup.receiver ?? root; const [key, ...path] = selection; @@ -216,7 +218,7 @@ export const semanticClassPropertyReferences = ( key == null || typeof key === "object" ? null : String(key); const pending: (t.ClassDeclaration | t.ClassExpression)[] = [root]; const seen = new Set(); - const possibleResults: ReferencedValue[] = []; + const possibleResults: JavaScriptSemanticObjectReference[] = []; // Instance fields are own properties, including fields initialized by base // constructors. They take precedence over every prototype descriptor. let instanceFields: ReadonlySet = new Set(); @@ -267,11 +269,11 @@ const objectReceiverReference = ( object: t.ObjectExpression, path: PropertyPath, state: JavaScriptSemanticAnalysisState, -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { // A literal's enclosing binding owns its slots. Returning the literal // itself would bypass invalidation of that binding's observed properties. let current: t.Node = object; - const projection: (string | number | null)[] = []; + const projection: PropertyPath[number][] = []; while (true) { const parent = state.parentsByNode.get(current); if ( @@ -279,7 +281,43 @@ const objectReceiverReference = ( parent.init === current && t.isIdentifier(parent.id) ) - return [{ node: parent.id, path: [...projection, ...path] }]; + return [ + { + node: parent.id, + path: [...projection, ...path], + ownerInitializer: current, + }, + ]; + if ( + t.isAssignmentExpression(parent, { operator: "=" }) && + parent.right === current && + t.isIdentifier(parent.left) + ) + return [ + { + node: parent.left, + path: [...projection, ...path], + ownerInitializer: current, + }, + ]; + if (t.isArrayExpression(parent)) { + let minimumIndex = 0; + let uncertainIndex = false; + for (const element of parent.elements) { + if (element === current) break; + if (t.isSpreadElement(element)) uncertainIndex = true; + else minimumIndex++; + } + // A spread contributes an unknown number of slots. Retain the known + // prefix without treating the element's syntax position as its index. + projection.unshift( + uncertainIndex + ? { excludedKeys: [], startIndex: minimumIndex } + : minimumIndex, + ); + current = parent; + continue; + } if (t.isObjectProperty(parent) && parent.value === current) { projection.unshift( semanticStaticPropertyKey(parent.key, parent.computed), @@ -312,7 +350,7 @@ const getterResultReferences = ( path: PropertyPath, state: JavaScriptSemanticAnalysisState, activeGetters: ReadonlyMap = new Map(), -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const { getter, owner, receiver = owner } = invocation; const previousLength = activeGetters.get(getter); if (previousLength !== undefined && previousLength <= path.length) return []; @@ -323,7 +361,7 @@ const getterResultReferences = ( const resolve = ( node: t.Node, remaining: PropertyPath, - ): readonly ReferencedValue[] => { + ): readonly JavaScriptSemanticObjectReference[] => { const expression = unwrapJavaScriptExpression(node).node; if (t.isThisExpression(expression)) { if (t.isClass(receiver)) { @@ -405,9 +443,9 @@ export const semanticObjectPropertyReferences = ( node: t.ObjectExpression, path: PropertyPath, state: JavaScriptSemanticAnalysisState, -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const [key, ...remaining] = path; - const references: ReferencedValue[] = []; + const references: JavaScriptSemanticObjectReference[] = []; const overwritten = new Set(); const dataOverwritten = new Set(); const accessorNames = new Set(); diff --git a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts index 71562e6ca..f2045a1eb 100644 --- a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts +++ b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts @@ -181,18 +181,37 @@ export const staticPath = (node: t.Node): string | undefined => { if (t.isStringLiteral(node)) return node.value; if (t.isTemplateLiteral(node) && node.expressions.length === 0) return stringValue(node); - if (t.isBinaryExpression(node, { operator: "+" })) { - const left = staticPath(node.left); - const right = staticPath(node.right); - return left === undefined || right === undefined - ? undefined - : `${left}${right}`; - } + if (t.isBinaryExpression(node, { operator: "+" })) + return staticConcatenation(node); if (t.isCallExpression(node) || t.isNewExpression(node)) return staticCallPath(node); return undefined; }; +/** + * Concatenate inert text. A leading source directory contributes no text: + * `__dirname + "/preload.js"` names a file inside it, while text without a + * leading separator extends the directory's own name and stays unknown. + */ +const staticConcatenation = (node: t.BinaryExpression): string | undefined => { + const operands: t.Node[] = []; + let current: t.Node = node; + while (t.isBinaryExpression(current, { operator: "+" })) { + operands.push(current.right); + current = current.left; + } + const anchored = isDirectoryPath(current); + if (!anchored) operands.push(current); + let text = ""; + for (const operand of operands.reverse()) { + const value = staticPath(operand); + if (value === undefined) return undefined; + text += value; + } + if (!anchored) return text; + return text.startsWith("/") ? posix.join(".", text) : undefined; +}; + const staticCallPath = ( node: t.CallExpression | t.NewExpression, ): string | undefined => { @@ -204,11 +223,6 @@ const staticCallPath = ( if (name === "fileURLToPath" || name.endsWith(".fileURLToPath")) { return staticFileUrlPath(argumentNode(node.arguments[0])); } - if ( - (name === "dirname" || name.endsWith(".dirname")) && - isFileIdentity(argumentNode(node.arguments[0])) - ) - return ""; if (name.endsWith(".resolve")) return name === "win32.resolve" || name.endsWith(".win32.resolve") ? undefined @@ -216,7 +230,12 @@ const staticCallPath = ( if (!name.endsWith(".join") && name !== "join") return undefined; const parts: string[] = []; for (const argument of node.arguments) { - if (isDirectoryIdentity(argumentNode(argument))) continue; + if (isDirectoryIdentity(argumentNode(argument))) { + // join never resets at an absolute segment, so `/preload.js` after the + // source directory stays inside it. + parts.push("."); + continue; + } const value = t.isNode(argument) ? staticPath(argument) : undefined; if (value === undefined) return undefined; parts.push(value); @@ -331,11 +350,20 @@ const isFilesystemPathExpression = (node: t.Node): boolean => { }; const isDirectoryIdentity = (node: t.Node | undefined): boolean => { - if (t.isIdentifier(node, { name: "__dirname" })) return true; + if (isDirectoryPath(node)) return true; if (!t.isCallExpression(node) && !t.isNewExpression(node)) return false; const name = calleeName(node.callee); - if (name === "URL" || name.endsWith(".URL")) - return isFileIdentity(argumentNode(node.arguments[1])); + return ( + (name === "URL" || name.endsWith(".URL")) && + isFileIdentity(argumentNode(node.arguments[1])) + ); +}; + +/** `__dirname` or the dirname of the source file, as a string. */ +const isDirectoryPath = (node: t.Node | undefined): boolean => { + if (t.isIdentifier(node, { name: "__dirname" })) return true; + if (!t.isCallExpression(node)) return false; + const name = calleeName(node.callee); return ( (name === "dirname" || name.endsWith(".dirname")) && isFileIdentity(argumentNode(node.arguments[0])) diff --git a/src/domain/javascript/javascriptStaticPathResolution.test.ts b/src/domain/javascript/javascriptStaticPathResolution.test.ts index e54122440..30bbc317f 100644 --- a/src/domain/javascript/javascriptStaticPathResolution.test.ts +++ b/src/domain/javascript/javascriptStaticPathResolution.test.ts @@ -38,6 +38,9 @@ const anchoredExpressions = [ ["path.resolve(__dirname)", "."], ['path.posix.resolve(`/app`, `/shared`, "preload.js")', "/shared/preload.js"], ['path.posix.resolve("" + "/app", "preload.js")', "/app/preload.js"], + ['__dirname + "/preload.js"', "preload.js"], + ['__dirname + "/" + "preload.js"', "preload.js"], + ['path.dirname(__filename) + "/lib" + "/preload.js"', "lib/preload.js"], ] as const; const unknownExpressions = [ @@ -58,6 +61,8 @@ const unknownExpressions = [ 'path.win32.resolve(__dirname, "preload.js")', 'win32.resolve("/app", "preload.js")', 'resolve("/app", "preload.js")', + '__dirname + "preload.js"', + 'path.dirname(__filename) + "preload.js"', ]; describe("inert JavaScript path resolution", () => { @@ -100,6 +105,16 @@ describe("inert JavaScript path resolution", () => { "preload.js", "filesystem-expression", ], + [ + 'path.join(__dirname, "/preload.js")', + "preload.js", + "filesystem-expression", + ], + [ + 'path.join(dirname(fileURLToPath(import.meta.url)), "/lib", "/preload.js")', + "lib/preload.js", + "filesystem-expression", + ], [ 'path.join("ignored", __dirname, "preload.js")', "ignored/preload.js", @@ -111,6 +126,7 @@ describe("inert JavaScript path resolution", () => { "filesystem-expression", ], ['join("sub", "preload.js")', "sub/preload.js", "module-specifier"], + ["path.join(__dirname)", ".", "filesystem-expression"], ] as const)("preserves join behavior in %s", (expression, path, context) => { assertPathProjection(expression, path, context); }); @@ -182,7 +198,6 @@ describe("inert JavaScript path resolution", () => { it.each([ "path.join()", - "path.join(__dirname)", 'path.join("/app", runtimeFile)', "path.join(...parts)", ])("preserves unresolved join syntax in %s", (expression) => diff --git a/src/domain/native/nativeAotPe.test.ts b/src/domain/native/nativeAotPe.test.ts index 8ae624d44..ab031cc85 100644 --- a/src/domain/native/nativeAotPe.test.ts +++ b/src/domain/native/nativeAotPe.test.ts @@ -159,3 +159,22 @@ describe("read-only NativeAOT PE parser", () => { }); }); }); + +describe("NativeAOT PE signature admission", () => { + it.each([ + ["MZ", "Not a PE image"], + ["PE", "Invalid PE signature"], + ])( + "rejects a high-bit lookalike of the %s signature", + (signature, reason) => { + const bytes = nativeAotPeFixture(); + const offset = signature === "MZ" ? 0 : bytes.readUInt32LE(0x3c); + for (const position of [offset, offset + 1]) + bytes.writeUInt8(bytes.readUInt8(position) | 0x80, position); + expect( + parseNativeAotPe(bytes, nativeAotPeDigest(bytes), SUMMARY_BUDGET) + .summary, + ).toMatchObject({ status: "not_applicable", reason }); + }, + ); +}); diff --git a/src/domain/native/nativeAotPe.ts b/src/domain/native/nativeAotPe.ts index b67a58398..7f9c5eb55 100644 --- a/src/domain/native/nativeAotPe.ts +++ b/src/domain/native/nativeAotPe.ts @@ -384,11 +384,11 @@ class PeImage { constructor(bytes: Buffer) { this.#bytes = bytes; - if (bytes.length < 0x40 || bytes.toString("ascii", 0, 2) !== "MZ") + if (bytes.length < 0x40 || bytes.toString("latin1", 0, 2) !== "MZ") throw new Error("Not a PE image"); const peOffset = this.#u32(0x3c); this.#range(peOffset, 24); - if (bytes.toString("ascii", peOffset, peOffset + 4) !== "PE\0\0") + if (bytes.toString("latin1", peOffset, peOffset + 4) !== "PE\0\0") throw new Error("Invalid PE signature"); this.#machine = this.#u16(peOffset + 4); const count = this.#u16(peOffset + 6); diff --git a/src/domain/native/nativeInvestigationGraph.test.ts b/src/domain/native/nativeInvestigationGraph.test.ts index 015458319..684b2a02e 100644 --- a/src/domain/native/nativeInvestigationGraph.test.ts +++ b/src/domain/native/nativeInvestigationGraph.test.ts @@ -83,6 +83,61 @@ const graph = nativeInvestigationGraphSchema.parse({ }); describe("native investigation graph traces", () => { + it.each(["forward", "backward"] as const)( + "keeps both endpoints of retained edges within the %s node budget", + (direction) => { + const trace = traceNativeInvestigationGraph(graph, { + start: direction === "forward" ? "a" : "c", + direction, + limits: { max_nodes: 1 }, + }); + expect(trace.nodes).toHaveLength(1); + expect(trace.edges).toEqual([]); + expect(trace).toMatchObject({ + truncated: true, + reason: "max_nodes_reached", + reached_depth: 0, + }); + }, + ); + + it.each([{ max_nodes: 2 }, { max_edges: 1 }])( + "reports retained node depth when a sibling hits the budget %j", + (limits) => { + const branching = { + ...graph, + edges: graph.edges + .filter((edge) => edge.resolution !== "unresolved") + .map((edge) => ({ ...edge, from: "a" })), + }; + const trace = traceNativeInvestigationGraph(branching, { + start: "a", + limits, + }); + expect(trace.nodes.map(({ id }) => id)).toEqual(["a", "b"]); + expect(trace.edges.map(({ id }) => id)).toEqual(["1"]); + expect(trace.reached_depth).toBe(1); + expect(trace.truncated).toBe(true); + }, + ); + + it("retains cycle edges to admitted nodes when the node budget is full", () => { + const cycle = { + ...graph, + nodes: graph.nodes.slice(0, 2), + edges: [ + graph.edges[0], + { ...graph.edges[0], id: "back", from: "b", to: "a" }, + ], + }; + const trace = traceNativeInvestigationGraph( + nativeInvestigationGraphSchema.parse(cycle), + { start: "a", limits: { max_nodes: 2 } }, + ); + expect(trace.edges.map(({ id }) => id)).toEqual(["1", "back"]); + expect(trace).toMatchObject({ reached_depth: 1, truncated: false }); + }); + it("retains observed and inferred route edges plus unresolved dispatch", () => { const trace = traceNativeInvestigationGraph(graph, { start: "a" }); expect(trace.nodes.map(({ id }) => id)).toEqual(["a", "b", "c"]); diff --git a/src/domain/native/nativeInvestigationGraph.ts b/src/domain/native/nativeInvestigationGraph.ts index 0dd0dbe8b..9b9efc412 100644 --- a/src/domain/native/nativeInvestigationGraph.ts +++ b/src/domain/native/nativeInvestigationGraph.ts @@ -207,7 +207,6 @@ export const traceNativeInvestigationGraph = ( while (queue.length > 0) { const current = queue.shift(); if (current === undefined) break; - reachedDepth = Math.max(reachedDepth, current.depth); const adjacent = adjacency.get(current.id) ?? []; for (const edge of adjacent) { @@ -227,19 +226,24 @@ export const traceNativeInvestigationGraph = ( reason ??= "max_depth_reached"; continue; } - selectedEdges.set(edge.id, edge); - if (alreadyVisited) continue; - if (visited.size >= limits.max_nodes) { + // Check destination capacity before retaining its edge. Otherwise the node + // bound leaves a resolved edge pointing outside the returned trace. + if (!alreadyVisited && visited.size >= limits.max_nodes) { truncated = true; reason = "max_nodes_reached"; break; } + selectedEdges.set(edge.id, edge); + if (alreadyVisited) continue; if (!nodeById.has(nextId)) { truncated = true; reason ??= "edge_target_missing_from_graph"; continue; } visited.add(nextId); + // A retained node has already been reached even if a later sibling + // exhausts a budget before this node is dequeued for expansion. + reachedDepth = Math.max(reachedDepth, current.depth + 1); queue.push({ id: nextId, depth: current.depth + 1 }); } if (reason === "max_edges_reached" || reason === "max_nodes_reached") break; diff --git a/src/dotnet/ManagedArtifactInspector.test.ts b/src/dotnet/ManagedArtifactInspector.test.ts index 2d30bf580..3b734c176 100644 --- a/src/dotnet/ManagedArtifactInspector.test.ts +++ b/src/dotnet/ManagedArtifactInspector.test.ts @@ -292,6 +292,30 @@ describe("managed artifact inventory", () => { ]); expect(result.coverage).toMatchObject({ state: "complete", issues: [] }); }); + + // dotnet/runtime corhdr.h: 32BITREQUIRED and 32BITPREFERRED form one + // two-bit field, so neither bit can be read on its own. + it.each([ + ["neither bit", 0x0000_0001, "anycpu", false], + ["32BITREQUIRED alone", 0x0000_0003, "x86", true], + ["both bits", 0x0002_0003, "anycpu-prefer-32", false], + ["the reserved 32BITPREFERRED alone", 0x0002_0001, "anycpu", false], + ] as const)( + "reads %s as one 32-bit requirement", + (_label, cliFlags, architecture, requires32Bit) => { + const bytes = buildManagedPeFixture({ cliFlags }); + const target = managedPeFixtureTarget(bytes); + + expect( + inspectManagedArtifactBytes(bytes, target).classification + .managed_architecture, + ).toBe(architecture); + expect( + inspectManagedNativeBoundariesBytes(bytes, target).cli_native + ?.requires_32bit, + ).toBe(requires32Bit); + }, + ); }); describe("managed artifact coded indexes", () => { diff --git a/src/dotnet/ManagedArtifactInspector.ts b/src/dotnet/ManagedArtifactInspector.ts index 7df8aae83..7e7acb087 100644 --- a/src/dotnet/ManagedArtifactInspector.ts +++ b/src/dotnet/ManagedArtifactInspector.ts @@ -9,6 +9,7 @@ import { type ManagedParseIssue, } from "../domain/managed/managedArtifact.js"; import { + managedCli32BitRequirement, readManagedPeLayout, type ManagedCliHeader, type ManagedPeLayout, @@ -249,9 +250,9 @@ const managedArchitectureFor = ( ): ManagedArtifactInspection["classification"]["managed_architecture"] => { if (layout.architecture !== "x86" || (cli.flags & 1) === 0) return layout.architecture; - if ((cli.flags & 0x0000_0002) !== 0) return "x86"; - if ((cli.flags & 0x0002_0000) !== 0) return "anycpu-prefer-32"; - return "anycpu"; + const requirement = managedCli32BitRequirement(cli.flags); + if (requirement === "required") return "x86"; + return requirement === "preferred" ? "anycpu-prefer-32" : "anycpu"; }; const runtimeClassification = ({ diff --git a/src/dotnet/ManagedNativeBoundaryHelpers.ts b/src/dotnet/ManagedNativeBoundaryHelpers.ts index fb81e1d28..e4b1a5960 100644 --- a/src/dotnet/ManagedNativeBoundaryHelpers.ts +++ b/src/dotnet/ManagedNativeBoundaryHelpers.ts @@ -18,7 +18,10 @@ import { } from "./ManagedMetadataHeaps.js"; import { managedTableRowCounts } from "./ManagedMetadataInventory.js"; import type { ManagedMetadataInventory } from "./ManagedMetadataInventory.js"; -import type { ManagedPeLayout } from "./ManagedPeReader.js"; +import { + managedCli32BitRequirement, + type ManagedPeLayout, +} from "./ManagedPeReader.js"; import { readManagedValue } from "./ManagedReaderFailure.js"; type ModuleRef = ManagedNativeBoundaryInspection["module_refs"][number]; @@ -380,7 +383,7 @@ export const cliNative = ( if (pe.cli === null) return null; return { il_only: (pe.cli.flags & 0x0000_0001) !== 0, - requires_32bit: (pe.cli.flags & 0x0000_0002) !== 0, + requires_32bit: managedCli32BitRequirement(pe.cli.flags) === "required", strong_name_signed: (pe.cli.flags & 0x0000_0008) !== 0, native_entry_point: (pe.cli.flags & 0x0000_0010) !== 0, ready_to_run_signature: pe.cli.readyToRunSignature, diff --git a/src/dotnet/ManagedPeReader.ts b/src/dotnet/ManagedPeReader.ts index 2f9bc3151..9a30c007b 100644 --- a/src/dotnet/ManagedPeReader.ts +++ b/src/dotnet/ManagedPeReader.ts @@ -31,6 +31,20 @@ export interface ManagedCliHeader { readonly readyToRunSignature: boolean; } +/** + * How the CLI header constrains process bitness. 32BITREQUIRED and + * 32BITPREFERRED are one two-bit field: required alone is x86-only, both set + * is platform-neutral code that prefers a 32-bit process, and preferred alone + * is reserved and constrains nothing. + */ +export const managedCli32BitRequirement = ( + flags: number, +): "required" | "preferred" | "none" => { + const pair = flags & 0x0002_0002; + if (pair === 0x0000_0002) return "required"; + return pair === 0x0002_0002 ? "preferred" : "none"; +}; + export interface ManagedPeLayout { readonly machine: number; readonly architecture: "x86" | "x86_64" | "arm" | "arm64"; diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 75d59c97f..333d2c913 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -195,6 +195,51 @@ describe("Ghidra function-analysis result values", () => { }); }); +describe("Ghidra Swift String literals", () => { + it("keeps a small Swift literal without storage apart from a large literal's bytes", () => { + const dossier = ghidraFunctionDossier(); + if ( + typeof dossier !== "object" || + dossier === null || + Array.isArray(dossier) + ) + throw new TypeError("Ghidra dossier fixture is invalid"); + const literals = [ + { + source_address: "0x401004", + value: "The user name ", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7265737520656854", "0xee0020656d616e20"], + }, + { + source_address: "0x401006", + value: "", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + { + source_address: "0x401008", + value: " cannot be written into a sudoers rule safely.", + representation: "large", + address: "0x402270", + registers: ["x0", "x1"], + words: ["0xd00000000000002e", "0x8000000000402250"], + }, + ]; + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + swift_string_literals: literals, + swift_string_literal_omissions: [], + }), + ).toMatchObject({ ok: true, value: { swift_string_literals: literals } }); + }); +}); + describe("Ghidra compact reference provenance", () => { it("distinguishes incoming references, examined emptiness and unavailable coverage at data addresses", () => { const instruction = { @@ -423,6 +468,34 @@ describe("Ghidra function-analysis malformed results", () => { ); }); +const malformedSwiftStringLiterals = ( + dossier: Record, +): Array<[string, GhidraFunctionOperation, JsonValue]> => + ( + [ + ["non-canonical Swift literal address", "large", "402240"], + ["small Swift literal with storage", "small", "0x402270"], + ["large Swift literal without storage", "large", null], + ] as const + ).map(([name, representation, address]) => [ + name, + "analyze_function", + { + ...dossier, + swift_string_literal_omissions: [], + swift_string_literals: [ + { + source_address: "0x401004", + value: "The power setting could not be changed.", + representation, + address, + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ]); + const malformedOutputs = (): Array< [string, GhidraFunctionOperation, JsonValue] > => { @@ -437,6 +510,11 @@ const malformedOutputs = (): Array< throw new TypeError("Ghidra native API fixture is invalid"); return [ ["non-canonical address", "procedure_callers", ["401000"]], + [ + "Swift literals without their omission record", + "analyze_function", + { ...dossier, swift_string_literals: [] }, + ], [ "missing classification", "procedure_info", @@ -467,6 +545,26 @@ const malformedOutputs = (): Array< Object.entries(dossier).filter(([key]) => key !== "native_api"), ), ], + ...malformedSwiftStringLiterals(dossier), + [ + "non-canonical Swift literal omission address", + "analyze_function", + { + ...dossier, + swift_string_literals: [], + swift_string_literal_omissions: [ + { + source_address: "0x401004", + representation: "large", + address: "402240", + byte_length: 39, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ], [ "non-canonical native API target", "analyze_function", diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index d45e41ba9..4e4c5054a 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -293,6 +293,15 @@ const ghidraFunctionDossier = functionDossierSchema }) .strict() .superRefine((value, context) => { + if ( + (value.swift_string_literals === undefined) !== + (value.swift_string_literal_omissions === undefined) + ) + context.addIssue({ + code: "custom", + message: + "Ghidra dossier must record Swift literals and their omissions together", + }); const addresses = [ value.procedure.address, ...value.comments.map(({ address }) => address), @@ -308,6 +317,14 @@ const ghidraFunctionDossier = functionDossierSchema address, source_address, ]), + ...(value.swift_string_literals ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), + ...(value.swift_string_literal_omissions ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), ...value.basic_blocks.flatMap(({ start, end, successors }) => [ start, end, diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index c6ca5212d..9675d133f 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -72,6 +72,15 @@ const NATIVE_ARCHITECTURES: readonly { /** Bound installation discovery reads; larger headers remain unknown. */ const EXECUTABLE_HEADER_PROBE_BYTES = 4096; +/** + * Bound the Java version probe so a chatty or hostile JVM cannot force an + * unbounded in-memory capture. Well-behaved JVMs print a few kilobytes for + * `-XshowSettings:properties`; the same 1 MiB budget used for other Java + * listings leaves ample headroom. A probe that exceeds the budget yields no + * observation and is reported through the existing failed-probe path. + */ +const JAVA_PROBE_MAX_OUTPUT_BYTES = 1024 * 1024; + /** Compatibility of one present native decompiler with the inspected host. */ type NativeDecompilerCompatibility = | { @@ -763,7 +772,7 @@ const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ encoding: "utf8", env: environment, timeout: 5_000, - maxBuffer: Number.POSITIVE_INFINITY, + maxBuffer: JAVA_PROBE_MAX_OUTPUT_BYTES, windowsHide: true, }, ); diff --git a/src/ghidra/GhidraLanguageCatalog.ts b/src/ghidra/GhidraLanguageCatalog.ts index 5da5943dc..6373f187f 100644 --- a/src/ghidra/GhidraLanguageCatalog.ts +++ b/src/ghidra/GhidraLanguageCatalog.ts @@ -75,8 +75,10 @@ export const readGhidraLanguageCatalog = async ( if (!entry.endsWith(".ldefs")) continue; const bytes = await readFile(join(directory, entry)); if (bytes.length > MAX_LDEFS_BYTES) continue; + // XML 1.0 section 4.3.3: a leading UTF-8 BOM is an encoding signature, + // not document content; TextDecoder consumes it, Buffer#toString does not. definitions.push( - ...parseGhidraLanguageDefinitions(bytes.toString("utf8")), + ...parseGhidraLanguageDefinitions(new TextDecoder().decode(bytes)), ); } } diff --git a/src/ghidra/GhidraLanguageOverride.test.ts b/src/ghidra/GhidraLanguageOverride.test.ts index 5f90a1454..02c5bf245 100644 --- a/src/ghidra/GhidraLanguageOverride.test.ts +++ b/src/ghidra/GhidraLanguageOverride.test.ts @@ -2,6 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { parseConfig } from "../config/parseConfig.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -190,6 +191,28 @@ describe("Ghidra language catalog", () => { ); }); + it("reads a UTF-8 signed definition beside an unsigned sibling", async () => { + const root = await createTestTempDirectory("rea-ghidra-bom-"); + const module = (name: string) => + join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(module("probe"), { recursive: true }); + await mkdir(module("x86"), { recursive: true }); + await writeFile( + join(module("probe"), "probe.ldefs"), + Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + Buffer.from( + '\n', + ), + ]), + ); + await writeFile(join(module("x86"), "x86.ldefs"), X86_LDEFS); + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "probe:LE:32:default", compilerSpecIds: [] }, + ...parseGhidraLanguageDefinitions(X86_LDEFS), + ]); + }); + it("admits only languages and compiler specs the installation defines", () => { const catalog = parseGhidraLanguageDefinitions(X86_LDEFS); expect( diff --git a/src/ghidra/GhidraLauncher.ts b/src/ghidra/GhidraLauncher.ts index 59c555606..e1527ea81 100644 --- a/src/ghidra/GhidraLauncher.ts +++ b/src/ghidra/GhidraLauncher.ts @@ -1,3 +1,4 @@ +import { parseGhidraJavaProperties } from "./GhidraJavaProperties.js"; import { GHIDRA_SEED_SCRIPT, snapshotGhidraAnalysisSeeds, @@ -5,7 +6,7 @@ import { } from "./GhidraAnalysisSeeds.js"; import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; -import { mkdir, realpath } from "node:fs/promises"; +import { mkdir, readFile, realpath } from "node:fs/promises"; import { basename, dirname, join, posix, win32 } from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -219,7 +220,11 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { "Installed Allegrex content changed since profile resolution; reopen the target to resolve a fresh profile.", ); } - await createGhidraRuntimeDirectories(paths, platform); + await createGhidraRuntimeDirectories( + paths, + platform, + await ghidraConfigReleaseDirectory(this.options.analyzeHeadlessPath), + ); const seedPath = this.options.analysisSeeds === undefined ? undefined @@ -721,9 +726,44 @@ const ghidraRuntimePaths = (runtimeRoot: string) => ({ scriptLogPath: join(runtimeRoot, "script.log"), }); +/** + * Ghidra's launch support resolves its JDK before the headless run and then + * persists that choice under `/ghidra/`. Inside + * REA's Windows private runtime a child JVM cannot create those directories + * itself, so startup aborted with "Failed to create directory" and reported + * that no supported JDK was found. Create the release directory while this + * process still owns the runtime, leaving Ghidra only a file write inside it. + */ +const ghidraConfigReleaseDirectory = async ( + analyzeHeadlessPath: string, +): Promise => { + const installRoot = dirname(dirname(analyzeHeadlessPath)); + try { + const properties = await readFile( + join(installRoot, "Ghidra", "application.properties"), + "latin1", + ); + const parsed = parseGhidraJavaProperties(properties); + if (!parsed.ok) return basename(installRoot); + const version = parsed.value.get("application.version"); + const release = parsed.value.get("application.release.name"); + return version === undefined || + release === undefined || + release.length === 0 + ? basename(installRoot) + : `ghidra_${version}_${release}`; + } catch (cause: unknown) { + // An unreadable properties file leaves the release coordinate unknown; the + // conventional extraction directory carries the same name. + void cause; + return basename(installRoot); + } +}; + const createGhidraRuntimeDirectories = async ( paths: ReturnType, platform: NodeJS.Platform, + configReleaseDirectory: string, ): Promise => { if (platform === "win32") { const runtime = windowsPrivateRuntime(dirname(paths.projectRoot)); @@ -736,6 +776,11 @@ const createGhidraRuntimeDirectories = async ( paths.dataRoot, ]) runtime.mkdir(basename(path)); + // Each level needs its parent first: the native owner opens every existing + // component and creates only the final one. + const configRoot = basename(paths.configRoot); + runtime.mkdir(win32.join(configRoot, "ghidra")); + runtime.mkdir(win32.join(configRoot, "ghidra", configReleaseDirectory)); return; } await Promise.all( @@ -746,6 +791,7 @@ const createGhidraRuntimeDirectories = async ( paths.cacheRoot, paths.configRoot, paths.dataRoot, + join(paths.configRoot, "ghidra", configReleaseDirectory), ].map((path) => mkdir(path, { recursive: true, mode: 0o700 })), ); }; diff --git a/src/ida/IdaContainment.test.ts b/src/ida/IdaContainment.test.ts new file mode 100644 index 000000000..069388df8 --- /dev/null +++ b/src/ida/IdaContainment.test.ts @@ -0,0 +1,623 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { decodeIdaToolResult } from "./IdaMcpConnection.js"; +import { idaCapabilities } from "./IdaProviderCapabilities.js"; +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const segment = (name: string, start: string, end: string) => ({ + name, + start, + end, + size: `0x${(BigInt(end) - BigInt(start)).toString(16)}`, + permissions: "r--", +}); + +/** Adjacent .text and .data, a gap, then a segment above 2^53. */ +const SEGMENTS = [ + segment(".text", "0x1000", "0x2000"), + segment(".data", "0x2000", "0x3000"), + segment(".far", "0x5000", "0x6000"), + segment(".huge", "0x20000000000001", "0x20000000000002"), +]; + +/** The same segments in descending order, which a scan may not rely on. */ +const UNSORTED = [...SEGMENTS].reverse(); + +const survey = (segments: JsonValue = SEGMENTS) => ({ + metadata: { path: "/db/target.i64", module: "target", arch: "metapc" }, + statistics: { total_segments: Array.isArray(segments) ? segments.length : 0 }, + segments, + entrypoints: [], +}); + +const function_ = { addr: "0x1000", name: "main", size: "0x20" }; + +const MODERN_WITHOUT_SURVEY = [ + "idb_open", + "idb_list", + "idb_close", + "server_health", + "list_funcs", + "lookup_funcs", + "find_regex", + "decompile", + "disasm", + "callees", + "xrefs_to", +]; + +interface Upstream { + /** Answer for the lookup of the queried address; default is "Not a function". */ + readonly lookup?: (query: string) => JsonValue; + readonly survey?: JsonValue; + readonly tools?: readonly string[]; + readonly surveyFailure?: "error" | "preview"; + /** A distinct source alias of the admitted target, as for an app bundle. */ + readonly sourcePath?: string; + /** Report a different input from the second health read on. */ + readonly identityChangesAfterOperation?: boolean; +} + +const session = async ( + upstream: Upstream = {}, + mode: "headless" | "attached" = "headless", +) => { + const created = await createIdaTarget("x86_64"); + const { root } = created; + const target = + upstream.sourcePath === undefined + ? created.target + : { ...created.target, sourcePath: upstream.sourcePath }; + roots.push(root); + const producer = new RecordingIdaMcp(target, mode); + if (upstream.tools !== undefined) producer.tools = upstream.tools; + producer.beforeCall = async (name) => { + if (name !== "survey_binary") return; + if (upstream.surveyFailure === "error") + decodeIdaToolResult({ + content: [{ type: "text", text: "survey failed" }], + isError: true, + }); + if (upstream.surveyFailure === "preview") + decodeIdaToolResult({ + content: [{ type: "text", text: "preview" }], + _meta: { ida_mcp: { output_truncated: true } }, + }); + }; + let healthReads = 0; + producer.overrideCall = (name, args) => { + if (name === "server_health" && upstream.identityChangesAfterOperation) { + healthReads += 1; + if (healthReads === 2) + return { + status: "ok", + input_path: "/different/target.elf", + idb_path: "/different/target.elf.i64", + module: "target.elf", + imagebase: "0x1000", + auto_analysis_ready: true, + }; + } + if (name === "lookup_funcs") { + const queries = Array.isArray(args.queries) ? args.queries : []; + return queries.map((query) => + upstream.lookup === undefined + ? { query, fn: null, error: "Not a function" } + : upstream.lookup(String(query)), + ); + } + if (name === "survey_binary") return upstream.survey ?? survey(); + return undefined; + }; + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode, + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, producer, target }; +}; + +const containing = async ( + client: IdaSessionClient, + address: string, + options?: Parameters[2], +) => client.execute("resolve_containing_procedure", { address }, options); + +const found = (query: string, fn: typeof function_) => ({ + query, + fn, + error: null, +}); + +describe("IDA containment capabilities", () => { + it("declares both operations for the headless profile only", () => { + const declared = (mode: "attached" | "headless", native = false) => + idaCapabilities(mode, false, native) + .map(({ operation }) => operation) + .filter( + (operation) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ); + + expect( + idaCapabilities("headless", false) + .filter( + ({ operation }) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ) + .map(({ available }) => available), + "both are declared available, not merely listed", + ).toEqual([true, true]); + expect(declared("headless")).toEqual([ + "resolve_containing_procedure", + "list_documents", + ]); + expect(declared("attached")).toEqual([]); + expect(declared("attached", true)).toEqual([]); + expect( + idaCapabilities("attached", false).map(({ operation }) => operation), + "attached profiles do not even declare them", + ).not.toContain("list_documents"); + }); + + it.each(["resolve_containing_procedure", "list_documents"] as const)( + "refuses %s on the legacy attached profile without calling upstream", + async (operation) => { + const { client, producer } = await session({}, "attached"); + try { + const result = await client.execute( + operation, + operation === "list_documents" ? {} : { address: "0x1000" }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( + false, + ); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containing procedure", () => { + it("returns the canonical entry for an interior address without reading segments", async () => { + const { client, producer } = await session({ + lookup: (query) => found(query, function_), + }); + try { + const result = await containing(client, "1ffe"); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: "0x1ffe", + found: true, + procedure: { + address: "0x1000", + name: "main", + classification: null, + body: { + available: false, + reason: "IDA MCP does not report complete function body ranges.", + }, + }, + }); + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "the normalized address is the only query", + ).toMatchObject([{ args: { queries: ["0x1ffe"] } }]); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "a positive answer needs no segment facts", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it.each([ + ["inside the first segment", "0x1000", "not_in_procedure"], + ["inside an adjacent segment", "0x2000", "not_in_procedure"], + [ + "at the exclusive end of the last adjacent segment", + "0x3000", + "outside_segments", + ], + ["in the gap between segments", "0x4000", "outside_segments"], + ["one below a segment start", "0xfff", "outside_segments"], + ["the last byte of a segment", "0x5fff", "not_in_procedure"], + ["above every segment", "0x6000", "outside_segments"], + [ + "inside a one-byte segment above 2^53", + "0x20000000000001", + "not_in_procedure", + ], + [ + "one below that segment, which a rounded number cannot tell apart", + "0x20000000000000", + "outside_segments", + ], + [ + "at the exclusive end of that segment", + "0x20000000000002", + "outside_segments", + ], + ] as const)( + "reports an address that is not a function %s as %s", + async (_description, address, reason) => { + const { client, producer } = await session(); + try { + const result = await containing(client, address); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: address, + found: false, + procedure: null, + reason, + }); + expect( + producer.calls.find(({ name }) => name === "survey_binary")?.args, + "the minimal survey is the only segment source", + ).toMatchObject({ detail_level: "minimal" }); + expect( + producer.calls.filter(({ name }) => name === "survey_binary"), + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containment failures", () => { + it.each([ + ["no segments field", { metadata: {}, statistics: {} }], + ["segments that are null", survey(null)], + ["an empty segment list", survey([])], + [ + "a row with a malformed address", + survey([ + { + name: ".x", + start: "0xzz", + end: "0x2000", + size: "0x1000", + permissions: "r--", + }, + ]), + ], + [ + "a reversed segment", + survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + ], + [ + "a size that disagrees with the bounds", + survey([{ ...segment(".x", "0x1000", "0x2000"), size: "0x10" }]), + ], + [ + "a total that disagrees with the rows", + { ...survey(), statistics: { total_segments: 9 } }, + ], + ["a malformed row after valid ones", survey([...SEGMENTS, { name: ".y" }])], + ] as const)( + "fails instead of guessing a reason when the survey has %s", + async (_description, reply) => { + const { client } = await session({ survey: reply as JsonValue }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisOutputError"); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it.each(["error", "preview"] as const)( + "fails when the survey tool reports %s", + async (surveyFailure) => { + const { client } = await session({ surveyFailure }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it("fails without a segment source when the server does not advertise the survey", async () => { + const { client, producer } = await session({ + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const negative = await containing(client, "0x4000"); + + expect(negative.ok).toBe(false); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "an unadvertised tool is never requested", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("still answers a positive lookup when the server does not advertise the survey", async () => { + const { client } = await session({ + lookup: (query) => found(query, function_), + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const result = await containing(client, "0x1010"); + + expect(result.ok).toBe(true); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA containment lookup failures", () => { + it.each([ + ["an answer for another query", () => found("0x9999", function_)], + [ + "a function together with an error", + (query: string) => ({ query, fn: function_, error: "Not found" }), + ], + [ + "neither a function nor an error", + (query: string) => ({ query, fn: null, error: null }), + ], + [ + "an unknown error text", + (query: string) => ({ query, fn: null, error: "boom" }), + ], + [ + "a padded missing answer", + (query: string) => ({ query, fn: null, error: " Not a function" }), + ], + ] as const)( + "treats %s as a producer failure and reads no segments", + async (_description, lookup) => { + const { client, producer } = await session({ lookup }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisProtocolError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it('treats "Not found" for an address like any expected missing answer', async () => { + const { client } = await session({ + lookup: (query) => ({ query, fn: null, error: "Not found" }), + }); + try { + const result = await containing(client, "0x4000"); + + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + found: false, + reason: "outside_segments", + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops before the survey when the request is cancelled after the lookup", async () => { + const controller = new AbortController(); + const { client, producer } = await session(); + const inner = producer.beforeCall; + producer.beforeCall = async (name, args) => { + await inner?.(name, args); + if (name === "lookup_funcs") controller.abort(); + }; + try { + const result = await containing(client, "0x4000", { + signal: controller.signal, + }); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "nothing is dispatched after cancellation", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("does not depend on the order of the surveyed segments", async () => { + const { client } = await session({ survey: survey(UNSORTED) }); + try { + const inside = await containing(client, "0x1000"); + const gap = await containing(client, "0x4000"); + + if (!inside.ok) throw inside.error; + if (!gap.ok) throw gap.error; + expect(inside.value.result).toMatchObject({ reason: "not_in_procedure" }); + expect(gap.value.result).toMatchObject({ reason: "outside_segments" }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("retains the completed replies when the survey is rejected", async () => { + const { client } = await session({ + survey: survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error.partialObservation).toMatchObject({ + normalized_result: { + status: "partial", + observations: [ + expect.objectContaining({ tool: "lookup_funcs" }), + expect.objectContaining({ tool: "survey_binary" }), + ], + }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("rejects an address that is not hexadecimal and a document that is not the admitted target", async () => { + const { client, producer } = await session(); + try { + const badAddress = await containing(client, "main"); + const badDocument = await client.execute("resolve_containing_procedure", { + address: "0x1000", + document: "/somewhere/else", + }); + + for (const result of [badAddress, badDocument]) { + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "invalid input never reaches upstream", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA document identity", () => { + it("accepts an explicit admitted document and rejects an unknown one for the same operations", async () => { + const { client, target, producer } = await session(); + try { + const admitted = await client.execute("resolve_containing_procedure", { + address: "0x4000", + document: target.path, + }); + const unknownContainment = await client.execute( + "resolve_containing_procedure", + { address: "0x4000", document: "/somewhere/else" }, + ); + const unknownListing = await client.execute("list_documents", { + document: "/somewhere/else", + }); + + expect(admitted.ok, "the admitted program is accepted").toBe(true); + for (const rejected of [unknownContainment, unknownListing]) { + expect(rejected.ok).toBe(false); + if (!rejected.ok) + expect(rejected.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "only the admitted request reached upstream", + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists the admitted path, not its source alias or the private copy", async () => { + const { client, target, producer } = await session({ + sourcePath: "/Applications/Fixture.app", + }); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + expect( + producer.calls.filter(({ name }) => name === "server_health"), + "the target identity is checked before and after the local answer", + ).toHaveLength(2); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("fails when the target identity changes during the operation", async () => { + const { client } = await session({ identityChangesAfterOperation: true }); + try { + const result = await client.execute("list_documents", {}); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.message).toContain("identity changed"); + } + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops without dispatching when the request is already cancelled", async () => { + const controller = new AbortController(); + controller.abort(); + const { client, producer } = await session(); + try { + const result = await client.execute( + "list_documents", + {}, + { signal: controller.signal }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(producer.calls).toEqual([]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists exactly the admitted program", async () => { + const { client, target } = await session(); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts index d59f598dd..5da9d40f7 100644 --- a/src/ida/IdaNativeProtocol.test.ts +++ b/src/ida/IdaNativeProtocol.test.ts @@ -758,3 +758,33 @@ describe("native IDA malformed observations", () => { await client.close(); }); }); + +describe("native IDA attached operation admission", () => { + it("forwards the native producer's explicit reference cap flag", async () => { + const { client } = await fixture(); + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBe(false); + await client.close(); + }); + + it("does not admit the headless-only containment and document operations", async () => { + const { client, producer } = await fixture(); + for (const [operation, parameters] of [ + ["resolve_containing_procedure", { address: "0x1000" }], + ["list_documents", {}], + ] as const) { + const result = await client.execute(operation, parameters); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "nothing is dispatched upstream", + ).toBe(false); + await client.close(); + }); +}); diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index c30ec81bc..25a49c268 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -45,6 +45,7 @@ import { modernStringsSchema, modernXrefsSchema, nextOffset, + surveySegmentsSchema, type IdaDisassembly, type IdaFunction, } from "./IdaProtocolValues.js"; @@ -86,6 +87,8 @@ export class IdaOperationRunner { readonly locations: EvidenceLocation[] = []; readonly limitations: string[] = []; relationshipCoverage: RelationshipCoverage | undefined; + /** The upstream cap flag of the last `xrefs_to` read, when the producer reported one. */ + referencesTruncated: boolean | undefined; constructor( private readonly connection: IdaMcpConnection, private readonly protocol: "legacy" | "modern" | "native", @@ -97,6 +100,10 @@ export class IdaOperationRunner { readonly x86Only?: boolean; /** False when the upstream server does not advertise `callgraph`. */ readonly callGraphAvailable?: boolean; + /** False when the upstream server does not advertise `survey_binary`. */ + readonly surveyAvailable?: boolean; + /** The program identity REA admitted, reported by `list_documents`. */ + readonly admittedDocument?: string; } = {}, ) {} @@ -177,41 +184,89 @@ export class IdaOperationRunner { return entry; } - private lookupFunction( + /** Parse one lookup row: a function, an expected "missing" answer, or a producer failure. */ + private lookupOutcome( query: string, item: z.infer[number] | undefined, - requestedBy?: string, - ): IdaFunction { + ): { readonly fn: IdaFunction } | { readonly missing: string } { if (item?.query !== query) throw new AnalysisProtocolError( `IDA function lookup answered for a different query than ${JSON.stringify(query)}.`, ); if ( - requestedBy !== undefined && item.fn === null && item.error !== null && MISSING_PROCEDURE_ANSWERS.has(item.error) ) - throw new AnalysisInputError( - requestedBy, - { - cause: new AnalysisProtocolError( - `IDA function lookup failed for ${query}: ${item.error}`, - ), - }, - [ - { - path: [], - reason: "invalid_value", - message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${item.error}.`, - }, - ], - ); + return { missing: item.error }; if (item.fn === null || item.error !== null) throw new AnalysisProtocolError( `IDA function lookup failed for ${query}: ${item.error ?? "missing function"}`, ); - return item.fn; + return { fn: item.fn }; + } + + private lookupFunction( + query: string, + item: z.infer[number] | undefined, + requestedBy?: string, + ): IdaFunction { + const outcome = this.lookupOutcome(query, item); + if ("fn" in outcome) return outcome.fn; + if (requestedBy === undefined) + throw new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ); + throw new AnalysisInputError( + requestedBy, + { + cause: new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ), + }, + [ + { + path: [], + reason: "invalid_value", + message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${outcome.missing}.`, + }, + ], + ); + } + + /** The containing function of an address, or why none contains it. */ + private async containingProcedure(address: string): Promise { + const item = modernLookupSchema.parse( + await this.call("lookup_funcs", { queries: [address] }), + )[0]; + const outcome = this.lookupOutcome(address, item); + if ("fn" in outcome) + return { + query_address: address, + found: true, + procedure: identity(outcome.fn), + }; + return { + query_address: address, + found: false, + procedure: null, + reason: (await this.addressInSegment(address)) + ? "not_in_procedure" + : "outside_segments", + }; + } + + /** Segment membership from the database-scoped minimal survey; a missing fact is a failure. */ + private async addressInSegment(address: string): Promise { + if (this.options.surveyAvailable === false) + throw new AnalysisProtocolError( + "The upstream server does not advertise survey_binary, so segment membership cannot be established.", + ); + const { segments } = surveySegmentsSchema.parse( + await this.call("survey_binary", { detail_level: "minimal" }), + ); + const value = BigInt(address); + return segments.some(({ start, end }) => start <= value && value < end); } /** True only for modern callee reads on targets not proven x86-only; x86 reads never reach the graph code. */ @@ -390,6 +445,7 @@ export class IdaOperationRunner { throw new AnalysisProtocolError( `IDA xrefs failed: ${page?.error ?? "missing references"}`, ); + if (page.more !== undefined) this.referencesTruncated = page.more; if (page.more === true) this.limitations.push( "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", @@ -536,6 +592,16 @@ export class IdaOperationRunner { address === idaAddressSchema.parse(parameters.address), ); } + if (operation === "list_documents") { + if (this.options.admittedDocument === undefined) + throw new AnalysisProtocolError("The admitted IDA program is unknown."); + return [this.options.admittedDocument]; + } + if (operation === "resolve_containing_procedure") { + const queried = idaAddressSchema.parse(parameters.address); + this.locations.push({ kind: "address", address: queried }); + return this.containingProcedure(queried); + } if (operation === "xrefs") { const inspectedAddress = idaAddressSchema.parse(parameters.address); this.locations.push({ kind: "address", address: inspectedAddress }); diff --git a/src/ida/IdaProtocolValues.ts b/src/ida/IdaProtocolValues.ts index 93ddbef88..f18e85ee0 100644 --- a/src/ida/IdaProtocolValues.ts +++ b/src/ida/IdaProtocolValues.ts @@ -218,6 +218,14 @@ export const openDatabaseSchema = z .passthrough(), }) .passthrough(); +/** Only an explicit completed rejection preserves legacy no-worker cleanup. */ +export const rejectedOpenDatabaseSchema = z + .object({ + error: z.string().min(1), + success: z.never().optional(), + }) + .passthrough(); + export const databaseListSchema = z .object({ sessions: z.array( @@ -247,3 +255,53 @@ export const absentDatabaseCloseSchema = (sessionId: string) => z .object({ error: z.literal(`Session not found: ${sessionId}`) }) .passthrough(); + +// Piped, so a malformed coordinate is a schema issue and never reaches BigInt. +const hexCoordinate = z + .string() + .regex(/^0x[0-9a-f]+$/iu) + .pipe(z.transform((value) => BigInt(value))); + +/** + * The minimal `survey_binary` reply, reduced to validated segment intervals + * (exclusive ends). Any malformed, reversed or inconsistent fact rejects the + * whole reply, because a membership answer needs every segment. + */ +export const surveySegmentsSchema = z + .object({ + statistics: z + .object({ total_segments: z.number().int().nonnegative().optional() }) + .passthrough() + .optional(), + segments: z + .array( + z + .object({ + start: hexCoordinate, + end: hexCoordinate, + size: hexCoordinate.optional(), + }) + .passthrough(), + ) + .min(1), + }) + .passthrough() + .superRefine(({ statistics, segments }, context) => { + if ( + statistics?.total_segments !== undefined && + statistics.total_segments !== segments.length + ) + context.addIssue({ + code: "custom", + message: "survey segment count disagrees with its statistics", + }); + for (const { start, end, size } of segments) { + // A malformed row has already reported its own issue; refinements still run. + if (typeof start !== "bigint" || typeof end !== "bigint") continue; + if (start >= end || (size !== undefined && size !== end - start)) + context.addIssue({ + code: "custom", + message: "survey segment bounds are reversed or inconsistent", + }); + } + }); diff --git a/src/ida/IdaProviderCapabilities.ts b/src/ida/IdaProviderCapabilities.ts index d52c949e0..0ccc016d2 100644 --- a/src/ida/IdaProviderCapabilities.ts +++ b/src/ida/IdaProviderCapabilities.ts @@ -24,9 +24,23 @@ export const IDA_OPERATIONS = Object.freeze([ "read_function_instructions", "xrefs", "analyze_function", + "resolve_containing_procedure", + "list_documents", ] as const satisfies readonly Exclude[]); export type IdaOperation = (typeof IDA_OPERATIONS)[number]; +/** Operations only the modern headless profile implements. */ +const HEADLESS_ONLY_OPERATIONS: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + +/** Whether the configured profile admits an operation. */ +export const idaProfileAdmits = ( + operation: IdaOperation, + mode: "attached" | "headless", +): boolean => mode === "headless" || !HEADLESS_ONLY_OPERATIONS.has(operation); + /** Shared limitations of observations from an externally mutable analysis database. */ export const IDA_LIMITATIONS = Object.freeze([ "Provider version identifies the REA IDA adapter. IDA engine and upstream distribution versions are unknown unless separately observed; MCP serverInfo is reported independently.", @@ -58,34 +72,37 @@ export const idaCapabilities = ( stdio: boolean, native = false, ): readonly CapabilityDescriptor[] => - IDA_OPERATIONS.map((operation) => ({ - provider: IDA_PROVIDER_IDENTITY, - operation, - ...((mode === "headless" || native) && operation === "procedure_callers" - ? ({ - available: false, - reason: native - ? "The native upstream profile reports code references without proving direct call edges." - : "The modern upstream profile reports code references without proving direct call edges.", - } as const) - : ({ available: true, reason: null } as const)), - cachePolicy: "live", - effects: { - mutatesArtifact: false, - launchesProcess: stdio || mode === "headless", - mayShowUi: - mode === "attached" && - (native - ? operation === "procedure_pseudo_code" || - operation === "analyze_function" - : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), - mayAccessNetwork: true, - mayWriteFilesystem: stdio || mode === "headless", - changesPermissions: false, - requiresRoot: false, - }, - limitations: - native && (operation === "list_strings" || operation === "search_strings") - ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] - : IDA_LIMITATIONS, - })); + IDA_OPERATIONS.filter((operation) => idaProfileAdmits(operation, mode)).map( + (operation) => ({ + provider: IDA_PROVIDER_IDENTITY, + operation, + ...((mode === "headless" || native) && operation === "procedure_callers" + ? ({ + available: false, + reason: native + ? "The native upstream profile reports code references without proving direct call edges." + : "The modern upstream profile reports code references without proving direct call edges.", + } as const) + : ({ available: true, reason: null } as const)), + cachePolicy: "live", + effects: { + mutatesArtifact: false, + launchesProcess: stdio || mode === "headless", + mayShowUi: + mode === "attached" && + (native + ? operation === "procedure_pseudo_code" || + operation === "analyze_function" + : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), + mayAccessNetwork: true, + mayWriteFilesystem: stdio || mode === "headless", + changesPermissions: false, + requiresRoot: false, + }, + limitations: + native && + (operation === "list_strings" || operation === "search_strings") + ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] + : IDA_LIMITATIONS, + }), + ); diff --git a/src/ida/IdaReferenceCoverage.test.ts b/src/ida/IdaReferenceCoverage.test.ts new file mode 100644 index 000000000..930c3a88a --- /dev/null +++ b/src/ida/IdaReferenceCoverage.test.ts @@ -0,0 +1,96 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { + callGraphSession as session, + removeCallGraphSessions, +} from "../../tests/fixtures/idaCallGraph.js"; + +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const attachedRoots: string[] = []; +afterEach(async () => { + await removeCallGraphSessions(); + for (const root of attachedRoots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const xrefs = async (more: boolean | undefined) => { + const { client, producer } = await session({ callees: [] }, "x86_64"); + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [{ xrefs: [], ...(more === undefined ? {} : { more }) }] + : inner?.(name, args); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + if (!result.ok) throw result.error; + return result.value; + } finally { + expect((await client.close()).ok).toBe(true); + } +}; + +describe("headless IDA reference coverage", () => { + it("reports a capped xrefs read as truncated", async () => { + const execution = await xrefs(true); + + expect(execution.referencesTruncated).toBe(true); + expect( + execution.limitations, + "the diagnostic limitation is still reported", + ).toContain( + "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", + ); + }); + + it("reports an explicitly uncapped xrefs read as not truncated", async () => { + expect((await xrefs(false)).referencesTruncated).toBe(false); + }); + + it("reports nothing when the producer does not say whether the read was capped", async () => { + expect((await xrefs(undefined)).referencesTruncated).toBeUndefined(); + }); + + it("does not report reference coverage for other operations", async () => { + const { client } = await session({ callees: [] }, "x86_64"); + try { + const result = await client.execute("procedure_callees", { + procedure: "main", + }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("reports nothing for the legacy profile, whose producer has no cap flag", async () => { + const { root, target } = await createIdaTarget("x86_64"); + attachedRoots.push(root); + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "attached", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + new RecordingIdaMcp(target, "attached"), + ); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaSessionClient.test.ts b/src/ida/IdaSessionClient.test.ts index e43f18cf7..b41a3c34a 100644 --- a/src/ida/IdaSessionClient.test.ts +++ b/src/ida/IdaSessionClient.test.ts @@ -515,7 +515,7 @@ describe("headless IDA transient busy health", () => { }); it.each([ - ["free-form text", "token=abc123 /private/home/user secret"], + ["free-form text", "token=abc123 /private secret"], ["an over-long name", "x".repeat(200)], ])( "reports %s from the worker as unknown instead of echoing it", @@ -899,6 +899,85 @@ describe("headless IDA reaped worker cleanup", () => { }); }); +/** A completed open response is not proof that its private Worker was ever released. */ +describe("headless IDA unresolved successful-open ownership", () => { + it.each(["matching", "foreign-input", "malformed"] as const)( + "retains the private input after %s success with incomplete initial discovery", + async (kind) => { + const { producer, client, root } = await fixture("headless"); + let hidden = true; + let ownedId: string | undefined; + let privateInput: string | undefined; + producer.beforeCall = async (name, args) => { + if (name !== "idb_open") return; + ownedId = z.string().parse(args.preferred_session_id); + privateInput = z.string().parse(args.input_path); + producer.database = ownedId; + producer.inputPath = privateInput; + }; + producer.overrideCall = (name) => { + if (name === "idb_open") { + const originalId = z.string().parse(ownedId); + const originalInput = z.string().parse(privateInput); + if (kind === "foreign-input") + return { + success: true, + session: { + session_id: "foreign-session", + input_path: "/foreign/input", + }, + }; + if (kind === "malformed") + return { success: true, session: { session_id: originalId } }; + return { + success: true, + session: { session_id: originalId, input_path: originalInput }, + }; + } + if (name === "idb_list" && hidden) return { sessions: [] }; + return undefined; + }; + + expect((await client.execute("health", {})).ok).toBe(false); + const incomplete = await client.close(); + expect(incomplete.ok).toBe(false); + if (!incomplete.ok) expect(incomplete.error.cleanupIncomplete).toBe(true); + expect(producer.closes).toBe(0); + expect(producer.database).toBe(ownedId); + expect( + (await readdir(root)).some((name) => name.startsWith("rea-ida-")), + ).toBe(true); + expect(producer.calls.some(({ name }) => name === "idb_close")).toBe( + false, + ); + + // A later positive private ownership record permits ordinary save=false + // cleanup, without ever adopting the foreign open-response identity. + hidden = false; + expect((await client.close()).ok).toBe(true); + expect(producer.calls.filter(({ name }) => name === "idb_close")).toEqual( + [{ name: "idb_close", args: { database: ownedId, save: false } }], + ); + expect(producer.closes).toBe(1); + expect(await readdir(root)).toEqual(["target.elf"]); + }, + ); + + it("preserves cleanup after a completed, explicit idb_open rejection", async () => { + const { producer, client, root } = await fixture("headless"); + producer.overrideCall = (name) => + name === "idb_open" + ? { error: "Input was rejected before creating a session" } + : undefined; + + expect((await client.execute("health", {})).ok).toBe(false); + expect((await client.close()).ok).toBe(true); + expect(producer.calls.some(({ name }) => name === "idb_close")).toBe(false); + expect(producer.closes).toBe(1); + expect(await readdir(root)).toEqual(["target.elf"]); + }); +}); + describe("headless IDA cleanup verification", () => { it("retries failed worker release through the retained connection before removing its workspace", async () => { const { producer, client, root } = await fixture("headless"); @@ -943,23 +1022,6 @@ describe("headless IDA cleanup verification", () => { ).toHaveLength(1); expect(await readdir(root)).toEqual(["target.elf"]); }); - it("never closes an unrelated database when startup returns a different input identity", async () => { - const { producer, client, root } = await fixture("headless"); - producer.overrideCall = (name) => - name === "idb_open" - ? { - success: true, - session: { - session_id: "user-database", - input_path: "/unrelated/input", - }, - } - : undefined; - expect((await client.execute("health", {})).ok).toBe(false); - expect((await client.close()).ok).toBe(true); - expect(producer.calls.some(({ name }) => name === "idb_close")).toBe(false); - expect(await readdir(root)).toEqual(["target.elf"]); - }); it("releases a privately owned worker after malformed open output using verified session inventory", async () => { const { producer, client, root } = await fixture("headless"); producer.beforeCall = async (name, args) => { diff --git a/src/ida/IdaSessionClient.ts b/src/ida/IdaSessionClient.ts index 7d3ec066c..14b20b107 100644 --- a/src/ida/IdaSessionClient.ts +++ b/src/ida/IdaSessionClient.ts @@ -33,6 +33,7 @@ import { IDA_LIMITATIONS, IDA_OPERATIONS, IDA_PROVIDER_IDENTITY, + idaProfileAdmits, type IdaOperation, } from "./IdaProviderCapabilities.js"; import { @@ -42,6 +43,7 @@ import { legacyMetadataSchema, modernHealthSchema, openDatabaseSchema, + rejectedOpenDatabaseSchema, } from "./IdaProtocolValues.js"; import { idaFileDigest, @@ -121,7 +123,9 @@ export class IdaSessionClient implements AnalysisClient { #databaseCloseCompleted = false; #openAttempted = false; #openCompleted = false; + #openRejected = false; #callGraphAdvertised = false; + #surveyAdvertised = false; #initialIdentity: string | undefined; #metadata: JsonValue = null; #lifecycle: JsonValue[] = []; @@ -194,6 +198,17 @@ export class IdaSessionClient implements AnalysisClient { "This integration admits read-only analysis operations only.", ), ); + if ( + operation !== "health" && + !idaProfileAdmits(operation, this.config.mode) + ) + return err( + new AnalysisCapabilityUnavailableError( + "ida", + operation, + "This operation is available for the modern headless profile only.", + ), + ); if ( operation === "procedure_callers" && idaProtocol(this.config) !== "legacy" @@ -269,6 +284,9 @@ export class IdaSessionClient implements AnalysisClient { ...(runner.relationshipCoverage === undefined ? {} : { relationshipCoverage: runner.relationshipCoverage }), + ...(runner.referencesTruncated !== undefined + ? { referencesTruncated: runner.referencesTruncated } + : {}), locations: [ { kind: "artifact-path", path: this.target.path }, ...runner.locations, @@ -293,6 +311,8 @@ export class IdaSessionClient implements AnalysisClient { selectorOrigin: options?.selectorOrigin, x86Only: isX86OnlyTarget(this.target), callGraphAvailable: this.#callGraphAdvertised, + surveyAvailable: this.#surveyAdvertised, + admittedDocument: this.target.path, }, ); } @@ -423,6 +443,7 @@ export class IdaSessionClient implements AnalysisClient { async #initialize(): Promise { const tools = new Set(await this.connection.connect()); this.#callGraphAdvertised = tools.has("callgraph"); + this.#surveyAdvertised = tools.has("survey_binary"); const expected = this.config.protocol === "native" ? nativeTools @@ -450,6 +471,9 @@ export class IdaSessionClient implements AnalysisClient { preferred_session_id: this.#workspace.sessionId, }); this.#openCompleted = true; + // A completed but malformed/positive open may still own a live Worker. + // Only an explicit producer rejection keeps the old no-worker branch. + this.#openRejected = rejectedOpenDatabaseSchema.safeParse(raw).success; this.#lifecycle.push(raw); const opened = openDatabaseSchema.parse(raw); if ( @@ -614,6 +638,14 @@ export class IdaSessionClient implements AnalysisClient { throw new AnalysisProtocolError( "An unowned worker still uses the private workspace; it was retained.", ); + if ( + this.#database === undefined && + this.#openCompleted && + !this.#openRejected + ) + throw new AnalysisProtocolError( + "IDA open returned without a verified private worker binding. An empty session inventory does not prove that its Worker exited; the private workspace was retained.", + ); } if (this.#database !== undefined) { if (!this.#databaseCloseCompleted) { diff --git a/src/native/pwntools/PwntoolsDecoder.ts b/src/native/pwntools/PwntoolsDecoder.ts index 169f25601..6aa456631 100644 --- a/src/native/pwntools/PwntoolsDecoder.ts +++ b/src/native/pwntools/PwntoolsDecoder.ts @@ -32,7 +32,10 @@ import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorClean import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { runOwnedCommand } from "../../process/OwnedCommand.js"; +import { + ownedCommandResources, + runOwnedCommand, +} from "../../process/OwnedCommand.js"; import type { ProviderProcessSupervisor, OwnedProviderProcessSpawnOptions, @@ -562,20 +565,16 @@ export class PwntoolsDecoder< const pending = this.#pendingCleanup; if (pending?.process === undefined) return undefined; const stopped = await pending.process.stop(); + const ownership = pending.process.launch.ownership; if (stopped.status === "incomplete") return new ProviderCleanupError( this.identity.id, - [ - pending.process.launch.ownership?.runId ?? "owned-decoder", - ...(pending.root === undefined ? [] : [pending.root.path]), - // The retained group may still be live; name it as OwnedCommand does. - ...(pending.process.launch.ownership === undefined - ? [] - : [ - `pid:${pending.process.launch.ownership.leaderPid}`, - `process-group:${pending.process.launch.ownership.processGroupId}`, - ]), - ], + ownership === undefined + ? [ + "owned-decoder", + ...(pending.root === undefined ? [] : [pending.root.path]), + ] + : ownedCommandResources(ownership, pending.root?.path), { reason: stopped.reason, captured_output: { diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts index 02fb4defc..d375971ae 100644 --- a/src/process/OwnedCommand.ts +++ b/src/process/OwnedCommand.ts @@ -9,6 +9,7 @@ import { } from "./ProviderProcess.js"; import { cleanupOwnedProcessGroup, + type OwnedProcessGroup, type ProcessCleanupFailure, } from "./ProcessOwnership.js"; import { @@ -34,6 +35,17 @@ export class OwnedCommandFailure extends Error { } } +/** Report recorded launch identities; these labels do not authorize signaling. */ +export const ownedCommandResources = ( + ownership: OwnedProcessGroup, + directory?: string, +): readonly string[] => [ + ownership.runId, + ...(directory === undefined ? [] : [directory]), + `pid:${ownership.leaderPid}`, + `process-group:${ownership.processGroupId}`, +]; + /** Execute a single owned process with independent cleanup and bounded diagnostic retention. */ export const runOwnedCommand = async ( spawn: OwnedProviderProcessSpawnOptions, @@ -208,12 +220,7 @@ const collectOwnedCommand = async ( ? {} : { cleanupResult: stopped.cleanupResult }), }, - [ - spawn.runId, - ...(spawn.cwd === undefined ? [] : [spawn.cwd]), - `pid:${launched.ownership.leaderPid}`, - `process-group:${launched.ownership.processGroupId}`, - ], + ownedCommandResources(launched.ownership, spawn.cwd), supervisor, ); if (failure !== undefined) diff --git a/src/process/capture/ProcessScenarioRuntimeValidation.test.ts b/src/process/capture/ProcessScenarioRuntimeValidation.test.ts index 1277b753f..7e5cb7e30 100644 --- a/src/process/capture/ProcessScenarioRuntimeValidation.test.ts +++ b/src/process/capture/ProcessScenarioRuntimeValidation.test.ts @@ -17,6 +17,28 @@ import { resolveProcessScenarioRuntimePaths, } from "./ProcessScenarioRuntimeValidation.js"; import { parseProcessScenario } from "../../domain/process/processScenario.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; + +it.skipIf(process.platform === "win32")( + "resolves a literal backslash in a POSIX executable basename through PATH", + async () => { + const root = await createTestTempDirectory("rea-process-posix-basename-"); + const bin = join(root, "bin"); + await mkdir(bin); + const command = "selected\\node"; + const selected = join(bin, command); + await symlink(process.execPath, selected); + await expect(resolveExecutable(command, root, bin)).resolves.toBe(selected); + const resolved = await resolveProcessScenarioRuntimePaths( + parseProcessScenario({ + executable: command, + working_directory: root, + environment: { PATH: bin }, + }), + ); + expect(resolved.executable).toBe(selected); + }, +); it("preserves the caller-selected executable symlink for process invocation", async () => { const root = await mkdtemp(join(tmpdir(), "rea-process-executable-alias-")); diff --git a/src/process/capture/ProcessScenarioRuntimeValidation.ts b/src/process/capture/ProcessScenarioRuntimeValidation.ts index 1ac9d2228..6d2978b5b 100644 --- a/src/process/capture/ProcessScenarioRuntimeValidation.ts +++ b/src/process/capture/ProcessScenarioRuntimeValidation.ts @@ -1,6 +1,6 @@ import { constants as fsConstants } from "node:fs"; import { access, realpath, stat } from "node:fs/promises"; -import { delimiter, isAbsolute, join, resolve } from "node:path"; +import { delimiter, isAbsolute, join, resolve, sep } from "node:path"; import type { ProcessScenario } from "../../domain/process/processScenario.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; @@ -71,7 +71,9 @@ export const resolveExecutable = async ( : resolve(workingDirectory, directory), ); const candidates = - isAbsolute(command) || command.includes("/") || command.includes("\\") + isAbsolute(command) || + command.includes("/") || + (sep === "\\" && command.includes("\\")) ? [resolve(workingDirectory, command)] : directories.map((directory) => join(directory, command)); const failures: string[] = []; diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index f5f92bdf4..510fbaafc 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -394,7 +394,7 @@ const readRows = ( for (const name of columns) if (!available.has(name)) throw new SqliteInspectionFailure( - "selection", + "column-selection", `Selected column does not exist with that exact name in table ${input.table}: ${name}`, ); const projection = columns diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 99df7ed9d..408fc4bc3 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -12,6 +12,7 @@ export class SqliteInspectionFailure extends Error { readonly reason: | "format" | "selection" + | "column-selection" | "unsupported" | "unavailable" | "output-limit" diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index 9474a6d27..cba180399 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -36,6 +36,7 @@ const replySchema = z.discriminatedUnion("ok", [ reason: z.enum([ "format", "selection", + "column-selection", "unsupported", "unavailable", "output-limit", @@ -49,6 +50,14 @@ export type SqliteWorkerLauncher = NonNullable< Parameters[2] >["launcher"]; +// Each input rejection names the caller argument that requires correction. +const rejectedArgument = { + format: "path", + unsupported: "path", + selection: "table", + "column-selection": "columns", +} as const; + const rejectedReply = ( reply: z.output & { ok: false }, capturedOutput: AnalysisCapturedOutput, @@ -83,7 +92,7 @@ const rejectedReply = ( ); return new AnalysisInputError(operation, { capturedOutput }, [ { - path: [reply.reason === "selection" ? "table" : "path"], + path: [rejectedArgument[reply.reason]], reason: "invalid_format", message: reply.message, }, diff --git a/src/windows/WindowsNativeLoader.test.ts b/src/windows/WindowsNativeLoader.test.ts index 1aa8d7375..c0fee8273 100644 --- a/src/windows/WindowsNativeLoader.test.ts +++ b/src/windows/WindowsNativeLoader.test.ts @@ -143,6 +143,22 @@ describe("packaged Windows native admission", () => { }, ); + it.each([ + { signature: "MZ", offset: 0 }, + { signature: "PE", offset: 64 }, + ])( + "rejects digest-valid high-bit $signature lookalikes before loading", + ({ offset }) => { + const value = fixture(); + value.bytes[offset] = (value.bytes[offset] ?? 0) | 0x80; + value.manifest.artifactSha256 = createHash("sha256") + .update(value.bytes) + .digest("hex"); + expect(loadWindowsNativeAuthority(value.host).available).toBe(false); + expect(value.loads()).toBe(0); + }, + ); + it("rejects a digest-valid artifact for another machine", () => { const value = fixture(); value.bytes.writeUInt16LE(0xaa64, 68); diff --git a/src/windows/WindowsNativeLoader.ts b/src/windows/WindowsNativeLoader.ts index 7719a0a85..2ab08e1ed 100644 --- a/src/windows/WindowsNativeLoader.ts +++ b/src/windows/WindowsNativeLoader.ts @@ -132,12 +132,12 @@ export const loadWindowsNativeAuthority = ( }; const assertWindowsX64Artifact = (bytes: Buffer): void => { - if (bytes.length < 64 || bytes.toString("ascii", 0, 2) !== "MZ") + if (bytes.length < 64 || bytes.toString("latin1", 0, 2) !== "MZ") throw new Error("Windows native artifact is not a PE image"); const offset = bytes.readUInt32LE(0x3c); if ( offset > bytes.length - 26 || - bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" || + bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" || bytes.readUInt16LE(offset + 4) !== 0x8664 || bytes.readUInt16LE(offset + 24) !== 0x20b ) diff --git a/tests/acceptance/applications/getterReceiverMutationsCli.test.ts b/tests/acceptance/applications/getterReceiverMutationsCli.test.ts index a5ba0bfeb..95b6e4862 100644 --- a/tests/acceptance/applications/getterReceiverMutationsCli.test.ts +++ b/tests/acceptance/applications/getterReceiverMutationsCli.test.ts @@ -39,6 +39,47 @@ export function superclass() { new B().v.x = 2; return shared.x; } +export function arrayReceiver() { + const arr = [{ x: 1, get self() { return this; } }]; + arr[0].self.x = 2; + return arr[0].x; +} +export function nestedArrayReceiver() { + const outer = { arr: [{ x: 1, get self() { return this; } }] }; + outer.arr[0].self.x = 2; + return outer.arr[0].x; +} +export function assignedReceiver() { + let box; + box = { x: 1, get self() { return this; } }; + box.self.x = 2; + return box.x; +} +export function savedArrayReceiver() { + const arr = [{ x: 1, get self() { return this; } }]; + const saved = arr[0].self; + saved.x = 2; + return arr[0].x; +} +export function arrayReadOnly() { + const arr = [{ x: 1, get self() { return this; } }]; + const saved = arr[0].self; + return arr[0].x; +} +export function replacementKeepsOldObject() { + const old = { x: 1 }; + let box = old; + box = { x: 1, old: box, get self() { return this; } }; + box.self.x = 2; + return old.x; +} +export function replacementMutatesCapturedObject() { + const old = { x: 1 }; + let box = old; + box = { old: box, get self() { return this; } }; + box.self.old.x = 2; + return old.x; +} export function untouched() { const box = { x: 1, y: 1, get self() { return this; } }; box.self.y = 2; @@ -66,13 +107,20 @@ export function untouched() { const { graph } = z .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) .parse(document).normalized_result; - for (const name of [ - "self", - "property", - "privateField", - "superclass", - "untouched", - ]) { + for (const [name, status] of [ + ["self", "unknown"], + ["property", "unknown"], + ["privateField", "unknown"], + ["superclass", "unknown"], + ["arrayReceiver", "unknown"], + ["nestedArrayReceiver", "unknown"], + ["assignedReceiver", "unknown"], + ["savedArrayReceiver", "unknown"], + ["arrayReadOnly", "literal"], + ["replacementKeepsOldObject", "literal"], + ["replacementMutatesCapturedObject", "unknown"], + ["untouched", "literal"], + ] as const) { const projection = graph.nodes .flatMap(({ observations }) => observations) .find( @@ -82,7 +130,16 @@ export function untouched() { ); expect(projection, name).toBeDefined(); expect(projection?.properties.static_return_shapes).toMatchObject([ - { value_status: name === "untouched" ? "literal" : "unknown" }, + { + value_status: status, + fields: [ + { + path: "", + state: status, + ...(status === "literal" ? { value: 1 } : {}), + }, + ], + }, ]); } }, diff --git a/tests/acceptance/artifactClassification.test.ts b/tests/acceptance/artifactClassification.test.ts new file mode 100644 index 000000000..7436a4967 --- /dev/null +++ b/tests/acceptance/artifactClassification.test.ts @@ -0,0 +1,111 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect } from "vitest"; + +import { + CPU, + fatImage, + machoImage, +} from "../../src/artifacts/apple/MachoImage.fixture.js"; +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest.for(["jar", "zip"])( + "distinguishes JVM classes from universal Mach-O files in %s inventory", + async (extension, { cli }) => { + const root = await createTestTempDirectory("rea-class-inventory-"); + const path = join(root, `mixed.${extension}`); + // The large class can hold the entire 65-entry table suggested by its + // version bytes. Checking only the table length does not distinguish it. + const classes = [ + ["Hello.class", javaClass()], + ["Big.class", javaClass("x".repeat(4_000))], + ["renamed.bin", javaClass()], + ] as const; + const writer = new ZipWriter(new Uint8ArrayWriter()); + for (const [name, bytes] of classes) + await writer.add(name, new Uint8ArrayReader(bytes)); + const nativePaths: string[] = []; + for (const wide of [false, true]) { + for (const littleEndian of [false, true]) { + const name = `fat${wide ? 64 : 32}-${littleEndian ? "le" : "be"}`; + nativePaths.push(name); + await writer.add( + name, + new Uint8ArrayReader( + fatImage( + [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], + wide, + littleEndian, + ), + ), + ); + } + } + await writeFile(path, await writer.close()); + + const reply = await cli.run({ + arguments: ["inspect-artifact", path, "--json"], + }); + expect(reply.exitCode, reply.stdout + reply.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.json).normalized_result, + ); + const inventory = artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ); + for (const name of nativePaths) { + expect(inventory.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: name, + artifact_kind: "executable", + artifact_format: "mach-o-universal", + limitations: [], + }), + ); + } + for (const [name, bytes] of classes) { + const occurrence = inventory.occurrences.find( + (item) => item.logical_path === name, + ); + expect(occurrence).toBeDefined(); + expect.soft(occurrence?.artifact_kind, name).not.toBe("executable"); + expect.soft(occurrence?.artifact_format, name).not.toMatch(/^mach-o/u); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const node = inventory.nodes.find((item) => item.sha256 === sha256); + expect(node).toBeDefined(); + expect.soft(node?.format, name).not.toMatch(/^mach-o/u); + } + }, +); + +/** Minimal valid Java 21 class, with an optional unused UTF-8 constant. */ +const javaClass = (extra?: string): Buffer => { + const utf8 = (value: string): Buffer => { + const text = Buffer.from(value); + const header = Buffer.alloc(3); + header[0] = 1; + header.writeUInt16BE(text.length, 1); + return Buffer.concat([header, text]); + }; + const constants = [ + Buffer.from([7, 0, 2]), + utf8("Hello"), + Buffer.from([7, 0, 4]), + utf8("java/lang/Object"), + ...(extra === undefined ? [] : [utf8(extra)]), + ]; + const header = Buffer.from("cafebabe000000410000", "hex"); + header.writeUInt16BE(constants.length + 1, 8); + return Buffer.concat([ + header, + ...constants, + Buffer.from("0021000100030000000000000000", "hex"), + ]); +}; diff --git a/tests/acceptance/setup/dshClientSetup.test.ts b/tests/acceptance/setup/dshClientSetup.test.ts index 4010369cd..b544c5816 100644 --- a/tests/acceptance/setup/dshClientSetup.test.ts +++ b/tests/acceptance/setup/dshClientSetup.test.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { expect } from "vitest"; @@ -8,6 +8,111 @@ import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +cliTest.for([ + { + label: "blank defaults", + harness: "", + agents: " \t ", + dshRoot: "home/.dsh", + agentsRoot: "home/.agents", + }, + { + label: "tilde roots", + harness: "~/selected harness", + agents: "~\\shared agents", + dshRoot: "home/selected harness", + agentsRoot: "home/shared agents", + }, + { + label: "relative roots", + harness: "./selected harness", + agents: "../shared agents", + dshRoot: "workspace/selected harness", + agentsRoot: "shared agents", + }, +])( + "DSH CLI uses Harness path semantics through setup, doctor and uninstall for $label", + async ({ harness, agents, dshRoot, agentsRoot }, { cli }) => { + const root = await createTestTempDirectory("rea-dsh-paths-"); + const home = join(root, "home"); + const cwd = join(root, "workspace"); + const config = join(root, dshRoot, "cordis.patch.yml"); + const skillRoot = join( + root, + agentsRoot, + "skills", + "reverse-engineer-anything", + ); + const skill = join(skillRoot, "SKILL.md"); + await mkdir(home); + await mkdir(cwd); + await mkdir(dirname(config), { recursive: true }); + const original = + "# caller patch\n- insert: [{id: caller, name: caller-plugin}]\n"; + await writeFile(config, original); + const run = (args: readonly string[]) => + cli.run({ + arguments: [...args, "--json"], + cwd, + environment: { + HOME: home, + USERPROFILE: home, + DSH_HOME: harness, + DSH_AGENTS_HOME: agents, + }, + }); + const planned = await run(["setup", "--client", "dsh", "--dry-run"]); + expect(planned.exitCode).toBe(0); + expect(planned.json).toMatchObject({ + status: "planned", + appliedActions: [], + plannedActions: expect.arrayContaining([ + expect.objectContaining({ id: "configure_client:dsh", target: config }), + expect.objectContaining({ kind: "install_skill", target: skillRoot }), + ]), + }); + expect(await readFile(config, "utf8")).toBe(original); + await expect(readFile(skill)).rejects.toMatchObject({ code: "ENOENT" }); + const configured = await run(["setup", "--client", "dsh", "--yes"]); + expect(configured.exitCode).toBe(0); + expect(configured.json).toMatchObject({ + status: "ready", + clients: { dsh: { status: "configured" } }, + }); + const text = await readFile(config, "utf8"); + expect(text).toContain("# caller patch"); + expect(text).toContain("rea-mcp"); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + expect(await readFile(skill, "utf8")).toContain( + "name: reverse-engineer-anything", + ); + const doctor = await run(["doctor", "--client", "dsh", "--skill"]); + expect(doctor.exitCode).toBe(0); + expect(doctor.json).toMatchObject({ healthy: true }); + const repeated = await run(["setup", "--client", "dsh", "--yes"]); + expect(repeated.exitCode).toBe(0); + expect(repeated.json).toMatchObject({ + status: "ready", + plannedActions: [], + appliedActions: [], + }); + expect(await readFile(config, "utf8")).toBe(text); + const removed = await run(["uninstall"]); + expect(removed.exitCode).toBe(0); + expect(removed.json).toMatchObject({ + status: "complete", + items: expect.arrayContaining([ + expect.objectContaining({ name: "dsh", status: "removed" }), + ]), + }); + expect(parse(await readFile(config, "utf8"))).toEqual([ + { insert: [{ id: "caller", name: "caller-plugin" }] }, + ]); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + await expect(readFile(skill)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + cliTest( "DSH CLI previews, approves, repairs, launches MCP, repeats and uninstalls while preserving caller YAML", async ({ cli }) => { diff --git a/tests/boundary/cli/cliOutput.test.ts b/tests/boundary/cli/cliOutput.test.ts index 43d517e16..44b3ac234 100644 --- a/tests/boundary/cli/cliOutput.test.ts +++ b/tests/boundary/cli/cliOutput.test.ts @@ -58,6 +58,95 @@ describe("CLI output argument boundary", () => { }); }); +describe("unclosed CLI output filter slices", () => { + it.each([ + "providers[", + "providers[0,1].name[", + "name,providers[", + "][,]", + "providers],name[0,1]", + ])("rejects %s before the filter parser runs", (filter) => { + expect( + validateCliOutputArguments([ + "providers", + "--json", + "--filter-output", + filter, + ]), + ).toMatchObject({ ok: false, code: "INVALID_OUTPUT_FILTER" }); + }); + + it("preserves effective-filter and earlier flag-error precedence", () => { + for (const arguments_ of [ + [ + "--filter-output", + "providers[", + "--filter-output", + "capabilities[0,1].provider.id,open", + ], + ["--filter-output", "capabilities[-1,].provider.id"], + ["--filter-output", "capabilities[[0,1]"], + ["--format", "invalid", "--filter-output", "providers["], + ["--token-limit", "invalid", "--filter-output", "providers["], + ]) + expect(validateCliOutputArguments(arguments_)).toEqual({ ok: true }); + }); + + workspaceCliTest( + "fails promptly across output formats without invoking the command", + async ({ cli }) => { + for (const format of ["json", "jsonl", "yaml", "toon", "md"]) { + const result = await cli.run({ + arguments: [ + "analyze", + "/absent/target", + "--format", + format, + "--filter-output", + "result[0,1].name[", + ], + timeoutMs: 3000, + }); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("INVALID_OUTPUT_FILTER"); + expect(result.stdout).toContain("closing ]"); + expect(result.stderr).toBe(""); + } + const splitPath = await cli.run({ + arguments: ["providers", "--json", "--filter-output", "][,]"], + timeoutMs: 3000, + }); + expect(splitPath.exitCode).toBe(1); + expect(splitPath.json).toMatchObject({ + ok: false, + error: { code: "INVALID_OUTPUT_FILTER" }, + }); + }, + CLI_INTEGRATION_TIMEOUT_MS, + ); + + workspaceCliTest( + "retains closed slices and multiple filter paths", + async ({ cli }) => { + const result = await cli.run({ + arguments: [ + "providers", + "--json", + "--filter-output", + "capabilities[0,1].provider.id,open", + ], + timeoutMs: 3000, + }); + expect(result.exitCode).toBe(0); + expect(result.json).toEqual({ + capabilities: [{ provider: { id: expect.any(String) } }], + open: false, + }); + }, + CLI_INTEGRATION_TIMEOUT_MS, + ); +}); + describe("CLI output valued-flag parsing", () => { workspaceCliTest( "preserves complete JSON errors when native builtin flags follow unusual arguments", diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index 5e8e86e7f..0d1ee5a59 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -561,3 +561,35 @@ describe("CLI result output surface", () => { }); }, 60_000); }); + +describe("streamed CLI filtered selections", () => { + it.each([ + ["raw_result", "json", "null\n"], + ["raw_result", "jsonl", "null\n"], + ["nested.value", "json", '{"nested":{"value":null}}\n'], + ["missing", "json", "{}\n"], + ["count", "json", "0\n"], + ["enabled", "json", "false\n"], + ] as const)( + "keeps the selected value of --filter-output %s in %s", + async (filter, format, expected) => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + ["--format", format, "--filter-output", filter], + destination, + ); + if (output === undefined) throw new Error("Missing JSON output surface"); + await output.write( + { raw_result: null, nested: { value: null }, count: 0, enabled: false }, + { command: "fixture", duration: "1ms", format }, + ); + expect(Buffer.concat(chunks).toString("utf8")).toBe(expected); + }, + ); +}); diff --git a/tests/boundary/cli/referenceSourceDirectoryImports.test.ts b/tests/boundary/cli/referenceSourceDirectoryImports.test.ts new file mode 100644 index 000000000..6270855cb --- /dev/null +++ b/tests/boundary/cli/referenceSourceDirectoryImports.test.ts @@ -0,0 +1,118 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; + +import { describe, expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +// Reference-source reads require no-follow opens; Windows reports unsupported. +describe.skipIf(process.platform === "win32")( + "compiled reference-source directory imports", + () => { + cliTest( + "resolves directory spellings without selecting similarly named files", + async ({ cli }) => { + const workspace = await createTestTempDirectory( + "rea-reference-directory-imports-", + ); + const root = join(workspace, "source"); + const imports = [ + ["slash.cjs", "./lib/", "lib/index.js"], + ["dot.cjs", "./lib/.", "lib/index.js"], + ["self.cjs", ".", "index.js"], + ["self-slash.cjs", "./", "index.js"], + ["nested/parent.cjs", "..", "index.js"], + ["nested/parent-slash.cjs", "../", "index.js"], + ["direct.cjs", "./lib.js", "lib.js"], + ["fallback.cjs", "./fallback", "fallback/index.js"], + ["legacy-order.cjs", "./dep", "dep/index.js"], + ] as const; + const unresolved = [ + ["missing.cjs", "./missing/", "missing/"], + ["escape.cjs", "../outside/", "../outside/"], + ["main.ts", "./folder.js/.", "folder.js"], + ] as const; + const files = new Map([ + ["index.js", 'module.exports = "root index";\n'], + ["lib/index.js", 'module.exports = "library index";\n'], + ["lib.js", 'module.exports = "sibling JavaScript file";\n'], + ["lib.ts", 'export const value = "sibling TypeScript file";\n'], + ["fallback/index.js", 'module.exports = "fallback index";\n'], + ["dep/index.js", 'module.exports = "existing index priority";\n'], + ["dep.cjs", 'module.exports = "later CommonJS candidate";\n'], + [ + "folder.ts", + 'export const value = "emitted filename counterpart";\n', + ], + ["missing.js", 'module.exports = "not a directory";\n'], + ...[...imports, ...unresolved].map( + ([path, specifier]) => + [path, `require(${JSON.stringify(specifier)});\n`] as const, + ), + ]); + await Promise.all( + [...files].map(async ([path, contents]) => { + const target = join(root, path); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, contents); + }), + ); + const outside = join(workspace, "outside", "index.js"); + await mkdir(dirname(outside), { recursive: true }); + await writeFile(outside, 'module.exports = "outside import root";\n'); + + // Node's loader establishes each fixture's meaning independently. + for (const [path, specifier, target] of imports) + expect(createRequire(join(root, path)).resolve(specifier)).toBe( + join(root, target), + ); + expect(() => + createRequire(join(root, "missing.cjs")).resolve("./missing/"), + ).toThrow(); + expect(() => + createRequire(join(root, "main.ts")).resolve("./folder.js/."), + ).toThrow(); + expect( + createRequire(join(root, "escape.cjs")).resolve("../outside/"), + ).toBe(outside); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: workspace, + XDG_CONFIG_HOME: workspace, + XDG_CACHE_HOME: workspace, + }, + }); + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.parse_failures).toEqual([]); + expect(graph.relationships).toHaveLength( + imports.length + unresolved.length, + ); + for (const [from, , to] of imports) + expect.soft(graph.relationships, from).toContainEqual({ + from_path: from, + to, + kind: "requires", + resolution: "internal", + parse_state: "parsed", + }); + for (const [from, , to] of unresolved) + expect.soft(graph.relationships, from).toContainEqual({ + from_path: from, + to, + kind: "requires", + resolution: "unresolved", + parse_state: "parsed", + }); + expect( + graph.entries.some((entry) => entry.path.includes("outside")), + ).toBe(false); + }, + ); + }, +); diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index 41397c520..cf7dcd595 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -366,16 +366,11 @@ it("rejects overlapping ZIP member data ranges through artifact inventory", asyn bytes.writeUInt32LE(firstLocalOffset, second + 42); await writeFile(path, bytes); - const target = await parseBinaryTarget(path); - if (!target.ok) throw target.error; - const result = await withArtifactClient(target.value, (client) => - client.execute("inventory_artifact", {}), + await expectInventoryFailure( + path, + "format", + "ZIP archive is malformed: Overlapping entry found", ); - - expect(result).toMatchObject({ - ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, - }); }); it("preserves encrypted-member and corrupt-CRC observations", async () => { @@ -438,16 +433,11 @@ it("preserves encrypted-member and corrupt-CRC observations", async () => { bytes[dataOffset] = firstByte ^ 1; await writeFile(corruptPath, bytes); - const corruptTarget = await parseBinaryTarget(corruptPath); - if (!corruptTarget.ok) throw corruptTarget.error; - const corruptResult = await withArtifactClient( - corruptTarget.value, - (client) => client.execute("inventory_artifact", {}), + await expectInventoryFailure( + corruptPath, + "format", + "ZIP archive is malformed: Invalid compressed data", ); - expect(corruptResult).toMatchObject({ - ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, - }); }); it("rejects a truncated ZIP through artifact inventory", async () => { @@ -455,17 +445,51 @@ it("rejects a truncated ZIP through artifact inventory", async () => { const path = join(root, "truncated.zip"); await writeFile(path, Buffer.from([0x50, 0x4b, 0x03, 0x04])); + await expectInventoryFailure( + path, + "format", + "ZIP archive is malformed: File format is not recognized", + ); +}); + +it("reports an unsupported ZIP compression method as unavailable", async () => { + const root = await createTestTempDirectory("rea-zip-method-"); + const path = join(root, "lzma.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("data.txt", new TextReader("member bytes"), { level: 0 }); + const bytes = Buffer.from(await writer.close()); + // Declare LZMA (14) in both headers; REA reads only stored and Deflate data. + const localHeader = bytes.indexOf(Buffer.from([0x50, 0x4b, 0x03, 0x04])); + const centralHeader = bytes.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + if (localHeader < 0 || centralHeader < 0) + throw new Error("Expected ZIP local and central headers"); + bytes.writeUInt16LE(14, localHeader + 8); + bytes.writeUInt16LE(14, centralHeader + 10); + await writeFile(path, bytes); + + await expectInventoryFailure( + path, + "unavailable", + "ZIP archive uses an unsupported feature: Compression method not supported", + ); +}); + +/** Inventory one archive through the provider and expect a typed failure. */ +const expectInventoryFailure = async ( + path: string, + reason: string, + detail: string, +): Promise => { const target = await parseBinaryTarget(path); if (!target.ok) throw target.error; const result = await withArtifactClient(target.value, (client) => client.execute("inventory_artifact", {}), ); - expect(result).toMatchObject({ ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, + error: { _tag: "ArtifactOperationError", reason, detail }, }); -}); +}; const writeSingleEntryZip = async ( path: string, diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 163219c39..321160a36 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -515,6 +515,10 @@ it.each([ "src/windows/Example.ts", { code: true, runtime: true, package: true, apple: false, evm: false }, ], + [ + "src/domain/javascript/javascriptRuntimeStaticCandidates.ts", + { code: true, runtime: true, package: true, javascript_recovery: true }, + ], ["install.sh", { code: true, runtime: true, package: true, evm: false }], [ ".agents/skills/reverse-engineer-anything/SKILL.md", diff --git a/tests/boundary/filesystem/dshClientConfiguration.test.ts b/tests/boundary/filesystem/dshClientConfiguration.test.ts index 72dbded9e..c0315f67b 100644 --- a/tests/boundary/filesystem/dshClientConfiguration.test.ts +++ b/tests/boundary/filesystem/dshClientConfiguration.test.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -37,28 +37,76 @@ const dshClient = ( env: Parameters[2], platform: NodeJS.Platform = "linux", ) => - supportedClients(join("/home/a"), platform, env).find( + supportedClients(resolve("/home/a"), platform, env).find( ({ name }) => name === "dsh", ); describe("DeepSeek Harness configuration paths", () => { + it.each([ + [undefined, ".dsh", ".agents", "shared"], + ["", ".dsh", ".agents", "shared"], + [" \t ", ".dsh", ".agents", "shared"], + ["~", "", "", "dsh"], + ["~/selected home", "selected home", "selected home", "dsh"], + ["~\\selected home", "selected home", "selected home", "dsh"], + ] as const)( + "resolves Harness and agents roots for %j", + (override, dshRoot, agentsRoot, client) => { + const home = resolve("/home/a"); + const environment = { DSH_HOME: override, DSH_AGENTS_HOME: override }; + expect( + supportedClients(home, process.platform, environment).find( + ({ name }) => name === "dsh", + ), + ).toMatchObject({ + configPath: join(home, dshRoot, "cordis.patch.yml"), + markerPath: join(home, dshRoot), + }); + expect(clientSkillDirectories(home, ["dsh"], environment)).toEqual([ + { client, directory: join(home, agentsRoot, "skills") }, + ]); + }, + ); + + it.each(["relative home", " literal home ", "~someone"])( + "resolves a literal root %j relative to cwd without trimming it", + (override) => { + const home = resolve("/home/a"); + const root = resolve(override); + const environment = { DSH_HOME: override, DSH_AGENTS_HOME: override }; + expect( + supportedClients(home, process.platform, environment).find( + ({ name }) => name === "dsh", + ), + ).toMatchObject({ + configPath: join(root, "cordis.patch.yml"), + markerPath: root, + }); + expect(clientSkillDirectories(home, ["dsh"], environment)).toEqual([ + { client: "dsh", directory: join(root, "skills") }, + ]); + }, + ); + it("resolves the home Cordis patch under the Harness home", () => { expect(dshClient({})).toEqual({ name: "dsh", displayName: "DeepSeek Harness", - configPath: join("/home/a", ".dsh", "cordis.patch.yml"), - markerPath: join("/home/a", ".dsh"), + configPath: resolve("/home/a", ".dsh", "cordis.patch.yml"), + markerPath: resolve("/home/a", ".dsh"), format: "dsh", }); // The home patch applies to every profile; DSH_HOME relocates it. expect(dshClient({ DSH_HOME: "/custom/harness" })?.configPath).toBe( - join("/custom/harness", "cordis.patch.yml"), + resolve("/custom/harness", "cordis.patch.yml"), ); expect( clientSkillDirectories(join("/home/a"), ["dsh"], { DSH_AGENTS_HOME: "/custom/agents", }), - ).toEqual([{ client: "dsh", directory: join("/custom/agents", "skills") }]); + ).toEqual([ + { client: "dsh", directory: resolve("/custom/agents", "skills") }, + ]); }); }); diff --git a/tests/boundary/filesystem/ghidraJavaProbe.test.ts b/tests/boundary/filesystem/ghidraJavaProbe.test.ts index c0a6974d5..415fbf3dd 100644 --- a/tests/boundary/filesystem/ghidraJavaProbe.test.ts +++ b/tests/boundary/filesystem/ghidraJavaProbe.test.ts @@ -87,3 +87,43 @@ it.each([ ); }, ); + +it("bounds the Java version probe and reports an over-budget probe as a failed Java check", async () => { + const root = await createTestTempDirectory("rea-java-probe-bound-"); + const configuredHome = join(root, "selected-jdk"); + const installDir = join(root, "ghidra"); + await mkdir(join(installDir, "Ghidra"), { recursive: true }); + await writeFile( + join(installDir, "Ghidra", "application.properties"), + `application.version=${SUPPORTED_GHIDRA_VERSION}\napplication.java.min=17\n`, + ); + // A chatty or hostile JVM that overruns the capture budget: spawnSync reports + // ENOBUFS, which the probe already treats as a failed observation. + mockSpawnSync.mockReturnValue({ + stdout: "", + stderr: "java.version = 21.0.11\n", + status: null, + error: Object.assign(new Error("spawnSync /java ENOBUFS"), { + code: "ENOBUFS", + }), + }); + + const inspection = inspectGhidraInstallation({ + environment: {}, + installDir, + javaHome: configuredHome, + }); + + const [, , options] = mockSpawnSync.mock.calls[0] as [ + string, + readonly string[], + { readonly maxBuffer?: number }, + ]; + expect(options.maxBuffer).toBe(1024 * 1024); + expect(inspection.javaVersion).toBeNull(); + expect(inspection.javaHome).toBe(configuredHome); + expect(inspection.checks.find(({ name }) => name === "java")).toMatchObject({ + status: "failed", + code: "runtime_missing", + }); +}); diff --git a/tests/boundary/filesystem/hermesClientConfiguration.test.ts b/tests/boundary/filesystem/hermesClientConfiguration.test.ts index 43a5f841c..218573d5c 100644 --- a/tests/boundary/filesystem/hermesClientConfiguration.test.ts +++ b/tests/boundary/filesystem/hermesClientConfiguration.test.ts @@ -237,6 +237,38 @@ mcp_servers: expect(await readFile(client.configPath, "utf8")).toBe(original); }); + it("keeps unrelated numbers as written, including Discord IDs", async () => { + const { home, client } = await hermesClient(); + // Hermes reads these exactly: two IDs one apart, an octal mode, a float. + const original = `discord: + free_response_channels: + - 1187654321098765432 + channel_prompts: + 1187654321098765433: Build questions. + 1187654321098765434: Release questions. + mode: 0600 + min_score: 0.0000001 +mcp_servers: + other: + command: other-server +`; + await writeFile(client.configPath, original); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text.startsWith(original)).toBe(true); + expect(parseYaml(text, { intAsBigInt: true })).toMatchObject({ + mcp_servers: { + rea: { command: command[0], args: command.slice(1), enabled: true }, + }, + }); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(await readFile(client.configPath, "utf8")).toBe(original); + }); + it("creates config.yaml when Hermes has no configuration yet", async () => { const { client } = await hermesClient(); expect( diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index b79407821..8393bad54 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -444,73 +444,97 @@ describe("JavaScript application workflow error diagnostics", () => { }); }); +const RUNTIME_PATH_CASES = [ + { + name: "case-sensitive POSIX input root", + root: null, + path: "/applications/Example.app/Contents/Resources/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "case-sensitive POSIX mapping root", + root: "/runtime/App", + path: "/runtime/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "matching POSIX mapping root", + root: "/runtime/App", + path: "/runtime/App/renderer.js", + status: "matched", + basis: "operator-file-mapping", + }, + { + name: "case-insensitive Windows mapping root", + root: String.raw`C:\runtime\App`, + path: String.raw`c:\runtime\app\renderer.js`, + status: "matched", + basis: "operator-file-mapping", + }, +] as const; + describe("JavaScript runtime reconciliation local paths", () => { - it.each([ - { - name: "case-sensitive POSIX input root", - root: null, - path: "/applications/Example.app/Contents/Resources/app/renderer.js", - status: "unknown", - }, - { - name: "case-sensitive POSIX mapping root", - root: "/runtime/App", - path: "/runtime/app/renderer.js", - status: "unknown", - }, - { - name: "matching POSIX mapping root", - root: "/runtime/App", - path: "/runtime/App/renderer.js", - status: "matched", - }, - { - name: "case-insensitive Windows mapping root", - root: String.raw`C:\runtime\App`, - path: String.raw`c:\runtime\app\renderer.js`, - status: "matched", - }, - ])("uses $name on every host", ({ root, path, status }) => { - const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; - const original = base.runtime_observations[0]!; - const inspection = electronPageInspectionSchema.parse( - original.normalized_result, - ); - const runtime = createEvidence(undefined, original.provider, { - predicateType: original.predicate_type, - operation: original.operation, - parameters: { ...original.parameters, include_script_sources: false }, - result: { - ...inspection, - scripts: { - ...inspection.scripts, - items: inspection.scripts.items.map((script) => ({ - ...script, - file_path: path, - source: { included: false, reason: "not-requested" }, - })), + it.each(RUNTIME_PATH_CASES)( + "uses $name on every host", + ({ root, path, status, basis }) => { + const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; + const original = base.runtime_observations[0]!; + const inspection = electronPageInspectionSchema.parse( + original.normalized_result, + ); + // Keep target and script coordinates consistent so an unrelated target + // cannot activate the static layer after its script mapping was rejected. + const runtime = createEvidence(undefined, original.provider, { + predicateType: original.predicate_type, + operation: original.operation, + parameters: { ...original.parameters, include_script_sources: false }, + result: { + ...inspection, + target: { ...inspection.target, file_path: path }, + scripts: { + ...inspection.scripts, + items: inspection.scripts.items.map((script) => ({ + ...script, + file_path: path, + source: { included: false, reason: "not-requested" }, + })), + }, }, - }, - confidence: original.confidence, - authority: original.authority, - }); - const reconciled = reconcileJavaScriptRuntimeEvidence({ - static_layers: base.static_layers.map((layer) => ({ - ...layer, - runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], - })), - runtime_observations: [runtime], - }); - if (!reconciled.ok) throw reconciled.error; - const result = javascriptRuntimeReconciliationResultSchema.parse( - reconciled.value.normalized_result, - ); - expect( - result.reconciliations.find( - ({ entity_kind }) => entity_kind === "script", - ), - ).toMatchObject({ status }); - }); + confidence: original.confidence, + authority: original.authority, + }); + const reconciled = reconcileJavaScriptRuntimeEvidence({ + static_layers: base.static_layers.map((layer) => ({ + ...layer, + runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], + })), + runtime_observations: [runtime], + }); + if (!reconciled.ok) throw reconciled.error; + const result = javascriptRuntimeReconciliationResultSchema.parse( + reconciled.value.normalized_result, + ); + expect( + result.reconciliations.find( + ({ entity_kind }) => entity_kind === "script", + ), + ).toMatchObject({ status, basis }); + expect( + result.static_load_states.find( + ({ kind }) => kind === "javascript-asset", + ), + ).toMatchObject( + status === "matched" + ? { status: "loaded", reason: "runtime-script-correspondence" } + : { status: "unknown", reason: "layer-outside-runtime-scope" }, + ); + expect(result.static_layers[0]?.runtime_mappings).toEqual( + root === null ? [] : [{ kind: "file-root", root, artifact_prefix: "" }], + ); + }, + ); it("resolves relative file mapping roots before deriving result Evidence", () => { const absoluteRoot = resolve(process.cwd(), "runtime-cache-fixture"); diff --git a/tests/boundary/filesystem/targetDiagnostics.test.ts b/tests/boundary/filesystem/targetDiagnostics.test.ts index 5d1953760..dacd21687 100644 --- a/tests/boundary/filesystem/targetDiagnostics.test.ts +++ b/tests/boundary/filesystem/targetDiagnostics.test.ts @@ -4,9 +4,35 @@ import { describe, expect, it } from "vitest"; import { systemDoctorHost } from "../../../src/application/Doctor.js"; import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { pe } from "../../../src/domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("executable format refusal diagnostics", () => { + it.each([ + [0x01a6, "0x01a6"], + [0x0000, "0x0000"], + [0xffff, "0xffff"], + ] as const)( + "retains unsupported PE machine %i in the refusal", + async (machine, expectedMachine) => { + const directory = await createTestTempDirectory("rea-target-pe-machine-"); + const path = join(directory, "unsupported.exe"); + const bytes = pe(0x014c); + bytes.writeUInt16LE(machine, 0x44); + await writeFile(path, bytes); + + const result = await parseBinaryTarget(path); + if (result.ok) throw new Error("unsupported PE target was admitted"); + const reason = `unsupported PE architecture (COFF machine ${expectedMachine})`; + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + message: `Cannot open artifact: ${reason}.`, + details: { reason, constraint: "file_format_rejected" }, + }); + }, + ); + it.each(["NE", "LE", "LX"] as const)( "reports unsupported %s executables without path advice", async (signature) => { diff --git a/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts new file mode 100644 index 000000000..8c94d5de0 --- /dev/null +++ b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts @@ -0,0 +1,249 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import type { AnalysisOperationPort } from "../../../src/application/AnalysisProvider.js"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; +import { EnhancedTools } from "../../../src/application/EnhancedTools.js"; +import { AnalysisOutputError } from "../../../src/domain/analysisErrorCore.js"; +import { err, ok } from "../../../src/domain/result.js"; + +import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; + +afterEach(closeEnhancedToolResources); + +const PROVIDER = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", +} as const; + +type Coverage = boolean | undefined; + +interface Behavior { + /** What the provider reports for the references of each match address. */ + readonly coverage?: Readonly>; + readonly limitations?: readonly string[]; + /** Also report the first string match as a procedure match. */ + readonly duplicateProcedure?: boolean; + /** Fail the references read of this address. */ + readonly failXrefsOf?: string; + readonly calls?: string[]; +} + +const containing = (address: string) => ({ + query_address: address, + found: false, + procedure: null, + reason: "not_in_procedure", +}); + +/** + * A provider with two string matches (0x10 and 0x11), each referenced from + * one address (0x20 and 0x21). + */ +const provider = (behavior: Behavior = {}): AnalysisOperationPort => ({ + execute: async (name, arguments_) => { + const address = String(arguments_.address); + behavior.calls?.push(`${name}:${address}`); + switch (name) { + case "search_strings": + return ok( + createAnalysisExecution( + [ + { address: "0x10", value: "hello" }, + { address: "0x11", value: "hello again" }, + ], + PROVIDER, + ), + ); + case "search_procedures": + return ok( + createAnalysisExecution( + behavior.duplicateProcedure === true + ? [{ address: "0x10", value: "hello" }] + : [], + PROVIDER, + ), + ); + case "xrefs": { + if (address === behavior.failXrefsOf) + return err(new AnalysisOutputError("xrefs", "provider failed")); + const reported = behavior.coverage?.[address]; + return ok( + createAnalysisExecution( + [address === "0x10" ? "0x20" : "0x21"], + PROVIDER, + { + limitations: behavior.limitations ?? [], + ...(reported === undefined + ? {} + : { referencesTruncated: reported }), + }, + ), + ); + } + case "resolve_containing_procedure": + return ok(createAnalysisExecution(containing(address), PROVIDER)); + default: + throw new Error(`Unexpected operation: ${name}`); + } + }, +}); + +const expectedTrace = ( + residual: readonly string[], + extraMatches: readonly object[] = [], +) => ({ + query: "hello", + search_mode: "literal", + matches: [ + { type: "string", address: "0x10", value: "hello" }, + { type: "string", address: "0x11", value: "hello again" }, + ...extraMatches, + ], + references: [ + { + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + }, + { + target_address: "0x11", + source_address: "0x21", + containing_procedure: containing("0x21"), + }, + ...extraMatches.map(() => ({ + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + })), + ], + truncated: residual.length > 0, + residual_unknowns: residual, +}); + +const unknownFor = (address: string) => + `References to ${address} were truncated by the provider; additional references may exist.`; + +const trace = async (analysis: AnalysisOperationPort) => { + const client = await connect(analysis); + const result = await client.callTool({ + name: "trace_feature", + arguments: { query: "hello" }, + }); + expect(result.isError).not.toBe(true); + return result.structuredContent as { normalized_result: unknown }; +}; + +describe("trace_feature reference coverage", () => { + it("marks the trace truncated for the target whose references the provider capped", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect( + output.normalized_result, + "the received references are kept and only the capped target is a residual unknown", + ).toEqual(expectedTrace([unknownFor("0x10")])); + }); + + it("does not carry a truncation over to the next target that reports nothing", async () => { + const output = await trace(provider({ coverage: { "0x10": true } })); + + expect(output.normalized_result).toEqual( + expectedTrace([unknownFor("0x10")]), + ); + }); + + it("reports a capped target once when it is matched twice", async () => { + const output = await trace( + provider({ coverage: { "0x10": true }, duplicateProcedure: true }), + ); + + expect(output.normalized_result).toEqual( + expectedTrace( + [unknownFor("0x10")], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("resolves the containing procedure of a repeated reference source once", async () => { + const calls: string[] = []; + const output = await trace(provider({ duplicateProcedure: true, calls })); + + expect( + calls.filter((call) => call.startsWith("resolve_containing_procedure:")), + "the source 0x20 is referenced by two matches of the same address and resolved once", + ).toEqual([ + "resolve_containing_procedure:0x20", + "resolve_containing_procedure:0x21", + ]); + expect( + output.normalized_result, + "every reference is still reported with its containing procedure", + ).toEqual( + expectedTrace( + [], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("keeps the trace complete when the provider explicitly reports no cap", async () => { + const output = await trace( + provider({ coverage: { "0x10": false, "0x11": false } }), + ); + + expect(output.normalized_result).toEqual(expectedTrace([])); + }); + + it("does not change the trace of a provider that reports no coverage", async () => { + const output = await trace( + provider({ + limitations: ["Upstream xrefs_to capped results at 1000 references."], + }), + ); + + expect( + output.normalized_result, + "a limitation text alone never truncates, and an absent field keeps the previous result", + ).toEqual(expectedTrace([])); + }); + + it("never exposes the coverage field in the tool output", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect(JSON.stringify(output)).not.toMatch( + /referencesTruncated|references_truncated/, + ); + }); + + it("fails the trace when a references read fails and reads nothing further", async () => { + const calls: string[] = []; + const result = await new EnhancedTools( + provider({ failXrefsOf: "0x10", calls }), + ).execute("trace_feature", { query: "hello" }); + + expect(result.ok).toBe(false); + expect( + calls.filter((call) => call.startsWith("xrefs:")), + "the next target is not read after the failure", + ).toEqual(["xrefs:0x10"]); + }); + + it("stops without reading references when the trace is cancelled", async () => { + const calls: string[] = []; + const controller = new AbortController(); + controller.abort(); + const result = await new EnhancedTools(provider({ calls })).execute( + "trace_feature", + { query: "hello" }, + controller.signal, + ); + + expect(result.ok).toBe(false); + expect(calls.filter((call) => call.startsWith("xrefs:"))).toEqual([]); + }); +}); diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts index 34205af3d..ca0cb2db6 100644 --- a/tests/boundary/process/ownedCommand.test.ts +++ b/tests/boundary/process/ownedCommand.test.ts @@ -257,9 +257,11 @@ it.skipIf(process.platform === "win32").each(["cancelled", "timeout"] as const)( failures: [expect.objectContaining({ reason: "run-token-mismatch" })], }); expect(failure.cleanupResult).toEqual(cleanup); - expect(failure.resources).toEqual( - expect.arrayContaining([request.runId]), - ); + expect(failure.resources).toEqual([ + request.runId, + `pid:${launched.ownership.leaderPid}`, + `process-group:${launched.ownership.processGroupId}`, + ]); // An unresolved member does not grant authority to signal the shared group. const acquiredPid = launched.ownership.leaderPid; expect(() => process.kill(acquiredPid, 0)).not.toThrow(); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index c3dc21542..01d8eba13 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -196,9 +196,9 @@ itWithCaptureCapability( result.value.frames.map(({ at_ms }) => at_ms), "capture-relative frame times stay ordered and nonnegative", ).toEqual( - [...result.value.frames.map(({ at_ms }) => at_ms)].sort( - (left, right) => left - right, - ), + result.value.frames + .map(({ at_ms }) => at_ms) + .sort((left, right) => left - right), ); expect(result.value.frames.every(({ at_ms }) => at_ms >= 0)).toBe(true); expect(result.value.exit.reason).toBe("exited"); diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 2837ccf40..159f84c93 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -28,6 +28,35 @@ const identitySchema = z.object({ const itWithLinuxCaptureCapability = process.platform === "linux" ? itWithCaptureCapability : it.skip; +itWithCaptureCapability( + "captures a POSIX PATH executable with a literal backslash in its basename", + async () => { + const root = await createTestTempDirectory("rea-process-backslash-name-"); + const bin = join(root, "bin"); + await mkdir(bin); + const command = "selected\\node"; + const selected = join(bin, command); + await symlink(process.execPath, selected); + const scenario = parseProcessScenario({ + executable: command, + working_directory: root, + environment: { PATH: bin }, + arguments: ["-e", 'process.stdout.write("selected executable")'], + settle_ms: 100, + }); + const result = await captureProcessScenario(scenario); + if (!result.ok) throw result.error; + expect(result.value.frames.map(({ data }) => data).join("")).toBe( + "selected executable", + ); + expect(result.value.exit).toMatchObject({ code: 0, reason: "exited" }); + expect(result.value.cleanup).toMatchObject({ + owned_process_group: "verified", + temporary_root: "removed", + }); + }, +); + itWithCaptureCapability( "preserves numeric reports and original normalized PTY text in Evidence", async () => { diff --git a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts index 270bf02b3..ea8fb964c 100644 --- a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts +++ b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts @@ -1,4 +1,11 @@ -import { access, chmod, readFile, stat } from "node:fs/promises"; +import { + access, + chmod, + mkdir, + readFile, + stat, + writeFile, +} from "node:fs/promises"; import { join, posix, win32 } from "node:path"; import { fileURLToPath } from "node:url"; @@ -480,6 +487,64 @@ describe("Ghidra Windows launcher", () => { ).toThrow(/metacharacters/u); }); + it.skipIf(process.platform !== "win32").each([ + { + properties: + "application.version=12.1.4\napplication.release.name=PUBLIC\n", + release: "ghidra_12.1.4_PUBLIC", + }, + { properties: undefined, release: "renamed-install" }, + { properties: "application.version=12.1.4\n", release: "renamed-install" }, + ])( + "prepares the release directory through the Windows runtime ($release)", + async ({ properties, release }) => { + const parent = await createTestTempDirectory("rea-launch-config-"); + const installRoot = join(parent, "renamed-install"); + if (properties !== undefined) { + await mkdir(join(installRoot, "Ghidra"), { recursive: true }); + await writeFile( + join(installRoot, "Ghidra", "application.properties"), + properties, + ); + } + const runtime = await createGhidraTestRuntime(parent); + runtimes.push(runtime); + const spawnProcess = vi.fn(async () => { + throw new Error("fixture stops before process creation"); + }); + const launcher = new GhidraHeadlessLauncher({ + environment: process.env, + platform: "win32", + analyzeHeadlessPath: join( + installRoot, + "support", + "analyzeHeadless.bat", + ), + javaHome: "C:\\Java\\jdk-21", + bridgeScriptPath: join(parent, "ReaGhidraBridge.java"), + spawnProcess, + }); + const launched = await launcher.launch({ + runtimeRoot: runtime.path, + transport: "unix-socket", + endpointPath: join(runtime.path, "bridge.sock"), + token: "fixture-token", + runId: "d6fcbb66-e829-4ff6-a535-0035aec63139", + targetPath: join(parent, "fixture.exe"), + targetSha256: "b".repeat(64), + providerVersion: "12.1.4", + profileDigest: "a".repeat(64), + }); + expect(launched.ok).toBe(false); + expect(spawnProcess).toHaveBeenCalledOnce(); + expect( + ( + await stat(join(runtime.path, "config", "ghidra", release)) + ).isDirectory(), + ).toBe(true); + }, + ); + it.skipIf(process.platform !== "win32")( "keeps authority in a private descriptor and isolates Ghidra state through the batch launcher", async () => { diff --git a/tests/boundary/providers/ida/containment.test.ts b/tests/boundary/providers/ida/containment.test.ts new file mode 100644 index 000000000..75eb78f42 --- /dev/null +++ b/tests/boundary/providers/ida/containment.test.ts @@ -0,0 +1,144 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectAddressContext } from "../../../../src/application/AnalysisContextQueries.js"; +import { EnhancedTools } from "../../../../src/application/EnhancedTools.js"; +import { IdaSessionClient } from "../../../../src/ida/IdaSessionClient.js"; +import { createIdaTarget, RecordingIdaMcp } from "../../../fixtures/idaMcp.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const session = async (cappedReferences = false) => { + const { root, target } = await createIdaTarget("arm64"); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + if (cappedReferences) { + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [ + { + xrefs: [ + { + addr: "0x1004", + type: "code", + fn: { addr: "0x1000", name: "main", size: "0x20" }, + }, + ], + more: true, + }, + ] + : inner?.(name, args); + } + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "headless", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, target }; +}; + +const MAIN = { + found: true, + procedure: { address: "0x1000", name: "main" }, +}; + +describe("trace_feature on headless IDA", () => { + it("attributes a reference to its containing procedure", async () => { + const { client } = await session(); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + matches: [{ type: "string", address: "0x3000" }], + references: [ + { + target_address: "0x3000", + source_address: "0x1004", + containing_procedure: MAIN, + }, + ], + truncated: false, + residual_unknowns: [], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("keeps an upstream reference cap visible once containment makes the trace run", async () => { + const { client } = await session(true); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + references: [{ source_address: "0x1004", containing_procedure: MAIN }], + truncated: true, + residual_unknowns: [ + "References to 0x3000 were truncated by the provider; additional references may exist.", + ], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("inspect_address_context on headless IDA", () => { + it("resolves the admitted program as the document and reports the containing procedure", async () => { + const { client, target } = await session(); + try { + const result = await inspectAddressContext(client, { + address: "0x1004", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + address: "0x1004", + document: target.path, + procedure: { state: "available", value: MAIN }, + name: { state: "unavailable" }, + comment: { state: "unavailable" }, + inline_comment: { state: "unavailable" }, + bookmarks: { state: "unavailable" }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("answers the same way for an explicit admitted document", async () => { + const { client, target } = await session(); + try { + const implicit = await inspectAddressContext(client, { + address: "0x1004", + }); + const explicit = await inspectAddressContext(client, { + address: "0x1004", + document: target.path, + }); + + if (!implicit.ok) throw implicit.error; + if (!explicit.ok) throw explicit.error; + expect(explicit.value).toEqual(implicit.value); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/tests/boundary/providers/ida/verifierFeatureTrace.test.ts b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts new file mode 100644 index 000000000..77f654ffc --- /dev/null +++ b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { + assertFeatureTrace, + findMappedNonFunction, +} from "../../../../scripts/lib/ida-feature-trace.mjs"; + +const procedure = { address: "0x1000", name: "main" }; + +const containing = (found: boolean) => + found + ? { + query_address: "0x1004", + found: true, + procedure: { address: "0x1000", name: "main" }, + } + : { + query_address: "0x3010", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + +const trace = ( + matches: readonly { type: string; address: string }[], + references: readonly { + target_address: string; + containing_procedure: unknown; + }[], +) => ({ + query: "main", + search_mode: "literal", + matches, + references, + truncated: false, + residual_unknowns: [], +}); + +const SELECTED = { type: "procedure", address: "0x1000" }; +const SUBSTRING = { type: "procedure", address: "0x2000" }; +const STRING = { type: "string", address: "0x3000" }; + +describe("the real IDA lane's feature trace assertions", () => { + it("accepts references to every matched address, whoever contains the source", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED, SUBSTRING, STRING], + [ + { + target_address: "0x1000", + containing_procedure: containing(true), + }, + // A substring match such as main_helper, referenced from code. + { + target_address: "0x2000", + containing_procedure: containing(true), + }, + // A string containing the query, referenced from data outside functions. + { + target_address: "0x3000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("accepts a reference to the selected procedure from data outside functions", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("rejects a reference whose target is not one of the matches", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x9999", + containing_procedure: containing(true), + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace without the selected procedure's match", () => { + expect(() => assertFeatureTrace(trace([STRING], []), procedure)).toThrow(); + }); + + it("rejects a containment answer that is neither found nor a known reason", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: { + query_address: "0x1004", + found: false, + procedure: null, + reason: "unknown", + }, + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace that does not say whether it was truncated", () => { + expect(() => + assertFeatureTrace( + { ...trace([SELECTED], []), truncated: undefined }, + procedure, + ), + ).toThrow(); + }); +}); + +describe("the real IDA lane's mapped non-function probe", () => { + const inFunction = { + query_address: "0x1100", + found: true, + procedure: { address: "0x1000", name: "main" }, + }; + const data = { + query_address: "0x3000", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + const resolver = + (answers: Readonly>, asked: string[] = []) => + async (address: string) => { + asked.push(address); + return answers[address]; + }; + + it("skips a string inside a function chunk and uses the next one", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x3000" }], + resolver({ "0x1100": inFunction, "0x3000": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked, "strings are tried in order until one qualifies").toEqual([ + "0x1100", + "0x3000", + ]); + }); + + it("stops at the first string that qualifies", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x3000" }, { address: "0x3100" }], + resolver({ "0x3000": data, "0x3100": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked).toEqual(["0x3000"]); + }); + + it("reports no qualifying string when every string is inside a function", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x1200" }], + resolver({ "0x1100": inFunction, "0x1200": inFunction }), + ), + ).resolves.toBeUndefined(); + }); + + it("reports no qualifying string when the target has no strings", async () => { + await expect( + findMappedNonFunction([], resolver({})), + ).resolves.toBeUndefined(); + }); + + it("rejects a listed string that resolves as unmapped, since IDA cannot list an unmapped address", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x4000" }, { address: "0x3000" }], + resolver({ + "0x4000": { ...data, reason: "outside_segments" }, + "0x3000": data, + }), + ), + ).rejects.toThrow(); + }); +}); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index 24dadc128..b5adf8297 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -5,6 +5,7 @@ import { DatabaseSync } from "node:sqlite"; import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; import { expect, it as nativeTest, onTestFinished } from "vitest"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; @@ -356,6 +357,49 @@ it("does not execute views when selecting rows", async () => { }); }); +it("attributes table and column selection failures to their own arguments", async () => { + const { database } = await createFixture(); + const provider = createProvider(); + onTestFinished(() => provider.close()); + const projection = async (selection: { + table: string; + columns?: string[]; + }) => { + const result = await provider.inspect({ + path: database.path, + row_limit: 1, + ...selection, + }); + if (result.ok) throw new Error("Expected a rejected selection"); + return projectAnalysisError(result.error); + }; + expect( + await projection({ table: "records", columns: ["nope"] }), + ).toMatchObject({ + details: { + issues: [ + { + path: ["columns"], + reason: "invalid_format", + message: + "Selected column does not exist with that exact name in table records: nope", + }, + ], + }, + }); + expect(await projection({ table: "absent" })).toMatchObject({ + details: { issues: [{ path: ["table"] }] }, + }); + const selected = await provider.inspect({ + path: database.path, + table: "records", + columns: ["id"], + row_limit: 1, + }); + if (!selected.ok) throw selected.error; + expect(selected.value.result).toMatchObject({ rows: { columns: ["id"] } }); +}); + it("rejects malformed databases without altering the selected file", async () => { const workspace = await createTestWorkspace("rea-sqlite-invalid-"); onTestFinished(() => removeTestWorkspace(workspace.root)); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index 813b90004..092f869e2 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -1278,3 +1278,102 @@ describe("selector origin forwarding", () => { ).toEqual(["provider", "provider", undefined]); }); }); + +describe("reference truncation forwarding", () => { + const referenceProvider = ( + referencesTruncated: boolean | undefined, + calls: string[], + ): AnalysisProvider => { + const identity = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + } as const; + return { + identity: () => identity, + resolveAnalysisProfile: () => + Promise.resolve( + resultOk({ + profile: createAnalysisProfile(identity, { fixture: true }), + }), + ), + capabilities: () => [ + { + provider: identity, + operation: "xrefs", + available: true, + reason: null, + effects: { + mutatesArtifact: false, + launchesProcess: false, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: false, + changesPermissions: false, + requiresRoot: false, + }, + limitations: [], + }, + ], + createClient: () => ({ + execute: (operation) => { + calls.push(operation); + return Promise.resolve( + resultOk( + createAnalysisExecution( + ["0x2000"], + identity, + referencesTruncated === undefined + ? {} + : { referencesTruncated }, + ), + ), + ); + }, + close: () => Promise.resolve(resultOk(null)), + }), + }; + }; + const parameters = { address: "0x1000", document: "fixture" }; + + it("forwards a truncation on both execution paths and never replays it from the snapshot", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(true, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const direct = await session.execute("xrefs", parameters); + const admitted = await session.withAdmittedAnalysis( + "trace_feature", + undefined, + (analysis) => analysis.execute("xrefs", parameters), + ); + + expect(direct.ok && direct.value.referencesTruncated).toBe(true); + if (!admitted.ok) throw admitted.error; + expect(admitted.value.ok && admitted.value.value.referencesTruncated).toBe( + true, + ); + expect( + calls, + "a truncated read is not recorded, so every call reaches the provider", + ).toEqual(["xrefs", "xrefs"]); + }); + + it("keeps an explicit complete report out of the way of snapshot replay", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(false, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const live = await session.execute("xrefs", parameters); + await session.execute("xrefs", parameters); + + expect(live.ok && live.value.referencesTruncated).toBe(false); + expect(calls, "the complete read is recorded and replayed").toEqual([ + "xrefs", + ]); + }); +}); diff --git a/tests/conformance/ida/providerContracts.test.ts b/tests/conformance/ida/providerContracts.test.ts index 39c524637..4e4b04c3b 100644 --- a/tests/conformance/ida/providerContracts.test.ts +++ b/tests/conformance/ida/providerContracts.test.ts @@ -29,15 +29,25 @@ const parametersFor = ( operation: string, ): Readonly> => { if (operation.startsWith("search_")) return { pattern: "a.b" }; - if (operation === "xrefs") return { address: "0x1000" }; + if (operation === "xrefs" || operation === "resolve_containing_procedure") + return { address: "0x1000" }; if (operation.startsWith("list_")) return {}; return { procedure: "main" }; }; +/** Operations only the headless profile declares. */ +const HEADLESS_ONLY: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + describe.each(["attached", "headless"] as const)( "IDA %s caller-visible contracts", (mode) => { - it.each(IDA_OPERATIONS)( + const operations = IDA_OPERATIONS.filter( + (operation) => mode === "headless" || !HEADLESS_ONLY.has(operation), + ); + it.each(operations)( "declares %s UI effects matching the upstream tools it reaches", async (operation) => { const { root, target } = await createIdaTarget(); @@ -69,7 +79,7 @@ describe.each(["attached", "headless"] as const)( }, ); - it.each(IDA_OPERATIONS)( + it.each(operations)( "serializes %s against its named advertised output contract", async (operation) => { const { root, target } = await createIdaTarget(); diff --git a/tests/conformance/swift/string-literals.swift b/tests/conformance/swift/string-literals.swift new file mode 100644 index 000000000..345c4cf74 --- /dev/null +++ b/tests/conformance/swift/string-literals.swift @@ -0,0 +1,23 @@ +@inline(never) +func consume(_ value: String) { print(value) } +@inline(never) +func smallLiteral() { consume("SleepDisabled") } +@inline(never) +func largeLiteral() { consume("The power setting could not be changed.") } +@inline(never) +func unicodeLiterals() { + consume("café") + consume("Résumé of the naïve décor, written twice") +} +@inline(never) +func emptyAndControlLiterals() { + consume("") + consume("tab\there") +} +@inline(never) +func returnedLiteral() -> String { "Holding awake" } +smallLiteral() +largeLiteral() +unicodeLiterals() +emptyAndControlLiterals() +print(returnedLiteral(), CommandLine.arguments.count) diff --git a/tests/fixtures/idaMcp.ts b/tests/fixtures/idaMcp.ts index 04bdb5f16..21e5e38f7 100644 --- a/tests/fixtures/idaMcp.ts +++ b/tests/fixtures/idaMcp.ts @@ -57,6 +57,7 @@ const modernNames = [ "callees", "xrefs_to", "callgraph", + "survey_binary", ]; /** Stateful producer fixture at the real connection seam; no module mocking. */ diff --git a/tests/process-global/pwntoolsStageRetirement.test.ts b/tests/process-global/pwntoolsStageRetirement.test.ts index 6ec062535..7cdf0b48a 100644 --- a/tests/process-global/pwntoolsStageRetirement.test.ts +++ b/tests/process-global/pwntoolsStageRetirement.test.ts @@ -18,7 +18,10 @@ import { PWNTOOLS_MEMORY_FAILURE_EXIT, PWNTOOLS_PROVIDER_IDENTITY, } from "../../src/native/pwntools/PwntoolsRelease.js"; -import { cleanupOwnedProcessGroup } from "../../src/process/ProcessOwnership.js"; +import { + cleanupOwnedProcessGroup, + type OwnedProcessGroup, +} from "../../src/process/ProcessOwnership.js"; import { prepareProcessOwnershipInspection } from "../../src/process/ProcessOwnershipObservation.js"; import { spawnOwnedProviderProcess } from "../../src/process/ProviderProcess.js"; import { PrivateRuntimeRoot } from "../../src/process/PrivateRuntimeRoot.js"; @@ -232,12 +235,13 @@ it.runIf(supportedHost).each([ const { path, root, environment, basic } = await fixture(); let launches = 0; let cleanupAllowed = false; + let retainedOwnership: OwnedProcessGroup | undefined; const read = vi.spyOn(artifacts, "readStableArtifact"); const provider = new PwntoolsRecordedCrashProvider( environment, async (spawn) => { if (stage === "debugger" && ++launches === 1) return basic(spawn); - return changingWorker( + const launched = await changingWorker( spawn, join(spawn.cwd ?? "", "resource.failure"), { @@ -247,6 +251,8 @@ it.runIf(supportedHost).each([ }, () => cleanupAllowed, ); + retainedOwnership = launched.ownership; + return launched; }, async () => root, ); @@ -261,14 +267,14 @@ it.runIf(supportedHost).each([ if (result.ok) throw new Error("Expected unresolved process ownership"); const projected = projectAnalysisError(result.error); expect(projected.code).toBe("cleanup_incomplete"); - expect(projected).toMatchObject({ - details: { - resources: expect.arrayContaining([ - expect.stringMatching(/^pid:\d+$/u), - expect.stringMatching(/^process-group:\d+$/u), - ]), - }, - }); + if (retainedOwnership === undefined) + throw new Error("Expected retained process ownership"); + expect(projected.details?.resources).toEqual([ + retainedOwnership.runId, + root.path, + `pid:${retainedOwnership.leaderPid}`, + `process-group:${retainedOwnership.processGroupId}`, + ]); if (stage === "debugger") expect(projected).toMatchObject({ details: {