From 599a8ee11de6b1a7fb270e669b88d78b6be982d2 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:19:07 +0800 Subject: [PATCH 01/26] fix(process): preserve cleanup ownership and normalize captures once Retain unresolved process identity and launch evidence through retryable cleanup. Share capture path normalization and preserve failed filesystem observations. Bound child output and drain Windows pipes without one-chunk throttling. Reviewed fixes include #1628, #2632, #2644, #2784, #2928, #2931, #2965, #2992, and #3011. Validation: 92 focused source tests passed; four platform-specific skips. Real Windows and Hopper lifecycle coverage was not rerun. --- docs/process-capture.md | 3 + native/windows/src/process.cc | 32 +- scripts/lib/clean-linux-hopper-runtime.mjs | 92 +++- src/domain/process/processCaptureCoverage.ts | 18 +- src/domain/process/processCaptureExample.ts | 2 + .../process/processCaptureValidation.ts | 12 + src/domain/process/processComparison.ts | 36 +- .../process/processTraceComparison.test.ts | 111 +++++ src/domain/process/processTraceEvaluation.ts | 98 +++-- src/hopper/BridgeLauncher.ts | 12 +- src/hopper/HopperCleanup.test.ts | 63 ++- src/hopper/HopperCleanup.ts | 39 +- src/hopper/HopperClient.ts | 27 ++ src/hopper/HopperSessionValues.ts | 10 +- src/hopper/HopperTargetLease.test.ts | 79 ++++ src/hopper/HopperTargetLease.ts | 69 ++- src/process/ArtifactAnalysisResources.test.ts | 85 ++++ src/process/ArtifactAnalysisResources.ts | 63 ++- src/process/ExecFileBoundedOutput.test.ts | 50 +++ src/process/ExecFileBoundedOutput.ts | 73 ++++ src/process/ProcessOwnership.test.ts | 179 ++++++++ src/process/ProcessOwnership.ts | 88 +++- .../ProcessOwnership.validation.test.ts | 34 ++ src/process/ProcessOwnershipObservation.ts | 16 + src/process/ProcessStartIdentity.test.ts | 21 + src/process/ProviderProcess.ts | 12 +- src/process/ProviderProcessOutput.test.ts | 24 + src/process/WindowsAuthority.ts | 139 +++--- .../capture/FilesystemCoverage.test.ts | 39 ++ .../capture/FilesystemSnapshot.test.ts | 63 +++ src/process/capture/FilesystemSnapshot.ts | 44 +- .../capture/ProcessCaptureLifecycle.ts | 10 +- src/process/capture/ProcessHarness.ts | 15 +- .../capture/ProcessNormalization.test.ts | 37 ++ src/process/capture/ProcessNormalization.ts | 97 +++-- src/windows/WindowsOwnedProcess.test.ts | 223 +++++++++- src/windows/WindowsOwnedProcess.ts | 87 +++- .../process/cleanLinuxHopperRuntime.test.ts | 411 ++++++++++++++++++ .../boundary/process/providerProcess.test.ts | 27 +- .../hopper/bridgeBootstrapCleanup.test.ts | 58 +++ .../cleanLinuxHopperRuntime.test.ts | 6 +- ...systemSnapshotFallbackCancellation.test.ts | 118 +++++ .../hopperLeaseDescriptorPermissions.test.ts | 79 ++++ 43 files changed, 2561 insertions(+), 240 deletions(-) create mode 100644 src/process/ArtifactAnalysisResources.test.ts create mode 100644 src/process/ExecFileBoundedOutput.test.ts create mode 100644 src/process/ExecFileBoundedOutput.ts create mode 100644 tests/boundary/process/cleanLinuxHopperRuntime.test.ts create mode 100644 tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts create mode 100644 tests/process-global/filesystemSnapshotFallbackCancellation.test.ts create mode 100644 tests/process-global/hopperLeaseDescriptorPermissions.test.ts diff --git a/docs/process-capture.md b/docs/process-capture.md index cc163f715..5b2bbbeeb 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -380,6 +380,9 @@ Every capture requires `truncation_details`, with separate accounting for: When a directory exceeds that capacity, the retained subset follows filesystem enumeration order and is then sorted; it is explicitly incomplete and cannot establish that an omitted path is absent. + An unreadable directory is recorded in `enumeration_omissions` with its + root-aliased path and operating-system error code. Entries observed before + that directory remain available, while descendants beneath it stay unknown. Each retained regular file without a digest has an aliased path, size, remaining budget and reason: `file_bytes_budget`, `file_changed_or_short_read`, or `file_unavailable`. Its `system_code` is diff --git a/native/windows/src/process.cc b/native/windows/src/process.cc index 42ee9582b..55e415345 100644 --- a/native/windows/src/process.cc +++ b/native/windows/src/process.cc @@ -214,18 +214,32 @@ std::unique_ptr spawnProcess(const std::wstring& command, const std::ws } static napi_value readOutput(napi_env env, Handle& read, bool& ended) { - std::array bytes; - DWORD available = 0, count = 0; + constexpr size_t kMaximumReadPerPoll = 1024 * 1024; + std::array chunk; + std::vector bytes; if (!read.valid()) ended = true; - else if (!PeekNamedPipe(read.get(), nullptr, 0, nullptr, &available, nullptr)) { - require(GetLastError() == ERROR_BROKEN_PIPE, "Observe owned process output failed"); - read.reset(); ended = true; - } else if (available > 0) { - require(ReadFile(read.get(), bytes.data(), std::min(available, bytes.size()), &count, nullptr), - "Read owned process output failed"); + else { + while (bytes.size() < kMaximumReadPerPoll) { + DWORD available = 0; + if (!PeekNamedPipe(read.get(), nullptr, 0, nullptr, &available, nullptr)) { + require(GetLastError() == ERROR_BROKEN_PIPE, + "Observe owned process output failed"); + read.reset(); + ended = true; + break; + } + if (available == 0) break; + const auto requested = static_cast(std::min( + {available, chunk.size(), kMaximumReadPerPoll - bytes.size()})); + DWORD count = 0; + require(ReadFile(read.get(), chunk.data(), requested, &count, nullptr) && + count > 0, + "Read owned process output failed"); + bytes.insert(bytes.end(), chunk.begin(), chunk.begin() + count); + } } napi_value result; - check(napi_create_buffer_copy(env, count, bytes.data(), nullptr, &result)); + check(napi_create_buffer_copy(env, bytes.size(), bytes.data(), nullptr, &result)); return result; } diff --git a/scripts/lib/clean-linux-hopper-runtime.mjs b/scripts/lib/clean-linux-hopper-runtime.mjs index 4d6bc8163..41ccfed6c 100644 --- a/scripts/lib/clean-linux-hopper-runtime.mjs +++ b/scripts/lib/clean-linux-hopper-runtime.mjs @@ -1,8 +1,36 @@ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { readdir, stat } from "node:fs/promises"; +import { readFile, readdir, stat } from "node:fs/promises"; +import { basename } from "node:path"; -const execFileAsync = promisify(execFile); +// CPython's interface option ends interpreter parsing; following values are +// script arguments. -W/-X consume either an attached value or the next operand. +// https://docs.python.org/3.12/using/cmdline.html +const pythonScriptOperand = (arguments_) => { + for (let index = 1; index < arguments_.length; index++) { + const option = arguments_[index]; + if (option === "--") return arguments_[index + 1]; + if (option === "-" || option === "") return undefined; + if (!option.startsWith("-")) return option; + if (option === "--check-hash-based-pycs") { + index++; + continue; + } + if (/^--(?:help(?:-env|-xoptions|-all)?|version)$/u.test(option)) + return undefined; + if (option.startsWith("--")) + throw new Error(`Unknown Python helper invocation option: ${option}`); + for (let character = 1; character < option.length; character++) { + const flag = option[character]; + if ("cmh?V".includes(flag)) return undefined; + if (flag === "W" || flag === "X") { + if (character === option.length - 1) index++; + break; + } + if (!"bBdEiIOPqRsSuvx".includes(flag)) + throw new Error(`Unknown Python helper invocation option: ${option}`); + } + } + return undefined; +}; /** * Observe leftover demo processes on a dedicated clean Linux runner, including @@ -18,9 +46,6 @@ export const assertCleanLinuxHopperRuntime = async ( ], procRoot = "/proc", ) => { - const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]); - if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout)) - throw new Error("Clean Linux runner retained a Hopper demo process"); const launchers = []; for (const path of launcherPaths) { try { @@ -29,20 +54,55 @@ export const assertCleanLinuxHopperRuntime = async ( if (error?.code !== "ENOENT") throw error; } } - if (launchers.length === 0) return; // /proc exposes the executed file even when argv[0] names a removed symlink. // Compare file identity, not a display path or a process name. let inspectionError; for (const entry of await readdir(procRoot, { withFileTypes: true })) { if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue; - let executable; + let retainedRuntime; try { - executable = await stat(`${procRoot}/${entry.name}/exe`, { - bigint: true, - }); + // procfs preserves argument boundaries. A caller merely mentioning a + // runtime in its script text or other arguments is not that runtime. + const commandLine = await readFile( + `${procRoot}/${entry.name}/cmdline`, + "utf8", + ); + const arguments_ = commandLine.split("\0"); + // The kernel command field identifies renamed runtime processes such + // as Xvfb. Keep it separate from arguments so ordinary script text does + // not count as a retained runtime. + const kernelCommand = ( + await readFile(`${procRoot}/${entry.name}/comm`, "utf8") + ).replace(/\n$/u, ""); + if ( + commandLine.length === 0 && + /^python3(?:\.\d+)?$/u.test(kernelCommand) + ) + throw new Error( + "Cannot identify a command-less Python process on the clean Linux runner", + ); + retainedRuntime = + kernelCommand === "Hopper" || + kernelCommand === "Xvfb" || + arguments_[0] === "/opt/hopper/bin/Hopper" || + basename(arguments_[0] ?? "") === "Xvfb" || + (/^python3(?:\.\d+)?$/u.test(basename(arguments_[0] ?? "")) && + arguments_.some( + (value) => basename(value) === "hopper-demo-x11.py", + ) && + basename(pythonScriptOperand(arguments_) ?? "") === + "hopper-demo-x11.py"); + if (!retainedRuntime && launchers.length > 0) { + const executable = await stat(`${procRoot}/${entry.name}/exe`, { + bigint: true, + }); + retainedRuntime = launchers.some( + ({ dev, ino }) => dev === executable.dev && ino === executable.ino, + ); + } } catch (error) { if (error?.code === "ENOENT" || error?.code === "ESRCH") continue; - // Other users' executables may be unreadable. The setup lane launches + // Other users' process facts may be unreadable. The setup lane launches // providers as the caller; unreadable caller-owned processes fail closed. if (error?.code === "EACCES") { const owner = await stat(`${procRoot}/${entry.name}`).catch((cause) => { @@ -56,11 +116,7 @@ export const assertCleanLinuxHopperRuntime = async ( inspectionError ??= error; continue; } - if ( - launchers.some( - ({ dev, ino }) => dev === executable.dev && ino === executable.ino, - ) - ) + if (retainedRuntime) throw new Error("Clean Linux runner retained a Hopper demo process"); } if (inspectionError !== undefined) throw inspectionError; diff --git a/src/domain/process/processCaptureCoverage.ts b/src/domain/process/processCaptureCoverage.ts index 72bbb9fb1..1a0e73ee5 100644 --- a/src/domain/process/processCaptureCoverage.ts +++ b/src/domain/process/processCaptureCoverage.ts @@ -16,7 +16,19 @@ export const filesystemCoverageSchema = z.strictObject({ depth_limit: z.number().int().nonnegative(), enumeration_truncated: z.boolean(), enumeration_reasons: z.array( - z.enum(["files_limit", "depth_limit", "identity_changed"]), + z.enum([ + "files_limit", + "depth_limit", + "identity_changed", + "directory_unavailable", + ]), + ), + enumeration_omissions: z.array( + z.strictObject({ + path: z.string(), + reason: z.literal("directory_unavailable"), + system_code: z.string().nullable(), + }), ), hash_budget_bytes: z.number().int().nonnegative(), hashed_bytes: z.number().int().nonnegative(), @@ -66,7 +78,9 @@ export type ProcessSamplingCoverage = const omitted = (retention: TerminalRetention): boolean => retention.observed_frames > retention.retained_frames; const filesystemTruncated = (coverage: FilesystemCoverage): boolean => - coverage.enumeration_truncated || coverage.hash_omissions.length > 0; + coverage.enumeration_truncated || + coverage.enumeration_omissions.length > 0 || + coverage.hash_omissions.length > 0; /** Derive the aggregate truncation flag from independent producer observations. */ export const hasCaptureTruncation = ( diff --git a/src/domain/process/processCaptureExample.ts b/src/domain/process/processCaptureExample.ts index c8dbff897..8259f3850 100644 --- a/src/domain/process/processCaptureExample.ts +++ b/src/domain/process/processCaptureExample.ts @@ -79,6 +79,7 @@ export const EMPTY_PROCESS_CAPTURE_EXAMPLE = { depth_limit: 16, enumeration_truncated: false, enumeration_reasons: [], + enumeration_omissions: [], hash_budget_bytes: 10_000_000, hashed_bytes: 0, hash_omissions: [], @@ -88,6 +89,7 @@ export const EMPTY_PROCESS_CAPTURE_EXAMPLE = { depth_limit: 16, enumeration_truncated: false, enumeration_reasons: [], + enumeration_omissions: [], hash_budget_bytes: 10_000_000, hashed_bytes: 0, hash_omissions: [], diff --git a/src/domain/process/processCaptureValidation.ts b/src/domain/process/processCaptureValidation.ts index 537f6f92f..53a783798 100644 --- a/src/domain/process/processCaptureValidation.ts +++ b/src/domain/process/processCaptureValidation.ts @@ -334,6 +334,9 @@ const validateCoverage = ( .filter((file) => file.type === "file" && file.sha256 === null) .map((file) => [file.path, file]), ); + const directories = new Set( + files.filter((file) => file.type === "directory").map(({ path }) => path), + ); require(coverage.hash_omissions.length === unhashedFiles.size && new Set(coverage.hash_omissions.map(({ path }) => path)).size === unhashedFiles.size && @@ -345,6 +348,14 @@ const validateCoverage = ( ? omission.system_code !== null : omission.system_code === null), ), `truncation_details.${name}.hash_omissions`, "every retained regular file without a digest must have one matching omission reason"); + require(new Set(coverage.enumeration_omissions.map(({ path }) => path)) + .size === coverage.enumeration_omissions.length && + (coverage.enumeration_omissions.length === 0 || + coverage.enumeration_reasons.includes("directory_unavailable")) && + coverage.enumeration_omissions.every( + (omission) => + directories.has(omission.path) && omission.system_code !== null, + ), `truncation_details.${name}.enumeration_omissions`, "every directory enumeration omission must identify one retained directory and its system error"); require(coverage.hashed_bytes === files .filter((file) => file.type === "file" && file.sha256 !== null) @@ -356,6 +367,7 @@ const validateCoverage = ( coverage.enumeration_reasons.length > 0 && checkpoint?.truncated === (coverage.enumeration_truncated || + coverage.enumeration_omissions.length > 0 || coverage.hash_omissions.length > 0), `truncation_details.${name}`, "filesystem checkpoint truncation must match enumeration and digest coverage"); } diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts index eafcd4fd8..06e134061 100644 --- a/src/domain/process/processComparison.ts +++ b/src/domain/process/processComparison.ts @@ -1,7 +1,10 @@ import canonicalize from "canonicalize"; import { z } from "zod"; import { processSourceTruncated } from "./processCaptureCoverage.js"; -import type { ProcessObservationSource } from "./processObservation.js"; +import type { + ProcessObservationLocation, + ProcessObservationSource, +} from "./processObservation.js"; import type { ProcessCapture } from "./processCaptureParsing.js"; import { comparableTerminalFrame } from "./processObservation.js"; @@ -299,6 +302,18 @@ const differingContractFields = ( ) .sort(); +const LOCATION_SOURCES = { + frames: "terminal_raw", + rendered_frames: "terminal_rendered", + interaction_events: "interaction", + lifecycle: "lifecycle", + process_samples: "process", + filesystem_checkpoints: "filesystem", +} as const satisfies Record< + ProcessObservationLocation["collection"], + ProcessObservationSource +>; + const applyTraceVerdict = ( dimensions: ComparisonDimensions, specification: ProcessTraceSpecification, @@ -314,6 +329,20 @@ const applyTraceVerdict = ( : "changed"; const sources = new Set(specification.events.map(({ source }) => source)); const covered = dimensionsForTraceSources(sources); + // A different verdict names the failing events and observations; attribute + // it only to their dimensions so equal observations keep their exact status. + const diagnosticSources = new Set([ + ...specification.events + .filter(({ id }) => trace.diagnostic?.event_ids.includes(id) === true) + .map(({ source }) => source), + ...(trace.diagnostic?.locations ?? []).map( + ({ collection }) => LOCATION_SOURCES[collection], + ), + ]); + const attributed = + diagnosticSources.size > 0 + ? dimensionsForTraceSources(diagnosticSources) + : covered; const statusFor = ( dimension: (typeof PROCESS_COMPARISON_DIMENSIONS)[number], ): ComparisonStatus => { @@ -321,7 +350,10 @@ const applyTraceVerdict = ( if (!covered.has(dimension)) return exact; if (trace.verdict === "nonconforming") return exact; if (trace.verdict === "different") - return traceShowsDifference ? "changed" : exact; + return traceShowsDifference && + (exact !== "unchanged" || attributed.has(dimension)) + ? "changed" + : exact; if ( trace.verdict === "unknown" && ["changed", "added", "removed"].includes(exact) diff --git a/src/domain/process/processTraceComparison.test.ts b/src/domain/process/processTraceComparison.test.ts index 2207352e3..07113bfad 100644 --- a/src/domain/process/processTraceComparison.test.ts +++ b/src/domain/process/processTraceComparison.test.ts @@ -11,6 +11,7 @@ import { values, } from "./processTraceComparison.fixture.js"; +import { compareProcessCaptures } from "./processComparison.js"; import { compareProcessTraces } from "./processTraceComparison.js"; describe("generic capture trace comparison", () => { it("requires explicit partial ordering and never infers causality from timestamps", () => { @@ -285,3 +286,113 @@ describe("unknown process trace evidence", () => { ).toThrow("event_journal"); }); }); + +describe("process trace dimension attribution", () => { + const traced = ( + left: ReturnType, + right: ReturnType, + ) => + compareProcessCaptures(left, right, { + traceSpecification: partialSpecification(), + }); + const make = (data: string, command: string) => + capture({ + ...values(["terminal", "process"]), + frames: [{ ...terminal, data }], + process_samples: [{ ...processStarted, command }], + }); + + it("keeps equal dimensions unchanged when one side fails a predicate", () => { + expect( + traced(make("Ready", "worker"), make("Different", "worker")), + ).toMatchObject({ + trace: { verdict: "different" }, + terminal: "changed", + process: "unchanged", + }); + expect( + traced(make("Ready", "worker"), make("Ready", "other-worker")), + ).toMatchObject({ + trace: { verdict: "different" }, + terminal: "unchanged", + process: "changed", + }); + }); + + it("still attributes an ordering difference to both ordered dimensions", () => { + expect( + traced( + capture(values(["terminal", "process"])), + capture(values(["process", "terminal"])), + ), + ).toMatchObject({ + trace: { verdict: "different" }, + terminal: "changed", + process: "changed", + }); + }); + + it("attributes every independent failed ordering edge", () => { + const terminals = [ + terminal, + { ...terminal, sequence: 1, data: "Ready again" }, + ]; + const workers = [processStarted, { ...processStarted, at_ms: 2, pid: 2 }]; + const events = ["ready1", "worker1", "ready2", "worker2"] as const; + const make = (order: readonly (typeof events)[number][]) => + capture({ + frames: terminals, + process_samples: workers, + event_journal: order.map((event) => ({ + collection: event.startsWith("ready") + ? ("frames" as const) + : ("process_samples" as const), + index: Number(event.slice(-1)) - 1, + })), + }); + const left = make(events); + const right = make(["worker1", "ready1", "worker2", "ready2"]); + const specification: ProcessTraceSpecification = { + events: [ + ...terminals.map((frame, index) => ({ + id: `ready${String(index + 1)}`, + source: "terminal_raw" as const, + exact: frame, + cardinality: { kind: "required" as const }, + })), + ...workers.map((sample, index) => ({ + id: `worker${String(index + 1)}`, + source: "process" as const, + exact: sample, + cardinality: { kind: "required" as const }, + })), + ], + language: { + kind: "partial_order", + happens_before: [ + { before: "ready1", after: "worker1" }, + { before: "ready2", after: "worker2" }, + ], + not_before: [], + unordered_groups: [ + { events: ["ready1", "ready2"] }, + { events: ["worker1", "worker2"] }, + { events: ["ready1", "worker2"] }, + { events: ["worker1", "ready2"] }, + ], + prefix: [], + suffix: [], + }, + }; + + expect( + compareProcessCaptures(left, right, { + traceSpecification: specification, + }), + ).toMatchObject({ + trace: { verdict: "different" }, + terminal: "changed", + process: "changed", + }); + }); +}); diff --git a/src/domain/process/processTraceEvaluation.ts b/src/domain/process/processTraceEvaluation.ts index 7c232a480..4cf42d8b7 100644 --- a/src/domain/process/processTraceEvaluation.ts +++ b/src/domain/process/processTraceEvaluation.ts @@ -248,38 +248,33 @@ const evaluateOrderingConstraints = ( language: PartialOrderLanguage, ): { readonly failure: EvaluatedSide | null; readonly satisfied: string[] } => { const satisfied: string[] = []; + const failures: FailureInput[] = []; for (const edge of language.happens_before) { const before = trace.locationsById.get(edge.before) ?? []; const after = trace.locationsById.get(edge.after) ?? []; const lastBefore = before.at(-1); const firstAfter = after[0]; if (lastBefore === undefined && firstAfter !== undefined) - return { - failure: failure({ - kind: "edge", - message: `${edge.before} is required when ${edge.after} occurs.`, - eventIds: [edge.before, edge.after], - locations: [firstAfter], - rawTrace: trace.rawTrace, - }), - satisfied, - }; - if ( + failures.push({ + kind: "edge", + message: `${edge.before} is required when ${edge.after} occurs.`, + eventIds: [edge.before, edge.after], + locations: [firstAfter], + rawTrace: trace.rawTrace, + }); + else if ( lastBefore !== undefined && firstAfter !== undefined && lastBefore.capture_order >= firstAfter.capture_order ) - return { - failure: failure({ - kind: "edge", - message: `${edge.before} must happen before ${edge.after}.`, - eventIds: [edge.before, edge.after], - locations: [lastBefore, firstAfter], - rawTrace: trace.rawTrace, - }), - satisfied, - }; - satisfied.push(`happens_before:${edge.before}:${edge.after}`); + failures.push({ + kind: "edge", + message: `${edge.before} must happen before ${edge.after}.`, + eventIds: [edge.before, edge.after], + locations: [lastBefore, firstAfter], + rawTrace: trace.rawTrace, + }); + else satisfied.push(`happens_before:${edge.before}:${edge.after}`); } for (const constraint of language.not_before) { const events = trace.locationsById.get(constraint.event) ?? []; @@ -287,34 +282,47 @@ const evaluateOrderingConstraints = ( const firstEvent = events[0]; const firstAnchor = anchors[0]; if (firstEvent !== undefined && firstAnchor === undefined) - return { - failure: failure({ - kind: "edge", - message: `${constraint.anchor} is required when ${constraint.event} occurs.`, - eventIds: [constraint.event, constraint.anchor], - locations: [firstEvent], - rawTrace: trace.rawTrace, - }), - satisfied, - }; - if ( + failures.push({ + kind: "edge", + message: `${constraint.anchor} is required when ${constraint.event} occurs.`, + eventIds: [constraint.event, constraint.anchor], + locations: [firstEvent], + rawTrace: trace.rawTrace, + }); + else if ( firstEvent !== undefined && firstAnchor !== undefined && firstEvent.capture_order < firstAnchor.capture_order ) - return { - failure: failure({ - kind: "edge", - message: `${constraint.event} must not occur before ${constraint.anchor}.`, - eventIds: [constraint.event, constraint.anchor], - locations: [firstEvent, firstAnchor], - rawTrace: trace.rawTrace, - }), - satisfied, - }; - satisfied.push(`not_before:${constraint.event}:${constraint.anchor}`); + failures.push({ + kind: "edge", + message: `${constraint.event} must not occur before ${constraint.anchor}.`, + eventIds: [constraint.event, constraint.anchor], + locations: [firstEvent, firstAnchor], + rawTrace: trace.rawTrace, + }); + else satisfied.push(`not_before:${constraint.event}:${constraint.anchor}`); } - return { failure: null, satisfied }; + if (failures.length === 0) return { failure: null, satisfied }; + const unique = (values: readonly T[], key: (value: T) => string): T[] => [ + ...new Map(values.map((value) => [key(value), value])).values(), + ]; + return { + failure: failure({ + kind: "edge", + message: failures.map(({ message }) => message).join(" "), + eventIds: unique( + failures.flatMap(({ eventIds }) => eventIds), + String, + ), + locations: unique( + failures.flatMap(({ locations }) => locations), + ({ collection, index }) => `${collection}:${String(index)}`, + ), + rawTrace: trace.rawTrace, + }), + satisfied, + }; }; const boundaryFailure = ( diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 3f83867bf..40cba4288 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { chmod, writeFile } from "node:fs/promises"; +import { chmod, rm, writeFile } from "node:fs/promises"; import { basename, dirname, extname, join } from "node:path"; import { promisify } from "node:util"; @@ -213,6 +213,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { action, input.loadedPath, ]; + let launched = false; try { if (signal?.aborted === true) return err(new HopperCancelledError()); const display = await this.#privateDisplay(signal); @@ -246,6 +247,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { signal, }); if (started === null) return err(new HopperCancelledError()); + launched = true; const cleanup = () => cleanupOwnedProcessGroup(started.ownership); const ownedLauncher = { ...(this.options.preparedImage === undefined @@ -283,6 +285,12 @@ export class HopperApplicationLauncher implements BridgeLauncher { }); } catch (cause: unknown) { return err(hopperLaunchFailure(cause, signal)); + } finally { + // Remove the token-bearing bootstrap file on every non-success path. + // Only the launched process needs the file; all error returns, early + // cancellations, and thrown exceptions must not leave it on disk. + if (!launched) + await rm(bootstrapPath, { force: true }).catch(() => undefined); } } @@ -347,6 +355,8 @@ export class HopperApplicationLauncher implements BridgeLauncher { await chmod(bootstrapPath, 0o600); return ok(bootstrapPath); } catch (cause: unknown) { + // Clean up any partially written bootstrap file that may contain the token. + await rm(bootstrapPath, { force: true }).catch(() => undefined); return err(new HopperStartError({ cause })); } } diff --git a/src/hopper/HopperCleanup.test.ts b/src/hopper/HopperCleanup.test.ts index 8e078b13d..b61009ca3 100644 --- a/src/hopper/HopperCleanup.test.ts +++ b/src/hopper/HopperCleanup.test.ts @@ -7,11 +7,15 @@ import { EventEmitter } from "node:events"; import { Socket } from "node:net"; import pino from "pino"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { ok } from "../domain/result.js"; import type { BridgeLaunch } from "./BridgeLauncher.js"; -import { cleanupHopperSession } from "./HopperCleanup.js"; +import { + cleanupHopperSession, + confirmExternalApplicationExit, + type HopperOwnedResources, +} from "./HopperCleanup.js"; const loggerHarness = () => { const logs: unknown[] = []; @@ -47,6 +51,58 @@ const processCleanupLaunch = (): BridgeLaunch => ({ const failure = (message: string) => Object.assign(new TypeError(message), { code: "ECONNRESET" }); +const externalApplicationResources = (): HopperOwnedResources => ({ + launch: { + process: Object.assign(new EventEmitter(), { + stdout: null, + stderr: null, + exitCode: null, + signalCode: null, + kill: () => true, + }), + ownsProcessLifetime: false, + providerLifetime: "external-application", + shutdownMode: "bridge-request", + }, + processSupervisor: undefined, + runtimeRoot: undefined, + shutdownConfirmed: false, + externalApplication: { + pid: 123, + identity: { state: "readable", identity: "captured-start" }, + }, +}); + +describe("external Hopper process exit confirmation", () => { + it("confirms only the captured GUI process exit and releases its transport", async () => { + const resources = externalApplicationResources(); + const released: unknown[] = []; + const observeExit = vi.fn().mockResolvedValue("exited"); + await confirmExternalApplicationExit( + resources, + undefined, + (socket) => released.push(socket), + observeExit, + ); + expect(observeExit).toHaveBeenCalledWith(123, "captured-start"); + expect(resources.shutdownConfirmed).toBe(true); + expect(released).toEqual([undefined]); + }); + + it("retains cleanup when process identity cannot be verified", async () => { + const resources = externalApplicationResources(); + const releaseTransport = vi.fn(); + await confirmExternalApplicationExit( + resources, + undefined, + releaseTransport, + async () => "unverified", + ); + expect(resources.shutdownConfirmed).toBe(false); + expect(releaseTransport).not.toHaveBeenCalled(); + }); +}); + describe("Hopper shutdown rejection diagnostics", () => { it("retains the primary shutdown operation and rejection cause", async () => { const { logger, logs } = loggerHarness(); @@ -57,6 +113,7 @@ describe("Hopper shutdown rejection diagnostics", () => { processSupervisor: undefined, runtimeRoot: undefined, shutdownConfirmed: false, + externalApplication: undefined, }, activeRequest: null, retainDocument: false, @@ -96,6 +153,7 @@ describe("Hopper shutdown rejection diagnostics", () => { processSupervisor: undefined, runtimeRoot: undefined, shutdownConfirmed: false, + externalApplication: undefined, }, activeRequest: null, retainDocument: false, @@ -152,6 +210,7 @@ it("keeps a prepared backing image when native document closure cannot be confir runtimeRoot, processSupervisor: undefined, shutdownConfirmed: false, + externalApplication: undefined, }, activeRequest: null, retainDocument: true, diff --git a/src/hopper/HopperCleanup.ts b/src/hopper/HopperCleanup.ts index eb155a9fb..ac4cce659 100644 --- a/src/hopper/HopperCleanup.ts +++ b/src/hopper/HopperCleanup.ts @@ -10,8 +10,12 @@ import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; import type { Logger } from "pino"; import type { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; -import type { ProcessCleanupResult } from "../process/ProcessOwnership.js"; +import type { + ProcessCleanupResult, + ProcessIdentityObservation, +} from "../process/ProcessOwnership.js"; import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import { observeProcessStartIdentityExit } from "../process/ProcessOwnershipObservation.js"; import type { BridgeLaunch } from "./BridgeLauncher.js"; import { createOwnedHopperShutdownDiagnostic, @@ -42,6 +46,12 @@ export interface HopperOwnedResources { processSupervisor: ProviderProcessSupervisor | undefined; runtimeRoot: PrivateRuntimeRoot | undefined; shutdownConfirmed: boolean; + externalApplication: + | { + readonly pid: number; + readonly identity: ProcessIdentityObservation; + } + | undefined; } export interface HopperCleanupInput { @@ -68,6 +78,11 @@ export const cleanupHopperSession = async ( cleanupResult: undefined, processStopped: input.resources.processSupervisor === undefined, }; + await confirmExternalApplicationExit( + input.resources, + input.socket, + input.releaseTransport, + ); await report( input.progress, 0, @@ -119,6 +134,28 @@ export const cleanupHopperSession = async ( return cleanupOutcome(input, state); }; +export const confirmExternalApplicationExit = async ( + resources: HopperOwnedResources, + socket: Socket | undefined, + releaseTransport: (socket: Socket | undefined) => void, + observeExit: typeof observeProcessStartIdentityExit = observeProcessStartIdentityExit, +): Promise => { + const application = resources.externalApplication; + if ( + resources.shutdownConfirmed || + resources.launch?.providerLifetime !== "external-application" || + application?.identity.state !== "readable" + ) + return; + if ( + (await observeExit(application.pid, application.identity.identity)) === + "exited" + ) { + resources.shutdownConfirmed = true; + releaseTransport(socket); + } +}; + const requestShutdown = async ( input: HopperCleanupInput, state: CleanupState, diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 2943839f4..c81a2f2fa 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -44,11 +44,13 @@ import { type ProviderProcessDiagnostic, ProviderProcessSupervisor, } from "../process/ProviderProcess.js"; +import { observeProcessStartIdentity } from "../process/ProcessOwnershipObservation.js"; import type { BridgeLaunch, BridgeLauncher } from "./BridgeLauncher.js"; import { hopperOperationStage } from "./HopperOperationStage.js"; import type { HopperDiagnostic } from "./HopperDiagnostics.js"; import { cleanupHopperSession, + confirmExternalApplicationExit, type HopperOwnedResources, } from "./HopperCleanup.js"; import { @@ -113,6 +115,7 @@ export class HopperClient { processSupervisor: undefined, runtimeRoot: undefined, shutdownConfirmed: false, + externalApplication: undefined, }; #token: string | undefined; readonly #lineage = new ProviderRunLineage(); @@ -434,6 +437,27 @@ export class HopperClient { } const parsed = parseHopperServerInfo(health.value, runId); if (!parsed.ok) return this.#startupFailure(parsed.error); + if (this.#resources.launch?.providerLifetime === "external-application") { + let identity; + try { + identity = await observeProcessStartIdentity(parsed.value.processId); + } catch (cause: unknown) { + identity = { + state: "unavailable" as const, + reason: + cause instanceof Error + ? cause.message + : "process identity observation failed", + }; + } + this.#resources.externalApplication = { + pid: parsed.value.processId, + identity: identity ?? { + state: "unavailable", + reason: "process identity was not returned while Hopper was running", + }, + }; + } await this.#lineage.observe(this.#resources.launch); return parsed; } @@ -535,6 +559,9 @@ export class HopperClient { // best-effort cleanup: a rejected startup must not mask close/cleanup; // #cleanup reports the authoritative close result. await starting?.catch(() => undefined); + await confirmExternalApplicationExit(this.#resources, this.#socket, () => + this.#releaseTransport(), + ); await this.#reconnectForCleanup(); return await this.#cleanup(options); } finally { diff --git a/src/hopper/HopperSessionValues.ts b/src/hopper/HopperSessionValues.ts index 86d15b135..abee06e04 100644 --- a/src/hopper/HopperSessionValues.ts +++ b/src/hopper/HopperSessionValues.ts @@ -9,6 +9,7 @@ export const HOPPER_SHUTDOWN_TIMEOUT_MS = 30_000; export interface HopperServerInfo { readonly name: string; readonly version: string; + readonly processId: number; } /** Validate the bridge identity and exact per-launch run identifier. */ @@ -22,9 +23,16 @@ export const parseHopperServerInfo = ( !Array.isArray(value) && value.name === "REA Hopper bridge" && typeof value.version === "string" && + typeof value.process_id === "number" && + Number.isSafeInteger(value.process_id) && + value.process_id > 0 && value.run_id === expectedRunId ) - return ok({ name: value.name, version: value.version }); + return ok({ + name: value.name, + version: value.version, + processId: value.process_id, + }); return err(new HopperProtocolError("Hopper bridge health result is invalid")); }; diff --git a/src/hopper/HopperTargetLease.test.ts b/src/hopper/HopperTargetLease.test.ts index 2c72909fa..a0c8e0296 100644 --- a/src/hopper/HopperTargetLease.test.ts +++ b/src/hopper/HopperTargetLease.test.ts @@ -1,6 +1,7 @@ import { execFile } from "node:child_process"; import { once } from "node:events"; import { + chmod, lstat, mkdir, mkdtemp, @@ -31,6 +32,84 @@ afterEach(async () => { ); }); +describe.skipIf(process.platform === "win32")( + "Hopper lease directory admission", + () => { + it("secures an existing owned directory without replacing it", async () => { + const directory = await temporaryDirectory(); + await chmod(directory, 0o755); + const before = await lstat(directory); + const acquired = await acquireLinuxHopperApplicationLease({ + directory, + runId: "permission-hardening", + }); + if (!acquired.acquired) + throw new Error("Expected the owned directory lease"); + try { + const after = await lstat(directory); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.mode & 0o777).toBe(0o700); + } finally { + await acquired.lease.release(); + } + }); + + it("rejects a directory symlink without changing its target", async () => { + const directory = await temporaryDirectory(); + const realDirectory = join(directory, "actual"); + const alias = join(directory, "selected"); + await mkdir(realDirectory); + await chmod(realDirectory, 0o755); + await symlink(realDirectory, alias); + await expect( + acquireLinuxHopperApplicationLease({ + directory: alias, + runId: "alias", + }), + ).rejects.toMatchObject({ _tag: "HopperStartError" }); + expect(await readdir(realDirectory)).toEqual([]); + expect((await lstat(realDirectory)).mode & 0o777).toBe(0o755); + expect((await lstat(alias)).isSymbolicLink()).toBe(true); + }); + + it.skipIf(process.getuid?.() === 0)( + "explains unreadable directory recovery without changing unverified permissions", + async () => { + const directory = await temporaryDirectory(); + await chmod(directory, 0o333); + try { + await expect( + acquireLinuxHopperApplicationLease({ + directory, + runId: "unreadable", + }), + ).rejects.toMatchObject({ + _tag: "HopperStartError", + cause: expect.objectContaining({ code: "EACCES" }), + userMessage: expect.stringContaining( + "restore owner-only read, write, and search permissions (0700)", + ), + }); + expect((await lstat(directory)).mode & 0o777).toBe(0o333); + // This fixture owns the directory. Apply the documented recovery, + // then exercise the same production acquisition again. + await chmod(directory, 0o700); + const recovered = await acquireLinuxHopperApplicationLease({ + directory, + runId: "recovered-access", + }); + if (!recovered.acquired) throw new Error("Expected recovered access"); + await recovered.lease.release(); + expect(await readdir(directory)).toEqual([]); + } finally { + await chmod(directory, 0o700); + } + }, + ); + }, +); + describe("Hopper target leases", () => { it("reports the owning REA session for a duplicate target and profile", async () => { const directory = await temporaryDirectory(); diff --git a/src/hopper/HopperTargetLease.ts b/src/hopper/HopperTargetLease.ts index 150700267..53d09aba1 100644 --- a/src/hopper/HopperTargetLease.ts +++ b/src/hopper/HopperTargetLease.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; -import { chmod, lstat, mkdir, realpath, rm, rmdir } from "node:fs/promises"; +import { chmod, mkdir, open, realpath, rm, rmdir } from "node:fs/promises"; +import { constants as fsConstants } from "node:fs"; import { createConnection, createServer, @@ -164,27 +165,69 @@ const tryAcquireLease = async ( }; const ensureLeaseDirectory = async (directory: string): Promise => { - let directoryStat: Awaited>; try { await mkdir(directory, { recursive: true, mode: 0o700 }); - directoryStat = await lstat(directory); } catch (cause: unknown) { throw new HopperStartError({ cause, - userMessage: - "REA could not create or inspect its private Hopper lease directory.", + userMessage: "REA could not create its private Hopper lease directory.", }); } - if ( - directoryStat.isSymbolicLink() || - directoryStat.isDirectory() === false || - (process.getuid !== undefined && directoryStat.uid !== process.getuid()) - ) + // Use O_DIRECTORY|O_NOFOLLOW to open the directory without following symlinks, + // then inspect and secure that descriptor. Later socket operations still use + // the path and are not made atomic by this directory validation. + let handle; + try { + handle = await open( + directory, + fsConstants.O_RDONLY | + (fsConstants.O_DIRECTORY ?? 0) | + (fsConstants.O_NOFOLLOW ?? 0), + ); + const directoryStat = await handle.stat(); + if ( + directoryStat.isSymbolicLink() || + directoryStat.isDirectory() === false || + (process.getuid !== undefined && directoryStat.uid !== process.getuid()) + ) + throw new HopperStartError({ + userMessage: + "REA's Hopper lease directory is not a private directory owned by this user.", + }); + // POSIX permission hardening: chmod on the already-opened descriptor avoids + // a second TOCTOU window where the directory could be swapped between check + // and permission fix. Windows DACL-based security is handled separately. + const posixPermissionsApply = process.getuid !== undefined; + if (posixPermissionsApply && (directoryStat.mode & 0o077) !== 0) { + await handle.chmod(0o700); + // Re-verify permissions on the same opened directory. + const afterChmod = await handle.stat(); + if ( + afterChmod.isSymbolicLink() || + afterChmod.isDirectory() === false || + (afterChmod.mode & 0o077) !== 0 + ) + throw new HopperStartError({ + userMessage: + "REA's Hopper lease directory permissions could not be secured.", + }); + } + } catch (cause: unknown) { + if (cause instanceof HopperStartError) throw cause; + const accessDenied = + process.getuid !== undefined && + cause instanceof Error && + "code" in cause && + cause.code === "EACCES"; throw new HopperStartError({ - userMessage: - "REA's Hopper lease directory is not a private directory owned by this user.", + cause, + userMessage: accessDenied + ? `REA could not access its Hopper lease directory at ${JSON.stringify(directory)}. Check its ownership and parent-directory access. For a directory you own, restore owner-only read, write, and search permissions (0700), then retry.` + : "REA could not inspect its private Hopper lease directory.", }); - if ((directoryStat.mode & 0o077) !== 0) await chmod(directory, 0o700); + } finally { + await handle?.close(); + } }; const bind = (server: Server, socketPath: string): Promise => diff --git a/src/process/ArtifactAnalysisResources.test.ts b/src/process/ArtifactAnalysisResources.test.ts new file mode 100644 index 000000000..84ce0e6d7 --- /dev/null +++ b/src/process/ArtifactAnalysisResources.test.ts @@ -0,0 +1,85 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { expect, it } from "vitest"; +import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { err } from "../domain/result.js"; +import { ArtifactAnalysisResources } from "./ArtifactAnalysisResources.js"; +import { readLinuxResourceLimits } from "./LinuxResourceLimits.js"; +import { ProviderProcessSupervisor } from "./ProviderProcess.js"; + +it("projects the specific reason and member evidence for retained cleanup", async () => { + const producer = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null, + signalCode: null, + kill: () => true, + }); + const owner = new ProviderProcessSupervisor({ + process: producer, + ownsProcessLifetime: true, + ownership: { + runId: "rea-tshark-run-1", + leaderPid: 4242, + processGroupId: 4242, + }, + cleanup: async () => ({ + cleaned: false as const, + reason: "process ownership token could not be read for 1 live process", + failures: [ + { + pid: 101, + reason: "environment-unreadable" as const, + diagnostic: "sysctl_errno_22", + }, + ], + unverified: [ + { pid: 900, diagnostic: "platform_binary_environment_withheld" }, + ], + }), + }); + const resources = new ArtifactAnalysisResources( + "fixture-provider", + "fixture-operation", + readStableArtifact, + readLinuxResourceLimits, + { message: "busy", details: {} }, + ); + resources.retain({ path: "runtime-root", close: async () => {} }, owner); + + const result = await resources.finish( + err(new ProviderAdapterError("fixture-provider", "fixture-operation")), + () => { + throw new Error("unexpected success observation"); + }, + ); + owner.dispose(); + + if (result.ok) throw new Error("expected cleanup failure"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { + resources: [ + "rea-tshark-run-1", + "runtime-root", + "pid:4242", + "process-group:4242", + ], + diagnostics: { + reason: "process ownership token could not be read for 1 live process", + failures: [ + { + pid: 101, + reason: "environment-unreadable", + diagnostic: "sysctl_errno_22", + }, + ], + unverified: [ + { pid: 900, diagnostic: "platform_binary_environment_withheld" }, + ], + }, + }, + }); +}); diff --git a/src/process/ArtifactAnalysisResources.ts b/src/process/ArtifactAnalysisResources.ts index ae3d9f803..ff7b63048 100644 --- a/src/process/ArtifactAnalysisResources.ts +++ b/src/process/ArtifactAnalysisResources.ts @@ -24,7 +24,7 @@ import { } from "./LinuxResourceLimits.js"; import type { PrivateRuntimeRoot } from "./PrivateRuntimeRoot.js"; import type { ProviderProcessSupervisor } from "./ProviderProcess.js"; -import { OwnedCommandFailure } from "./OwnedCommand.js"; +import { OwnedCommandFailure, ownedCommandResources } from "./OwnedCommand.js"; export type ArtifactAnalysisRuntime = Pick< PrivateRuntimeRoot, @@ -129,22 +129,67 @@ export class ArtifactAnalysisResources { return [...this.#pending.keys()].map((root) => root.path); } + #resourcesFor(root: ArtifactAnalysisRuntime): readonly string[] { + const ownership = this.#pending.get(root)?.owner?.launch.ownership; + return ownership === undefined + ? [root.path] + : ownedCommandResources(ownership, root.path); + } + + get #pendingResources(): readonly string[] { + return [...this.#pending.keys()].flatMap((root) => + this.#resourcesFor(root), + ); + } + async release(): Promise { for (const [root, retained] of this.#pending) { try { const { owner } = retained; if (owner !== undefined) { const stopped = await owner.stop(); - if (stopped.status === "incomplete") throw new Error(stopped.reason); + if (stopped.status === "incomplete") { + const cleanup = stopped.cleanupResult; + throw new ProviderCleanupError( + this.providerId, + this.#resourcesFor(root), + { + reason: stopped.reason, + ...(cleanup?.failures === undefined + ? {} + : { + failures: cleanup.failures.map((failure) => ({ + ...failure, + })), + }), + ...(cleanup?.unverified === undefined + ? {} + : { + unverified: cleanup.unverified.map((process) => ({ + ...process, + })), + }), + }, + { operation: retained.operation }, + ); + } this.#pending.set(root, { ...retained, owner: undefined }); } await root.close(); this.#pending.delete(root); } catch (cause: unknown) { + const diagnostics = + cause instanceof ProviderCleanupError ? cause.diagnostics : undefined; throw new ProviderCleanupError( this.providerId, - [root.path], - { reason: String(cause) }, + this.#resourcesFor(root), + { + ...diagnostics, + reason: + cause instanceof ProviderCleanupError + ? String(diagnostics?.reason ?? cause.message) + : String(cause), + }, { operation: retained.operation, cause }, ); } @@ -236,12 +281,18 @@ export class ArtifactAnalysisResources { const partialObservation = result.ok ? observation(result.value) : result.error.partialObservation; + const cleanupDiagnostics = + cause instanceof ProviderCleanupError ? cause.diagnostics : undefined; return err( new ProviderCleanupError( this.providerId, - this.pendingPaths, + this.#pendingResources, { - reason: String(cause), + ...cleanupDiagnostics, + reason: + cause instanceof ProviderCleanupError + ? String(cleanupDiagnostics?.reason ?? cause.message) + : String(cause), previous_error: result.ok ? null : projectAnalysisError(result.error), diff --git a/src/process/ExecFileBoundedOutput.test.ts b/src/process/ExecFileBoundedOutput.test.ts new file mode 100644 index 000000000..c055bc002 --- /dev/null +++ b/src/process/ExecFileBoundedOutput.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; + +import { execFileBoundedOutput } from "./ExecFileBoundedOutput.js"; + +describe("execFileBoundedOutput", () => { + it("lets a command finish after its output exceeds the capture limit", async () => { + const result = await execFileBoundedOutput( + process.execPath, + [ + "-e", + "process.stdout.write('x'.repeat(70000)); process.stderr.write('failure detail');", + ], + { env: process.env, limit: 64 * 1024 }, + ); + expect(result.exitCode).toBe(0); + expect(Buffer.byteLength(result.stdout)).toBeLessThan(64 * 1024 + 50); + expect(result.stdout).toContain("[output omitted]"); + expect(result.stderr).toBe("failure detail"); + expect(result.outputTruncated).toBe(true); + }); + + it("retains a failing command's status and stderr after verbose stdout", async () => { + const result = await execFileBoundedOutput( + process.execPath, + [ + "-e", + "process.stdout.write('x'.repeat(70000)); process.stderr.write('package conflict'); process.exitCode = 1;", + ], + { env: process.env, limit: 64 * 1024 }, + ); + expect(result.exitCode).toBe(1); + expect(result.stderr).toBe("package conflict"); + expect(result.outputTruncated).toBe(true); + }); + + it("retains a final error after a long prefix in the same stream", async () => { + const result = await execFileBoundedOutput( + process.execPath, + [ + "-e", + "process.stderr.write('warning '.repeat(10000)); process.stderr.write('error: final conflict'); process.exitCode = 1;", + ], + { env: process.env, limit: 64 * 1024 }, + ); + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("[output omitted]"); + expect(result.stderr).toContain("error: final conflict"); + expect(result.outputTruncated).toBe(true); + }); +}); diff --git a/src/process/ExecFileBoundedOutput.ts b/src/process/ExecFileBoundedOutput.ts new file mode 100644 index 000000000..fad51e2bc --- /dev/null +++ b/src/process/ExecFileBoundedOutput.ts @@ -0,0 +1,73 @@ +import { spawn } from "node:child_process"; + +export interface BoundedCommandOutput { + readonly exitCode: string | number | null; + readonly stdout: string; + readonly stderr: string; + readonly outputTruncated: boolean; +} + +/** Drain both streams while retaining bounded head and tail evidence. */ +export const execFileBoundedOutput = ( + executable: string, + args: readonly string[], + options: { + readonly env: Readonly; + readonly limit: number; + }, +): Promise => + new Promise((resolve) => { + const child = spawn(executable, [...args], { + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); + const headLimit = Math.ceil(options.limit / 2); + const tailLimit = options.limit - headLimit; + const stdout = { head: Buffer.alloc(0), tail: Buffer.alloc(0), total: 0 }; + const stderr = { head: Buffer.alloc(0), tail: Buffer.alloc(0), total: 0 }; + let outputTruncated = false; + const capture = + (stream: typeof stdout) => + (chunk: Buffer): void => { + stream.total += chunk.length; + const headCount = Math.min( + chunk.length, + Math.max(0, headLimit - stream.head.length), + ); + if (headCount > 0) + stream.head = Buffer.concat([ + stream.head, + chunk.subarray(0, headCount), + ]); + const rest = chunk.subarray(headCount); + if (rest.length > 0 && tailLimit > 0) + stream.tail = Buffer.concat([stream.tail, rest]).subarray(-tailLimit); + if (stream.total > options.limit) outputTruncated = true; + }; + child.stdout.on("data", capture(stdout)); + child.stderr.on("data", capture(stderr)); + let spawnError: NodeJS.ErrnoException | undefined; + child.on("error", (error: NodeJS.ErrnoException) => { + spawnError = error; + }); + child.on("close", (code, signal) => { + resolve({ + exitCode: spawnError?.code ?? code ?? signal, + stdout: retainedOutput(stdout, options.limit), + stderr: retainedOutput(stderr, options.limit), + outputTruncated, + }); + }); + }); + +const retainedOutput = ( + stream: { + readonly head: Buffer; + readonly tail: Buffer; + readonly total: number; + }, + limit: number, +): string => + stream.total > limit + ? `${stream.head.toString("utf8")} … [output omitted] … ${stream.tail.toString("utf8")}` + : Buffer.concat([stream.head, stream.tail]).toString("utf8"); diff --git a/src/process/ProcessOwnership.test.ts b/src/process/ProcessOwnership.test.ts index de233698c..fafa39950 100644 --- a/src/process/ProcessOwnership.test.ts +++ b/src/process/ProcessOwnership.test.ts @@ -209,6 +209,185 @@ describe("opaque process neighbors during cleanup", () => { }); describe("related sweep candidates during cleanup", () => { + it("counts each unreadable live member once", async () => { + const processes = [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + command: "capture", + }, + { + pid: 101, + parentPid: 100, + processGroupId: 100, + state: "S", + command: "child-a", + }, + { + pid: 102, + parentPid: 100, + processGroupId: 100, + state: "S", + command: "child-b", + }, + ]; + const signalGroup = vi.fn(); + const adapter: ProcessOwnershipHost = { + listProcesses: async () => processes, + environment: async () => ({}), + runTokens: async (members) => + new Map( + members.map(({ pid }) => [ + pid, + pid === 100 + ? { state: "readable" as const, runId: ownership.runId } + : { + state: "unavailable" as const, + reason: "environment_unavailable", + }, + ]), + ), + signalGroup, + }; + const result = await cleanupOwnedProcessGroup( + { ...ownership, sweepTokenOwnedProcesses: true, captureBaseline: [] }, + adapter, + ); + if (result.cleaned) throw new Error("expected cleanup failure"); + expect(signalGroup).not.toHaveBeenCalled(); + expect(result.failures?.map(({ pid }) => pid).sort()).toEqual([101, 102]); + expect(result.reason).toContain("2 live process(es)"); + }); + + it("upgrades unreadable evidence when later group validation finds a foreign token", async () => { + const processes = [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + command: "capture", + }, + { + pid: 200, + parentPid: 100, + processGroupId: 200, + state: "S", + command: "detached-child", + }, + ]; + const signalGroup = vi.fn(); + let childReads = 0; + const adapter: ProcessOwnershipHost = { + listProcesses: async () => processes, + processIdentities: async (members) => + new Map( + members.map(({ pid }) => [ + pid, + { state: "readable" as const, identity: `start-${String(pid)}` }, + ]), + ), + environment: async () => ({}), + runTokens: async (members) => + new Map( + members.map(({ pid }) => [ + pid, + pid === 100 + ? { state: "readable" as const, runId: ownership.runId } + : ++childReads <= 4 + ? { + state: "unavailable" as const, + reason: "environment_unavailable", + } + : { state: "readable" as const, runId: "foreign-token" }, + ]), + ), + signalGroup, + }; + const result = await cleanupOwnedProcessGroup( + { ...ownership, sweepTokenOwnedProcesses: true, captureBaseline: [] }, + adapter, + ); + expect(result).toMatchObject({ + cleaned: false, + reason: "process ownership token mismatch for PID(s): 200", + failures: [{ pid: 200, reason: "run-token-mismatch" }], + }); + expect(signalGroup).not.toHaveBeenCalledWith(200, "SIGKILL"); + }); +}); + +describe("withheld process evidence during cleanup", () => { + it.each(["revalidation", "signal-error"] as const)( + "retains withheld-process evidence after %s failure", + async (mode) => { + const processes = [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + command: "fixture", + }, + { + pid: 900, + parentPid: 1, + processGroupId: 900, + state: "S", + command: "opaque", + }, + ]; + let rootReads = 0; + const signalGroup = vi.fn(() => { + if (mode === "signal-error") + throw Object.assign(new Error("signal denied"), { code: "EACCES" }); + }); + const adapter: ProcessOwnershipHost = { + listProcesses: async () => processes, + environment: async () => ({}), + runTokens: async (members) => + new Map( + members.map(({ pid }) => [ + pid, + pid === 900 + ? { + state: "unavailable" as const, + reason: "platform_binary_environment_withheld", + } + : { + state: "readable" as const, + runId: + mode === "revalidation" && ++rootReads > 2 + ? "other-run" + : "run-token", + }, + ]), + ), + signalGroup, + }; + + const result = await cleanupOwnedProcessGroup( + { ...ownership, sweepTokenOwnedProcesses: true, captureBaseline: [] }, + adapter, + ); + + expect(result.unverified).toEqual([ + { pid: 900, diagnostic: "platform_binary_environment_withheld" }, + ]); + if (result.cleaned) throw new Error("expected cleanup failure"); + if (mode === "revalidation") + expect(result.reason).toBe( + "process ownership token mismatch for PID(s): 100", + ); + else + expect(result.reason).toBe( + "owned process group signal failed: signal denied", + ); + }, + ); + it.each([ "environment_unavailable", "platform_binary_environment_withheld", diff --git a/src/process/ProcessOwnership.ts b/src/process/ProcessOwnership.ts index 399b9cd06..91d681f79 100644 --- a/src/process/ProcessOwnership.ts +++ b/src/process/ProcessOwnership.ts @@ -227,6 +227,15 @@ export const readProcessRunId = async ( ): Promise => (await host.environment(pid, signal)).REA_PROCESS_RUN_ID; +/** Keep unrelated-process uncertainty attached to every outcome of this cleanup attempt. */ +const withUnverifiedObservations = ( + result: ProcessCleanupFailure, + unverified: readonly OwnershipSweepUnverifiedProcess[], +): ProcessCleanupFailure => + unverified.length === 0 || (result.unverified?.length ?? 0) > 0 + ? result + : { ...result, unverified }; + /** Token-validate every rooted POSIX process group before signaling any. */ export const cleanupOwnedProcessGroup = async ( ownership: OwnedProcessGroup, @@ -252,7 +261,16 @@ export const cleanupOwnedProcessGroup = async ( host, plan.tokenOwnedIdentities, ); - if ("cleaned" in revalidation) return revalidation; + if ("cleaned" in revalidation) { + if (!revalidation.cleaned) + return withUnverifiedObservations(revalidation, plan.unverified); + return { + ...revalidation, + ...(revalidation.unverified?.length || plan.unverified.length === 0 + ? {} + : { unverified: plan.unverified }), + }; + } if (revalidation.empty) continue; try { host.signalGroup(processGroupId, "SIGKILL"); @@ -261,10 +279,13 @@ export const cleanupOwnedProcessGroup = async ( const code = cause instanceof Error && "code" in cause ? cause.code : undefined; if (code !== "ESRCH") - return { - cleaned: false, - reason: `owned process group signal failed: ${errorMessage(cause)}`, - }; + return withUnverifiedObservations( + { + cleaned: false, + reason: `owned process group signal failed: ${errorMessage(cause)}`, + }, + plan.unverified, + ); } } if (plan.unresolved.length > 0) @@ -507,7 +528,8 @@ const createOwnedCleanupPlan = async ( const scanFailures = tokenOwned.failures.filter(({ pid }) => members.some((member) => member.pid === pid), ); - if (failures.length > 0) unresolved.push(...failures); + for (const failure of failures) + mergeProcessOwnershipFailure(unresolved, failure); if ( failures.length === 0 && scanFailures.length === 0 && @@ -533,6 +555,36 @@ const createOwnedCleanupPlan = async ( }; }; +/** Keep one current, most useful ownership diagnosis for each live PID. */ +const mergeProcessOwnershipFailure = ( + failures: ProcessOwnershipValidationFailure[], + next: ProcessOwnershipValidationFailure, +): void => { + const index = failures.findIndex(({ pid }) => pid === next.pid); + if (index < 0) { + failures.push(next); + return; + } + const previous = failures[index]; + if ( + previous !== undefined && + ownershipFailurePriority(next.reason) >= + ownershipFailurePriority(previous.reason) + ) + failures[index] = next; +}; + +const ownershipFailurePriority = ( + reason: ProcessOwnershipValidationFailure["reason"], +): number => + reason === "run-token-mismatch" + ? 3 + : reason === "process-identity-unavailable" + ? 2 + : reason === "environment-unreadable" + ? 1 + : 0; + const revalidateOwnedProcessGroup = async ( processGroupId: number, ownership: OwnedProcessGroup, @@ -768,9 +820,27 @@ const recheckOwnershipFailures = async ( if (!failures.some(({ reason }) => reason !== "run-token-mismatch")) return failures; try { - const live = new Set( - liveProcesses(await host.listProcesses()).map(({ pid }) => pid), - ); + let processes = await host.listProcesses(); + const isDarwin = (host.platform ?? process.platform) === "darwin"; + for ( + let attempt = 0; + isDarwin && + attempt < TOKEN_READ_RETRY_ATTEMPTS && + processes.some( + ({ pid, state }) => + failures.some( + (failure) => + failure.pid === pid && failure.reason !== "run-token-mismatch", + ) && state.slice(1).includes("E"), + ); + attempt += 1 + ) { + await new Promise((resolve) => + setTimeout(resolve, TOKEN_READ_RETRY_DELAY_MS), + ); + processes = await host.listProcesses(); + } + const live = new Set(liveProcesses(processes).map(({ pid }) => pid)); return failures.filter( ({ pid, reason }) => reason === "run-token-mismatch" || live.has(pid), ); diff --git a/src/process/ProcessOwnership.validation.test.ts b/src/process/ProcessOwnership.validation.test.ts index c718f4b93..ed6179a77 100644 --- a/src/process/ProcessOwnership.validation.test.ts +++ b/src/process/ProcessOwnership.validation.test.ts @@ -21,6 +21,40 @@ const launcher: ProcessTableEntry = { }; describe("owned process-group cleanup validation: ownership and lineage", () => { + it("settles unreadable observations for Darwin processes exiting in the kernel", async () => { + let snapshots = 0; + const child = { + pid: 101, + parentPid: 100, + processGroupId: 100, + state: "?E", + command: "fixture-child", + }; + const adapter: ProcessOwnershipHost = { + platform: "darwin", + listProcesses: () => { + snapshots += 1; + return Promise.resolve( + snapshots === 1 + ? [launcher, child] + : snapshots === 2 + ? [launcher, child] + : [launcher, { ...child, state: "Z" }], + ); + }, + environment: (pid) => + pid === 100 + ? Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }) + : Promise.reject(new Error("sysctl_errno_22")), + signalGroup: vi.fn(), + }; + + await expect(cleanupOwnedProcessGroup(ownership, adapter)).resolves.toEqual( + { cleaned: true, signaled: true }, + ); + expect(snapshots).toBeGreaterThanOrEqual(3); + }); + it("fails closed when a descendant in another process group lacks the token", async () => { const adapter: ProcessOwnershipHost = { listProcesses: () => diff --git a/src/process/ProcessOwnershipObservation.ts b/src/process/ProcessOwnershipObservation.ts index 5747cded8..3b4311c94 100644 --- a/src/process/ProcessOwnershipObservation.ts +++ b/src/process/ProcessOwnershipObservation.ts @@ -510,6 +510,22 @@ export const observeProcessStartIdentity = async ( }; }; +/** Confirm whether the process that held an authenticated bridge has exited. */ +export const observeProcessStartIdentityExit = async ( + pid: number, + expectedIdentity: string, + host: ProcessOwnershipHost = systemProcessOwnershipHost, +): Promise<"exited" | "present" | "unverified"> => { + try { + const observed = await observeProcessStartIdentity(pid, host); + if (observed === undefined) return "exited"; + if (observed.state !== "readable") return "unverified"; + return observed.identity === expectedIdentity ? "present" : "exited"; + } catch { + return "unverified"; + } +}; + /** Revalidate a launch-time start identity immediately before signaling a PID. */ export const signalProcessWithStartIdentity = async ( pid: number, diff --git a/src/process/ProcessStartIdentity.test.ts b/src/process/ProcessStartIdentity.test.ts index 20b2862ee..8d2f53261 100644 --- a/src/process/ProcessStartIdentity.test.ts +++ b/src/process/ProcessStartIdentity.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ProcessOwnershipHost } from "./ProcessOwnership.js"; import { observeProcessStartIdentity, + observeProcessStartIdentityExit, signalProcessWithStartIdentity, } from "./ProcessOwnershipObservation.js"; @@ -37,6 +38,26 @@ describe("process start identity lease", () => { }); }); + it("confirms exit or PID replacement without treating unreadable state as exit", async () => { + await expect( + observeProcessStartIdentityExit(42, "start-1", host(undefined)), + ).resolves.toBe("exited"); + await expect( + observeProcessStartIdentityExit(42, "start-1", host("replacement-start")), + ).resolves.toBe("exited"); + await expect( + observeProcessStartIdentityExit(42, "start-1", host("start-1")), + ).resolves.toBe("present"); + const unavailable: ProcessOwnershipHost = { + ...host("start-1"), + processIdentities: async () => + new Map([[42, { state: "unavailable", reason: "reader unavailable" }]]), + }; + await expect( + observeProcessStartIdentityExit(42, "start-1", unavailable), + ).resolves.toBe("unverified"); + }); + it("does not adopt a replacement PID's first identity for an owned launch", async () => { const replacement: ProcessOwnershipHost = { ...host("replacement-start"), diff --git a/src/process/ProviderProcess.ts b/src/process/ProviderProcess.ts index c5e547ac7..17eda48d7 100644 --- a/src/process/ProviderProcess.ts +++ b/src/process/ProviderProcess.ts @@ -66,6 +66,8 @@ export interface ProviderProcessHandle extends Pick< readonly signalCode: NodeJS.Signals | null; /** True once an enforced process memory limit reported an allocation failure. */ readonly memoryLimitReached?: boolean; + /** True if a Windows backpressured stream was destroyed before output drained. */ + readonly outputIncomplete?: boolean; kill(signal?: NodeJS.Signals | number): boolean; unref?(): void; ref?(): void; @@ -97,7 +99,7 @@ export interface ProviderProcessExitObservation { /** Detached diagnostics captured for one supervised provider process. */ export interface ProviderProcessSnapshot { - /** True only when a configured diagnostic budget excluded output bytes. */ + /** True when retained diagnostics omit bytes due to the budget or provider output loss. */ readonly diagnosticTruncated?: boolean; readonly stdout: { readonly text: string; @@ -120,6 +122,8 @@ export interface ProviderProcessSnapshot { readonly signal: NodeJS.Signals | null | undefined; /** Present only when the process tree reached its enforced memory limit. */ readonly memoryLimitReached?: true; + /** Present when a provider stream was destroyed before all bytes were observed. */ + readonly outputIncomplete?: true; } /** Resource-safe events emitted without imposing provider protocol semantics. */ @@ -338,8 +342,11 @@ export class ProviderProcessSupervisor { /** Latest retained output and exit observation, with explicit configured truncation. */ snapshot(): ProviderProcessSnapshot { + const outputIncomplete = this.launch.process.outputIncomplete === true; return { - ...(this.#diagnosticTruncated ? { diagnosticTruncated: true } : {}), + ...(this.#diagnosticTruncated || outputIncomplete + ? { diagnosticTruncated: true } + : {}), stdout: this.#stdout.snapshot(this.#stdoutObservedBytes), stderr: this.#stderr.snapshot(this.#stderrObservedBytes), exitCode: this.#exitObservation?.code, @@ -347,6 +354,7 @@ export class ProviderProcessSupervisor { ...(this.launch.process.memoryLimitReached === true ? { memoryLimitReached: true } : {}), + ...(outputIncomplete ? { outputIncomplete: true } : {}), }; } diff --git a/src/process/ProviderProcessOutput.test.ts b/src/process/ProviderProcessOutput.test.ts index 3ca2621f0..993386e79 100644 --- a/src/process/ProviderProcessOutput.test.ts +++ b/src/process/ProviderProcessOutput.test.ts @@ -122,6 +122,30 @@ it("reports a reached process memory limit only when the handle observed one", ( } }); +it("projects provider output loss as truncated diagnostics without inventing a byte count", () => { + const producer = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null, + signalCode: null, + kill: () => false, + }); + Object.defineProperty(producer, "outputIncomplete", { value: true }); + const supervisor = new ProviderProcessSupervisor({ + process: producer, + ownsProcessLifetime: false, + }); + try { + expect(supervisor.snapshot()).toMatchObject({ + diagnosticTruncated: true, + outputIncomplete: true, + }); + expect(supervisor.snapshot()).not.toHaveProperty("discardedOutputBytes"); + } finally { + supervisor.dispose(); + } +}); + it("unreferences process and stream handles when stop reports incomplete cleanup", async () => { const unrefProcess = vi.fn(); const unrefStdin = vi.fn(); diff --git a/src/process/WindowsAuthority.ts b/src/process/WindowsAuthority.ts index b25a22863..e05080739 100644 --- a/src/process/WindowsAuthority.ts +++ b/src/process/WindowsAuthority.ts @@ -1,4 +1,7 @@ -import { systemWindowsNativeAuthority } from "../windows/WindowsNativeLoader.js"; +import { + systemWindowsNativeAuthority, + type WindowsNativeLoadResult, +} from "../windows/WindowsNativeLoader.js"; /** Windows controls are proven by the packaged native boundary, not POSIX flags. */ export type WindowsNativeCapability = @@ -12,6 +15,15 @@ export type WindowsNativeCapability = readonly reason: string; readonly proof: "not-proven" | "not-applicable"; readonly retryable?: true; + readonly code: + | "unsupported_host" + | "runtime_missing" + | "runtime_unavailable"; + readonly failureStage: Exclude< + WindowsNativeLoadResult, + { available: true } + >["failureStage"]; + readonly remediation: string; }; /** Security controls required before REA may claim Windows isolation. */ @@ -25,64 +37,87 @@ export interface WindowsNativeCapabilitySet { export const WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON = "Windows native controls are unavailable in this runtime; use the Windows x64 REA package with its matching bundled native artifact."; +/** One readiness observation, shared by diagnostics and security controls. */ +export type WindowsNativeReadiness = + | { readonly available: true } + | { + readonly available: false; + readonly reason: string; + readonly failureStage: Exclude< + WindowsNativeLoadResult, + { available: true } + >["failureStage"]; + readonly code: + | "unsupported_host" + | "runtime_missing" + | "runtime_unavailable"; + readonly retryable?: true; + readonly remediation: string; + readonly proof: "not-proven" | "not-applicable"; + }; + +/** Interpret the loader's tagged failure without guessing from OS error text. */ +export const inspectWindowsNativeReadiness = ( + platform: NodeJS.Platform, + observation?: WindowsNativeLoadResult, +): WindowsNativeReadiness => { + if (platform !== "win32") + return { + available: false, + failureStage: "host", + reason: "Windows native authority is not applicable on this host", + proof: "not-applicable", + code: "unsupported_host", + remediation: "Use a Windows x64 host for Windows native controls.", + }; + const loaded = observation ?? systemWindowsNativeAuthority(); + if (loaded.available) return { available: true }; + const failureStage = loaded.failureStage; + return { + available: false, + failureStage, + reason: loaded.reason, + ...(loaded.retryable === true ? { retryable: true as const } : {}), + proof: "not-proven", + code: + failureStage === "host" + ? "unsupported_host" + : failureStage === "artifact" + ? "runtime_missing" + : "runtime_unavailable", + remediation: + failureStage === "host" + ? "Use a Windows x64 host and a supported Node.js runtime with Node-API 8 or newer." + : failureStage === "artifact" + ? "Install the matching Windows x64 REA package with its bundled native addon; check the reported artifact load or integrity failure." + : failureStage === "inspection" + ? "Install the matching Windows x64 REA package; its native inspection must prove every required security control." + : "Inspect the preserved native self-test error. Check that the process's temporary directory is writable local NTFS and that its filesystem, DACL, Job Object, and cleanup operations are permitted. Resolve the reported constraint, then rerun rea doctor; reinstalling Ghidra or Java cannot fix this self-test.", + }; +}; + /** * Describe native Windows controls without inferring them from taskkill, * chmod, symlink creation, or a numeric POSIX open flag. */ export const windowsNativeCapabilities = ( platform: NodeJS.Platform, + readiness: WindowsNativeReadiness = inspectWindowsNativeReadiness(platform), ): WindowsNativeCapabilitySet => { - if (platform !== "win32") { - const notApplicable = { - available: false, - reason: "Windows native authority is not applicable on this host", - proof: "not-applicable", - } as const; - return { - job_object_process_ownership: notApplicable, - private_runtime_dacl: notApplicable, - reparse_safe_path_admission: notApplicable, - }; - } - - if (process.platform === "win32") { - const loaded = systemWindowsNativeAuthority(); - const capability = loaded.available - ? ({ available: true, reason: null, proof: "native-authority" } as const) - : ({ - available: false, - reason: loaded.reason, - proof: "not-proven", - ...(loaded.retryable === true ? { retryable: true as const } : {}), - } as const); - return { - job_object_process_ownership: capability, - private_runtime_dacl: capability, - reparse_safe_path_admission: capability, - }; - } - - const notProven = { - available: false, - reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, - proof: "not-proven", - } as const; + const capability = readiness.available + ? ({ available: true, reason: null, proof: "native-authority" } as const) + : ({ + available: false, + reason: readiness.reason, + proof: readiness.proof, + code: readiness.code, + failureStage: readiness.failureStage, + remediation: readiness.remediation, + ...(readiness.retryable === true ? { retryable: true as const } : {}), + } as const); return { - job_object_process_ownership: notProven, - private_runtime_dacl: notProven, - reparse_safe_path_admission: notProven, + job_object_process_ownership: capability, + private_runtime_dacl: capability, + reparse_safe_path_admission: capability, }; }; - -/** Whether every native control needed for Windows isolation is proven. */ -export const hasWindowsNativeAuthority = (platform: NodeJS.Platform): boolean => - Object.values(windowsNativeCapabilities(platform)).every( - ({ available }) => available, - ); - -/** Preserve the actual loader or operating-system constraint in provider errors. */ -export const windowsNativeAuthorityUnavailableReason = ( - platform: NodeJS.Platform, -): string => - windowsNativeCapabilities(platform).job_object_process_ownership.reason ?? - WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON; diff --git a/src/process/capture/FilesystemCoverage.test.ts b/src/process/capture/FilesystemCoverage.test.ts index baa5f5c94..4d77ebfaa 100644 --- a/src/process/capture/FilesystemCoverage.test.ts +++ b/src/process/capture/FilesystemCoverage.test.ts @@ -100,6 +100,45 @@ it("distinguishes unavailable content hashes from completed path enumeration", a ).toMatchObject({ status: "deleted", before: { sha256: null }, after: null }); }); +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "retains readable entries and marks an unreadable directory subtree unknown", + async () => { + const root = await createTestTempDirectory("rea-fs-unreadable-directory-"); + const visible = join(root, "a-visible.txt"); + const blocked = join(root, "z-blocked"); + await writeFile(visible, "visible"); + await mkdir(blocked); + await chmod(blocked, 0); + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + }); + + try { + const snapshot = await snapshotRoots(scenario); + expect(snapshot.files.map(({ path }) => path)).toContain( + "root_0:a-visible.txt", + ); + expect(snapshot.files.map(({ path }) => path)).toContain( + "root_0:z-blocked", + ); + expect(snapshot.completeRoots).toEqual([]); + expect(snapshot.coverage.enumeration_omissions).toEqual([ + { + path: "root_0:z-blocked", + reason: "directory_unavailable", + system_code: "EACCES", + }, + ]); + expect(snapshot.truncated).toBe(true); + } finally { + await chmod(blocked, 0o700); + await rm(root, { recursive: true, force: true }); + } + }, +); + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( "retains unreadable-file and sibling observations in a complete snapshot", async () => { diff --git a/src/process/capture/FilesystemSnapshot.test.ts b/src/process/capture/FilesystemSnapshot.test.ts index 16799b973..3aeab0391 100644 --- a/src/process/capture/FilesystemSnapshot.test.ts +++ b/src/process/capture/FilesystemSnapshot.test.ts @@ -1,6 +1,8 @@ import { fstatSync } from "node:fs"; import { mkdtemp, + mkdir, + symlink, lstat, open, readdir, @@ -35,6 +37,27 @@ const countOpenDescriptorsFor = async (path: string): Promise => { return matches.filter(Boolean).length; }; +it.skipIf(process.platform === "win32")( + "omits a captured regular file replaced by a symlink", + async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-snapshot-symlink-")); + const selected = join(directory, "selected"); + const replacement = join(directory, "replacement"); + try { + await writeFile(selected, "captured bytes"); + await writeFile(replacement, "unselected bytes"); + const expected = await lstat(selected); + await rm(selected); + await symlink(replacement, selected); + const outcome = await hashFile(selected, expected, 1_000); + expect(outcome.state).toBe("omitted"); + expect(outcome).not.toHaveProperty("sha256"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, +); + it("does not hash a replacement file using the earlier path metadata", async () => { const directory = await mkdtemp(join(tmpdir(), "rea-snapshot-identity-")); const path = join(directory, "observed-file"); @@ -154,3 +177,43 @@ it.skipIf(process.platform === "win32")( } }, ); + +for (const variant of [ + "unchanged", + "case-alias", + "ancestor-junction", + "replaced-junction", +] as const) { + it.runIf(process.platform === "win32")( + `hashes only the captured Windows file through ${variant}`, + async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-snapshot-windows-")); + try { + const original = join(directory, "original"); + const replacement = join(directory, "replacement"); + await mkdir(original); + await mkdir(replacement); + await writeFile(join(original, "TestFile.txt"), "captured bytes"); + await writeFile(join(replacement, "TestFile.txt"), "replaced bytes"); + const alias = join(directory, "alias"); + if (variant === "ancestor-junction" || variant === "replaced-junction") + await symlink(original, alias, "junction"); + const path = join( + variant.includes("junction") ? alias : original, + variant === "case-alias" ? "testfile.txt" : "TestFile.txt", + ); + const expected = await lstat(path); + if (variant === "replaced-junction") { + await rm(alias); + await symlink(replacement, alias, "junction"); + } + const result = await hashFile(path, expected, 1_000); + expect(result.state).toBe( + variant === "replaced-junction" ? "omitted" : "hashed", + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, + ); +} diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.ts index 5740f6525..3ec2e0716 100644 --- a/src/process/capture/FilesystemSnapshot.ts +++ b/src/process/capture/FilesystemSnapshot.ts @@ -74,12 +74,14 @@ export const hashFile = async ( signal?: AbortSignal, ): Promise => { signal?.throwIfAborted(); + const noFollowAvailable = + typeof fsConstants.O_NOFOLLOW === "number" && fsConstants.O_NOFOLLOW !== 0; let handle: FileHandle; try { handle = await open( path, fsConstants.O_RDONLY | - (fsConstants.O_NOFOLLOW ?? 0) | + (noFollowAvailable ? fsConstants.O_NOFOLLOW : 0) | (fsConstants.O_NONBLOCK ?? 0), ); } catch (cause: unknown) { @@ -94,6 +96,18 @@ export const hashFile = async ( } try { signal?.throwIfAborted(); + // On hosts without O_NOFOLLOW, compare the leaf's identity with the + // captured regular file. Ancestor aliases do not change file identity. + if (!noFollowAvailable) { + const named = await lstat(path); + signal?.throwIfAborted(); + if (!hasSameFileState(expected, named)) + return { + state: "omitted", + reason: "file_changed_or_short_read", + system_code: null, + }; + } const stats = await handle.stat(); signal?.throwIfAborted(); if (!hasSameFileState(expected, stats) || stats.size > maxBytes) @@ -124,7 +138,13 @@ export const hashFile = async ( } const after = await handle.stat(); signal?.throwIfAborted(); - if (!hasSameFileState(stats, after)) + let unchanged = hasSameFileState(stats, after); + if (unchanged && !noFollowAvailable) { + const namedAfter = await lstat(path); + signal?.throwIfAborted(); + unchanged = hasSameFileState(after, namedAfter); + } + if (!unchanged) return { state: "omitted", reason: "file_changed_or_short_read", @@ -158,6 +178,7 @@ export const snapshotRoots = async ( const enumerationReasons = new Set< FilesystemCoverage["enumeration_reasons"][number] >(); + const enumerationOmissions: FilesystemCoverage["enumeration_omissions"] = []; const hashOmissions: FilesystemCoverage["hash_omissions"] = []; const visit = async ( root: string, @@ -252,7 +273,23 @@ export const snapshotRoots = async ( let complete = true; const remainingEntries = scenario.limits.files - entries.length - pendingNames; - const directory = await opendir(path); + let directory; + try { + directory = await opendir(path); + } catch (cause: unknown) { + signal?.throwIfAborted(); + const systemCode = fileUnavailableSystemCode(cause); + if (systemCode === undefined) throw cause; + truncated = true; + complete = false; + enumerationReasons.add("directory_unavailable"); + enumerationOmissions.push({ + path: `${rootAlias}:${relativePath}`, + reason: "directory_unavailable", + system_code: systemCode, + }); + return false; + } for await (const child of directory) { signal?.throwIfAborted(); if ( @@ -305,6 +342,7 @@ export const snapshotRoots = async ( enumeration_truncated: completeRoots.length !== scenario.filesystem_observation_paths.length, enumeration_reasons: [...enumerationReasons], + enumeration_omissions: enumerationOmissions, hash_budget_bytes: scenario.limits.file_bytes, hashed_bytes: scenario.limits.file_bytes - remainingBytes, hash_omissions: hashOmissions, diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index ab9271dda..0e1ca73ee 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -38,7 +38,6 @@ import { assertNotCancelled } from "./ProcessScenarioRuntimeValidation.js"; import { normalizeProcessElapsedTime, normalizeProcessSamples, - normalizeProcessText, } from "./ProcessNormalization.js"; import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js"; import { TerminalRenderer } from "./TerminalRenderer.js"; @@ -1302,9 +1301,9 @@ export const captureTerminalFrames = (options: { readonly scenario: ProcessScenario; readonly frames: TerminalFrame[]; readonly started: number; - readonly temporaryRoot: string; readonly onOutput: () => void; readonly renderer: TerminalRenderer; + readonly normalizeText: (value: string) => string; readonly recordEvent: RecordProcessCaptureEvent; }): (() => TerminalRetention) => { let outputBytes = 0; @@ -1323,12 +1322,7 @@ export const captureTerminalFrames = (options: { monotonicProcessCaptureClock() - options.started, options.scenario.normalization.time_bucket_ms, ); - const normalized = normalizeProcessText( - data, - options.scenario, - options.temporaryRoot, - options.terminal.pid, - ); + const normalized = options.normalizeText(data); const sequence = options.frames.length; options.frames.push({ sequence, diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.ts index 847e512f8..d12e66837 100644 --- a/src/process/capture/ProcessHarness.ts +++ b/src/process/capture/ProcessHarness.ts @@ -24,7 +24,7 @@ import { startProcessSampler } from "./ProcessSampling.js"; import { snapshotRoots } from "./FilesystemSnapshot.js"; import { TerminalRenderer } from "./TerminalRenderer.js"; import type { ProcessTimer } from "./ProcessTimer.js"; -import { normalizeProcessText } from "./ProcessNormalization.js"; +import { createProcessTextNormalizer } from "./ProcessNormalization.js"; import { selectCapturedProcessGroupIds } from "../ProcessOwnershipProcessTree.js"; import { awaitTerminalExit, @@ -222,8 +222,7 @@ const cleanupFailedStartup = async (options: { const createTerminalRenderer = ( scenario: ProcessScenario, - temporaryRoot: string, - terminalPid: () => number, + normalizeText: (value: string) => string, recordEvent: RecordProcessCaptureEvent, ): TerminalRenderer => new TerminalRenderer({ @@ -231,8 +230,7 @@ const createTerminalRenderer = ( rows: scenario.terminal.rows, scrollback: scenario.terminal.scrollback, maxBytes: scenario.limits.output_bytes, - normalize: (value) => - normalizeProcessText(value, scenario, temporaryRoot, terminalPid()), + normalize: normalizeText, recordEvent, }); @@ -271,10 +269,14 @@ const startCaptureRuntime = async ( const startedAt = new Date(); const started = monotonicProcessCaptureClock(); let lastOutput = started; - renderer = createTerminalRenderer( + const normalizeText = createProcessTextNormalizer( scenario, options.temporaryRoot, () => terminal?.pid ?? -1, + ); + renderer = createTerminalRenderer( + scenario, + normalizeText, options.recordEvent, ); terminal = spawn(scenario.executable, [...scenario.arguments], { @@ -299,6 +301,7 @@ const startCaptureRuntime = async ( started, onOutput: () => (lastOutput = monotonicProcessCaptureClock()), renderer, + normalizeText, }); scheduleScenarioInteractions({ ...options, diff --git a/src/process/capture/ProcessNormalization.test.ts b/src/process/capture/ProcessNormalization.test.ts index e5ef61062..54b042dd4 100644 --- a/src/process/capture/ProcessNormalization.test.ts +++ b/src/process/capture/ProcessNormalization.test.ts @@ -1,6 +1,7 @@ import { expect, it } from "vitest"; import { + createProcessTextNormalizer, normalizeProcessSamples, normalizeProcessText, } from "./ProcessNormalization.js"; @@ -94,6 +95,42 @@ it("replaces caller-declared patterns literally, without expanding replacement p ).toBe("listening on $&$&; $$; $1=alpha; tail $1"); }); +it("reuses matchers while observing the terminal PID after launch", () => { + let pid = -1; + const normalize = createProcessTextNormalizer( + scenario({ ports: false, patterns: [] }), + "/temporary", + () => pid, + ); + expect(normalize("root 41010 /workspace/item")).toBe( + "root 41010 /item", + ); + pid = 41_010; + expect(normalize("root 41010 /workspace/item")).toBe( + "root /item", + ); + pid = 42_020; + expect(normalize("root 41010 peer 42020")).toBe("root 41010 peer "); +}); + +it("keeps equal-path priority and replacement aliases local to each capture", () => { + const declared = parseProcessScenario({ + executable: "/selected/tool", + working_directory: "/selected", + filesystem_observation_paths: ["/selected", ""], + normalization: { paths: true, pids: false, ports: false, patterns: [] }, + }); + const normalize = createProcessTextNormalizer(declared, "/selected", -1); + const other = createProcessTextNormalizer(declared, "/other", -1); + const text = "/selected/item /source"; + expect(normalize(text)).toBe( + "/item /source", + ); + expect(other(text)).toBe( + "/item /source", + ); +}); + it("preserves empty-pattern boundaries, including empty text", () => { const declared = scenario({ patterns: [{ pattern: "", replacement: "$$" }], diff --git a/src/process/capture/ProcessNormalization.ts b/src/process/capture/ProcessNormalization.ts index 74db31b1c..85dc40929 100644 --- a/src/process/capture/ProcessNormalization.ts +++ b/src/process/capture/ProcessNormalization.ts @@ -9,36 +9,51 @@ export const normalizeProcessElapsedTime = ( timeBucketMs: number, ): number => Math.floor(elapsedMs / timeBucketMs) * timeBucketMs; -/** Normalize one terminal/protocol payload under explicitly selected scenario rules. */ +/** Prepare capture-local matchers while retaining a live terminal PID lookup. */ +export const createProcessTextNormalizer = ( + scenario: ProcessScenario, + temporaryRoot: string, + pid: number | (() => number), +): ((value: string) => string) => { + const normalizePaths = scenario.normalization.paths + ? createPathNormalizer(scenario, temporaryRoot) + : (value: string): string => value; + let previousPid: number | undefined; + let normalizePids = (value: string): string => value; + return (value) => { + let normalized = normalizePaths(value); + if (scenario.normalization.ports) + normalized = normalizePortTokens(normalized); + if (scenario.normalization.pids) { + const currentPid = typeof pid === "number" ? pid : pid(); + if (currentPid !== previousPid) { + previousPid = currentPid; + normalizePids = createPidNormalizer([currentPid]); + } + normalized = normalizePids(normalized); + } + for (const pattern of scenario.normalization.patterns) + normalized = normalized.replaceAll( + pattern.pattern, + () => pattern.replacement, + ); + return normalized; + }; +}; + +/** Normalize one payload when its caller does not retain a capture-local matcher. */ export const normalizeProcessText = ( value: string, scenario: ProcessScenario, temporaryRoot: string, pid: number, -): string => { - let normalized = value; - if (scenario.normalization.paths) - normalized = normalizePathTokens(normalized, scenario, temporaryRoot); - if (scenario.normalization.ports) - normalized = normalizePortTokens(normalized); - if (scenario.normalization.pids) - normalized = normalizePidTokens(normalized, [pid]); - // A callback keeps replacement text literal while preserving replaceAll's - // empty-pattern boundaries, which the process scenario contract permits. - for (const pattern of scenario.normalization.patterns) - normalized = normalized.replaceAll( - pattern.pattern, - () => pattern.replacement, - ); - return normalized; -}; +): string => createProcessTextNormalizer(scenario, temporaryRoot, pid)(value); /** Match original paths once, preferring the most specific selected identity. */ -const normalizePathTokens = ( - value: string, +const createPathNormalizer = ( scenario: ProcessScenario, temporaryRoot: string, -): string => { +): ((value: string) => string) => { const aliases = new Map(); const selected: readonly (readonly [string, string])[] = [ [temporaryRoot, ""], @@ -56,14 +71,12 @@ const normalizePathTokens = ( const paths = [...aliases.keys()].sort( (left, right) => right.length - left.length, ); - if (paths.length === 0) return value; + if (paths.length === 0) return (value) => value; const pattern = paths .map((path) => path.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) .join("|"); - return value.replace( - new RegExp(pattern, "gu"), - (path) => aliases.get(path) ?? path, - ); + const matcher = new RegExp(pattern, "gu"); + return (value) => value.replace(matcher, (path) => aliases.get(path) ?? path); }; // A port may end a sentence, but a period followed by a letter, digit, or @@ -109,16 +122,17 @@ export const normalizeProcessSamples = ( for (const identifier of identifiers) if (identifier > 0 && !mapping.has(identifier)) mapping.set(identifier, mapping.size + 1); + const normalizeText = createProcessTextNormalizer( + scenario, + "", + rootPid, + ); + const normalizeSamplePids = scenario.normalization.pids + ? createPidNormalizer(identifiers) + : undefined; const normalizeCommand = (command: string): string => { - const normalized = normalizeProcessText( - command, - scenario, - "", - rootPid, - ); - const withNormalizedPids = scenario.normalization.pids - ? normalizePidTokens(normalized, identifiers) - : normalized; + const normalized = normalizeText(command); + const withNormalizedPids = normalizeSamplePids?.(normalized) ?? normalized; return withNormalizedPids.trim(); }; return samples.map((sample) => ({ @@ -144,17 +158,14 @@ export const normalizeProcessSamples = ( })); }; -const normalizePidTokens = ( - value: string, +const createPidNormalizer = ( identifiers: readonly number[], -): string => { +): ((value: string) => string) => { const tokens = [...new Set(identifiers)] .filter((identifier) => Number.isSafeInteger(identifier) && identifier > 0) .sort((left, right) => right - left) .map(String); - if (tokens.length === 0) return value; - return value.replace( - new RegExp(`(?", - ); + if (tokens.length === 0) return (value) => value; + const matcher = new RegExp(`(? value.replace(matcher, ""); }; diff --git a/src/windows/WindowsOwnedProcess.test.ts b/src/windows/WindowsOwnedProcess.test.ts index 5beec9520..2ec1452bb 100644 --- a/src/windows/WindowsOwnedProcess.test.ts +++ b/src/windows/WindowsOwnedProcess.test.ts @@ -1,5 +1,5 @@ import { once } from "node:events"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { WindowsNativeAuthority } from "./WindowsNativeLoader.js"; import { @@ -82,6 +82,7 @@ const authority = ( : {}), }; if (operation === "process_poll") return polls.shift(); + if (operation === "process_terminate") return true; if (operation === "process_close") return null; throw new Error(`Unmodeled native operation ${operation}`); }, @@ -205,3 +206,223 @@ describe("Windows Job Object memory limits", () => { expect(() => launch(native, LIMIT)).toThrow(); }); }); + +describe("Windows output capture drain", () => { + it("repolls without the 25ms idle delay while the child keeps producing output", async () => { + vi.useFakeTimers(); + try { + const chunk = Buffer.alloc(8192, 0x61); + const polls: Array<{ + stdout: Buffer; + stderr: Buffer; + stdoutEnded: boolean; + stderrEnded: boolean; + exitCode: number | null; + activeProcesses: number; + }> = Array.from({ length: 10 }, () => ({ + stdout: chunk, + stderr: Buffer.alloc(0), + stdoutEnded: false, + stderrEnded: false, + exitCode: null, + activeProcesses: 1, + })); + polls.push({ ...settledPoll }); + const { native, calls } = authority({ polls }); + const child = launch(native); + child.stdout.resume(); + const closed = once(child, "close"); + // 100ms of simulated time: the fixed 25ms poll cadence can complete + // at most ~4 polls, so all 10 data polls plus the settle poll must + // still be pending. Draining repolls immediately while data flows. + await vi.advanceTimersByTimeAsync(100); + expect( + calls.filter((call) => call.operation === "process_poll"), + ).toHaveLength(11); + await closed; + } finally { + vi.useRealTimers(); + } + }); + + it("keeps the 25ms idle delay after a poll with no output", async () => { + vi.useFakeTimers(); + try { + const idle = { + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + stdoutEnded: false, + stderrEnded: false, + exitCode: null, + activeProcesses: 1, + }; + const { native, calls } = authority({ + polls: [idle, { ...settledPoll }], + }); + const child = launch(native); + const closed = once(child, "close"); + const pollCount = () => + calls.filter((call) => call.operation === "process_poll").length; + // The constructor schedules the first poll 25ms out. + await vi.advanceTimersByTimeAsync(25); + expect(pollCount()).toBe(1); + // An idle child must not be repolled early: 10ms of simulated time + // must not reach the next poll, the remaining 15ms must. + await vi.advanceTimersByTimeAsync(10); + expect(pollCount()).toBe(1); + await vi.advanceTimersByTimeAsync(15); + expect(pollCount()).toBe(2); + await closed; + } finally { + vi.useRealTimers(); + } + }); + + it("repolls without the 25ms idle delay while only stderr carries output", async () => { + vi.useFakeTimers(); + try { + const chunk = Buffer.alloc(4096, 0x62); + const { native, calls } = authority({ + polls: [ + { + stdout: Buffer.alloc(0), + stderr: chunk, + stdoutEnded: false, + stderrEnded: false, + exitCode: null, + activeProcesses: 1, + }, + { ...settledPoll }, + ], + }); + const child = launch(native); + const closed = once(child, "close"); + const pollCount = () => + calls.filter((call) => call.operation === "process_poll").length; + await vi.advanceTimersByTimeAsync(25); + expect(pollCount()).toBe(1); + // stderr-only output still means the child is producing: the next + // poll must be immediate, not 25ms out. + await vi.advanceTimersByTimeAsync(1); + expect(pollCount()).toBe(2); + await closed; + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("Windows output backpressure and cleanup", () => { + it("stops native reads until both paused output consumers drain", async () => { + vi.useFakeTimers(); + try { + const chunk = Buffer.alloc(65536, 0x63); + const { native, calls } = authority({ + polls: [ + { + stdout: chunk, + stderr: chunk, + stdoutEnded: false, + stderrEnded: false, + exitCode: null, + activeProcesses: 1, + }, + { ...settledPoll, stderr: Buffer.alloc(0) }, + ], + }); + const child = launch(native); + const closed = once(child, "close"); + child.stdout.pause(); + child.stderr.pause(); + const pollCount = () => + calls.filter((call) => call.operation === "process_poll").length; + await vi.advanceTimersByTimeAsync(25); + expect(child.stdout.writableNeedDrain).toBe(true); + expect(child.stderr.writableNeedDrain).toBe(true); + await vi.advanceTimersByTimeAsync(5000); + expect(pollCount()).toBe(1); + const stdoutDrained = once(child.stdout, "drain"); + child.stdout.resume(); + await stdoutDrained; + await vi.advanceTimersByTimeAsync(25); + expect(pollCount()).toBe(1); + const stderrDrained = once(child.stderr, "drain"); + child.stderr.resume(); + await stderrDrained; + await vi.advanceTimersByTimeAsync(1); + await closed; + expect(pollCount()).toBe(2); + } finally { + vi.useRealTimers(); + } + }); + + it("discards blocked output during cleanup so the owned job can settle", async () => { + vi.useFakeTimers(); + try { + const chunk = Buffer.alloc(65536, 0x64); + const { native, calls } = authority({ + polls: [ + { + stdout: chunk, + stderr: Buffer.alloc(0), + stdoutEnded: false, + stderrEnded: false, + exitCode: null, + activeProcesses: 1, + }, + { ...settledPoll, stderr: Buffer.alloc(0) }, + ], + }); + const child = launch(native); + child.stdout.pause(); + await vi.advanceTimersByTimeAsync(25); + expect(child.stdout.writableNeedDrain).toBe(true); + const closed = once(child, "close"); + const cleanup = child.cleanup(); + await vi.advanceTimersByTimeAsync(25); + expect(await cleanup).toEqual({ cleaned: true, signaled: true }); + await closed; + expect( + calls.filter((call) => call.operation === "process_poll"), + ).toHaveLength(2); + expect(child.stdout.destroyed).toBe(true); + expect(child.outputIncomplete).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it("resumes polling when a blocked output stream is destroyed", async () => { + vi.useFakeTimers(); + try { + const chunk = Buffer.alloc(65536, 0x65); + const { native, calls } = authority({ + polls: [ + { + stdout: chunk, + stderr: Buffer.alloc(0), + stdoutEnded: false, + stderrEnded: false, + exitCode: null, + activeProcesses: 1, + }, + { ...settledPoll, stderr: Buffer.alloc(0) }, + ], + }); + const child = launch(native); + const closed = once(child, "close"); + await vi.advanceTimersByTimeAsync(25); + expect(child.stdout.writableNeedDrain).toBe(true); + child.stdout.destroy(); + await vi.advanceTimersByTimeAsync(0); + await closed; + expect( + calls.filter((call) => call.operation === "process_poll"), + ).toHaveLength(2); + expect(child.outputIncomplete).toBe(true); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/src/windows/WindowsOwnedProcess.ts b/src/windows/WindowsOwnedProcess.ts index b6a530461..062f3b78e 100644 --- a/src/windows/WindowsOwnedProcess.ts +++ b/src/windows/WindowsOwnedProcess.ts @@ -67,11 +67,14 @@ export class WindowsOwnedProcess extends EventEmitter { #closed = false; #failure: string | undefined; #timer: NodeJS.Timeout | undefined; + #blockedWake: NodeJS.Timeout | undefined; + readonly #blockedOutput = new Set(); #unreferenced = false; #cleanup: Promise | undefined; readonly #memoryLimitBytes: number | undefined; #memoryLimitReached = false; #peakJobMemoryBytes: number | undefined; + #outputIncomplete = false; constructor( command: string, @@ -179,6 +182,11 @@ export class WindowsOwnedProcess extends EventEmitter { return this.#peakJobMemoryBytes; } + /** True when cleanup destroyed a backpressured stream before all output was observed. */ + get outputIncomplete(): boolean { + return this.#outputIncomplete; + } + async #writeInput(chunk: unknown): Promise { const bytes = z.instanceof(Buffer).parse(chunk); const written = z @@ -227,12 +235,14 @@ export class WindowsOwnedProcess extends EventEmitter { unref(): void { this.#unreferenced = true; this.#timer?.unref(); + this.#blockedWake?.unref(); } /** Restore event-loop references when retrying process lifecycle observation. */ ref(): void { this.#unreferenced = false; this.#timer?.ref(); + this.#blockedWake?.ref(); } async #stop(): Promise { @@ -240,6 +250,7 @@ export class WindowsOwnedProcess extends EventEmitter { let timer: NodeJS.Timeout | undefined; try { const signaled = this.kill(); + this.#discardBlockedOutput(); this.#schedule(); const settled = await Promise.race([ this.#settled.then(() => true), @@ -265,12 +276,50 @@ export class WindowsOwnedProcess extends EventEmitter { } } - #schedule(): void { - if (this.#closed || this.#timer !== undefined) return; - this.#timer = setTimeout(() => this.#poll(), 25); + #schedule(delay = 25): void { + if ( + this.#closed || + this.#timer !== undefined || + this.#blockedOutput.size > 0 + ) + return; + this.#timer = setTimeout(() => this.#poll(), delay); if (this.#unreferenced) this.#timer.unref(); } + #scheduleBlockedWake(): void { + if ( + this.#closed || + this.#blockedOutput.size === 0 || + this.#blockedWake !== undefined + ) + return; + this.#blockedWake = setTimeout(() => { + this.#blockedWake = undefined; + this.#scheduleBlockedWake(); + }, 1_000); + if (this.#unreferenced) this.#blockedWake.unref(); + } + + #clearBlockedWake(): void { + if (this.#blockedWake !== undefined) clearTimeout(this.#blockedWake); + this.#blockedWake = undefined; + } + + #unblockOutput(stream: PassThrough): void { + if (stream.destroyed) this.#outputIncomplete = true; + this.#blockedOutput.delete(stream); + if (this.#blockedOutput.size === 0) this.#clearBlockedWake(); + this.#schedule(0); + } + + #discardBlockedOutput(): void { + if (this.#blockedOutput.size > 0) this.#outputIncomplete = true; + for (const stream of this.#blockedOutput) stream.destroy(); + this.#blockedOutput.clear(); + this.#clearBlockedWake(); + } + #poll(): void { this.#timer = undefined; if (this.#closed) return; @@ -289,8 +338,11 @@ export class WindowsOwnedProcess extends EventEmitter { state = limited; } this.#failure = undefined; - if (state.stdout.length > 0) this.stdout.write(state.stdout); - if (state.stderr.length > 0) this.stderr.write(state.stderr); + // Stop native reads while either consumer is backpressured. A slower + // cadence still accumulates unbounded unread output in the parent heap. + const idle = state.stdout.length === 0 && state.stderr.length === 0; + this.#writeOutput(this.stdout, state.stdout); + this.#writeOutput(this.stderr, state.stderr); if (state.exitCode !== null && this.exitCode === null) { this.exitCode = state.exitCode; this.emit("exit", state.exitCode, null); @@ -302,7 +354,7 @@ export class WindowsOwnedProcess extends EventEmitter { state.activeProcesses === 0 ) { this.#release(); - } else this.#schedule(); + } else this.#schedule(idle ? 25 : 0); } catch (cause: unknown) { const failure = cause instanceof Error ? cause : new Error(String(cause)); this.#failure = failure.message; @@ -313,14 +365,33 @@ export class WindowsOwnedProcess extends EventEmitter { } } + #writeOutput(stream: PassThrough, bytes: Buffer): void { + if (bytes.length === 0 || stream.writableEnded) return; + if (stream.destroyed) { + this.#outputIncomplete = true; + return; + } + if (stream.write(bytes)) return; + this.#blockedOutput.add(stream); + const unblock = () => { + stream.off("drain", unblock); + stream.off("close", unblock); + this.#unblockOutput(stream); + }; + stream.once("drain", unblock); + stream.once("close", unblock); + this.#scheduleBlockedWake(); + } + #release(): void { if (this.#closed) return; if (this.#timer !== undefined) clearTimeout(this.#timer); + this.#clearBlockedWake(); this.#authority.call("process_close", [this.#handle]); this.#closed = true; this.stdin?.destroy(); - this.stdout.end(); - this.stderr.end(); + if (!this.stdout.destroyed) this.stdout.end(); + if (!this.stderr.destroyed) this.stderr.end(); this.#settle?.(); this.emit("close", this.exitCode, null); } diff --git a/tests/boundary/process/cleanLinuxHopperRuntime.test.ts b/tests/boundary/process/cleanLinuxHopperRuntime.test.ts new file mode 100644 index 000000000..410d713e4 --- /dev/null +++ b/tests/boundary/process/cleanLinuxHopperRuntime.test.ts @@ -0,0 +1,411 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { + chmod, + copyFile, + mkdir, + readFile, + symlink, + unlink, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { assertCleanLinuxHopperRuntime } from "../../../scripts/lib/clean-linux-hopper-runtime.mjs"; +import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execFileAsync = promisify(execFile); + +it.runIf(process.platform === "linux").each([ + ["Xvfb", "Clean Linux runner retained a Hopper demo process"], + ["Hopper", "Clean Linux runner retained a Hopper demo process"], + ["python3", "Cannot identify a command-less Python process"], + ["unrelated", undefined], +] as const)( + "observes an unreaped %s without signaling or reaping it", + async (name, refusal) => { + const directory = await createTestTempDirectory("rea-runtime-zombie-"); + const procRoot = join(directory, "proc"); + await mkdir(procRoot); + const executable = join(directory, name); + await copyFile("/usr/bin/true", executable); + await chmod(executable, 0o700); + // WNOWAIT observes exit without reaping. The parent keeps the zombie until + // the test closes its stdin, then waitpid performs the actual cleanup. + const parent = execFileAsync( + "/usr/bin/python3", + [ + "-c", + ` +import os, sys +pid = os.fork() +if pid == 0: + os.execl(sys.argv[1], sys.argv[1]) +try: + os.waitid(os.P_PID, pid, os.WEXITED | os.WNOWAIT) + print(pid, flush=True) + sys.stdin.buffer.read(1) +finally: + os.waitpid(pid, 0) +`, + executable, + ], + { timeout: 10_000 }, + ); + const ready = new Promise((resolve, reject) => { + parent.child.stdout?.once("data", (data: Buffer) => + resolve(Number(data.toString().trim())), + ); + parent.catch(reject); + }); + let zombiePid: number | undefined; + try { + zombiePid = await ready; + expect(Number.isSafeInteger(zombiePid)).toBe(true); + const prefix = `/proc/${String(zombiePid)}`; + const procEntry = join(procRoot, String(zombiePid)); + await mkdir(procEntry); + for (const name of ["cmdline", "comm"]) + await symlink(`${prefix}/${name}`, join(procEntry, name)); + expect(await readFile(`${prefix}/cmdline`, "utf8")).toBe(""); + expect((await readFile(`${prefix}/comm`, "utf8")).trim()).toBe(name); + const observation = assertCleanLinuxHopperRuntime([], procRoot); + if (refusal === undefined) + await expect(observation).resolves.toBeUndefined(); + else await expect(observation).rejects.toThrow(refusal); + expect(await readFile(`${prefix}/stat`, "utf8")).toContain( + `(${name}) Z `, + ); + } finally { + parent.child.stdin?.end("reap"); + await parent; + if (zombiePid !== undefined) + await expect( + readFile(`/proc/${String(zombiePid)}/stat`, "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + } + }, +); + +it.runIf(process.platform === "linux")( + "does not treat runtime names in another process's arguments as a runtime", + async () => { + const directory = await createTestTempDirectory("rea-runtime-arguments-"); + const procRoot = join(directory, "proc"); + await mkdir(procRoot); + const observer = new URL( + "../../../scripts/lib/clean-linux-hopper-runtime.mjs", + import.meta.url, + ).href; + const code = ` + import { mkdir, symlink } from "node:fs/promises"; + import { join } from "node:path"; + const { assertCleanLinuxHopperRuntime } = await import(${JSON.stringify(observer)}); + const entry = join(${JSON.stringify(procRoot)}, String(process.pid)); + await mkdir(entry); + await symlink("/proc/" + process.pid + "/cmdline", join(entry, "cmdline")); + await symlink("/proc/" + process.pid + "/comm", join(entry, "comm")); + await assertCleanLinuxHopperRuntime([], ${JSON.stringify(procRoot)}); + console.log('clean'); + `; + const { stdout } = await execFileAsync( + process.execPath, + [ + "--input-type=module", + "-e", + code, + procRoot, + "Xvfb", + "/opt/hopper/bin/Hopper", + "hopper-demo-x11.py", + ], + { timeout: 10_000 }, + ); + expect(stdout.trim()).toBe("clean"); + }, +); + +it.runIf(process.platform === "linux")( + "distinguishes an ordinary Node script with the helper's filename", + async () => { + const directory = await createTestTempDirectory("rea-runtime-arguments-"); + const procRoot = join(directory, "proc"); + await mkdir(procRoot); + const script = join(directory, "hopper-demo-x11.py"); + const observer = new URL( + "../../../scripts/lib/clean-linux-hopper-runtime.mjs", + import.meta.url, + ).href; + const code = ` + import { mkdir, symlink } from "node:fs/promises"; + import { join } from "node:path"; + const entry = join(${JSON.stringify(procRoot)}, String(process.pid)); + await mkdir(entry); + await symlink("/proc/" + process.pid + "/cmdline", join(entry, "cmdline")); + await symlink("/proc/" + process.pid + "/comm", join(entry, "comm")); + const { assertCleanLinuxHopperRuntime } = await import(${JSON.stringify(observer)}); + await assertCleanLinuxHopperRuntime([], ${JSON.stringify(procRoot)}); + console.log('clean'); + `; + const { stdout } = await execFileAsync( + process.execPath, + ["--input-type=module", "--eval", code, script], + { timeout: 10_000 }, + ); + expect(stdout.trim()).toBe("clean"); + }, +); + +const launchReapedOrphan = (runId: string, parentCode: string) => { + // The short-lived Node launcher still exits before observation. A retained + // subreaper adopts its child and waits after owned cleanup, even when the + // container's PID 1 does not reap. It is a sibling of the verifier below. + const reaper = execFileAsync( + "/usr/bin/python3", + [ + "-c", + ` +import ctypes, json, os, subprocess, sys +libc = ctypes.CDLL(None, use_errno=True) +if libc.prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER + raise OSError(ctypes.get_errno(), "PR_SET_CHILD_SUBREAPER") +launcher = subprocess.run(sys.argv[1:], capture_output=True, text=True, check=True) +child = json.loads(launcher.stdout) +try: + print(json.dumps(child), flush=True) + sys.stdin.buffer.read(1) +finally: + os.waitpid(child["pid"], 0) +`, + process.execPath, + "-e", + parentCode, + ], + { + env: { ...process.env, REA_PROCESS_RUN_ID: runId }, + timeout: 40_000, + }, + ); + const ready = new Promise<{ pid: number; parentPid: number }>( + (resolve, reject) => { + reaper.child.stdout?.once("data", (data: Buffer) => { + try { + resolve(JSON.parse(data.toString())); + } catch (error) { + reject(error); + } + }); + reaper.then( + () => reject(new Error("Fixture reaper exited before readiness")), + reject, + ); + }, + ); + return { reaper, ready }; +}; + +const observeFromSeparateVerifier = async ( + runId: string, + launcherPaths: string[], + procRoot: string, +) => { + const observer = new URL( + "../../../scripts/lib/clean-linux-hopper-runtime.mjs", + import.meta.url, + ).href; + const verifierRun = new URL( + "../../../scripts/lib/verifier-run.mjs", + import.meta.url, + ).href; + const { stdout: observed } = await execFileAsync( + process.execPath, + [ + "--input-type=module", + "-e", + ` +const { assertCleanLinuxHopperRuntime } = await import(${JSON.stringify(observer)}); +const { completeVerifierRun } = await import(${JSON.stringify(verifierRun)}); +const { process_lineage } = await completeVerifierRun({ + run_id: ${JSON.stringify(runId)}, verifier_pid: process.pid, parent_pid: process.ppid, +}); +let refusal; +try { await assertCleanLinuxHopperRuntime(${JSON.stringify(launcherPaths)}, ${JSON.stringify(procRoot)}); } +catch (error) { refusal = error.message; } +console.log(JSON.stringify({ process_lineage, refusal })); +`, + ], + { timeout: 10_000 }, + ); + return JSON.parse(observed) as { + process_lineage: { + status: string; + descendants: { pid: number }[]; + }; + refusal?: string; + }; +}; + +it.runIf(process.platform === "linux").each([ + ["Xvfb-named Node", null, true], + ["Python helper", [], true], + ["Python helper with -u", ["-u"], true], + [ + "Python helper with combined/value options", + ["-Bu", "-W", "ignore", "-X", "dev", "--check-hash-based-pycs", "always"], + true, + ], + ["Python helper after --", ["--"], true], + [ + "Python -c script argument", + ["-c", "import time; print('ready', flush=True); time.sleep(30)"], + false, + ], + ["Python -m script argument", ["-m", "unrelated"], false], + ["alternate launcher", null, true], +] as const)( + "classifies a reparented %s without signaling it", + async (mode, pythonOptions, retained) => { + const runId = randomUUID(); + const childCode = + "process.title = 'Xvfb'; setInterval(() => {}, 1000); process.send('ready');"; + const directory = await createTestTempDirectory("rea-clean-runtime-test-"); + const procRoot = join(directory, "proc"); + await mkdir(procRoot); + const launcher = join(directory, "Hopper"); + const alternate = join(directory, "Hopper-alternate"); + const helper = join(directory, "hopper-demo-x11.py"); + const alternateMode = mode === "alternate launcher"; + const pythonMode = pythonOptions !== null; + if (alternateMode) { + await copyFile("/usr/bin/sleep", launcher); + await chmod(launcher, 0o700); + await symlink(launcher, alternate); + } + if (pythonMode) { + await writeFile( + helper, + "import time\nprint('ready', flush=True)\ntime.sleep(30)\n", + ); + await copyFile(helper, join(directory, "unrelated.py")); + } + // The fixture must be the cause of a refusal; a retained host runtime + // would otherwise mask a broken classification or executable-identity check. + await assertCleanLinuxHopperRuntime( + alternateMode ? [launcher] : [], + procRoot, + ); + const command = alternateMode + ? alternate + : pythonMode + ? "/usr/bin/python3" + : process.execPath; + const arguments_ = alternateMode + ? ["30"] + : pythonMode + ? [...pythonOptions, helper] + : ["-e", childCode]; + const stdio = alternateMode + ? "ignore" + : pythonMode + ? ["ignore", "pipe", "ignore"] + : ["ignore", "ignore", "ignore", "ipc"]; + const parentCode = ` + const { spawn } = require('node:child_process'); + const child = spawn(${JSON.stringify(command)}, ${JSON.stringify(arguments_)}, { + detached: true, stdio: ${JSON.stringify(stdio)}, cwd: ${JSON.stringify(directory)} + }); + child.once('error', () => process.exit(1)); + const ready = () => { + process.stdout.write(JSON.stringify({ pid: child.pid, parentPid: process.pid })); + if (child.connected) child.disconnect(); + child.stdout?.destroy(); + child.unref(); + }; + ${pythonMode ? "child.stdout.once('data', ready);" : `child.once(${JSON.stringify(alternateMode ? "spawn" : "message")}, ready);`} + `; + const { reaper, ready } = launchReapedOrphan(runId, parentCode); + let orphanPid: number | undefined; + try { + const { pid, parentPid } = await ready; + orphanPid = pid; + const procEntry = join(procRoot, String(pid)); + await mkdir(procEntry); + for (const name of ["cmdline", "comm", "exe"]) + await symlink(`/proc/${String(pid)}/${name}`, join(procEntry, name)); + const { stdout: coordinates } = await execFileAsync("ps", [ + "-p", + String(pid), + "-o", + "ppid=,pgid=", + ]); + const [currentParent, group] = coordinates + .trim() + .split(/\s+/u) + .map(Number); + expect(currentParent).not.toBe(parentPid); + expect(currentParent).toBe(reaper.child.pid); + expect(group).toBe(pid); + + if (alternateMode) { + const commandLine = await readFile( + `/proc/${String(pid)}/cmdline`, + "utf8", + ); + expect(commandLine.split("\0")[0]).toBe(alternate); + await unlink(alternate); + } + if (pythonMode) { + const commandLine = await readFile( + `/proc/${String(pid)}/cmdline`, + "utf8", + ); + expect( + commandLine.split("\0").slice(0, pythonOptions.length + 2), + ).toEqual([command, ...pythonOptions, helper]); + } + const { process_lineage: lineage, refusal } = + await observeFromSeparateVerifier( + runId, + alternateMode ? [launcher] : [], + procRoot, + ); + expect(lineage.status).toBe("verified"); + expect(lineage.descendants.some((p) => p.pid === pid)).toBe(false); + if (retained) + expect(refusal).toBe( + "Clean Linux runner retained a Hopper demo process", + ); + else expect(refusal).toBeUndefined(); + // Observation must not terminate the fixture, even though it resembles Xvfb. + const { stdout: alive } = await execFileAsync("ps", [ + "-p", + String(pid), + "-o", + "stat=", + ]); + expect(alive.trim()).not.toMatch(/^Z/u); + } finally { + try { + if (orphanPid !== undefined) { + const cleanup = await cleanupOwnedProcessGroup({ + runId, + leaderPid: orphanPid, + processGroupId: orphanPid, + }); + expect(cleanup.cleaned, JSON.stringify(cleanup)).toBe(true); + } + } finally { + reaper.child.stdin?.end("reap"); + await reaper; + if (orphanPid !== undefined) + await expect( + readFile(`/proc/${String(orphanPid)}/stat`, "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + } + } + }, +); diff --git a/tests/boundary/process/providerProcess.test.ts b/tests/boundary/process/providerProcess.test.ts index b283df2a1..6e3b70995 100644 --- a/tests/boundary/process/providerProcess.test.ts +++ b/tests/boundary/process/providerProcess.test.ts @@ -18,7 +18,7 @@ import { spawnOwnedProviderProcess, type ProviderProcessDiagnostic, } from "../../../src/process/ProviderProcess.js"; -import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../../../src/process/WindowsAuthority.js"; +import * as nativeLoader from "../../../src/windows/WindowsNativeLoader.js"; import { spawnProviderProcessFixture, stopProviderProcessFixture, @@ -71,12 +71,25 @@ describe("provider process runtime and wait primitives", () => { }); it("fails closed for a Windows runtime root without native DACL authority", async () => { - await expect( - PrivateRuntimeRoot.create({ platform: "win32" }), - ).rejects.toMatchObject({ - code: "private-runtime-root-authority-unavailable", - message: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, - }); + const authority = vi + .spyOn(nativeLoader, "systemWindowsNativeAuthority") + .mockReturnValue({ + available: false, + failureStage: "self-test", + reason: "Native DACL self-test failed with Win32 error 145", + retryable: true, + }); + try { + await expect( + PrivateRuntimeRoot.create({ platform: "win32" }), + ).rejects.toMatchObject({ + code: "private-runtime-root-authority-unavailable", + message: "Native DACL self-test failed with Win32 error 145", + retryable: true, + }); + } finally { + authority.mockRestore(); + } }); it("classifies external cancellation during startup", async () => { diff --git a/tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts b/tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts new file mode 100644 index 000000000..81a8ffc5e --- /dev/null +++ b/tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts @@ -0,0 +1,58 @@ +import { lstat } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { HopperApplicationLauncher } from "../../../../src/hopper/BridgeLauncher.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; + +it.each(["failed", "cancelled"] as const)( + "removes the token-bearing bootstrap and releases the lease after a %s launch", + async (outcome) => { + const directory = await createTestTempDirectory("rea-bootstrap-cleanup-"); + const controller = new AbortController(); + if (outcome === "cancelled") controller.abort(); + let releases = 0; + const launcher = new HopperApplicationLauncher( + { + environment: {}, + launcherPath: join(directory, "nonexistent-hopper"), + targetPath: join(directory, "test-binary"), + targetKind: "executable", + loaderArgs: [], + bridgeScriptPath: join(directory, "bridge.py"), + launchMode: "native", + }, + { + platform: "darwin", + acquireTargetLease: async () => ({ + acquired: true, + lease: { + release: async () => { + releases++; + }, + }, + }), + }, + ); + const result = await launcher.launch( + { + directory, + socketPath: join(directory, "bridge.sock"), + token: "fixture-token", + runId: "bootstrap-cleanup", + }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (outcome === "cancelled") + expect(result).toMatchObject({ + ok: false, + error: { _tag: "HopperCancelledError" }, + }); + await expect(lstat(join(directory, "bootstrap.py"))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(releases).toBe(1); + }, +); diff --git a/tests/process-global/cleanLinuxHopperRuntime.test.ts b/tests/process-global/cleanLinuxHopperRuntime.test.ts index fdd19e8ee..55fdf0ec9 100644 --- a/tests/process-global/cleanLinuxHopperRuntime.test.ts +++ b/tests/process-global/cleanLinuxHopperRuntime.test.ts @@ -77,7 +77,11 @@ it if (alternateMode) { const processEntry = join(procRoot, String(pid)); await mkdir(processEntry); - await symlink(`/proc/${String(pid)}/exe`, join(processEntry, "exe")); + for (const name of ["cmdline", "comm", "exe"]) + await symlink( + `/proc/${String(pid)}/${name}`, + join(processEntry, name), + ); } const { stdout: coordinates } = await execFileAsync("ps", [ "-p", diff --git a/tests/process-global/filesystemSnapshotFallbackCancellation.test.ts b/tests/process-global/filesystemSnapshotFallbackCancellation.test.ts new file mode 100644 index 000000000..cc984dceb --- /dev/null +++ b/tests/process-global/filesystemSnapshotFallbackCancellation.test.ts @@ -0,0 +1,118 @@ +import { createHash } from "node:crypto"; +import { lstat, writeFile } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, expect, it, vi } from "vitest"; + +const observation = vi.hoisted(() => ({ + path: "", + namedReads: 0, + cancelAt: 0, + controller: undefined as AbortController | undefined, + reason: undefined as Error | undefined, + handles: [] as FileHandle[], +})); + +// Force the capability fallback while keeping actual filesystem IO. These +// cases establish cancellation semantics, not native Windows compatibility. +vi.mock("node:fs", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + constants: { ...original.constants, O_NOFOLLOW: 0 }, + }; +}); + +vi.mock("node:fs/promises", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + open: async (...args: Parameters) => { + const handle = await original.open(...args); + if (args[0] === observation.path) observation.handles.push(handle); + return handle; + }, + lstat: async (...args: Parameters) => { + const stats = await original.lstat(...args); + if ( + args[0] === observation.path && + ++observation.namedReads === observation.cancelAt + ) + observation.controller?.abort(observation.reason); + return stats; + }, + }; +}); + +import { parseProcessScenario } from "../../src/domain/process/processScenario.js"; +import { + hashFile, + snapshotRoots, +} from "../../src/process/capture/FilesystemSnapshot.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +afterEach(() => { + observation.path = ""; + observation.namedReads = 0; + observation.cancelAt = 0; + observation.controller = undefined; + observation.reason = undefined; + observation.handles.length = 0; +}); + +it.each(["hash", "snapshot"] as const)( + "rejects cancellation during the final %s fallback inspection and closes the descriptor", + async (operation) => { + const directory = await createTestTempDirectory("rea-fs-fallback-cancel-"); + const path = join(directory, "observed-file"); + await writeFile(path, "captured bytes"); + const expected = await lstat(path); + const controller = new AbortController(); + const reason = Object.assign( + new Error("cancelled during final named-file inspection"), + { code: "EACCES" }, + ); + observation.path = path; + observation.controller = controller; + observation.reason = reason; + // A root file snapshot first inspects presence and entry metadata, then + // reaches the same two named-file inspections as direct hashing. + observation.cancelAt = operation === "hash" ? 2 : 4; + const result = + operation === "hash" + ? hashFile(path, expected, 1_000, controller.signal) + : snapshotRoots( + parseProcessScenario({ + executable: process.execPath, + working_directory: directory, + filesystem_observation_paths: [path], + }), + controller.signal, + ); + await expect(result).rejects.toBe(reason); + expect(controller.signal.aborted).toBe(true); + expect(observation.handles).toHaveLength(1); + await expect(observation.handles[0]?.stat()).rejects.toMatchObject({ + code: "EBADF", + }); + }, +); + +it("keeps an unchanged fallback digest and closes its descriptor", async () => { + const directory = await createTestTempDirectory("rea-fs-fallback-digest-"); + const path = join(directory, "observed-file"); + const bytes = Buffer.from("captured bytes"); + await writeFile(path, bytes); + const expected = await lstat(path); + observation.path = path; + await expect(hashFile(path, expected, 1_000)).resolves.toEqual({ + state: "hashed", + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + expect(observation.namedReads).toBe(2); + expect(observation.handles).toHaveLength(1); + await expect(observation.handles[0]?.stat()).rejects.toMatchObject({ + code: "EBADF", + }); +}); diff --git a/tests/process-global/hopperLeaseDescriptorPermissions.test.ts b/tests/process-global/hopperLeaseDescriptorPermissions.test.ts new file mode 100644 index 000000000..174097abe --- /dev/null +++ b/tests/process-global/hopperLeaseDescriptorPermissions.test.ts @@ -0,0 +1,79 @@ +import { chmod, lstat, mkdir, mkdtemp, rm } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, expect, it, vi } from "vitest"; + +const replacement = vi.hoisted(() => ({ + selected: "", + retained: "", + handle: undefined as FileHandle | undefined, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + open: async (...args: Parameters) => { + const handle = await original.open(...args); + if (args[0] === replacement.selected) { + replacement.handle = handle; + try { + await original.rename(replacement.selected, replacement.retained); + await original.mkdir(replacement.selected); + await original.chmod(replacement.selected, 0o755); + } catch (cause) { + await handle.close(); + throw cause; + } + } + return handle; + }, + }; +}); + +import { acquireLinuxHopperApplicationLease } from "../../src/hopper/HopperTargetLease.js"; + +afterEach(() => { + replacement.selected = ""; + replacement.retained = ""; + replacement.handle = undefined; +}); + +it.skipIf(process.platform === "win32")( + "secures only the admitted directory descriptor when its path is replaced", + async () => { + // macOS limits Unix-domain socket paths to 104 bytes. Keep the selected + // directory path short enough for the production lease socket beneath it. + const root = await mkdtemp("/tmp/r-"); + const selected = join(root, "selected"); + const retained = join(root, "retained"); + await mkdir(selected); + await chmod(selected, 0o755); + const before = await lstat(selected); + replacement.selected = selected; + replacement.retained = retained; + const acquired = await acquireLinuxHopperApplicationLease({ + directory: selected, + runId: "descriptor-permissions", + }); + if (!acquired.acquired) { + await rm(root, { recursive: true, force: true }); + throw new Error("Expected the fixture lease"); + } + try { + expect((await lstat(retained)).ino).toBe(before.ino); + expect((await lstat(retained)).mode & 0o777).toBe(0o700); + expect((await lstat(selected)).ino).not.toBe(before.ino); + expect((await lstat(selected)).mode & 0o777).toBe(0o755); + await expect(replacement.handle?.stat()).rejects.toMatchObject({ + code: "EBADF", + }); + } finally { + await acquired.lease.release(); + await rm(root, { recursive: true, force: true }); + } + // This asserts descriptor permission ownership only. Later socket + // operations still resolve the path, as the production comment states. + }, +); From ec56c1da04c6c25825721a45e596e8cc237a6dd3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:25:53 +0800 Subject: [PATCH 02/26] fix(setup): preserve registration identity and recovery ownership Keep layered environment choices, current byte-exact recovery snapshots, and package-manager failure details. Report pinned package drift without claiming runner resolution. Retain failed Linux installer cleanup for host-owned retry while preserving the install outcome. Reviewed fixes include #2632, #2769, #2783, #2959, #2979, #2983, #2995, #3009, and #2559. Validation: 98 focused setup/registration tests passed; 45 Linux installer tests passed. Real Linux package installation remains unverified. --- docs/client-configuration.md | 12 +- docs/installation.md | 42 +- install.sh | 5 +- src/android/JadxDoctor.ts | 155 +++++++ .../ClientConfigurationDocument.ts | 3 +- src/application/ClientConfigurationFiles.ts | 56 ++- .../ClientRegistrationEnvironment.ts | 13 +- src/application/ClientRegistrationIdentity.ts | 24 +- src/application/ClientRegistrationStatus.ts | 39 +- src/application/Doctor.test.ts | 29 ++ src/application/Doctor.ts | 2 + src/application/DoctorDiagnostics.ts | 13 +- src/application/DoctorProviderIds.ts | 2 +- src/application/DshCordisPatch.ts | 17 +- src/application/LinuxHopper.test.ts | 115 ++++- src/application/LinuxHopper.ts | 227 +++++++--- src/application/Setup.configuration.test.ts | 46 ++ src/application/Setup.fixture.ts | 4 + src/application/Setup.ts | 1 + src/application/SetupClientConfiguration.ts | 158 ++++--- src/application/SetupHost.ts | 32 +- src/application/SetupInstallFailure.test.ts | 48 +++ src/application/SetupInstallFailure.ts | 48 ++- src/application/SetupPlan.ts | 15 +- src/application/SetupRecoveryBackup.ts | 88 ++++ src/application/SetupSkill.ts | 68 ++- src/application/SetupTypes.ts | 2 + src/application/SupportedClients.ts | 1 + src/application/Uninstall.ts | 15 +- src/doctorRuntime.test.ts | 56 +++ src/doctorRuntime.ts | 2 + .../registrationRuntimeEnvironmentCli.test.ts | 235 +++++++++++ .../android/jadxConfiguration.test.ts | 7 + tests/boundary/android/jadxDoctor.test.ts | 396 ++++++++++++++++++ .../clientConfigurationEnvironment.test.ts | 42 +- ...entConfigurationLayeredEnvironment.test.ts | 219 ++++++++++ .../clientRegistrationStatus.test.ts | 26 ++ .../filesystem/dshClientConfiguration.test.ts | 115 ++++- .../boundary/filesystem/installScript.test.ts | 28 +- .../filesystem/packageClientBackup.test.ts | 47 +++ tests/boundary/filesystem/setupSkill.test.ts | 88 +++- .../zcodeClientConfiguration.test.ts | 90 +++- 42 files changed, 2418 insertions(+), 213 deletions(-) create mode 100644 src/android/JadxDoctor.ts create mode 100644 src/application/SetupRecoveryBackup.ts create mode 100644 src/doctorRuntime.test.ts create mode 100644 tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts create mode 100644 tests/boundary/android/jadxDoctor.test.ts create mode 100644 tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts create mode 100644 tests/boundary/filesystem/packageClientBackup.test.ts diff --git a/docs/client-configuration.md b/docs/client-configuration.md index 409bc7f19..41d239681 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -129,6 +129,12 @@ and Claude Code reports that autocompact is thrashing. To avoid this: The sizes above come from REA's own `tools/list`. A live Claude Code session against DeepSeek's API is unverified. +For a client whose stdio reader rejects the complete catalog frame, add +`"REA_MCP_OUTPUT_SCHEMA_PROFILE": "off"` to REA's `env` object and restart the +client. This omits only advertised output schemas; REA still validates complete +structured results against its canonical contracts. See [MCP output schema +profiles](mcp-contracts.md#output-schemas). + ## Kimi Code Setup registers `rea` in `~/.kimi-code/mcp.json`, or @@ -279,7 +285,11 @@ the first native server into an empty table, it copies the fallback's `mcpServers` entries into the native table so they keep loading, and leaves the fallback file untouched. Existing REA environment overrides are preserved when refreshing its command. A malformed or unreadable fallback file fails setup instead of -being silently suppressed; repair or remove it and rerun. ZCode disables a +being silently suppressed; repair or remove it and rerun. Uninstall removes an +REA-owned `rea` entry from both the native table and the fallback, backing up +each changed file, so an emptied native table cannot load a leftover fallback +registration; other fallback servers, settings and a foreign `rea` entry are +kept. ZCode disables a server with `enable: false` on its entry; doctor reports such a registration as stale, and rerunning setup removes the flag to repair it. Native configuration uses strict JSON; setup refuses comments, trailing commas and a BOM rather than diff --git a/docs/installation.md b/docs/installation.md index 39584b51d..7abefac3d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -64,11 +64,21 @@ preserves a verified `.rea.backup..recovery` copy of the current bytes. Both snapshots remain available; a failed rollback reports the current recovery copy's path and the restore failure. Setup also retains valid environment overrides in REA's existing server entry, -such as `REA_MCP_INPUT_SCHEMA_PROFILE` and `REA_MCP_MAX_RESPONSE_BYTES`. +such as `REA_MCP_INPUT_SCHEMA_PROFILE`, `REA_MCP_OUTPUT_SCHEMA_PROFILE`, and +`REA_MCP_MAX_RESPONSE_BYTES`. Detected provider settings replace only matching environment keys. Setup and doctor report a malformed REA environment map or invalid runtime setting before setup writes files. Repair the named setting in the client configuration; setup does not silently discard the rest of the map. +This includes malformed loader/exclusion JSON arrays and a Ghidra compiler +specification without its required language identifier. Valid overrides keep +their original strings, including empty Hopper arguments. DeepSeek Harness +`!!js` environment expressions remain unevaluated and preserved; registration +alignment does not verify the runtime values those expressions will produce. +For layered OpenCode files, each file's values are checked before merging, while +setting dependencies are checked against the effective merged environment. +Setup preserves each layer's overrides instead of copying inherited settings +into the file it updates. After setup, REA reports which agents, analysis tools, and workflow files passed its final checks. Restart any agent named in the completion message, then begin @@ -334,6 +344,20 @@ document for the same target. REA does not currently identify, focus, or reuse that existing GUI document, and the supported launcher exposes no attach or reuse action for it. +REA currently declines saved Hopper database (`.hop`) targets on macOS because +the native `open database` launch route does not run REA's bridge script. To +analyze that program with REA, select its original executable as the target and +open it through Hopper; this starts the supported bridge route. This does not +recover database-only annotations or analysis changes. Saved-database analysis +on Linux continues to use its supported demo-launch route. + +Hopper lease directories must be readable and owned by the current user. +REA validates and secures an opened directory before reserving a lease. If +directory access fails, the error identifies the path and recovery steps; +REA leaves its permissions unchanged. Check ownership and parent-directory +access, then restore owner-only read, write, and search permissions (`0700`) +on a directory you own before retrying. + On supported Linux distributions, approved setup verifies Hopper's official `.deb`, `.rpm`, or Arch package before invoking the native package manager. REA runs the supported demo build on a private Xvfb display and selects Hopper's @@ -346,6 +370,13 @@ host mount and the rest of `/tmp` remain unchanged; this fallback never invokes `sudo`. `rea doctor --provider hopper --json` reports the selected strategy and both host and effective mount facts. +When the package-manager step fails, setup reports the failing command and its +captured output, so a package conflict is distinguishable from an authorization +failure. On CachyOS, an unversioned compatibility package such as +`zlib-ng-compat` may conflict with Hopper's versioned `zlib` dependency. Resolve +the package conflict yourself or install Hopper's official package manually, +then rerun `rea setup`. + The supported Linux demo forwards additional launches into its existing application, even across private displays. REA therefore reserves one Linux Hopper application per user across cooperating REA processes. A competing @@ -370,8 +401,9 @@ sessions before removing that recovery directory and retrying. On macOS, REA uses `/Applications/Hopper Disassembler.app/Contents/MacOS/hopper` by default. On Linux, it prefers executable `/opt/hopper/bin/Hopper`, then executable -`~/.local/share/rea/hopper/bin/Hopper`; if neither exists, the former remains -the diagnostic fallback. `HOPPER_LAUNCHER_PATH` overrides those choices: +`~/.local/share/rea/hopper/bin/Hopper`, followed by `/opt/hopper-v4/bin/hopper` +and `/usr/bin/hopper`; if none exists, the former remains the diagnostic +fallback. `HOPPER_LAUNCHER_PATH` overrides those choices: ```bash export HOPPER_LAUNCHER_PATH=/absolute/path/to/Hopper @@ -511,7 +543,7 @@ reported as incompatible instead of being admitted from its directory name, and a header that cannot be read or recognized is reported as unknown. REA does not build or install native tools, or change Gatekeeper quarantine settings. -The adapter exposes 25 read-only operations: thirteen inventory/name/search +The adapter exposes 26 read-only operations: fourteen inventory/name/search operations and twelve function-analysis operations. These cover metadata, decompilation, assembly, resolved calls, typed references, xrefs, function dossiers, instructions, recovered data types, measured load mappings, loaded @@ -799,7 +831,7 @@ On a controlled Windows x64 runner, use `npm run verify:ghidra:windows`. The verifier generates a deterministic native PE fixture from source bytes and requires the Windows native authority before opening the provider. The controls are implemented on main; see the -[release boundary](#released-package-and-main). The lane verifies all 25 admitted +[release boundary](#released-package-and-main). The lane verifies all 26 admitted read-only operations, digest identity, and runtime cleanup. The separate `verify:ghidra:windows:package` lane checks the packed CLI/MCP boundary with an ordinary user. It requires the built Windows native bundle and an existing diff --git a/install.sh b/install.sh index c18726611..0da8ef15f 100755 --- a/install.sh +++ b/install.sh @@ -16,6 +16,7 @@ prerelease_identifier="(${numeric_identifier}|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-] build_identifier='[0-9A-Za-z-]+' version_pattern="${numeric_identifier}\\.${numeric_identifier}\\.${numeric_identifier}(-${prerelease_identifier}(\\.${prerelease_identifier})*)?(\\+${build_identifier}(\\.${build_identifier})*)?" version="${REA_VERSION:-}" +version_requested=false dry_run=false start_setup=true verbose=false @@ -81,6 +82,7 @@ main() { --version) (($# >= 2)) || fail "--version requires a value." version="$2" + version_requested=true shift 2 ;; --dry-run) @@ -121,7 +123,7 @@ main() { fail_node_version "$node_version" fi - if [ -n "$version" ]; then + if [ "$version_requested" = true ] || [ -n "$version" ]; then matches "$version" "^${version_pattern}$" || fail "version must be an exact semantic version." else release_json="$(curl -fsSL --proto '=https' --tlsv1.2 --retry 3 -H 'Accept: application/vnd.github+json' "https://api.github.com/repos/$REPOSITORY/releases/latest")" || fail "the latest REA release could not be resolved. Check network access or pass --version VERSION, then retry." @@ -164,6 +166,7 @@ main() { npm "${install_args[@]}" || fail "npm could not install REA. Check registry access and npm permissions, then retry." [ -x "$install_bin" ] || fail "npm completed without installing the rea command. Check the npm global bin directory and PATH, then retry." installed_version="$("$install_bin" --version 2>/dev/null | tr -d '[:space:]')" || fail "the installed REA version could not be read. Reinstall the requested version, then retry." + [ -n "$installed_version" ] || fail "the installed REA version could not be read. Reinstall the requested version, then retry." [ "$installed_version" = "$version" ] || fail "installed version $installed_version does not match $version. Reinstall the requested version, then retry." printf 'REA %s is installed.\n' "$version" diff --git a/src/android/JadxDoctor.ts b/src/android/JadxDoctor.ts new file mode 100644 index 000000000..b357d1e85 --- /dev/null +++ b/src/android/JadxDoctor.ts @@ -0,0 +1,155 @@ +import type { + DoctorProviderCheck, + DoctorProviderInspection, +} from "../application/Doctor.js"; +import type { ProviderAvailability } from "../application/AnalysisProvider.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { + inspectJadxAvailability, + JADX_FAILURE_KINDS, + projectJadxFailureOutcome, + type JadxFailureKind, +} from "./JadxConfiguration.js"; + +/** Project the validated JADX configuration into the doctor contract. */ +export const inspectJadxProvider = async ( + environment: Readonly>, +): Promise => { + const configured = environment.REA_JADX_MCP_JAR !== undefined; + const availability = await inspectJadxAvailability(environment); + // diagnostics is a loosely-typed JsonValue map; guard the published path + // even though the success path always sets jar_path today. + const jarPath = + availability.status === "available" && + typeof availability.diagnostics["jar_path"] === "string" + ? availability.diagnostics["jar_path"] + : undefined; + const available = + availability.status === "available" && jarPath !== undefined; + // Preserve the caller-selected JAR path (it may be a stable symlink) and + // the validated JAVA_HOME selection; the resolved jarPath stays in + // diagnostics as inspection evidence. + const configuredJar = environment.REA_JADX_MCP_JAR; + return { + id: "jadx", + configured, + available, + providerVersion: null, + registrationEnvironment: + jarPath === undefined || configuredJar === undefined + ? {} + : { + REA_JADX_MCP_JAR: configuredJar, + ...optionalEntry("JAVA_HOME", environment.JAVA_HOME), + ...optionalEntry( + "REA_JADX_HEAP_MIB", + environment.REA_JADX_HEAP_MIB, + ), + ...optionalEntry( + "REA_JADX_ACTIVE_PROCESSOR_COUNT", + environment.REA_JADX_ACTIVE_PROCESSOR_COUNT, + ), + }, + checks: [configurationCheck(configured, availability, jarPath)], + }; +}; + +/** + * Publish a caller-selected variable only when explicitly set; never persist + * ambient inheritance. Values are validated before this point. + */ +const optionalEntry = ( + name: string, + value: string | undefined, +): Record => (value === undefined ? {} : { [name]: value }); + +const configurationCheck = ( + configured: boolean, + availability: ProviderAvailability, + jarPath: string | undefined, +): DoctorProviderCheck => { + const code = availability.status === "unavailable" ? availability.code : null; + if (availability.status === "available" && jarPath !== undefined) { + return { + name: "configuration", + ok: true, + code: null, + detail: `JADX MCP bridge JAR validated at ${jarPath}; its path is recorded for agent registrations.`, + remediation: null, + classification: "config_drift", + }; + } + // The failure kind is tagged at the configuration boundary; project its + // remediation and classification here instead of reinterpreting the + // phase/setting evidence. Every failed check keeps a remediation. + const kind = + availability.status === "unavailable" + ? asFailureKind(availability.diagnostics["failure_kind"]) + : null; + const outcome = projectJadxFailureOutcome(kind, configured); + return { + name: "configuration", + ok: false, + code, + detail: + availability.status === "available" + ? "Validated JADX configuration reported no JAR path." + : availability.reason, + remediation: outcome.remediation, + classification: outcome.classification, + ...(kind === "java_probe" && availability.status === "unavailable" + ? { details: probeDetails(availability.diagnostics) } + : kind !== null && + kind.startsWith("windows_native_") && + availability.status === "unavailable" + ? { details: windowsNativeFailureDetails(availability.diagnostics) } + : {}), + }; +}; + +const windowsNativeFailureDetails = ( + diagnostics: Readonly>, +): Record => { + const details: Record = {}; + for (const key of ["failure_stage", "remediation"] as const) { + const value = diagnostics[key]; + if (value !== undefined) details[key] = value; + } + return details; +}; + +/** Validate the loose diagnostics tag against the boundary's failure kinds. */ +const asFailureKind = (value: JsonValue | undefined): JadxFailureKind | null => + typeof value === "string" && + (JADX_FAILURE_KINDS as readonly string[]).includes(value) + ? (value as JadxFailureKind) + : null; + +/** + * Project bounded Java probe metadata for the doctor report. The full + * stdout/stderr can reach the 1 MiB probe budget, so only the bounded + * outcome metadata is carried; the message already references the retained + * output. + */ +const probeDetails = ( + diagnostics: Readonly>, +): Record => { + const details: Record = {}; + for (const key of [ + "java_executable", + "failure", + "output_truncated", + "exit_code", + "process_error_code", + "signal", + "killed", + "output_limit_bytes", + "deadline_ms", + "jdk_compiler", + "java_major", + ] as const) { + const value = diagnostics[key]; + if (value !== undefined) details[key] = value; + } + return details; +}; diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index f7ab8f0e2..896442835 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -167,7 +167,7 @@ export const parseZcodeFallbackConfiguration = ( ? {} : requireObject(document.mcpServers, "ZCode fallback mcpServers"), serversPath: ["mcpServers"], - dialect: "json", + dialect: "zcode_fallback", legacyServers: {}, }; }; @@ -183,6 +183,7 @@ export const parseClientConfiguration = ( if (format === undefined || format === "unsupported") throw new TypeError("client does not have a supported MCP config format"); // DeepSeek Harness stores a Cordis patch list, not a server table. + if (format === "zcode_fallback") return parseZcodeFallbackConfiguration(text); if (format === "dsh") return parseDshCordisDocument(text); const document = parseDocument(text, format); const serversKey = clientConfigurationServersKey(format); diff --git a/src/application/ClientConfigurationFiles.ts b/src/application/ClientConfigurationFiles.ts index fc3c2d4ea..5b1665f29 100644 --- a/src/application/ClientConfigurationFiles.ts +++ b/src/application/ClientConfigurationFiles.ts @@ -11,6 +11,21 @@ import { import type { SetupClient } from "./SupportedClients.js"; import { clientRegistrationEnvironment } from "./ClientRegistrationEnvironment.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import type { ConfigurationError } from "../domain/configurationErrors.js"; + +/** A parsed layer retains its path so rereading the write target replaces only that layer. */ +export interface ClientConfigurationSource extends ClientConfigurationDocument { + readonly sourcePath: string; +} + +const invalidEnvironment = ( + path: string, + error: ConfigurationError, +): ClientConfigurationFileError => ({ + kind: "malformed", + path, + detail: `Configuration ${path} has an invalid REA server environment: ${error.settings.map(({ setting, constraint }) => `${setting}: ${constraint}`).join("; ")}. Repair it before rerunning setup.`, +}); /** A named source failure that setup can explain without losing the bad path. */ export interface ClientConfigurationFileError { @@ -74,9 +89,9 @@ export const readClientConfigurationFiles = async ( client: SetupClient, resources?: ArtifactResourceScope, ): Promise< - Result + Result > => { - const documents: ClientConfigurationDocument[] = []; + const documents: ClientConfigurationSource[] = []; for (const path of client.configPaths ?? [client.configPath]) { const document = await readConfigurationFile( path, @@ -85,16 +100,21 @@ export const readClientConfigurationFiles = async ( ); if (!document.ok) return document; if (document.value !== undefined) { - const environment = clientRegistrationEnvironment(document.value); + const environment = clientRegistrationEnvironment( + document.value, + {}, + "fragment", + ); if (!environment.ok) - return err({ - kind: "malformed", - path, - detail: `Configuration ${path} has an invalid REA server environment: ${environment.error.settings.map(({ setting, constraint }) => `${setting}: ${constraint}`).join("; ")}. Repair it before rerunning setup.`, - }); - documents.push(document.value); + return err(invalidEnvironment(path, environment.error)); + documents.push({ ...document.value, sourcePath: path }); } } + const environment = clientRegistrationEnvironment( + effectiveClientConfiguration(documents), + ); + if (!environment.ok) + return err(invalidEnvironment(client.configPath, environment.error)); return ok(documents); }; @@ -117,8 +137,24 @@ const isObject = (value: unknown): value is Record => /** Project a client's effective user registration without modifying its source files. */ export const effectiveClientConfiguration = ( - documents: readonly ClientConfigurationDocument[], + sources: readonly ClientConfigurationSource[], + target?: { + readonly client: Pick; + readonly document: ClientConfigurationDocument | undefined; + }, ): ClientConfigurationDocument | undefined => { + const documents = + target === undefined + ? sources + : (target.client.configPaths ?? [target.client.configPath]).flatMap( + (path) => { + const document = + path === target.client.configPath + ? target.document + : sources.find(({ sourcePath }) => sourcePath === path); + return document === undefined ? [] : [document]; + }, + ); const last = documents.at(-1); if (last === undefined) return undefined; if (documents.length === 1) return last; diff --git a/src/application/ClientRegistrationEnvironment.ts b/src/application/ClientRegistrationEnvironment.ts index 86e95404d..efe6838a4 100644 --- a/src/application/ClientRegistrationEnvironment.ts +++ b/src/application/ClientRegistrationEnvironment.ts @@ -1,6 +1,9 @@ import { z } from "zod"; -import { parseEnvironment } from "../config/environment.js"; +import { + parseEnvironment, + parseEnvironmentValues, +} from "../config/environment.js"; import { ConfigurationError } from "../domain/configurationErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -15,9 +18,10 @@ import { dshDocument } from "./DshCordisPatch.js"; export const clientRegistrationEnvironment = ( parsed: ClientConfigurationDocument | undefined, overrides: Readonly> = {}, + validation: "fragment" | "effective" = "effective", ): Result>, ConfigurationError> => { if (parsed?.dialect === "dsh") - return dshDocument(parsed.document).environment(overrides); + return dshDocument(parsed.document).environment(overrides, validation); const key = parsed?.dialect === "opencode" || parsed?.dialect === "opencode_v2" ? "environment" @@ -46,7 +50,10 @@ export const clientRegistrationEnvironment = ( // Validate known settings with the runtime's parser, but retain the original // string overrides, including unknown client-specific keys and no defaults. const environment = { ...existing.data, ...overrides }; - const validated = parseEnvironment(environment); + const validated = + validation === "fragment" + ? parseEnvironmentValues(environment) + : parseEnvironment(environment); if (!validated.ok) return validated; return ok( Object.fromEntries( diff --git a/src/application/ClientRegistrationIdentity.ts b/src/application/ClientRegistrationIdentity.ts index 6e7744698..bdc94297e 100644 --- a/src/application/ClientRegistrationIdentity.ts +++ b/src/application/ClientRegistrationIdentity.ts @@ -76,18 +76,24 @@ const unwrapCmdCommand = (command: readonly string[]): readonly string[] => { return []; }; -const packageRunnerReference = ( +/** Preserve the package argument from a supported package-runner command. */ +export const clientRegistrationPackageReference = ( command: readonly string[], ): string | undefined => { + const arguments_ = unwrapCmdCommand(command); if ( - command.length === 4 && - command[0] === "npx" && - command[1] === "-y" && - command[3] === "mcp" + arguments_.length === 4 && + arguments_[0] === "npx" && + arguments_[1] === "-y" && + arguments_[3] === "mcp" ) - return command[2]; - if (command.length === 3 && command[0] === "bunx" && command[2] === "mcp") - return command[1]; + return arguments_[2]; + if ( + arguments_.length === 3 && + arguments_[0] === "bunx" && + arguments_[2] === "mcp" + ) + return arguments_[1]; return undefined; }; @@ -129,7 +135,7 @@ export const isOwnedClientRegistrationCommand = ( return true; } - const packageReference = packageRunnerReference(unwrapCmdCommand(command)); + const packageReference = clientRegistrationPackageReference(command); return ( packageReference !== undefined && isOwnedPackageReference(packageReference) ); diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index 56a3e81ba..878abd0d5 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -13,10 +13,12 @@ import { access } from "node:fs/promises"; import { resolve } from "node:path"; import { z } from "zod"; +import { valid } from "semver"; import { PRODUCT_IDENTITY } from "../identity.js"; import { MCP_STARTUP_POLICY } from "../mcpStartupPolicy.js"; import { + clientRegistrationPackageReference, isOwnedClientRegistrationCommand, packageRunnerRegistrationCommand, } from "./ClientRegistrationIdentity.js"; @@ -33,6 +35,8 @@ import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.j interface ClientRegistrationStatusBase { readonly client: string; readonly config_path: string; + /** Exact package reference requested by npx/bunx, when registration uses one. */ + readonly package_reference?: string; } type RegistrationCommand = readonly [string, ...string[]]; @@ -201,12 +205,18 @@ export const readClientRegistrationInspections = async ( registration.command, ...registration.args, ]; + const packageReference = clientRegistrationPackageReference(command); const activation = registrationActivation(registration, client, parsed); + const packageVersion = exactRegisteredPackageVersion(packageReference); + const packageVersionDrift = + packageVersion !== undefined && + packageVersion !== PRODUCT_IDENTITY.packageVersion; record( configuredStatus( client, command, activation === "enabled" && + !packageVersionDrift && registrationAligned( registration, client, @@ -217,7 +227,11 @@ export const readClientRegistrationInspections = async ( policyBlock === undefined ? "aligned" : "stale", - policyBlock, + policyBlock ?? + (packageVersionDrift + ? packageVersionRemediation(packageReference ?? "", client) + : undefined), + packageReference, ), activation, ); @@ -378,12 +392,16 @@ const configuredStatus = ( command: RegistrationCommand, state: "aligned" | "stale", policyRemediation?: string, + packageReference?: string, ): ClientRegistrationStatus => state === "aligned" ? { client: client.name, config_path: client.configPath, command, + ...(packageReference === undefined + ? {} + : { package_reference: packageReference }), state, remediation: null, } @@ -391,10 +409,29 @@ const configuredStatus = ( client: client.name, config_path: client.configPath, command, + ...(packageReference === undefined + ? {} + : { package_reference: packageReference }), state, remediation: policyRemediation ?? remediation(client), }; +const exactRegisteredPackageVersion = ( + packageReference: string | undefined, +): string | undefined => { + if (packageReference === undefined) return undefined; + const prefix = `${PRODUCT_IDENTITY.packageName}@`; + if (!packageReference.startsWith(prefix)) return undefined; + const version = packageReference.slice(prefix.length); + return valid(version) === version ? version : undefined; +}; + +const packageVersionRemediation = ( + packageReference: string, + client: SetupClient, +): string => + `This registration requests ${packageReference}, while the current REA CLI package is ${PRODUCT_IDENTITY.packageVersion}. REA does not resolve package-runner requests during doctor; update the registration with rea setup --client ${client.name}, then restart the client.`; + const unavailableStatus = ( client: SetupClient, state: "missing" | "invalid", diff --git a/src/application/Doctor.test.ts b/src/application/Doctor.test.ts index a39f81e35..63e0eef19 100644 --- a/src/application/Doctor.test.ts +++ b/src/application/Doctor.test.ts @@ -588,6 +588,9 @@ describe("doctor Linux and Hopper discovery", () => { result.checks.find(({ name }) => name === "hopper-version")?.remediation, ).toContain("Unset or update HOPPER_LAUNCHER_PATH"); }); +}); + +describe("doctor installed Hopper discovery", () => { it("detects a Homebrew cask installed outside /Applications", async () => { const path = "/opt/homebrew/Caskroom/hopper/Hopper Disassembler.app/Contents/MacOS/hopper"; @@ -612,7 +615,9 @@ describe("doctor Linux and Hopper discovery", () => { ); expect(result.hopperPath).toBe(path); }); +}); +describe("doctor target admission on Linux", () => { it("uses shared app target validation when a target is supplied", async () => { const result = await runDoctor( "/Applications/Notes.app", @@ -652,6 +657,30 @@ describe("doctor Linux and Hopper discovery", () => { }); }); +describe("doctor standard Linux Hopper paths", () => { + it.each(["/opt/hopper-v4/bin/hopper", "/usr/bin/hopper"])( + "detects Hopper at the standard path %s", + async (path) => { + const result = await runDoctor( + undefined, + host({ + platform: "linux", + macosVersion: () => Promise.resolve(undefined), + linuxDistribution: () => + Promise.resolve({ + id: "ubuntu", + versionId: "24.04", + packageFamily: "deb", + hopperSetupSupported: true, + }), + executable: (candidate) => Promise.resolve(candidate === path), + }), + ); + expect(result.hopperPath).toBe(path); + }, + ); +}); + describe("doctor target admission diagnostics", () => { it("preserves route-specific target diagnostics from the host", async () => { const targetCheck = { diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 3e8659dca..24adc5500 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -48,6 +48,7 @@ export interface DoctorCheck { readonly classification: | "healthy" | "missing_dependency" + | "runtime_failure" | "unsupported_host" | "unsupported_target" | "missing_analysis_engine" @@ -65,6 +66,7 @@ export interface DoctorProviderCheck { readonly detail: string; readonly remediation: string | null; readonly classification: Exclude; + readonly details?: Readonly>; } /** Read-only provider inspection supplied by an outer composition adapter. */ diff --git a/src/application/DoctorDiagnostics.ts b/src/application/DoctorDiagnostics.ts index 1d8a5e621..069525a33 100644 --- a/src/application/DoctorDiagnostics.ts +++ b/src/application/DoctorDiagnostics.ts @@ -2,7 +2,11 @@ import { SUPPORTED_NODE_VERSION_PROSE, supportsNodeVersion, } from "../domain/runtimeVersion.js"; -import { linuxHopperLauncherPath, supportsReaHost } from "./LinuxHopper.js"; +import { + linuxHopperLauncherCandidates, + systemLinuxHopperLauncherCandidates, + supportsReaHost, +} from "./LinuxHopper.js"; import type { DoctorCheck, DoctorHost, @@ -11,7 +15,6 @@ import type { const DEFAULT_HOPPER = "/Applications/Hopper Disassembler.app/Contents/MacOS/hopper"; -const SYSTEM_LINUX_HOPPER = "/opt/hopper/bin/Hopper"; /** Ordered environment diagnostics consumed by the Doctor report assembler. */ export interface DoctorDiagnostics { @@ -148,8 +151,9 @@ const findHopper = async (host: DoctorHost): Promise => { host.configuredHopperPath === undefined ? [ DEFAULT_HOPPER, - SYSTEM_LINUX_HOPPER, - linuxHopperLauncherPath(host.homeDirectory), + ...(host.platform === "linux" + ? linuxHopperLauncherCandidates(host.homeDirectory) + : [...systemLinuxHopperLauncherCandidates]), ...(await host.manualHopperPaths()), ] : [host.configuredHopperPath]; @@ -280,6 +284,7 @@ const providerDoctorChecks = ( ...(candidate.remediation === null ? {} : { remediation: candidate.remediation }), + ...(candidate.details === undefined ? {} : { details: candidate.details }), })); const unsupportedHopperRemediation = (configured: boolean): string => diff --git a/src/application/DoctorProviderIds.ts b/src/application/DoctorProviderIds.ts index 4b7f40331..af31f4c68 100644 --- a/src/application/DoctorProviderIds.ts +++ b/src/application/DoctorProviderIds.ts @@ -1,2 +1,2 @@ /** Deep-analysis providers whose readiness can be selected explicitly by doctor. */ -export const DOCTOR_PROVIDER_IDS = ["hopper", "ghidra", "ida"] as const; +export const DOCTOR_PROVIDER_IDS = ["hopper", "ghidra", "ida", "jadx"] as const; diff --git a/src/application/DshCordisPatch.ts b/src/application/DshCordisPatch.ts index 0e34da23e..fe88e5f16 100644 --- a/src/application/DshCordisPatch.ts +++ b/src/application/DshCordisPatch.ts @@ -1,5 +1,8 @@ import { isMap, isSeq, type Node, type YAMLMap } from "yaml"; -import { parseEnvironment } from "../config/environment.js"; +import { + parseEnvironment, + parseEnvironmentValues, +} from "../config/environment.js"; import { ConfigurationError } from "../domain/configurationErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import type { ClientConfigurationDocument } from "./ClientConfigurationDocument.js"; @@ -181,6 +184,7 @@ export class DshCordisDocument { /** Validate literals without treating expression source as an environment value. */ environment( overrides: Readonly> = {}, + validation: "fragment" | "effective" = "effective", ): Result>, ConfigurationError> { const config = this.#field("config"); if (config instanceof CordisExpression) @@ -215,8 +219,12 @@ export class DshCordisDocument { new ConfigurationError("Cordis server environment must be a mapping"), ); const literals: Record = {}; + const unresolvedSettings = new Set(); for (const [key, value] of Object.entries(raw ?? {})) { - if (value instanceof CordisExpression) continue; + if (value instanceof CordisExpression) { + if (overrides[key] === undefined) unresolvedSettings.add(key); + continue; + } if (typeof value !== "string") return err( new ConfigurationError("Invalid Cordis server environment", { @@ -232,7 +240,10 @@ export class DshCordisDocument { literals[key] = value; } const environment = { ...literals, ...overrides }; - const validated = parseEnvironment(environment); + const validated = + validation === "fragment" + ? parseEnvironmentValues(environment) + : parseEnvironment(environment, unresolvedSettings); return validated.ok ? ok(environment) : validated; } diff --git a/src/application/LinuxHopper.test.ts b/src/application/LinuxHopper.test.ts index 1c37a03ea..3819ecfed 100644 --- a/src/application/LinuxHopper.test.ts +++ b/src/application/LinuxHopper.test.ts @@ -2,19 +2,36 @@ import { createHash } from "node:crypto"; import { describe, expect, it } from "vitest"; import { + formatCommandForDisplay, installLinuxHopper, linuxHopperLauncherDigestSupported, linuxPackageManagerCommands, linuxSharedLibrariesAvailable, parseLinuxDistribution, supportsReaHost, + LinuxHopperTemporaryCleanupOwner, type LinuxDistribution, type LinuxHopperDownload, type LinuxHopperInstallHost, type LinuxHopperLauncherStatus, type LinuxPackageFamily, + type LinuxPackageInstallFailure, + type LinuxPackageInstallOutcome, } from "./LinuxHopper.js"; +describe("Linux package-manager command display", () => { + it("preserves argument boundaries for paths with spaces and quotes", () => { + expect( + formatCommandForDisplay("pkexec", [ + "pacman", + "-U", + "/tmp/rea test/hopper.arch", + "apostrophe'path", + ]), + ).toBe('pkexec pacman -U "/tmp/rea test/hopper.arch" "apostrophe\'path"'); + }); +}); + class RecordingLinuxHost implements LinuxHopperInstallHost { distributionValue: LinuxDistribution | undefined = { id: "ubuntu", @@ -25,6 +42,14 @@ class RecordingLinuxHost implements LinuxHopperInstallHost { archive = new Uint8Array([1, 2, 3]); archiveOk = true; installSucceeds = true; + cleanupFails = false; + packageFailure: LinuxPackageInstallFailure = { + command: "pacman -U --noconfirm /tmp/rea test/hopper.arch", + exitCode: 1, + stdout: "", + stderr: "error: failed to prepare transaction", + outputTruncated: false, + }; launcherStatusValue: LinuxHopperLauncherStatus = "ready"; downloads: string[] = []; installed: Array<{ family: LinuxPackageFamily; archive: string }> = []; @@ -42,15 +67,20 @@ class RecordingLinuxHost implements LinuxHopperInstallHost { installPackage = ( family: LinuxPackageFamily, archive: string, - ): Promise => { + ): Promise => { this.installed.push({ family, archive }); - return Promise.resolve(this.installSucceeds); + const outcome: LinuxPackageInstallOutcome = this.installSucceeds + ? { status: "installed" } + : { status: "failed", failure: this.packageFailure }; + return Promise.resolve(outcome); }; launcherStatus = (): Promise => Promise.resolve(this.launcherStatusValue); cleanup = (path: string): Promise => { this.cleaned.push(path); - return Promise.resolve(); + return this.cleanupFails + ? Promise.reject(new Error("temporary path is busy")) + : Promise.resolve(); }; } @@ -183,7 +213,9 @@ describe("Linux Hopper installation", () => { ); }, ); +}); +describe("Linux Hopper package installation", () => { it.each(["deb", "rpm", "arch"] as const)( "downloads, verifies, installs, reads back, and cleans a %s package", async (family) => { @@ -204,7 +236,9 @@ describe("Linux Hopper installation", () => { expect(host.cleaned).toEqual(["/tmp/rea test"]); }, ); +}); +describe("Linux Hopper installation failure classification", () => { it.each([ ["unsupported host", "unsupported_host"], ["package request failure", "download"], @@ -232,11 +266,65 @@ describe("Linux Hopper installation", () => { expect(await installFixture(host)).toEqual({ status: "failed", reason, + ...(scenario === "package manager failure" + ? { packageFailure: host.packageFailure } + : {}), }); if (host.installed.length > 0) expect(host.cleaned).toEqual(["/tmp/rea test"]); }); +}); +describe("Linux Hopper cleanup and failure details", () => { + it("preserves the failing package-manager command and its output", async () => { + const host = new RecordingLinuxHost(); + host.installSucceeds = false; + host.packageFailure = { + command: "pkexec pacman -U --noconfirm /tmp/rea test/hopper.arch", + exitCode: 1, + stdout: "loading packages...", + stderr: + "\u001b[31merror: unresolvable package conflicts detected\u001b[0m", + outputTruncated: false, + }; + expect(await installFixture(host)).toEqual({ + status: "failed", + reason: "authorization_or_package_manager", + packageFailure: host.packageFailure, + }); + expect(host.cleaned).toEqual(["/tmp/rea test"]); + }); + + it("preserves a successful install result when temporary cleanup fails", async () => { + const host = new RecordingLinuxHost(); + host.cleanupFails = true; + expect(await installFixture(host)).toEqual({ + status: "installed", + launcherPath: "/opt/hopper/bin/Hopper", + cleanupFailure: { + path: "/tmp/rea test", + reason: "temporary path is busy", + }, + }); + }); + + it("preserves an install failure when temporary cleanup also fails", async () => { + const host = new RecordingLinuxHost(); + host.installSucceeds = false; + host.cleanupFails = true; + expect(await installFixture(host)).toMatchObject({ + status: "failed", + reason: "authorization_or_package_manager", + packageFailure: host.packageFailure, + cleanupFailure: { + path: "/tmp/rea test", + reason: "temporary path is busy", + }, + }); + }); +}); + +describe("Linux Hopper cancellation", () => { it("cancels before network or mutation", async () => { const host = new RecordingLinuxHost(); const controller = new AbortController(); @@ -252,6 +340,27 @@ describe("Linux Hopper installation", () => { }); }); +describe("Linux Hopper temporary cleanup ownership", () => { + it("retains a failed cleanup for a later successful retry", async () => { + const owner = new LinuxHopperTemporaryCleanupOwner(); + owner.retain({ path: "/tmp/rea-hopper", reason: "busy" }); + let attempts = 0; + + expect( + await owner.retry(async () => { + attempts += 1; + throw new Error("still busy"); + }), + ).toContain("/tmp/rea-hopper"); + expect( + await owner.retry(async () => { + attempts += 1; + }), + ).toBeUndefined(); + expect(attempts).toBe(2); + }); +}); + function releasesFor( bytes: Uint8Array, ): Record< diff --git a/src/application/LinuxHopper.ts b/src/application/LinuxHopper.ts index 765f31d29..b3503ae7e 100644 --- a/src/application/LinuxHopper.ts +++ b/src/application/LinuxHopper.ts @@ -1,18 +1,18 @@ import { createHash } from "node:crypto"; -import { execFile } from "node:child_process"; import { createReadStream } from "node:fs"; import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { promisify } from "node:util"; +import { execFileBoundedOutput } from "../process/ExecFileBoundedOutput.js"; import { execFileOutput } from "../process/ExecFileOutput.js"; import { downloadHopperPackage, HOPPER_DOWNLOAD_BYTES, } from "./HopperDownload.js"; -const execFileAsync = promisify(execFile); +/** Package-manager output retained per stream without stopping the command. */ +const PACKAGE_MANAGER_OUTPUT_BYTES = 64 * 1024; const DOWNLOAD_PREFIX = "https://www.hopperapp.com:443/downloader/public/"; const SUPPORTED_HOPPER_SHA256 = new Set([ "0294ced141cc373468ee22d8343e7dac41980cb05a937994ca81c9f09afe7ded", @@ -67,7 +67,7 @@ export const linuxHopperInstallDisclosure = ( family, ``, isRoot, - ).map(({ executable, args }) => `${executable} ${args.join(" ")}`), + ).map(({ executable, args }) => formatCommandForDisplay(executable, args)), }; }; @@ -111,6 +111,25 @@ export interface LinuxHopperDownload { readonly bytes: Uint8Array; } +/** One failed package-manager command, with the output that explains it. */ +export interface LinuxPackageInstallFailure { + /** The executed executable and arguments, joined for display. */ + readonly command: string; + readonly exitCode: string | number | null; + readonly stdout: string; + readonly stderr: string; + /** Whether the command produced more output than was retained. */ + readonly outputTruncated: boolean; +} + +/** Whether the package-manager step installed the verified package. */ +export type LinuxPackageInstallOutcome = + | { readonly status: "installed" } + | { + readonly status: "failed"; + readonly failure: LinuxPackageInstallFailure; + }; + /** External effects required to install the official Linux package. */ export interface LinuxHopperInstallHost { distribution(): Promise; @@ -120,15 +139,20 @@ export interface LinuxHopperInstallHost { ): Promise; createTemporaryDirectory(): Promise; writeArchive(path: string, bytes: Uint8Array): Promise; - installPackage(family: LinuxPackageFamily, archive: string): Promise; + installPackage( + family: LinuxPackageFamily, + archive: string, + ): Promise; launcherStatus(path: string): Promise; cleanup(path: string): Promise; } /** Structured outcome for every expected Linux installation failure. */ export type LinuxHopperInstallResult = - | { readonly status: "installed"; readonly launcherPath: string } - | { + | ({ readonly status: "installed"; readonly launcherPath: string } & { + readonly cleanupFailure?: LinuxHopperTemporaryCleanupFailure; + }) + | ({ readonly status: "failed"; readonly reason: | "unsupported_host" @@ -140,7 +164,49 @@ export type LinuxHopperInstallResult = | "runtime_dependencies" | "unsupported_hopper_build" | "cancelled"; - }; + /** + * The failing package-manager command and its output. Present only when + * the package-manager step itself failed, so setup can report what the + * package manager said instead of assuming an authorization problem. + */ + readonly packageFailure?: LinuxPackageInstallFailure; + } & { readonly cleanupFailure?: LinuxHopperTemporaryCleanupFailure }); + +/** Failed removal of an owned temporary install directory. */ +export interface LinuxHopperTemporaryCleanupFailure { + readonly path: string; + readonly reason: string; +} + +/** Own failed temporary cleanup until a later retry succeeds. */ +export class LinuxHopperTemporaryCleanupOwner { + private readonly pending = new Map< + string, + LinuxHopperTemporaryCleanupFailure + >(); + + retain(failure: LinuxHopperTemporaryCleanupFailure): void { + this.pending.set(failure.path, failure); + } + + async retry( + cleanup: (path: string) => Promise, + ): Promise { + const failures: string[] = []; + for (const path of this.pending.keys()) { + try { + await cleanup(path); + this.pending.delete(path); + } catch (cause: unknown) { + const reason = cause instanceof Error ? cause.message : String(cause); + failures.push( + `Linux Hopper temporary cleanup failed for ${path}: ${reason}`, + ); + } + } + return failures.length === 0 ? undefined : failures.join("; "); + } +} export type LinuxHopperLauncherStatus = | "ready" @@ -215,33 +281,57 @@ export const installLinuxHopper = async ( if (distribution?.hopperSetupSupported !== true) return { status: "failed", reason: "unsupported_host" }; let temporary: string | undefined; + let result: LinuxHopperInstallResult; try { - const release = (options.releases ?? SUPPORTED_RELEASES)[ - distribution.packageFamily - ]; - const archiveDownload = await host.download(release.filename, options); - if (!archiveDownload.ok) return { status: "failed", reason: "download" }; - if (!packageIntegrityMatches(archiveDownload.bytes, release)) - return { status: "failed", reason: "integrity" }; - temporary = await host.createTemporaryDirectory(); - const archive = join(temporary, `hopper.${distribution.packageFamily}`); - await host.writeArchive(archive, archiveDownload.bytes); - if (!(await host.installPackage(distribution.packageFamily, archive))) - return { status: "failed", reason: "authorization_or_package_manager" }; - const launcherPath = "/opt/hopper/bin/Hopper"; - const launcherStatus = await host.launcherStatus(launcherPath); - if (launcherStatus !== "ready") - return { - status: "failed", - reason: - launcherStatus === "missing" ? "launcher_missing" : launcherStatus, - }; - return { status: "installed", launcherPath }; + result = await (async (): Promise => { + const release = (options.releases ?? SUPPORTED_RELEASES)[ + distribution.packageFamily + ]; + const archiveDownload = await host.download(release.filename, options); + if (!archiveDownload.ok) return { status: "failed", reason: "download" }; + if (!packageIntegrityMatches(archiveDownload.bytes, release)) + return { status: "failed", reason: "integrity" }; + temporary = await host.createTemporaryDirectory(); + const archive = join(temporary, `hopper.${distribution.packageFamily}`); + await host.writeArchive(archive, archiveDownload.bytes); + const install = await host.installPackage( + distribution.packageFamily, + archive, + ); + if (install.status === "failed") + return { + status: "failed", + reason: "authorization_or_package_manager", + packageFailure: install.failure, + }; + const launcherPath = "/opt/hopper/bin/Hopper"; + const launcherStatus = await host.launcherStatus(launcherPath); + if (launcherStatus !== "ready") + return { + status: "failed", + reason: + launcherStatus === "missing" ? "launcher_missing" : launcherStatus, + }; + return { status: "installed", launcherPath }; + })(); } catch (cause: unknown) { - if (isAbortCause(cause)) return { status: "failed", reason: "cancelled" }; - return { status: "failed", reason: "download" }; - } finally { - if (temporary !== undefined) await host.cleanup(temporary); + result = { + status: "failed", + reason: isAbortCause(cause) ? "cancelled" : "download", + }; + } + if (temporary === undefined) return result; + try { + await host.cleanup(temporary); + return result; + } catch (cause: unknown) { + return { + ...result, + cleanupFailure: { + path: temporary, + reason: cause instanceof Error ? cause.message : String(cause), + }, + }; } }; @@ -249,6 +339,24 @@ export const installLinuxHopper = async ( export const linuxHopperLauncherPath = (home: string): string => join(home, ".local/share/rea/hopper/bin/Hopper"); +/** REA's managed Linux install path, preferred when no explicit launcher is set. */ +export const REA_MANAGED_LINUX_HOPPER_LAUNCHER = "/opt/hopper/bin/Hopper"; + +/** Other system-wide Hopper paths used by package installations. */ +export const systemLinuxHopperLauncherCandidates: readonly string[] = [ + "/opt/hopper-v4/bin/hopper", + "/usr/bin/hopper", +]; + +/** Probe REA-managed and user-local installs before arbitrary system builds. */ +export const linuxHopperLauncherCandidates = ( + home: string, +): readonly string[] => [ + REA_MANAGED_LINUX_HOPPER_LAUNCHER, + linuxHopperLauncherPath(home), + ...systemLinuxHopperLauncherCandidates, +]; + /** Compose package-manager and linker checks from the selected caller environment. */ export const systemLinuxHopperInstallHost = ( environment: Readonly, @@ -304,24 +412,49 @@ const installSystemPackage = async ( family: LinuxPackageFamily, archive: string, environment: Readonly, -): Promise => { - try { - for (const command of linuxPackageManagerCommands( - family, - archive, - process.getuid?.() === 0, - )) - await execFileAsync(command.executable, command.args, { +): Promise => { + for (const command of linuxPackageManagerCommands( + family, + archive, + process.getuid?.() === 0, + )) { + const output = await execFileBoundedOutput( + command.executable, + command.args, + { env: environment, - }); - return true; - } catch (cause: unknown) { - // Package-manager failure is reported by the false return. - void cause; - return false; + limit: PACKAGE_MANAGER_OUTPUT_BYTES, + }, + ); + if (output.exitCode !== 0) { + return { + status: "failed", + failure: { + command: formatCommandForDisplay(command.executable, command.args), + exitCode: output.exitCode, + // The streams stay separate; their original interleaved order + // is not recoverable after capture. + stdout: output.stdout, + stderr: output.stderr, + outputTruncated: output.outputTruncated, + }, + }; + } } + return { status: "installed" }; }; +/** Preserve argument boundaries in a command shown to the user. */ +export const formatCommandForDisplay = ( + executable: string, + args: readonly string[], +): string => + [executable, ...args] + .map((part) => + /^[A-Za-z0-9_./:@%+=,-]+$/u.test(part) ? part : JSON.stringify(part), + ) + .join(" "); + /** Select the native package manager and authorization boundary without shell evaluation. */ export const linuxPackageManagerCommands = ( family: LinuxPackageFamily, diff --git a/src/application/Setup.configuration.test.ts b/src/application/Setup.configuration.test.ts index a3f46ed6f..8fe8e9ee8 100644 --- a/src/application/Setup.configuration.test.ts +++ b/src/application/Setup.configuration.test.ts @@ -4,6 +4,7 @@ import { join, resolve } from "node:path"; import { FakeSetupHost, options } from "./Setup.fixture.js"; import { runSetup } from "./Setup.js"; import { setupRegistrationCommand } from "./SetupHost.js"; +import { discoverSetupState } from "./SetupPlan.js"; import type { MsixPackageView } from "./MsixPackageStorage.js"; import type { SetupClient } from "./SupportedClients.js"; @@ -191,6 +192,51 @@ describe("setup workflow", () => { }); }); +describe("configured Linux Hopper selection", () => { + it("keeps an explicitly configured launcher even when demo checks fail", async () => { + const host = new FakeSetupHost("linux"); + host.distribution = { + id: "ubuntu", + versionId: "24.04", + packageFamily: "deb", + hopperSetupSupported: true, + }; + host.hopper = "/opt/hopper/bin/Hopper"; + host.hopperLauncherConfigured = true; + host.linuxDemoRuntimeMissing = true; + const doctor = await host.doctor(); + const discovery = await discoverSetupState({ + host, + providerEnvironment: { HOPPER_LAUNCHER_PATH: host.hopper }, + forceHopperInstall: false, + proposeHopper: true, + hopperLauncherConfigured: true, + initialDoctor: doctor, + }); + expect(discovery.installHopper).toBe(false); + }); + + it("still offers repair for an unconfigured REA managed install", async () => { + const host = new FakeSetupHost("linux"); + host.distribution = { + id: "ubuntu", + versionId: "24.04", + packageFamily: "deb", + hopperSetupSupported: true, + }; + host.hopper = "/opt/hopper/bin/Hopper"; + host.unsupportedHopperVersion = true; + const discovery = await discoverSetupState({ + host, + providerEnvironment: { HOPPER_LAUNCHER_PATH: host.hopper }, + forceHopperInstall: false, + proposeHopper: true, + initialDoctor: await host.doctor(), + }); + expect(discovery.installHopper).toBe(true); + }); +}); + describe("setup workflow action selection", () => { it("executes only actions selected by the interactive adapter", async () => { const host = new FakeSetupHost(); diff --git a/src/application/Setup.fixture.ts b/src/application/Setup.fixture.ts index 3c7c21d35..ef7f37d3a 100644 --- a/src/application/Setup.fixture.ts +++ b/src/application/Setup.fixture.ts @@ -52,6 +52,7 @@ export class FakeSetupHost implements SetupHost { scopedDoctorHealthy: boolean | undefined; linuxDemoRuntimeMissing = false; unsupportedHopperVersion = false; + hopperLauncherConfigured = false; constructor(platform: NodeJS.Platform = "darwin") { this.platform = platform; @@ -64,6 +65,9 @@ export class FakeSetupHost implements SetupHost { Promise.resolve(this.distribution); initialSetupState = async (scope?: DoctorScope) => ({ ...(this.hopper === undefined ? {} : { hopperPath: this.hopper }), + ...(this.hopperLauncherConfigured + ? { hopperLauncherConfigured: true } + : {}), providerEnvironment: { ...(this.hopper === undefined ? {} diff --git a/src/application/Setup.ts b/src/application/Setup.ts index 7244824db..2b2fb5d45 100644 --- a/src/application/Setup.ts +++ b/src/application/Setup.ts @@ -107,6 +107,7 @@ const runSetupWorkflow = async ( host, providerEnvironment, forceHopperInstall: options.installHopper, + hopperLauncherConfigured: initialState.hopperLauncherConfigured === true, proposeHopper: options.proposeHopper ?? (confirm !== undefined && !options.structured), initialDoctor: initialState.doctor, diff --git a/src/application/SetupClientConfiguration.ts b/src/application/SetupClientConfiguration.ts index 90f4bf4ca..0af9702ab 100644 --- a/src/application/SetupClientConfiguration.ts +++ b/src/application/SetupClientConfiguration.ts @@ -1,7 +1,9 @@ import { readClientPolicyBlock } from "./GeminiClientSettings.js"; import { readClientConfigurationFiles, + effectiveClientConfiguration, readConfigurationText, + type ClientConfigurationSource, } from "./ClientConfigurationFiles.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -18,9 +20,7 @@ import { type ClientConfigurationDocument, } from "./ClientConfigurationDocument.js"; import { dshDocument, DshCordisDocument } from "./DshCordisPatch.js"; -import { constants as fsConstants } from "node:fs"; -import { createHash } from "node:crypto"; -import { copyFile, mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, rm, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import writeFileAtomic from "write-file-atomic"; import { clientRegistrationEnvironment } from "./ClientRegistrationEnvironment.js"; @@ -31,10 +31,13 @@ import { MCP_STARTUP_POLICY } from "../mcpStartupPolicy.js"; import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js"; import { packageReadTargetChanged } from "./MsixPackageStorage.js"; import { - readRegularFile, readRegularFileText, RegularFileCleanupFailure, } from "./RegularFileRead.js"; +import { + preserveRecoveryBackup, + selectCurrentRecoveryBackup, +} from "./SetupRecoveryBackup.js"; import type { ClientConfigurationFileError } from "./ClientConfigurationFiles.js"; import type { ClientConfigurationInspection, @@ -90,7 +93,9 @@ const configureClientDocument = async ( throwCleanupFailure(files.error); return { status: "failed", reason: "readback" }; } - const existing = files.value.at(-1); + const existing = files.value.find( + ({ sourcePath }) => sourcePath === client.configPath, + ); const editabilityError = existing?.dialect === "dsh" ? dshDocument(existing.document).editabilityError @@ -104,7 +109,8 @@ const configureClientDocument = async ( client, environment, command, - files.value.at(-1), + existing, + files.value, ), ); if (!policy.ok) { @@ -137,7 +143,13 @@ const configureClientDocument = async ( const policy = await readClientPolicyBlock( client, parsed, - clientConfigurationDesired(client, environment, command, parsed), + clientConfigurationDesired( + client, + environment, + command, + parsed, + files.value, + ), ); if (!policy.ok) { throwCleanupFailure(policy.error); @@ -156,10 +168,16 @@ const configureClientDocument = async ( throwCleanupFailure(fallback.error); return { status: "failed", reason: "readback" }; } - const desired = clientConfigurationDesired(client, environment, command, { - ...parsed, - servers: { ...fallback.value, ...parsed.servers }, - }); + const desired = clientConfigurationDesired( + client, + environment, + command, + { + ...parsed, + servers: { ...fallback.value, ...parsed.servers }, + }, + files.value, + ); if (registrationCurrent(parsed, desired)) return { status: "unchanged" }; return persistClientConfiguration({ client, @@ -205,7 +223,10 @@ const persistClientConfiguration = async ({ const selection = await selectCurrentRecoveryBackup(backupPath, original); if (!selection.ok) return { status: "failed", reason: "backup" }; if ( - !(await preserveConfigBackup(transactionPath, selection.value.backupPath)) + !(await preserveRecoveryBackup( + transactionPath, + selection.value.backupPath, + )) ) return { status: "failed", reason: "backup" }; const verified = await selectCurrentRecoveryBackup(backupPath, original); @@ -338,7 +359,13 @@ export const clientConfigurationAligned = async ( const policy = await readClientPolicyBlock( client, parsed, - clientConfigurationDesired(client, providerEnvironment, command, parsed), + clientConfigurationDesired( + client, + providerEnvironment, + command, + parsed, + files.value, + ), ); if (!policy.ok) { throwCleanupFailure(policy.error); @@ -353,6 +380,7 @@ export const clientConfigurationAligned = async ( providerEnvironment, command, parsed, + files.value, ), ) ); @@ -400,7 +428,9 @@ const inspectClientDocument = async ( remediation: files.error.detail, }; } - const existing = files.value.at(-1); + const existing = files.value.find( + ({ sourcePath }) => sourcePath === client.configPath, + ); const editabilityError = existing?.dialect === "dsh" ? dshDocument(existing.document).editabilityError @@ -413,7 +443,13 @@ const inspectClientDocument = async ( const policy = await readClientPolicyBlock( client, undefined, - clientConfigurationDesired(client, providerEnvironment, command, existing), + clientConfigurationDesired( + client, + providerEnvironment, + command, + existing, + files.value, + ), ); if (!policy.ok) { throwCleanupFailure(policy.error); @@ -456,7 +492,13 @@ const inspectClientDocument = async ( const policy = await readClientPolicyBlock( client, parsed, - clientConfigurationDesired(client, providerEnvironment, command, parsed), + clientConfigurationDesired( + client, + providerEnvironment, + command, + parsed, + files.value, + ), ); if (!policy.ok) { throwCleanupFailure(policy.error); @@ -472,6 +514,7 @@ const inspectClientDocument = async ( providerEnvironment, command, parsed, + files.value, ); if (registrationCurrent(parsed, desired)) return { status: "already_current" }; @@ -554,76 +597,6 @@ const readZcodeFallbackServers = async ( } }; -const preserveConfigBackup = async ( - source: string, - destination: string, -): Promise => { - try { - await copyFile(source, destination, fsConstants.COPYFILE_EXCL); - return true; - } catch (cause: unknown) { - return cause instanceof Error && "code" in cause && cause.code === "EEXIST"; - } -}; - -interface RecoveryBackupSelection { - readonly backupPath: string; - readonly present: boolean; -} - -interface RecoveryBackupSelectionFailure { - readonly kind: "unreadable" | "occupied"; - readonly path: string; - readonly detail: string; -} - -const selectCurrentRecoveryBackup = async ( - firstBackupPath: string, - original: string, -): Promise> => { - const expected = Buffer.from(original, "utf8"); - let firstBackup: Buffer | undefined; - try { - firstBackup = await readRegularFile(firstBackupPath, { - symlinks: "reject", - }); - } catch (cause: unknown) { - if (cause instanceof RegularFileCleanupFailure) throw cause; - if (!isMissing(cause)) - return err({ - kind: "unreadable", - path: firstBackupPath, - detail: `The recovery backup could not be checked: ${errorDetail(cause)}`, - }); - } - if (firstBackup === undefined) - return ok({ backupPath: firstBackupPath, present: false }); - if (firstBackup.equals(expected)) - return ok({ backupPath: firstBackupPath, present: true }); - - const digest = createHash("sha256").update(original, "utf8").digest("hex"); - const recoveryPath = `${firstBackupPath}.${digest}.recovery`; - let recovery: Buffer | undefined; - try { - recovery = await readRegularFile(recoveryPath, { symlinks: "reject" }); - } catch (cause: unknown) { - if (cause instanceof RegularFileCleanupFailure) throw cause; - if (!isMissing(cause)) - return err({ - kind: "unreadable", - path: recoveryPath, - detail: `The recovery backup could not be checked: ${errorDetail(cause)}`, - }); - } - if (recovery !== undefined && !recovery.equals(expected)) - return err({ - kind: "occupied", - path: recoveryPath, - detail: `The recovery path already contains different data: ${recoveryPath}`, - }); - return ok({ backupPath: recoveryPath, present: recovery !== undefined }); -}; - const restoreConfig = async ( path: string, original: string | undefined, @@ -678,8 +651,18 @@ const clientConfigurationDesired = ( client: SetupClient, providerEnvironment: SetupProviderEnvironment, command: readonly string[], - parsed?: ClientConfigurationDocument, + parsed: ClientConfigurationDocument | undefined, + sources: readonly ClientConfigurationSource[], ) => { + const effective = effectiveClientConfiguration(sources, { + client, + document: parsed, + }); + const validated = clientRegistrationEnvironment( + effective, + providerEnvironment, + ); + if (!validated.ok) throw validated.error; const format = parsed?.dialect ?? client.format ?? "json"; const launcher = command.length === 0 ? [PRODUCT_IDENTITY.cliBinary, "mcp"] : command; @@ -697,6 +680,7 @@ const clientConfigurationDesired = ( const environment = clientRegistrationEnvironment( parsed, providerEnvironment, + "fragment", ); if (!environment.ok) throw environment.error; const registration = clientRegistrationEntry( diff --git a/src/application/SetupHost.ts b/src/application/SetupHost.ts index 84d41229e..623202fdd 100644 --- a/src/application/SetupHost.ts +++ b/src/application/SetupHost.ts @@ -17,6 +17,7 @@ import { import { runDoctor, systemDoctorHost, type DoctorHost } from "./Doctor.js"; import { installLinuxHopper, + LinuxHopperTemporaryCleanupOwner, readLinuxDistribution, systemLinuxHopperInstallHost, } from "./LinuxHopper.js"; @@ -142,6 +143,8 @@ export const systemSetupHost = ( ? systemMacHopperInstallHost(homeDirectory, environment) : undefined; const resources = new ArtifactResourceScope(); + const linuxHopperHost = systemLinuxHopperInstallHost(environment); + const pendingLinuxHopperCleanup = new LinuxHopperTemporaryCleanupOwner(); return { platform, homeDirectory, @@ -151,7 +154,10 @@ export const systemSetupHost = ( nodeVersion: process.versions.node, macosVersion: () => doctorHost.macosVersion(), linuxDistribution: readLinuxDistribution, - close: () => closeSetupResources(resources, macHopperHost, doctorHost), + close: () => + closeSetupResources(resources, macHopperHost, doctorHost, () => + pendingLinuxHopperCleanup.retry(linuxHopperHost.cleanup), + ), initialSetupState: async ( scope?: DoctorScope, ): Promise => { @@ -160,6 +166,11 @@ export const systemSetupHost = ( ...(diagnosis.hopperPath === undefined ? {} : { hopperPath: diagnosis.hopperPath }), + ...(typeof environment.HOPPER_LAUNCHER_PATH === "string" && + environment.HOPPER_LAUNCHER_PATH.trim() !== "" && + diagnosis.hopperPath === environment.HOPPER_LAUNCHER_PATH + ? { hopperLauncherConfigured: true } + : {}), providerEnvironment: { ...providerRegistrationEnvironment( diagnosis.providerInspections ?? [], @@ -173,12 +184,12 @@ export const systemSetupHost = ( }, installHopper: async (replaceExisting) => { if (platform === "linux") { - const result = await installLinuxHopper( - systemLinuxHopperInstallHost(environment), - ); + const result = await installLinuxHopper(linuxHopperHost); + if (result.cleanupFailure !== undefined) + pendingLinuxHopperCleanup.retain(result.cleanupFailure); return result.status === "installed" - ? result - : setupInstallFailure(result.reason); + ? { status: "installed", launcherPath: result.launcherPath } + : setupInstallFailure(result.reason, result.packageFailure); } if (macHopperHost === undefined) return setupInstallFailure("unsupported_host"); @@ -264,12 +275,14 @@ const closeSetupResources = async ( resources: ArtifactResourceScope, macHopperHost: ReturnType | undefined, doctorHost: DoctorHost, + retryLinuxHopperCleanup: () => Promise, ): Promise => { - const [resourceCleanup, hopperCleanup, doctorCleanup] = + const [resourceCleanup, hopperCleanup, doctorCleanup, linuxCleanup] = await Promise.allSettled([ resources.close(), Promise.resolve().then(() => macHopperHost?.close()), Promise.resolve().then(() => doctorHost.close?.()), + retryLinuxHopperCleanup(), ]); const failures: string[] = []; if (hopperCleanup.status === "rejected") @@ -284,6 +297,11 @@ const closeSetupResources = async ( ); else if (doctorCleanup.value !== undefined) failures.push(doctorCleanup.value); + if (linuxCleanup.status === "rejected") + failures.push( + `Linux Hopper cleanup retry failed: ${errorMessage(linuxCleanup.reason)}`, + ); + else if (linuxCleanup.value !== undefined) failures.push(linuxCleanup.value); if (resourceCleanup.status === "rejected") failures.push(errorMessage(resourceCleanup.reason)); return failures.length === 0 ? undefined : failures.join("; "); diff --git a/src/application/SetupInstallFailure.test.ts b/src/application/SetupInstallFailure.test.ts index d8ba27bdb..939593c3f 100644 --- a/src/application/SetupInstallFailure.test.ts +++ b/src/application/SetupInstallFailure.test.ts @@ -21,3 +21,51 @@ describe("macOS Hopper setup failure projection", () => { }); }); }); + +describe("Linux package-manager setup failure projection", () => { + it("quotes the failing command and a sanitized bounded diagnostic", () => { + const result = setupInstallFailure("authorization_or_package_manager", { + command: 'pkexec pacman -U "/tmp/hopper package.pkg.tar.xz"', + exitCode: 1, + stdout: "", + stderr: "\u001b[31merror:\u001b[0m package conflict\nrefused", + outputTruncated: false, + }); + expect(result).toMatchObject({ + status: "failed", + code: "authorization_or_package_manager_failed", + }); + if (result.status !== "failed") throw new Error("expected failure"); + expect(result.remediation).toContain( + 'pkexec pacman -U "/tmp/hopper package.pkg.tar.xz"; exit 1', + ); + expect(result.remediation).toContain("error: package conflict refused"); + expect(result.remediation).not.toContain("\u001b"); + expect(result.remediation).not.toContain("\n"); + }); + + it("preserves the established remediation if no package output exists", () => { + expect(setupInstallFailure("authorization_or_package_manager")).toEqual({ + status: "failed", + code: "authorization_or_package_manager_failed", + remediation: + "Allow the system package-manager operation or install the planned Hopper package and Linux runtime dependencies, then rerun rea setup.", + }); + }); + + it("keeps truncated package output within its excerpt budget", () => { + const result = setupInstallFailure("authorization_or_package_manager", { + command: "pacman -U hopper.pkg.tar.xz", + exitCode: 1, + stdout: "", + stderr: `${"warning ".repeat(100)}final package conflict`, + outputTruncated: true, + }); + if (result.status !== "failed") throw new Error("expected failure"); + const excerpt = result.remediation + .split("its output: ")[1] + ?.split(" Resolve")[0]; + expect(excerpt).toContain("output truncated"); + expect(excerpt?.length).toBeLessThanOrEqual(400); + }); +}); diff --git a/src/application/SetupInstallFailure.ts b/src/application/SetupInstallFailure.ts index 944a60343..25f020e86 100644 --- a/src/application/SetupInstallFailure.ts +++ b/src/application/SetupInstallFailure.ts @@ -1,4 +1,5 @@ import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; +import type { LinuxPackageInstallFailure } from "./LinuxHopper.js"; export type SetupFailureCode = | "unsupported_host" @@ -45,9 +46,53 @@ export type HopperInstallFailureReason = | "publisher_verification_failed" | "copy"; +const normalizePackageOutput = (output: string): string => + output + .replace(/\u001b(?:\[[0-9;?]*[ -/]*[@-~]|\([0-9A-Za-z]|[@-Z\\^_])/gu, "") + .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, "") + .replace(/\s+/gu, " ") + .trim(); + +const packageOutputExcerpt = ( + failure: LinuxPackageInstallFailure, + limit = 400, +): string => { + const output = [failure.stderr, failure.stdout] + .map(normalizePackageOutput) + .filter((text) => text !== "") + .join(" "); + const suffix = failure.outputTruncated ? " (output truncated)" : ""; + const bodyLimit = limit - suffix.length; + if (output.length <= bodyLimit) return `${output}${suffix}`; + const half = Math.floor((bodyLimit - 3) / 2); + const head = output + .slice(0, half) + .replace(/\s+\S*$/u, "") + .trimEnd(); + const tail = output + .slice(-half) + .replace(/^\S*\s+/u, "") + .trimStart(); + return `${head} … ${tail}${suffix}`; +}; + +const packageManagerRemediation = ( + failure: LinuxPackageInstallFailure | undefined, +): string => { + if (failure === undefined) + return "Allow the system package-manager operation or install the planned Hopper package and Linux runtime dependencies, then rerun rea setup."; + const status = + failure.exitCode === null ? "no exit status" : `exit ${failure.exitCode}`; + const excerpt = packageOutputExcerpt(failure); + const observed = + excerpt === "" ? "no output was captured." : `its output: ${excerpt}`; + return `The system package-manager command failed (${failure.command}; ${status}); ${observed} Resolve the reported problem, grant the authorization the command needs, or install the planned Hopper package and Linux runtime dependencies manually, then rerun rea setup.`; +}; + /** Map provider installer failures to stable setup codes and safe recovery. */ export const setupInstallFailure = ( reason: HopperInstallFailureReason, + packageFailure?: LinuxPackageInstallFailure, ): SetupHopperInstallResult => { switch (reason) { case "unsupported_host": @@ -67,8 +112,7 @@ export const setupInstallFailure = ( return { status: "failed", code: "authorization_or_package_manager_failed", - remediation: - "Allow the system package-manager operation or install the planned Hopper package and Linux runtime dependencies, then rerun rea setup.", + remediation: packageManagerRemediation(packageFailure), }; case "launcher_missing": return { diff --git a/src/application/SetupPlan.ts b/src/application/SetupPlan.ts index f428ecade..345a05c45 100644 --- a/src/application/SetupPlan.ts +++ b/src/application/SetupPlan.ts @@ -39,16 +39,19 @@ export const discoverSetupState = async (input: { readonly host: SetupHost; readonly providerEnvironment: SetupProviderEnvironment; readonly forceHopperInstall: boolean; + readonly hopperLauncherConfigured?: boolean; readonly proposeHopper: boolean; readonly initialDoctor: DoctorReport; }): Promise => { const { initialDoctor } = input; - const linuxHopperRepairNeeded = initialDoctor.checks.some( - ({ name, ok, detail }) => - !ok && - (name === "hopper-demo-runtime" || - (name === "hopper-version" && detail === "/opt/hopper/bin/Hopper")), - ); + const linuxHopperRepairNeeded = + input.hopperLauncherConfigured !== true && + initialDoctor.checks.some( + ({ name, ok, detail }) => + !ok && + (name === "hopper-demo-runtime" || + (name === "hopper-version" && detail === "/opt/hopper/bin/Hopper")), + ); const linuxDistribution = input.host.platform === "linux" ? await input.host.linuxDistribution() diff --git a/src/application/SetupRecoveryBackup.ts b/src/application/SetupRecoveryBackup.ts new file mode 100644 index 000000000..23f324210 --- /dev/null +++ b/src/application/SetupRecoveryBackup.ts @@ -0,0 +1,88 @@ +import { createHash } from "node:crypto"; +import { constants as fsConstants } from "node:fs"; +import { copyFile } from "node:fs/promises"; +import { err, ok, type Result } from "../domain/result.js"; +import { + readRegularFile, + RegularFileCleanupFailure, +} from "./RegularFileRead.js"; + +export interface RecoveryBackupSelection { + readonly backupPath: string; + readonly present: boolean; +} + +export interface RecoveryBackupSelectionFailure { + readonly kind: "unreadable" | "occupied"; + readonly path: string; + readonly detail: string; +} + +const isMissing = (cause: unknown): boolean => + cause instanceof Error && "code" in cause && cause.code === "ENOENT"; + +const errorDetail = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +/** Select a persistent snapshot only when its bytes match this transaction's original. */ +export const selectCurrentRecoveryBackup = async ( + firstBackupPath: string, + original: string | Buffer, +): Promise> => { + const expected = Buffer.isBuffer(original) + ? original + : Buffer.from(original, "utf8"); + let firstBackup: Buffer | undefined; + try { + firstBackup = await readRegularFile(firstBackupPath, { + symlinks: "reject", + }); + } catch (cause: unknown) { + if (cause instanceof RegularFileCleanupFailure) throw cause; + if (!isMissing(cause)) + return err({ + kind: "unreadable", + path: firstBackupPath, + detail: `The recovery backup could not be checked: ${errorDetail(cause)}`, + }); + } + if (firstBackup === undefined) + return ok({ backupPath: firstBackupPath, present: false }); + if (firstBackup.equals(expected)) + return ok({ backupPath: firstBackupPath, present: true }); + + const digest = createHash("sha256").update(expected).digest("hex"); + const recoveryPath = `${firstBackupPath}.${digest}.recovery`; + let recovery: Buffer | undefined; + try { + recovery = await readRegularFile(recoveryPath, { symlinks: "reject" }); + } catch (cause: unknown) { + if (cause instanceof RegularFileCleanupFailure) throw cause; + if (!isMissing(cause)) + return err({ + kind: "unreadable", + path: recoveryPath, + detail: `The recovery backup could not be checked: ${errorDetail(cause)}`, + }); + } + if (recovery !== undefined && !recovery.equals(expected)) + return err({ + kind: "occupied", + path: recoveryPath, + detail: `The recovery path already contains different data: ${recoveryPath}`, + }); + return ok({ backupPath: recoveryPath, present: recovery !== undefined }); +}; + +/** Preserve a snapshot without replacing an earlier recovery file. */ +export const preserveRecoveryBackup = async ( + source: string, + destination: string, +): Promise => { + try { + await copyFile(source, destination, fsConstants.COPYFILE_EXCL); + return true; + } catch (cause: unknown) { + return cause instanceof Error && "code" in cause && cause.code === "EEXIST"; + } +}; diff --git a/src/application/SetupSkill.ts b/src/application/SetupSkill.ts index 794202b48..7eeec9456 100644 --- a/src/application/SetupSkill.ts +++ b/src/application/SetupSkill.ts @@ -17,6 +17,7 @@ import { withRegularFile, } from "./RegularFileRead.js"; import { clientSkillDirectories } from "./SupportedClients.js"; +import { selectCurrentRecoveryBackup } from "./SetupRecoveryBackup.js"; export const MANAGED_SKILL_FILES = [ "SKILL.md", @@ -36,10 +37,12 @@ export const readSkillFile = ( path: string, resources?: ArtifactResourceScope, ): Promise => - readSkillFileState(path, resources).then(({ content }) => content); + readSkillFileState(path, resources).then(({ bytes }) => + bytes.toString("utf8"), + ); interface SkillFileState { - readonly content: string; + readonly bytes: Buffer; readonly mode: number; } @@ -58,7 +61,7 @@ const readSkillFileState = async ( ); if (bytes === undefined || bytes.length !== stats.size) throw new RegularFileChangedError(path); - return { content: bytes.toString("utf8"), mode: stats.mode & 0o7777 }; + return { bytes, mode: stats.mode & 0o7777 }; }, { symlinks: "reject", @@ -86,7 +89,7 @@ export interface SkillInstallFileOperations { readonly backup: (source: string, destination: string) => Promise; readonly write: ( path: string, - content: string, + content: string | Buffer, mode: number, ) => Promise; readonly remove: (path: string) => Promise; @@ -277,7 +280,8 @@ const canonicalSkillNeedsInstallByReading = async ( const destination = join(root, relativePath); const content = await canonicalSkillContent(relativePath); const original = await readOptionalSkillState(destination, resources); - if (original?.content !== content) return true; + if (original?.bytes.equals(Buffer.from(content, "utf8")) !== true) + return true; } } return false; @@ -327,7 +331,7 @@ const restoreSkillFiles = async ( for (const { destination, original } of [...attempted].reverse()) { try { if (original === undefined) await operations.remove(destination); - else await operations.write(destination, original.content, original.mode); + else await operations.write(destination, original.bytes, original.mode); } catch (cause: unknown) { failures.push({ path: destination, reason: causeReason(cause) }); } @@ -335,6 +339,32 @@ const restoreSkillFiles = async ( return failures; }; +const preserveCurrentSkillBackup = async ( + destination: string, + original: Buffer, + operations: SkillInstallFileOperations, +): Promise => { + const firstBackupPath = `${destination}.rea.backup`; + const selection = await selectCurrentRecoveryBackup( + firstBackupPath, + original, + ); + if (!selection.ok) throw new Error(selection.error.detail); + if (selection.value.present) return selection.value.backupPath; + + await operations.backup(destination, selection.value.backupPath); + const verified = await selectCurrentRecoveryBackup(firstBackupPath, original); + if ( + !verified.ok || + verified.value.backupPath !== selection.value.backupPath || + !verified.value.present + ) + throw new Error( + `The current file was not preserved at ${selection.value.backupPath}`, + ); + return verified.value.backupPath; +}; + /** Transactionally install the canonical skill bundle and optional references. */ export const installCanonicalSkill = async ( home: string, @@ -347,6 +377,7 @@ export const installCanonicalSkill = async ( let attempted: CanonicalSkillFile[] = []; let stage: "prepare" | "backup" | "write" | "readback" = "prepare"; let failingPath = "skill bundle"; + const recoveryPaths = new Map(); try { const canonical = await canonicalSkillFiles( home, @@ -356,7 +387,8 @@ export const installCanonicalSkill = async ( resources, ); const changed = canonical.filter( - ({ content, original }) => original?.content !== content, + ({ content, original }) => + original?.bytes.equals(Buffer.from(content, "utf8")) !== true, ); if (changed.length === 0) return ok("unchanged"); @@ -364,8 +396,16 @@ export const installCanonicalSkill = async ( for (const { destination, original } of changed) { failingPath = destination; await operations.mkdir(dirname(destination)); - if (original !== undefined) - await operations.backup(destination, `${destination}.rea.backup`); + if (original !== undefined) { + recoveryPaths.set( + destination, + await preserveCurrentSkillBackup( + destination, + original.bytes, + operations, + ), + ); + } } stage = "write"; for (const file of changed) { @@ -382,7 +422,11 @@ export const installCanonicalSkill = async ( stage = "readback"; for (const { destination, content } of changed) { failingPath = destination; - if ((await readSkillFile(destination, resources)) !== content) + if ( + !(await readSkillFileState(destination, resources)).bytes.equals( + Buffer.from(content, "utf8"), + ) + ) throw new Error(`skill readback mismatch: ${destination}`); } return ok("installed"); @@ -396,7 +440,9 @@ export const installCanonicalSkill = async ( reason: causeReason(cause), rollbackFailures, recoveryBackupPaths: attempted.flatMap(({ destination, original }) => - original === undefined ? [] : [`${destination}.rea.backup`], + original === undefined || recoveryPaths.get(destination) === undefined + ? [] + : [recoveryPaths.get(destination)!], ), cause, }), diff --git a/src/application/SetupTypes.ts b/src/application/SetupTypes.ts index f0cf61c59..9d7bc44c6 100644 --- a/src/application/SetupTypes.ts +++ b/src/application/SetupTypes.ts @@ -13,6 +13,8 @@ export type SetupProviderEnvironment = Readonly>; /** Read-only host facts shared by the initial setup plan. */ export interface SetupInitialState { readonly hopperPath?: string; + /** Whether the caller explicitly selected a Hopper launcher in its environment. */ + readonly hopperLauncherConfigured?: boolean; readonly providerEnvironment: SetupProviderEnvironment; readonly doctor: DoctorReport; } diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 55a651a52..582c48ba7 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -47,6 +47,7 @@ export interface SetupClient { | "pi" | "kimi" | "hermes" + | "zcode_fallback" | "zcode" | "dsh" | "unsupported"; diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index eac36c73d..d6447a8f6 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -231,12 +231,21 @@ export const systemUninstallHost = ( const clients = supportedClients(home, platform, environment); redirection = appDataRedirection(clients); return Promise.resolve( - clients.flatMap((client) => - uninstallConfigPaths(client).map((configPath) => ({ + clients.flatMap((client) => [ + ...uninstallConfigPaths(client).map((configPath) => ({ ...client, configPath, })), - ), + ...(client.zcodeFallbackPath === undefined + ? [] + : [ + { + ...client, + configPath: client.zcodeFallbackPath, + format: "zcode_fallback" as const, + }, + ]), + ]), ); }, inspectClient: async (client) => { diff --git a/src/doctorRuntime.test.ts b/src/doctorRuntime.test.ts new file mode 100644 index 000000000..c62bd0f5c --- /dev/null +++ b/src/doctorRuntime.test.ts @@ -0,0 +1,56 @@ +import { chmod, mkdir, realpath, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it as test } from "vitest"; + +import { createSystemDoctorHost } from "./doctorRuntime.js"; +import { DOCTOR_PROVIDER_IDS } from "./application/DoctorProviderIds.js"; +import { providerRegistrationEnvironment } from "./application/SetupRegistrationEnvironment.js"; +import { createTestTempDirectory } from "../tests/fixtures/temporaryDirectory.js"; +import { writeJadxJarInventory } from "../tests/fixtures/android/jadxJar.js"; + +const it = test.skipIf(process.platform === "win32"); + +it("exposes jadx through the doctor provider selector", () => { + expect(DOCTOR_PROVIDER_IDS).toContain("jadx"); +}); + +it("carries a validated REA_JADX_MCP_JAR into the provider registration environment", async () => { + const root = await createTestTempDirectory("rea-jadx-registration-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = await createFakeJdk(root); + await writeJadxJarInventory(jar); + + const host = createSystemDoctorHost({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + const inspections = await host.providerInspections?.(); + expect(inspections?.map(({ id }) => id)).toContain("jadx"); + + const environment = providerRegistrationEnvironment(inspections ?? []); + expect(environment).toMatchObject({ + REA_JADX_MCP_JAR: await realpath(jar), + }); +}); + +it("omits REA_JADX_MCP_JAR from the provider registration environment when it is not configured", async () => { + const host = createSystemDoctorHost({}); + const inspections = await host.providerInspections?.(); + + const environment = providerRegistrationEnvironment(inspections ?? []); + expect(environment).not.toHaveProperty("REA_JADX_MCP_JAR"); +}); + +const createFakeJdk = async (root: string): Promise => { + const javaHome = join(root, "fake-jdk"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + await mkdir(bin, { recursive: true }); + await writeFile( + java, + "#!/bin/sh\nprintf 'java.base@21\\njdk.compiler@21\\n'\n", + ); + await chmod(java, 0o700); + return javaHome; +}; diff --git a/src/doctorRuntime.ts b/src/doctorRuntime.ts index 1605c3691..75abfe2aa 100644 --- a/src/doctorRuntime.ts +++ b/src/doctorRuntime.ts @@ -4,6 +4,7 @@ import { type DoctorHost, } from "./application/Doctor.js"; import { fileURLToPath } from "node:url"; +import { inspectJadxProvider } from "./android/JadxDoctor.js"; import { inspectSystemGhidraProvider } from "./ghidra/GhidraDoctor.js"; import { inspectIdaRegistration } from "./ida/IdaDoctor.js"; import { selectLinuxPrivateDisplayStrategy } from "./hopper/LinuxPrivateDisplayProbe.js"; @@ -17,6 +18,7 @@ export const createSystemDoctorHost = ( providerInspections: async () => [ await inspectSystemGhidraProvider(environment), inspectIdaRegistration(environment.REA_IDA_MCP_CONFIG), + await inspectJadxProvider(environment), ], linuxDemoRuntimeCheck: () => inspectLinuxPrivateDisplay(environment), }); diff --git a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts new file mode 100644 index 000000000..c3ea4ed5a --- /dev/null +++ b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts @@ -0,0 +1,235 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { expect } from "vitest"; +import { stringify } from "yaml"; + +import { + effectiveClientServer, + parseClientConfiguration, + clientRegistrationEntry, +} from "../../../src/application/ClientConfigurationDocument.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const scenarios = [ + { + client: "cursor", + format: "json", + path: ".cursor/mcp.json", + setting: "HOPPER_LOADER_ARGS_JSON", + invalid: { HOPPER_LOADER_ARGS_JSON: "{" }, + repaired: { HOPPER_LOADER_ARGS_JSON: '["", "--loader", "Mach-O"]' }, + }, + { + client: "dsh", + format: "dsh", + path: ".dsh/cordis.patch.yml", + setting: "REA_GHIDRA_COMPILER_SPEC_ID", + invalid: { REA_GHIDRA_COMPILER_SPEC_ID: "borlandcpp" }, + repaired: { + REA_GHIDRA_COMPILER_SPEC_ID: "borlandcpp", + REA_GHIDRA_LANGUAGE_ID: "x86:LE:32:default", + }, + }, +] as const; + +for (const scenario of scenarios) + cliTest( + `setup and doctor reject ${scenario.client} overrides rejected by MCP startup`, + async ({ cli }) => { + const home = await createTestTempDirectory("rea-registration-runtime-"); + const config = join(home, scenario.path); + await mkdir(dirname(config), { recursive: true }); + const environment = { + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, ".config"), + DSH_HOME: join(home, ".dsh"), + }; + const registration = (env: Readonly>) => { + const server = { + command: "stale-launcher", + args: ["mcp"], + env: { CALLER_SETTING: "keep exactly", ...env }, + }; + return scenario.client === "dsh" + ? stringify([ + { + insert: [ + { + id: "rea-mcp", + name: "@deepseek-ai/dsh-mcp-client", + config: { + serverName: "rea", + transport: "stdio", + ...server, + }, + }, + ], + }, + ]) + : JSON.stringify({ mcpServers: { rea: server } }); + }; + const original = registration(scenario.invalid); + await writeFile(config, original); + const startup = await cli.run({ + arguments: ["mcp"], + cwd: home, + environment: { ...environment, ...scenario.invalid }, + }); + expect(startup.exitCode).toBe(1); + expect(startup.stderr).toContain(scenario.setting); + expect(startup.stdout).toBe(""); + const run = (args: readonly string[]) => + cli.run({ arguments: [...args, "--json"], cwd: home, environment }); + const setup = await run([ + "setup", + "--client", + scenario.client, + "--skill=false", + "--yes", + ]); + expect(setup.exitCode).toBe(1); + expect(setup.json).toMatchObject({ + status: "needs_human", + plannedActions: [], + appliedActions: [], + remediation: expect.stringContaining(scenario.setting), + }); + const doctor = await run(["doctor", "--client", scenario.client]); + expect(doctor.exitCode).toBe(1); + expect(doctor.json).toMatchObject({ + identity: { + registrations: expect.arrayContaining([ + expect.objectContaining({ + client: scenario.client, + state: "invalid", + remediation: expect.stringContaining(scenario.setting), + }), + ]), + }, + }); + expect(await readFile(config, "utf8")).toBe(original); + await expect(access(`${config}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + + // Model the caller repairing the named setting before rerunning setup. + const repaired = registration(scenario.repaired); + await writeFile(config, repaired); + const configured = await run([ + "setup", + "--client", + scenario.client, + "--skill=false", + "--yes", + ]); + expect(configured.exitCode, configured.stdout).toBe(0); + const text = await readFile(config, "utf8"); + const parsed = parseClientConfiguration(text, scenario.format); + expect(effectiveClientServer(parsed, "rea")).toMatchObject({ + env: { CALLER_SETTING: "keep exactly", ...scenario.repaired }, + }); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(repaired); + const repeated = await run([ + "setup", + "--client", + scenario.client, + "--skill=false", + "--yes", + ]); + expect(repeated.json).toMatchObject({ + status: "ready", + appliedActions: [], + plannedActions: [], + }); + expect(await readFile(config, "utf8")).toBe(text); + }, + ); + +cliTest( + "OpenCode CLI validates dependent settings after merging JSON and JSONC", + async ({ cli }) => { + const home = await createTestTempDirectory("rea-layered-runtime-cli-"); + const directory = join(home, ".config/opencode"); + await mkdir(directory, { recursive: true }); + const lower = join(directory, "opencode.json"); + const upper = join(directory, "opencode.jsonc"); + const content = (env: Record) => + JSON.stringify({ + mcp: { + rea: clientRegistrationEntry( + "opencode", + [resolve("scripts/rea.mjs"), "mcp"], + env, + ), + }, + }); + const lowerContent = content({ + REA_GHIDRA_LANGUAGE_ID: "x86:LE:32:default", + }); + const upperContent = content({ REA_GHIDRA_COMPILER_SPEC_ID: "borlandcpp" }); + await writeFile(lower, lowerContent); + await writeFile(upper, upperContent); + const environment = { + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, ".config"), + }; + const run = (args: readonly string[]) => + cli.run({ arguments: [...args, "--json"], cwd: home, environment }); + const setup = await run([ + "setup", + "--client", + "opencode", + "--skill=false", + "--yes", + ]); + expect(setup.exitCode, setup.stdout).toBe(0); + expect(setup.json).toMatchObject({ + status: "ready", + plannedActions: [], + appliedActions: [], + }); + const doctor = await run(["doctor", "--client", "opencode"]); + expect(doctor.exitCode, doctor.stdout).toBe(0); + expect(doctor.json).toMatchObject({ + identity: { + registrations: expect.arrayContaining([ + expect.objectContaining({ client: "opencode", state: "aligned" }), + ]), + }, + }); + expect(await readFile(lower, "utf8")).toBe(lowerContent); + expect(await readFile(upper, "utf8")).toBe(upperContent); + await writeFile(lower, content({ CALLER_SETTING: "retained" })); + const missing = await run([ + "setup", + "--client", + "opencode", + "--skill=false", + "--yes", + ]); + expect(missing.exitCode).toBe(1); + expect(missing.json).toMatchObject({ + status: "needs_human", + plannedActions: [], + appliedActions: [], + remediation: expect.stringContaining("REA_GHIDRA_COMPILER_SPEC_ID"), + }); + const invalidDoctor = await run(["doctor", "--client", "opencode"]); + expect(invalidDoctor.exitCode).toBe(1); + expect(invalidDoctor.json).toMatchObject({ + identity: { + registrations: expect.arrayContaining([ + expect.objectContaining({ client: "opencode", state: "invalid" }), + ]), + }, + }); + expect(await readFile(upper, "utf8")).toBe(upperContent); + for (const path of [lower, upper]) + await expect(access(`${path}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + }, +); diff --git a/tests/boundary/android/jadxConfiguration.test.ts b/tests/boundary/android/jadxConfiguration.test.ts index 3ebba3271..7600ce44f 100644 --- a/tests/boundary/android/jadxConfiguration.test.ts +++ b/tests/boundary/android/jadxConfiguration.test.ts @@ -380,3 +380,10 @@ const createFakeJdk = async ( await chmod(java, 0o700); return javaHome; }; + +it("tags unavailable diagnostics with the boundary failure kind", async () => { + const availability = await inspectJadxAvailability({}); + expect(availability.status).toBe("unavailable"); + if (availability.status !== "unavailable") throw new Error("unreachable"); + expect(availability.diagnostics["failure_kind"]).toBe("jar"); +}); diff --git a/tests/boundary/android/jadxDoctor.test.ts b/tests/boundary/android/jadxDoctor.test.ts new file mode 100644 index 000000000..e053f66b5 --- /dev/null +++ b/tests/boundary/android/jadxDoctor.test.ts @@ -0,0 +1,396 @@ +import { chmod, mkdir, realpath, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it as test } from "vitest"; + +import { inspectJadxProvider } from "../../../src/android/JadxDoctor.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { writeJadxJarInventory } from "../../fixtures/android/jadxJar.js"; +import { writeOrderedZip } from "../../fixtures/artifactEntryOrder.js"; + +const it = test.skipIf(process.platform === "win32"); + +it("publishes REA_JADX_MCP_JAR as registration environment when the configured JAR validates", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = await createFakeJdk(root); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + + expect(inspection).toMatchObject({ + id: "jadx", + configured: true, + available: true, + providerVersion: null, + registrationEnvironment: { REA_JADX_MCP_JAR: await realpath(jar) }, + checks: [ + { + name: "configuration", + ok: true, + code: null, + classification: "config_drift", + }, + ], + }); +}); + +it("publishes JAVA_HOME alongside the JAR when the configured JAR validates", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-javahome-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = await createFakeJdk(root); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + + expect(inspection.registrationEnvironment).toEqual({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); +}); + +it("preserves the caller-selected JAR path when it is a symlink", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-symlink-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const link = join(root, "current.jar"); + const javaHome = await createFakeJdk(root); + await writeJadxJarInventory(jar); + await symlink(jar, link); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: link, + JAVA_HOME: javaHome, + }); + + expect(inspection.available).toBe(true); + expect(inspection.registrationEnvironment["REA_JADX_MCP_JAR"]).toBe(link); +}); + +it("guides Java selection when the Java runtime is missing", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-remediation-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: join(root, "missing-jdk"), + }); + + expect(inspection.checks[0]?.code).toBe("runtime_missing"); + expect(inspection.checks[0]?.remediation).toContain("JAVA_HOME"); + expect(inspection.checks[0]?.remediation).not.toContain("REA_JADX_MCP_JAR"); +}); + +it("guides Java selection when JAVA_HOME is not an absolute path", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-options-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: "relative/path", + }); + + expect(inspection.checks[0]?.code).toBe("open_options_invalid"); + expect(inspection.checks[0]?.remediation).toContain("JAVA_HOME"); + expect(inspection.checks[0]?.remediation).not.toContain("REA_JADX_MCP_JAR"); +}); + +it("publishes no registration environment when REA_JADX_MCP_JAR is unset", async () => { + const inspection = await inspectJadxProvider({}); + + expect(inspection).toMatchObject({ + id: "jadx", + configured: false, + available: false, + providerVersion: null, + registrationEnvironment: {}, + checks: [ + { + name: "configuration", + ok: false, + code: "not_configured", + classification: "missing_analysis_engine", + }, + ], + }); +}); + +it("publishes no registration environment when the configured JAR fails validation", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-invalid-"); + const jar = join(root, "engine.jar"); + const javaHome = await createFakeJdk(root); + await writeFile(jar, "not a Java archive"); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + + expect(inspection).toMatchObject({ + id: "jadx", + configured: true, + available: false, + registrationEnvironment: {}, + checks: [ + { + name: "configuration", + ok: false, + code: "version_unresolved", + classification: "config_drift", + }, + ], + }); + expect(inspection.checks[0]?.detail).toContain("REA_JADX_MCP_JAR"); +}); + +it("classifies a missing Java runtime as a missing dependency", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-runtime-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: join(root, "missing-jdk"), + }); + + expect(inspection).toMatchObject({ + configured: true, + available: false, + registrationEnvironment: {}, + checks: [ + { + name: "configuration", + ok: false, + code: "runtime_missing", + classification: "missing_dependency", + }, + ], + }); +}); + +it("guides JVM option correction when REA_JADX_HEAP_MIB is invalid", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-heap-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = await createFakeJdk(root); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + REA_JADX_HEAP_MIB: "not-a-number", + }); + + expect(inspection.checks[0]?.code).toBe("open_options_invalid"); + expect(inspection.checks[0]?.remediation).toContain("REA_JADX_HEAP_MIB"); + expect(inspection.checks[0]?.remediation).not.toContain("JAVA_HOME"); +}); + +it("publishes validated JVM resource overrides in the registration environment", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-overrides-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = await createFakeJdk(root); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + REA_JADX_HEAP_MIB: "4096", + REA_JADX_ACTIVE_PROCESSOR_COUNT: "8", + }); + + expect(inspection.available).toBe(true); + expect(inspection.registrationEnvironment).toEqual({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + REA_JADX_HEAP_MIB: "4096", + REA_JADX_ACTIVE_PROCESSOR_COUNT: "8", + }); +}); + +it("guides JDK selection when the Java runtime is older than 17", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-oldjdk-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = await createFakeJdk(root, "11"); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + + expect(inspection.checks[0]?.code).toBe("unsupported_version"); + expect(inspection.checks[0]?.remediation).toContain("JDK"); + expect(inspection.checks[0]?.remediation).not.toContain("REA_JADX_MCP_JAR"); +}); + +it("guides JAR replacement when the JAR misses bridge classes", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-classes-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = await createFakeJdk(root); + await writeOrderedZip(jar, ["com/atxx/jhmcp/JadxSession.class"]); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + + expect(inspection.checks[0]?.code).toBe("unsupported_version"); + expect(inspection.checks[0]?.remediation).toContain("REA_JADX_MCP_JAR"); +}); + +it("remediates an unsupported host with platform guidance", async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, "platform", { + value: "sunos", + configurable: true, + }); + try { + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: "/opt/jadx-headless-mcp.jar", + }); + + expect(inspection.checks[0]?.code).toBe("unsupported_host"); + expect(inspection.checks[0]?.remediation).toContain("Linux"); + } finally { + Object.defineProperty(process, "platform", { + value: originalPlatform, + configurable: true, + }); + } +}); + +it("guides Windows native package correction when process ownership is unavailable", async () => { + const originalPlatform = process.platform; + const originalArch = process.arch; + Object.defineProperty(process, "platform", { + value: "win32", + configurable: true, + }); + Object.defineProperty(process, "arch", { value: "x64", configurable: true }); + try { + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: "/opt/jadx-headless-mcp.jar", + }); + + expect(inspection.checks[0]?.code).toBe("runtime_missing"); + expect(inspection.checks[0]?.remediation).toContain( + "Windows x64 REA package", + ); + expect(inspection.checks[0]?.remediation).not.toContain("JAVA_HOME"); + } finally { + Object.defineProperty(process, "platform", { + value: originalPlatform, + configurable: true, + }); + Object.defineProperty(process, "arch", { + value: originalArch, + configurable: true, + }); + } +}); + +it("guides Java probe correction when the module listing is unparseable", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-unparseable-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeJadxJarInventory(jar); + // Fake JDK whose module listing lacks a parseable java.base version. + const javaHome = join(root, "fake-jdk"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + await mkdir(bin, { recursive: true }); + await writeFile(java, "#!/bin/sh\nprintf 'jdk.compiler@21\\n'\n"); + await chmod(java, 0o700); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + + expect(inspection.checks[0]?.code).toBe("version_unresolved"); + expect(inspection.checks[0]?.remediation).toContain("java --list-modules"); + expect(inspection.checks[0]?.remediation).not.toContain("REA_JADX_MCP_JAR"); +}); + +const createFakeJdk = async (root: string, major = "21"): Promise => { + const javaHome = join(root, "fake-jdk"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + await mkdir(bin, { recursive: true }); + await writeFile( + java, + `#!/bin/sh\nprintf 'java.base@${major}\\njdk.compiler@${major}\\n'\n`, + ); + await chmod(java, 0o700); + return javaHome; +}; + +it("classifies missing Windows native artifacts as missing dependencies", async () => { + const originalPlatform = process.platform; + const originalArch = process.arch; + Object.defineProperty(process, "platform", { + value: "win32", + configurable: true, + }); + Object.defineProperty(process, "arch", { value: "x64", configurable: true }); + try { + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: "/opt/jadx-headless-mcp.jar", + }); + + expect(inspection.checks[0]?.code).toBe("runtime_missing"); + expect(inspection.checks[0]?.classification).toBe("missing_dependency"); + } finally { + Object.defineProperty(process, "platform", { + value: originalPlatform, + configurable: true, + }); + Object.defineProperty(process, "arch", { + value: originalArch, + configurable: true, + }); + } +}); + +it("classifies malformed JAVA_HOME as configuration drift", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-javahome-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeJadxJarInventory(jar); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: "relative/path", + }); + + expect(inspection.checks[0]?.code).toBe("open_options_invalid"); + expect(inspection.checks[0]?.classification).toBe("config_drift"); +}); + +it("preserves Java probe diagnostics in doctor output", async () => { + const root = await createTestTempDirectory("rea-jadx-doctor-probe-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeJadxJarInventory(jar); + const javaHome = join(root, "fake-jdk-unparseable"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + await mkdir(bin, { recursive: true }); + // Emits jdk.compiler but no parseable java.base version. + await writeFile(java, `#!/bin/sh\nprintf 'jdk.compiler@21\\n'\n`); + await chmod(java, 0o700); + + const inspection = await inspectJadxProvider({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + + expect(inspection.checks[0]?.code).toBe("version_unresolved"); + expect(inspection.checks[0]?.details).toMatchObject({ + java_executable: expect.any(String), + }); +}); diff --git a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts index 9e0b340c5..74cbf612d 100644 --- a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts +++ b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts @@ -32,6 +32,11 @@ it.each([ REA_MCP_MAX_RESPONSE_BYTES: "33554432", REA_BROWSER_EXECUTABLE: join(root, "custom-browser"), GHIDRA_INSTALL_DIR: join(root, "old-ghidra"), + HOPPER_LOADER_ARGS_JSON: '["", "--loader", "Mach-O"]', + REA_REFERENCE_SECRET_PATTERNS_JSON: '["*.private", "local/**"]', + REA_GHIDRA_LANGUAGE_ID: "x86:LE:32:default", + REA_GHIDRA_COMPILER_SPEC_ID: "borlandcpp", + CALLER_SETTING: "preserve exactly", }; const original = serializeClientConfiguration( { @@ -53,9 +58,10 @@ it.each([ ).toMatchObject({ status: "configured" }); const configured = await readFile(configPath, "utf8"); const parsed = parseClientConfiguration(configured, format); - expect(effectiveClientServer(parsed, "rea")).toMatchObject({ - [environmentKey]: { ...custom, ...providers }, - }); + expect(effectiveClientServer(parsed, "rea")).toHaveProperty( + environmentKey, + { ...custom, ...providers }, + ); expect(effectiveClientServer(parsed, "other")).toEqual( effectiveClientServer( parseClientConfiguration(original, format), @@ -142,6 +148,36 @@ it.each([ { REA_MCP_MAX_RESPONSE_BYTES: "1" }, "REA_MCP_MAX_RESPONSE_BYTES", ], + [ + "malformed loader JSON", + { HOPPER_LOADER_ARGS_JSON: "{" }, + "HOPPER_LOADER_ARGS_JSON", + ], + [ + "non-array loader JSON", + { HOPPER_LOADER_ARGS_JSON: "{}" }, + "HOPPER_LOADER_ARGS_JSON", + ], + [ + "NUL loader argument", + { HOPPER_LOADER_ARGS_JSON: JSON.stringify(["a\0b"]) }, + "HOPPER_LOADER_ARGS_JSON", + ], + [ + "malformed exclusion JSON", + { REA_REFERENCE_SECRET_PATTERNS_JSON: "{" }, + "REA_REFERENCE_SECRET_PATTERNS_JSON", + ], + [ + "empty exclusion pattern", + { REA_REFERENCE_SECRET_PATTERNS_JSON: '[""]' }, + "REA_REFERENCE_SECRET_PATTERNS_JSON", + ], + [ + "unpaired compiler specification", + { REA_GHIDRA_COMPILER_SPEC_ID: "borlandcpp" }, + "REA_GHIDRA_COMPILER_SPEC_ID", + ], ] as const)( "refuses %s before changing the configuration", async (_name, env, setting) => { diff --git a/tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts b/tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts new file mode 100644 index 000000000..41f0b984c --- /dev/null +++ b/tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts @@ -0,0 +1,219 @@ +import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { + clientRegistrationEntry, + parseClientConfiguration, + effectiveClientServer, +} from "../../../src/application/ClientConfigurationDocument.js"; +import { + configureClientConfiguration, + inspectClientConfiguration, + clientConfigurationAligned, +} from "../../../src/application/SetupClientConfiguration.js"; +import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { npxRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const command = npxRegistrationCommand(process.platform); +const language = "x86:LE:32:default"; +const compiler = "borlandcpp"; + +const layered = async (lower: unknown, upper: unknown) => { + const home = await createTestTempDirectory("rea-layered-environment-"); + const directory = join(home, ".config/opencode"); + await mkdir(directory, { recursive: true }); + const paths = [ + join(directory, "opencode.json"), + join(directory, "opencode.jsonc"), + ]; + const sources = [lower, upper].map((environment) => + JSON.stringify({ + mcp: { + rea: { + ...clientRegistrationEntry("opencode", command, {}), + environment, + }, + }, + }), + ); + for (const [index, path] of paths.entries()) + await writeFile(path, sources[index]!); + const client = { + name: "opencode", + format: "opencode", + configPath: paths[1]!, + configPaths: paths, + } as const; + const status = async () => + ( + await readClientRegistrationStatuses(home, "rea", { + environment: { XDG_CONFIG_HOME: join(home, ".config") }, + }) + ).find(({ client }) => client === "opencode"); + const unchanged = async () => { + for (const [index, path] of paths.entries()) { + expect(await readFile(path, "utf8")).toBe(sources[index]); + await expect(access(`${path}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + } + }; + return { client, paths, sources, status, unchanged }; +}; + +it("validates language/compiler settings in the effective OpenCode environment", async () => { + const fixture = await layered( + { REA_GHIDRA_LANGUAGE_ID: language }, + { REA_GHIDRA_COMPILER_SPEC_ID: compiler }, + ); + expect(await inspectClientConfiguration(fixture.client, {}, command)).toEqual( + { status: "already_current" }, + ); + expect(await clientConfigurationAligned(fixture.client, {}, command)).toBe( + true, + ); + expect(await fixture.status()).toMatchObject({ state: "aligned" }); + expect( + await configureClientConfiguration(fixture.client, {}, command), + ).toEqual({ status: "unchanged" }); + await fixture.unchanged(); +}); + +it("rejects a companion absent from the complete layered environment", async () => { + const fixture = await layered( + { CALLER_SETTING: "keep" }, + { REA_GHIDRA_COMPILER_SPEC_ID: compiler }, + ); + expect( + await inspectClientConfiguration(fixture.client, {}, command), + ).toMatchObject({ + status: "invalid", + remediation: expect.stringContaining("REA_GHIDRA_COMPILER_SPEC_ID"), + }); + expect(await clientConfigurationAligned(fixture.client, {}, command)).toBe( + false, + ); + expect(await fixture.status()).toMatchObject({ + state: "invalid", + remediation: expect.stringContaining("REA_GHIDRA_COMPILER_SPEC_ID"), + }); + expect( + await configureClientConfiguration(fixture.client, {}, command), + ).toMatchObject({ status: "failed" }); + await fixture.unchanged(); +}); + +it("keeps higher-priority values and does not copy inherited settings into the write layer", async () => { + const fixture = await layered( + { REA_GHIDRA_LANGUAGE_ID: language, HOPPER_LOADER_ARGS_JSON: '["lower"]' }, + { REA_GHIDRA_COMPILER_SPEC_ID: compiler, HOPPER_LOADER_ARGS_JSON: '[""]' }, + ); + expect( + await configureClientConfiguration( + fixture.client, + { REA_LOG_LEVEL: "debug" }, + command, + ), + ).toMatchObject({ status: "configured" }); + expect(await readFile(fixture.paths[0]!, "utf8")).toBe(fixture.sources[0]); + const configured = parseClientConfiguration( + await readFile(fixture.client.configPath, "utf8"), + "opencode", + ); + expect(effectiveClientServer(configured, "rea")).toHaveProperty( + "environment", + { + REA_GHIDRA_COMPILER_SPEC_ID: compiler, + HOPPER_LOADER_ARGS_JSON: '[""]', + REA_LOG_LEVEL: "debug", + }, + ); + expect( + await readFile(`${fixture.client.configPath}.rea.backup`, "utf8"), + ).toBe(fixture.sources[1]); + expect(await fixture.status()).toMatchObject({ state: "aligned" }); + expect(await clientConfigurationAligned(fixture.client, {}, command)).toBe( + true, + ); + expect( + await configureClientConfiguration(fixture.client, {}, command), + ).toEqual({ status: "unchanged" }); +}); + +it.each([ + ["invalid JSON", { HOPPER_LOADER_ARGS_JSON: "{" }], + ["invalid value type", { HOPPER_LOADER_ARGS_JSON: 42 }], + ["invalid environment map", []], +])( + "does not hide a lower-priority %s behind a valid override", + async (_name, lower) => { + const fixture = await layered(lower, { HOPPER_LOADER_ARGS_JSON: "[]" }); + expect( + await inspectClientConfiguration(fixture.client, {}, command), + ).toMatchObject({ + status: "invalid", + remediation: expect.stringContaining(fixture.paths[0]!), + }); + + expect(await fixture.status()).toMatchObject({ state: "invalid" }); + expect( + await configureClientConfiguration(fixture.client, {}, command), + ).toMatchObject({ status: "failed" }); + await fixture.unchanged(); + }, +); + +it("uses inherited settings when creating a missing write layer", async () => { + const fixture = await layered({ REA_GHIDRA_LANGUAGE_ID: language }, {}); + await rm(fixture.client.configPath); + const overrides = { REA_GHIDRA_COMPILER_SPEC_ID: compiler }; + expect( + await inspectClientConfiguration(fixture.client, overrides, command), + ).toEqual({ status: "create" }); + expect( + await configureClientConfiguration(fixture.client, overrides, command), + ).toMatchObject({ status: "configured" }); + const parsed = parseClientConfiguration( + await readFile(fixture.client.configPath, "utf8"), + "opencode", + ); + expect(effectiveClientServer(parsed, "rea")).toHaveProperty( + "environment", + overrides, + ); + expect(await readFile(fixture.paths[0]!, "utf8")).toBe(fixture.sources[0]); + expect(await fixture.status()).toMatchObject({ state: "aligned" }); + await expect( + access(`${fixture.client.configPath}.rea.backup`), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("replaces the selected write layer rather than the last loaded layer", async () => { + const fixture = await layered( + { REA_GHIDRA_LANGUAGE_ID: language }, + { REA_GHIDRA_COMPILER_SPEC_ID: compiler }, + ); + const client = { ...fixture.client, configPath: fixture.paths[0]! }; + const overrides = { REA_LOG_LEVEL: "debug" }; + expect(await inspectClientConfiguration(client, {}, command)).toEqual({ + status: "already_current", + }); + expect( + await configureClientConfiguration(client, overrides, command), + ).toMatchObject({ status: "configured" }); + const parsed = parseClientConfiguration( + await readFile(client.configPath, "utf8"), + "opencode", + ); + expect(effectiveClientServer(parsed, "rea")).toHaveProperty("environment", { + REA_GHIDRA_LANGUAGE_ID: language, + ...overrides, + }); + expect(await readFile(fixture.paths[1]!, "utf8")).toBe(fixture.sources[1]); + expect(await readFile(`${client.configPath}.rea.backup`, "utf8")).toBe( + fixture.sources[0], + ); + expect(await clientConfigurationAligned(client, {}, command)).toBe(true); + expect(await fixture.status()).toMatchObject({ state: "aligned" }); +}); diff --git a/tests/boundary/filesystem/clientRegistrationStatus.test.ts b/tests/boundary/filesystem/clientRegistrationStatus.test.ts index d3cb002ea..13aca1826 100644 --- a/tests/boundary/filesystem/clientRegistrationStatus.test.ts +++ b/tests/boundary/filesystem/clientRegistrationStatus.test.ts @@ -131,6 +131,32 @@ describe("client registration status", () => { }), ]); }); + + it("reports the exact requested package version when it differs from this CLI", async () => { + const home = await createTestTempDirectory( + "rea-registration-version-drift-", + ); + await mkdir(join(home, ".codex")); + const command = ["npx", "-y", "rea-agents@4.1.0", "mcp"]; + await writeFile( + join(home, ".codex/config.toml"), + `[mcp_servers.rea]\ncommand = "${command[0]}"\nargs = ["${command[1]}", "${command[2]}", "${command[3]}"]\n`, + ); + + const [status] = await readClientRegistrationStatuses(home); + + expect(status).toMatchObject({ + client: "codex", + command, + package_reference: "rea-agents@4.1.0", + state: "stale", + remediation: expect.stringContaining( + `current REA CLI package is ${PRODUCT_IDENTITY.packageVersion}`, + ), + }); + expect(status?.remediation).toContain("does not resolve package-runner"); + expect(status?.remediation).toContain("rea setup --client codex"); + }); }); describe("commandcode registration status", () => { diff --git a/tests/boundary/filesystem/dshClientConfiguration.test.ts b/tests/boundary/filesystem/dshClientConfiguration.test.ts index fd65e6b00..0142716b6 100644 --- a/tests/boundary/filesystem/dshClientConfiguration.test.ts +++ b/tests/boundary/filesystem/dshClientConfiguration.test.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -1704,6 +1704,119 @@ const dshStatus = async (home: string) => }) ).find(({ client }) => client === "dsh"); +describe("DeepSeek Harness runtime environment validation", () => { + it.each([ + ["HOPPER_LOADER_ARGS_JSON", "{"], + ["REA_REFERENCE_SECRET_PATTERNS_JSON", '[""]'], + ["REA_GHIDRA_COMPILER_SPEC_ID", "borlandcpp"], + ])("refuses invalid literal %s before writing", async (setting, value) => { + const { home, client } = await harnessClient(); + const entry = canonicalEntry(); + const original = stringifyYaml([ + { + insert: [ + { ...entry, config: { ...entry.config, env: { [setting]: value } } }, + ], + }, + ]); + await writeFile(client.configPath, original); + expect(await inspectClientConfiguration(client, {}, command)).toMatchObject( + { status: "invalid", remediation: expect.stringContaining(setting) }, + ); + expect(await dshStatus(home)).toMatchObject({ + state: "invalid", + remediation: expect.stringContaining(setting), + }); + expect( + await configureClientConfiguration(client, {}, command), + ).toMatchObject({ status: "failed" }); + expect(await readFile(client.configPath, "utf8")).toBe(original); + await expect( + access(`${client.configPath}.rea.backup`), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("keeps an expression-backed companion unknown until a literal override replaces it", async () => { + const { home, client } = await harnessClient(); + const entry = canonicalEntry(); + const document = parseDocument( + stringifyYaml([ + { + insert: [ + { + ...entry, + config: { + ...entry.config, + command: "stale", + env: { + REA_GHIDRA_COMPILER_SPEC_ID: "borlandcpp", + CALLER_SETTING: "keep", + }, + }, + }, + ], + }, + ]), + ); + for (const setting of [ + "REA_GHIDRA_LANGUAGE_ID", + "HOPPER_LOADER_ARGS_JSON", + ]) { + const expression = document.createNode(`process.env.${setting}`); + expression.tag = "tag:yaml.org,2002:js"; + document.setIn([0, "insert", 0, "config", "env", setting], expression); + } + const original = document.toString(); + await writeFile(client.configPath, original); + expect(await inspectClientConfiguration(client, {}, command)).toMatchObject( + { status: "update" }, + ); + expect( + await configureClientConfiguration(client, {}, command), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(client.configPath, "utf8"); + expect(configured).toContain("!!js process.env.REA_GHIDRA_LANGUAGE_ID"); + expect(configured).toContain("!!js process.env.HOPPER_LOADER_ARGS_JSON"); + expect(configured).toContain("CALLER_SETTING: keep"); + expect(await dshStatus(home)).toMatchObject({ state: "aligned" }); + expect( + await configureClientConfiguration(client, {}, command), + ).toMatchObject({ status: "unchanged" }); + await expect( + inspectClientConfiguration( + client, + { HOPPER_LOADER_ARGS_JSON: "{" }, + command, + ), + ).rejects.toMatchObject({ + settings: [{ setting: "HOPPER_LOADER_ARGS_JSON" }], + }); + await expect( + inspectClientConfiguration( + client, + { REA_GHIDRA_LANGUAGE_ID: "" }, + command, + ), + ).rejects.toMatchObject({ + settings: [{ setting: "REA_GHIDRA_LANGUAGE_ID" }], + }); + expect( + await inspectClientConfiguration( + client, + { + REA_GHIDRA_LANGUAGE_ID: "x86:LE:32:default", + HOPPER_LOADER_ARGS_JSON: '[""]', + }, + command, + ), + ).toMatchObject({ status: "update" }); + expect(await readFile(client.configPath, "utf8")).toBe(configured); + expect(await readFile(`${client.configPath}.rea.backup`, "utf8")).toBe( + original, + ); + }); +}); + describe("DeepSeek Harness source ownership", () => { it.each(["setup", "uninstall"])( "preserves unrelated rows in a later externally anchored operation during %s", diff --git a/tests/boundary/filesystem/installScript.test.ts b/tests/boundary/filesystem/installScript.test.ts index d7e7f78a3..861a9bdfc 100644 --- a/tests/boundary/filesystem/installScript.test.ts +++ b/tests/boundary/filesystem/installScript.test.ts @@ -186,6 +186,18 @@ describe("curl installer scenarios", { timeout: 20_000 }, () => { }); }); +describe("installer reported version", { timeout: 20_000 }, () => { + it("rejects an empty version from a successful installed command", async () => { + const fixture = await createFixture(); + await expect( + runInstaller(fixture, ["--version", "0.3.0"], { FAKE_REA_VERSION: "" }), + ).rejects.toMatchObject({ + stderr: + "REA installation failed: the installed REA version could not be read. Reinstall the requested version, then retry.\n", + }); + }); +}); + describe("installer TTY fallback", { timeout: 20_000 }, () => { it("skips automatic setup when stderr is redirected", async () => { const fixture = await createFixture(); @@ -392,6 +404,20 @@ describe("native macOS installer", { timeout: 20_000 }, () => { }); describe("installer semantic version parsing", { timeout: 20_000 }, () => { + it("rejects an explicitly empty version instead of selecting latest", async () => { + const fixture = await createFixture(); + await expect( + runInstaller(fixture, ["--version", "", "--dry-run"]), + ).rejects.toMatchObject({ + stdout: "", + stderr: + "REA installation failed: version must be an exact semantic version.\n", + }); + await expect(readFile(fixture.npmLog, "utf8")).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + it.each([ "1.2.3-01", "1.2.3-$(printf unexpected)", @@ -559,7 +585,7 @@ if [ -n "\${FAKE_CURL_BODY:-}" ]; then printf '%s' "$FAKE_CURL_BODY"; else print `#!/bin/sh printf '%s\n' "$*" >> "$FAKE_REA_LOG" [ "\${FAKE_REA_VERSION_FAIL:-}" = "1" ] && exit 1 -[ "$1" = "--version" ] && printf '%s\n' "\${FAKE_REA_VERSION:-0.3.0}" +[ "$1" = "--version" ] && printf '%s\n' "\${FAKE_REA_VERSION-0.3.0}" exit 0 `, ), diff --git a/tests/boundary/filesystem/packageClientBackup.test.ts b/tests/boundary/filesystem/packageClientBackup.test.ts new file mode 100644 index 000000000..cd57fda10 --- /dev/null +++ b/tests/boundary/filesystem/packageClientBackup.test.ts @@ -0,0 +1,47 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { assertClientConfig } from "../../../scripts/verify-package-setup.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("rejects a backup replaced with the post-setup client configuration", async () => { + const directory = await createTestTempDirectory("rea-client-backup-"); + const path = join(directory, "client.json"); + const original = Buffer.from('{"existing":true,"model":"caller"}\n'); + const configured = JSON.stringify({ + existing: true, + model: "caller", + mcpServers: { rea: { command: process.execPath, args: ["mcp"] } }, + }); + await writeFile(path, configured); + await writeFile(`${path}.rea.backup`, original); + await assertClientConfig(process.execPath, path, original); + + await writeFile(`${path}.rea.backup`, configured); + await expect( + assertClientConfig(process.execPath, path, original), + ).rejects.toThrow("packaged client backup failed"); + expect(await readFile(path, "utf8")).toBe(configured); +}); + +it("compares backup bytes even when invalid UTF-8 decodes to the same text", async () => { + const directory = await createTestTempDirectory("rea-client-backup-bytes-"); + const path = join(directory, "client.json"); + const original = Buffer.from('{"existing":true,"note":"\uFFFD"}\n'); + const configured = JSON.stringify({ + existing: true, + mcpServers: { rea: { command: process.execPath, args: ["mcp"] } }, + }); + const changed = Buffer.concat([ + Buffer.from('{"existing":true,"note":"'), + Buffer.from([0xff]), + Buffer.from('"}\n'), + ]); + expect(changed.toString("utf8")).toBe(original.toString("utf8")); + await writeFile(path, configured); + await writeFile(`${path}.rea.backup`, changed); + await expect( + assertClientConfig(process.execPath, path, original), + ).rejects.toThrow("packaged client backup failed"); +}); diff --git a/tests/boundary/filesystem/setupSkill.test.ts b/tests/boundary/filesystem/setupSkill.test.ts index af4df0625..83b4d48cd 100644 --- a/tests/boundary/filesystem/setupSkill.test.ts +++ b/tests/boundary/filesystem/setupSkill.test.ts @@ -39,6 +39,90 @@ const expectPermissionMode = async ( expect((await stat(path)).mode & 0o777).toBe(expected); }; +describe("skill recovery backup selection", () => { + it("advertises a current recovery snapshot when the first backup is historical", async () => { + const home = await createTestTempDirectory("rea-skill-historical-backup-"); + const [destination] = skillDestinations(home, []); + if (destination === undefined) throw new Error("Missing shared skill root"); + const manifest = join(destination.path, "SKILL.md"); + const current = "currently installed instructions V2\n"; + const oldSnapshot = "historical instructions V0\n"; + await mkdir(dirname(manifest), { recursive: true }); + await writeFile(manifest, current); + await writeFile(`${manifest}.rea.backup`, oldSnapshot); + let manifestWrites = 0; + const operations: SkillInstallFileOperations = { + mkdir: async (path) => + mkdir(path, { recursive: true }).then(() => undefined), + backup: async (source, target) => + copyFile(source, target, fsConstants.COPYFILE_EXCL), + write: async (path, content, mode) => { + if (path === manifest && ++manifestWrites === 2) + throw new Error("injected rollback failure"); + await writeFileAtomic(path, content, { encoding: "utf8", mode }); + if (path === manifest && manifestWrites === 1) + throw new Error("injected failure after publication"); + }, + remove: async (path) => rm(path, { force: true }), + }; + + const result = await installCanonicalSkill( + home, + [], + {}, + process.platform, + operations, + ); + + if (result.ok) throw new Error("Expected the injected write failure"); + expect(result.error.rollbackFailures).toEqual([ + { path: manifest, reason: "injected rollback failure" }, + ]); + const recovery = result.error.recoveryBackupPaths.find((path) => + path.startsWith(`${manifest}.rea.backup.`), + ); + expect(recovery).toBeDefined(); + expect(await readFile(recovery!, "utf8")).toBe(current); + expect(await readFile(`${manifest}.rea.backup`, "utf8")).toBe(oldSnapshot); + }); +}); + +describe("skill recovery byte preservation", () => { + it("backs up and restores the exact bytes of invalid UTF-8 before replacing a skill", async () => { + const home = await createTestTempDirectory("rea-skill-damaged-bytes-"); + const [destination] = skillDestinations(home, []); + if (destination === undefined) throw new Error("Missing shared skill root"); + const manifest = join(destination.path, "SKILL.md"); + const original = Buffer.from([0x63, 0x61, 0x66, 0xe9, 0xff, 0x0a]); + const firstBackup = `${manifest}.rea.backup`; + const recovery = `${firstBackup}.${createHash("sha256").update(original).digest("hex")}.recovery`; + await mkdir(dirname(manifest), { recursive: true }); + await writeFile(manifest, original); + await writeFile(firstBackup, "historical instructions V0\n"); + let manifestWrites = 0; + const result = await installCanonicalSkill(home, [], {}, process.platform, { + mkdir: async (path) => + mkdir(path, { recursive: true }).then(() => undefined), + backup: async (source, target) => + copyFile(source, target, fsConstants.COPYFILE_EXCL), + write: async (path, content, mode) => { + if (path === manifest && ++manifestWrites === 2) + throw new Error("injected rollback failure"); + await writeFileAtomic(path, content, { encoding: "utf8", mode }); + if (path === manifest && manifestWrites === 1) + throw new Error("injected failure after publication"); + }, + remove: async (path) => rm(path, { force: true }), + }); + if (result.ok) throw new Error("Expected the injected write failure"); + expect(result.error.recoveryBackupPaths).toContain(recovery); + expect(await readFile(recovery)).toEqual(original); + expect(await readFile(firstBackup, "utf8")).toBe( + "historical instructions V0\n", + ); + }); +}); + describe("canonical skill transaction", () => { it("does not roll back files when backup preparation fails", async () => { const home = await createTestTempDirectory("rea-skill-backup-failure-"); @@ -105,7 +189,7 @@ describe("canonical skill transaction", () => { await writeFile(connection, "old connection\n", { mode: 0o644 }); await writeFile(native, "old native\n", { mode: 0o600 }); const writes = new Map(); - const nativeWrites: string[] = []; + const nativeWrites: (string | Buffer)[] = []; const operations: SkillInstallFileOperations = { mkdir: async (path) => { await mkdir(path, { recursive: true }); @@ -154,7 +238,7 @@ describe("canonical skill transaction", () => { expect(await readFile(native, "utf8")).not.toBe("old native\n"); expect(await readFile(`${native}.rea.backup`, "utf8")).toBe("old native\n"); expect(nativeWrites).toHaveLength(2); - expect(nativeWrites[1]).toBe("old native\n"); + expect(nativeWrites[1]).toEqual(Buffer.from("old native\n", "utf8")); expect(result.error.message).toContain( `Existing-file backups remain at ${manifest}.rea.backup`, ); diff --git a/tests/boundary/filesystem/zcodeClientConfiguration.test.ts b/tests/boundary/filesystem/zcodeClientConfiguration.test.ts index 4b00a2e01..268747356 100644 --- a/tests/boundary/filesystem/zcodeClientConfiguration.test.ts +++ b/tests/boundary/filesystem/zcodeClientConfiguration.test.ts @@ -13,7 +13,10 @@ import { } from "../../../src/application/SetupClientConfiguration.js"; import { skillDestinations } from "../../../src/application/SetupSkill.js"; import { supportedClients } from "../../../src/application/SupportedClients.js"; -import { systemUninstallHost } from "../../../src/application/Uninstall.js"; +import { + runUninstall, + systemUninstallHost, +} from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; @@ -257,6 +260,91 @@ describe("ZCode .agents fallback preservation", () => { }); }); +describe("ZCode fallback uninstall", () => { + const owned = { command: command[0], args: command.slice(1) }; + const writeFallback = async (home: string, servers: object) => { + await mkdir(join(home, ".agents"), { recursive: true }); + const text = `${JSON.stringify({ ui: { keep: true }, mcpServers: servers }, null, 2)}\n`; + await writeFile(fallbackPath(home), text); + return text; + }; + + it.each(["comment", "trailing-comma", "bom"])( + "fails closed on strict fallback syntax: %s", + async (mode) => { + const { home } = await zcodeClient(); + const valid = await writeFallback(home, { + rea: owned, + other: { command: "other-server" }, + }); + const original = + mode === "comment" + ? `// unsupported comment\n${valid}` + : mode === "bom" + ? `\uFEFF${valid}` + : valid.replace(/\}\s*$/u, ",}"); + await writeFile(fallbackPath(home), original); + const result = await runUninstall(false, systemUninstallHost(home)); + expect(result.items).toContainEqual( + expect.objectContaining({ name: "zcode", status: "failed" }), + ); + expect(await readFile(fallbackPath(home), "utf8")).toBe(original); + await expect( + readFile(`${fallbackPath(home)}.rea.backup`), + ).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it("removes the REA-owned fallback entry with the native registration", async () => { + const { home, client } = await zcodeClient(); + await writeFile( + client.configPath, + JSON.stringify({ mcp: { servers: { rea: owned } } }), + ); + const original = await writeFallback(home, { + rea: owned, + other: { command: "other-server" }, + }); + + const result = await runUninstall(false, systemUninstallHost(home)); + + expect(result.status).toBe("complete"); + expect(result.items.find(({ name }) => name === "zcode")?.status).toBe( + "removed", + ); + const native = JSON.parse(await readFile(client.configPath, "utf8")); + expect(native.mcp.servers).toEqual({}); + const fallback = JSON.parse(await readFile(fallbackPath(home), "utf8")); + expect(fallback).toEqual({ + ui: { keep: true }, + mcpServers: { other: { command: "other-server" } }, + }); + expect(await readFile(`${fallbackPath(home)}.rea.backup`, "utf8")).toBe( + original, + ); + }); + + it("removes a fallback-only REA entry and keeps a foreign rea entry", async () => { + for (const [servers, expected] of [ + [{ rea: owned }, {}], + [ + { rea: { command: "someone-else" } }, + { rea: { command: "someone-else" } }, + ], + ] as const) { + const { home } = await zcodeClient(); + await writeFallback(home, servers); + + expect( + (await runUninstall(false, systemUninstallHost(home))).status, + ).toBe("complete"); + expect( + JSON.parse(await readFile(fallbackPath(home), "utf8")).mcpServers, + ).toEqual(expected); + } + }); +}); + describe("ZCode fallback admission", () => { it.each([{ PORT: 123 }, { REA_MCP_INPUT_SCHEMA_PROFILE: "unknown" }])( "refuses malformed fallback REA environment %j before native writes", From 2dc89f81be807de88b7b19b5f4ffcc7bff171068 Mon Sep 17 00:00:00 2001 From: JANG BYUNGKUN Date: Wed, 14 Oct 2026 09:27:37 +0900 Subject: [PATCH 03/26] docs(readme): sync Korean README with the English README (#3046) --- docs/readme/README_ko.md | 47 +++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index 1c1133254..e6affdb19 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -20,7 +20,7 @@ **[웹사이트](https://rea.tools/ko/) · [가이드](https://rea.tools/ko/guides/) · [사례](https://rea.tools/ko/showcase/)** -[빠른 시작](#빠른-시작) · [REA의 작동 방식](#rea의-작동-방식) · [분석할 수 있는 대상](#분석할-수-있는-대상) · [사례](#사례) · [자주 묻는 질문](#자주-묻는-질문) · [GNU objdump 및 Rizin 도구](../reverse-engineering-tools.md) · [문서](#문서) +[빠른 시작](#빠른-시작) · [REA의 작동 방식](#rea의-작동-방식) · [분석할 수 있는 대상](#분석할-수-있는-대상) · [사례](#사례) · [자주 묻는 질문](#자주-묻는-질문) · [문서](#문서) npx rea-agents setup @@ -56,7 +56,7 @@ REA는 에이전트를 분석 도구에 연결해 네이티브 바이너리, JavaScript와 Electron 앱, .NET 어셈블리, 웹사이트를 검사할 수 있게 합니다. 터미널에서도 같은 도구를 사용할 수 있습니다. 분석은 로컬에서 실행되며, 결과에는 각 결론의 근거와 한계가 포함됩니다. -설정 과정은 에이전트에 REA를 등록하고 해당 버전에 맞는 워크플로 지침을 설치합니다. 네이티브 분석에는 기존 Hopper 또는 Ghidra 설치를 사용할 수 있습니다. 승인하면 설정 과정에서 Hopper를 설치할 수도 있습니다. 정적 JavaScript 분석에는 두 엔진 모두 필요하지 않습니다. +설정 과정은 에이전트에 REA를 등록하고 해당 버전에 맞는 워크플로 지침을 설치합니다. 네이티브 분석에는 기존 Hopper, Ghidra 또는 IDA 설치를 사용할 수 있습니다. 승인하면 설정 과정에서 Hopper를 설치할 수도 있습니다. 정적 JavaScript 분석에는 네이티브 분석 엔진이 필요하지 않습니다. > **[REA 웹사이트](https://rea.tools/ko/)**에서 설정 방법, 그림으로 설명한 가이드, 실제 사례를 확인하세요. @@ -105,12 +105,14 @@ rea --help REA는 빠르게 바뀌며 새 릴리스에는 버그 수정이 자주 포함됩니다. 설치된 버전을 최신으로 유지하세요. -npm으로 설치한 CLI에서는 다음 명령을 사용합니다. +`npm install --global` 또는 `bun add --global`로 설치한 CLI에서는 다음 명령을 사용합니다. ```bash rea update ``` +Bun 전역 업데이트에는 Bun 1.2.17 이상이 필요합니다. 이전 버전의 Bun을 사용한다면 `rea update`를 실행하기 전에 Bun을 직접 업그레이드하세요. + 에이전트 등록과 스킬을 갱신하려면 업데이트 과정에서 출력된 설정 명령을 실행하세요. `npx`를 사용한다면 다음 명령으로 에이전트 설정을 갱신하세요. @@ -119,7 +121,7 @@ rea update npx rea-agents@latest setup ``` -설정 변경 사항을 검토하고 에이전트를 다시 시작하세요. 일회성 CLI 작업에는 `npx rea-agents@latest` 뒤에 필요한 명령을 붙여 실행하세요. +설정 변경 사항을 검토하고 에이전트를 다시 시작하세요. 일회성 CLI 작업에는 `npx rea-agents@latest` 뒤에 필요한 명령을 붙여 실행하세요. Bun을 사용한다면 위 명령의 `npx rea-agents@latest`를 `bunx rea-agents@latest`로 바꿔 실행하세요. ## REA의 작동 방식 @@ -139,14 +141,16 @@ REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니 | --------------------- | ----------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | | 네이티브 바이너리 | 의사 코드, 어셈블리, 문자열, 심볼, 호출, 참조 | Hopper, Ghidra 또는 IDA; [네이티브 분석](https://rea.tools/ko/guides/native/) | | 오프라인 ELF 레이아웃 | 섹션, 세그먼트, 원본 심볼/재배치 정보, 정적 분석으로 얻은 보안 완화 기법 후보 | Linux x64에서 호출자가 제공하는 pwntools; [바이너리 진단](../binary-diagnostics.md) | +| Go 바이너리 | 내장된 컴파일러 버전, 모듈, `replace` 지시어, 빌드 설정 | REA 내장 ELF/PE/thin Mach-O 정적 검사; Go 설치 불필요 | +| Rust 바이너리 | cargo-auditable이 내장한 크레이트 이름, 버전, 소스, 의존성 관계 | REA 내장 ELF/PE/thin Mach-O 정적 검사; [Rust 바이너리](../rust-binaries.md) | +| WASM 아티팩트 | 유효성 검사, 섹션, import/export의 정확한 WAT 형식, 디코딩된 WAT | 호출자 제공 WABT 1.0.42; `inspect_wasm_artifact` | | EVM 바이트코드 | 디스패치 선택자, 바이트 오프셋, 추론한 인자와 상태 변경 가능 여부 | Linux x64의 로컬 원시 바이트/16진수 입력 파일; [오프라인 EVM 가이드](../evm-bytecode.md) | -| Rust binaries | Embedded cargo-auditable dependency list | [Rust binaries](../rust-binaries.md) | +| SQLite 데이터베이스 | 테이블, 열, 인덱스, 뷰, 트리거, 선택한 일반 테이블의 행 | 로컬 데이터베이스/WAL 스냅샷; Node.js 24.15 이상의 24.x 또는 26 이상(네이티브 SQLite 한도 적용) | | 기록된 Linux 크래시 | 원시 note 레코드, 기록된 모든 스레드의 레지스터/시그널, 선택적 매핑 후보 | Linux x64에서 호출자가 제공하는 pwntools; GDB/pwndbg는 선택 사항; [기록된 크래시](../recorded-crashes.md) | | JavaScript / Electron | 모듈, 임포트, 소스 맵, 라우트, IPC, 네이티브 애드온 관계 | Node.js와 npm; [애플리케이션 분석](https://rea.tools/ko/guides/javascript/) | | 웹사이트 | 페이지 구조, 스크립트, 네트워크 관찰 결과, 요청한 스크린샷 | Chrome 계열 브라우저; [브라우저 분석](https://rea.tools/ko/guides/browser/) | -| 저장된 네트워크 캡처 | 요청, 응답, 접근 가능한 페이로드, 소스 위치 | HAR; mitmproxy 네이티브 캡처에는 Linux의 mitmdump; [캡처 가이드](../web-network-captures.md) | -| 패킷 캡처 | 엔드포인트, DNS, TLS SNI, 프로토콜 분포, HTTP 요청 | 호출자 제공 tshark 및 capinfos; [pcap 가이드](../pcap-analysis.md) | -| .NET 어셈블리 | 메타데이터, CIL 명령, 선언된 네이티브 의존성, 빌드 비교 | 정적 검사; [관리 코드 가이드](../managed-code-analysis.md) | +| 저장된 네트워크 캡처 | 요청, 응답, 접근 가능한 페이로드, 소스 위치 | HAR 또는 Fiddler Everywhere SAZ; mitmproxy 네이티브 캡처에는 Linux의 mitmdump; [캡처 가이드](../web-network-captures.md) | +| .NET 어셈블리 | 메타데이터, CIL 명령, 선언된 네이티브 의존성, 빌드 비교 | 별도 도구 불필요(정적 검사만); [관리 코드 가이드](../managed-code-analysis.md) | | Android APK | 매니페스트 선언, 클래스, 디컴파일된 메서드, 참조 | Linux/macOS/Windows x64의 헤드리스 JADX와 전체 JDK; [Android 가이드](../android-analysis.md) | | Android 기기 | 기기, 패키지, 프로세스, 로그, 덤프, 화면 캡처, APK/파일 전송 | 호출자 제공 adb(에뮬레이터 또는 실제 기기); [ADB 가이드](../adb-device-analysis.md) | | JEB 프로젝트 | 엔진 기반 프로젝트 유닛, 아티팩트 해시, 타입/메서드 의사코드 | 호출자가 시작하고 MCP를 제공하는 JEB 클라이언트(REA_JEB_MCP_URL로 선택); [JEB 가이드](../jeb-analysis.md) | @@ -154,15 +158,18 @@ REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니 | Flutter 앱 | Dart 스냅샷 해시, ABI별 라이브러리 해시, 엔진 빌드 출처 | 없음 (정적 APK 파싱); [Flutter 가이드](../flutter-analysis.md) | | 펌웨어 | 영역, 추출 결과, 네이티브 분석으로 전달할 정보 | Linux의 Binwalk / Unblob; [펌웨어 가이드](../firmware-analysis.md) | | 기록된 패킷 | 선택 필드, 반복 값, 필터 및 디코딩 규칙 | Linux: TShark 4.x + prlimit; [캡처 가이드](../packet-captures.md) | +| 패킷 요약 | 엔드포인트, DNS, TLS SNI, 프로토콜 분포, HTTP 요청 | 호출자 제공 tshark 및 capinfos(Linux에서 검증됨); [pcap 가이드](../pcap-analysis.md) | | 기록된 논리 신호 | 디코더 주석, 샘플 범위, 진단 정보 | Linux: sigrok-cli 0.7.2 + prlimit; [신호 기록 가이드](../signal-recordings.md) | | 패키지 및 리소스 | 파일 목록, 다이제스트, plist, Apple 번들 구조, 추출한 리소스 | [아티팩트 및 JavaScript 가이드](../javascript-artifact-reconstruction.md), [Apple 애플리케이션](../apple-application-analysis.md) | | 프로세스 동작 | 터미널 출력, 상호작용, 종료 및 파일 시스템 관찰 결과, 실행 비교 | 네이티브 PTY를 지원하는 Linux/macOS; [프로세스 캡처](../process-capture.md) | 정적 JavaScript 및 .NET 검사는 제공된 파일을 읽으며 애플리케이션을 실행하지 않습니다. 런타임 캡처는 사용자의 권한으로 선택한 대상을 실행하거나 대상과 상호작용합니다. 각 런타임 가이드에서 그 영향을 설명합니다. +오프라인 네이티브 바이너리 검사에는 호출자가 제공하는 GNU objdump와 Rizin도 사용할 수 있습니다. 제공자 설정, CLI 명령, Evidence의 한계는 [리버스 엔지니어링 도구 가이드](../reverse-engineering-tools.md)를 참고하세요. REA는 objdump와 Rizin을 설치하거나 업데이트하지 않습니다. + -지원하는 네이티브 형식과 호스트는 제공자마다 다릅니다. [Hopper 및 Ghidra 설정](../installation.md#hopper), [IDA 가이드](../ida-provider.md), [실험적 Windows Ghidra 지원](../windows-ghidra-p0.md)을 참고하세요. Ghidra는 [16비트 DOS 분석](../ghidra-dos.md)도 지원합니다. 제공자 선택은 [CLI 가이드](../cli.md#choose-a-provider)를 참고하세요. 최신 npm 릴리스 이후 추가된 기능은 [릴리스에서의 사용 가능 여부](../installation.md#released-package-and-main)를 확인하세요. +지원하는 네이티브 형식과 호스트는 제공자마다 다릅니다. [Hopper 및 Ghidra 설정](../installation.md#hopper), [IDA 가이드](../ida-provider.md)(선택 사항인 네이티브 MCP 브리지 포함), [실험적 Windows Ghidra 지원](../windows-ghidra-p0.md)을 참고하세요. Ghidra는 [16비트 DOS 분석](../ghidra-dos.md)도 지원합니다. 큰 바이너리를 분석할 때는 `REA_GHIDRA_STARTUP_TIMEOUT_MS` 또는 `REA_HOPPER_STARTUP_TIMEOUT_MS`로 제공자의 시작 제한 시간을 늘리세요. 제공자 선택은 [CLI 가이드](../cli.md#choose-a-provider)를 참고하세요. 최신 npm 릴리스 이후 추가된 기능은 [릴리스에서의 사용 가능 여부](../installation.md#released-package-and-main)를 확인하세요. ## 사례 @@ -209,7 +216,7 @@ REA로 흥미로운 대상을 분석했다면 알려 주세요. [issue](https://
Hopper를 먼저 실행해야 하나요? -REA는 작업에 필요할 때 Hopper를 실행합니다. macOS에서는 처음 실행할 때 데모 모드를 선택하거나 라이선스를 활성화하라는 대화 상자가 나타날 수 있습니다. [Hopper 실행 및 문제 해결](../installation.md#launcher-paths-and-troubleshooting)을 참고하세요. +REA는 작업에 필요할 때 Hopper를 실행합니다. macOS에서는 처음 실행할 때 데모 모드를 선택하거나 라이선스를 활성화하라는 대화 상자가 나타날 수 있습니다. 첫 CLI/MCP 작업이나 무인 작업을 실행하기 전에 같은 사용자 세션에서 이 대화 상자를 완료하세요. 필요하면 Hopper를 직접 여세요. [Hopper 실행 및 문제 해결](../installation.md#launcher-paths-and-troubleshooting)을 참고하세요.
@@ -239,7 +246,7 @@ REA는 대상을 로컬에서 분석합니다. 에이전트는 도구 결과를 먼저 업데이트하세요. 최근 릴리스에서 이미 수정되었을 수 있습니다. -npm으로 설치한 CLI에서는 다음 명령을 사용합니다. +`npm install --global` 또는 `bun add --global`로 설치한 CLI에서는 다음 명령을 사용합니다. ```bash rea update @@ -286,11 +293,9 @@ REA를 사용하고, 버그를 보고하고, 기능을 요청하고, 빌드를 책임 있는 사용에 관한 안내는 [DISCLAIMER.md](../../DISCLAIMER.md)를 참고하세요. -REA는 법을 준수하는 연구, 교육, 상호운용성 확보, 소프트웨어 보존, 그리고 본인이 소유하거나 조사 권한을 가진 소프트웨어의 분석을 위한 도구입니다. +불법적이거나 허가되지 않은 사용에 대한 프로젝트의 입장은 모든 REA 버전, 배포 채널, 플랫폼에 적용되며, 각 사본에는 해당 라이선스가 그대로 유지됩니다. -The project's position on illegal or unauthorized use covers all REA -versions, distribution channels, and platforms; each copy retains its -applicable licenses. +REA는 법을 준수하는 연구, 교육, 상호운용성 확보, 소프트웨어 보존, 그리고 본인이 소유하거나 조사 권한을 가진 소프트웨어의 분석을 위한 도구입니다. ## 기여 @@ -302,14 +307,6 @@ REA에 도움을 보태 주세요! [issue를 등록](https://github.com/morluto/ [MIT](../../LICENSE) -REA-authored code is licensed under MIT. Third-party material retains its own -licenses and copyright notices; see [Third-party notices](../../THIRD_PARTY_NOTICES.md) -for the dependency inventory, preserved license texts, and external-tool terms. +REA가 작성한 코드는 MIT 라이선스를 따릅니다. 제3자 자료에는 각자의 라이선스와 저작권 고지가 그대로 유지됩니다. 의존성 목록, 보존된 라이선스 원문, 외부 도구 약관은 [제3자 고지](../../THIRD_PARTY_NOTICES.md)를 참고하세요. -**You must legally obtain and be authorized to use commercial tools**, including -IDA Pro / Hex-Rays decompilers, Hopper, and JEB, under their applicable licenses. -REA grants no tool license, key, or activation entitlement. Trials and demos -remain subject to vendor terms, including user, feature, automation, and output -sharing restrictions. Cracked copies, stolen keys, unauthorized key sharing, -and bypassing license restrictions are not authorized. This guidance does not -amend MIT or third-party licenses. +IDA Pro / Hex-Rays 디컴파일러, Hopper, JEB 등 상용 도구는 해당 라이선스에 따라 **합법적으로 취득하고 사용 권한을 갖춰야 합니다.** REA는 어떤 도구의 라이선스, 키, 활성화 권한도 부여하지 않습니다. 체험판과 데모는 사용자, 기능, 자동화, 출력 공유 제한을 포함한 공급업체 약관을 따릅니다. 크랙 버전, 도난 키, 무단 키 공유, 라이선스 제한 우회는 허용되지 않습니다. 이 안내는 MIT 라이선스나 제3자 라이선스를 변경하지 않습니다. From d4bd65c53c0254d0195cc7ab8da163dcadfd3b9b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:28:14 +0800 Subject: [PATCH 04/26] fix(providers)!: preserve readiness, coverage, and diagnostic ownership Share tagged readiness observations, retain partial provider facts, authenticate bridge identity, bound diagnostics, and preserve cancellation and lost-session reasons. Fail closed for unparsed MIPS ABI attributes and macOS saved Hopper databases without a script-capable bridge route. Reviewed fixes: #2010, #2108, #2513, #2695, #2731, #2771, #2828, #2867, #2868, #2870, #2880, #2919, #2945, #2961, #2981, #2998, #3000, #3002. Validation: 285 focused source/boundary tests passed. Real Ghidra, IDA, Linux Hopper, and Windows provider runs were not repeated. BREAKING CHANGE: MIPS targets with unresolved GNU ABI attributes and saved Hopper databases on macOS are refused rather than advertised as supported. --- bridge/ghidra/ReaGhidraBridge.java | 574 +++++-- bridge/hopper_bridge.py | 7 +- docs/adb-device-analysis.md | 68 +- docs/ghidra-mips.md | 16 +- docs/ghidra-provider.md | 17 +- docs/go-binaries.md | 14 +- docs/ida-provider.md | 4 +- docs/windows-ghidra-p0.md | 15 +- src/adb/AdbDeviceOutput.ts | 19 + src/adb/AdbProvider.test.ts | 99 +- src/adb/AdbProvider.ts | 119 +- src/adb/AdbShellOutput.ts | 2 +- src/android/JadxConfiguration.ts | 166 +- src/android/JadxRelease.ts | 20 + src/apktool/ApktoolProvider.test.ts | 35 +- src/apktool/ApktoolProvider.ts | 6 +- src/application/AnalysisProvider.ts | 7 + src/application/CapabilityInventory.test.ts | 44 +- src/application/CapabilityInventory.ts | 6 +- src/application/EnhancedTools.ts | 44 +- src/application/MipsElfMetadata.ts | 6 + src/application/WindowsCapabilities.ts | 38 +- .../binary/AnalysisProviderRegistry.test.ts | 75 +- .../binary/AnalysisProviderRegistry.ts | 44 +- src/ghidra/GhidraAnalysisProfile.ts | 10 +- src/ghidra/GhidraClient.ts | 63 +- src/ghidra/GhidraDoctor.test.ts | 122 +- src/ghidra/GhidraDoctor.ts | 33 +- src/ghidra/GhidraFunctionValues.test.ts | 145 ++ src/ghidra/GhidraFunctionValues.ts | 17 +- src/ghidra/GhidraInstallation.ts | 47 +- src/ghidra/GhidraJavaLaunch.ts | 72 +- src/ghidra/GhidraLanguageCatalog.ts | 95 +- src/ghidra/GhidraLaunchFailure.test.ts | 32 +- src/ghidra/GhidraMipsProfile.ts | 5 + src/ghidra/GhidraProvider.behavior.test.ts | 254 +++- src/ghidra/GhidraProvider.ts | 63 +- src/ghidra/GhidraProviderCapabilities.ts | 32 +- src/ghidra/GhidraProviderClient.ts | 70 +- src/ghidra/GhidraSessionError.ts | 53 +- src/ghidra/GhidraSnapshotCache.test.ts | 158 +- src/go/GoBinaryProvider.ts | 4 +- src/hopper/HopperProvider.ts | 8 + src/hopper/HopperProviderDiscovery.test.ts | 25 +- src/ida/IdaContainment.test.ts | 47 +- src/ida/IdaDiagnosticRedactor.test.ts | 563 +++++++ src/ida/IdaDiagnosticRedactor.ts | 243 +++ src/ida/IdaDiagnosticWriter.ts | 85 ++ src/ida/IdaDoctor.ts | 13 +- src/ida/IdaJsonDiagnostic.test.ts | 337 +++++ src/ida/IdaJsonDiagnostic.ts | 216 +++ src/ida/IdaMcpConnection.test.ts | 286 +++- src/ida/IdaMcpConnection.ts | 322 +++- src/ida/IdaMcpEnvironment.test.ts | 13 +- src/ida/IdaOperationFailures.test.ts | 192 ++- src/ida/IdaOperationRunner.ts | 12 +- src/ida/IdaProvider.ts | 6 +- src/ida/IdaProviderCapabilities.ts | 1 - src/ida/IdaSessionClient.test.ts | 61 +- src/jeb/JebMcpEndpoint.test.ts | 9 + src/jeb/JebMcpEndpoint.ts | 6 + src/wasm/WabtArtifactProvider.test.ts | 43 + src/wasm/WabtArtifactProvider.ts | 1 + src/windows/WindowsNativeLoader.test.ts | 30 +- src/windows/WindowsNativeLoader.ts | 23 +- .../filesystem/ghidraJavaProbe.test.ts | 103 ++ .../ghidraLanguageCatalogBound.test.ts | 86 ++ .../ghidraLanguageCatalogCleanup.test.ts | 203 +++ .../boundary/filesystem/mipsAbiFlags.test.ts | 27 +- tests/boundary/mcp/ghidraEvidenceMcp.test.ts | 3 +- tests/boundary/mcp/idaTransport.test.ts | 1329 ++++++++++++++++- .../providers/ghidra/ghidraClient.test.ts | 30 + .../hopper/hopperClient.cleanupRetry.test.ts | 3 + .../hopper/hopperClient.restart.test.ts | 12 +- tests/composition/idaProvider.test.ts | 23 +- .../conformance/ghidra/ReaNoReturnProbe.java | 115 ++ .../ghidra/windowsCapabilities.test.ts | 18 + tests/fixtures/fakeGhidra.mjs | 8 + tests/fixtures/fakeHopper.mjs | 1 + tests/fixtures/ghidraHandshakeProbe.py | 107 ++ 80 files changed, 6696 insertions(+), 634 deletions(-) create mode 100644 src/ida/IdaDiagnosticRedactor.test.ts create mode 100644 src/ida/IdaDiagnosticRedactor.ts create mode 100644 src/ida/IdaDiagnosticWriter.ts create mode 100644 src/ida/IdaJsonDiagnostic.test.ts create mode 100644 src/ida/IdaJsonDiagnostic.ts create mode 100644 tests/boundary/filesystem/ghidraLanguageCatalogBound.test.ts create mode 100644 tests/boundary/filesystem/ghidraLanguageCatalogCleanup.test.ts create mode 100644 tests/fixtures/ghidraHandshakeProbe.py diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index f91f003e4..df639a673 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -5,9 +5,11 @@ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; import java.net.StandardProtocolFamily; import java.net.UnixDomainSocketAddress; import java.nio.channels.Channels; @@ -81,6 +83,7 @@ import ghidra.program.model.listing.Function; import ghidra.program.model.listing.FunctionIterator; import ghidra.program.model.listing.Instruction; import ghidra.program.model.listing.InstructionIterator; +import ghidra.program.model.listing.Listing; import ghidra.program.model.listing.Variable; import ghidra.program.model.mem.MemoryBlock; import ghidra.program.model.mem.MemoryBlockSourceInfo; @@ -297,7 +300,7 @@ public final class ReaGhidraBridge extends HeadlessScript { Files.move(pendingPath, endpointPath); } } - acceptClient(server, descriptor); + acceptAuthenticatedLoopbackClient(server, descriptor); } finally { Files.deleteIfExists(pendingPath); @@ -309,6 +312,50 @@ public final class ReaGhidraBridge extends HeadlessScript { return System.getProperty("os.name", "").startsWith("Windows"); } + private static String readLoopbackHandshake( + SocketChannel client, int maxBytes, long timeoutMillis) throws IOException { + long deadline = System.nanoTime() + timeoutMillis * 1_000_000L; + InputStream input = client.socket().getInputStream(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + while (true) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) throw new SocketTimeoutException("Bridge authentication timed out"); + client.socket().setSoTimeout((int) Math.max(1, + Math.min(Integer.MAX_VALUE, (remaining + 999_999L) / 1_000_000L))); + int value = input.read(); + if (value == -1) return null; + if (value == '\n') return bytes.toString(StandardCharsets.UTF_8); + if (bytes.size() == maxBytes) throw new IOException("Bridge authentication frame is too large"); + bytes.write(value); + } + } + + private void acceptAuthenticatedLoopbackClient( + ServerSocketChannel server, SessionDescriptor descriptor) throws Exception { + while (true) { + try (SocketChannel client = server.accept()) { + Request first; + try { + String line = readLoopbackHandshake(client, 64 * 1024, 1_000); + if (line == null) continue; + first = parseRequest(line, descriptor.token); + } + catch (IOException | RuntimeException rejected) { + // A rejected unauthenticated peer cannot consume the bridge session. + continue; + } + client.socket().setSoTimeout(0); + try (BufferedReader reader = new BufferedReader( + Channels.newReader(client, StandardCharsets.UTF_8)); + BufferedWriter writer = new BufferedWriter( + Channels.newWriter(client, StandardCharsets.UTF_8))) { + serveClient(descriptor, reader, writer, first); + } + return; + } + } + } + private void acceptClient( ServerSocketChannel server, SessionDescriptor descriptor) throws Exception { @@ -319,26 +366,28 @@ public final class ReaGhidraBridge extends HeadlessScript { BufferedWriter writer = new BufferedWriter( Channels.newWriter(client, StandardCharsets.UTF_8) )) { - serveClient(descriptor, reader, writer); + serveClient(descriptor, reader, writer, null); } } private void serveClient( SessionDescriptor descriptor, BufferedReader reader, - BufferedWriter writer) throws Exception { + BufferedWriter writer, + Request first) throws Exception { while (true) { - String line = reader.readLine(); - if (line == null) { - return; - } - Request request; - try { - request = parseRequest(line, descriptor.token); - } - catch (RuntimeException exception) { - writeFailure(writer, 1, "invalid_request", "Bridge request is invalid"); - return; + Request request = first; + first = null; + if (request == null) { + String line = reader.readLine(); + if (line == null) return; + try { + request = parseRequest(line, descriptor.token); + } + catch (RuntimeException exception) { + writeFailure(writer, 1, "invalid_request", "Bridge request is invalid"); + return; + } } try { if (request.method.equals("shutdown")) { @@ -405,7 +454,9 @@ public final class ReaGhidraBridge extends HeadlessScript { case "xrefs" -> xrefs(request.params); case "analyze_function" -> { FunctionAnalysisObservation observation = analyzeFunctionObservation(request.params); - yield functionObservation(observation.dossier(), observation.limitations()); + JsonObject result = functionObservation(observation.dossier(), observation.limitations()); + result.add("relationship_coverage", observation.relationshipCoverage()); + yield result; } case "set_address_name", "set_addresses_names" -> { if (!descriptor.transport.equals("unix-socket")) @@ -884,6 +935,7 @@ public final class ReaGhidraBridge extends HeadlessScript { JsonObject result = functionObservation( value == null ? JsonNull.INSTANCE : GSON.toJsonTree(value), noReturnCallLimitations( + function, currentProgram.getListing().getInstructions(function.getBody(), true) ) ); @@ -975,7 +1027,7 @@ public final class ReaGhidraBridge extends HeadlessScript { InstructionScan scan = scanInstructions(function, Integer.MAX_VALUE); return functionObservation( GSON.toJsonTree(renderAssembly(scan.instructions)), - noReturnCallLimitations(scan.instructions) + noReturnCallLimitations(function, scan.instructions) ); } @@ -994,7 +1046,7 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add( "The fast path does not invoke the decompiler or scan whole-program names and strings." ); - List boundaryLimitations = noReturnCallLimitations(scan.instructions); + List boundaryLimitations = noReturnCallLimitations(function, scan.instructions); for (String limitation : boundaryLimitations) limitations.add(limitation); JsonObject result = new JsonObject(); result.add("procedure", procedureIdentity(function)); @@ -1016,12 +1068,19 @@ public final class ReaGhidraBridge extends HeadlessScript { for (Function item : ordered) { result.add(canonicalAddress(item.getEntryPoint())); } - return functionObservation( - result, - noReturnCallLimitations( + if (callers) { + JsonObject observation = functionObservation(result, noReturnCallLimitations(function, currentProgram.getListing().getInstructions(function.getBody(), true) - ) + )); + observation.add("relationship_coverage", callRelationshipCoverage(null, + unrepresentedIncomingCalls(function, observed))); + monitor.checkCancelled(); + return observation; + } + CallBoundaryObservation calls = callBoundaryObservation( + function, currentProgram.getListing().getInstructions(function.getBody(), true), observed, false, null ); + return callRelationshipObservation(result, calls); } private JsonObject procedureInfo(JsonObject params) throws Exception { @@ -1041,6 +1100,7 @@ public final class ReaGhidraBridge extends HeadlessScript { return functionObservation( result, noReturnCallLimitations( + function, currentProgram.getListing().getInstructions(function.getBody(), true) ) ); @@ -1071,18 +1131,10 @@ public final class ReaGhidraBridge extends HeadlessScript { result.addProperty("direction", direction); result.add("references", edges); result.addProperty("reference_kinds_available", true); - JsonArray unresolvedCalls = new JsonArray(); - if (direction.equals("outgoing")) for (Instruction instruction : scanInstructions(function, Integer.MAX_VALUE).instructions) { - if (!instruction.getFlowType().isCall()) continue; - boolean resolved = false; - for (Reference reference : instruction.getReferencesFrom()) if (reference.getReferenceType().isCall()) resolved = true; - if (!resolved) { - JsonObject site = new JsonObject(); - site.addProperty("address", canonicalAddress(instruction.getAddress())); - site.addProperty("reason", "Instruction decoder identifies a call but ReferenceManager supplies no call destination"); - unresolvedCalls.add(site); - } - } + JsonArray unresolvedCalls = direction.equals("outgoing") + ? callBoundaryObservation( + function, currentProgram.getListing().getInstructions(function.getBody(), true), null, true, references + ).unresolvedCalls() : new JsonArray(); result.add("unresolved_calls", unresolvedCalls); return result; } @@ -1490,16 +1542,28 @@ public final class ReaGhidraBridge extends HeadlessScript { InstructionScan scan = scanInstructions(function, Integer.MAX_VALUE); DecompileResults decompilation = decompile(function); String pseudocode = decompiledText(decompilation); - List incomingReferences = collectReferences(function, "incoming") - .stream() + JsonObject availability = new JsonObject(); + availability.add("pseudocode", facetAvailability( + pseudocode != null, + function.isExternal() ? "Ghidra does not decompile external functions." + : function.getBody().isEmpty() ? "Ghidra has no function body to decompile." + : "Ghidra completed decompilation without returning pseudocode." + )); + if (pseudocode == null) { + pseudocode = ""; + } + List bodyIncoming = collectReferences(function, "incoming"); + List incomingReferences = bodyIncoming.stream() .filter(reference -> !function.getBody().contains(reference.getFromAddress())) .toList(); List outgoingReferences = collectReferences(function, "outgoing"); JsonArray incoming = referenceEdges(incomingReferences); JsonArray outgoing = referenceEdges(outgoingReferences); - JsonArray comments = comments(scan.instructions); - JsonArray callers = procedureIdentities(function.getCallingFunctions(monitor)); - JsonArray callees = procedureIdentities(function.getCalledFunctions(monitor)); + JsonArray comments = comments(function); + Set callerFunctions = function.getCallingFunctions(monitor); + JsonArray callers = procedureIdentities(callerFunctions); + Set calleeFunctions = function.getCalledFunctions(monitor); + JsonArray callees = procedureIdentities(calleeFunctions); JsonArray referencedStrings = referencedStrings(outgoingReferences); JsonArray referencedNames = referencedNames(outgoingReferences); JsonArray blocks = basicBlockValues(function); @@ -1511,7 +1575,7 @@ public final class ReaGhidraBridge extends HeadlessScript { procedure.add("locals", functionLocals(function)); JsonObject result = new JsonObject(); result.add("procedure", procedure); - result.addProperty("pseudocode", pseudocode == null ? "" : pseudocode); + result.addProperty("pseudocode", pseudocode); result.add("assembly", assembly); result.add("comments", comments); result.add("callers", callers); @@ -1520,20 +1584,6 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("outgoing_references", outgoing); result.add("referenced_strings", referencedStrings); result.add("referenced_names", referencedNames); - JsonObject facets = new JsonObject(); - JsonObject pseudocodeAvailability = new JsonObject(); - pseudocodeAvailability.addProperty("available", pseudocode != null); - if (pseudocode == null) { - pseudocodeAvailability.addProperty("reason", "Ghidra did not supply pseudocode."); - } - facets.add("pseudocode", pseudocodeAvailability); - for (String facet : List.of("assembly", "comments", "callers", "callees", - "references", "strings_names", "cfg")) { - JsonObject available = new JsonObject(); - available.addProperty("available", true); - facets.add(facet, available); - } - result.add("facet_availability", facets); boolean swift = swiftProgram(); boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:"); SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null; @@ -1547,6 +1597,15 @@ public final class ReaGhidraBridge extends HeadlessScript { nativeApiBoundary(function, decompilation) ); result.add("native_value_flow", nativeValueFlow(decompilation)); + // Successful full queries distinguish collected empty sets from missing facts. + // Availability does not claim that Ghidra resolved every runtime relationship. + for (String facet : List.of("comments", "callers", "callees", "references", "strings_names", "cfg")) { + availability.add(facet, facetAvailability(true, null)); + } + availability.add("assembly", facetAvailability( + !scan.truncated(), "The function instruction scan was truncated." + )); + result.add("facet_availability", availability); JsonArray limitations = new JsonArray(); limitations.add( "Unresolved computed or indirect flows without target addresses are not represented as reference edges." @@ -1562,7 +1621,7 @@ public final class ReaGhidraBridge extends HeadlessScript { ); if (swift && aarch64) { limitations.add( - "Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by conditional selects, passed through unresolved jumps, or built at runtime are not reported." + "Swift String literals are decoded from constant register pairs at calls, returns and paired stores. Literals stored one word at a time, chosen by conditional selects, or built at runtime are not reported; candidate pairs at unresolved jumps are retained as unknown omissions." ); if (swiftLiterals.omissions().size() > 0) { limitations.add( @@ -1574,8 +1633,10 @@ public final class ReaGhidraBridge extends HeadlessScript { "Swift String literal decoding is implemented for AArch64 register pairs only; no Swift literals were decoded for this architecture." ); } - List boundaryLimitations = noReturnCallLimitations(scan.instructions); - for (String limitation : boundaryLimitations) { + CallBoundaryObservation calls = callBoundaryObservation( + function, scan.instructions, calleeFunctions, true, outgoingReferences); + result.add("unresolved_calls", calls.unresolvedCalls()); + for (String limitation : calls.limitations()) { limitations.add(limitation); } if (decompilation != null && decompilation.getErrorMessage() != null && @@ -1583,7 +1644,14 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add("Ghidra decompiler diagnostic: " + decompilation.getErrorMessage()); } result.add("limitations", limitations); - return new FunctionAnalysisObservation(result, boundaryLimitations); + monitor.checkCancelled(); + int missingCallers = 0; + for (Reference reference : bodyIncoming) { + monitor.checkCancelled(); + if (unrepresentedCaller(reference, callerFunctions)) missingCallers++; + } + return new FunctionAnalysisObservation(result, calls.limitations(), + callRelationshipCoverage(calls, missingCallers)); } private JsonObject nativeValueFlow(DecompileResults decompilation) throws Exception { @@ -3287,54 +3355,276 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } - private List noReturnCallLimitations(Iterable instructions) + private static JsonObject facetAvailability(boolean available, String reason) { + JsonObject result = new JsonObject(); + result.addProperty("available", available); + if (!available) result.addProperty("reason", reason); + return result; + } + + private JsonObject callRelationshipObservation(JsonElement value, CallBoundaryObservation calls) + throws Exception { + monitor.checkCancelled(); + JsonObject result = functionObservation(value, calls.limitations()); + result.add("relationship_coverage", callRelationshipCoverage(calls, null)); + return result; + } + + private static JsonObject callRelationshipCoverage( + CallBoundaryObservation calls, Integer missingCallers) { + JsonObject coverage = completeResolvedCallCoverage(); + Integer omitted = calls == null ? null : calls.unrepresentedCalleeCalls(); + JsonArray directions = new JsonArray(); + if (missingCallers != null && missingCallers > 0) directions.add("callers"); + String callerLimitation = missingCallers != null && missingCallers > 0 + ? "Ghidra's caller inventory omits sources of " + missingCallers + + " call reference(s) into this function's body." : null; + if ((calls != null && omitted == null) || (calls == null && missingCallers == null)) { + coverage.addProperty("status", "unknown"); + String reason = calls == null ? "No call inventory was examined." + : "The callee inventory was not examined alongside decoded call targets."; + coverage.addProperty("reason", reason + (callerLimitation == null ? "" : " " + callerLimitation)); + if (calls == null && missingCallers == null) directions.add("callers"); + directions.add("callees"); + coverage.add("limited_directions", directions); + return coverage; + } + JsonArray limitations = new JsonArray(); + if (callerLimitation != null) limitations.add(callerLimitation); + if (omitted != null && omitted > 0) { + limitations.add("Ghidra's callee inventory omits targets of " + omitted + + " call reference(s) or decoded call(s) without destinations."); + directions.add("callees"); + } + if (!limitations.isEmpty()) { + coverage.addProperty("status", "partial"); + coverage.add("limitations", limitations); + coverage.add("limited_directions", directions); + } + return coverage; + } + + private static JsonObject completeResolvedCallCoverage() { + JsonObject coverage = new JsonObject(); + coverage.addProperty("status", "complete"); + JsonArray diagnostics = new JsonArray(); + diagnostics.add( + "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals." + ); + coverage.add("diagnostics", diagnostics); + return coverage; + } + + private boolean unrepresentedCaller(Reference reference, Set callers) { + if (!reference.getReferenceType().isCall()) return false; + Function caller = containingFunction(reference.getFromAddress()); + return caller == null || !callers.contains(caller); + } + + private int unrepresentedIncomingCalls(Function function, Set callers) + throws Exception { + int[] omitted = { 0 }; + visitReferences(function, "incoming", reference -> { + if (unrepresentedCaller(reference, callers)) omitted[0]++; + }); + return omitted[0]; + } + + private List noReturnCallLimitations( + Function function, Iterable instructions) throws Exception { List limitations = new ArrayList<>(); + Instruction beforePrevious = null; + Instruction previous = null; for (Instruction instruction : instructions) { monitor.checkCancelled(); - if (!instruction.getFlowType().isCall() || !instruction.getFlowType().isTerminal()) { - continue; + String limitation = terminalCallLimitation(instruction); + if (limitation != null) limitations.add(limitation); + if (beforePrevious != null && previous != null) { + String seh = borlandSehHandlerLimitation( + function, beforePrevious, previous, instruction); + if (seh != null) limitations.add(seh); } - Address[] destinations = instruction.getFlows(); - StringBuilder limitation = new StringBuilder(); - limitation.append("Ghidra Listing reports a terminal call at ") - .append(canonicalAddress(instruction.getAddress())); - if (destinations.length == 0) { - limitation.append(" with no resolved callee; its no-return basis is unknown."); - } - else { - limitation.append(" to "); - for (int index = 0; index < destinations.length; index++) { - if (index > 0) limitation.append(", "); - Address destination = destinations[index]; - limitation.append(canonicalAddress(destination)); - Function target = currentProgram.getFunctionManager().getFunctionAt(destination); - if (target == null) { - limitation.append(" (callee identity unknown)"); - } - else { - limitation.append(" ").append(procedureName(target)); - limitation.append(" (external=").append(target.isExternal()); - limitation.append(", thunk=").append(target.isThunk()); - limitation.append(", hasNoReturn=").append(target.hasNoReturn()).append(")"); - } - } - limitation.append("; fallthrough is excluded by Ghidra's flow model."); - Function target = destinations.length == 1 - ? currentProgram.getFunctionManager().getFunctionAt(destinations[0]) - : null; - if (target != null && target.hasNoReturn()) { - limitation.append(" This records Ghidra's FunctionManager flag, not an independent verification that the callee cannot return."); - } - else { - limitation.append(" The reason for the terminal flow is not established by this observation."); - } - } - limitations.add(limitation.toString()); + beforePrevious = previous; + previous = instruction; } return limitations; } + private CallBoundaryObservation callBoundaryObservation( + Function function, Iterable instructions, Set returnedCallees, + boolean includeUnresolvedSites, List observedReferences) + throws Exception { + List limitations = new ArrayList<>(); + CallBoundaryReferences calls = new CallBoundaryReferences(returnedCallees, includeUnresolvedSites); + Instruction beforePrevious = null; + Instruction previous = null; + if (observedReferences == null) visitReferences(function, "outgoing", calls::observe); + else for (Reference reference : observedReferences) { + monitor.checkCancelled(); + calls.observe(reference); + } + for (Instruction instruction : instructions) { + monitor.checkCancelled(); + String limitation = terminalCallLimitation(instruction); + if (limitation != null) limitations.add(limitation); + if (beforePrevious != null && previous != null) { + String seh = borlandSehHandlerLimitation( + function, beforePrevious, previous, instruction); + if (seh != null) limitations.add(seh); + } + beforePrevious = previous; + previous = instruction; + if (!instruction.getFlowType().isCall()) continue; + boolean destination = false; + for (Reference reference : instruction.getReferencesFrom()) { + if (!reference.getReferenceType().isCall()) continue; + destination = true; + break; + } + if (!destination) { + calls.unrepresented++; + calls.unresolved(instruction.getAddress(), + "Instruction decoder identifies a call but ReferenceManager supplies no call destination"); + } + } + return new CallBoundaryObservation(List.copyOf(limitations), + returnedCallees == null ? null : calls.unrepresented, calls.unresolvedCalls()); + } + + private String borlandSehHandlerLimitation( + Function function, Instruction handlerPush, Instruction framePush, + Instruction frameLink) { + if (!immediatelyPrecedes(handlerPush, framePush) || + !immediatelyPrecedes(framePush, frameLink) || + !handlerPush.getMnemonicString().equalsIgnoreCase("PUSH") || + handlerPush.getNumOperands() != 1 || + !framePush.getMnemonicString().equalsIgnoreCase("PUSH") || + framePush.getNumOperands() != 1 || + !frameLink.getMnemonicString().equalsIgnoreCase("MOV") || + frameLink.getNumOperands() != 2 || + !sameFsMemoryOperand(framePush, frameLink) || + !hasRegisterOperand(frameLink, 1, "ESP")) + return null; + Object[] handlerOperand = handlerPush.getOpObjects(0); + if (handlerOperand.length != 1 || !(handlerOperand[0] instanceof Scalar scalar) || + scalar.bitLength() != 32) + return null; + Address handler = currentProgram.getAddressFactory().getDefaultAddressSpace() + .getAddress(scalar.getUnsignedValue()); + if (function.getBody().contains(handler)) return null; + return "Ghidra Listing shows a Borland/Delphi-style SEH frame registration at " + + canonicalAddress(frameLink.getAddress()) + " (push " + canonicalAddress(handler) + + " followed by fs-segment stack chaining); handler " + canonicalAddress(handler) + + " lies outside this procedure's body, so callees, strings, and cross-references may omit the handler region."; + } + + private static boolean immediatelyPrecedes(Instruction first, Instruction second) { + Address fallThrough = first.getFallThrough(); + return fallThrough != null && fallThrough.equals(second.getAddress()); + } + + private static boolean sameFsMemoryOperand(Instruction first, Instruction second) { + String firstOperand = normalizedFsMemoryOperand(first); + String secondOperand = normalizedFsMemoryOperand(second); + return firstOperand != null && firstOperand.equalsIgnoreCase(secondOperand); + } + + private static String normalizedFsMemoryOperand(Instruction instruction) { + String representation = instruction.getDefaultOperandRepresentation(0); + representation = representation.replaceAll("(?i)\\b(?:byte|word|dword|qword)\\s+ptr\\s+", "") + .replaceAll("\\s+", ""); + return representation.regionMatches(true, 0, "fs:", 0, 3) && + representation.indexOf('[') >= 0 ? representation : null; + } + + private static boolean hasRegisterOperand( + Instruction instruction, int operand, String name) { + for (Object object : instruction.getOpObjects(operand)) { + if (object instanceof Register register && register.getName().equalsIgnoreCase(name)) + return true; + } + return false; + } + + private final class CallBoundaryReferences { + private final Set returnedCallees; + private final TreeMap unresolved; + private int unrepresented; + + CallBoundaryReferences(Set returnedCallees, boolean includeUnresolvedSites) { + this.returnedCallees = returnedCallees; + unresolved = includeUnresolvedSites ? new TreeMap<>() : null; + } + + void observe(Reference reference) { + if (!reference.getReferenceType().isCall()) return; + // Reference identity follows the same containing-function owner as referenceEdge. + Function target = containingFunction(reference.getToAddress()); + if (returnedCallees != null && (target == null || !returnedCallees.contains(target))) + unrepresented++; + if (target == null) unresolved(reference.getFromAddress(), + "A call destination has no known containing Ghidra procedure"); + } + + void unresolved(Address address, String reason) { + if (unresolved != null) unresolved.put(address, reason); + } + + JsonArray unresolvedCalls() { + if (unresolved == null) return null; + JsonArray result = new JsonArray(); + for (var item : unresolved.entrySet()) { + JsonObject site = new JsonObject(); + site.addProperty("address", canonicalAddress(item.getKey())); + site.addProperty("reason", item.getValue()); + result.add(site); + } + return result; + } + } + + private String terminalCallLimitation(Instruction instruction) { + if (!instruction.getFlowType().isCall() || !instruction.getFlowType().isTerminal()) + return null; + Address[] destinations = instruction.getFlows(); + StringBuilder limitation = new StringBuilder(); + limitation.append("Ghidra Listing reports a terminal call at ") + .append(canonicalAddress(instruction.getAddress())); + if (destinations.length == 0) { + limitation.append(" with no resolved callee; its no-return basis is unknown."); + } + else { + limitation.append(" to "); + for (int index = 0; index < destinations.length; index++) { + if (index > 0) limitation.append(", "); + Address destination = destinations[index]; + limitation.append(canonicalAddress(destination)); + Function target = currentProgram.getFunctionManager().getFunctionAt(destination); + if (target == null) { + limitation.append(" (callee identity unknown)"); + } + else { + limitation.append(" ").append(procedureName(target)); + limitation.append(" (external=").append(target.isExternal()); + limitation.append(", thunk=").append(target.isThunk()); + limitation.append(", hasNoReturn=").append(target.hasNoReturn()).append(")"); + } + } + limitation.append("; fallthrough is excluded by Ghidra's flow model."); + Function target = destinations.length == 1 + ? currentProgram.getFunctionManager().getFunctionAt(destinations[0]) + : null; + if (target != null && target.hasNoReturn()) { + limitation.append(" This records Ghidra's FunctionManager flag, not an independent verification that the callee cannot return."); + } + else { + limitation.append(" The reason for the terminal flow is not established by this observation."); + } + } + return limitation.toString(); + } + private static String renderAssembly(List instructions) { return String.join("\n", renderAssemblyLines(instructions)); } @@ -3358,15 +3648,27 @@ public final class ReaGhidraBridge extends HeadlessScript { private List collectReferences( Function function, String direction) throws Exception { + TreeMap observed = new TreeMap<>(); + visitReferences(function, direction, reference -> addReference(observed, reference)); + List result = new ArrayList<>(observed.values()); + result.sort(REFERENCE_ORDER); + return result; + } + + @FunctionalInterface + private interface ReferenceVisitor { + void accept(Reference reference) throws Exception; + } + + private void visitReferences(Function function, String direction, ReferenceVisitor visitor) + throws Exception { // References can target instruction interiors or originate in embedded // data. Use actual body ownership, including an external's exact entry. - AddressSet scope = new AddressSet(function.getBody()); - scope.add(function.getEntryPoint()); + AddressSet scope = functionObservationScope(function); boolean outgoing = direction.equals("outgoing"); AddressIterator addresses = outgoing ? currentProgram.getReferenceManager().getReferenceSourceIterator(scope, true) : currentProgram.getReferenceManager().getReferenceDestinationIterator(scope, true); - TreeMap observed = new TreeMap<>(); while (addresses.hasNext()) { monitor.checkCancelled(); Address address = addresses.next(); @@ -3374,7 +3676,7 @@ public final class ReaGhidraBridge extends HeadlessScript { for (Reference reference : currentProgram.getReferenceManager() .getReferencesFrom(address)) { monitor.checkCancelled(); - addReference(observed, reference); + if (!reference.isEntryPointReference()) visitor.accept(reference); } } else { @@ -3382,19 +3684,14 @@ public final class ReaGhidraBridge extends HeadlessScript { .getReferencesTo(address); while (iterator.hasNext()) { monitor.checkCancelled(); - addReference(observed, iterator.next()); + Reference reference = iterator.next(); + if (!reference.isEntryPointReference()) visitor.accept(reference); } } } - List result = new ArrayList<>(observed.values()); - result.sort(REFERENCE_ORDER); - return result; } private static void addReference(TreeMap observed, Reference reference) { - if (reference.isEntryPointReference()) { - return; - } String key = canonicalAddress(reference.getFromAddress()) + "\u0000" + canonicalAddress(reference.getToAddress()) + "\u0000" + reference.getReferenceType().getName() + "\u0000" + @@ -3403,6 +3700,12 @@ public final class ReaGhidraBridge extends HeadlessScript { observed.putIfAbsent(key, reference); } + private static AddressSet functionObservationScope(Function function) { + AddressSet scope = new AddressSet(function.getBody()); + scope.add(function.getEntryPoint()); + return scope; + } + private JsonArray referenceEdges(List references) { JsonArray result = new JsonArray(); for (Reference reference : references) { @@ -3556,23 +3859,28 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } - private static JsonArray comments(List instructions) { + private JsonArray comments(Function function) throws Exception { + Listing listing = currentProgram.getListing(); + AddressSet scope = functionObservationScope(function); + AddressIterator addresses = listing.getCommentAddressIterator(scope, true); TreeMap observed = new TreeMap<>(); - for (Instruction instruction : instructions) { + while (addresses.hasNext()) { + monitor.checkCancelled(); + Address address = addresses.next(); for (CommentType type : CommentType.values()) { - String text = instruction.getComment(type); + String text = listing.getComment(type, address); if (text == null || text.isEmpty()) { continue; } String kind = type == CommentType.EOL ? "inline" : "comment"; JsonObject item = new JsonObject(); - item.addProperty("address", canonicalAddress(instruction.getAddress())); + item.addProperty("address", canonicalAddress(address)); item.addProperty("kind", kind); item.addProperty("text", text); - String key = canonicalAddress(instruction.getAddress()) + "\u0000" + + String key = canonicalAddress(address) + "\u0000" + kind + "\u0000" + type.name() + "\u0000" + text; observed.put(key, item); - } + } } JsonArray result = new JsonArray(); observed.values().forEach(result::add); @@ -3643,13 +3951,20 @@ public final class ReaGhidraBridge extends HeadlessScript { * The canonical empty string is a small literal with a zero count. */ private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception { + return swiftStringLiterals(function, new SwiftLiteralBudget()); + } + + // An explicit budget makes capacity omissions deterministic in producer tests. + private SwiftStringLiteralDecode swiftStringLiterals( + Function function, + SwiftLiteralBudget budget + ) throws Exception { Register[] registers = new Register[31]; for (int index = 0; index < registers.length; index++) { registers[index] = currentProgram.getRegister("x" + index); } TreeMap observed = new TreeMap<>(); TreeMap omitted = new TreeMap<>(); - SwiftLiteralBudget budget = new SwiftLiteralBudget(); Map selectedValues = new HashMap<>(); SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false); propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() { @@ -3657,6 +3972,9 @@ public final class ReaGhidraBridge extends HeadlessScript { public boolean evaluateContextBefore(VarnodeContext context, Instruction instruction) { // Returning true ends only the current flow, so every queued flow checks the shared stop. if (budget.stopped()) return true; + FlowType flow = instruction.getFlowType(); + Address[] flowTargets = instruction.getFlows(); + boolean unresolvedJump = flow.isJump() && !flow.isConditional() && flowTargets.length == 0; for (Register[] pair : swiftStringPairs(function, instruction, registers)) { Long word = registerConstant(context, pair[0]); Long object = registerConstant(context, pair[1]); @@ -3706,12 +4024,14 @@ public final class ReaGhidraBridge extends HeadlessScript { JsonObject item = overCapacity ? swiftStringOmission(layout, word, object) : swiftStringItem(value, layout, word, object); + if (unresolvedJump && !overCapacity) + item = swiftStringOmission(layout, word, object, "unresolved_jump"); item.addProperty("source_address", canonicalAddress(instruction.getAddress())); JsonArray pairRegisters = new JsonArray(); pairRegisters.add(pair[0].getName()); pairRegisters.add(pair[1].getName()); item.add("registers", pairRegisters); - (overCapacity ? omitted : observed).put(key, item); + (overCapacity || unresolvedJump ? omitted : observed).put(key, item); if (overCapacity) { budget.stop(); return true; @@ -3747,11 +4067,12 @@ public final class ReaGhidraBridge extends HeadlessScript { if (operation.getOpcode() == PcodeOp.RETURN) returns = true; } // A jump is a proven tail call only when it has resolved targets and all leave the - // body. A targetless computed jump can still dispatch inside the function. + // body. For unresolved targetless jumps, candidate literals remain unknown omissions. Address[] targets = instruction.getFlows(); boolean tailCall = flow.isJump() && !flow.isConditional() && targets.length > 0 && Arrays.stream(targets).allMatch(target -> !function.getBody().contains(target)); - if (!flow.isCall() && !returns && !tailCall) return pairs; + boolean unresolvedJump = flow.isJump() && !flow.isConditional() && targets.length == 0; + if (!flow.isCall() && !returns && !tailCall && !unresolvedJump) return pairs; // A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned. int lastFirstRegister = returns ? 0 : 6; for (int index = 0; index <= lastFirstRegister; index++) { @@ -3894,6 +4215,11 @@ public final class ReaGhidraBridge extends HeadlessScript { remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2); } + SwiftLiteralBudget(long remaining) { + if (remaining < 0) throw new IllegalArgumentException("Swift literal budget must be nonnegative"); + this.remaining = remaining; + } + boolean takeState() { if (remaining < BYTES_PER_RECORD) return false; remaining -= BYTES_PER_RECORD; @@ -3932,6 +4258,15 @@ public final class ReaGhidraBridge extends HeadlessScript { } private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { + return swiftStringOmission(layout, word, object, "capacity"); + } + + private static JsonObject swiftStringOmission( + SwiftLiteralLayout layout, + long word, + long object, + String reason + ) { JsonObject item = new JsonObject(); item.addProperty("representation", layout.representation()); if (layout.address() == null) { @@ -3940,7 +4275,7 @@ public final class ReaGhidraBridge extends HeadlessScript { item.addProperty("address", canonicalAddress(layout.address())); } item.addProperty("byte_length", layout.byteLength()); - item.addProperty("reason", "capacity"); + item.addProperty("reason", reason); item.add("words", swiftStringWords(word, object)); return item; } @@ -4508,7 +4843,8 @@ public final class ReaGhidraBridge extends HeadlessScript { private record InventoryItem(Address address, String value, JsonObject facts) {} private record AddressNameRequest(String inputAddress, Address address, String name) {} private record FunctionEntry(Function function, InventoryItem item) {} - private record FunctionAnalysisObservation(JsonObject dossier, List limitations) {} + private record FunctionAnalysisObservation(JsonObject dossier, List limitations, JsonObject relationshipCoverage) {} + private record CallBoundaryObservation(List limitations, Integer unrepresentedCalleeCalls, JsonArray unresolvedCalls) {} private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {} private record SwiftLiteralLayout( String representation, diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 4188fe0da..0425cc0d9 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -736,7 +736,12 @@ def _analyze_function(document, params): def _dispatch(method, params): """Dispatch only the closed operation set implemented by REA's public tools.""" if method == "health": - return {"name": "REA Hopper bridge", "version": "1.0.0", "run_id": REA_RUN_ID} + return { + "name": "REA Hopper bridge", + "version": "1.0.0", + "run_id": REA_RUN_ID, + "process_id": os.getpid(), + } if method in ("shutdown", "shutdown_document"): document = _session_document() if document is None: diff --git a/docs/adb-device-analysis.md b/docs/adb-device-analysis.md index 12fd77cd2..8c94df25f 100644 --- a/docs/adb-device-analysis.md +++ b/docs/adb-device-analysis.md @@ -73,32 +73,32 @@ existing analysis families directly, for example ## Operations -| Tool | Kind | What it does | -| ----------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `inspect_adb_client` | read | Resolved binary, version, install path; never contacts the server | -| `list_adb_devices` | read | Serial, state, transport, model metadata, inferred emulator/physical kind with basis | -| `inspect_adb_device` | read | Fixed `getprop` whitelist: build, SDK, patch, fingerprint, product, ABIs, debuggable, `ro.kernel.qemu` | -| `inspect_adb_display` | read | `wm size` and `wm density` as declared | -| `list_adb_features` | read | `pm list features` (hardware/software features, hex GL versions) | -| `list_adb_services` | read | `service list` binder services and interfaces | -| `list_adb_processes` | read | `ps -A`: user, pid, ppid, rss, state, name | -| `inspect_adb_window` | read | `dumpsys window` focus declarations (what is on screen) | -| `read_adb_setting` | read | One `settings get` value; device-side null is reported as such | -| `read_adb_logcat` | read | Bounded `logcat -d -t` dump for one buffer, optional pid filter | -| `list_adb_packages` | read | `pm list packages -f` with scope; base APK device paths | -| `resolve_adb_packages` | read | Case-insensitive package-name substring search with exact-match flag | -| `inspect_adb_package` | read | `dumpsys package` projection: versions, install times, installer, flags, permission count | -| `list_adb_directory` | read | NUL-framed immediate children, including hidden entries: kinds, permissions, owner, group, sizes, symlink targets | -| `pull_adb_package` | read | Complete split APK set from `pm path`, each file digested | -| `pull_adb_file` | read | One absolute device path pulled and digested | -| `capture_adb_screen` | read | One `screencap -p` frame (exec-out, no device-side file) with dimensions | -| `collect_adb_bugreport` | read | Device-generated bugreport zip, digested | -| `push_adb_file` | **mutates device** | One local file to a device path (e.g. pushing tools into `/data/local/tmp`); refuses existing targets without explicit overwrite; verifies via the device's own sha256sum | -| `install_adb_package` | **mutates device** | Local APK via `adb install [-r]`; preserves the device's Success/Failure verdict | -| `uninstall_adb_package` | **mutates device** | Removes a package and its data for user 0 | -| `start_adb_app` | **mutates device** | Resolves the package's launcher activity (`cmd package resolve-activity --brief`) and starts it (`am start -n`) | -| `start_adb_activity` | **mutates device** | Starts an intent (`am start -a` with optional URI, component, and up to 16 typed extras); preserves the activity manager's verdict | -| `stop_adb_app` | **mutates device** | Force-stops every process of a package, ending a dynamic-analysis run | +| Tool | Kind | What it does | +| ----------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `inspect_adb_client` | read | Resolved binary, version, install path; never contacts the server | +| `list_adb_devices` | read | Serial, state, transport, model metadata, inferred emulator/physical kind with basis | +| `inspect_adb_device` | read | Fixed `getprop` whitelist: build, SDK, patch, fingerprint, product, ABIs, debuggable, `ro.kernel.qemu` | +| `inspect_adb_display` | read | `wm size` and `wm density` as declared | +| `list_adb_features` | read | `pm list features` (hardware/software features, hex GL versions) | +| `list_adb_services` | read | `service list` binder services and interfaces | +| `list_adb_processes` | read | `ps -A`: user, pid, ppid, rss, state, name | +| `inspect_adb_window` | read | `dumpsys window` focus declarations (what is on screen) | +| `read_adb_setting` | read | One `settings get` value; device-side null is reported as such | +| `read_adb_logcat` | read | Bounded `logcat -d -t` dump for one buffer, optional pid filter | +| `list_adb_packages` | read | `pm list packages -f` with scope; base APK device paths | +| `resolve_adb_packages` | read | Case-insensitive package-name substring search with exact-match flag | +| `inspect_adb_package` | read | `dumpsys package` projection: versions, install times, installer, flags, permission count | +| `list_adb_directory` | read | NUL-framed immediate children, including hidden entries: kinds, permissions, owner, group, sizes, symlink targets | +| `pull_adb_package` | read | Complete split APK set from `pm path`, each file digested | +| `pull_adb_file` | read | One absolute device path pulled and digested | +| `capture_adb_screen` | read | One `screencap -p` frame (exec-out, no device-side file) with dimensions | +| `collect_adb_bugreport` | read | Device-generated bugreport zip, digested | +| `push_adb_file` | **mutates device** | One local file to a device path (e.g. pushing tools into `/data/local/tmp`); checks for existing targets and refuses them without explicit overwrite (the check is not atomic); compares digests when device `sha256sum` is available | +| `install_adb_package` | **mutates device** | Local APK via `adb install [-r]`; preserves the device's Success/Failure verdict | +| `uninstall_adb_package` | **mutates device** | Removes a package and its data for user 0 | +| `start_adb_app` | **mutates device** | Resolves the package's launcher activity (`cmd package resolve-activity --brief`) and starts it (`am start -n`) | +| `start_adb_activity` | **mutates device** | Starts an intent (`am start -a` with optional URI, component, and up to 16 typed extras); preserves the activity manager's verdict | +| `stop_adb_app` | **mutates device** | Force-stops every process of a package, ending a dynamic-analysis run | ## Interpret the result @@ -118,6 +118,16 @@ existing analysis families directly, for example while POSIX hosts retain valid POSIX filenames such as names with spaces or colons. A package pull checks reported names against the destination filesystem before pulling, and refuses names that would share a local path. +- **Package path refusals are diagnosable.** Refusals from `pull_adb_package` + return `unreadable_output` with `retryable: false`, keep the `pm path` output + in `details.captured_output`, and name the refusal in `details.reason`. + `unsafe_device_path` lists refused entries in + `details.diagnostics.rejected_paths` as `{device_path, reason}`; + `duplicate_local_destination` reports the shared `file_name` and both + `device_paths`; `local_destination_unavailable` reports the `device_path`, + `file_name`, and filesystem `error_code`. Diagnostics retain raw device + strings, while the caller-facing message escapes invisible and bidirectional + characters so they cannot hide text or reorder the path. - **Splits are part of the answer.** `pm path` enumerates the base APK and every installed split; pulls cover the whole set. System packages whose APKs keep other names report the `unknown` role rather than guessing. @@ -127,8 +137,10 @@ existing analysis families directly, for example partial coverage instead of dropping rows. - **Mutations are explicit.** `push_adb_file`, `install_adb_package`, and `uninstall_adb_package` declare `mutatesTarget` and never run implicitly: - pushes refuse existing device files unless `overwrite` is set and verify - the transfer with the device's own `sha256sum`; installs preserve the + pushes check for existing device files before transfer and refuse them + unless `overwrite` is set. The check is not atomic, so a different process + can create the path before `adb push` writes it. When available, the device's + `sha256sum` verifies the transfer; installs preserve the device's `Failure [...]` reason verbatim. - **Producer identity stays attached.** Every result embeds the producing adb client (`binary_path`, `path_source`, `version`, `installed_path`). diff --git a/docs/ghidra-mips.md b/docs/ghidra-mips.md index f010b4f4c..397741a6a 100644 --- a/docs/ghidra-mips.md +++ b/docs/ghidra-mips.md @@ -52,13 +52,15 @@ section names or the record being inside the initial 4 KiB header probe. Program and section declarations must agree; duplicate, conflicting, truncated or out-of-file records are rejected. Stripped section tables and section-only records are supported. Extended table numbering is explicitly unsupported. -For carved ELFs whose section table extends beyond the captured file, REA -examines every complete captured section header and records the unavailable -headers as a limitation. A valid program-header ABI record can still establish -the supported declaration. Available section declarations must agree with it; -their duplicate, conflicting or unreadable records remain errors. A missing -program-header record does not make ABI declarations in uncaptured sections -absent or supply a default ABI. +GNU `SHT_GNU_ATTRIBUTES` sections are refused by numeric section type until REA +parses and reconciles their declarations; section names and payload sizes do not +control detection or allocation. For carved ELFs whose section table extends +beyond the captured file, REA examines every complete captured section header +and records the unavailable headers as a limitation. The provider refuses an +incomplete ABI declaration inspection even when a program-header ABI record is +valid, because an unavailable section header could carry an unsupported or +conflicting declaration. A missing program-header record does not make ABI +declarations in uncaptured sections absent or supply a default ABI. The field layout follows [`Elf_MIPS_ABIFlags_v0`](https://github.com/bminor/glibc/blob/master/elf/elf.h). diff --git a/docs/ghidra-provider.md b/docs/ghidra-provider.md index fd23dcd07..2f70b48bc 100644 --- a/docs/ghidra-provider.md +++ b/docs/ghidra-provider.md @@ -154,12 +154,15 @@ An inferred no-return call can terminate that body before subsequent bytes. Pseudocode, assembly, procedure info, instruction windows, callee and caller reads, and dossiers report observed terminal call sites in Evidence limitations, including resolved callee names, addresses, and FunctionManager flags when -available. Instruction windows -and dossiers also retain these warnings in their result limitations. An -unresolved callee or unestablished terminal-flow basis remains explicit. These -observations qualify the analysis; they do not prove that a no-return flag is -incorrect or recover excluded code. Analysis profiles version this observation -contract so older snapshots cannot replay unqualified results. +available. On x86-32, the same views also report the exact Borland/Delphi-style +FS exception-registration pattern when its handler address falls outside the +function body. This is a static registration observation; it does not establish +handler execution or change the body. Instruction windows and dossiers also +retain these warnings in their result limitations. An unresolved callee or +unestablished terminal-flow basis remains explicit. These observations qualify +the analysis; they do not prove that a no-return flag is incorrect or recover +excluded code. Analysis profiles version this observation contract so older +snapshots cannot replay unqualified results. On non-Windows hosts, the ephemeral import repair clears no-return flags on the known returning external ABI names `malloc`, `__tls_get_addr`, `memcpy`, `memset`, @@ -189,7 +192,7 @@ host/provider acceptance run with only the host compiler. `npm run verify:ghidra:windows` checks source-owned native x86-64 PE applications; `-- --x86` selects x86 and `-- --dll` selects DLLs, and the flags can be combined. -These lanes check all 25 read-only operations, target/snapshot/import SHA-256 linkage, +These lanes check all 26 read-only operations, target/snapshot/import SHA-256 linkage, and project, endpoint, process, and runtime cleanup. The independent native lane checks DACLs, handle admission, cancellation, and Job Object lifecycle. `npm run verify:ghidra:windows:package` packs and installs REA into an isolated diff --git a/docs/go-binaries.md b/docs/go-binaries.md index 155780984..134a8c74d 100644 --- a/docs/go-binaries.md +++ b/docs/go-binaries.md @@ -58,6 +58,12 @@ original stable file snapshot, including pointer-backed strings. It does not scan arbitrary byte strings as compiler metadata or interpret a fat Mach-O container as one selected architecture. +For ELF images without a named build-info section, discovery checks each +nonempty writable, non-executable load segment in program-header order. This +includes sectionless PIE images with metadata in a later data segment. Named +sections retain priority, and alignment, file mapping and record validation +remain required. + Historical moduleless Go binaries can store their empty module-string header in declared zero-initialized virtual memory rather than file-backed bytes. The reader recognizes this only when the complete optional module header lies in that memory @@ -78,9 +84,13 @@ host permission denial, changed artifacts and cancellation retain distinct errors. The selected file is never modified; this inspection creates no temporary files, starts no subprocesses and accesses no network endpoint. -Resource guards bound the selected file to 256 MiB, aggregate embedded strings +Resource guards bound the selected file and cumulative header searches to 256 MiB +each, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and the complete normalized metadata -report to 8 MiB. Oversized results fail without partial success. MCP also applies +report to 8 MiB. Identical search spans with the same virtual alignment are +searched once; partially overlapping spans consume the cumulative search budget +and fail with a CPU resource constraint when it is exhausted. Oversized results +fail without partial success. MCP also applies its [response transport budget](mcp-contracts.md), which counts both text and structured content; a report within the metadata limit can still require export from retained Evidence. These limits are not operating-system memory or CPU diff --git a/docs/ida-provider.md b/docs/ida-provider.md index 012f75bfb..fca4d0bb5 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -137,11 +137,11 @@ For guided registration, set `REA_IDA_MCP_CONFIG` before `rea setup --client { + let excerpt = ""; + let count = 0; + for (const character of devicePath) { + if (count === 160) break; + excerpt += character; + count += 1; + } + const quoted = JSON.stringify(excerpt).replace( + /[\p{C}\p{Z}\p{Default_Ignorable_Code_Point}\u2800]/gu, + (character) => + character === " " + ? character + : `\\u{${character.codePointAt(0)!.toString(16).toUpperCase()}}`, + ); + return excerpt.length === devicePath.length ? quoted : `${quoted} (excerpt)`; +}; + /** One APK of an installed package exactly as `pm path` reports it. */ export interface AdbPackagePathEntry { readonly device_path: string; diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index 3d15325fd..9c126785c 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -240,11 +240,12 @@ const shellStub = ( ): ((root: string) => string) => { const cases = Object.entries(dispatch) .map(([key, output]) => { - const [first, second] = key.split(":"); - const guard = - second === undefined - ? `[ "$4" = "${first}" ]` - : `[ "$4" = "${first}" ] && [ "$5" = "${second}" ]`; + const [first, second, third] = key.split(":"); + const guard = [ + `[ "$4" = "${first}" ]`, + ...(second === undefined ? [] : [`[ "$5" = "${second}" ]`]), + ...(third === undefined ? [] : [`[ "$6" = "${third}" ]`]), + ].join(" && "); return ` if ${guard}; then printf '%b' '${output.replaceAll("\\", "\\\\").replaceAll("\0", "\\000").replace(/'/gu, "'\\''")}'; exit 0; fi`; }) .join("\n"); @@ -379,6 +380,30 @@ exit 1`, expect(result.error.capturedOutput?.stdout).toContain( pmOutput.split("\n")[0], ); + const projected = projectAnalysisError(result.error); + const duplicate = pmOutput.includes("\n"); + expect(projected).toMatchObject({ + code: "unreadable_output", + message: expect.stringContaining("pm path for com.example"), + remediation: { action: expect.stringContaining("details.diagnostics") }, + details: { + reason: duplicate ? "duplicate_local_destination" : "unsafe_device_path", + diagnostics: { + package_name: "com.example", + ...(duplicate + ? { device_paths: ["/data/app/x/base.apk", "/data/app/y/base.apk"] } + : { + rejected_paths: [ + { + device_path: "/data/app/x/", + reason: "empty_file_name", + }, + ], + }), + }, + captured_output: { stdout: expect.stringContaining("package:") }, + }, + }); await expect(readFile(join(root, "pull-ran"))).rejects.toThrow(); await expect(lstat(output)).rejects.toMatchObject({ code: "ENOENT" }); }); @@ -410,7 +435,15 @@ it.each([ if (aliases) { expect(result.ok).toBe(false); if (result.ok) return; - expect(result.error.message).toContain("unique local destinations"); + expect(projectAnalysisError(result.error)).toMatchObject({ + message: expect.stringContaining("unique local destination"), + details: { + reason: "local_destination_unavailable", + diagnostics: { + device_path: `/data/app/y/${secondName}`, + }, + }, + }); await expect(readFile(pullLog)).rejects.toThrow(); await expect(lstat(packageDirectory)).rejects.toThrow(); return; @@ -1015,6 +1048,60 @@ it("inspects the display and window focus", async () => { ); }); +it("reads Android 16 focus declarations from displays when windows omits them", async () => { + const { binary } = await writeAdbStub( + shellStub({ + "dumpsys:window:windows": "Window state has no focus fields\\n", + "dumpsys:window:displays": + "mCurrentFocus=Window{abc u0 com.example/.Main}\nmFocusedApp=ActivityRecord{def u0 com.example/.Main t1}\n", + }), + ); + const result = await evidenceResult(providerOver(binary), { + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(result).toMatchObject({ + focused_window: "Window{abc u0 com.example/.Main}", + focused_app: "ActivityRecord{def u0 com.example/.Main t1}", + observed_lines: [ + "Window{abc u0 com.example/.Main}", + "ActivityRecord{def u0 com.example/.Main t1}", + ], + }); + const execution = await providerOver(binary).execute({ + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(execution.ok).toBe(true); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContainEqual( + expect.stringContaining("dumpsys window displays"), + ); +}); + +it("describes missing window focus as unknown after both sections omit declarations", async () => { + const { binary } = await writeAdbStub( + shellStub({ + "dumpsys:window:windows": "no focus\\n", + "dumpsys:window:displays": "no focus\\n", + }), + ); + const result = await evidenceResult(providerOver(binary), { + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(result).toMatchObject({ focused_window: null, focused_app: null }); + const execution = await providerOver(binary).execute({ + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(execution.ok).toBe(true); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContainEqual( + expect.stringContaining("focus is unknown"), + ); +}); + it("reads one settings value and reports device-side nulls", async () => { const { binary } = await writeAdbStub(shellStub({ "settings:get": "1\n" })); const enabled = await evidenceResult(providerOver(binary), { diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index efe3eb3fa..b7bc545ea 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -56,6 +56,7 @@ import { type AdbClientIdentity, } from "./AdbConfiguration.js"; import { + displayDevicePath, parseAdbDevicesOutput, parseGetpropOutput, parsePackageListOutput, @@ -417,8 +418,12 @@ const removeOwnedOutputDirectory = async ( const reservePackageDestinations = async ( operation: string, + packageName: string, packageDirectory: string, - entries: readonly { readonly file_name: string }[], + entries: readonly { + readonly file_name: string; + readonly device_path: string; + }[], capturedOutput: { readonly stdout: string; readonly stderr: string; @@ -427,6 +432,8 @@ const reservePackageDestinations = async ( ): Promise => { const reservations: { readonly path: string; + readonly file_name: string; + readonly device_path: string; readonly identity: OutputDirectoryIdentity; }[] = []; for (const entry of entries) { @@ -438,6 +445,8 @@ const reservePackageDestinations = async ( throw new Error(`Could not reserve local filename ${entry.file_name}`); reservations.push({ path, + file_name: entry.file_name, + device_path: entry.device_path, identity: { dev: created.dev, ino: created.ino, @@ -445,10 +454,28 @@ const reservePackageDestinations = async ( }, }); } catch (cause: unknown) { + const errorCode = + cause instanceof Error && + "code" in cause && + typeof cause.code === "string" + ? cause.code + : null; return new AnalysisOutputError( operation, - `Package Manager paths cannot be written to unique local destinations at ${JSON.stringify(entry.file_name)}; refusing before pulling any files`, - { cause, capturedOutput }, + "local_destination_unavailable", + { + cause, + capturedOutput, + userMessage: `pm path for ${packageName} reported ${displayDevicePath(entry.device_path)}, whose file name ${displayDevicePath(entry.file_name)} cannot be written to a unique local destination (${errorCode ?? "unknown error"}). No APK was pulled.`, + remediationAction: + "Review details.diagnostics and details.captured_output for the refused destination. Correct the output filesystem conflict before retrying; REA does not rename APKs.", + diagnostics: { + package_name: packageName, + device_path: entry.device_path, + file_name: entry.file_name, + error_code: errorCode, + }, + }, ); } } @@ -467,10 +494,28 @@ const reservePackageDestinations = async ( ); await rmdir(reservation.path); } catch (cause: unknown) { + const errorCode = + cause instanceof Error && + "code" in cause && + typeof cause.code === "string" + ? cause.code + : null; return new AnalysisOutputError( operation, - `Could not release a local filename reservation before pulling: ${reservation.path}`, - { cause, capturedOutput }, + "local_destination_unavailable", + { + cause, + capturedOutput, + userMessage: `Could not release the local filename reservation for ${displayDevicePath(reservation.device_path)} (${errorCode ?? "unknown error"}); no APK was pulled.`, + remediationAction: + "Review details.diagnostics and details.captured_output for the refused destination. Correct the output filesystem conflict before retrying; REA does not rename APKs.", + diagnostics: { + package_name: packageName, + device_path: reservation.device_path, + file_name: reservation.file_name, + error_code: errorCode, + }, + }, ); } } @@ -1095,14 +1140,27 @@ export class AdbProvider { ); } case "inspect_adb_window": { - const ran = await shell(["dumpsys", "window", "windows"], { + let section = "windows"; + const ran = await shell(["dumpsys", "window", section], { timeoutMs: SHELL_TIMEOUT_MS, maxOutputBytes: DUMPSYS_WINDOW_MAX_OUTPUT_BYTES, }); if (!ran.ok) return ran; - const { focusedWindow, focusedApp } = parseWindowFocusOutput( + let { focusedWindow, focusedApp } = parseWindowFocusOutput( ran.value.stdout, ); + if (focusedWindow === null && focusedApp === null) { + section = "displays"; + const fallback = await shell(["dumpsys", "window", section], { + timeoutMs: SHELL_TIMEOUT_MS, + maxOutputBytes: DUMPSYS_WINDOW_MAX_OUTPUT_BYTES, + }); + if (!fallback.ok) return fallback; + ({ focusedWindow, focusedApp } = parseWindowFocusOutput( + fallback.value.stdout, + )); + } + const declared = focusedWindow !== null || focusedApp !== null; return ok( createAnalysisExecution( { @@ -1119,6 +1177,9 @@ export class AdbProvider { { limitations: [ "Focus values are the window manager's declarations at read time; no interaction occurred", + declared + ? `Focus declarations were read from \`dumpsys window ${section}\`` + : "Neither `dumpsys window windows` nor `dumpsys window displays` declared mCurrentFocus or mFocusedApp; focus is unknown, not absent", ], }, ), @@ -1269,14 +1330,27 @@ export class AdbProvider { stderr: paths.value.stderr, truncated: false, }; - if (parsed.rejectedPaths.length > 0) + if (parsed.rejectedPaths.length > 0) { + const first = parsed.rejectedPaths[0]!; + const otherCount = parsed.rejectedPaths.length - 1; return err( - new AnalysisOutputError( - operation, - `Package Manager reported paths that cannot be represented as host files: ${parsed.rejectedPaths.map((entry) => `${JSON.stringify(entry.device_path)} (${entry.reason})`).join(", ")}`, - { capturedOutput }, - ), + new AnalysisOutputError(operation, "unsafe_device_path", { + capturedOutput, + userMessage: `pm path for ${packageName} reported an APK path that cannot be written as a file on this host: ${displayDevicePath(first.device_path)} (${first.reason})${otherCount > 0 ? ` and ${String(otherCount)} other path(s)` : ""}. No APK was pulled.`, + remediationAction: + "Review details.diagnostics for the refused device paths and details.captured_output for the retained pm path output. REA does not rename device-reported APK paths; correct the device path policy before retrying.", + diagnostics: { + package_name: packageName, + rejected_paths: parsed.rejectedPaths.map( + ({ device_path, reason }) => ({ + device_path, + reason, + }), + ), + }, + }), ); + } if (parsed.paths.length === 0 && parsed.unparsedLines.length === 0) return err( new AnalysisUnsupportedTargetError( @@ -1296,11 +1370,19 @@ export class AdbProvider { for (const entry of parsed.paths) { if (destinationNames.has(entry.file_name)) return err( - new AnalysisOutputError( - operation, - `Package Manager reported multiple paths with the same local destination ${JSON.stringify(entry.file_name)}; refusing before pulling any files`, - { capturedOutput }, - ), + new AnalysisOutputError(operation, "duplicate_local_destination", { + capturedOutput, + userMessage: `pm path for ${packageName} reported multiple APK paths with the same local file name ${displayDevicePath(entry.file_name)}; no APK was pulled.`, + remediationAction: + "Review details.diagnostics and details.captured_output for the conflicting device paths. REA does not rename device-reported paths; correct Package Manager's path response before retrying.", + diagnostics: { + package_name: packageName, + file_name: entry.file_name, + device_paths: parsed.paths + .filter((candidate) => candidate.file_name === entry.file_name) + .map((candidate) => candidate.device_path), + }, + }), ); destinationNames.add(entry.file_name); } @@ -1395,6 +1477,7 @@ export class AdbProvider { }; const destinationError = await reservePackageDestinations( operation, + packageName, packageDirectory, parsed.paths, capturedOutput, diff --git a/src/adb/AdbShellOutput.ts b/src/adb/AdbShellOutput.ts index 6dac4cc0c..70a1584a2 100644 --- a/src/adb/AdbShellOutput.ts +++ b/src/adb/AdbShellOutput.ts @@ -172,7 +172,7 @@ export const parseWmDensityOutput = ( return { physical, override }; }; -/** Parse the focus declarations from `dumpsys window windows`. */ +/** Parse focus declarations from `dumpsys window windows` or `displays`. */ export const parseWindowFocusOutput = ( stdout: string, ): { diff --git a/src/android/JadxConfiguration.ts b/src/android/JadxConfiguration.ts index b44d1f28b..e07e2c211 100644 --- a/src/android/JadxConfiguration.ts +++ b/src/android/JadxConfiguration.ts @@ -1,6 +1,7 @@ import { constants } from "node:fs"; import { access, realpath, stat } from "node:fs/promises"; import { delimiter, isAbsolute, join, resolve } from "node:path"; +import { inspectWindowsNativeReadiness } from "../process/WindowsAuthority.js"; import { systemWindowsNativeAuthority } from "../windows/WindowsNativeLoader.js"; import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; import type { AndroidOperation } from "../domain/android/androidAnalysis.js"; @@ -11,7 +12,15 @@ import { execFileOutput, execFileOutputFailure, } from "../process/ExecFileOutput.js"; -import { JADX_JAR_CONFIGURATION_REMEDIATION } from "./JadxRelease.js"; +import type { DoctorProviderCheck } from "../application/Doctor.js"; +import { + JADX_JAR_CONFIGURATION_REMEDIATION, + JADX_JAVA_PROBE_REMEDIATION, + JADX_JAVA_RUNTIME_REMEDIATION, + JADX_JVM_OPTIONS_REMEDIATION, + JADX_UNSUPPORTED_HOST_REMEDIATION, + JADX_WINDOWS_NATIVE_REMEDIATION, +} from "./JadxRelease.js"; const JAVA_MODULE_LISTING_TIMEOUT_MS = 10_000; const JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES = 1024 * 1024; @@ -30,16 +39,110 @@ type ConfigurationFailureCode = | "not_configured" | "executable_missing" | "runtime_missing" + | "runtime_unavailable" | "unsupported_host" | "unsupported_version" | "version_unresolved" | "open_options_invalid"; +/** + * Failure kinds tagged at the JADX configuration boundary. Consumers (the + * doctor adapter) project the boundary's remediation and classification for + * a kind; they must not reinterpret the phase/setting evidence to derive + * them. + */ +export type JadxFailureKind = + | "windows_native_artifact" + | "windows_native_runtime" + | "java_runtime" + | "java_home" + | "jvm_options" + | "jar" + | "java_probe" + | "unsupported_host"; + +/** Runtime mirror of JadxFailureKind for validating the loose diagnostics tag. */ +export const JADX_FAILURE_KINDS: readonly JadxFailureKind[] = [ + "windows_native_artifact", + "windows_native_runtime", + "java_runtime", + "java_home", + "jvm_options", + "jar", + "java_probe", + "unsupported_host", +]; + +export interface JadxFailureOutcome { + readonly remediation: string; + readonly classification: DoctorProviderCheck["classification"]; +} + +/** + * Project the boundary-owned remediation and classification for a failure + * kind. A null kind (missing or corrupt tag) keeps JAR guidance, matching + * the previous fallthrough for unclassified failures. + */ +export const projectJadxFailureOutcome = ( + kind: JadxFailureKind | null, + configured: boolean, +): JadxFailureOutcome => { + const drift = configured ? "config_drift" : "missing_analysis_engine"; + switch (kind) { + case "windows_native_artifact": + return { + remediation: JADX_WINDOWS_NATIVE_REMEDIATION, + classification: "missing_dependency", + }; + case "windows_native_runtime": + return { + remediation: JADX_WINDOWS_NATIVE_REMEDIATION, + classification: "runtime_failure", + }; + case "java_runtime": + return { + remediation: JADX_JAVA_RUNTIME_REMEDIATION, + classification: "missing_dependency", + }; + case "java_home": + return { + remediation: JADX_JAVA_RUNTIME_REMEDIATION, + classification: drift, + }; + case "jvm_options": + return { + remediation: JADX_JVM_OPTIONS_REMEDIATION, + classification: drift, + }; + case "jar": + return { + remediation: JADX_JAR_CONFIGURATION_REMEDIATION, + classification: drift, + }; + case "java_probe": + return { + remediation: JADX_JAVA_PROBE_REMEDIATION, + classification: drift, + }; + case "unsupported_host": + return { + remediation: JADX_UNSUPPORTED_HOST_REMEDIATION, + classification: "unsupported_host", + }; + case null: + return { + remediation: JADX_JAR_CONFIGURATION_REMEDIATION, + classification: drift, + }; + } +}; + class JadxConfigurationFailure extends Error { constructor( readonly code: ConfigurationFailureCode, message: string, readonly diagnostics: Readonly>, + readonly kind: JadxFailureKind, options?: ErrorOptions, ) { super(message, options); @@ -75,7 +178,7 @@ export const inspectJadxAvailability = async ( status: "unavailable", code: cause.code, reason: cause.message, - diagnostics: cause.diagnostics, + diagnostics: { ...cause.diagnostics, failure_kind: cause.kind }, }; throw cause; } @@ -191,6 +294,7 @@ const requireEngineArchive = async ( archive_error: reason, maximum_metadata_read_bytes: JAR_METADATA_READ_MAX_BYTES, }, + "jar", cause, ); } finally { @@ -200,6 +304,7 @@ const requireEngineArchive = async ( "unsupported_version", `REA_JADX_MCP_JAR ${jar} does not contain the required metadata bridge classes: ${[...missing].join(", ")}. ${JADX_JAR_CONFIGURATION_REMEDIATION}`, { jar_path: jar, phase: "jar-inspection", missing_classes: [...missing] }, + "jar", ); }; @@ -207,25 +312,52 @@ const requireSupportedHost = (): void => { if (process.platform === "linux" || process.platform === "darwin") return; if (process.platform === "win32" && process.arch === "x64") { const loaded = systemWindowsNativeAuthority(); - if (loaded.available && loaded.authority.inspection.protocolStdin === true) - return; + if (loaded.available) { + if (loaded.authority.inspection.protocolStdin === true) return; + throw configurationFailure( + "runtime_missing", + "The bundled Windows native artifact lacks owned protocol stdin; use a matching current Windows x64 REA package.", + { + platform: process.platform, + architecture: process.arch, + phase: "process-ownership", + owned_protocol_stdin: false, + }, + "windows_native_artifact", + ); + } + const readiness = inspectWindowsNativeReadiness(process.platform, loaded); throw configurationFailure( - "runtime_missing", - loaded.available - ? "The bundled Windows native artifact lacks owned protocol stdin; use a matching current Windows x64 REA package." - : loaded.reason, + readiness.available ? "runtime_unavailable" : readiness.code, + readiness.available + ? "Windows native readiness disagreed with the native loader result." + : readiness.reason, { platform: process.platform, architecture: process.arch, phase: "process-ownership", owned_protocol_stdin: false, + ...(readiness.available + ? {} + : { + failure_stage: readiness.failureStage, + remediation: readiness.remediation, + }), }, + readiness.available + ? "windows_native_runtime" + : readiness.code === "unsupported_host" + ? "unsupported_host" + : readiness.code === "runtime_missing" + ? "windows_native_artifact" + : "windows_native_runtime", ); } throw configurationFailure( "unsupported_host", `Android JADX subprocess ownership is unsupported on ${process.platform}/${process.arch}; use Linux, macOS, or Windows x64 with its bundled native controls.`, { platform: process.platform }, + "unsupported_host", ); }; @@ -238,6 +370,7 @@ const resolveJar = async ( "not_configured", JADX_JAR_CONFIGURATION_REMEDIATION, { jar_configured: false }, + "jar", ); try { await access(jar, constants.R_OK); @@ -246,6 +379,7 @@ const resolveJar = async ( "executable_missing", `REA_JADX_MCP_JAR is not a regular file: ${jar}`, { jar_path: jar }, + "jar", ); return await realpath(jar); } catch (cause) { @@ -254,6 +388,7 @@ const resolveJar = async ( "executable_missing", `Cannot read REA_JADX_MCP_JAR ${jar}: ${cause instanceof Error ? cause.message : String(cause)}`, { jar_path: jar }, + "jar", cause, ); } @@ -293,6 +428,7 @@ const resolveJava = async ( "runtime_missing", `Cannot inspect selected Java executable ${candidate}: ${errorText(cause)}`, { java_executable: candidate }, + "java_runtime", cause, ); } @@ -301,6 +437,7 @@ const resolveJava = async ( "runtime_missing", "Cannot find java.exe on the selected Windows PATH. Select a full JDK with JAVA_HOME or PATH, then verify it with java --list-modules. REA does not install Java.", { java_executable: executable }, + "java_runtime", ); } if (isAbsolute(environment.JAVA_HOME)) @@ -309,6 +446,7 @@ const resolveJava = async ( "open_options_invalid", "JAVA_HOME must select an existing JDK by absolute path; omit it to use java on PATH.", { java_home: environment.JAVA_HOME }, + "java_home", ); }; @@ -324,6 +462,7 @@ const requireJavaHomeExecutable = async ( "runtime_missing", `JAVA_HOME java is not a regular file: ${java}`, { java_executable: java }, + "java_runtime", ); } catch (cause) { if (cause instanceof JadxConfigurationFailure) throw cause; @@ -331,6 +470,7 @@ const requireJavaHomeExecutable = async ( "runtime_missing", `Cannot execute JAVA_HOME java ${java}: ${cause instanceof Error ? cause.message : String(cause)}`, { java_executable: java }, + "java_runtime", cause, ); } @@ -351,6 +491,7 @@ const resolveJvmArguments = ( "open_options_invalid", `${name} must be a positive safe integer; received ${value}.`, { setting: name, value }, + "jvm_options", ); if ( name === "REA_JADX_ACTIVE_PROCESSOR_COUNT" && @@ -360,6 +501,7 @@ const resolveJvmArguments = ( "open_options_invalid", `${name} exceeds the JVM's signed 32-bit processor-count range.`, { setting: name, value }, + "jvm_options", ); result.push(`${prefix}${value}${suffix}`); } @@ -405,6 +547,7 @@ const inspectJavaRuntime = async ( failure: "output_limit", output_limit_bytes: JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES, }, + "java_probe", cause, ); if (output.killed) @@ -416,12 +559,14 @@ const inspectJavaRuntime = async ( failure: "probe_terminated", deadline_ms: JAVA_MODULE_LISTING_TIMEOUT_MS, }, + "java_probe", cause, ); throw configurationFailure( "runtime_missing", `Cannot inspect selected Java runtime ${java}: ${output.stderr.trim() || errorText(cause)}. Select a full JDK including jdk.compiler via JAVA_HOME or PATH; verify it with java --list-modules.`, diagnostics, + "java_runtime", cause, ); } @@ -438,6 +583,7 @@ const inspectJavaRuntime = async ( stderr, output_truncated: false, }, + "java_runtime", ); const major = /^java\.base@(\d+)(?:[.+_-]|$)/mu.exec(stdout)?.[1]; if (major === undefined) @@ -453,6 +599,7 @@ const inspectJavaRuntime = async ( stderr, output_truncated: false, }, + "java_probe", ); if (Number(major) < 17) throw configurationFailure( @@ -467,6 +614,7 @@ const inspectJavaRuntime = async ( stderr, output_truncated: false, }, + "java_runtime", ); }; @@ -474,12 +622,14 @@ const configurationFailure = ( code: ConfigurationFailureCode, message: string, diagnostics: Readonly>, + kind: JadxFailureKind, cause?: unknown, ): JadxConfigurationFailure => new JadxConfigurationFailure( code, message, diagnostics, + kind, cause === undefined ? undefined : { cause }, ); diff --git a/src/android/JadxRelease.ts b/src/android/JadxRelease.ts index 8c7d16840..447886b52 100644 --- a/src/android/JadxRelease.ts +++ b/src/android/JadxRelease.ts @@ -15,6 +15,26 @@ export const JADX_BRIDGE_IDENTITY = Object.freeze({ /** Remediation when `REA_JADX_MCP_JAR` is missing or not an absolute path. */ export const JADX_JAR_CONFIGURATION_REMEDIATION = `Set REA_JADX_MCP_JAR to the absolute path of a caller-supplied jadx-headless-mcp ${JADX_RELEASE.version} JAR. REA does not download or install it.`; +/** Remediation when the selected Java runtime cannot satisfy the JADX bridge. */ +export const JADX_JAVA_RUNTIME_REMEDIATION = + "Select a full JDK including jdk.compiler via JAVA_HOME or PATH, then verify it with java --list-modules. REA does not install Java."; + +/** Remediation when a JADX JVM resource override is malformed. */ +export const JADX_JVM_OPTIONS_REMEDIATION = + "Set REA_JADX_HEAP_MIB to a positive integer (heap size in MiB) and REA_JADX_ACTIVE_PROCESSOR_COUNT to a positive integer within the JVM's signed 32-bit range."; + +/** Remediation when the host cannot run the Android JADX subprocess. */ +export const JADX_UNSUPPORTED_HOST_REMEDIATION = + "Use Linux, macOS, or Windows x64 with its bundled native controls for Android JADX analysis."; + +/** Remediation when the Windows native backend cannot take process ownership. */ +export const JADX_WINDOWS_NATIVE_REMEDIATION = + "Use a matching current Windows x64 REA package with its bundled native controls. The native failure above identifies the failed constraint; Java installation or registration changes cannot enable a missing native backend."; + +/** Remediation when the Java module-listing probe cannot establish the runtime. */ +export const JADX_JAVA_PROBE_REMEDIATION = + "Verify the selected Java launcher with java --list-modules: it must start promptly and emit a concise module listing including a parseable java.base version. REA does not install Java."; + /** Producing engine identity used by Evidence and the generated product catalog. */ export const JADX_PROVIDER_IDENTITY = Object.freeze({ id: "jadx", diff --git a/src/apktool/ApktoolProvider.test.ts b/src/apktool/ApktoolProvider.test.ts index b6ee249e5..5ac36876a 100644 --- a/src/apktool/ApktoolProvider.test.ts +++ b/src/apktool/ApktoolProvider.test.ts @@ -100,10 +100,12 @@ const decodeStub = ( readonly manifestFile?: string; readonly stringsFile?: string; readonly stringsXml?: string; + readonly extraValuesDirectories?: readonly string[]; } = {}, ): ((root: string) => string) => { const withStrings = options.withStrings ?? true; const withGerman = options.withGerman ?? true; + const extraValuesDirectories = options.extraValuesDirectories ?? []; const stringsXml = options.stringsXml ?? REAL_STRINGS; const ymlWrite = options.ymlFile === undefined @@ -117,6 +119,12 @@ const decodeStub = ( options.stringsFile === undefined ? `printf '%s\\n' '${stringsXml.replace(/'/gu, "'\\''")}' > "$5/res/values/strings.xml"` : `cp '${options.stringsFile}' "$5/res/values/strings.xml"`; + const qualifierStrings = extraValuesDirectories + .map( + (name) => + `printf '%s\\n' 'Wrong' > "$5/res/values-${name}/strings.xml"`, + ) + .join("\\n "); return () => `if [ "$1" = "--version" ]; then ${VERSION_BLOCK}; exit 0; fi if [ "$1" = "d" ]; then ${ @@ -124,9 +132,10 @@ if [ "$1" = "d" ]; then ? `printf 'Exception in thread "main" brut.androlib.AndrolibException: Could not decode the APK\\n' >&2; exit 1;` : `${ymlWrite} ${manifestWrite} - mkdir -p "$5/res/values"${withGerman ? ' "$5/res/values-de"' : ""} + mkdir -p "$5/res/values"${withGerman ? ' "$5/res/values-de"' : ""}${extraValuesDirectories.map((name) => ` "$5/res/values-${name}"`).join("")} ${withStrings ? stringsWrite : ""} ${withGerman ? `printf '%s\\n' '${DE_STRINGS.replace(/'/gu, "'\\''")}' > "$5/res/values-de/strings.xml"` : ""} + ${qualifierStrings} printf 'I: Using Apktool on stub\\n' exit 0` } @@ -254,6 +263,30 @@ it("preserves empty resource strings and reports markup omissions as partial", a expect(result.coverage).toBe("partial"); }); +it("ignores Android resource qualifiers when discovering locales", async () => { + const { command, root } = await writeApktoolStub( + decodeStub({ extraValuesDirectories: ["car", "night", "v21"] }), + ); + const apk = await apkFixture(root); + const result = await evidenceResult(providerOver(command), { + operation: "decode_android_resources", + input: { path: apk.path, include_strings: true }, + }); + expect(result.locales).toEqual(["de"]); +}); + +it.each(["car", "night", "v21"])( + "rejects %s as a requested locale", + (locale) => { + expect( + apktoolRequestSchema.safeParse({ + operation: "decode_android_resources", + input: { path: "/tmp/probe.apk", locale }, + }).success, + ).toBe(false); + }, +); + it("projects one locale's strings on request", async () => { const { command, root } = await writeApktoolStub(decodeStub()); const apk = await apkFixture(root); diff --git a/src/apktool/ApktoolProvider.ts b/src/apktool/ApktoolProvider.ts index c58ea1f49..eec08fe43 100644 --- a/src/apktool/ApktoolProvider.ts +++ b/src/apktool/ApktoolProvider.ts @@ -25,7 +25,10 @@ import { createAnalysisExecution, type AnalysisExecution, } from "../application/AnalysisProvider.js"; -import type { ApktoolRequest } from "../domain/apktool/apktoolResourceAnalysis.js"; +import { + androidLocaleQualifierSchema, + type ApktoolRequest, +} from "../domain/apktool/apktoolResourceAnalysis.js"; import { AnalysisCapabilityUnavailableError, AnalysisCancelledError, @@ -291,6 +294,7 @@ const countDecodedFiles = async ( const match = /^values-(.+)$/u.exec(entry.name); if ( match !== null && + androidLocaleQualifierSchema.safeParse(match[1]).success && (await access(join(path, "strings.xml")).then( () => true, () => false, diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 376036fa1..d1d41800c 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -275,6 +275,12 @@ export type ProviderTargetSupport = readonly diagnostics: Readonly>; }; +/** One owner-observed availability/capability snapshot for provider discovery. */ +export interface AnalysisProviderReadinessSnapshot { + readonly availability: ProviderAvailability; + readonly capabilities: readonly CapabilityDescriptor[]; +} + /** Deep provider candidate discoverable before one target-bound route is chosen. */ export interface AnalysisProviderCandidate extends AnalysisProvider { resolveAnalysisProfile( @@ -282,5 +288,6 @@ export interface AnalysisProviderCandidate extends AnalysisProvider { options?: AnalysisProfileResolutionOptions, ): Promise>; inspectAvailability(): ProviderAvailability; + inspectReadinessSnapshot?(): AnalysisProviderReadinessSnapshot; inspectTargetSupport(target: BinaryTarget): ProviderTargetSupport; } diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 94b9ac659..4f1a217f0 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -98,6 +98,21 @@ const entry = ( }; describe("capability failure reasons", () => { + it("keeps an unavailable Android runtime distinct from a missing provider", () => { + const inventory = buildCapabilityInventory(status(), { + ...enabledPolicy, + androidAnalysisAvailability: { + status: "unavailable", + code: "runtime_unavailable", + reason: "The JADX native self-test failed.", + diagnostics: {}, + }, + }); + expect( + inventory.find(({ name }) => name === "inspect_android_package"), + ).toMatchObject({ available: false, reason: "provider_unavailable" }); + }); + it("admits physical universal Mach-O targets for native Mach-O operations", () => { expect( entry( @@ -501,7 +516,6 @@ it("reports host denial before target mismatch but keeps composed workflow prere describe("binary_overview capability requirements", () => { it("admits a provider with current_document and other mandatory capabilities", () => { const capabilities = [ - "list_segments", "current_document", "list_procedures", "list_strings", @@ -520,7 +534,6 @@ describe("binary_overview capability requirements", () => { it("admits a provider with list_documents and other mandatory capabilities", () => { const capabilities = [ - "list_segments", "list_documents", "list_procedures", "list_strings", @@ -556,7 +569,7 @@ describe("binary_overview capability requirements", () => { }); }); - it("reports missing segment enumeration independently when segments are missing", () => { + it("admits a provider without optional segment enumeration", () => { const capabilities = [ "current_document", "list_procedures", @@ -568,12 +581,31 @@ describe("binary_overview capability requirements", () => { status({ open: true, kind: "executable", capabilities }), ), ).toMatchObject({ - available: false, - reason: "provider_missing", - remediation: "Configure a provider for required operation list_segments.", + available: true, + reason: "available", + remediation: null, }); }); + it("keeps binary overview available when a declared segment facet is unavailable", () => { + const capabilities = [ + { + operation: "list_segments", + available: false, + reason: "This upstream profile does not expose segment enumeration.", + }, + { operation: "list_documents", available: true, reason: null }, + { operation: "list_procedures", available: true, reason: null }, + { operation: "list_strings", available: true, reason: null }, + ]; + expect( + entry( + "binary_overview", + status({ open: true, kind: "executable", capabilities }), + ), + ).toMatchObject({ available: true, reason: "available" }); + }); + it("reports provider_unavailable when declared document selection capability is unavailable", () => { const capabilities = [ { operation: "list_segments", available: true, reason: null }, diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 87734fbd1..029b23357 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -115,7 +115,6 @@ const ENHANCED_REQUIREMENTS: Readonly< analyze_swift_types: ["list_procedures"], find_xrefs_to_name: ["list_names", "xrefs"], binary_overview: [ - "list_segments", ["current_document", "list_documents"], "list_procedures", "list_strings", @@ -462,7 +461,10 @@ const workflowAvailabilityFor = ({ reason: policy.androidAnalysisAvailability?.code === "unsupported_host" ? "unsupported_host" - : "provider_missing", + : policy.androidAnalysisAvailability?.code === + "runtime_unavailable" + ? "provider_unavailable" + : "provider_missing", remediation: policy.androidAnalysisAvailability?.reason ?? "Set REA_JADX_MCP_JAR to a caller-supplied jadx-headless-mcp 0.7.1 JAR and provide a full JDK on Linux, macOS, or Windows x64 with its matching bundled native controls.", diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index 207186ccc..7bd340d3b 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -525,33 +525,47 @@ export class EnhancedTools { this.#call("list_strings", {}, signal), ], ); - if (!segmentsResult.ok) return segmentsResult; + if ( + !segmentsResult.ok && + !(segmentsResult.error instanceof AnalysisCapabilityUnavailableError) + ) + return segmentsResult; if (!proceduresResult.ok) return proceduresResult; if (!stringsResult.ok) return stringsResult; - const segments = parseSegments(segmentsResult.value); - if (!segments.ok) return segments; + const segments = segmentsResult.ok + ? parseSegments(segmentsResult.value) + : undefined; + if (segments !== undefined && !segments.ok) return segments; const procedureCount = parseListCount(proceduresResult.value, "procedures"); if (!procedureCount.ok) return procedureCount; const stringCount = parseListCount(stringsResult.value, "strings"); if (!stringCount.ok) return stringCount; - const imageBase = parseSegmentImageBase(segments.value); + const observedSegments = segments?.ok ? segments.value : undefined; + const imageBase = + observedSegments === undefined + ? undefined + : parseSegmentImageBase(observedSegments); return ok({ document: document.value, ...(imageBase === undefined ? {} : { image_base: imageBase }), - segments: segments.value.map( - ({ name, start, end, image_base, address_space }) => ({ - name, - start, - end, - length: addressDistance(start, end), - ...(image_base === undefined ? {} : { image_base }), - ...(address_space === undefined ? {} : { address_space }), - }), - ), - segment_count: segments.value.length, + ...(observedSegments === undefined + ? {} + : { + segments: observedSegments.map( + ({ name, start, end, image_base, address_space }) => ({ + name, + start, + end, + length: addressDistance(start, end), + ...(image_base === undefined ? {} : { image_base }), + ...(address_space === undefined ? {} : { address_space }), + }), + ), + segment_count: observedSegments.length, + }), procedure_count: procedureCount.value, string_count: stringCount.value, }); diff --git a/src/application/MipsElfMetadata.ts b/src/application/MipsElfMetadata.ts index 8fd9d0e89..38b2bccb8 100644 --- a/src/application/MipsElfMetadata.ts +++ b/src/application/MipsElfMetadata.ts @@ -10,6 +10,8 @@ import { type MipsElfRelocationSection, } from "./MipsElfSharedLoad.js"; +const SHT_GNU_ATTRIBUTES = 0x6ffffff5; + /** Observed ABI and load declarations, retaining unavailable source coverage. */ export interface MipsElfInspection { readonly abiFlags: MipsAbiFlags | null; @@ -151,6 +153,10 @@ export const readMipsElfMetadata = async ( checkCancelled(); const position = i * table.entrySize; const type = u32(contents.value, position + table.typeOffset); + if (table.optional && type === SHT_GNU_ATTRIBUTES) + return err( + "MIPS ELF declares GNU ABI attributes that REA does not inspect; refusing provider selection until the declared ABI is parsed and reconciled", + ); if (metadata.type === 3 && table.optional && (type === 4 || type === 9)) relocationSections.push({ type, diff --git a/src/application/WindowsCapabilities.ts b/src/application/WindowsCapabilities.ts index dfd092c69..61b96c40f 100644 --- a/src/application/WindowsCapabilities.ts +++ b/src/application/WindowsCapabilities.ts @@ -70,24 +70,26 @@ export const probeWindowsCapabilities = async ( }; }; -const systemDependencies = (): WindowsCapabilityDependencies => ({ - platform: process.platform, - architecture: process.arch, - probeSymlinkCreation, - probeNoFollowOpen: () => - systemNoFollowOpenCapability(process.platform, constants.O_NOFOLLOW), - probePrivateAcl: () => - windowsNativeCapabilities(process.platform).private_runtime_dacl, - nativeCapabilities: () => windowsNativeCapabilities(process.platform), - probeUnixDomainSocket: () => - process.platform === "win32" - ? { - available: false, - reason: "Node path-based IPC uses named pipes on Windows", - } - : { available: true, reason: null }, - probePty: probeProcessCaptureCapability, -}); +const systemDependencies = (): WindowsCapabilityDependencies => { + const security = windowsNativeCapabilities(process.platform); + return { + platform: process.platform, + architecture: process.arch, + probeSymlinkCreation, + probeNoFollowOpen: () => + systemNoFollowOpenCapability(process.platform, constants.O_NOFOLLOW), + probePrivateAcl: () => security.private_runtime_dacl, + nativeCapabilities: () => security, + probeUnixDomainSocket: () => + process.platform === "win32" + ? { + available: false, + reason: "Node path-based IPC uses named pipes on Windows", + } + : { available: true, reason: null }, + probePty: probeProcessCaptureCapability, + }; +}; /** Report pathname no-follow support without treating it as Windows authority. */ export const systemNoFollowOpenCapability = ( diff --git a/src/application/binary/AnalysisProviderRegistry.test.ts b/src/application/binary/AnalysisProviderRegistry.test.ts index fde2f6718..60d3ab62a 100644 --- a/src/application/binary/AnalysisProviderRegistry.test.ts +++ b/src/application/binary/AnalysisProviderRegistry.test.ts @@ -13,6 +13,7 @@ import { import { createAnalysisProfile } from "../../domain/analysisProfile.js"; import type { BinaryTarget } from "../../domain/binaryTargetTypes.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { ProviderSelectionError } from "../../domain/providerSelectionError.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { err, ok } from "../../domain/result.js"; @@ -63,6 +64,46 @@ it("rejects malformed provider capability declarations", () => { ).toThrow(/Invalid analysis provider ID: auto/u); }); +it("uses one owner readiness snapshot for candidate availability and capabilities", () => { + const base = candidate("alpha").provider; + let observed = 0; + let capabilityReads = 0; + const provider: AnalysisProviderCandidate = { + ...base, + inspectAvailability: () => { + throw new Error("availability must come from the owner snapshot"); + }, + capabilities: () => { + capabilityReads += 1; + return [capability(base.identity())]; + }, + inspectReadinessSnapshot: () => { + observed += 1; + return { + availability: { + status: "unavailable", + code: "runtime_unavailable", + reason: "native inspection failed", + diagnostics: { failure_stage: "self-test" }, + }, + capabilities: [capability(base.identity())], + }; + }, + }; + + const [status] = new AnalysisProviderRegistry([provider]).candidates(); + expect(observed).toBe(1); + expect(capabilityReads).toBe(1); + expect(status).toMatchObject({ + availability: { + status: "unavailable", + code: "runtime_unavailable", + diagnostics: { failure_stage: "self-test" }, + }, + capabilities: [{ operation: "address_name" }], + }); +}); + it("selects the sole available provider and honors explicit selection precedence", async () => { const alpha = candidate("alpha"); const beta = candidate("beta", { available: false }); @@ -104,7 +145,7 @@ it("selects the sole available provider and honors explicit selection precedence }); }); -describe("analysis provider registry: binding failures and cancellation", () => { +describe("analysis provider registry: selection failures", () => { it("returns actionable unknown, unavailable, and unsupported failures", async () => { const unavailable = candidate("alpha", { available: false }); const registry = new AnalysisProviderRegistry([unavailable.provider]); @@ -207,7 +248,9 @@ describe("analysis provider registry: binding failures and cancellation", () => }); } }); +}); +describe("analysis provider registry: profile and cleanup failures", () => { it("does not bind a profile with unresolved version or adapter failure", async () => { const unresolved = candidate("unresolved", { profile: "missing" }); const failed = candidate("failed", { profile: "error" }); @@ -240,6 +283,36 @@ describe("analysis provider registry: binding failures and cancellation", () => }); }); + it("aborts automatic selection when a provider reports unconfirmed cleanup", async () => { + const ghidra = candidate("ghidra"); + const alternate = candidate("hopper"); + const cleanupFailure = new ProviderCleanupError( + "ghidra", + ["/opt/ghidra/data/languages/x86.ldefs"], + { reason: "descriptor close failed" }, + { operation: "resolve_analysis_profile" }, + ); + const provider: AnalysisProviderCandidate = { + ...ghidra.provider, + resolveAnalysisProfile: () => Promise.resolve(err(cleanupFailure)), + }; + + const selected = await new AnalysisProviderRegistry([ + alternate.provider, + provider, + ]).select(DATABASE_TARGET); + + expect(selected).toMatchObject({ + ok: false, + error: { + _tag: "ProviderAdapterError", + providerId: "ghidra", + cleanupIncomplete: true, + }, + }); + expect(alternate.created).toEqual([]); + }); + it("cancels a pending profile probe even when the adapter ignores its signal", async () => { const alpha = candidate("alpha"); let observedSignal: AbortSignal | undefined; diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index 033677881..c6a1739a3 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -306,6 +306,10 @@ export class AnalysisProviderRegistry { throw new TypeError("Cancelled candidate evaluation disappeared"); return result.value; }); + const cleanupFailure = evaluations.find( + ({ profileError }) => profileError?.cleanupIncomplete === true, + )?.profileError; + if (cleanupFailure !== undefined) return err(cleanupFailure); const statuses = evaluations.map(({ status }) => status); const usable = evaluations.filter( ( @@ -334,23 +338,29 @@ export class AnalysisProviderRegistry { const candidateStatus = ( provider: AnalysisProviderCandidate, target: BinaryTarget | undefined, -): AnalysisProviderCandidateStatus => ({ - provider: structuredClone(provider.identity()), - availability: cloneAvailability(provider.inspectAvailability()), - targetSupport: - target === undefined - ? { - status: "unknown", - code: null, - reason: "No target is open.", - diagnostics: {}, - } - : cloneTargetSupport(provider.inspectTargetSupport(target)), - selected: false, - capabilities: provider - .capabilities() - .map((capability) => structuredClone(capability)), -}); +): AnalysisProviderCandidateStatus => { + const readiness = provider.inspectReadinessSnapshot?.() ?? { + availability: provider.inspectAvailability(), + capabilities: provider.capabilities(), + }; + return { + provider: structuredClone(provider.identity()), + availability: cloneAvailability(readiness.availability), + targetSupport: + target === undefined + ? { + status: "unknown", + code: null, + reason: "No target is open.", + diagnostics: {}, + } + : cloneTargetSupport(provider.inspectTargetSupport(target)), + selected: false, + capabilities: readiness.capabilities.map((capability) => + structuredClone(capability), + ), + }; +}; const signalIsAborted = (signal: AbortSignal | undefined): boolean => signal?.aborted === true; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index da39ce53c..24158e4a1 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -125,7 +125,7 @@ export const resolveGhidraAnalysisProfile = async ( load_image_observations: "source-mappings-entry-context-v2", function_body_evidence: "complete-inclusive-ranges-v1", function_references: "complete-body-and-entry-reference-manager-v2", - call_relationship_coverage: "resolved-call-reference-diagnostics-v1", + call_relationship_coverage: "function-body-call-coverage-v5", function_dossier_coverage: "observed-facets-and-call-relationships-v1", string_inventory_evidence: "defined-data-coverage-v1", @@ -134,7 +134,8 @@ export const resolveGhidraAnalysisProfile = async ( "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", - function_boundary_observations: "ghidra-terminal-call-limitations-v2", + function_boundary_observations: + "ghidra-function-boundary-limitations-v3", no_return_repair: installation.platform === "win32" ? "disabled-windows-p0" @@ -222,7 +223,7 @@ export const resolveGhidraAnalysisProfile = async ( load_image_observations: "source-mappings-entry-context-v2", function_body_evidence: "complete-inclusive-ranges-v1", function_references: "complete-body-and-entry-reference-manager-v2", - call_relationship_coverage: "resolved-call-reference-diagnostics-v1", + call_relationship_coverage: "function-body-call-coverage-v5", function_dossier_coverage: "observed-facets-and-call-relationships-v1", string_inventory_evidence: "defined-data-coverage-v1", location_resolution: "explicit-address-exact-entry-symbol-first-v3", @@ -230,7 +231,8 @@ export const resolveGhidraAnalysisProfile = async ( "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", - function_boundary_observations: "ghidra-terminal-call-limitations-v2", + function_boundary_observations: + "ghidra-function-boundary-limitations-v3", no_return_repair: installation.platform === "win32" ? "disabled-windows-p0" diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index d1dee3f01..acbd58ee6 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -58,6 +58,7 @@ import { sameRegularFileState, } from "../filesystem/RegularFile.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js"; export type GhidraTargetSnapshot = | { readonly kind: "captured"; readonly bytes: Buffer } @@ -119,7 +120,7 @@ export class GhidraClient { #closePromise: Promise | undefined; readonly #failure = bindGhidraSessionFailure( () => this.#diagnostics(), - (value) => this.#redactAuthentication(value), + (value, incomplete) => this.#redactAuthentication(value, incomplete), ); readonly #wire: GhidraWire; readonly #requestQueue: GhidraRequestQueue; @@ -662,25 +663,47 @@ export class GhidraClient { maxDiagnosticBytes: GHIDRA_PROCESS_DIAGNOSTIC_BYTES, onDiagnostic: (event) => this.#onProcessDiagnostic(event), }); - const connected = await this.#connect(endpoint, deadline); - if (!connected.ok) { - const failure = connected.error; - return this.#startupFailure(failure); - } - const completed = await completeGhidraStartupHandshake({ - deadline, - request: (method, params, requestOptions) => - this.#wire.request(method, params, requestOptions), - parseSessionInfo: (value) => this.#parseSessionInfo(value), - failure: this.#failure, - startupTimeoutMs: this.#options.startupTimeoutMs, - }); + const completed = await this.#connectAndHandshake(endpoint, deadline); if (!completed.ok) return this.#startupFailure(completed.error); await this.#lineage.observe(this.#launch); this.#requestQueue.reopen(); return completed; } + async #connectAndHandshake( + endpoint: GhidraEndpoint, + deadline: ProviderStartupDeadline, + ): Promise { + // Retry only a dropped loopback transport, never identity or protocol failures. + for (let attempt = 0; ; attempt += 1) { + const connected = await this.#connect(endpoint, deadline); + if (!connected.ok) return connected; + const completed = await completeGhidraStartupHandshake({ + deadline, + request: (method, params, requestOptions) => + this.#wire.request(method, params, requestOptions), + parseSessionInfo: (value) => this.#parseSessionInfo(value), + failure: this.#failure, + startupTimeoutMs: this.#options.startupTimeoutMs, + }); + if ( + completed.ok || + completed.error.kind !== "process" || + endpoint.transport !== "authenticated-loopback-tcp" || + attempt >= 2 || + deadline.signal.aborted || + this.#closing || + this.#process?.observedExit() !== undefined + ) + return completed; + this.#detachSocket?.(); + this.#detachSocket = undefined; + this.#socket?.destroy(); + this.#socket = undefined; + this.#responseBuffer.reset(); + } + } + async #connect( endpoint: GhidraEndpoint, deadline: ProviderStartupDeadline, @@ -880,9 +903,15 @@ export class GhidraClient { ); } - #redactAuthentication(value: string): string { - for (const token of this.#authenticationTokens) - value = value.replaceAll(token, "[REDACTED]"); + #redactAuthentication(value: string, incomplete = false): string { + for (const token of this.#authenticationTokens) { + const bytes = Buffer.byteLength(value); + value = redactCapturedTransportCredential( + { text: value, bytes, observedBytes: bytes + (incomplete ? 1 : 0) }, + token, + "[REDACTED]", + ); + } return value; } diff --git a/src/ghidra/GhidraDoctor.test.ts b/src/ghidra/GhidraDoctor.test.ts index 47ec9c882..6a8504547 100644 --- a/src/ghidra/GhidraDoctor.test.ts +++ b/src/ghidra/GhidraDoctor.test.ts @@ -4,7 +4,7 @@ import { runDoctor, type DoctorHost } from "../application/Doctor.js"; import { createDoctorHostFixture } from "../application/Doctor.fixture.js"; import { providerRegistrationEnvironment } from "../application/SetupRegistrationEnvironment.js"; import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; -import * as windowsAuthority from "../process/WindowsAuthority.js"; +import * as nativeLoader from "../windows/WindowsNativeLoader.js"; import { inspectGhidraInstallation, type GhidraInstallationHost, @@ -180,13 +180,11 @@ describe("Ghidra doctor integration", () => { describe("Windows Ghidra doctor authority", () => { beforeEach(() => { - vi.spyOn(windowsAuthority, "hasWindowsNativeAuthority").mockReturnValue( - false, - ); - vi.spyOn( - windowsAuthority, - "windowsNativeAuthorityUnavailableReason", - ).mockReturnValue(WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON); + vi.spyOn(nativeLoader, "systemWindowsNativeAuthority").mockReturnValue({ + available: false, + failureStage: "artifact", + reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, + }); }); afterEach(() => vi.restoreAllMocks()); it("distinguishes a valid Windows installation from unavailable native authority", async () => { @@ -228,9 +226,9 @@ describe("Windows Ghidra doctor authority", () => { result.checks.find(({ name }) => name === "ghidra-native_authority"), ).toMatchObject({ ok: false, - classification: "unsupported_host", + classification: "missing_dependency", detail: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, - remediation: expect.stringContaining("cannot enable"), + remediation: expect.stringContaining("matching Windows x64 REA package"), }); }); @@ -252,7 +250,109 @@ describe("Windows Ghidra doctor authority", () => { expect(result.scope_checks).toContainEqual( expect.objectContaining({ name: "ghidra-native_authority", - classification: "unsupported_host", + classification: "missing_dependency", + detail: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, + }), + ); + }); +}); + +describe("Windows native doctor failure diagnostics", () => { + beforeEach(() => { + vi.spyOn(nativeLoader, "systemWindowsNativeAuthority").mockReturnValue({ + available: false, + failureStage: "artifact", + reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, + }); + }); + afterEach(() => vi.restoreAllMocks()); + + it.each([ + ["host", "unsupported_host", "unsupported_host", "Windows x64 host"], + [ + "artifact", + "runtime_missing", + "missing_dependency", + "bundled native addon", + ], + [ + "self-test", + "runtime_unavailable", + "runtime_failure", + "preserved native self-test error", + ], + [ + "inspection", + "runtime_unavailable", + "runtime_failure", + "every required security control", + ], + ] as const)( + "projects %s failure without blaming Ghidra", + async (failureStage, code, classification, remediation) => { + const reason = + "Windows native failure: remove_owned_directory: directory is not empty (145)"; + const load = vi.mocked(nativeLoader.systemWindowsNativeAuthority); + load.mockReturnValue({ available: false, failureStage, reason }); + const result = await runDoctor( + undefined, + { + ...doctorHost(inspection({ installDir: INSTALL, platform: "win32" })), + platform: "win32", + linuxDistribution: () => Promise.resolve(undefined), + }, + { providers: ["ghidra"] }, + ); + expect(load).toHaveBeenCalledTimes(1); + expect(result.healthy).toBe(false); + expect(result.providerInspections?.[0]).toMatchObject({ + available: false, + registrationEnvironment: { GHIDRA_INSTALL_DIR: INSTALL }, + checks: expect.arrayContaining([ + expect.objectContaining({ + name: "native_authority", + ok: false, + code, + detail: reason, + }), + ]), + }); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: "ghidra-native_authority", + classification, + detail: reason, + remediation: expect.stringContaining(remediation), + }), + ); + }, + ); + + it("keeps one failure observation even when the next query would differ", () => { + const load = vi.mocked(nativeLoader.systemWindowsNativeAuthority); + load.mockReturnValueOnce({ + available: false, + failureStage: "self-test", + retryable: true, + reason: "cleanup failed (145)", + }); + const ghidra = inspection({ installDir: INSTALL, platform: "win32" }); + const failed = projectGhidraDoctorInspection(ghidra); + expect(load).toHaveBeenCalledTimes(1); + expect(failed.checks).toContainEqual( + expect.objectContaining({ + name: "native_authority", + ok: false, + code: "runtime_unavailable", + detail: "cleanup failed (145)", + }), + ); + const later = projectGhidraDoctorInspection(ghidra); + expect(load).toHaveBeenCalledTimes(2); + expect(later.checks).toContainEqual( + expect.objectContaining({ + name: "native_authority", + code: "runtime_missing", detail: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, }), ); diff --git a/src/ghidra/GhidraDoctor.ts b/src/ghidra/GhidraDoctor.ts index 5e74a9199..1590ea14b 100644 --- a/src/ghidra/GhidraDoctor.ts +++ b/src/ghidra/GhidraDoctor.ts @@ -2,10 +2,7 @@ import type { DoctorProviderCheck, DoctorProviderInspection, } from "../application/Doctor.js"; -import { - hasWindowsNativeAuthority, - windowsNativeAuthorityUnavailableReason, -} from "../process/WindowsAuthority.js"; +import { inspectWindowsNativeReadiness } from "../process/WindowsAuthority.js"; import { inspectGhidraInstallation, type GhidraInstallationCheck, @@ -16,10 +13,11 @@ import { export const projectGhidraDoctorInspection = ( inspection: GhidraInstallationInspection, ): DoctorProviderInspection => { - const windowsAuthorityUnavailable = - inspection.status === "available" && - inspection.platform === "win32" && - !hasWindowsNativeAuthority(inspection.platform); + const readiness = + inspection.status === "available" && inspection.platform === "win32" + ? inspectWindowsNativeReadiness(inspection.platform) + : undefined; + const windowsAuthorityUnavailable = readiness?.available === false; return { id: "ghidra", configured: inspection.installDir !== null, @@ -37,19 +35,22 @@ export const projectGhidraDoctorInspection = ( ...selectedChecks(inspection).map((check) => projectCheck(check, inspection.installDir !== null), ), - ...(inspection.status === "available" && inspection.platform === "win32" + ...(readiness !== undefined ? [ { name: "native_authority", ok: !windowsAuthorityUnavailable, - code: windowsAuthorityUnavailable ? "unsupported_host" : null, - detail: windowsAuthorityUnavailable - ? windowsNativeAuthorityUnavailableReason(inspection.platform) + code: !readiness.available ? readiness.code : null, + detail: !readiness.available + ? readiness.reason : "Verified Windows native Job Object, private DACL, and reparse-safe handle controls; local NTFS only.", - remediation: windowsAuthorityUnavailable - ? "Use the matching Windows x64 REA package with its bundled native addon and a writable local NTFS temporary directory. The native failure above identifies the failed constraint; Ghidra installation or registration changes cannot enable a missing native backend." - : null, - classification: "unsupported_host" as const, + remediation: !readiness.available ? readiness.remediation : null, + classification: + !readiness.available && readiness.code === "runtime_unavailable" + ? ("runtime_failure" as const) + : !readiness.available && readiness.code === "runtime_missing" + ? ("missing_dependency" as const) + : ("unsupported_host" as const), }, ] : []), diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index e7e5d8e9d..f40cc1ca3 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -43,7 +43,152 @@ const parseFunctionValueFixture = ( return parsed.ok ? { ok: true as const, value: parsed.value.value } : parsed; }; +describe("Ghidra unresolved call locations", () => { + it("rejects a noncanonical unresolved call site rather than retaining unverifiable location evidence", () => { + expect( + parseGhidraFunctionResult("analyze_function", { + value: { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + unresolved_calls: [ + { address: "not-an-address", reason: "No call destination." }, + ], + }, + limitations: [], + relationship_coverage: { + status: "partial", + limitations: ["An unresolved call remains."], + }, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); +}); + describe("Ghidra function-analysis result values", () => { + it("preserves unresolved Swift jump omissions", () => { + const dossier = { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + swift_string_literals: [], + swift_string_literal_omissions: [ + { + source_address: "0x401004", + representation: "small", + address: null, + byte_length: 5, + reason: "unresolved_jump", + registers: ["x0", "x1"], + words: ["0x5000000000000005", "0x0000000000000000"], + }, + ], + }; + expect( + parseGhidraFunctionResult("analyze_function", { + value: dossier, + limitations: [], + }), + ).toMatchObject({ + ok: true, + value: { + value: { + swift_string_literal_omissions: + dossier.swift_string_literal_omissions, + }, + }, + }); + }); + + it("retains unavailable pseudocode instead of treating its empty representation as observed source", () => { + const availability = { + pseudocode: { + available: false, + reason: "Ghidra does not decompile external functions.", + }, + comments: { available: true }, + }; + const parsed = parseGhidraFunctionResult("analyze_function", { + value: { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + pseudocode: "", + facet_availability: availability, + }, + limitations: [], + }); + expect(parsed).toMatchObject({ + ok: true, + value: { value: { pseudocode: "", facet_availability: availability } }, + }); + }); + it.each([ + { status: "complete", diagnostics: ["Resolved static references only."] }, + { + status: "partial", + limitations: ["Only the first instruction range was examined."], + }, + { status: "unknown", reason: "Relationship inventory was unavailable." }, + { + status: "partial", + limitations: ["Missing callers."], + limited_directions: ["callers"], + }, + { + status: "unknown", + reason: "Missing callees.", + limited_directions: ["callees"], + }, + ])("preserves observed $status relationship coverage", (coverage) => { + const parsed = parseGhidraFunctionResult("analyze_function", { + value: ghidraFunctionDossier(), + limitations: [], + relationship_coverage: coverage, + }); + expect(parsed).toMatchObject({ + ok: true, + value: { relationshipCoverage: coverage }, + }); + }); + + it("preserves omitted relationship coverage for legacy observations", () => { + const parsed = parseGhidraFunctionResult("analyze_function", { + value: ghidraFunctionDossier(), + limitations: [], + }); + if (!parsed.ok) throw parsed.error; + expect(parsed.value).not.toHaveProperty("relationshipCoverage"); + }); +}); + +describe("Ghidra relationship metadata validation", () => { + it.each([ + { status: "partial", limitations: [] }, + { status: "unknown", reason: "" }, + { status: "complete", limitations: ["Truncated."] }, + { status: "partial", limitations: ["Truncated."], limited_directions: [] }, + { + status: "partial", + limitations: ["Truncated."], + limited_directions: ["callers", "callers"], + }, + { status: "unknown", reason: "Missing.", limited_directions: ["incoming"] }, + { status: "complete", limited_directions: ["callers"] }, + ])("rejects malformed relationship coverage %j", (coverage) => { + expect( + parseGhidraFunctionResult("analyze_function", { + value: ghidraFunctionDossier(), + limitations: [], + relationship_coverage: coverage, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); + + it("rejects relationship metadata on unrelated function operations", () => { + expect( + parseGhidraFunctionResult("procedure_assembly", { + value: "ret", + limitations: [], + relationship_coverage: { status: "complete" }, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); + it("preserves file-offset resolution in Evidence locations and rejects inexact offsets", () => { const selection = { file_offset: 0x210, diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index a6e62c93c..d038ac82e 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -27,6 +27,10 @@ import { import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js"; import { nativeValueFlowSchema } from "../domain/native/nativeValueFlow.js"; import type { EvidenceLocation } from "../domain/evidence.js"; +import { + relationshipCoverageSchema, + type RelationshipCoverage, +} from "../domain/relationshipCoverage.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import { @@ -307,6 +311,7 @@ const ghidraFunctionDossier = functionDossierSchema ...value.comments.map(({ address }) => address), ...value.callers.map(({ address }) => address), ...value.callees.map(({ address }) => address), + ...(value.unresolved_calls ?? []).map(({ address }) => address), ...value.incoming_references.flatMap(referenceAddresses), ...value.outgoing_references.flatMap(referenceAddresses), ...value.referenced_strings.flatMap(({ address, source_address }) => [ @@ -419,6 +424,7 @@ const functionBoundaryObservationOperations: ReadonlySet["selection"]; @@ -450,13 +458,17 @@ export const parseGhidraFunctionResult = ( const envelope = functionObservationEnvelope.safeParse(value); if ( !envelope.success || + (operation !== "analyze_function" && + operation !== "procedure_callers" && + operation !== "procedure_callees" && + envelope.data.relationship_coverage !== undefined) || (operation !== "procedure_pseudo_code" && envelope.data.selection !== undefined) ) return err( new AnalysisOutputError( operation, - "Ghidra bridge omitted the function observation envelope", + "Ghidra bridge returned an invalid function observation envelope for this operation", envelope.success ? undefined : { cause: envelope.error }, ), ); @@ -469,6 +481,9 @@ export const parseGhidraFunctionResult = ( ? ok({ value: jsonValueSchema.parse(parsed.data), limitations: observation.limitations, + ...(observation.relationship_coverage === undefined + ? {} + : { relationshipCoverage: observation.relationship_coverage }), ...(observation.selection === undefined ? {} : { diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index 95c1967b7..f30f6ef7a 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -252,7 +252,9 @@ export interface GhidraJavaObservation { readonly version: string; readonly major: number; readonly home: string; - readonly bits: number; + readonly bits: number | null; + /** Present when Java reported a bitness value that could not be parsed exactly. */ + readonly unparsedBits?: string; readonly runtime: "jdk" | "jre"; readonly vmName?: string; /** @@ -803,12 +805,23 @@ const javaCheck = ( ghidraJavaMajorAccepted(observation.major, requirement) && observation.bits === 64 && observation.runtime === "jdk"; + const bitness = + observation === undefined + ? undefined + : observation.bits === null + ? `bitness unknown${observation.unparsedBits === undefined ? "" : ` (sun.arch.data.model = ${JSON.stringify(observation.unparsedBits)})`}` + : `${String(observation.bits)}-bit`; + const knownMismatch = + observation !== undefined && + (!ghidraJavaMajorAccepted(observation.major, requirement) || + (observation.bits !== null && observation.bits !== 64) || + observation.runtime !== "jdk"); const detail = observation === undefined ? requirement.unresolved === null ? command : `${command}; ${requirement.unresolved}` - : `${observation.version}; ${String(observation.bits)}-bit; ${observation.runtime === "jdk" ? "JDK" : "runtime only"}; ${observation.home}; accepted ${range}`; + : `${observation.version}; ${bitness}; ${observation.runtime === "jdk" ? "JDK" : "runtime only"}; ${observation.home}; accepted ${range}`; return installationCheck({ name: "java", passed: supported, @@ -817,7 +830,9 @@ const javaCheck = ( ? "version_unresolved" : observation === undefined ? "runtime_missing" - : "unsupported_version", + : knownMismatch + ? "unsupported_version" + : "version_unresolved", detail, remediation: javaRemediation(requirement, configuredHome, range), }); @@ -905,11 +920,22 @@ const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ const vmName = javaProperty(output, "java.vm.name"); const osArch = javaProperty(output, "os.arch"); const osName = javaProperty(output, "os.name"); - const bits = Number.parseInt( - javaProperty(output, "sun.arch.data.model") ?? "0", - 10, - ); - if (version === null || home === null) return undefined; + const reportedBits = javaProperty(output, "sun.arch.data.model"); + const parsedBits = + reportedBits !== null && /^\d+$/.test(reportedBits) + ? Number(reportedBits) + : null; + const bits = + parsedBits !== null && Number.isSafeInteger(parsedBits) + ? parsedBits + : null; + if ( + version === null || + version.length === 0 || + home === null || + home.length === 0 + ) + return undefined; return { version, major: javaMajor(version), @@ -918,6 +944,9 @@ const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ ...(osArch === null ? {} : { osArch }), ...(osName === null ? {} : { osName }), bits, + ...(reportedBits === null || bits !== null + ? {} + : { unparsedBits: reportedBits }), runtime: executablePath( (process.platform === "win32" ? win32 : posix).join( home, @@ -944,7 +973,7 @@ const executablePath = (path: string): boolean => { const javaProperty = (output: string, name: string): string | null => new RegExp( - `^\\s*${name.replaceAll(".", "\\.")}\\s*=\\s*(.+?)\\s*$`, + `^[ \\t]*${name.replaceAll(".", "\\.")}[ \\t]*=[ \\t]*([^\\r\\n]*?)[ \\t]*\\r?$`, "mu", ).exec(output)?.[1] ?? null; diff --git a/src/ghidra/GhidraJavaLaunch.ts b/src/ghidra/GhidraJavaLaunch.ts index 317c0939f..598a152c2 100644 --- a/src/ghidra/GhidraJavaLaunch.ts +++ b/src/ghidra/GhidraJavaLaunch.ts @@ -1,6 +1,39 @@ import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { dirname, join } from "node:path"; -import { execFileOutput } from "../process/ExecFileOutput.js"; +import { + execFileOutput, + execFileOutputFailure, + type ExecFileOutputFailure, +} from "../process/ExecFileOutput.js"; + +export const GHIDRA_LAUNCH_SUPPORT_CAPTURE_BYTES = 64 * 1024; +type LaunchSupportPhase = "-envvars" | "-vmargs"; +type LaunchSupportOutcome = Omit< + ExecFileOutputFailure, + "killed" | "outputTruncated" +> & { + readonly killed: boolean | null; + readonly outputTruncated: boolean | null; +}; + +export class GhidraLaunchSupportError extends Error { + readonly outcome: LaunchSupportOutcome | undefined; + + constructor(phase: LaunchSupportPhase, cause: unknown) { + super(`Ghidra LaunchSupport ${phase} failed`, { cause }); + this.name = "GhidraLaunchSupportError"; + const outcome = execFileOutputFailure(cause); + this.outcome = + outcome === undefined + ? undefined + : { + ...outcome, + killed: outcome.code === "ABORT_ERR" ? null : outcome.killed, + outputTruncated: + outcome.code === "ABORT_ERR" ? null : outcome.outputTruncated, + }; + } +} /** JVM option environments remove quotes, including quotes inside an option. */ const jvmEnvironmentArguments = (value: string): readonly string[] => { @@ -85,17 +118,20 @@ export const ghidraJavaLaunch = async (options: { const environment = { ...snapshotEnvironment(options.environment, options.platform), }; - const configuredEnvironment = await execFileOutput( - command, - [...supportArguments, "-envvars"], - { - env: environment, - // Wait for the helper's exit on abort so a cancelled or timed-out - // startup never leaves the probe running without an owning process. - stopSignal: "SIGTERM", - ...(options.signal === undefined ? {} : { signal: options.signal }), - }, - ); + const supportOutput = async (phase: LaunchSupportPhase) => { + try { + return await execFileOutput(command, [...supportArguments, phase], { + env: environment, + maxBuffer: GHIDRA_LAUNCH_SUPPORT_CAPTURE_BYTES, + // Wait for the helper's exit on abort so no probe outlives its owner. + stopSignal: "SIGTERM", + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + } catch (cause: unknown) { + throw new GhidraLaunchSupportError(phase, cause); + } + }; + const configuredEnvironment = await supportOutput("-envvars"); options.onStderr("-envvars", configuredEnvironment.stderr); for (const line of configuredEnvironment.stdout.split(/\r?\n/u)) { if (line === "") continue; @@ -121,17 +157,7 @@ export const ghidraJavaLaunch = async (options: { ] as const) { rejectJvmPlatformOverrides(jvmEnvironmentArguments(environment[key] ?? "")); } - const configuredArguments = await execFileOutput( - command, - [...supportArguments, "-vmargs"], - { - env: environment, - // Wait for the helper's exit on abort so a cancelled or timed-out - // startup never leaves the probe running without an owning process. - stopSignal: "SIGTERM", - ...(options.signal === undefined ? {} : { signal: options.signal }), - }, - ); + const configuredArguments = await supportOutput("-vmargs"); options.onStderr("-vmargs", configuredArguments.stderr); const vmArguments = configuredArguments.stdout .split(/\r?\n/u) diff --git a/src/ghidra/GhidraLanguageCatalog.ts b/src/ghidra/GhidraLanguageCatalog.ts index 6373f187f..1cde8769f 100644 --- a/src/ghidra/GhidraLanguageCatalog.ts +++ b/src/ghidra/GhidraLanguageCatalog.ts @@ -1,9 +1,12 @@ -import { readdir, readFile } from "node:fs/promises"; +import { open, readdir } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; import { join } from "node:path"; import { DOMParser, onWarningStopParsing } from "@xmldom/xmldom"; import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; import { ConfigurationError } from "../domain/configurationErrors.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; /** One language definition and the compiler specs it declares. */ export interface GhidraLanguageDefinition { @@ -16,6 +19,62 @@ const MAX_LDEFS_BYTES = 4 * 1024 * 1024; // Processor modules ship under Processors; installed extensions may add more. const LANGUAGE_ROOTS = ["Ghidra/Processors", "Ghidra/Extensions"] as const; +interface RetainedLanguageFile { + readonly path: string; + readonly owner: OwnedFileHandle; +} + +/** Language definition handles whose close is unconfirmed but retryable. */ +const pendingLanguageFileCleanup = new Set(); + +/** + * Cleanup of one or more language definition handles is unconfirmed. Any + * read failure that preceded it is kept as the primary failure. + */ +export class GhidraLanguageCatalogCleanupError extends Error { + constructor( + readonly paths: readonly string[], + readonly causes: readonly unknown[], + readonly readFailure?: unknown, + ) { + super( + `Language definition cleanup is unconfirmed: ${paths.join(", ")}`, + readFailure === undefined ? undefined : { cause: readFailure }, + ); + } +} + +/** Close one owner; retain it only while a later close can still succeed. */ +const closeLanguageFile = async ( + file: RetainedLanguageFile, +): Promise => { + try { + await file.owner.close(); + pendingLanguageFileCleanup.delete(file); + return undefined; + } catch (cause: unknown) { + // Descriptor invalidation does not prove that this owner confirmed close. + // Keep even sticky failures in the lifecycle scope so later reads surface + // the unresolved cleanup instead of silently abandoning ownership. + pendingLanguageFileCleanup.add(file); + return cause; + } +}; + +/** Retry every retained owner before reporting the failures together. */ +const retryLanguageFileCleanup = async (): Promise => { + const failed: { readonly path: string; readonly cause: unknown }[] = []; + for (const file of pendingLanguageFileCleanup) { + const cause = await closeLanguageFile(file); + if (cause !== undefined) failed.push({ path: file.path, cause }); + } + if (failed.length > 0) + throw new GhidraLanguageCatalogCleanupError( + failed.map(({ path }) => path), + failed.map(({ cause }) => cause), + ); +}; + /** Parse the `` elements of one `.ldefs` document. */ export const parseGhidraLanguageDefinitions = ( text: string, @@ -59,10 +118,40 @@ const languageDirectories = async (installDir: string): Promise => { return directories; }; +/** Read at most the catalog limit; undefined marks an oversized file. */ +const readLanguageDefinitionFile = async ( + path: string, +): Promise => { + await retryLanguageFileCleanup(); + const file = { path, owner: new OwnedFileHandle(await open(path, "r")) }; + let read: + | { readonly ok: true; readonly bytes: Buffer | undefined } + | { readonly ok: false; readonly cause: unknown }; + try { + read = { + ok: true, + bytes: await readBoundedFileBytes(file.owner.handle, MAX_LDEFS_BYTES), + }; + } catch (cause: unknown) { + read = { ok: false, cause }; + } + const cleanup = await closeLanguageFile(file); + if (cleanup !== undefined) + throw new GhidraLanguageCatalogCleanupError( + [path], + [cleanup], + read.ok ? undefined : read.cause, + ); + if (!read.ok) throw read.cause; + return read.bytes; +}; + /** Read every language definition shipped by one Ghidra installation. */ export const readGhidraLanguageCatalog = async ( installDir: string, ): Promise => { + // Surface retained cleanup even when this catalog has no definitions to read. + await retryLanguageFileCleanup(); const definitions: GhidraLanguageDefinition[] = []; for (const directory of await languageDirectories(installDir)) { let entries: string[]; @@ -73,8 +162,8 @@ export const readGhidraLanguageCatalog = async ( } for (const entry of entries.sort()) { if (!entry.endsWith(".ldefs")) continue; - const bytes = await readFile(join(directory, entry)); - if (bytes.length > MAX_LDEFS_BYTES) continue; + const bytes = await readLanguageDefinitionFile(join(directory, entry)); + if (bytes === undefined) continue; // XML 1.0 section 4.3.3: a leading UTF-8 BOM is an encoding signature, // not document content; TextDecoder consumes it, Buffer#toString does not. definitions.push( diff --git a/src/ghidra/GhidraLaunchFailure.test.ts b/src/ghidra/GhidraLaunchFailure.test.ts index 246ec82d0..d25cd9932 100644 --- a/src/ghidra/GhidraLaunchFailure.test.ts +++ b/src/ghidra/GhidraLaunchFailure.test.ts @@ -56,7 +56,8 @@ const launchOutcomeOf = (error: Error): Record | undefined => { string, unknown >; - const outcome = cause["launch_outcome"] as Record; + const nested = cause["cause"] as Record | undefined; + const outcome = nested?.["command_outcome"] as Record; return outcome === undefined ? undefined : outcome; }; @@ -78,12 +79,16 @@ describe("Ghidra launch failure diagnostics", () => { launched.error instanceof GhidraLaunchError ? launched.error.capturedOutput : undefined; - expect(captured).toBeDefined(); - expect(captured?.exit_code).toBe(3); - expect(captured?.signal).toBeNull(); - expect(captured?.stdout).toBe(""); - expect(captured?.stderr).toContain("jvm probe exploded"); - expect(captured?.truncated).toBe(false); + expect(captured).toBeUndefined(); + const outcome = launchOutcomeOf(launched.error); + expect(outcome).toMatchObject({ + killed: false, + outputTruncated: false, + stdoutTruncated: false, + stderrTruncated: false, + stdout: "", + }); + expect(outcome?.["stderr"]).toContain("jvm probe exploded"); } finally { await rm(javaHome, { recursive: true, force: true }); await rm(runtimeRoot, { recursive: true, force: true }); @@ -103,10 +108,11 @@ describe("Ghidra launch failure diagnostics", () => { if (launched.ok) throw new Error("expected the launch to fail"); const outcome = launchOutcomeOf(launched.error); expect(outcome).toBeDefined(); - expect(outcome?.["exit_code"]).toBe(3); - expect(outcome?.["signal"]).toBeNull(); + expect(outcome?.["killed"]).toBe(false); + expect(outcome?.["outputTruncated"]).toBe(false); + expect(outcome?.["stderrTruncated"]).toBe(false); expect(outcome?.["stderr"]).toContain("jvm probe exploded"); - expect(outcome?.["truncated"]).toBe(false); + expect(outcome?.["outputTruncated"]).toBe(false); } finally { await rm(javaHome, { recursive: true, force: true }); await rm(runtimeRoot, { recursive: true, force: true }); @@ -126,9 +132,9 @@ describe("Ghidra launch failure diagnostics", () => { expect(launched.ok).toBe(false); if (launched.ok) throw new Error("expected the launch to fail"); const outcome = launchOutcomeOf(launched.error); - expect(outcome?.["truncated"]).toBe(true); - expect(outcome?.["exit_code"]).toBe(1); - expect(outcome?.["stderr_bytes"]).toBe(oversized.length); + expect(outcome?.["stderrTruncated"]).toBe(true); + expect(outcome?.["outputTruncated"]).toBe(false); + expect(outcome?.["stderrBytes"]).toBe(oversized.length); expect((outcome?.["stderr"] as string).length).toBeLessThanOrEqual( EXPECTED_OUTPUT_BOUND_BYTES, ); diff --git a/src/ghidra/GhidraMipsProfile.ts b/src/ghidra/GhidraMipsProfile.ts index 76a753ccb..51ac546f2 100644 --- a/src/ghidra/GhidraMipsProfile.ts +++ b/src/ghidra/GhidraMipsProfile.ts @@ -137,6 +137,11 @@ export const ghidraMipsUnsupportedReason = ( if (load.symbolRelativeCount !== 0) return "Symbol-bearing R_MIPS_REL32 is unsupported: Ghidra normalizes MIPS16 function addresses and can lose the source function-pointer mode tag. Symbol-index-zero REL32 is verified separately."; } + if ((metadata.limitations?.length ?? 0) > 0) + return [ + "MIPS ABI declaration inspection is incomplete; unsupported attributes or conflicting declaration sources may remain unknown.", + ...(metadata.limitations ?? []), + ].join(" "); const abi = metadata.abiFlags; if (shared && abi === null && (metadata.limitations?.length ?? 0) === 0) return null; // Inspected absence leaves FP semantics unknown, never inferred. diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index c69c83bd9..9d4892fc8 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -8,7 +8,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { describe, expect, it, onTestFinished, vi } from "vitest"; +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; +import * as nativeLoader from "../windows/WindowsNativeLoader.js"; +import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; import { fixtureDosLoadImage } from "./GhidraLoadImage.fixture.js"; import { fixtureRawBinaryLoadImage } from "./GhidraRawBinaryLoadImage.fixture.js"; import { @@ -30,13 +32,12 @@ import { err, ok } from "../domain/result.js"; import type { Result } from "../domain/result.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; import { silentLogger } from "../logger.js"; -import * as WindowsNativeLoader from "../windows/WindowsNativeLoader.js"; -import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { ProviderStartupDeadline } from "../process/ProviderDeadline.js"; const INSTALL = "/opt/ghidra_12.1.4_PUBLIC"; +afterEach(() => vi.restoreAllMocks()); const installationHost = ( platform: NodeJS.Platform = "linux", ): GhidraInstallationHost => ({ @@ -360,18 +361,20 @@ describe("Ghidra platform support", () => { availabilityCode: "unsupported_host", }); }); +}); +describe("Ghidra Windows platform support", () => { it("keeps Windows P0 unavailable until native isolation authority exists", () => { const native = vi - .spyOn(WindowsNativeLoader, "systemWindowsNativeAuthority") + .spyOn(nativeLoader, "systemWindowsNativeAuthority") .mockReturnValue({ available: false, + failureStage: "artifact", reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, }); onTestFinished(() => native.mockRestore()); const ghidra = provider(installationHost("win32")); const nativeApplication = peTarget("x86_64"); - for (const architecture of ["x86", "x86_64"] as const) expect(ghidra.inspectTargetSupport(peTarget(architecture))).toMatchObject( { @@ -386,11 +389,15 @@ describe("Ghidra platform support", () => { ); expect(ghidra.inspectAvailability()).toMatchObject({ status: "unavailable", - code: "unsupported_host", + code: "runtime_missing", reason: expect.stringContaining( "Windows native controls are unavailable", ), diagnostics: { + windows_native_failure: { + stage: "artifact", + remediation: expect.any(String), + }, windows_security: { job_object_process_ownership: expect.objectContaining({ available: false, @@ -412,7 +419,7 @@ describe("Ghidra platform support", () => { expect.objectContaining({ operation: "list_procedures", available: false, - availabilityCode: "unsupported_host", + availabilityCode: "runtime_missing", limitations: expect.arrayContaining([ expect.stringContaining( "matching packaged Windows x64 native addon", @@ -454,6 +461,125 @@ describe("Ghidra platform support", () => { }); }); +describe("Windows Ghidra native failure diagnostics", () => { + it.each([ + ["artifact", "runtime_missing"], + ["self-test", "runtime_unavailable"], + ["inspection", "runtime_unavailable"], + ] as const)( + "keeps Windows P0 unavailable after a %s failure", + async (failureStage, code) => { + const load = vi + .spyOn(nativeLoader, "systemWindowsNativeAuthority") + .mockReturnValue({ + available: false, + failureStage, + reason: "Windows native controls are unavailable: missing manifest", + }); + const ghidra = provider(installationHost("win32")); + const nativeApplication = peTarget("x86_64"); + + for (const architecture of ["x86", "x86_64"] as const) + expect( + ghidra.inspectTargetSupport(peTarget(architecture)), + ).toMatchObject({ + status: "supported", + diagnostics: { + host_platform: "win32", + architecture, + executable_role: "application", + managed: false, + }, + }); + expect(ghidra.inspectAvailability()).toMatchObject({ + status: "unavailable", + code, + reason: expect.stringContaining( + "Windows native controls are unavailable", + ), + diagnostics: { + windows_native_failure: { + stage: failureStage, + remediation: expect.any(String), + }, + windows_security: { + job_object_process_ownership: expect.objectContaining({ + available: false, + proof: "not-proven", + }), + private_runtime_dacl: expect.objectContaining({ + available: false, + proof: "not-proven", + }), + reparse_safe_path_admission: expect.objectContaining({ + available: false, + proof: "not-proven", + }), + }, + }, + }); + expect(load).toHaveBeenCalledTimes(1); + expect(ghidra.capabilities()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + operation: "list_procedures", + available: false, + availabilityCode: code, + limitations: expect.arrayContaining([ + expect.stringContaining( + "matching packaged Windows x64 native addon", + ), + expect.stringContaining( + "private DACLs, and Job Object ownership automatically", + ), + ]), + }), + ]), + ); + expect( + ghidra.inspectTargetSupport({ + ...nativeApplication, + executableRole: "non-executable", + }), + ).toMatchObject({ + status: "unsupported", + code: "target_role_unsupported", + }); + expect( + ghidra.inspectTargetSupport({ ...nativeApplication, managed: true }), + ).toMatchObject({ + status: "unsupported", + code: "managed_target_unsupported", + }); + expect( + ghidra.inspectTargetSupport(executableTarget("elf", "x86_64")), + ).toMatchObject({ + status: "unsupported", + code: "target_format_unsupported", + }); + expect( + ghidra.inspectTargetSupport(executableTarget("pe", "arm64")), + ).toMatchObject({ + status: "unsupported", + code: "architecture_unsupported", + }); + const client = ghidra.createClient(nativeApplication); + const health = await client.execute("health", {}); + if (health.ok) + throw new Error( + "Failed native admission must not create a ready client", + ); + expect(projectAnalysisError(health.error)).toMatchObject({ + code: "capability_unavailable", + details: { + reason: "Windows native controls are unavailable: missing manifest", + }, + }); + expect(load).toHaveBeenCalledTimes(3); + }, + ); +}); + describe("Ghidra Windows PE fixture admission", () => { it("admits generated PE32 and PE32+ DLLs and applications while preserving native-only Windows admission", async () => { const root = fileURLToPath(new URL("../../", import.meta.url)); @@ -633,7 +759,7 @@ describe("Ghidra client projection", () => { expect(resolved.value.profile.parameters).toMatchObject({ language_id: "auto-from-header", compiler_spec_id: "auto-default", - call_relationship_coverage: "resolved-call-reference-diagnostics-v1", + call_relationship_coverage: "function-body-call-coverage-v5", function_dossier_coverage: "observed-facets-and-call-relationships-v1", }); @@ -870,7 +996,16 @@ describe("Ghidra relationship coverage", () => { Promise.resolve( ok( operation === "procedure_callees" - ? { value: ["0x2000"], limitations: [] } + ? { + value: ["0x2000"], + limitations: [], + relationship_coverage: { + status: "complete", + diagnostics: [ + "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals.", + ], + }, + } : { value: [], limitations: [] }, ), ), @@ -2228,3 +2363,104 @@ describe("Ghidra disabled analyzer options", () => { } }); }); + +describe("Ghidra queries after the owned provider is lost", () => { + const lostProviderSession = async ( + state: "exited" | "unreachable" | "idle", + failureKind: "process" | "protocol" | "remote" = "process", + answeredBeforeLoss = true, + ) => { + let calls = 0; + const ghidra = provider(installationHost(), () => ({ + start: async () => + err(new GhidraSessionError("start", "startup is not exercised")), + callTool: async () => { + calls += 1; + if (answeredBeforeLoss && calls === 1) return ok(null); + return err( + new GhidraSessionError( + failureKind, + "Ghidra bridge socket closed", + { failure_kind: failureKind }, + failureKind === "remote" ? { remoteCode: "internal" } : {}, + ), + ); + }, + close: async () => ok(null), + operationHealthSnapshot: () => + state === "idle" + ? undefined + : { + state, + stage: null, + retryAction: "restart_provider", + exitCode: state === "exited" ? 137 : null, + requests: [], + }, + })); + const target = executableTarget("elf", "x86_64"); + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("Expected Ghidra analysis profile"); + const client = ghidra.createClient(target, profile.value.profile); + if (answeredBeforeLoss) await client.execute("list_documents", {}); + return client; + }; + + it("tells the caller to reopen the target when Ghidra exited", async () => { + const client = await lostProviderSession("exited"); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "execution_failure", + retryable: false, + details: { provider_id: "ghidra", operation: "list_documents" }, + }); + expect(projected.message).toContain("exited with code 137"); + expect(projected.message).toContain("Call open_binary"); + expect(projected.remediation.action).toContain("Call open_binary"); + expect(projected.remediation.action).not.toContain("rea doctor"); + }); + + it("tells the caller to reopen the target when the bridge became unreachable", async () => { + const client = await lostProviderSession("unreachable", "protocol"); + const result = await client.execute("list_procedures", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + const projected = projectAnalysisError(result.error); + expect(projected.message).toContain("became unreachable"); + expect(projected.remediation.action).toContain("Call open_binary"); + }); + + it("keeps installation guidance when Ghidra dies before it ever answered", async () => { + const client = await lostProviderSession("exited", "process", false); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + const projected = projectAnalysisError(result.error); + expect(projected.message).not.toContain("Call open_binary"); + expect(projected.remediation.action).toContain("rea doctor"); + }); + + it("keeps the generic failure when the provider is not known to be lost", async () => { + const client = await lostProviderSession("idle"); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(projectAnalysisError(result.error).remediation.action).toContain( + "rea doctor", + ); + }); + + it("does not reinterpret a bridge-reported failure as a lost provider", async () => { + const client = await lostProviderSession("exited", "remote"); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(projectAnalysisError(result.error).message).not.toContain( + "Call open_binary", + ); + }); +}); diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index b0ee8787d..3cee8bace 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -4,6 +4,7 @@ import type { AnalysisClientContext, AnalysisProfileResolutionOptions, AnalysisProviderCandidate, + AnalysisProviderReadinessSnapshot, CapabilityDescriptor, ProviderAvailability, ProviderIdentity, @@ -25,6 +26,7 @@ import { } from "../domain/jsonValue.js"; import { err, ok } from "../domain/result.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import type { Logger } from "pino"; import { GhidraClient } from "./GhidraClient.js"; import { @@ -38,6 +40,7 @@ import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { resolveGhidraAnalysisSeeds } from "./GhidraAnalysisSeeds.js"; import { admitGhidraLanguageOverride, + GhidraLanguageCatalogCleanupError, readGhidraLanguageCatalog, } from "./GhidraLanguageCatalog.js"; import { @@ -55,8 +58,8 @@ import { type GhidraProviderClientFactory, } from "./GhidraProviderClient.js"; import { - windowsNativeAuthorityUnavailableReason, - hasWindowsNativeAuthority, + inspectWindowsNativeReadiness, + type WindowsNativeReadiness, windowsNativeCapabilities, } from "../process/WindowsAuthority.js"; @@ -101,22 +104,43 @@ export class GhidraProvider implements AnalysisProviderCandidate { } inspectAvailability(): ProviderAvailability { + return this.inspectReadinessSnapshot().availability; + } + + inspectReadinessSnapshot(): AnalysisProviderReadinessSnapshot { const installation = this.#inspectInstallation(); + const readiness = + installation.platform === "win32" + ? inspectWindowsNativeReadiness(installation.platform) + : undefined; + return { + availability: this.#availability(installation, readiness), + capabilities: + readiness === undefined + ? CAPABILITIES + : windowsP0Capabilities(readiness), + }; + } + + #availability( + installation: GhidraInstallationInspection, + readiness: WindowsNativeReadiness | undefined, + ): ProviderAvailability { const diagnostics = ghidraInstallationDiagnostics(installation); - if ( - installation.status === "available" && - installation.platform === "win32" && - !hasWindowsNativeAuthority(installation.platform) - ) + if (installation.status === "available" && readiness?.available === false) return { status: "unavailable", - code: "unsupported_host", - reason: windowsNativeAuthorityUnavailableReason(installation.platform), + code: readiness.code, + reason: readiness.reason, diagnostics: { ...diagnostics, windows_security: jsonObjectSchema.parse( - windowsNativeCapabilities(installation.platform), + windowsNativeCapabilities(installation.platform, readiness), ), + windows_native_failure: { + stage: readiness.failureStage, + remediation: readiness.remediation, + }, }, }; return installation.status === "available" @@ -217,6 +241,22 @@ export class GhidraProvider implements AnalysisProviderCandidate { try { catalog = await readGhidraLanguageCatalog(installation.installDir); } catch (cause: unknown) { + if (cause instanceof GhidraLanguageCatalogCleanupError) + return err( + new ProviderCleanupError( + "ghidra", + cause.paths, + { + reason: cause.message, + paths: [...cause.paths], + close_failures: cause.causes.map(providerFailureText), + ...(cause.readFailure === undefined + ? {} + : { read_failure: providerFailureText(cause.readFailure) }), + }, + { operation: "resolve_analysis_profile", cause }, + ), + ); return err( new ProviderAdapterError("ghidra", "resolve_analysis_profile", { cause, @@ -302,6 +342,9 @@ export class GhidraProvider implements AnalysisProviderCandidate { } } +const providerFailureText = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + const inspectWindowsP0TargetSupport = ( target: BinaryTarget, diagnostics: Readonly>, diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index f4bff8d9c..6272802d3 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -2,10 +2,9 @@ import type { CapabilityDescriptor, ProviderIdentity, } from "../application/AnalysisProvider.js"; -import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; import { - hasWindowsNativeAuthority, - windowsNativeAuthorityUnavailableReason, + inspectWindowsNativeReadiness, + type WindowsNativeReadiness, } from "../process/WindowsAuthority.js"; import { GHIDRA_FUNCTION_OPERATIONS } from "./GhidraFunctionValues.js"; import { GHIDRA_INVENTORY_OPERATIONS } from "./GhidraInventoryValues.js"; @@ -44,22 +43,6 @@ const stringInventoryLimitation = const searchLimitation = "Regex mode uses Java Pattern semantics. Matcher or compiler stack exhaustion returns a resource constraint without closing the session; literal mode avoids regex recursion and returns complete matching values."; -const resolvedCallReferenceLimitation = - "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals."; - -/** Relationship-only caveats belong to each observed caller/callee result. */ -export const relationshipCoverageFor = ( - operation: string, -): RelationshipCoverage | undefined => - operation === "procedure_callers" || - operation === "procedure_callees" || - operation === "analyze_function" - ? { - status: "complete", - diagnostics: [resolvedCallReferenceLimitation], - } - : undefined; - /** Limitation text for one admitted operation, including the common base. */ export const limitationsFor = (operation: string): readonly string[] => { const common = [ @@ -214,8 +197,9 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( ); /** Capabilities advertised for the experimental Windows x64 P0 boundary. */ -export const windowsP0Capabilities = (): readonly CapabilityDescriptor[] => { - const available = hasWindowsNativeAuthority("win32"); +export const windowsP0Capabilities = ( + readiness: WindowsNativeReadiness = inspectWindowsNativeReadiness("win32"), +): readonly CapabilityDescriptor[] => { return Object.freeze( CAPABILITIES.map((capability): CapabilityDescriptor => { const limitations = Object.freeze([ @@ -230,13 +214,13 @@ export const windowsP0Capabilities = (): readonly CapabilityDescriptor[] => { availabilityCode: "unsupported_host", limitations, }); - return available + return readiness.available ? Object.freeze({ ...capability, limitations }) : Object.freeze({ ...capability, available: false, - reason: windowsNativeAuthorityUnavailableReason("win32"), - availabilityCode: "unsupported_host", + reason: readiness.reason, + availabilityCode: readiness.code, limitations, }); }), diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 9db1b0c5f..80c37c0be 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -34,6 +34,11 @@ import { } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { EvidenceLocation } from "../domain/evidence.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; +import { + providerRetryAction, + type ProviderOperationHealth, +} from "../domain/providerOperationHealth.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -86,7 +91,6 @@ import { healthLimitations, windowsP0Limitations, limitationsFor, - relationshipCoverageFor, } from "./GhidraProviderCapabilities.js"; import type { GhidraSessionError } from "./GhidraSessionError.js"; import { @@ -99,10 +103,7 @@ import { validateGhidraExtensionProfile, ghidraExtensionLimitations, } from "./extensions/GhidraExtensions.js"; -import { - windowsNativeAuthorityUnavailableReason, - hasWindowsNativeAuthority, -} from "../process/WindowsAuthority.js"; +import { inspectWindowsNativeReadiness } from "../process/WindowsAuthority.js"; import { ghidraRawBinaryLanguage } from "./GhidraRawBinaryLanguage.js"; /** Production seam for exercising provider projection without a real process. */ @@ -652,6 +653,11 @@ export const createGhidraProviderClient = (input: { ...(context === undefined ? {} : { runId: context.runId }), logger: logger.child({ layer: "ghidra-bridge" }), }); + // Startup failures retain installation guidance; only a previously running + // provider can be described as lost and reopened. + let providerStarted = false; + const lostProviderHealth = (): ProviderOperationHealth | undefined => + providerStarted ? client.operationHealthSnapshot?.() : undefined; const checkSessionObservations = async ( operation: AnalysisOperation, info: GhidraSessionInfo, @@ -733,8 +739,8 @@ export const createGhidraProviderClient = (input: { rawResult: JsonValue, observationLimitations: readonly string[] = [], locations: readonly EvidenceLocation[] = [], + relationshipCoverage?: RelationshipCoverage, ) => { - const relationshipCoverage = relationshipCoverageFor(operation); return createAnalysisExecution(normalized, committedProfile.provider, { rawResult, locations, @@ -757,6 +763,7 @@ export const createGhidraProviderClient = (input: { operation, called.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const parsed = parseGhidraInventoryResult(operation, called.value); @@ -780,12 +787,14 @@ export const createGhidraProviderClient = (input: { ); if (operation === "health") { const started = await client.start(options?.signal, options?.progress); + if (started.ok) providerStarted = true; if (!started.ok) return err( projectSessionError( operation, started.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const failed = await checkSessionObservations(operation, started.value); @@ -808,12 +817,14 @@ export const createGhidraProviderClient = (input: { softFloat ) { const started = await client.start(options?.signal, options?.progress); + if (started.ok) providerStarted = true; if (!started.ok) return err( projectSessionError( operation, started.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const failed = await checkSessionObservations(operation, started.value); @@ -825,6 +836,7 @@ export const createGhidraProviderClient = (input: { ? {} : { progress: options.progress }), }); + if (called.ok) providerStarted = true; if (operation === "list_documents" || operation === "list_procedures") return inventoryExecution(operation, called); if (!called.ok) @@ -833,17 +845,20 @@ export const createGhidraProviderClient = (input: { operation, called.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); let normalized: JsonValue; let observationLimitations: readonly string[] = []; let observationLocations: readonly EvidenceLocation[] = []; + let relationshipCoverage: RelationshipCoverage | undefined; if (isGhidraFunctionOperation(operation)) { const result = parseGhidraFunctionResult(operation, called.value); if (!result.ok) return result; normalized = result.value.value; observationLimitations = result.value.limitations; observationLocations = result.value.locations ?? []; + relationshipCoverage = result.value.relationshipCoverage; } else { const result = parseGhidraInventoryResult(operation, called.value); if (!result.ok) return result; @@ -863,6 +878,7 @@ export const createGhidraProviderClient = (input: { operation, started.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const derived = await nativeAotOwner.derive( @@ -925,6 +941,7 @@ export const createGhidraProviderClient = (input: { operation, failure, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), nativeAotResult: derived, signal: options?.signal, @@ -939,6 +956,7 @@ export const createGhidraProviderClient = (input: { called.value, observationLimitations, observationLocations, + relationshipCoverage, ), ); }, @@ -997,15 +1015,16 @@ const ghidraClientPrerequisites = ( diagnostics: ghidraInstallationDiagnostics(installation), }), ); - if ( - installation.platform === "win32" && - !hasWindowsNativeAuthority(installation.platform) - ) + const readiness = + installation.platform === "win32" + ? inspectWindowsNativeReadiness(installation.platform) + : undefined; + if (readiness?.available === false) return err( new AnalysisCapabilityUnavailableError( "ghidra", "health", - windowsNativeAuthorityUnavailableReason(installation.platform), + readiness.reason, ), ); if (profile === undefined || profile.provider.id !== "ghidra") @@ -1027,11 +1046,13 @@ const projectSessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, startupTimeoutMs: number, + health?: ProviderOperationHealth, ): AnalysisError => { const primary = projectPrimarySessionError( operation, failure, startupTimeoutMs, + health, ); return failure.cleanupFailure === undefined ? primary @@ -1046,6 +1067,7 @@ const projectPrimarySessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, startupTimeoutMs: number, + health?: ProviderOperationHealth, ): AnalysisError => { const context = { cause: failure, diagnostics: failure.diagnostics }; if (failure.cause instanceof AnalysisAccessDeniedError) @@ -1112,5 +1134,31 @@ const projectPrimarySessionError = ( failure.message, context, ); + const exited = exitedProviderMessage(operation, failure, health); + if (exited !== undefined) + return new ProviderAdapterError("ghidra", operation, { + ...context, + userMessage: exited, + }); return new ProviderAdapterError("ghidra", operation, context); }; + +/** A provider already observed as running can be restarted by reopening. */ +const exitedProviderMessage = ( + operation: AnalysisOperation, + failure: GhidraSessionError, + health: ProviderOperationHealth | undefined, +): string | undefined => { + if (failure.kind !== "process" && failure.kind !== "protocol") + return undefined; + if ( + health === undefined || + providerRetryAction(health.state) !== "restart_provider" + ) + return undefined; + const observed = + health.state === "exited" + ? `exited${health.exitCode === null ? "" : ` with code ${String(health.exitCode)}`}` + : "became unreachable"; + return `The Ghidra provider for this session ${observed} before ${operation} could complete. Call open_binary for the same target to start a new Ghidra session, then retry; session-only annotations from the previous provider are not retained.`; +}; diff --git a/src/ghidra/GhidraSessionError.ts b/src/ghidra/GhidraSessionError.ts index ca3b4f6ac..dedf27bd4 100644 --- a/src/ghidra/GhidraSessionError.ts +++ b/src/ghidra/GhidraSessionError.ts @@ -2,6 +2,7 @@ import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import { GhidraLaunchSupportError } from "./GhidraJavaLaunch.js"; import { GhidraLaunchError, GHIDRA_LAUNCH_COMMAND_OUTPUT_BYTES, @@ -11,6 +12,7 @@ const MAX_CAUSE_DEPTH = 3; const MAX_CAUSE_MESSAGE_BYTES = 2 * 1024; const MAX_CAUSE_FIELD_BYTES = 1024; const MAX_COMPILER_STDERR_BYTES = 8 * 1024; +const MAX_COMMAND_OUTPUT_BYTES = 8 * 1024; /** Stable failure phase for one private Ghidra headless session. */ export type GhidraSessionFailureKind = @@ -59,7 +61,7 @@ const createGhidraSessionError = (failure: { readonly cause?: unknown; readonly timeoutMs?: number; readonly remoteCode?: string; - readonly redact: (value: string) => string; + readonly redact: (value: string, incomplete?: boolean) => string; }): GhidraSessionError => new GhidraSessionError( failure.kind, @@ -90,7 +92,7 @@ const createGhidraSessionError = (failure: { export const bindGhidraSessionFailure = ( diagnostics: () => Readonly>, - redact: (value: string) => string = (value) => value, + redact: (value: string, incomplete?: boolean) => string = (value) => value, ) => ( kind: GhidraSessionFailureKind, @@ -114,7 +116,7 @@ export const bindGhidraSessionFailure = const describeCause = ( cause: unknown, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, depth = 0, ): JsonValue => { if (cause instanceof Error) { @@ -137,7 +139,7 @@ const describeCause = ( const describeErrorCause = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, depth: number, ): JsonValue => { const message = boundedUtf8Text( @@ -152,15 +154,45 @@ const describeErrorCause = ( ...(message.truncated ? { message_truncated: true } : {}), ...causeCode(cause, redact), ...causeSignal(cause, redact), + ...causeCommandOutcome(cause, redact), ...causeCompilerOutcome(cause, redact), ...causeLaunchOutcome(cause, redact), ...nestedErrorCause(cause, redact, depth), }; }; +const causeCommandOutcome = ( + cause: Error, + redact: (value: string, incomplete?: boolean) => string, +): Record => { + if (!(cause instanceof GhidraLaunchSupportError)) return {}; + const outcome = cause.outcome; + if (outcome === undefined) return {}; + const stdout = boundedUtf8Text( + redact(outcome.stdout, outcome.outputTruncated !== false), + MAX_COMMAND_OUTPUT_BYTES, + ); + const stderr = boundedUtf8Text( + redact(outcome.stderr, outcome.outputTruncated !== false), + MAX_COMMAND_OUTPUT_BYTES, + ); + return { + command_outcome: { + stdout: stdout.text, + stderr: stderr.text, + stdoutBytes: outcome.stdoutBytes ?? Buffer.byteLength(outcome.stdout), + stderrBytes: outcome.stderrBytes ?? Buffer.byteLength(outcome.stderr), + stdoutTruncated: stdout.truncated, + stderrTruncated: stderr.truncated, + outputTruncated: outcome.outputTruncated, + killed: outcome.killed, + }, + }; +}; + const causeCode = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { const code = "code" in cause ? cause.code : undefined; if (typeof code === "string") { @@ -176,7 +208,7 @@ const causeCode = ( const causeSignal = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { const signal = "signal" in cause ? cause.signal : undefined; if (typeof signal === "string") { @@ -192,7 +224,7 @@ const causeSignal = ( const causeCompilerOutcome = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { if (!(cause instanceof DarwinProcessOwnershipInspectionError)) return {}; const outcome = cause.compilerOutcome; @@ -218,7 +250,7 @@ const causeCompilerOutcome = ( const boundedCompilerField = ( value: string | number | null, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): { readonly value: string | number | null; readonly truncated: boolean } => { if (typeof value !== "string") return { value, truncated: false }; const bounded = boundedUtf8Text(redact(value), MAX_CAUSE_FIELD_BYTES); @@ -228,9 +260,10 @@ const boundedCompilerField = ( /** Retain a launch failure's launcher command outcome in its cause envelope. */ const causeLaunchOutcome = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { if (!(cause instanceof GhidraLaunchError)) return {}; + if (cause.cause instanceof GhidraLaunchSupportError) return {}; const launchSupport = cause.launchSupport.length === 0 ? {} @@ -273,7 +306,7 @@ const causeLaunchOutcome = ( const nestedErrorCause = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, depth: number, ): Record => { if (!("cause" in cause)) return {}; diff --git a/src/ghidra/GhidraSnapshotCache.test.ts b/src/ghidra/GhidraSnapshotCache.test.ts index 83d293aa2..fa7b35c22 100644 --- a/src/ghidra/GhidraSnapshotCache.test.ts +++ b/src/ghidra/GhidraSnapshotCache.test.ts @@ -11,6 +11,7 @@ import { silentLogger } from "../logger.js"; import { GhidraProvider } from "./GhidraProvider.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; import { GHIDRA_SESSION_CAPABILITIES } from "./GhidraSessionValues.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; const TARGET: BinaryTarget = { path: "/tmp/ghidra-cache-fixture", @@ -23,7 +24,7 @@ const TARGET: BinaryTarget = { // Exercise the real provider descriptors, profile, input/output projection and // session cache; only the installed Ghidra process is replaced at its seam. -const fixture = (version = "12.1.4") => { +const fixture = (version = "12.1.4", coverage?: RelationshipCoverage) => { const config = parseConfig({ GHIDRA_INSTALL_DIR: "/opt/ghidra" }); if (!config.ok) throw config.error; const calls: string[] = []; @@ -113,7 +114,13 @@ const fixture = (version = "12.1.4") => { return ok( operation === "procedure_pseudo_code" || operation === "analyze_function" - ? { value, limitations: [] } + ? { + value, + limitations: [], + ...(operation === "analyze_function" && coverage !== undefined + ? { relationship_coverage: coverage } + : {}), + } : value, ); }, @@ -133,42 +140,119 @@ const open = async (session: ReturnType["session"]) => { expect((await session.openResolvedTarget(preview.value)).ok).toBe(true); }; -describe("Ghidra snapshot cache", () => { - it("refuses snapshots whose profile predates typed incoming references", async () => { - const initial = fixture(); - await open(initial.session); - const snapshot = initial.session.exportAnalysisSnapshot(); - if (!snapshot.ok) throw snapshot.error; - await initial.session.close(); - const profile = snapshot.value.binding.analysis_profile; - expect(profile.parameters.instruction_references).toBe( - "exact-address-incoming-v1", - ); - const legacy = { - ...snapshot.value, - binding: { - ...snapshot.value.binding, - analysis_profile: createAnalysisProfile( - profile.provider, - Object.fromEntries( - Object.entries(profile.parameters).filter( - ([key]) => key !== "instruction_references", - ), - ), - ), - }, - }; - const replay = fixture(); - const imported = replay.session.importAnalysisSnapshot(legacy); - if (!imported.ok) throw imported.error; - expect(await replay.session.previewTarget(TARGET)).toMatchObject({ - ok: false, - error: { _tag: "EvidenceIntegrityError" }, - }); - expect(replay.calls).toEqual([]); - await replay.session.close(); - }); +describe("Ghidra relationship coverage replay", () => { + it.each([ + undefined, + { status: "complete", diagnostics: ["Resolved static references only."] }, + { status: "partial", limitations: ["Instruction coverage was truncated."] }, + { status: "unknown", reason: "Relationship inventory was unavailable." }, + { + status: "partial", + limitations: ["Missing callers."], + limited_directions: ["callers"], + }, + ] satisfies (RelationshipCoverage | undefined)[])( + "preserves relationship coverage %j through live execution, cache and snapshot replay", + async (coverage) => { + const initial = fixture("12.1.4", coverage); + await open(initial.session); + const parameters = { procedure: "0x1000" }; + for (let i = 0; i < 2; i++) { + const execution = await initial.session.execute( + "analyze_function", + parameters, + ); + if (!execution.ok) throw execution.error; + expect(execution.value.relationshipCoverage).toEqual(coverage); + } + expect(initial.calls).toEqual(["analyze_function"]); + const snapshot = initial.session.exportAnalysisSnapshot(); + if (!snapshot.ok) throw snapshot.error; + await initial.session.close(); + const replay = fixture(); + expect(replay.session.importAnalysisSnapshot(snapshot.value).ok).toBe( + true, + ); + await open(replay.session); + const execution = await replay.session.execute( + "analyze_function", + parameters, + ); + if (!execution.ok) throw execution.error; + expect(execution.value.relationshipCoverage ?? undefined).toEqual( + coverage, + ); + expect(replay.calls).toEqual([]); + await replay.session.close(); + }, + ); +}); +describe("Ghidra snapshot profile admission", () => { + it.each([ + { + parameter: "instruction_references", + current: "exact-address-incoming-v1", + legacy: undefined, + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "resolved-call-reference-diagnostics-v1", + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "decoded-call-target-coverage-v2", + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "decoded-call-target-coverage-v3", + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "function-body-call-coverage-v4", + }, + ])( + "refuses snapshots with outdated $parameter semantics", + async ({ parameter, current, legacy: prior }) => { + const initial = fixture(); + await open(initial.session); + const snapshot = initial.session.exportAnalysisSnapshot(); + if (!snapshot.ok) throw snapshot.error; + await initial.session.close(); + const profile = snapshot.value.binding.analysis_profile; + expect(profile.parameters[parameter]).toBe(current); + const legacy = { + ...snapshot.value, + binding: { + ...snapshot.value.binding, + analysis_profile: createAnalysisProfile(profile.provider, { + ...Object.fromEntries( + Object.entries(profile.parameters).filter( + ([key]) => key !== parameter, + ), + ), + ...(prior === undefined ? {} : { [parameter]: prior }), + }), + }, + }; + const replay = fixture(); + const imported = replay.session.importAnalysisSnapshot(legacy); + if (!imported.ok) throw imported.error; + expect(await replay.session.previewTarget(TARGET)).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); + expect(replay.calls).toEqual([]); + await replay.session.close(); + }, + ); +}); + +describe("Ghidra snapshot cache", () => { it.each([ { operation: "analyze_function", parameters: { procedure: "0x1000" } }, { diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 466136f3f..9e2ac1f30 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -55,7 +55,7 @@ const failureForPrimary = ( if (cause instanceof GoBinaryResourceFailure) return new AnalysisResourceConstraintError( OPERATION, - "memory", + cause.boundary === "search" ? "cpu" : "memory", `Selected artifact ${path}: ${cause.message}`, { boundary: cause.boundary, maximum_bytes: cause.maximum_bytes }, { cause }, @@ -194,7 +194,7 @@ export class GoBinaryProvider implements GoBinaryPort { limitations: [ "Embedded build metadata is observed file content, not proof of compiler identity, dependency authenticity or runtime behavior.", "Only Go build information is inspected; function names, source mappings, type metadata and decompiled code are not recovered.", - "Resource guards bound stable artifact reads to 256 MiB, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits.", + "Resource guards bound stable artifact reads and cumulative header searches to 256 MiB each, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits.", ...(info !== null && (info.go_version === null || info.module_text === null) ? [ diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index e4d7c8381..e446e4044 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -114,6 +114,14 @@ export class HopperProvider implements AnalysisProviderCandidate { reason: `Hopper cannot directly analyze ${target.kind} targets.`, diagnostics, }; + if (target.kind === "database" && this.platform === "darwin") + return { + status: "unsupported", + code: "target_kind_unsupported", + reason: + "This Hopper macOS launcher cannot start REA's analysis bridge when opening a saved Hopper database. Open an executable target instead; saved-database analysis is unavailable until Hopper exposes a database launch route that runs the bridge script.", + diagnostics, + }; if (target.kind === "database") return { status: "supported", diff --git a/src/hopper/HopperProviderDiscovery.test.ts b/src/hopper/HopperProviderDiscovery.test.ts index 82acf8888..5df7eacad 100644 --- a/src/hopper/HopperProviderDiscovery.test.ts +++ b/src/hopper/HopperProviderDiscovery.test.ts @@ -46,7 +46,7 @@ describe("Hopper provider discovery", () => { }); it("separates target kind, format, and architecture support", () => { - const hopper = provider(process.execPath); + const hopper = provider(process.execPath, "linux"); expect(hopper.inspectTargetSupport(databaseTarget())).toMatchObject({ status: "supported", }); @@ -59,12 +59,31 @@ describe("Hopper provider discovery", () => { diagnostics: { target_kind: "archive", target_format: "asar" }, }); }); + + it("fails closed for saved Hopper databases on macOS without a bridge launch route", () => { + expect( + provider(process.execPath, "darwin").inspectTargetSupport( + databaseTarget(), + ), + ).toMatchObject({ + status: "unsupported", + code: "target_kind_unsupported", + reason: expect.stringContaining("cannot start REA's analysis bridge"), + diagnostics: { + target_kind: "database", + target_format: "analysis-database", + }, + }); + }); }); -const provider = (launcherPath: string): HopperProvider => { +const provider = ( + launcherPath: string, + platform: NodeJS.Platform = process.platform, +): HopperProvider => { const config = parseConfig({ HOPPER_LAUNCHER_PATH: launcherPath }); if (!config.ok) throw config.error; - return new HopperProvider(config.value, silentLogger, {}); + return new HopperProvider(config.value, silentLogger, {}, platform); }; const databaseTarget = (): BinaryTarget => ({ diff --git a/src/ida/IdaContainment.test.ts b/src/ida/IdaContainment.test.ts index 069388df8..f81d7b9cd 100644 --- a/src/ida/IdaContainment.test.ts +++ b/src/ida/IdaContainment.test.ts @@ -148,7 +148,7 @@ const found = (query: string, fn: typeof function_) => ({ }); describe("IDA containment capabilities", () => { - it("declares both operations for the headless profile only", () => { + it("declares document identity in both profiles and address resolution only in headless", () => { const declared = (mode: "attached" | "headless", native = false) => idaCapabilities(mode, false, native) .map(({ operation }) => operation) @@ -172,35 +172,30 @@ describe("IDA containment capabilities", () => { "resolve_containing_procedure", "list_documents", ]); - expect(declared("attached")).toEqual([]); - expect(declared("attached", true)).toEqual([]); + expect(declared("attached")).toEqual(["list_documents"]); + expect(declared("attached", true)).toEqual(["list_documents"]); expect( idaCapabilities("attached", false).map(({ operation }) => operation), - "attached profiles do not even declare them", - ).not.toContain("list_documents"); + "attached profiles declare the bound document but not address resolution", + ).toContain("list_documents"); }); - it.each(["resolve_containing_procedure", "list_documents"] as const)( - "refuses %s on the legacy attached profile without calling upstream", - async (operation) => { - const { client, producer } = await session({}, "attached"); - try { - const result = await client.execute( - operation, - operation === "list_documents" ? {} : { address: "0x1000" }, - ); - - expect(result.ok).toBe(false); - if (!result.ok) - expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); - expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( - false, - ); - } finally { - expect((await client.close()).ok).toBe(true); - } - }, - ); + it("refuses address resolution on the legacy attached profile", async () => { + const { client, producer } = await session({}, "attached"); + try { + const result = await client.execute("resolve_containing_procedure", { + address: "0x1000", + }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( + false, + ); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); }); describe("headless IDA containing procedure", () => { diff --git a/src/ida/IdaDiagnosticRedactor.test.ts b/src/ida/IdaDiagnosticRedactor.test.ts new file mode 100644 index 000000000..0c65e37d3 --- /dev/null +++ b/src/ida/IdaDiagnosticRedactor.test.ts @@ -0,0 +1,563 @@ +import { describe, expect, it } from "vitest"; +import { + createIdaDiagnosticRedactor, + IDA_DIAGNOSTIC_SOURCE_BYTES, +} from "./IdaDiagnosticRedactor.js"; + +describe("IDA diagnostic credential redaction", () => { + it.each([false, true])( + "redacts literal credentials once regardless of header order (reversed: %s)", + (reversed) => { + const entries = [ + ["Authorization", "Bearer fixture"], + ["proxy-AUTHORIZATION", "bAsIc fixture-long.$[token]"], + ["cOoKiE", "session=fixture-long.$[token]; marker=redacted"], + ["X-Evidence", "caller-selected-evidence"], + ]; + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: Object.fromEntries(reversed ? entries.reverse() : entries), + mode: "attached", + timeoutMs: 1000, + }).text; + expect( + redact( + "Bearer fixture / fixture / bAsIc fixture-long.$[token] / fixture-long.$[token] / session=fixture-long.$[token]; marker=redacted /caller/input?token=visible API_KEY=visible caller-selected-evidence", + ), + ).toBe( + "␟ / ␟ / ␟ / ␟ / ␟ /caller/input?token=visible API_KEY=visible caller-selected-evidence", + ); + }, + ); + it("redacts configured and HTTP-normalized header values", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { + aUtHoRiZaTiOn: "\r\n \tBearer fixture-secret \t\r\n", + "pRoXy-AuThOrIzAtIoN": "\tBasic proxy-secret ", + CoOkIe: " session=cookie-secret\t", + }, + mode: "attached", + timeoutMs: 1000, + }).text; + expect( + redact( + "raw=\r\n \tBearer fixture-secret \t\r\n; wire=Bearer fixture-secret; token=fixture-secret; proxy=Basic proxy-secret; bare=proxy-secret; cookie=session=cookie-secret", + ), + ).toBe( + "raw=[redacted transport credential]; wire=[redacted transport credential]; token=[redacted transport credential]; proxy=[redacted transport credential]; bare=[redacted transport credential]; cookie=[redacted transport credential]", + ); + }); + it("does not treat empty HTTP authentication as diagnostic content", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { + Authorization: "\r\n \t \r\n", + "Proxy-Authorization": "", + Cookie: "\t", + }, + mode: "attached", + timeoutMs: 1000, + }).text; + const message = "Failed at /caller/input \t with ordinary whitespace"; + expect(redact(message)).toBe(message); + }); + it("does not reprocess the replacement marker as another credential", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Bearer alpha", Cookie: "redacted" }, + mode: "attached", + timeoutMs: 1000, + }).text; + expect(redact("alpha redacted")).toBe("␟ ␟"); + }); +}); + +const credentialCases: readonly { + readonly name: string; + readonly headers: Readonly>; + readonly message: string; + readonly expected: string; +}[] = [ + { + name: "duplicate cookie names, quoted values, and equals padding", + headers: { + Cookie: + 'session=first-secret; session=second-secret; signed="padded=value=="', + }, + message: "first-secret / second-secret / padded=value== / session signed", + expected: + "[redacted transport credential] / [redacted transport credential] / [redacted transport credential] / session signed", + }, + { + name: "empty cookie and auth-param values", + headers: { Cookie: 'a=; b=""', Authorization: 'Custom empty=""' }, + message: 'Ordinary spaces and "" remain; a b empty are names', + expected: 'Ordinary spaces and "" remain; a b empty are names', + }, + { + name: "opaque token68 and explicit one-character credentials", + headers: { Authorization: "Negotiate opaque+/==", Cookie: "a=Q" }, + message: "opaque+/== Q Negotiate a", + expected: + "[redacted transport credential] [redacted transport credential] Negotiate a", + }, + { + name: "auth-param values retain quoted commas and decode quoted pairs", + headers: { + Authorization: String.raw`Digest first="nonce,part", response="quote\"part", opaque="back\\part", tail=last-secret`, + }, + message: String.raw`nonce,part / quote"part / quote\"part / back\part / back\\part / last-secret / Digest first response opaque tail`, + expected: + "[redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / Digest first response opaque tail", + }, + { + name: "complete parameters before a malformed quoted member", + headers: { + Authorization: + 'Digest first=known-secret, broken="unterminated, second=visible', + }, + message: "known-secret visible broken second", + expected: "[redacted transport credential] visible broken second", + }, + { + name: "unclosed quotes do not create parameters from their contents", + headers: { Authorization: 'Digest broken="unterminated, second=visible' }, + message: "unterminated visible broken second", + expected: "unterminated visible broken second", + }, + { + name: "malformed cookie members do not hide valid later pairs", + headers: { + Cookie: 'broken; =not-a-value; malformed="unclosed; session=valid-secret', + }, + message: "valid-secret broken not-a-value unclosed session", + expected: + "[redacted transport credential] broken not-a-value unclosed session", + }, + { + name: "cookie octets remain opaque without URL decoding", + headers: { Cookie: "session=encoded%2Fcredential" }, + message: "encoded%2Fcredential / encoded/credential", + expected: "[redacted transport credential] / encoded/credential", + }, +]; +it.each(credentialCases)( + "parses $name at the configured credential boundary", + ({ headers, message, expected }) => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers, + mode: "attached", + timeoutMs: 1000, + }).text; + expect(redact(message)).toBe(expected); + }, +); + +it("redacts full configured malformed headers without guessing their internal values", () => { + const header = 'Digest broken="unterminated, second=visible'; + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: header }, + mode: "attached", + timeoutMs: 1000, + }); + expect(redactor.text(header)).toBe("[redacted transport credential]"); + expect(redactor.json(JSON.stringify(header))).toBe( + '"[redacted transport credential]"', + ); +}); + +const marker = "[redacted transport credential]"; +const redactorFor = (header: string, name = "Authorization") => + createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { [name]: header }, + mode: "attached", + timeoutMs: 1000, + }); + +describe("IDA Basic authentication payload components", () => { + it.each(["aUtHoRiZaTiOn", "pRoXy-AuThOrIzAtIoN"])( + "protects decoded Basic pairs and passwords while preserving user names for %s", + (name) => { + const pair = "fixture-user:fixture-password:tail"; + const encoded = Buffer.from(pair, "utf8").toString("base64"); + const redactor = redactorFor("bAsIc " + encoded, name); + const reason = + "pair=" + + pair + + "; user=fixture-user; password=fixture-password:tail; wire=" + + encoded; + const safe = + "pair=" + + marker + + "; user=fixture-user; password=" + + marker + + "; wire=" + + marker; + expect(redactor.text(reason + "; path=/caller/input?token=visible")).toBe( + safe + "; path=/caller/input?token=visible", + ); + expect( + redactor.json(JSON.stringify({ reason, path: "/caller/input" })), + ).toBe(JSON.stringify({ reason: safe, path: "/caller/input" })); + }, + ); + + it.each([ + [ + ":fixture-password", + "pair=:fixture-password; password=fixture-password; empty values leave spaces", + "pair=" + marker + "; password=" + marker + "; empty values leave spaces", + ], + [ + "fixture-user:", + "pair=fixture-user:; user=fixture-user; empty values leave spaces", + "pair=" + marker + "; user=fixture-user; empty values leave spaces", + ], + [ + ":", + "pair=:; empty values leave spaces", + "pair=" + marker + "; empty values leave spaces", + ], + ])( + "protects a decoded pair with empty components: %s", + (pair, reason, safe) => { + const encoded = Buffer.from(pair, "utf8").toString("base64"); + expect(redactorFor("Basic " + encoded).text(reason)).toBe(safe); + }, + ); +}); + +describe("IDA Basic authentication charset and invalid data", () => { + it("retains both strict UTF-8 and exact octet-compatible interpretations when charset is unknown", () => { + const pair = "tést-user:päss£word"; + const bytes = Buffer.from(pair, "utf8"); + const octets = bytes.toString("latin1"); + const redactor = redactorFor("Basic " + bytes.toString("base64")); + const reason = + "utf8=" + + pair + + "; octets=" + + octets + + "; user=tést-user; password=päss£word; octet-user=tést-user; octet-password=päss£word; caller=/caller/input"; + expect(redactor.text(reason)).toBe( + "utf8=" + + marker + + "; octets=" + + marker + + "; user=tést-user; password=" + + marker + + "; octet-user=tést-user; octet-password=" + + marker + + "; caller=/caller/input", + ); + }); + + it("preserves exact Latin-1 octets without inventing lossy UTF-8 replacement credentials", () => { + const bytes = Buffer.from("latin-user:pässword", "latin1"); + const redactor = redactorFor("Basic " + bytes.toString("base64")); + expect( + redactor.text( + "latin-user:pässword / latin-user / pässword / p\ufffdssword /caller/input", + ), + ).toBe( + marker + " / latin-user / " + marker + " / p\ufffdssword /caller/input", + ); + }); + + it("preserves a UTF-8 BOM as credential data rather than dropping it", () => { + const pair = "\ufefffixture-user:fixture-password"; + const encoded = Buffer.from(pair, "utf8").toString("base64"); + expect( + redactorFor("Basic " + encoded).text(pair + " / \ufefffixture-user"), + ).toBe(marker + " / \ufefffixture-user"); + }); + + it("keeps decoded Basic values opaque without URL decoding", () => { + const encoded = Buffer.from("fixture-user:p%2Fsecret", "utf8").toString( + "base64", + ); + expect(redactorFor("Basic " + encoded).text("p%2Fsecret / p/secret")).toBe( + marker + " / p/secret", + ); + }); + + const canonical = Buffer.from( + "fixture-user:fixture-password", + "utf8", + ).toString("base64"); + const alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const nonzeroPadBits = canonical.replace( + /(.)(=)$/u, + (_match, symbol: string, padding: string) => + (alphabet[alphabet.indexOf(symbol) + 1] ?? "!") + padding, + ); + it.each([ + { name: "unpadded", payload: canonical.slice(0, -1) }, + { name: "at most two padding characters", payload: canonical + "=" }, + ])( + "accepts the supported normalized Basic Base64 spelling: $name", + ({ payload }) => { + const redactor = redactorFor("Basic " + payload); + expect( + redactor.text( + "fixture-user:fixture-password / fixture-password / fixture-user", + ), + ).toBe(marker + " / " + marker + " / fixture-user"); + }, + ); + + it.each([ + { + name: "non-alphabet character", + payload: canonical.slice(0, 4) + "!" + canonical.slice(4), + }, + { + name: "embedded whitespace", + payload: canonical.slice(0, 4) + " " + canonical.slice(4), + }, + { name: "more than two pad characters", payload: canonical + "==" }, + { name: "nonzero pad bits", payload: nonzeroPadBits }, + { + name: "missing colon separator", + payload: Buffer.from("fixture-user", "utf8").toString("base64"), + }, + { + name: "ASCII control in user-pass", + payload: Buffer.from("fixture-user:fixture-password\n", "utf8").toString( + "base64", + ), + }, + { + name: "auth-param-shaped malformed Basic data", + payload: "credential=fixture-password", + }, + ])("retains only opaque configured Basic data for $name", ({ payload }) => { + const header = "Basic " + payload; + const redactor = redactorFor(header); + expect(redactor.text(header + " / " + payload)).toBe( + marker + " / " + marker, + ); + expect(redactor.text("fixture-user / fixture-password /caller/input")).toBe( + "fixture-user / fixture-password /caller/input", + ); + }); + + it("does not decode another scheme's base64-shaped opaque payload", () => { + const encoded = Buffer.from( + "fixture-user:fixture-password", + "utf8", + ).toString("base64"); + const redactor = redactorFor("Custom " + encoded); + expect(redactor.text(encoded + " / fixture-user / fixture-password")).toBe( + marker + " / fixture-user / fixture-password", + ); + }); +}); + +it("redacts protocol-decoded Basic credentials and password without treating ordinary names as secrets", () => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { + Authorization: `Basic ${Buffer.from("user:secret").toString("base64")}`, + }, + }); + expect(redactor.text("user:secret / secret / user")).toBe( + "[redacted transport credential] / [redacted transport credential] / user", + ); + expect(redactor.json('"\\u0073ecret"')).toBe( + '"[redacted transport credential]"', + ); +}); + +it.each(["text", "json"] as const)( + "bounds short-secret diagnostic expansion before %s redaction", + (format) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Cookie: "a=Q" }, + }); + const result = redactor[format]("Q".repeat(2 * 1024 * 1024)); + expect(result).toBe("[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]"); + }, +); + +it.each(["text", "json"] as const)( + "applies the UTF-8 source-byte budget before %s processing even without credentials", + (format) => { + const atBudget = "£".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES / 2); + const overBudget = atBudget + "£"; + const configuredHeaders: readonly Readonly>[] = [ + {}, + { Cookie: "a=Q" }, + ]; + for (const headers of configuredHeaders) { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers, + }); + expect(redactor[format](atBudget)).toBe(atBudget); + expect(redactor[format](overBudget)).toBe( + "[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]", + ); + } + }, +); + +it.each(["dXNlcjpzZWNyZXQ=", "dXNlcjpzZWNyZXQ", "dXNlcjpzZWNyZXQ=="])( + "redacts accepted canonical Basic spellings for %s", + (payload) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Authorization: `Basic ${payload}` }, + }); + const marker = "[redacted transport credential]"; + expect( + redactor.text( + "dXNlcjpzZWNyZXQ= / dXNlcjpzZWNyZXQ / user:secret / secret", + ), + ).toBe(`${marker} / ${marker} / ${marker} / ${marker}`); + }, +); + +describe("overlapping IDA credential source spans", () => { + it.each([false, true])( + "unions all overlapping plain credentials before writing (reversed: %s)", + (reversed) => { + const entries = [ + ["Authorization", "Custom X]"], + ["Cookie", "a=]SECRET"], + ]; + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: Object.fromEntries(reversed ? entries.reverse() : entries), + mode: "attached", + timeoutMs: 1000, + }); + const result = redactor.text( + "Rejected X]SECRET at /caller/input?cursor=visible", + ); + expect(result).toBe("Rejected ␟ at /caller/input?cursor=visible"); + expect(result).not.toContain("]SECRET"); + }, + ); + + it("unions a chain of different overlapping credentials", () => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Custom abc", Cookie: "a=cde; b=def" }, + mode: "attached", + timeoutMs: 1000, + }); + expect(redactor.text("prefix abcdefg suffix")).toBe( + "prefix " + marker + "g suffix", + ); + }); + + it("merges dense same-credential overlaps without retaining a per-match collection", () => { + const redactor = redactorFor("Custom AA"); + const result = redactor.text("A".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES)); + expect(result.length).toBe(marker.length); + expect(result).toBe(marker); + }); + + it("retains adjacent disjoint credential occurrences", () => { + expect(redactorFor("Custom AA").text("AA-AA")).toBe(marker + "-" + marker); + }); +}); + +describe("credential-safe redaction markers", () => { + it.each([ + { secret: "]SECRET", source: "XSECRET", expected: "␟SECRET" }, + { secret: "SECRET[", source: "SECRETX", expected: "SECRET␟" }, + { + secret: "a[redacted transport credential]b", + source: "aXb", + expected: "a␟b", + }, + { secret: "redacted", source: "redacted", expected: "␟" }, + ])( + "does not create $secret from replacement text or boundaries", + ({ secret, source, expected }) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Authorization: "Custom " + secret, Cookie: "a=X" }, + }); + expect(redactor.text(source)).toBe(expected); + expect( + redactor.json( + JSON.stringify({ reason: source, context: "/caller/input" }), + ), + ).toBe(JSON.stringify({ reason: expected, context: "/caller/input" })); + expect(redactor.json('"' + source + '",x')).toBe('"' + expected + '",x'); + }, + ); + it("chooses a different graphic scalar when a decoded Basic credential contains the first marker", () => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { + Authorization: + "Basic " + Buffer.from("user:␟", "utf8").toString("base64"), + Cookie: "a=]SECRET", + }, + }); + expect(redactor.text("␟ at /caller/input")).toBe("␠ at /caller/input"); + expect(redactor.json(String.raw`{"reason":"␟","context":"kept"}`)).toBe( + '{"reason":"␠","context":"kept"}', + ); + }); +}); + +describe("unclassified diagnostic blocks and message ownership", () => { + it.each>>([{}, { Cookie: "a=Q" }])( + "budgets the entire source block list before redaction: %s", + (headers) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers, + }); + const first = "£".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES / 4); + expect(redactor.blocks([first, first])).toBe( + "[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]", + ); + expect(redactor.blocks([first, first.slice(1)])).toBe( + first + "\n" + first.slice(1), + ); + }, + ); + it("keeps complete quoted plain credentials while preserving explicit JSON semantics", () => { + const redactor = redactorFor(String.raw`Custom folder\tmp`); + const source = String.raw`"folder\tmp at /caller/input"`; + expect(redactor.blocks([source])).toBe( + '"[redacted transport credential] at /caller/input"', + ); + expect(redactor.json(source)).toBe(source); + }); + it("sanitizes the omission notice as diagnostic text", () => { + const redactor = redactorFor("Custom omitted"); + expect( + redactor.blocks(["Q".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES + 1)]), + ).toContain("[redacted transport credential]"); + expect( + redactor.text("Q".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES + 1)), + ).not.toContain("omitted"); + }); +}); diff --git a/src/ida/IdaDiagnosticRedactor.ts b/src/ida/IdaDiagnosticRedactor.ts new file mode 100644 index 000000000..d332d1548 --- /dev/null +++ b/src/ida/IdaDiagnosticRedactor.ts @@ -0,0 +1,243 @@ +import { + redactIdaJsonDiagnostic, + redactIdaUnclassifiedDiagnostic, +} from "./IdaJsonDiagnostic.js"; +import { + IDA_REDACTION_MARKER, + joinIdaDiagnosticParts, + writeIdaDiagnostic, + type IdaDiagnosticMatch, +} from "./IdaDiagnosticWriter.js"; +import type { IdaConfiguration } from "./IdaConfiguration.js"; +import { AnalysisProtocolError } from "../domain/analysisErrorCore.js"; + +// Diagnostic text is bounded before decoding or replacement expansion. +export const IDA_DIAGNOSTIC_SOURCE_BYTES = 64 * 1024; +const omittedDiagnostic = "[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]"; + +const boundedDiagnostic = + (redact: (message: string) => string) => + (message: string): string => + Buffer.byteLength(message, "utf8") > IDA_DIAGNOSTIC_SOURCE_BYTES + ? redact(omittedDiagnostic) + : redact(message); + +/** RFC 7617 user-pass candidates; the configured request carries no charset. */ +const basicPayloads = (payload: string): readonly string[] => { + if (!/^[A-Za-z\d+/]+={0,2}$/u.test(payload)) return []; + const octets = Buffer.from(payload, "base64"); + const canonical = octets.toString("base64"); + // Match the supported padded/unpadded policy without accepting Buffer's + // permissive alphabet or discarding nonzero pad bits. + if ( + canonical.replace(/=+$/u, "") !== payload.replace(/=+$/u, "") || + octets.indexOf(0x3a) < 0 + ) + return []; + // Basic user-id and password exclude ASCII control characters. + if (octets.some((value) => value < 0x20 || value === 0x7f)) return []; + const candidates = [octets.toString("latin1")]; + try { + candidates.push( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(octets), + ); + } catch { + // Invalid UTF-8 has no decoded Unicode candidate. Exact Latin-1 preserves + // each octet; do not invent a replacement-character credential. + } + // The default charset is unknown. Retain both supported interpretations + // rather than selecting one or normalizing their configured source bytes. + return [ + canonical, + canonical.replace(/=+$/u, ""), + ...candidates.flatMap((pair) => { + const separator = pair.indexOf(":"); + // The complete pair and password are credentials; a bare user-id is not. + return [pair, pair.slice(separator + 1)].filter( + (value) => value.length > 0, + ); + }), + ]; +}; + +/** RFC 9110 credentials: any scheme, followed by opaque data or auth-params. */ +const authorizationPayloads = (header: string): readonly string[] => { + const credentials = header.match( + /^([!#$%&'*+.^_`|~\dA-Za-z-]+)[ \t]+([\s\S]+)$/u, + ); + const payload = credentials?.[2]; + if (payload === undefined) return []; + if (credentials?.[1]?.toLowerCase() === "basic") + return [payload, ...basicPayloads(payload)]; + const values = [payload]; + // A sticky parse cannot reinterpret commas inside an incomplete quoted value. + // Keep complete parameters preceding malformed input; the opaque payload above + // remains a known credential even when the parameter grammar is not satisfied. + const parameter = + /[ \t,]*[!#$%&'*+.^_`|~\dA-Za-z-]+[ \t]*=[ \t]*("(?:[^"\\\r\n]|\\[^\r\n])*"|[!#$%&'*+.^_`|~\dA-Za-z-]+)[ \t]*(?:,|$)/uy; + let cursor = 0; + while (cursor < payload.length) { + parameter.lastIndex = cursor; + const match = parameter.exec(payload); + if (match === null) break; + cursor = parameter.lastIndex; + const value = match[1] ?? ""; + if (!value.startsWith('"')) { + values.push(value); + continue; + } + const quoted = value.slice(1, -1); + if (quoted.length > 0) + values.push(value, quoted, quoted.replace(/\\([^\r\n])/gu, "$1")); + } + return values; +}; + +/** RFC 6265 cookie-pairs: retain duplicates and '=' inside values. */ +const cookiePayloads = (header: string): readonly string[] => + header.split(";").flatMap((pair) => { + const value = pair.match( + /^[ \t]*[!#$%&'*+.^_`|~\dA-Za-z-]+[ \t]*=[ \t]*(.*?)[ \t]*$/u, + )?.[1]; + if (value === undefined) return []; + const payload = + value.startsWith('"') && value.endsWith('"') ? value.slice(1, -1) : value; + // Cookie quotes wrap cookie-octets; they do not introduce escape syntax. + if (!/^[\x21\x23-\x2b\x2d-\x3a\x3c-\x5b\x5d-\x7e]+$/u.test(payload)) + return []; + return [value, payload]; + }); + +/** Choose one marker for this credential owner, including replacement boundaries. */ +const redactionMarker = (secrets: readonly string[]): string => { + // Crossing either edge of the default marker requires a square bracket. + // A credential wholly inside the marker must also be excluded. + if ( + secrets.every( + (secret) => + !secret.includes("[") && + !secret.includes("]") && + !IDA_REDACTION_MARKER.includes(secret), + ) + ) + return IDA_REDACTION_MARKER; + const characters = new Set(); + for (const secret of secrets) + for (const character of secret) characters.add(character); + // A graphic scalar absent from every credential cannot contain or connect + // a credential, and can be written verbatim inside a JSON string. + const graphic = /[\p{L}\p{N}\p{P}\p{S}]/u; + for (let point = 0x241f; point <= 0x10ffff; point++) { + if (point >= 0xd800 && point <= 0xdfff) continue; + const character = String.fromCodePoint(point); + if (graphic.test(character) && !characters.has(character)) return character; + } + throw new AnalysisProtocolError( + "Configured IDA credentials leave no available diagnostic redaction character.", + ); +}; + +/** Own configured credential interpretation for all IDA error diagnostics. */ +export const createIdaDiagnosticRedactor = ( + config?: IdaConfiguration, +): Readonly<{ + text(message: string): string; + json(message: string): string; + blocks(messages: readonly string[]): string; + message(prefix: string, ownedBody: string): string; +}> => { + const headers = + config !== undefined && "headers" in config + ? Object.entries(config.headers) + : []; + const values = headers.flatMap(([name, value]) => { + if (!/^(?:authorization|proxy-authorization|cookie)$/iu.test(name)) + return []; + // Fetch normalizes surrounding HTTP whitespace before sending headers. + const normalized = value.replace(/^[ \t\r\n]+|[ \t\r\n]+$/gu, ""); + if (normalized.length === 0) return []; + return [ + value, + normalized, + ...(name.toLowerCase() === "cookie" + ? cookiePayloads(normalized) + : authorizationPayloads(normalized)), + ]; + }); + const secrets = [...new Set(values.filter((value) => value.length > 0))].sort( + (left, right) => right.length - left.length, + ); + const replacement = + secrets.length === 0 ? IDA_REDACTION_MARKER : redactionMarker(secrets); + const alternatives = secrets + .map((secret) => secret.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + .join("|"); + // Look ahead at each source position; a global consuming match loses overlap. + // Longest-first alternatives cover every shorter match at the same position. + const pattern = + alternatives.length === 0 + ? undefined + : new RegExp("(?=(" + alternatives + "))", "gu"); + const matches = function* (value: string): Iterable { + if (pattern === undefined) return; + let pending: IdaDiagnosticMatch | undefined; + for (const match of value.matchAll(pattern)) { + const credential = match[1]; + if (credential === undefined || credential.length === 0) + throw new AnalysisProtocolError( + "IDA diagnostic matching omitted its credential span.", + ); + const span = { start: match.index, end: match.index + credential.length }; + if (pending !== undefined && span.start < pending.end) { + pending = { + start: pending.start, + end: Math.max(pending.end, span.end), + }; + } else { + if (pending !== undefined) yield pending; + pending = span; + } + } + if (pending !== undefined) yield pending; + }; + const text = boundedDiagnostic((message) => + writeIdaDiagnostic(message, matches(message), replacement), + ); + const json = boundedDiagnostic((message) => + secrets.length === 0 + ? message + : redactIdaJsonDiagnostic(message, matches, replacement), + ); + return { + text, + json, + blocks(messages) { + let bytes = 0; + let sourceUnits = 0; + for (let index = 0; index < messages.length; index++) { + const value = messages[index]!; + bytes += Buffer.byteLength(value, "utf8") + (index === 0 ? 0 : 1); + sourceUnits += value.length + (index === 0 ? 0 : 1); + if (bytes > IDA_DIAGNOSTIC_SOURCE_BYTES) return text(omittedDiagnostic); + } + function* parts(): Generator { + for (let index = 0; index < messages.length; index++) { + if (index > 0) yield "\n"; + const value = messages[index]!; + yield secrets.length === 0 || value.length === 0 + ? value + : redactIdaUnclassifiedDiagnostic(value, matches, replacement); + } + } + return joinIdaDiagnosticParts(parts(), sourceUnits); + }, + message(prefix, ownedBody) { + // Normalized wire headers and supported decoded candidates have no LF. + // Preserve the already-owned body without inspecting its replacement text. + return joinIdaDiagnosticParts( + [text(prefix), "\n", ownedBody], + prefix.length + 1 + ownedBody.length, + ); + }, + }; +}; diff --git a/src/ida/IdaDiagnosticWriter.ts b/src/ida/IdaDiagnosticWriter.ts new file mode 100644 index 000000000..750740507 --- /dev/null +++ b/src/ida/IdaDiagnosticWriter.ts @@ -0,0 +1,85 @@ +import { constants } from "node:buffer"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { bufferedJsonParts } from "../domain/jsonSerialization.js"; + +export const IDA_REDACTION_MARKER = "[redacted transport credential]"; +export type IdaDiagnosticMatch = Readonly<{ start: number; end: number }>; + +type DiagnosticRepresentation = "redacted-output" | "json-character-run"; + +/** An owned diagnostic cannot be represented as one runtime string. */ +export class IdaDiagnosticResourceError extends AnalysisResourceConstraintError { + constructor( + inputCodeUnits: number, + representedCodeUnits: number, + representation: DiagnosticRepresentation = "redacted-output", + ) { + super( + "ida_diagnostic_redaction", + "memory", + representation === "redacted-output" + ? "Redacted IDA diagnostics exceed the runtime's maximum string length." + : "Temporary IDA diagnostic JSON character decoding exceeds the runtime's maximum string length.", + { + boundary: "ida-diagnostic-redaction", + representation, + input_code_units: inputCodeUnits, + ...(representation === "redacted-output" + ? { output_code_units_at_least: representedCodeUnits } + : { json_decoding_code_units_at_least: representedCodeUnits }), + max_string_code_units: constants.MAX_STRING_LENGTH, + }, + ); + } +} + +/** Account for the complete representation before allocating its final string. */ +export const joinIdaDiagnosticParts = ( + parts: Iterable, + inputCodeUnits: number, +): string => { + function* checkedParts(): Generator { + let outputCodeUnits = 0; + for (const part of parts) { + outputCodeUnits += part.length; + if (outputCodeUnits > constants.MAX_STRING_LENGTH) + throw new IdaDiagnosticResourceError(inputCodeUnits, outputCodeUnits); + yield part; + } + } + return [...bufferedJsonParts(checkedParts())].join(""); +}; + +/** Write ordered source spans without retaining a per-match collection. */ +export const writeIdaDiagnostic = ( + text: string, + matches: Iterable, + replacement: string = IDA_REDACTION_MARKER, +): string => { + function* parts(): Generator { + let cursor = 0; + let pending: IdaDiagnosticMatch | undefined; + for (const match of matches) { + if (pending !== undefined && match.start < pending.end) { + pending = { + start: pending.start, + end: Math.max(pending.end, match.end), + }; + continue; + } + if (pending !== undefined) { + yield text.slice(cursor, pending.start); + yield replacement; + cursor = pending.end; + } + pending = match; + } + if (pending !== undefined) { + yield text.slice(cursor, pending.start); + yield replacement; + cursor = pending.end; + } + yield text.slice(cursor); + } + return joinIdaDiagnosticParts(parts(), text.length); +}; diff --git a/src/ida/IdaDoctor.ts b/src/ida/IdaDoctor.ts index 750ca757e..8a711aff1 100644 --- a/src/ida/IdaDoctor.ts +++ b/src/ida/IdaDoctor.ts @@ -52,13 +52,16 @@ const inspectPrivateWorkspace = (): DoctorProviderCheck => { return { name: "private_workspace", ok: privacy.available, - code: privacy.available ? null : "unsupported_host", + code: privacy.available ? null : privacy.code, detail: privacy.available ? `Private workspace authority is available (${privacy.proof}).` : privacy.reason, - remediation: privacy.available - ? null - : "Use the matching Windows x64 REA package and a writable local NTFS temporary directory.", - classification: "unsupported_host", + remediation: privacy.available ? null : privacy.remediation, + classification: + !privacy.available && privacy.code === "runtime_missing" + ? "missing_dependency" + : !privacy.available && privacy.code === "runtime_unavailable" + ? "runtime_failure" + : "unsupported_host", }; }; diff --git a/src/ida/IdaJsonDiagnostic.test.ts b/src/ida/IdaJsonDiagnostic.test.ts new file mode 100644 index 000000000..3929551d3 --- /dev/null +++ b/src/ida/IdaJsonDiagnostic.test.ts @@ -0,0 +1,337 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { + IdaDiagnosticResourceError, + writeIdaDiagnostic, +} from "./IdaDiagnosticWriter.js"; +import { createIdaDiagnosticRedactor } from "./IdaDiagnosticRedactor.js"; +import { redactIdaJsonDiagnostic } from "./IdaJsonDiagnostic.js"; + +const runtime = vi.hoisted(() => ({ limit: undefined as number | undefined })); +vi.mock("node:buffer", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + constants: { + ...original.constants, + get MAX_STRING_LENGTH() { + return runtime.limit ?? original.constants.MAX_STRING_LENGTH; + }, + }, + }; +}); +afterEach(() => { + runtime.limit = undefined; +}); + +const marker = "[redacted transport credential]"; +const forCredential = (credential: string) => + createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: `Custom ${credential}` }, + mode: "attached", + timeoutMs: 1000, + }); + +describe("IDA JSON diagnostic source preservation", () => { + it("matches equivalent JSON encodings without enumerating credential spellings", () => { + const redactor = forCredential(""); + const encodings = [ + ["<", String.raw`\u003c`, String.raw`\u003C`], + ["s", String.raw`\u0073`], + ["/", String.raw`\/`, String.raw`\u002f`, String.raw`\u002F`], + [">", String.raw`\u003e`, String.raw`\u003E`], + ].reduce( + (prefixes, units) => + prefixes.flatMap((prefix) => units.map((unit) => prefix + unit)), + [""], + ); + for (const encoded of encodings) + expect(redactor.json(`{"reason":"${encoded}"}`)).toBe( + `{"reason":"${marker}"}`, + ); + }); + + it("preserves unrelated spelling inside a matching string", () => { + const source = String.raw`{"reason":"\u003cpublic\u003e\/asset \u003csecret\u003e \ud83d\uDE00"}`; + expect(forCredential("").json(source)).toBe( + String.raw`{"reason":"\u003cpublic\u003e\/asset ` + + marker + + String.raw` \ud83d\uDE00"}`, + ); + }); + + it("retains duplicate fields, whitespace and exact numeric lexemes", () => { + const source = + '{\n "\\u003csecret\\u003e": 9007199254740993, "n": -0, "n": 1.00e+03, "value": ""\n}'; + expect(forCredential("").json(source)).toBe( + `{\n "${marker}": 9007199254740993, "n": -0, "n": 1.00e+03, "value": "${marker}"\n}`, + ); + }); + + it.each([ + { credential: "a😀b", encoded: String.raw`a\ud83d\uDE00b` }, + { credential: "a\ud800b", encoded: String.raw`a\uD800b` }, + { credential: "a\nb", encoded: String.raw`a\nb` }, + { credential: 'a"b', encoded: String.raw`a\u0022b` }, + ])("maps UTF-16 source spans for $encoded", ({ credential, encoded }) => { + expect(forCredential(credential).json(`"before ${encoded} after"`)).toBe( + `"before ${marker} after"`, + ); + }); + + it.each([ + { + source: String.raw`{broken: "\u003csecret\u003e",`, + expected: `{broken: "${marker}",`, + }, + { + source: String.raw`{"x":"\u003csecret\u003e`, + expected: `{"x":"${marker}`, + }, + { + source: String.raw`{"x":"\u003csecret\u003e\q"}`, + expected: `{"x":"${marker}\\q"}`, + }, + ])( + "preserves malformed source around known complete characters: $source", + ({ source, expected }) => { + expect(forCredential("").json(source)).toBe(expected); + }, + ); + + it("does not join decoded fragments across invalid escapes", () => { + const source = String.raw`{"x":""}`; + expect(forCredential("").json(source)).toBe(source); + }); + + it("retains literal matches crossing invalid JSON units", () => { + expect( + forCredential(String.raw`foo\qbar`).json(String.raw`{"x":"foo\qbar"}`), + ).toBe(`{"x":"${marker}"}`); + }); + + it.each( + ["b", "f", "n", "r", "t", "/", '"', "\\", "u0061"].map( + (escape) => "folder\\" + escape + "tmp", + ), + )( + "retains literal credentials in ambiguous malformed fragments: %s", + (credential) => { + const redactor = forCredential(credential); + for (const [before, after] of [ + ["Rejected ", ""], + ['Rejected "', '"'], + ['{"reason":"Rejected ', ""], + ['{"reason":"Rejected ', String.raw`\q"}`], + ]) + expect(redactor.json(before + credential + after)).toBe( + before + marker + after, + ); + expect(redactor.json('"' + credential + '"')).toBe( + '"' + credential + '"', + ); + expect(redactor.json(JSON.stringify("Rejected " + credential))).toBe( + JSON.stringify("Rejected " + marker), + ); + }, + ); + + it("uses decoded values instead of matching unrelated escape syntax", () => { + const source = String.raw`{"x":"\u003cpublic\u003e","y":"u003c"}`; + expect(forCredential("u003c").json(source)).toBe( + String.raw`{"x":"\u003cpublic\u003e","y":"` + marker + '"}', + ); + }); + + it("retains literal opaque matches spanning quoted fragments in plain bodies", () => { + const credential = 'prefix"quoted"suffix'; + expect( + forCredential(credential).json( + `Auth Custom ${credential}; payload ${credential}; path /caller/input`, + ), + ).toBe(`Auth ${marker}; payload ${marker}; path /caller/input`); + }); + + it("replaces adjacent matches once without rescanning the marker", () => { + expect( + forCredential("redacted").json(String.raw`"redacted\u0072edacted"`), + ).toBe('"␟␟"'); + }); + + it("handles unquoted JSON fragments while retaining ordinary text semantics", () => { + const source = String.raw`\u003csecret\u003e`; + const redactor = forCredential(""); + expect(redactor.text(source)).toBe(source); + expect(redactor.json(source)).toBe(marker); + const invalid = String.raw``; + expect(redactor.json(invalid)).toBe(invalid); + const unrelated = String.raw`"\u003cpublic\u003e"`; + expect(forCredential("u003c").json(unrelated)).toBe(unrelated); + expect(forCredential("u003c").json(unrelated.slice(1, -1))).toBe( + "\\" + marker + String.raw`public\u003e`, + ); + }); + it("leaves ordinary text diagnostics and credential-free configurations unchanged", () => { + const source = String.raw`{"x":"\u003csecret\u003e"}`; + expect(forCredential("").text(source)).toBe(source); + const redactor = createIdaDiagnosticRedactor({ + headers: {}, + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + }); + expect(redactor.json(source)).toBe(source); + }); +}); + +describe("IDA diagnostic output resource bounds", () => { + it.each(["text", "json"] as const)( + "bounds %s expansion using the runtime string limit", + (format) => { + runtime.limit = 64; + const redactor = forCredential("Q"); + const prefix = format === "json" ? '"' : ""; + const suffix = prefix; + expect(redactor[format](prefix + "QQ" + suffix)).toBe( + prefix + marker + marker + suffix, + ); + let failure: unknown; + try { + redactor[format](prefix + "QQQ" + suffix); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(IdaDiagnosticResourceError); + if (!(failure instanceof IdaDiagnosticResourceError)) + throw new Error("Expected an owned diagnostic resource failure"); + expect(failure.reportedLimits).toMatchObject({ + boundary: "ida-diagnostic-redaction", + max_string_code_units: 64, + }); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "resource_constraint", + details: { resource: "memory" }, + }); + }, + ); + + it("bounds temporary JSON character decoding before wrapping an oversized run", () => { + runtime.limit = 64; + const redactor = forCredential("Z"); + expect(redactor.json("Q".repeat(62))).toBe("Q".repeat(62)); + let failure: unknown; + try { + redactor.json("Q".repeat(63)); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(IdaDiagnosticResourceError); + if (!(failure instanceof IdaDiagnosticResourceError)) + throw new Error("Expected a temporary decoding resource failure"); + expect(failure.message).toContain("Temporary"); + expect(failure.reportedLimits).toEqual({ + boundary: "ida-diagnostic-redaction", + representation: "json-character-run", + input_code_units: 63, + json_decoding_code_units_at_least: 65, + max_string_code_units: 64, + }); + }); + + it("closes the first match stream if priming the second stream fails", () => { + runtime.limit = 64; + const source = "Q".repeat(63); + let closed = false; + function* matches() { + try { + yield { start: 0, end: 1 }; + } finally { + closed = true; + } + } + expect(() => redactIdaJsonDiagnostic(source, matches)).toThrow( + IdaDiagnosticResourceError, + ); + expect(closed).toBe(true); + }); + + it("stops consuming a lazy match stream before oversized output is allocated", () => { + runtime.limit = 32; + let consumed = 0; + function* matches() { + for (let index = 0; index < 10000; index++) { + consumed++; + yield { start: index, end: index + 1 }; + } + } + expect(() => writeIdaDiagnostic("Q".repeat(10000), matches())).toThrow( + IdaDiagnosticResourceError, + ); + expect(consumed).toBeLessThan(10); + }); +}); + +describe("overlapping raw and decoded JSON credential spans", () => { + const overlappingMarker = "␟"; + const overlappingRedactor = () => + createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Custom X]", Cookie: "a=]SECRET" }, + mode: "attached", + timeoutMs: 1000, + }); + it.each([ + String.raw`{"reason":"X]SECRET","n":1.00e+03}`, + String.raw`{"reason":"X\u005dSECRET","n":1.00e+03}`, + ])("unions decoded overlaps before monotone source mapping: %s", (source) => { + expect(overlappingRedactor().json(source)).toBe( + '{"reason":"' + overlappingMarker + '","n":1.00e+03}', + ); + }); + it.each(["X]SECRET", String.raw`X\u005dSECRET`])( + "unions raw/recovered overlaps in a malformed document: %s", + (source) => { + expect( + overlappingRedactor().json("Rejected " + source + " at /caller/input"), + ).toBe("Rejected " + overlappingMarker + " at /caller/input"); + }, + ); + it("maps a merged overlap containing an escaped surrogate pair", () => { + const encoded = Buffer.from("fixture-user:😀X]", "utf8").toString("base64"); + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Basic " + encoded, Cookie: "a=]SECRET" }, + mode: "attached", + timeoutMs: 1000, + }); + expect( + redactor.json( + String.raw`{"reason":"\ud83d\ude00X\u005dSECRET","context":"kept"}`, + ), + ).toBe('{"reason":"' + overlappingMarker + '","context":"kept"}'); + }); + it("unions dense escaped decoded overlaps lazily", () => { + const source = '"' + String.raw`\u0041`.repeat(4096) + '"'; + const result = forCredential("AA").json(source); + expect(result.length).toBe(marker.length + 2); + expect(result).toBe('"' + marker + '"'); + }); +}); + +it("bounds the final multi-block diagnostic representation", () => { + runtime.limit = 64; + const redactor = forCredential("Q"); + expect(() => redactor.blocks(["QQ", "QQ"])).toThrow( + IdaDiagnosticResourceError, + ); +}); + +it("bounds context plus the already-owned diagnostic body before joining", () => { + runtime.limit = 64; + const redactor = forCredential("Q"); + const body = redactor.text("QQ"); + expect(() => redactor.message("ctx", body)).toThrow( + IdaDiagnosticResourceError, + ); +}); diff --git a/src/ida/IdaJsonDiagnostic.ts b/src/ida/IdaJsonDiagnostic.ts new file mode 100644 index 000000000..4f0823b44 --- /dev/null +++ b/src/ida/IdaJsonDiagnostic.ts @@ -0,0 +1,216 @@ +import { constants } from "node:buffer"; +import { createScanner, type ScanError, type SyntaxKind } from "jsonc-parser"; +import { + IDA_REDACTION_MARKER, + IdaDiagnosticResourceError, + writeIdaDiagnostic, + type IdaDiagnosticMatch, +} from "./IdaDiagnosticWriter.js"; + +// jsonc-parser declares ambient const enums, which cannot be imported as values +// under verbatimModuleSyntax. Keep its scanner codes local and type-checked. +const JSON_STRING_LITERAL = 10 satisfies SyntaxKind; +const JSON_EOF = 17 satisfies SyntaxKind; +const JSON_NO_ERROR = 0 satisfies ScanError; + +/** Matches are non-overlapping and ordered by their UTF-16 source offsets. */ +type Matches = (value: string) => Iterable; + +// Complete JSON character runs remain meaningful even in an unfinished or +// malformed string. Invalid units separate runs instead of joining their text. +const JSON_CHARACTER_RUN = + /(?:[^"\\\u0000-\u001f]|\\(?:["\\/bfnrt]|u[\dA-Fa-f]{4}))+/gu; + +function* stringMatches( + encoded: string, + decoded: string, + offset: number, + matches: Matches, +): Generator { + // Match offsets and JSON escape widths are both UTF-16 units. No source bytes + // outside a matched value need to be normalized or serialized again. + let sourceCursor = 0; + let decodedCursor = 0; + const sourceOffset = (boundary: number): number => { + while (decodedCursor < boundary) { + sourceCursor += + encoded[sourceCursor] === "\\" + ? encoded[sourceCursor + 1] === "u" + ? 6 + : 2 + : 1; + decodedCursor++; + } + return offset + sourceCursor; + }; + for (const match of matches(decoded)) + yield { + start: sourceOffset(match.start), + end: sourceOffset(match.end), + }; +} + +function* characterRunMatches( + encoded: string, + offset: number, + matches: Matches, + inputCodeUnits: number, +): Generator { + for (const run of encoded.matchAll(JSON_CHARACTER_RUN)) { + // The run grammar admits only complete JSON characters. JSON.parse owns + // their decoded meaning; malformed units remain verbatim between runs. + const decodingCodeUnits = run[0].length + 2; + if (decodingCodeUnits > constants.MAX_STRING_LENGTH) + throw new IdaDiagnosticResourceError( + inputCodeUnits, + decodingCodeUnits, + "json-character-run", + ); + const decoded: unknown = JSON.parse('"' + run[0] + '"'); + if (typeof decoded !== "string") + throw new TypeError("A JSON character run did not decode to a string."); + yield* stringMatches(run[0], decoded, offset + run.index, matches); + } +} + +function* decodedMatches( + text: string, + matches: Matches, +): Generator { + let scannedEnd = 0; + const scanner = createScanner(text); + while (scanner.scan() !== JSON_EOF) { + if (scanner.getToken() !== JSON_STRING_LITERAL) continue; + const start = scanner.getTokenOffset(); + const end = start + scanner.getTokenLength(); + if (scannedEnd < start) + yield* characterRunMatches( + text.slice(scannedEnd, start), + scannedEnd, + matches, + text.length, + ); + if (scanner.getTokenError() === JSON_NO_ERROR) + yield* stringMatches( + text.slice(start + 1, end - 1), + scanner.getTokenValue(), + start + 1, + matches, + ); + else + yield* characterRunMatches( + text.slice(start + 1, end), + start + 1, + matches, + text.length, + ); + scannedEnd = end; + } + yield* characterRunMatches( + text.slice(scannedEnd), + scannedEnd, + matches, + text.length, + ); +} + +function* literalMatches( + text: string, + matches: Matches, + validDocument: boolean, +): Generator { + if (!validDocument) { + // Recovery cannot establish a unique meaning for a malformed document. + yield* matches(text); + return; + } + const scanner = createScanner(text); + let complete: IdaDiagnosticMatch | undefined; + let ended = false; + for (const match of matches(text)) { + while (!ended && (complete === undefined || complete.end <= match.start)) { + complete = undefined; + while (scanner.scan() !== JSON_EOF) { + if (scanner.getToken() !== JSON_STRING_LITERAL) continue; + const start = scanner.getTokenOffset(); + complete = { start, end: start + scanner.getTokenLength() }; + break; + } + if (complete === undefined) ended = true; + } + // Only a complete string in a valid document owns decoded-only semantics. + // Raw matches crossing tokens remain visible to the credential boundary. + if ( + complete === undefined || + match.start < complete.start || + match.end > complete.end + ) + yield match; + } +} + +function* mergeMatches( + left: Iterable, + right: Iterable, +): Generator { + const leftIterator = left[Symbol.iterator](); + const rightIterator = right[Symbol.iterator](); + try { + let leftMatch = leftIterator.next(); + let rightMatch = rightIterator.next(); + while (!leftMatch.done || !rightMatch.done) { + if ( + rightMatch.done || + (!leftMatch.done && leftMatch.value.start <= rightMatch.value.start) + ) { + if (leftMatch.done) break; + yield leftMatch.value; + leftMatch = leftIterator.next(); + } else { + yield rightMatch.value; + rightMatch = rightIterator.next(); + } + } + } finally { + try { + leftIterator.return?.(); + } finally { + rightIterator.return?.(); + } + } +} + +/** Redact complete or malformed JSON diagnostics without rewriting unrelated source. */ +export const redactIdaJsonDiagnostic = ( + text: string, + matches: Matches, + replacement: string = IDA_REDACTION_MARKER, +): string => { + let validDocument = false; + try { + JSON.parse(text); + validDocument = true; + } catch { + // Keep both raw and decoded interpretations when the document is incomplete. + } + return writeIdaDiagnostic( + text, + mergeMatches( + literalMatches(text, matches, validDocument), + decodedMatches(text, matches), + ), + replacement, + ); +}; + +/** Untyped producer text retains both literal and JSON character interpretations. */ +export const redactIdaUnclassifiedDiagnostic = ( + text: string, + matches: Matches, + replacement: string = IDA_REDACTION_MARKER, +): string => + writeIdaDiagnostic( + text, + mergeMatches(matches(text), decodedMatches(text, matches)), + replacement, + ); diff --git a/src/ida/IdaMcpConnection.test.ts b/src/ida/IdaMcpConnection.test.ts index a582ed902..139c377e2 100644 --- a/src/ida/IdaMcpConnection.test.ts +++ b/src/ida/IdaMcpConnection.test.ts @@ -1,31 +1,68 @@ -import { describe, expect, it } from "vitest"; +import { inspect } from "node:util"; +import { SdkErrorCode, SdkHttpError } from "@modelcontextprotocol/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { + createIdaMcpConnection, decodeIdaToolResult, - redactIdaTransportFailure, } from "./IdaMcpConnection.js"; +import { createIdaDiagnosticRedactor } from "./IdaDiagnosticRedactor.js"; describe("upstream IDA tool result decoding", () => { - it("redacts configured transport authentication without redacting caller evidence", () => { - const error = redactIdaTransportFailure( - new Error( - "Invalid Authorization value Bearer fixture-secret at /caller/input", - ), - { - url: "http://127.0.0.1:12345/mcp", - headers: { Authorization: "Bearer fixture-secret" }, - mode: "attached", - timeoutMs: 1000, - }, - ); - expect(error.message).not.toContain("fixture-secret"); - expect(error.message).toContain("/caller/input"); + it("leaves successful legacy, structured, and native evidence unchanged", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Bearer fixture-secret" }, + mode: "attached", + timeoutMs: 1000, + }); + const evidence = "Caller-selected fixture-secret evidence"; expect( - decodeIdaToolResult({ - content: [ - { type: "text", text: "Caller-selected fixture-secret evidence" }, - ], - }), - ).toBe("Caller-selected fixture-secret evidence"); + decodeIdaToolResult( + { content: [{ type: "text", text: evidence }] }, + undefined, + redact, + ), + ).toBe(evidence); + expect( + decodeIdaToolResult( + { content: [], structuredContent: { evidence } }, + undefined, + redact, + ), + ).toEqual({ evidence }); + expect( + decodeIdaToolResult( + { content: [{ type: "text", text: JSON.stringify({ evidence }) }] }, + "native", + redact, + ), + ).toEqual({ evidence }); + }); + it("redacts credentials in truncated-output diagnostics before constructing the error", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Bearer fixture-secret" }, + mode: "attached", + timeoutMs: 1000, + }); + expect(() => + decodeIdaToolResult( + { + content: [], + _meta: { + ida_mcp: { + output_truncated: true, + download_hint: + "Download /caller/output?cursor=kept using Bearer fixture-secret", + }, + }, + }, + undefined, + redact, + ), + ).toThrow( + "Download /caller/output?cursor=kept using [redacted transport credential]", + ); }); it("decodes scalar wrappers and legacy text without dropping structured dictionaries", () => { expect( @@ -79,3 +116,208 @@ describe("upstream IDA tool result decoding", () => { ).toThrow("Download the full upstream output"); }); }); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +const failingHttpConnection = ( + credential: string, + response: Response | Error, +) => { + // Exercise the real SDK consumer with an explicitly simulated Fetch boundary. + vi.stubGlobal( + "fetch", + response instanceof Error + ? vi.fn().mockRejectedValue(response) + : vi.fn().mockResolvedValue(response), + ); + return createIdaMcpConnection( + { + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Custom " + credential }, + mode: "attached", + timeoutMs: 1000, + }, + {}, + ); +}; + +const inspectRetainedError = (error: unknown): string => + inspect(error, { depth: null, maxStringLength: null, breakLength: Infinity }); + +describe("retained IDA diagnostic errors", () => { + it.each([ + { + credential: "fixture-secret", + body: '{"reason":"fixture-secret at /caller/input",', + forbidden: "fixture-secret", + }, + { + credential: "", + body: String.raw`["\u003ctoken\u003e",x]`, + forbidden: "u003ctoken", + }, + ])( + "retains only sanitized malformed HTTP JSON: $credential", + async ({ credential, body, forbidden }) => { + const connection = failingHttpConnection( + credential, + new Response(body, { headers: { "Content-Type": "application/json" } }), + ); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected a malformed HTTP JSON failure"); + expect(failure.message).toContain("[redacted transport credential]"); + expect(inspectRetainedError(failure)).not.toContain(forbidden); + expect(inspectRetainedError(failure)).not.toContain(credential); + await connection.close(); + }, + ); + + it("does not rescan an owned malformed JSON replacement marker", async () => { + const connection = failingHttpConnection( + "redacted", + new Response("Rejected redacted", { + headers: { "Content-Type": "application/json" }, + }), + ); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected a malformed HTTP JSON failure"); + expect(failure.message).toBe( + "IDA MCP returned malformed HTTP JSON. Response body: \nRejected ␟", + ); + expect(inspectRetainedError(failure)).not.toContain("[[redacted"); + await connection.close(); + }); + + it("replaces an unknown network error graph with a safe owned reason", async () => { + const upstream = Object.assign( + new Error("fixture-secret refused at /caller/input", { + cause: new Error("nested fixture-secret"), + }), + { headers: { Authorization: "Custom fixture-secret" } }, + ); + const connection = failingHttpConnection("fixture-secret", upstream); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected a network failure"); + expect(failure.message).toContain("refused at /caller/input"); + expect(failure.cause).not.toBe(upstream); + expect(inspectRetainedError(failure)).not.toContain("fixture-secret"); + await connection.close(); + }); + + it("retains meaningful HTTP status without unsafe SDK metadata", async () => { + const connection = failingHttpConnection( + "fixture-secret", + new Response("unavailable fixture-secret at /caller/input", { + status: 500, + statusText: "fixture-secret", + headers: { "Content-Type": "text/plain" }, + }), + ); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected an HTTP failure"); + expect(failure.cause).toMatchObject({ data: { status: 500 } }); + expect(failure.message).toContain( + "unavailable [redacted transport credential] at /caller/input", + ); + expect(inspectRetainedError(failure)).not.toContain("fixture-secret"); + await connection.close(); + }); +}); + +it.each([ + { + code: SdkErrorCode.ClientHttpNotImplemented, + message: "Changed SDK context: fixture-secret", + }, + { + code: SdkErrorCode.ClientHttpFailedToOpenStream, + message: "Error POSTing to endpoint: fixture-secret", + }, +])( + "does not infer owned body spans from an unknown SDK shape: $code / $message", + async ({ code, message }) => { + const connection = failingHttpConnection( + "fixture-secret", + new SdkHttpError(code, message, { status: 500, text: "fixture-secret" }), + ); + try { + const failure = await connection + .connect() + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected SDK failure"); + expect(inspectRetainedError(failure)).not.toContain("fixture-secret"); + expect(failure.message).toContain("[redacted transport credential]"); + expect(failure.cause).toMatchObject({ code, data: { status: 500 } }); + } finally { + await connection.close(); + } + }, +); + +it("redacts JSON character candidates in unclassified SDK errors", async () => { + const connection = failingHttpConnection( + 'q"uote', + new Error(String.raw`network context {"error":"q\"uote"} at /caller/input`), + ); + try { + const failure = await connection.connect().catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected SDK failure"); + expect(failure.message).toContain( + '{"error":"[redacted transport credential]"} at /caller/input', + ); + expect(inspectRetainedError(failure)).not.toContain(String.raw`q\"uote`); + } finally { + await connection.close(); + } +}); + +it("owns unclassified truncated-output hints separately from their context", () => { + const diagnostic = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Authorization: 'Custom q"uote' }, + }); + let failure: unknown; + try { + decodeIdaToolResult( + { + content: [], + _meta: { + ida_mcp: { + output_truncated: true, + download_hint: String.raw`{"error":"q\"uote","path":"/caller/output"}`, + }, + }, + }, + undefined, + diagnostic, + ); + } catch (cause: unknown) { + failure = cause; + } + if (!(failure instanceof Error)) + throw new Error("Expected truncation failure"); + expect(failure.message).toContain( + '{"error":"[redacted transport credential]","path":"/caller/output"}', + ); + expect(inspectRetainedError(failure)).not.toContain(String.raw`q\"uote`); +}); diff --git a/src/ida/IdaMcpConnection.ts b/src/ida/IdaMcpConnection.ts index 8137edcc9..1db07b861 100644 --- a/src/ida/IdaMcpConnection.ts +++ b/src/ida/IdaMcpConnection.ts @@ -1,8 +1,16 @@ +import { + createIdaDiagnosticRedactor, + IDA_DIAGNOSTIC_SOURCE_BYTES, +} from "./IdaDiagnosticRedactor.js"; +import { IdaDiagnosticResourceError } from "./IdaDiagnosticWriter.js"; import { ChildProcess } from "node:child_process"; import { Socket } from "node:net"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { Client, + ProtocolError, + SdkErrorCode, + SdkHttpError, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client"; import { @@ -12,7 +20,11 @@ import { import { z } from "zod"; import type { IdaConfiguration } from "./IdaConfiguration.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; -import { AnalysisProtocolError } from "../domain/analysisErrorCore.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { + AnalysisProtocolError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; /** Injectable MCP boundary used by the provider and transport conformance tests. */ @@ -53,14 +65,14 @@ const toolResultSchema = z.object({ .optional(), }); +const IDA_STDIO_MAX_MESSAGE_BYTES = 64 * 1024 * 1024; + const parseIdaJsonText = (text: string): JsonValue => { let value: unknown; try { value = JSON.parse(text); - } catch (cause: unknown) { - throw new AnalysisProtocolError("IDA MCP returned malformed JSON text.", { - cause, - }); + } catch { + throw new AnalysisProtocolError("IDA MCP returned malformed JSON text."); } return jsonValueSchema.parse(value); }; @@ -69,18 +81,30 @@ const parseIdaJsonText = (text: string): JsonValue => { export const decodeIdaToolResult = ( input: unknown, protocol?: "native", + diagnostic: IdaDiagnosticRedactor = createIdaDiagnosticRedactor(), ): JsonValue => { const result = toolResultSchema.parse(input); const blocks = result.content.flatMap((block) => block.type === "text" && block.text !== undefined ? [block.text] : [], ); - if (result.isError === true) + if (result.isError === true) { + const reason = diagnostic.blocks(blocks); throw new AnalysisProtocolError( - `IDA MCP tool failed: ${blocks.join("\n")}`, + diagnostic.message("IDA MCP tool failed: ", reason), + { + diagnostics: { reason }, + }, ); + } if (result._meta?.ida_mcp?.output_truncated === true) throw new AnalysisProtocolError( - `IDA MCP truncated its output; the preview cannot establish a complete observation. ${result._meta.ida_mcp.download_hint ?? "Use the upstream output download workflow to retrieve the full result."}`, + diagnostic.message( + "IDA MCP truncated its output; the preview cannot establish a complete observation. ", + diagnostic.blocks([ + result._meta.ida_mcp.download_hint ?? + "Use the upstream output download workflow to retrieve the full result.", + ]), + ), ); // Native MCP retains the complete JSON text while structuredContent may // contain a shortened preview. Parse that producer-owned representation once. @@ -113,59 +137,264 @@ export const decodeIdaToolResult = ( return values[0] ?? null; }; -/** Redact configured transport authentication values from SDK diagnostics only. */ -export const redactIdaTransportFailure = ( - cause: unknown, - config: IdaConfiguration, -): Error => { - let message = - cause instanceof Error - ? cause.message - : "Unknown IDA MCP transport failure"; - if ("headers" in config) { - for (const [name, value] of Object.entries(config.headers)) { - if (!/^(?:authorization|proxy-authorization|cookie)$/iu.test(name)) - continue; - const credential = value.match(/^(?:Bearer|Basic)\s+([\s\S]+)$/iu)?.[1]; - for (const secret of [value, credential]) - if (secret !== undefined && secret.length > 0) - message = message - .split(secret) - .join("[redacted transport credential]"); - } +// Retain only producer-sanitized JSON diagnostics, never a native parser cause. +class IdaHttpJsonError extends Error { + constructor(text: string, redactor: IdaDiagnosticRedactor) { + super( + redactor.message( + "IDA MCP returned malformed HTTP JSON. Response body: ", + text, + ), + ); } - return new Error(message, { cause }); +} + +type IdaDiagnosticRedactor = ReturnType; + +type IdaContentType = Readonly<{ + essence: string; + subtype: string; + charset: string | undefined; + parameters: ReadonlyMap; +}>; + +const httpQuotedOctet = (code: number): boolean => + code === 9 || + code === 32 || + (code >= 33 && code <= 126) || + (code >= 128 && code <= 255); + +/** Consume one complete RFC 9110 media type; never discard a malformed parameter. */ +const parseIdaContentType = ( + text: string, +): Result => { + let cursor = 0; + const token = new RegExp("[!#$%&'*+.^_`|~0-9A-Za-z-]+", "y"); + const readToken = (): string | undefined => { + token.lastIndex = cursor; + const match = token.exec(text); + if (match === null) return undefined; + cursor = token.lastIndex; + return match[0]; + }; + const readOws = (): void => { + while (text[cursor] === " " || text[cursor] === "\t") cursor++; + }; + const readQuoted = (): string | undefined => { + cursor++; + let value = ""; + while (cursor < text.length) { + let code = text.charCodeAt(cursor++); + if (code === 34) return value; + if (code === 92) { + code = text.charCodeAt(cursor++); + if (!httpQuotedOctet(code)) return undefined; + } else if (!httpQuotedOctet(code)) return undefined; + value += String.fromCharCode(code); + } + return undefined; + }; + readOws(); + const type = readToken()?.toLowerCase(); + if (type === undefined || text[cursor++] !== "/") return err("malformed"); + const subtype = readToken()?.toLowerCase(); + if (subtype === undefined) return err("malformed"); + const parameters = new Map(); + let ambiguous = false; + while (true) { + readOws(); + if (cursor === text.length) break; + if (text[cursor++] !== ";") return err("malformed"); + readOws(); + // RFC 9110 parameters admit an empty item after a semicolon. + if (cursor === text.length || text[cursor] === ";") continue; + const name = readToken()?.toLowerCase(); + if (name === undefined || text[cursor++] !== "=") return err("malformed"); + const value = text[cursor] === '"' ? readQuoted() : readToken(); + if (value === undefined) return err("malformed"); + // Repetition is ambiguous for this owner, even if both values agree. + // Keep parsing so syntax errors anywhere in the field cannot be discarded. + if (parameters.has(name)) ambiguous = true; + else parameters.set(name, value); + } + return ambiguous + ? err("ambiguous") + : ok({ + essence: type + "/" + subtype, + subtype, + charset: parameters.get("charset"), + parameters, + }); +}; +const httpErrorBody = ( + response: Response, + text: string, + redactor: IdaDiagnosticRedactor, +): string => { + const omitted = redactor.text( + "IDA MCP response body omitted: unsupported or unknown format or character encoding.", + ); + const contentType = response.headers.get("content-type"); + if (contentType === null) return omitted; + const mediaType = parseIdaContentType(contentType); + if (!mediaType.ok) return omitted; + const charset = mediaType.value.charset?.toLowerCase(); + if ( + charset !== undefined && + !["utf-8", "utf8", "us-ascii", "ascii"].includes(charset) + ) + return omitted; + if ( + mediaType.value.subtype === "json" || + mediaType.value.subtype.endsWith("+json") + ) + return redactor.json(text); + if (mediaType.value.essence === "text/plain") return redactor.text(text); + return omitted; }; +/** Keep error-body materialization within the diagnostic source-byte budget. */ +const readBoundedResponseText = async ( + response: Response, +): Promise => { + if (response.body === null) return ""; + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) return Buffer.concat(chunks, total).toString("utf8"); + total += value.byteLength; + if (total > IDA_DIAGNOSTIC_SOURCE_BYTES) { + await reader.cancel().catch(() => undefined); + return undefined; + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } +}; + +const diagnosticFetch = + (redactor: IdaDiagnosticRedactor): typeof fetch => + async (input, init) => { + const response = await fetch(input, init); + const readText = response.text.bind(response); + let safeBody: string | undefined; + if (!response.ok) { + // Normalize before the SDK can swallow a body-consumer failure or drop + // its format metadata. Both consumers use this one owned diagnostic. + const sourceBody = await readBoundedResponseText(response); + safeBody = + sourceBody === undefined + ? redactor.text( + `IDA MCP response body omitted: exceeds ${IDA_DIAGNOSTIC_SOURCE_BYTES} UTF-8 bytes.`, + ) + : httpErrorBody(response, sourceBody, redactor); + const body = safeBody; + Object.defineProperty(response, "text", { + value: async (): Promise => body, + }); + } + // Successful JSON remains lazy, and SSE keeps its original body stream. + Object.defineProperty(response, "json", { + value: async (): Promise => { + const text = safeBody ?? (await readText()); + try { + const value: unknown = JSON.parse(text); + return value; + } catch { + // Parsing failure owns diagnostics even when the HTTP status succeeded. + throw new IdaHttpJsonError( + safeBody ?? httpErrorBody(response, text, redactor), + redactor, + ); + } + }, + }); + return response; + }; + +const retainedTransportCause = (cause: unknown, message: string): Error => { + // Do not clone arbitrary upstream fields, stacks, data or nested causes. + if (cause instanceof SdkHttpError) + return new SdkHttpError(cause.code, message, { + status: cause.data.status, + statusText: "", + }); + if (cause instanceof ProtocolError) + return new ProtocolError(cause.code, message); + return new Error(message); +}; /** Create a standard SDK connection to an existing upstream registration. */ export const createIdaMcpConnection = ( config: IdaConfiguration, environment: Readonly, ): IdaMcpConnection => { const client = new Client({ name: "rea-ida-provider", version: "1" }); + const redactor = createIdaDiagnosticRedactor(config); let stderr = Buffer.alloc(0); let stderrBytes = 0; - const transportFailure = (cause: unknown): never => { - const failure = redactIdaTransportFailure(cause, config); - if (stderrBytes === 0) throw failure; + const transportFailure = ( + cause: unknown, + operation = "ida_mcp_response", + ): never => { + const bufferLimit = + cause instanceof Error + ? /ReadBuffer exceeded maximum size of (\d+) bytes/u.exec(cause.message) + : null; + if (cause instanceof IdaDiagnosticResourceError) throw cause; + let failure: Error; + if (cause instanceof IdaHttpJsonError) + failure = new Error(cause.message, { cause }); + else { + const message = + cause instanceof Error + ? cause.message + : "Unknown IDA MCP transport failure"; + const body = cause instanceof SdkHttpError ? cause.data.text : undefined; + // Only the pinned SDK POST producer places this owned body at a known + // span. Equal substrings elsewhere remain untrusted SDK context. + const prefix = "Error POSTing to endpoint: "; + const ownedPostBody = + cause instanceof SdkHttpError && + cause.code === SdkErrorCode.ClientHttpNotImplemented && + typeof body === "string" && + message === prefix + body; + // Normalized wire headers and supported decoded candidates have no LF. + // Separating owners prevents their concatenation from spelling a credential. + const safeMessage = ownedPostBody + ? redactor.message(prefix, body) + : redactor.blocks([message]); + failure = new Error(safeMessage, { + cause: retainedTransportCause(cause, safeMessage), + }); + } let start = 0; - // The byte tail may start inside a UTF-8 code point. Retain whole units. if (stderrBytes > stderr.length) while (start < stderr.length && (stderr[start]! & 0xc0) === 0x80) start++; - const diagnostic = redactIdaTransportFailure( - new Error(stderr.subarray(start).toString("utf8")), - config, - ).message; + const capturedOutput = { + stdout: "", + stderr: redactor.blocks([stderr.subarray(start).toString("utf8")]), + truncated: stderrBytes > stderr.length - start, + stdout_bytes: 0, + stderr_bytes: stderrBytes, + }; + if (bufferLimit !== null) + throw new AnalysisResourceConstraintError( + operation, + "transport", + `Upstream IDA MCP reply exceeded the ${bufferLimit[1]}-byte transport limit; no partial Evidence was returned.`, + { max_response_bytes: Number(bufferLimit[1]) }, + { cause, capturedOutput }, + ); + if (stderrBytes === 0) throw failure; throw new ProviderAdapterError("ida", "transport", { cause: failure, diagnostics: { reason: failure.message }, - capturedOutput: { - stdout: "", - stderr: diagnostic, - truncated: stderrBytes > stderr.length - start, - stdout_bytes: 0, - stderr_bytes: stderrBytes, - }, + capturedOutput, }); }; const transport = @@ -188,9 +417,11 @@ export const createIdaMcpConnection = ( ...snapshotEnvironment(config.env), }, stderr: "pipe", + maxBufferSize: IDA_STDIO_MAX_MESSAGE_BYTES, }) : new StreamableHTTPClientTransport(new URL(config.url), { requestInit: { headers: config.headers, redirect: "error" }, + fetch: diagnosticFetch(redactor), }); // Keep a bounded diagnostic tail for genuine failures; never persist ambient variables. if (transport instanceof StdioClientTransport) @@ -217,8 +448,9 @@ export const createIdaMcpConnection = ( return decodeIdaToolResult( await client .callTool({ name, arguments: args }, { timeout: config.timeoutMs }) - .catch(transportFailure), + .catch((cause: unknown) => transportFailure(cause, name)), config.protocol, + redactor, ); }, async readResource(uri) { diff --git a/src/ida/IdaMcpEnvironment.test.ts b/src/ida/IdaMcpEnvironment.test.ts index b744779ae..5e7a746d0 100644 --- a/src/ida/IdaMcpEnvironment.test.ts +++ b/src/ida/IdaMcpEnvironment.test.ts @@ -43,12 +43,23 @@ input.on('line', (line) => { HOME: "/selected-home", PATH: "/selected-path", }; + const registrationEnvironment: Record = { + REA_REGISTRATION_CONTEXT: "registration", + }; + if (process.platform === "win32") { + const systemRoot = process.env.SYSTEMROOT; + if (systemRoot === undefined || systemRoot.length === 0) + throw new Error( + "The Windows Node fixture requires its actual SYSTEMROOT.", + ); + registrationEnvironment.SYSTEMROOT = systemRoot; + } for (const selected of [environment, {}]) { const connection = createIdaMcpConnection( { command: process.execPath, args: [script], - env: { REA_REGISTRATION_CONTEXT: "registration" }, + env: registrationEnvironment, mode: "attached", timeoutMs: 5000, }, diff --git a/src/ida/IdaOperationFailures.test.ts b/src/ida/IdaOperationFailures.test.ts index 1f8a83c6b..8886f6861 100644 --- a/src/ida/IdaOperationFailures.test.ts +++ b/src/ida/IdaOperationFailures.test.ts @@ -1,5 +1,7 @@ -import { rm } from "node:fs/promises"; +import { rm, stat } from "node:fs/promises"; +import { dirname } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { z } from "zod"; import { createIdaTarget, RecordingIdaMcp, @@ -7,6 +9,7 @@ import { import type { JsonValue } from "../domain/jsonValue.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { createIdaDiagnosticRedactor } from "./IdaDiagnosticRedactor.js"; import { decodeIdaToolResult } from "./IdaMcpConnection.js"; import type { AnalysisOperation } from "../application/AnalysisProvider.js"; import { IdaSessionClient } from "./IdaSessionClient.js"; @@ -17,6 +20,193 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }); }); +it("preserves a rejected headless decompile diagnostic with completed lookup evidence and releases its owned worker", async () => { + const { root, target } = await createIdaTarget(); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + const reason = + "Invalid params: unexpected parameters: ['offset', 'max_lines']"; + const config = { + url: "http://127.0.0.1:12345/mcp", + headers: { + Authorization: "Token custom-fixture-token", + Cookie: "session=cookie-fixture-secret", + }, + mode: "headless" as const, + timeoutMs: 1000, + workspaceRoot: root, + }; + const redactor = createIdaDiagnosticRedactor(config); + producer.beforeCall = async (name) => { + if (name === "decompile") + decodeIdaToolResult( + { + isError: true, + content: [ + { + type: "text", + text: reason + " (custom-fixture-token / cookie-fixture-secret)", + }, + ], + }, + undefined, + redactor, + ); + }; + const safeReason = + reason + + " ([redacted transport credential] / [redacted transport credential])"; + const client = new IdaSessionClient(config, target, producer); + try { + const result = await client.execute("procedure_pseudo_code", { + procedure: "main", + }); + if (result.ok) throw new Error("Expected the upstream rejection"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "unreadable_output", + details: { + diagnostics: { reason: safeReason }, + partial_observation: { + provider: { id: "ida" }, + operation: "procedure_pseudo_code", + normalized_result: { + status: "partial", + observations: [expect.objectContaining({ tool: "lookup_funcs" })], + target_binding: { + status: "not_verified", + reason: "IDA MCP tool failed: \n" + safeReason, + }, + }, + }, + }, + }); + expect(JSON.stringify(projectAnalysisError(result.error))).not.toContain( + "custom-fixture-token", + ); + expect(JSON.stringify(projectAnalysisError(result.error))).not.toContain( + "cookie-fixture-secret", + ); + expect( + producer.calls.find(({ name }) => name === "decompile"), + ).toMatchObject({ + args: { addr: "0x1000", offset: 0, max_lines: 100 }, + }); + } finally { + const database = producer.database; + expect((await client.close()).ok).toBe(true); + expect(producer.calls.filter(({ name }) => name === "idb_close")).toEqual([ + { name: "idb_close", args: { database, save: false } }, + ]); + expect(producer.closes).toBe(1); + await expect(stat(dirname(producer.inputPath))).rejects.toMatchObject({ + code: "ENOENT", + }); + } +}); + +describe("headless IDA tool diagnostics in partial evidence", () => { + const callerPath = "/caller/input?token=visible"; + const marker = "[redacted transport credential]"; + it.each([ + { + name: "a decoded Basic password matching the tool failure prefix", + password: "failed", + upstream: "Rejected decompile request for " + callerPath, + safeReason: "Rejected decompile request for " + callerPath, + forbiddenInBinding: ["failed"], + }, + { + name: "a JSON-encoded tool explanation", + password: "", + upstream: String.raw`{"reason":"\u003csecret\u003e","user":"user","path":"/caller/input?token=visible"}`, + safeReason: + '{"reason":"' + + marker + + '","user":"user","path":"/caller/input?token=visible"}', + forbiddenInBinding: ["", String.raw`\u003csecret\u003e`], + }, + ])( + "redacts $name before retaining the public target-binding reason", + async ({ password, upstream, safeReason, forbiddenInBinding }) => { + const { root, target } = await createIdaTarget(); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + const config = { + url: "http://127.0.0.1:12345/mcp", + headers: { + Authorization: + "Basic " + + Buffer.from("user:" + password, "utf8").toString("base64"), + }, + mode: "headless" as const, + timeoutMs: 1000, + workspaceRoot: root, + }; + const redactor = createIdaDiagnosticRedactor(config); + producer.beforeCall = async (name) => { + if (name === "decompile") + decodeIdaToolResult( + { isError: true, content: [{ type: "text", text: upstream }] }, + undefined, + redactor, + ); + }; + const client = new IdaSessionClient(config, target, producer); + try { + const result = await client.execute("procedure_pseudo_code", { + procedure: "main", + }); + if (result.ok) throw new Error("Expected the producer's tool error"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "unreadable_output", + details: { + partial_observation: { + provider: { id: "ida" }, + operation: "procedure_pseudo_code", + normalized_result: { + status: "partial", + observations: [ + expect.objectContaining({ tool: "lookup_funcs" }), + ], + target_binding: { + status: "not_verified", + reason: expect.any(String), + }, + }, + }, + }, + }); + const details = z + .object({ + diagnostics: z.object({ reason: z.string() }), + partial_observation: z.object({ + normalized_result: z.object({ + target_binding: z.object({ reason: z.string() }), + }), + }), + }) + .parse(projected.details); + const bindingReason = + details.partial_observation.normalized_result.target_binding.reason; + for (const credential of forbiddenInBinding) + expect.soft(bindingReason).not.toContain(credential); + expect.soft(bindingReason).toContain(marker); + expect(bindingReason).toContain(callerPath); + expect.soft(details.diagnostics.reason).toBe(safeReason); + } finally { + expect((await client.close()).ok).toBe(true); + expect( + producer.calls.filter(({ name }) => name === "idb_close"), + ).toHaveLength(1); + await expect(stat(dirname(producer.inputPath))).rejects.toMatchObject({ + code: "ENOENT", + }); + } + }, + ); +}); + describe("headless IDA partial operation evidence", () => { it("retains completed rows when post-operation target identity changes", async () => { const { root, target } = await createIdaTarget(); diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index 576dac1e7..252bdbe3a 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -311,14 +311,12 @@ export class IdaOperationRunner { const current: number = offset; if (this.protocol === "legacy") { const page = legacyPageSchema.parse( - await this.call("list_strings", { offset, count: 100 }), - ); - result.push( - ...page.data.map((item) => { - const s = legacyStringSchema.parse(item); - return { address: s.address, value: s.string }; - }), + await this.call("list_strings", { offset, count: 0 }), ); + for (const item of page.data) { + const s = legacyStringSchema.parse(item); + result.push({ address: s.address, value: s.string }); + } offset = page.next_offset; } else { const page = ( diff --git a/src/ida/IdaProvider.ts b/src/ida/IdaProvider.ts index 5ce924fcd..d2bf97060 100644 --- a/src/ida/IdaProvider.ts +++ b/src/ida/IdaProvider.ts @@ -68,13 +68,17 @@ export class IdaProvider implements AnalysisProviderCandidate { if (!privacy.available) return { status: "unavailable", - code: "unsupported_host", + code: privacy.code, reason: privacy.reason, diagnostics: { configured: true, mode: "headless", live_connection_probed: false, private_workspace_available: false, + windows_native_failure: { + stage: privacy.failureStage, + remediation: privacy.remediation, + }, }, }; } diff --git a/src/ida/IdaProviderCapabilities.ts b/src/ida/IdaProviderCapabilities.ts index 0ccc016d2..1394310d9 100644 --- a/src/ida/IdaProviderCapabilities.ts +++ b/src/ida/IdaProviderCapabilities.ts @@ -32,7 +32,6 @@ export type IdaOperation = (typeof IDA_OPERATIONS)[number]; /** Operations only the modern headless profile implements. */ const HEADLESS_ONLY_OPERATIONS: ReadonlySet = new Set([ "resolve_containing_procedure", - "list_documents", ]); /** Whether the configured profile admits an operation. */ diff --git a/src/ida/IdaSessionClient.test.ts b/src/ida/IdaSessionClient.test.ts index 1a35dd7c7..3d0feeed2 100644 --- a/src/ida/IdaSessionClient.test.ts +++ b/src/ida/IdaSessionClient.test.ts @@ -1,6 +1,6 @@ import { readFile, readdir, rm } from "node:fs/promises"; import { sep } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { z } from "zod"; import { createIdaTarget, @@ -11,11 +11,34 @@ import { functionDossierSchema } from "../domain/hopperValues.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { WindowsPrivateRuntime } from "../windows/WindowsPrivateRuntime.js"; const roots: string[] = []; +const windowsRuntimes = new Set(); +beforeEach(() => { + if (process.platform !== "win32") return; + const create = WindowsPrivateRuntime.create; + vi.spyOn(WindowsPrivateRuntime, "create").mockImplementation( + (parent, prefix) => { + const runtime = create(parent, prefix); + windowsRuntimes.add(runtime); + return runtime; + }, + ); +}); afterEach(async () => { - for (const root of roots.splice(0)) - await rm(root, { recursive: true, force: true }); + try { + // These pure producer fixtures spawn no IDA worker. After the retention + // assertions, release only the actual native owners this test created. + for (const runtime of windowsRuntimes) { + await runtime.close(); + windowsRuntimes.delete(runtime); + } + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); + } finally { + vi.restoreAllMocks(); + } }); /** A failed close whose diagnostic names exactly these unverified sessions. */ const reportsSessions = (...sessions: readonly object[]) => ({ @@ -1495,6 +1518,38 @@ it("binds retained transport diagnostics to the requested operation and recovery await client.close(); }); +describe("attached IDA legacy string inventory", () => { + it("fetches legacy strings and literal searches in single upstream calls", async () => { + const { producer, client } = await fixture(); + const all = Array.from({ length: 250 }, (_, i) => ({ + address: `0x${(0x3000 + i).toString(16)}`, + length: 5, + string: `hello-${i}`, + })); + producer.overrideCall = (name, args) => { + if (name !== "list_strings") return undefined; + const offset = Number(args.offset ?? 0); + const count = Number(args.count ?? 100); + const data = count === 0 ? all : all.slice(offset, offset + count); + return { + data, + next_offset: + count === 0 || offset + count >= all.length ? null : offset + count, + }; + }; + const listed = await client.execute("list_strings", {}); + expect(listed.ok && listed.value.result).toHaveLength(250); + expect( + producer.calls.filter(({ name }) => name === "list_strings")[0]?.args + .count, + ).toBe(0); + expect( + producer.calls.filter(({ name }) => name === "list_strings"), + ).toHaveLength(1); + await client.close(); + }); +}); + it("retains transport diagnostics on a cancellation regardless of stderr", async () => { const { producer, client } = await fixture("attached"); await client.execute("health", {}); diff --git a/src/jeb/JebMcpEndpoint.test.ts b/src/jeb/JebMcpEndpoint.test.ts index 0252632d8..d07ad0d58 100644 --- a/src/jeb/JebMcpEndpoint.test.ts +++ b/src/jeb/JebMcpEndpoint.test.ts @@ -5,6 +5,7 @@ import { parseJebMcpEndpoint } from "./JebMcpEndpoint.js"; describe("JEB MCP endpoint", () => { it("rejects an explicit empty query or fragment instead of dropping it", () => { expect(parseJebMcpEndpoint("http://127.0.0.1:8425/mcp").ok).toBe(true); + expect(parseJebMcpEndpoint("http://127.0.0.1:8425/mcp/").ok).toBe(true); for (const value of [ "http://127.0.0.1:8425/mcp?", "http://127.0.0.1:8425/mcp#", @@ -12,4 +13,12 @@ describe("JEB MCP endpoint", () => { ]) expect(parseJebMcpEndpoint(value).ok, value).toBe(false); }); + + it("accepts only the declared MCP route", () => { + expect(parseJebMcpEndpoint("http://127.0.0.1:8425/mcp").ok).toBe(true); + for (const path of ["/mcp/extra", "/not-mcp", "/"]) + expect(parseJebMcpEndpoint(`http://127.0.0.1:8425${path}`).ok, path).toBe( + false, + ); + }); }); diff --git a/src/jeb/JebMcpEndpoint.ts b/src/jeb/JebMcpEndpoint.ts index bae4a00a9..97bca88f0 100644 --- a/src/jeb/JebMcpEndpoint.ts +++ b/src/jeb/JebMcpEndpoint.ts @@ -47,6 +47,12 @@ export const parseJebMcpEndpoint = ( "Embedding credentials in the JEB MCP endpoint is not supported; configure authentication with the engine", }), ); + if (url.pathname !== "/mcp" && url.pathname !== "/mcp/") + return err( + new AnalysisInputError("jeb_mcp_endpoint", { + cause: "JEB MCP endpoint must use the /mcp route", + }), + ); if (urlDeclaresQueryOrFragment(url)) return err( new AnalysisInputError("jeb_mcp_endpoint", { diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index 7d1eca0f7..e7dc2a515 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -544,6 +544,49 @@ it("reports WABT output beyond the budget as a resource constraint with a bounde signal: "SIGKILL", }); }); +it("does not attribute wasm-objdump output to a wasm2wat launch failure", async () => { + const selectedRoot = await createTestTempDirectory("rea-wabt-launch-"); + const path = join(selectedRoot, "selected.wasm"); + await writeFile(path, Buffer.alloc(76, 1)); + for (const tool of ["wasm-validate", "wasm-objdump", "wasm2wat"]) + await writeFile( + join(selectedRoot, tool + (process.platform === "win32" ? ".exe" : "")), + "source-owned executable seam", + { mode: 0o700 }, + ); + const dump = await readFile( + new URL("../../tests/fixtures/wasm/objdump.txt", import.meta.url), + "utf8", + ); + const provider = new WabtArtifactProvider( + { REA_WABT_BIN_DIRECTORY: selectedRoot }, + (input) => { + const version = input.arguments[0] === "--version"; + if (!version && input.command.includes("wasm2wat")) + throw Object.assign(new Error("spawn E2BIG"), { code: "E2BIG" }); + const script = version + ? 'process.stdout.write("1.0.42\\n")' + : input.command.includes("objdump") + ? `process.stdout.write(${JSON.stringify(dump)})` + : ""; + return spawnOwnedProviderProcess({ + ...input, + command: process.execPath, + arguments: ["-e", script], + expectedCommand: null, + }); + }, + ); + const result = await provider.inspect({ + path, + glue_paths: [], + candidate_paths: [], + }); + if (result.ok) throw new Error("Expected the wasm2wat launch to fail"); + expect(result.error.message).toContain("spawn E2BIG"); + expect(result.error.capturedOutput).toBeUndefined(); +}); + it("bounds an oversized stderr to an excerpt", () => { expect( projectAnalysisError( diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index d706c97e8..fae455c24 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -180,6 +180,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { args: readonly string[], acceptNonZeroExit = false, ) => { + capturedOutput = undefined; await verifySnapshot(); const snapshot = await runOwnedCommand( { diff --git a/src/windows/WindowsNativeLoader.test.ts b/src/windows/WindowsNativeLoader.test.ts index c81e8e9a8..9f2b5623d 100644 --- a/src/windows/WindowsNativeLoader.test.ts +++ b/src/windows/WindowsNativeLoader.test.ts @@ -111,24 +111,28 @@ describe("packaged Windows native admission", () => { }); it.each([ - { platform: "linux" as const }, - { architecture: "arm64" }, - { nodeApiVersion: "7" }, - { nodeApiVersion: undefined }, - { packageVersion: "2.0.0" }, - ])("rejects incompatible host metadata before loading: %j", (override) => { - const value = fixture(); - expect( - loadWindowsNativeAuthority({ ...value.host, ...override }).available, - ).toBe(false); - expect(value.loads()).toBe(0); - }); + [{ platform: "linux" as const }, "host"], + [{ architecture: "arm64" }, "host"], + [{ nodeApiVersion: "7" }, "host"], + [{ nodeApiVersion: undefined }, "host"], + [{ packageVersion: "2.0.0" }, "artifact"], + ] as const)( + "rejects incompatible host metadata before loading: %j", + (override, failureStage) => { + const value = fixture(); + expect( + loadWindowsNativeAuthority({ ...value.host, ...override }), + ).toMatchObject({ available: false, failureStage }); + expect(value.loads()).toBe(0); + }, + ); it("rejects tampering even when the original file was a valid PE image", () => { const value = fixture(); value.bytes[127] = 1; expect(loadWindowsNativeAuthority(value.host)).toMatchObject({ available: false, + failureStage: "artifact", reason: expect.stringContaining("SHA-256"), }); expect(value.loads()).toBe(0); @@ -190,6 +194,7 @@ describe("packaged Windows native admission", () => { }), ).toMatchObject({ available: false, + failureStage: "self-test", retryable: true, reason: expect.stringContaining("private DACL: access denied"), }); @@ -202,6 +207,7 @@ describe("packaged Windows native admission", () => { loadArtifact: () => ({ call: () => ({ privateDacl: true }) }), }); expect(loaded.available).toBe(false); + expect(loaded).toMatchObject({ failureStage: "inspection" }); expect(loaded).not.toHaveProperty("retryable"); }); diff --git a/src/windows/WindowsNativeLoader.ts b/src/windows/WindowsNativeLoader.ts index 13bf4e9f3..37620f53a 100644 --- a/src/windows/WindowsNativeLoader.ts +++ b/src/windows/WindowsNativeLoader.ts @@ -71,6 +71,7 @@ export type WindowsNativeLoadResult = | { readonly available: false; readonly reason: string; + readonly failureStage: "host" | "artifact" | "self-test" | "inspection"; /** Only a failed native invocation can recover on a later query. */ readonly retryable?: true; }; @@ -93,15 +94,20 @@ export const loadWindowsNativeAuthority = ( if (host.platform !== "win32" || host.architecture !== "x64") return { available: false, + failureStage: "host", reason: "REA Windows native controls require a Windows x64 host.", }; + const nodeApiVersion = Number(host.nodeApiVersion); + if (!Number.isSafeInteger(nodeApiVersion) || nodeApiVersion < 8) + return { + available: false, + failureStage: "host", + reason: "REA Windows native controls require Node-API 8 or newer.", + }; try { const manifest = windowsNativeManifestSchema.parse(host.readManifest()); if (manifest.packageVersion !== host.packageVersion) throw new Error("Windows native artifact belongs to another REA version"); - const nodeApiVersion = Number(host.nodeApiVersion); - if (!Number.isSafeInteger(nodeApiVersion) || nodeApiVersion < 8) - throw new Error("Windows native artifact requires Node-API 8 or newer"); const artifact = host.readArtifact(); if ( createHash("sha256").update(artifact).digest("hex") !== @@ -118,15 +124,24 @@ export const loadWindowsNativeAuthority = ( } catch (cause: unknown) { return { available: false, + failureStage: "self-test", retryable: true, reason: `REA Windows native self-test failed: ${cause instanceof Error ? cause.message : String(cause)}`, }; } - const inspection = inspectionSchema.parse(observed); + const parsed = inspectionSchema.safeParse(observed); + if (!parsed.success) + return { + available: false, + failureStage: "inspection", + reason: `REA Windows native inspection is invalid: ${parsed.error.message}`, + }; + const inspection = parsed.data; return { available: true, authority: { call: binding.call, inspection } }; } catch (cause: unknown) { return { available: false, + failureStage: "artifact", reason: `REA Windows native controls are unavailable: ${cause instanceof Error ? cause.message : String(cause)}`, }; } diff --git a/tests/boundary/filesystem/ghidraJavaProbe.test.ts b/tests/boundary/filesystem/ghidraJavaProbe.test.ts index 415fbf3dd..d48dc1ece 100644 --- a/tests/boundary/filesystem/ghidraJavaProbe.test.ts +++ b/tests/boundary/filesystem/ghidraJavaProbe.test.ts @@ -88,6 +88,109 @@ it.each([ }, ); +it.each([ + { label: "omitted", property: "", shown: "bitness unknown" }, + { + label: "empty", + property: "sun.arch.data.model =\n", + shown: 'bitness unknown (sun.arch.data.model = "")', + }, + { + label: "malformed", + property: "sun.arch.data.model = 64junk\n", + shown: 'bitness unknown (sun.arch.data.model = "64junk")', + }, + { + label: "not exactly representable", + property: "sun.arch.data.model = 9007199254740993\n", + shown: 'bitness unknown (sun.arch.data.model = "9007199254740993")', + }, +])("keeps $label Java bitness unresolved", async ({ property, shown }) => { + const root = await createTestTempDirectory("rea-java-bitness-"); + const configuredHome = join(root, "selected-jdk"); + const reportedHome = join(root, "reported-jdk"); + const installDir = join(root, "ghidra"); + await mkdir(join(installDir, "Ghidra"), { recursive: true }); + await mkdir(join(reportedHome, "bin"), { recursive: true }); + await writeFile( + join( + reportedHome, + "bin", + process.platform === "win32" ? "javac.exe" : "javac", + ), + "fixture", + { + mode: 0o755, + }, + ); + await writeFile( + join(installDir, "Ghidra", "application.properties"), + `application.version=${SUPPORTED_GHIDRA_VERSION}\napplication.java.min=17\n`, + ); + mockSpawnSync.mockReturnValue({ + stdout: "", + stderr: ` java.version = 21.0.11\n java.home = ${reportedHome}\n${property}`, + status: 0, + }); + + const inspection = inspectGhidraInstallation({ + environment: {}, + installDir, + javaHome: configuredHome, + }); + + expect(inspection.status).toBe("unavailable"); + expect(inspection.checks.find(({ name }) => name === "java")).toMatchObject({ + status: "failed", + code: "version_unresolved", + detail: expect.stringContaining(shown), + }); + expect( + inspection.checks.find(({ name }) => name === "java")?.detail, + ).not.toMatch(/(?:0-bit|9007199254740992-bit)/); +}); + +it("keeps a known 32-bit Java report classified as unsupported", async () => { + const root = await createTestTempDirectory("rea-java-bitness-known-"); + const configuredHome = join(root, "selected-jdk"); + const reportedHome = join(root, "reported-jdk"); + const installDir = join(root, "ghidra"); + await mkdir(join(installDir, "Ghidra"), { recursive: true }); + await mkdir(join(reportedHome, "bin"), { recursive: true }); + await writeFile( + join( + reportedHome, + "bin", + process.platform === "win32" ? "javac.exe" : "javac", + ), + "fixture", + { + mode: 0o755, + }, + ); + await writeFile( + join(installDir, "Ghidra", "application.properties"), + `application.version=${SUPPORTED_GHIDRA_VERSION}\napplication.java.min=17\n`, + ); + mockSpawnSync.mockReturnValue({ + stdout: "", + stderr: ` java.version = 21.0.11\n java.home = ${reportedHome}\n sun.arch.data.model = 32\n`, + status: 0, + }); + + const inspection = inspectGhidraInstallation({ + environment: {}, + installDir, + javaHome: configuredHome, + }); + + expect(inspection.checks.find(({ name }) => name === "java")).toMatchObject({ + status: "failed", + code: "unsupported_version", + detail: expect.stringContaining("32-bit"), + }); +}); + it("bounds the Java version probe and reports an over-budget probe as a failed Java check", async () => { const root = await createTestTempDirectory("rea-java-probe-bound-"); const configuredHome = join(root, "selected-jdk"); diff --git a/tests/boundary/filesystem/ghidraLanguageCatalogBound.test.ts b/tests/boundary/filesystem/ghidraLanguageCatalogBound.test.ts new file mode 100644 index 000000000..8555c1dbd --- /dev/null +++ b/tests/boundary/filesystem/ghidraLanguageCatalogBound.test.ts @@ -0,0 +1,86 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; + +import { readGhidraLanguageCatalog } from "../../../src/ghidra/GhidraLanguageCatalog.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const LIMIT = 4 * 1024 * 1024; +const OVERSIZED = "oversized.ldefs"; + +// Count bytes delivered from the oversized definition, whichever read API is used. +const { readBytes } = vi.hoisted(() => ({ readBytes: { value: 0 } })); +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + const tracked = (path: unknown) => String(path).endsWith(OVERSIZED); + return { + ...actual, + readFile: async (...args: Parameters) => { + const bytes = await actual.readFile(...args); + if (tracked(args[0])) readBytes.value += bytes.length; + return bytes; + }, + open: async (...args: Parameters) => { + const handle = await actual.open(...args); + if (!tracked(args[0])) return handle; + const read = handle.read.bind(handle) as ( + ...readArgs: unknown[] + ) => Promise<{ bytesRead: number }>; + return Object.assign(handle, { + read: async (...readArgs: unknown[]) => { + const result = await read(...readArgs); + readBytes.value += result.bytesRead; + return result; + }, + }); + }, + }; +}); + +afterEach(() => { + readBytes.value = 0; +}); + +it("skips an oversized definition without buffering it beyond the catalog limit", async () => { + const root = await createTestTempDirectory("rea-ghidra-ldefs-bound-"); + const module = (name: string) => + join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(module("huge"), { recursive: true }); + await mkdir(module("x86"), { recursive: true }); + await writeFile( + join(module("huge"), OVERSIZED), + Buffer.concat([ + Buffer.from( + '', + ), + Buffer.alloc(2 * LIMIT, 0x20), + ]), + ); + await writeFile( + join(module("x86"), "x86.ldefs"), + '', + ); + + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "x86:LE:32:default", compilerSpecIds: [] }, + ]); + expect(readBytes.value).toBeGreaterThan(0); + expect(readBytes.value).toBeLessThanOrEqual(LIMIT + 1); +}); + +it("keeps a definition at exactly the catalog limit", async () => { + const root = await createTestTempDirectory("rea-ghidra-ldefs-limit-"); + const languages = join(root, "Ghidra/Processors/edge/data/languages"); + await mkdir(languages, { recursive: true }); + const document = Buffer.from( + '', + ); + await writeFile( + join(languages, "edge.ldefs"), + Buffer.concat([document, Buffer.alloc(LIMIT - document.length, 0x20)]), + ); + + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "edge:LE:32:default", compilerSpecIds: [] }, + ]); +}); diff --git a/tests/boundary/filesystem/ghidraLanguageCatalogCleanup.test.ts b/tests/boundary/filesystem/ghidraLanguageCatalogCleanup.test.ts new file mode 100644 index 000000000..c00983765 --- /dev/null +++ b/tests/boundary/filesystem/ghidraLanguageCatalogCleanup.test.ts @@ -0,0 +1,203 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; + +import { + GhidraLanguageCatalogCleanupError, + readGhidraLanguageCatalog, +} from "../../../src/ghidra/GhidraLanguageCatalog.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +interface FileControl { + closeFails: boolean; + /** Hold close() until every synchronized handle has reached it. */ + synchronizedClose: boolean; + invalidateOnFailure: boolean; + readFails: boolean; + invalidated: boolean; + closeCalls: number; +} + +// Per-file failure controls, keyed by definition file name, and a barrier +// that releases synchronized closes only after all of them are pending. +const { controls, barrier } = vi.hoisted(() => { + const state = { expected: 0, arrived: 0, release: () => {} }; + const gate = new Promise((resolve) => { + state.release = resolve; + }); + return { + controls: new Map(), + barrier: { + state, + arrive: async () => { + state.arrived += 1; + if (state.arrived >= state.expected) state.release(); + await gate; + }, + }, + }; +}); +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: async (...args: Parameters) => { + const real = await actual.open(...args); + const control = controls.get(basename(String(args[0]))); + if (control === undefined) return real; + let released = false; + const release = async () => { + if (released) return; + released = true; + await real.close(); + }; + return { + get fd() { + return control.invalidated ? -1 : real.fd; + }, + read: (...readArgs: Parameters) => + control.readFails + ? Promise.reject(new Error("definition read failed")) + : real.read(...readArgs), + close: async () => { + control.closeCalls += 1; + if (control.synchronizedClose && control.closeCalls === 1) + await barrier.arrive(); + if (control.closeFails) { + if (control.invalidateOnFailure) { + await release(); + control.invalidated = true; + } + throw new Error(`definition close failed: ${String(args[0])}`); + } + await release(); + }, + }; + }, + }; +}); + +let preserveUnconfirmedOwner = false; + +const control = (name: string, overrides: Partial = {}) => { + const value: FileControl = { + closeFails: false, + synchronizedClose: false, + invalidateOnFailure: false, + readFails: false, + invalidated: false, + closeCalls: 0, + ...overrides, + }; + controls.set(name, value); + return value; +}; + +const installation = async (name: string): Promise => { + const root = await createTestTempDirectory("rea-ghidra-ldefs-cleanup-"); + const languages = join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(languages, { recursive: true }); + await writeFile( + join(languages, `${name}.ldefs`), + ``, + ); + return root; +}; + +// Let every retained owner close, then drain them through a healthy read. +afterEach(async () => { + if (preserveUnconfirmedOwner) { + preserveUnconfirmedOwner = false; + controls.clear(); + return; + } + for (const value of controls.values()) value.closeFails = false; + await readGhidraLanguageCatalog(await installation("drain")); + controls.clear(); +}); + +it("retries a retained close before the next read", async () => { + const retained = control("flaky.ldefs", { closeFails: true }); + await expect( + readGhidraLanguageCatalog(await installation("flaky")), + ).rejects.toMatchObject({ paths: [expect.stringContaining("flaky.ldefs")] }); + retained.closeFails = false; + expect(await readGhidraLanguageCatalog(await installation("x86"))).toEqual([ + { id: "x86:LE:32:default", compilerSpecIds: [] }, + ]); + expect(retained.closeCalls).toBe(2); +}); + +it("attempts every retained close and reports the remaining failures together", async () => { + const first = control("first.ldefs", { + closeFails: true, + synchronizedClose: true, + }); + const second = control("second.ldefs", { + closeFails: true, + synchronizedClose: true, + }); + barrier.state.expected = 2; + // Concurrent resolutions can each retain an owner. + const roots = [await installation("first"), await installation("second")]; + const results = await Promise.allSettled( + roots.map((root) => readGhidraLanguageCatalog(root)), + ); + expect(results.map(({ status }) => status)).toEqual(["rejected", "rejected"]); + second.closeFails = false; + const failure = await readGhidraLanguageCatalog( + await installation("x86"), + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(failure).toMatchObject({ + paths: [expect.stringContaining("first.ldefs")], + }); + expect(second.closeCalls).toBe(2); + expect(first.closeCalls).toBe(2); +}); + +it("keeps the read failure when the close also fails", async () => { + control("damaged.ldefs", { readFails: true, closeFails: true }); + const failure = await readGhidraLanguageCatalog( + await installation("damaged"), + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(failure).toMatchObject({ + readFailure: expect.objectContaining({ message: "definition read failed" }), + }); +}); + +it("reports retained cleanup for a catalog without definitions", async () => { + control("pending.ldefs", { closeFails: true }); + await expect( + readGhidraLanguageCatalog(await installation("pending")), + ).rejects.toBeInstanceOf(GhidraLanguageCatalogCleanupError); + const empty = await createTestTempDirectory("rea-ghidra-ldefs-empty-"); + await expect(readGhidraLanguageCatalog(empty)).rejects.toMatchObject({ + paths: [expect.stringContaining("pending.ldefs")], + }); +}); + +it("retains an invalidated descriptor owner and keeps later reads blocked", async () => { + const broken = control("broken.ldefs", { + closeFails: true, + invalidateOnFailure: true, + }); + const first = await readGhidraLanguageCatalog( + await installation("broken"), + ).catch((cause: unknown) => cause); + expect(first).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(first).toMatchObject({ + paths: [expect.stringContaining("broken.ldefs")], + }); + + const later = await readGhidraLanguageCatalog( + await installation("x86"), + ).catch((cause: unknown) => cause); + expect(later).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(later).toMatchObject({ + paths: [expect.stringContaining("broken.ldefs")], + }); + expect(broken.closeCalls).toBe(1); + preserveUnconfirmedOwner = true; +}); diff --git a/tests/boundary/filesystem/mipsAbiFlags.test.ts b/tests/boundary/filesystem/mipsAbiFlags.test.ts index 1b724d31b..e89f5cdbf 100644 --- a/tests/boundary/filesystem/mipsAbiFlags.test.ts +++ b/tests/boundary/filesystem/mipsAbiFlags.test.ts @@ -156,7 +156,10 @@ for (const little of [true, false]) { else bytes.writeUInt32BE(sectionOffset, 32); const resolved = await resolveFixture(bytes); assert.equal(resolved.mips.abiFlags?.fpAbi, 5); - assert.equal(ghidraMipsUnsupportedReason(resolved), null); + assert.match( + ghidraMipsUnsupportedReason(resolved) ?? "", + /ABI declaration inspection is incomplete/u, + ); assert.match( resolved.mips.limitations?.join(" ") ?? "", /section header table/u, @@ -172,7 +175,7 @@ it("keeps declarations in uncaptured section headers unknown when program ABI da assert.equal(resolved.mips.abiFlags, null); assert.match( ghidraMipsUnsupportedReason(resolved) ?? "", - /inspected ABI.*unavailable headers remain unknown/u, + /ABI declaration inspection is incomplete.*unavailable headers remain unknown/u, ); }); for (const failure of ["conflicting", "outside", "duplicate"]) { @@ -370,6 +373,26 @@ it("projects a malformed out-of-probe ABI record as a target error", async () => assert.deepEqual(await readFile(path), bytes); }); +for (const little of [true, false]) { + it(`refuses GNU ABI attributes by section type without reading attacker-controlled names or sizes (${little ? "LE" : "BE"})`, async () => { + const bytes = fixture(little); + const write16 = little + ? (value: number, offset: number) => bytes.writeUInt16LE(value, offset) + : (value: number, offset: number) => bytes.writeUInt16BE(value, offset); + const write32 = little + ? (value: number, offset: number) => bytes.writeUInt32LE(value, offset) + : (value: number, offset: number) => bytes.writeUInt32BE(value, offset); + write32(0xffffffff, 168); // Unknown name; the section type is authoritative. + write32(0x6ffffff5, 172); // SHT_GNU_ATTRIBUTES. + write32(0xfffffff0, 184); // Must not be read as a payload offset. + write32(0xffffffff, 188); // Must not control allocation. + write16(0, 50); // No section-name string table is required for detection. + const result = await inspect(bytes, little); + assert.ok(!result.ok); + assert.match(result.error, /GNU ABI attributes that REA does not inspect/u); + }); +} + it("preserves cancellation instead of turning it into malformed target data", async () => { const directory = await createTestTempDirectory("rea-mips-resolution-"); const path = join(directory, "fixture.elf"); diff --git a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts index e786fc3cc..f7643f337 100644 --- a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts +++ b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts @@ -175,7 +175,8 @@ it("retains terminal-call qualifications across function views and snapshot repl }); expect(evidence.limitations, name).toContain(limitation); expect(evidence.analysis_profile?.parameters).toMatchObject({ - function_boundary_observations: "ghidra-terminal-call-limitations-v2", + function_boundary_observations: + "ghidra-function-boundary-limitations-v3", }); const replay = await harness.session.execute(name, evidence.parameters); if (!replay.ok) throw replay.error; diff --git a/tests/boundary/mcp/idaTransport.test.ts b/tests/boundary/mcp/idaTransport.test.ts index 4a43bdb2e..2844d8eca 100644 --- a/tests/boundary/mcp/idaTransport.test.ts +++ b/tests/boundary/mcp/idaTransport.test.ts @@ -1,9 +1,46 @@ import { createServer } from "node:http"; -import { afterEach, expect, it } from "vitest"; +import { inspect } from "node:util"; +import { ProtocolError } from "@modelcontextprotocol/client"; +import { afterEach, expect, it, vi } from "vitest"; +import { constants } from "node:buffer"; +import { IdaDiagnosticResourceError } from "../../../src/ida/IdaDiagnosticWriter.js"; import { z } from "zod"; +import { AnalysisProtocolError } from "../../../src/domain/analysisErrorCore.js"; +import type { JsonValue } from "../../../src/domain/jsonValue.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js"; import { createIdaMcpConnection } from "../../../src/ida/IdaMcpConnection.js"; import type { IdaMcpConnection } from "../../../src/ida/IdaMcpConnection.js"; +// Inject an owned capacity failure at the producer while retaining the real +// SDK. Small writer-limit tests separately exercise actual expansion checks. +vi.mock("../../../src/ida/IdaDiagnosticRedactor.js", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("../../../src/ida/IdaDiagnosticRedactor.js") + >(); + return { + ...actual, + createIdaDiagnosticRedactor: ( + config: Parameters[0], + ) => { + const redactor = actual.createIdaDiagnosticRedactor(config); + return { + ...redactor, + text: redactor.text, + json: (message: string) => { + if (message === "fixture-resource-boundary") + throw new IdaDiagnosticResourceError( + message.length, + constants.MAX_STRING_LENGTH + 1, + ); + return redactor.json(message); + }, + }; + }, + }; +}); + const cleanups: (() => Promise)[] = []; afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); @@ -19,7 +56,36 @@ const requestSchema = z.object({ .passthrough() .optional(), }); -const fixture = async (redirectUrl?: string, cycle = false, native = false) => { +type IdaTransportFixtureOptions = { + readonly redirectUrl?: string; + readonly cycle?: boolean; + readonly native?: boolean; + readonly toolResult?: JsonValue; + readonly resourceResponseText?: string; + readonly toolRpcError?: { + readonly code: number; + readonly message: string; + readonly data?: JsonValue; + }; + readonly toolStatus?: number; + readonly toolStatusText?: string; + readonly toolContentType?: string; + readonly toolResponseBody?: string | Buffer; + readonly configurationHeaders?: Readonly>; +}; +const fixture = async ({ + redirectUrl, + cycle = false, + native = false, + toolResult, + resourceResponseText, + toolRpcError, + toolStatus = 200, + toolStatusText, + toolContentType = "application/json", + toolResponseBody, + configurationHeaders = { Authorization: "Bearer transport-fixture-secret" }, +}: IdaTransportFixtureOptions = {}) => { const methods: string[] = []; const headers: (string | undefined)[] = []; const server = createServer(async (request, response) => { @@ -78,28 +144,47 @@ const fixture = async (redirectUrl?: string, cycle = false, native = false) => { { uri: "ida://idb/metadata", mimeType: "application/json", - text: JSON.stringify({ - path: "/fixture/input.i64", - sha256: "a".repeat(64), - base: "0x1000", - module: "input", - }), + text: + resourceResponseText ?? + JSON.stringify({ + path: "/fixture/input.i64", + sha256: "a".repeat(64), + base: "0x1000", + module: "input", + }), }, ], } - : native - ? { - content: [ - { - type: "text", - text: JSON.stringify([{ observed: true }]), - }, - ], - structuredContent: { result: [], _output_truncated: true }, - } - : { content: [], structuredContent: { observed: true } }; - response.writeHead(200, { "Content-Type": "application/json" }); - response.end(JSON.stringify({ jsonrpc: "2.0", id: input.id, result })); + : toolResult !== undefined + ? toolResult + : native + ? { + content: [ + { + type: "text", + text: JSON.stringify([{ observed: true }]), + }, + ], + structuredContent: { result: [], _output_truncated: true }, + } + : { content: [], structuredContent: { observed: true } }; + if (input.method === "tools/call" && toolStatusText !== undefined) + response.statusMessage = toolStatusText; + response.writeHead(input.method === "tools/call" ? toolStatus : 200, { + "Content-Type": + input.method === "tools/call" ? toolContentType : "application/json", + }); + response.end( + input.method === "tools/call" && toolResponseBody !== undefined + ? toolResponseBody + : JSON.stringify({ + jsonrpc: "2.0", + id: input.id, + ...(input.method === "tools/call" && toolRpcError !== undefined + ? { error: toolRpcError } + : { result }), + }), + ); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); cleanups.push( @@ -117,7 +202,7 @@ const fixture = async (redirectUrl?: string, cycle = false, native = false) => { const connection: IdaMcpConnection = createIdaMcpConnection( { url: `http://127.0.0.1:${address.port}/mcp`, - headers: { Authorization: "Bearer transport-fixture-secret" }, + headers: configurationHeaders, mode: native ? "attached" : "headless", ...(native ? { protocol: "native" as const } : {}), timeoutMs: 2000, @@ -148,15 +233,30 @@ it("uses the pinned SDK for a local Streamable HTTP handshake, paged discovery, ).toBe(true); }); +it("omits an oversized HTTP error body before retaining or redacting it", async () => { + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + toolResponseBody: "owned-diagnostic-" + "x".repeat(70 * 1024), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("response body omitted"); + expect(failure.message).not.toContain("owned-diagnostic-"); +}); + it("rejects redirects instead of forwarding transport credentials to another endpoint", async () => { const destination = await fixture(); - const { connection } = await fixture(destination.url); + const { connection } = await fixture({ redirectUrl: destination.url }); await expect(connection.connect()).rejects.toThrow(); expect(destination.headers).toEqual([]); }); it("rejects nonterminating SDK pagination instead of returning a partial tool catalog", async () => { - const { connection, methods } = await fixture(undefined, true); + const { connection, methods } = await fixture({ cycle: true }); await expect(connection.connect()).rejects.toThrow( "pagination did not terminate", ); @@ -166,7 +266,7 @@ it("rejects nonterminating SDK pagination instead of returning a partial tool ca }); it("reads native metadata resources and complete JSON text through the real SDK transport", async () => { - const { connection, methods } = await fixture(undefined, false, true); + const { connection, methods } = await fixture({ native: true }); expect(await connection.connect()).toEqual(["first", "second"]); expect(await connection.call("first", {})).toEqual([{ observed: true }]); expect(await connection.readResource?.("ida://idb/metadata")).toMatchObject({ @@ -176,3 +276,1182 @@ it("reads native metadata resources and complete JSON text through the real SDK await expect(connection.readResource?.("ida://other")).rejects.toThrow(); expect(methods).toContain("resources/read"); }); + +it("accepts a legacy stdio reply over the SDK default while staying bounded", async () => { + const script = String.raw` +let input = ""; +const send = (message) => process.stdout.write(JSON.stringify(message) + "\n"); +process.stdin.setEncoding("utf8"); +process.stdin.on("data", (chunk) => { + input += chunk; + for (;;) { + const newline = input.indexOf("\n"); + if (newline < 0) break; + const request = JSON.parse(input.slice(0, newline)); + input = input.slice(newline + 1); + if (request.id === undefined) continue; + const result = request.method === "initialize" + ? { protocolVersion: request.params.protocolVersion, capabilities: { tools: {} }, serverInfo: { name: "large-reply-fixture", version: "1" } } + : request.method === "tools/list" + ? { tools: [{ name: "list_strings", inputSchema: { type: "object", properties: {} } }] } + : { content: [{ type: "text", text: "fixture" }], structuredContent: { result: "x".repeat(11 * 1024 * 1024) } }; + send({ jsonrpc: "2.0", id: request.id, result }); + } +}); +`; + const connection = createIdaMcpConnection( + { + command: process.execPath, + args: ["-e", script], + env: {}, + mode: "attached", + timeoutMs: 10_000, + }, + {}, + ); + cleanups.push(() => connection.close()); + expect(await connection.connect()).toEqual(["list_strings"]); + expect(await connection.call("list_strings", {})).toEqual( + "x".repeat(11 * 1024 * 1024), + ); +}); + +it.each([false, true])( + "preserves an upstream tool rejection through SDK transport and public error projection (native: %s)", + async (native) => { + const reason = + "Invalid params: unexpected parameters: ['offset', 'max_lines']"; + const { connection } = await fixture({ + native, + toolResult: { + isError: true, + content: [{ type: "text", text: reason }], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", { + addr: "0x1000", + offset: 0, + max_lines: 100, + }) + .catch((cause: unknown) => cause); + + expect(failure).toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error( + "Expected the upstream rejection to remain a protocol error", + ); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); + +it.each([false, true])( + "redacts configured credentials from SDK tool errors while retaining caller context (native: %s)", + async (native) => { + const configurationHeaders = { + aUtHoRiZaTiOn: "\r\n \tBearer transport-fixture-secret \r\n", + "Proxy-Authorization": "Basic proxy-fixture-secret", + CoOkIe: "session=cookie-fixture-secret", + "X-Evidence": "selected-header-evidence", + }; + const context = + "/caller/input?cursor=visible API_KEY=caller-evidence selected-header-evidence"; + const { connection } = await fixture({ + native, + configurationHeaders, + toolResult: { + isError: true, + content: [ + { + type: "text", + text: + "Rejected Bearer transport-fixture-secret / transport-fixture-secret / Basic proxy-fixture-secret / proxy-fixture-secret / session=cookie-fixture-secret at " + + context, + }, + ], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error( + "Expected the upstream rejection to remain a protocol error", + ); + const reason = + "Rejected [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] at " + + context; + expect(failure.message).toBe("IDA MCP tool failed: \n" + reason); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); + +type ParsedCredentialCase = { + readonly label: string; + readonly headers: Readonly>; + readonly payloads: readonly string[]; +}; +const basicCredentialCases: readonly ParsedCredentialCase[] = [ + { + label: "decoded Basic ASCII credentials", + headers: { + Authorization: + "Basic " + + Buffer.from("basic-fixture-user:basic-fixture:password").toString( + "base64", + ), + }, + payloads: [ + "basic-fixture-user:basic-fixture:password", + "basic-fixture:password", + ], + }, + { + label: "decoded Proxy Basic UTF-8 credentials", + headers: { + "Proxy-Authorization": + "bAsIc " + + Buffer.from("utf8-fixture-user:páss-fixture:☃", "utf8").toString( + "base64", + ), + }, + payloads: ["utf8-fixture-user:páss-fixture:☃", "páss-fixture:☃"], + }, + { + label: "decoded Basic Latin-1 credentials", + headers: { + Authorization: + "Basic " + + Buffer.from( + "latin-fixture-user:café-fixture:password", + "latin1", + ).toString("base64"), + }, + payloads: [ + "latin-fixture-user:café-fixture:password", + "café-fixture:password", + ], + }, +]; +const parsedCredentialCases: readonly ParsedCredentialCase[] = [ + ...basicCredentialCases, + { + label: "Cookie values", + headers: { + Cookie: + 'session=cookie-fixture-secret; quoted="quoted-fixture=value=="; empty=', + }, + payloads: ["cookie-fixture-secret", "quoted-fixture=value=="], + }, + { + label: "Authorization Token payload", + headers: { Authorization: "Token custom-fixture-token+/==" }, + payloads: ["custom-fixture-token+/=="], + }, + { + label: "Proxy-Authorization custom token68", + headers: { "Proxy-Authorization": "Custom proxy-fixture-token._~+/-==" }, + payloads: ["proxy-fixture-token._~+/-=="], + }, + { + label: "Digest auth-param values", + headers: { + Authorization: + 'Digest username="fixture-user", nonce="nonce,fixture", response="response\\\"secret", opaque="back\\\\slash"', + }, + payloads: [ + "fixture-user", + "nonce,fixture", + 'response"secret', + "back\\slash", + ], + }, +]; + +it.each( + parsedCredentialCases.flatMap((testCase) => + [false, true].map((native) => ({ ...testCase, native })), + ), +)( + "redacts parsed $label from tool errors (native: $native)", + async ({ headers, payloads, native }) => { + const context = "/caller/input?cursor=visible API_KEY=caller-evidence"; + const { connection } = await fixture({ + native, + configurationHeaders: headers, + toolResult: { + isError: true, + content: [ + { + type: "text", + text: "Rejected " + payloads.join(" / ") + " at " + context, + }, + ], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error( + "Expected the upstream rejection to remain a protocol error", + ); + const reason = + "Rejected " + + payloads.map(() => "[redacted transport credential]").join(" / ") + + " at " + + context; + expect(failure.message).toBe("IDA MCP tool failed: \n" + reason); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); + +it.each([false, true])( + "retains successful credential-like evidence through SDK transport (native: %s)", + async (native) => { + const evidence = { + observed: "custom-fixture-token+/==", + cookie: "quoted-fixture=value==", + digest: 'response"secret', + }; + const { connection } = await fixture({ + native, + configurationHeaders: { + Authorization: "Token custom-fixture-token+/==", + Cookie: 'session="quoted-fixture=value=="', + "Proxy-Authorization": 'Digest response="response\\\"secret"', + }, + toolResult: native + ? { content: [{ type: "text", text: JSON.stringify(evidence) }] } + : { content: [], structuredContent: evidence }, + }); + await connection.connect(); + expect(await connection.call("first", {})).toEqual(evidence); + }, +); + +it.each<{ + label: string; + headers: Readonly>; + payloads: readonly string[]; +}>([ + { + label: "Cookie and custom token payloads", + headers: { + Authorization: "Token custom-fixture-token", + Cookie: "session=cookie-fixture-secret", + }, + payloads: ["custom-fixture-token", "cookie-fixture-secret"], + }, + { + label: "JSON-escaped Digest payloads and header", + headers: { + Authorization: 'Digest response="q\\\"uote", opaque="back\\\\slash"', + }, + payloads: [ + 'q"uote', + "back\\slash", + 'Digest response="q\\\"uote", opaque="back\\\\slash"', + ], + }, + ...basicCredentialCases, +])( + "redacts $label echoed in an HTTP failure through the SDK rejection path", + async ({ headers, payloads }) => { + const context = "/caller/input?cursor=visible API_KEY=caller-evidence"; + const { connection } = await fixture({ + toolStatus: 500, + configurationHeaders: headers, + toolResult: { + isError: true, + content: [ + { + type: "text", + text: + "Transport rejected " + payloads.join(" / ") + " at " + context, + }, + ], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + "Transport rejected " + + payloads.map(() => "[redacted transport credential]").join(" / ") + + " at " + + context, + ); + for (const payload of payloads) + expect(failure.message).not.toContain( + JSON.stringify(payload).slice(1, -1), + ); + }, +); + +it.each<{ + label: string; + headers: Readonly>; + credential: string; + wire: string; + toolContentType: string; +}>([ + { + label: "Go-style HTML escapes", + toolContentType: "application/problem+json; charset=utf-8", + headers: { Cookie: "session=&/fixture" }, + credential: "&/fixture", + wire: String.raw`\u003csecret\u003e\u0026/fixture`, + }, + { + label: "fully escaped ASCII with mixed hex case and solidus", + toolContentType: "application/json", + headers: { Authorization: "Token aBz/secret" }, + credential: "aBz/secret", + wire: String.raw`\u0061\u0042\u007A\/\u0073\u0065\u0063\u0072\u0065\u0074`, + }, + { + label: "Latin-1 auth-param Unicode and solidus escapes", + toolContentType: "application/json", + headers: { Authorization: 'Digest response="café/secret"' }, + credential: "café/secret", + wire: String.raw`caf\u00E9\/secret`, + }, +])( + "redacts $label in SDK HTTP failures without rewriting unrelated wire evidence", + async ({ headers, credential, wire, toolContentType }) => { + const prefix = '{\n "reason": "Rejected '; + const suffix = + String.raw`", "numeric": 9007199254740993, "duplicate": "first", "duplicate": "second", "context": "\u003cpublic\u003e\/asset", "symbol": "\ud83d\uDE00"` + + "\n}\n"; + const toolResponseBody = prefix + wire + suffix; + expect(JSON.parse(toolResponseBody)).toMatchObject({ + reason: "Rejected " + credential, + }); + const { connection } = await fixture({ + toolStatus: 500, + toolContentType, + configurationHeaders: headers, + toolResponseBody, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + prefix + "[redacted transport credential]" + suffix, + ); + expect(failure.message).not.toContain(wire); + }, +); + +const jsonFailureResponses = [ + { toolStatus: 200, toolContentType: "application/json" }, + { toolStatus: 500, toolContentType: "application/json" }, + { + toolStatus: 500, + toolContentType: "application/problem+json; charset=utf-8", + }, +] as const; + +it.each( + jsonFailureResponses.flatMap((response) => + [ + { + label: "unquoted escaped body", + credential: "", + wire: String.raw`\u003ctoken\u003e`, + toolResponseBody: String.raw`\u003ctoken\u003e`, + safeBody: "[redacted transport credential]", + }, + { + label: "quoted escaped array entry before invalid JSON", + credential: "", + wire: String.raw`\u003cs\u003e`, + toolResponseBody: String.raw`["\u003cs\u003e",x]`, + safeBody: '["[redacted transport credential]",x]', + }, + ].map((body) => ({ ...response, ...body })), + ), +)( + "redacts $label from malformed HTTP $toolStatus $toolContentType responses", + async ({ + credential, + wire, + toolResponseBody, + safeBody, + toolStatus, + toolContentType, + }) => { + const { connection } = await fixture({ + configurationHeaders: { Cookie: "token=" + credential }, + toolResponseBody, + toolStatus, + toolContentType, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject malformed JSON"); + expect(failure.message).toContain(safeBody); + if (toolStatus === 200) + expect(failure.message).toMatch(/malformed.*HTTP.*JSON/iu); + else expect(failure.cause).toMatchObject({ data: { status: 500 } }); + expect(failure.message).not.toContain(toolResponseBody); + expect(failure.message).not.toContain(wire); + expect(failure.message).not.toContain(credential); + }, +); + +it.each( + jsonFailureResponses.flatMap((response) => + [ + { + credentialLabel: "literal short escape", + credential: String.raw`folder\tmp`, + }, + { + credentialLabel: "literal Unicode escape", + credential: String.raw`folder\u0061`, + }, + ].flatMap((credentialCase) => + [ + { + bodyLabel: "unquoted body", + prefix: "Rejected ", + suffix: " at /caller/input?cursor=visible", + }, + { + bodyLabel: "quoted value in an invalid document", + prefix: 'Rejected "', + suffix: '" at /caller/input?cursor=visible', + }, + { + bodyLabel: "unfinished JSON string", + prefix: '{"reason":"Rejected ', + suffix: " at /caller/input?cursor=visible", + }, + ].map((body) => ({ ...response, ...credentialCase, ...body })), + ), + ), +)( + "redacts $credentialLabel in $bodyLabel from HTTP $toolStatus $toolContentType diagnostics and their public projection", + async ({ credential, toolStatus, toolContentType, prefix, suffix }) => { + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Custom " + credential }, + toolStatus, + toolContentType, + toolResponseBody: prefix + credential + suffix, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject malformed JSON"); + expect(failure.message).toContain( + prefix + "[redacted transport credential]" + suffix, + ); + expect(failure.message).not.toContain(credential); + if (toolStatus === 200) + expect(failure.message).toMatch(/malformed.*HTTP.*JSON/iu); + else expect(failure.cause).toMatchObject({ data: { status: 500 } }); + const projected = projectAnalysisError( + new ProviderAdapterError("ida", "decompile", { + cause: failure, + diagnostics: { reason: failure.message }, + }), + ); + expect(projected).toMatchObject({ + code: "execution_failure", + details: { diagnostics: { reason: failure.message } }, + }); + expect(JSON.stringify(projected)).not.toContain( + JSON.stringify(credential).slice(1, -1), + ); + }, +); + +it("preserves JSON escape syntax that spells a configured credential without containing its value", async () => { + const toolResponseBody = String.raw`{"reason":"\u003cpublic\u003e","numeric":9007199254740993}`; + const { connection } = await fixture({ + toolStatus: 500, + configurationHeaders: { Cookie: "token=u003c" }, + toolResponseBody, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain(toolResponseBody); + expect(failure.message).not.toContain("[redacted transport credential]"); +}); + +it("redacts complete opaque authentication values containing quotes from plain HTTP error bodies", async () => { + const credential = 'prefix"quoted"suffix'; + const header = "Custom " + credential; + const context = "/caller/input?cursor=visible API_KEY=caller-evidence"; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: header }, + toolResponseBody: + "Rejected " + header + " / " + credential + " at " + context, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + "Rejected [redacted transport credential] / [redacted transport credential] at " + + context, + ); + expect(failure.message).not.toContain(credential); +}); + +it.each(["Rejected ", ""])( + "redacts quoted literal credentials from text/plain HTTP failures with prefix %j", + async (prefix) => { + const credential = String.raw`folder\tmp`; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody: prefix + '"' + credential + '"', + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + prefix + '"[redacted transport credential]"', + ); + expect(failure.message).not.toContain(credential); + }, +); + +it("redacts a JSON-RPC error exactly once after successful HTTP delivery", async () => { + const { connection } = await fixture({ + configurationHeaders: { Cookie: "token=redacted" }, + toolRpcError: { code: -32602, message: "redacted at /caller/input" }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the JSON-RPC error"); + expect(failure.cause).toBeInstanceOf(ProtocolError); + expect(failure.message.endsWith("␟ at /caller/input")).toBe(true); + expect(failure.message.match(/␟/gu)).toHaveLength(1); + expect(failure.message).not.toContain("redacted"); +}); + +it.each([ + "", + String.raw`\u003ccause-fixture-secret\u003e`, +])( + "does not retain sensitive malformed HTTP JSON in inspected causes: %s", + async (wire) => { + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" }, + toolResponseBody: '{"reason":"Rejected ' + wire + " at /caller/input", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected malformed JSON rejection"); + expect(failure.message).toContain( + "[redacted transport credential] at /caller/input", + ); + const inspected = inspect(failure, { + depth: null, + showHidden: true, + maxStringLength: null, + }); + for (const value of [ + "", + wire, + JSON.stringify(wire).slice(1, -1), + ]) + expect(inspected).not.toContain(value); + }, +); + +it("retains safe JSON-RPC code and message without unknown remote cause data", async () => { + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Bearer rpc-cause-fixture-secret" }, + toolRpcError: { + code: -32602, + message: "Rejected request at /caller/input", + data: { + echo: "rpc-cause-fixture-secret", + nested: { caller: "/caller/input" }, + }, + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected JSON-RPC rejection"); + expect(failure.message).toContain("Rejected request at /caller/input"); + expect(failure.cause).toBeInstanceOf(ProtocolError); + expect(failure.cause).toMatchObject({ code: -32602 }); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "rpc-cause-fixture-secret", + ); +}); + +it.each([ + "text/html", + "application/problem+xml", + "application/octet-stream", + "text/plain; charset=utf-16", + "not a media type", +])( + "omits HTTP error bodies whose declared encoding is unsupported: %s", + async (toolContentType) => { + const body = + "<format-fixture-secret>&/fixture at /caller/input"; + const { connection } = await fixture({ + configurationHeaders: { + Cookie: "session=&/fixture", + }, + toolStatus: 500, + toolContentType, + toolResponseBody: body, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP error"); + expect(failure.message).toMatch(/body.*omitted|omitted.*body/iu); + expect(failure.cause).toMatchObject({ data: { status: 500 } }); + const inspected = inspect(failure, { + depth: null, + showHidden: true, + maxStringLength: null, + }); + expect(inspected).not.toContain(body); + expect(inspected).not.toContain("format-fixture-secret"); + }, +); + +it.each(["native tool", "resource"] as const)( + "does not retain the producer JSON parse cause for %s text", + async (source) => { + const malformedText = '["token",x]'; + const { connection } = await fixture({ + native: true, + configurationHeaders: { Authorization: "Bearer token" }, + ...(source === "native tool" + ? { toolResult: { content: [{ type: "text", text: malformedText }] } } + : { resourceResponseText: malformedText }), + }); + await connection.connect(); + const failure = await ( + source === "native tool" + ? connection.call("first", {}) + : connection.readResource?.("ida://idb/metadata") + )?.catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(AnalysisProtocolError); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "token", + ); + }, +); + +it.each([200, 500])( + "preserves owned diagnostic capacity failures through HTTP %s SDK handling", + async (toolStatus) => { + const { connection } = await fixture({ + toolStatus, + toolResponseBody: "fixture-resource-boundary", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(IdaDiagnosticResourceError); + if (!(failure instanceof IdaDiagnosticResourceError)) + throw new Error("Expected an owned diagnostic capacity failure"); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "resource_constraint", + details: { + resource: "memory", + reported_limits: { boundary: "ida-diagnostic-redaction" }, + }, + }); + }, +); + +it.each( + [200, 500].flatMap((toolStatus) => + ["utf-16le", "utf-16", " = utf-16", "=utf-8; charset=utf-16"].map( + (charset) => ({ toolStatus, charset }), + ), + ), +)( + "omits unsupported $charset diagnostic bytes from HTTP $toolStatus JSON failures", + async ({ toolStatus, charset }) => { + const credential = "charset-fixture-secret"; + const utf16 = Buffer.from('{"reason":"' + credential + '"}', "utf16le"); + const encoded = + charset === "utf-16" + ? Buffer.concat([Buffer.from([0xff, 0xfe]), utf16]) + : utf16; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + credential }, + toolStatus, + toolContentType: + "application/json; charset" + + (charset.startsWith("=") || charset.startsWith(" ") + ? charset + : "=" + charset), + toolResponseBody: encoded, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected unsupported encoded JSON error"); + expect(failure.message).toMatch(/body.*omitted|omitted.*body/iu); + expect(failure.message).not.toContain(encoded.toString("utf8")); + const inspected = inspect(failure, { + depth: null, + showHidden: true, + maxStringLength: null, + }); + expect( + inspected + .replaceAll("\\x00", "") + .replaceAll("\\u0000", "") + .replaceAll("\0", ""), + ).not.toContain(credential); + }, +); + +it("retains successfully parsed analysis values despite unsupported declared charset metadata", async () => { + const observed = "charset-fixture-secret"; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + observed }, + toolContentType: "application/json; charset=utf-16le", + toolResult: { content: [], structuredContent: { observed } }, + }); + await connection.connect(); + expect(await connection.call("first", {})).toEqual({ observed }); +}); + +it.each( + [200, 500].flatMap((toolStatus) => + [ + "application/json; charset = utf-16", + "application/json; charset\t=utf-16le", + "application/json; charset=utf-8; CHARSET=utf-16le", + 'application/json; note="unterminated; charset=utf-16le', + ].map((toolContentType) => ({ toolStatus, toolContentType })), + ), +)( + "omits encoded diagnostic bytes when Content-Type is incomplete or ambiguous: HTTP $toolStatus $toolContentType", + async ({ toolStatus, toolContentType }) => { + const credential = "mime-fixture-secret"; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + credential }, + toolStatus, + toolContentType, + toolResponseBody: Buffer.from( + '{"reason":"' + credential + '"}', + "utf16le", + ), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected malformed Content-Type failure"); + expect(failure.message).toMatch(/body.*omitted|omitted.*body/iu); + expect( + inspect(failure, { depth: null, showHidden: true }).replaceAll( + "\\x00", + "", + ), + ).not.toContain(credential); + }, +); + +it.each([ + "application/json; charset=", + 'application/json; charset="utf-8', + 'application/json; charset="utf-8"junk', + "application/json; note=a,b", + "application/json; note=a; NOTE=b", + "text/plain; charset = utf-16", +])( + "omits error bodies whose whole Content-Type is invalid or conflicting: %s", + async (toolContentType) => { + const { connection } = await fixture({ + toolStatus: 500, + toolContentType, + toolResponseBody: "untrusted /caller/body", + }); + await connection.connect(); + await expect(connection.call("first", {})).rejects.toThrow( + /body.*omitted|omitted.*body/iu, + ); + }, +); + +it.each([ + "application/json", + 'Application/Problem+JSON\t; Note="a; charset=utf-16"; ChArSeT="UTF-8"', + String.raw`application/json; charset="utf\-8"; note="escaped\"quote; slash\\"`, + 'application/json; note=""; charset=utf-8;', + "application/json;; ;charset=utf-8;", + "text/plain; note=caller; charset=utf8", +])( + "retains safe producer diagnostics for complete legal Content-Type: %s", + async (toolContentType) => { + const credential = "mime-fixture-secret"; + const reason = "Rejected " + credential + " at /caller/input"; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + credential }, + toolStatus: 500, + toolContentType, + toolResponseBody: toolContentType.startsWith("text/plain") + ? reason + : JSON.stringify({ reason }), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected upstream error"); + expect(failure.message).toContain( + "Rejected [redacted transport credential] at /caller/input", + ); + expect(failure.message).not.toContain("omitted"); + expect(failure.message).not.toContain(credential); + }, +); + +it.each(["endpoint", ""])( + "redacts the complete SDK context when the trusted body repeats its text: %s", + async (toolResponseBody) => { + const credential = "Error POSTing to endpoint"; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toBe( + "[redacted transport credential]: \n" + toolResponseBody, + ); + expect(failure.cause).toMatchObject({ data: { status: 500 } }); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); + }, +); + +it("keeps a sanitized plain HTTP body intact without exempting matching status text", async () => { + const { connection } = await fixture({ + toolStatus: 500, + toolStatusText: "abcXdef", + toolContentType: "text/plain", + configurationHeaders: { + Authorization: "Custom abcXdef", + Cookie: "token=redacted", + }, + toolResponseBody: "X redacted at /caller/input", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("X ␟ at /caller/input"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "abcXdef", + ); +}); + +it("keeps SDK context and the owned body separate when joining them would form a credential", async () => { + const credential = "endpoint: public"; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody: "public", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("endpoint: \npublic"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); +}); + +it("keeps owned HTTP JSON failure context separate from the sanitized body", async () => { + const credential = "Response body: public"; + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody: "public", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected malformed JSON failure"); + expect(failure.message).toContain("Response body: \npublic"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); +}); + +it.each(["tool rejection", "HTTP rejection", "malformed HTTP JSON"] as const)( + "redacts overlapping credential spans through the SDK: %s", + async (source) => { + const reason = "Rejected X]SECRET at /caller/input"; + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Custom X]", Cookie: "a=]SECRET" }, + ...(source === "tool rejection" + ? { + toolResult: { + isError: true, + content: [{ type: "text", text: reason }], + }, + } + : { + toolStatus: source === "HTTP rejection" ? 500 : 200, + toolResponseBody: + String.raw`{"reason":"Rejected X\u005dSECRET at /caller/input"` + + (source === "HTTP rejection" ? "}" : ",x"), + }), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected producer failure"); + expect(failure.message).toContain("at /caller/input"); + const retained = inspect(failure, { depth: null, showHidden: true }); + expect(retained).not.toContain("SECRET"); + expect(retained).not.toContain("X]"); + }, +); + +it("does not introduce a credential at a redaction-marker boundary in a real HTTP failure", async () => { + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom X", Cookie: "a=]SECRET" }, + toolResponseBody: "Rejected XSECRET at /caller/input", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("SECRET at /caller/input"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "]SECRET", + ); +}); + +it.each( + [ + { + label: "Basic pair and leading-space password", + authorization: "Basic ZmFpbGVkOiBwdWJsaWM=", + forbidden: ["failed: public", " public"], + }, + { + label: "opaque payload spanning the tool prefix and body", + authorization: "Custom failed: public", + forbidden: ["failed: public"], + }, + { + label: "credential equal to the fixed tool prefix", + authorization: "Custom IDA MCP tool failed", + forbidden: ["IDA MCP tool failed"], + }, + ].flatMap((testCase) => + [false, true].map((native) => ({ ...testCase, native })), + ), +)( + "keeps tool-error owners separate for $label (native: $native)", + async ({ authorization, forbidden, native }) => { + const { connection } = await fixture({ + native, + configurationHeaders: { Authorization: authorization }, + toolResult: { + isError: true, + content: [{ type: "text", text: "public" }], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error("Expected the tool rejection to remain a protocol error"); + expect(failure.message).toContain("public"); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason: "public" } }, + }); + for (const credential of forbidden) + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); + }, +); + +const toolDiagnosticBlockCases: readonly { + label: string; + headers: Readonly>; + blocks: readonly string[]; + safeBlocks: readonly string[]; +}[] = [ + { + label: "JSON-escaped quoted credential without rewriting other source", + headers: { Authorization: 'Custom q"uote' }, + blocks: [ + '{"error":"Rejected q\\"uote at /caller/input","amount":9007199254740993,"duplicate":1,"duplicate":2}', + ], + safeBlocks: [ + '{"error":"Rejected [redacted transport credential] at /caller/input","amount":9007199254740993,"duplicate":1,"duplicate":2}', + ], + }, + { + label: "independent JSON arrays, escaped keys and surrogate pairs", + headers: { + Authorization: + "Basic " + Buffer.from("user:😀", "utf8").toString("base64"), + Cookie: "token=; syntax=u003c", + }, + blocks: [ + "Plain status at /caller/plain", + '[{"path":"/caller/array"},"\\u003cjson-secret\\u003e"]', + '{"\\u003cjson-secret\\u003e":"key context","emoji":"\\uD83d\\uDE00 at /caller/emoji"}', + '{"public":"\\u003cpublic\\u003e","amount":9007199254740993}', + ], + safeBlocks: [ + "Plain status at /caller/plain", + '[{"path":"/caller/array"},"[redacted transport credential]"]', + '{"[redacted transport credential]":"key context","emoji":"[redacted transport credential] at /caller/emoji"}', + '{"public":"\\[redacted transport credential]public\\u003e","amount":9007199254740993}', + ], + }, + { + label: + "unclassified text retains both raw and decoded credential candidates", + headers: { + Authorization: "Custom folder\\tmp", + Cookie: "token=", + }, + blocks: [ + "Rejected folder\\tmp at /caller/plain; literal \\u003cpublic\\u003e", + ], + safeBlocks: [ + "Rejected [redacted transport credential] at /caller/plain; literal [redacted transport credential]", + ], + }, + { + label: "quoted plain backslash credential despite valid JSON appearance", + headers: { Authorization: "Custom folder\\tmp" }, + blocks: [String.raw`"folder\tmp at /caller/input"`], + safeBlocks: ['"[redacted transport credential] at /caller/input"'], + }, + { + label: "malformed JSON quote escapes and independent context", + headers: { Authorization: 'Custom q"uote' }, + blocks: [ + String.raw`{"error":"Rejected q\"uote at /caller/input`, + "Second context at /caller/second", + ], + safeBlocks: [ + '{"error":"Rejected [redacted transport credential] at /caller/input', + "Second context at /caller/second", + ], + }, +]; + +it.each( + toolDiagnosticBlockCases.flatMap((testCase) => + [false, true].map((native) => ({ ...testCase, native })), + ), +)( + "owns each SDK tool diagnostic block as $label (native: $native)", + async ({ headers, blocks, safeBlocks, native }) => { + const { connection } = await fixture({ + native, + configurationHeaders: headers, + toolResult: { + isError: true, + content: blocks.map((text) => ({ type: "text", text })), + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error("Expected the tool rejection to remain a protocol error"); + const reason = safeBlocks.join("\n"); + expect(failure.message.endsWith(reason)).toBe(true); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); diff --git a/tests/boundary/providers/ghidra/ghidraClient.test.ts b/tests/boundary/providers/ghidra/ghidraClient.test.ts index 9ce3f77ce..4bd24f5f7 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.test.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.test.ts @@ -31,6 +31,7 @@ import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js const fixturePath = fileURLToPath( new URL("../../../fixtures/fakeGhidra.mjs", import.meta.url), ); + const PROFILE_DIGEST = "a".repeat(64); const PROVIDER_VERSION = "12.1.4"; const TARGET_SHA256 = createHash("sha256") @@ -40,6 +41,8 @@ const HOST_TRANSPORT: GhidraTransportKind = process.platform === "win32" ? "authenticated-loopback-tcp" : "unix-socket"; type FixtureMode = + | "reject_first_connection" + | "reject_all_connections" | "success" | "fragmented" | "oversized_result" @@ -1131,3 +1134,30 @@ describe.skipIf(process.platform === "win32")( ); }, ); + +it("reconnects a dropped preauthentication loopback socket within the original startup deadline", async () => { + const launcher = new FixtureLauncher("reject_first_connection"); + const client = clientFor(launcher, { + transport: "authenticated-loopback-tcp", + }); + const started = await client.start(); + expect(started.ok).toBe(true); + expect(launcher.processes).toHaveLength(1); + const result = await client.callToolIfReady("list_strings", {}); + expect(result).toMatchObject({ ok: true }); +}); + +it("bounds retries when loopback preauthentication keeps failing", async () => { + const launcher = new FixtureLauncher("reject_all_connections"); + const client = clientFor(launcher, { + transport: "authenticated-loopback-tcp", + }); + expect(await client.start()).toMatchObject({ + ok: false, + error: { kind: "process" }, + }); + expect(launcher.processes).toHaveLength(1); + await expect(waitForChildExit(launcher.processes[0]!, 5_000)).resolves.toBe( + true, + ); +}); diff --git a/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts b/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts index 62310facf..6188f0a39 100644 --- a/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts @@ -184,6 +184,7 @@ it.each([false, true])( processSupervisor: supervisor, runtimeRoot, shutdownConfirmed: false, + externalApplication: undefined, }; const socket = new Socket(); onTestFinished(() => { @@ -393,6 +394,7 @@ it("preserves confirmed document shutdown while retrying owned process cleanup", processSupervisor: supervisor, runtimeRoot, shutdownConfirmed: false, + externalApplication: undefined, } satisfies HopperOwnedResources, activeRequest: null, retainDocument: false, @@ -457,6 +459,7 @@ it.each([ processSupervisor: supervisor, runtimeRoot, shutdownConfirmed: false, + externalApplication: undefined, }; const result = await cleanupHopperSession({ socket: new Socket(), diff --git a/tests/boundary/providers/hopper/hopperClient.restart.test.ts b/tests/boundary/providers/hopper/hopperClient.restart.test.ts index 5acf382f6..ef22cf0f4 100644 --- a/tests/boundary/providers/hopper/hopperClient.restart.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.restart.test.ts @@ -30,7 +30,11 @@ describe("HopperClient restart", () => { await expect(client.start()).resolves.toEqual({ ok: true, - value: { name: "REA Hopper bridge", version: "1.0.0" }, + value: { + name: "REA Hopper bridge", + version: "1.0.0", + processId: expect.any(Number), + }, }); }); @@ -42,7 +46,11 @@ describe("HopperClient restart", () => { await closing; await expect(restarting).resolves.toEqual({ ok: true, - value: { name: "REA Hopper bridge", version: "1.0.0" }, + value: { + name: "REA Hopper bridge", + version: "1.0.0", + processId: expect.any(Number), + }, }); await expect(client.callTool("echo", { value: "fresh" })).resolves.toEqual({ ok: true, diff --git a/tests/composition/idaProvider.test.ts b/tests/composition/idaProvider.test.ts index 538c9d30e..58f3b78da 100644 --- a/tests/composition/idaProvider.test.ts +++ b/tests/composition/idaProvider.test.ts @@ -186,7 +186,7 @@ describe("IDA provider composition", () => { }); describe("IDA binary_overview execution", () => { - it("reports document selection unavailable when binary_overview executes against an IDA session", async () => { + it("uses the bound IDA document and omits unsupported segment coverage", async () => { const { root, target } = await createIdaTarget(); roots.push(root); const path = join(root, "registration.json"); @@ -200,17 +200,16 @@ describe("IDA binary_overview execution", () => { expect((await session.open(target.path)).ok).toBe(true); const tools = new EnhancedTools(session); const overview = await tools.execute("binary_overview", {}); - expect(overview).toMatchObject({ - ok: false, - error: { - _tag: "AnalysisCapabilityUnavailableError", - providerId: "ida", - operation: "list_documents", - message: expect.stringContaining( - "operation is not declared by this provider", - ), - }, - }); + expect(overview.ok).toBe(true); + if (overview.ok) { + expect(overview.value).toMatchObject({ + document: target.path, + procedure_count: 1, + string_count: 1, + }); + expect(overview.value).not.toHaveProperty("segments"); + expect(overview.value).not.toHaveProperty("segment_count"); + } const procedures = await session.execute("list_procedures", {}); expect(procedures.ok).toBe(true); expect((await session.close()).ok).toBe(true); diff --git a/tests/conformance/ghidra/ReaNoReturnProbe.java b/tests/conformance/ghidra/ReaNoReturnProbe.java index b13c2b5e9..19f98f346 100644 --- a/tests/conformance/ghidra/ReaNoReturnProbe.java +++ b/tests/conformance/ghidra/ReaNoReturnProbe.java @@ -12,6 +12,8 @@ import ghidra.program.model.listing.FlowOverride; import ghidra.program.model.listing.Function; import ghidra.program.model.listing.Instruction; import ghidra.program.model.symbol.SourceType; +import ghidra.program.model.symbol.Reference; +import ghidra.program.model.symbol.RefType; import java.io.File; import java.lang.reflect.Method; import java.nio.file.Files; @@ -20,9 +22,12 @@ import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.ArrayList; /** Injects reported analysis flags into imported ELF/Mach-O functions; never executes target code. */ public class ReaNoReturnProbe extends GhidraScript { + private record MemoryCallReference(Address source, Address target, RefType type, + SourceType sourceType, int operandIndex, boolean primary) {} private final JsonArray checks = new JsonArray(); private final Map originalEnds = new LinkedHashMap<>(); private final Map originalSignatures = new LinkedHashMap<>(); @@ -85,6 +90,114 @@ public class ReaNoReturnProbe extends GhidraScript { return instance; } + private void knownInteriorCall(GhidraScript bridge) throws Exception { + Function caller = function("rea_tls_chain"); + Function target = function("rea_tls"); + Address interior = target.getEntryPoint().add(1); + require("interior_target_is_owned", currentProgram.getFunctionManager().getFunctionAt(interior) == null && + target.equals(currentProgram.getFunctionManager().getFunctionContaining(interior))); + Instruction call = null; + for (Instruction instruction : currentProgram.getListing().getInstructions(caller.getBody(), true)) { + if (instruction.getFlowType().isCall()) { call = instruction; break; } + } + if (call == null) throw new IllegalStateException("Missing real call instruction"); + var references = currentProgram.getReferenceManager(); + List originals = new ArrayList<>(); + List saved = new ArrayList<>(); + for (Reference reference : call.getReferencesFrom()) { + if (!reference.getReferenceType().isCall()) continue; + originals.add(reference); + saved.add(new MemoryCallReference(reference.getFromAddress(), reference.getToAddress(), + reference.getReferenceType(), reference.getSource(), reference.getOperandIndex(), reference.isPrimary())); + } + require("original_call_reference_present", !originals.isEmpty()); + Reference changed = null; + try { + for (Reference reference : originals) references.delete(reference); + changed = references.addMemoryReference(call.getAddress(), interior, + RefType.UNCONDITIONAL_CALL, SourceType.USER_DEFINED, 0); + require("interior_target_omitted_by_callee_getter", !caller.getCalledFunctions(monitor).contains(target)); + JsonObject query = new JsonObject(); + query.addProperty("document", currentProgram.getName()); + query.addProperty("procedure", caller.getName()); + query.addProperty("direction", "outgoing"); + Method read = bridge.getClass().getDeclaredMethod("procedureReferences", JsonObject.class); + read.setAccessible(true); + JsonObject output = (JsonObject) read.invoke(bridge, query); + require("interior_reference_keeps_known_procedure", output.getAsJsonArray("references").asList().stream().anyMatch( + edge -> !edge.getAsJsonObject().get("target_procedure").isJsonNull() && + edge.getAsJsonObject().getAsJsonObject("target_procedure").get("name").getAsString().equals(target.getName()))); + require("known_reference_is_not_also_unresolved", output.getAsJsonArray("unresolved_calls").isEmpty()); + query.remove("direction"); + Method callees = bridge.getClass().getDeclaredMethod("procedureCalls", JsonObject.class, boolean.class); + callees.setAccessible(true); + JsonObject inventory = (JsonObject) callees.invoke(bridge, query, false); + require("omitted_callee_inventory_stays_partial", inventory.getAsJsonObject("relationship_coverage").get("status").getAsString().equals("partial")); + require("omitted_callee_direction_is_explicit", + inventory.getAsJsonObject("relationship_coverage").getAsJsonArray("limited_directions").toString().equals("[\"callees\"]")); + require("interior_caller_omitted_by_entry_getter", !target.getCallingFunctions(monitor).contains(caller)); + query.addProperty("procedure", target.getName()); + JsonObject incoming = (JsonObject) callees.invoke(bridge, query, true); + require("off_entry_caller_inventory_stays_partial", + incoming.getAsJsonObject("relationship_coverage").get("status").getAsString().equals("partial")); + require("off_entry_caller_direction_is_explicit", + incoming.getAsJsonObject("relationship_coverage").getAsJsonArray("limited_directions").toString().equals("[\"callers\"]")); + JsonObject dossierQuery = new JsonObject(); + dossierQuery.addProperty("procedure", target.getName()); + Method analyze = bridge.getClass().getDeclaredMethod("analyzeFunctionObservation", JsonObject.class); + analyze.setAccessible(true); + Object observation = analyze.invoke(bridge, dossierQuery); + Method coverage = observation.getClass().getDeclaredMethod("relationshipCoverage"); + coverage.setAccessible(true); + require("dossier_preserves_caller_only_gap", + ((JsonObject) coverage.invoke(observation)).getAsJsonArray("limited_directions").toString().equals("[\"callers\"]")); + } finally { + if (changed != null) references.delete(changed); + for (MemoryCallReference reference : saved) { + Reference restored = references.addMemoryReference(reference.source(), reference.target(), + reference.type(), reference.sourceType(), reference.operandIndex()); + references.setPrimary(restored, reference.primary()); + } + } + } + + private void nonCallInstructionReference(GhidraScript bridge) throws Exception { + Function caller = function("rea_tls_chain"); + Function target = function("rea_memcpy"); + Instruction source = null; + for (Instruction instruction : currentProgram.getListing().getInstructions(caller.getBody(), true)) { + if (!instruction.getFlowType().isCall()) { source = instruction; break; } + } + if (source == null) throw new IllegalStateException("Missing non-call instruction"); + var references = currentProgram.getReferenceManager(); + Reference added = references.addMemoryReference(source.getAddress(), target.getEntryPoint().add(1), + RefType.UNCONDITIONAL_CALL, SourceType.USER_DEFINED, -1); + try { + require("non_call_reference_target_omitted", !caller.getCalledFunctions(monitor).contains(target)); + JsonObject query = new JsonObject(); + query.addProperty("document", currentProgram.getName()); + query.addProperty("procedure", caller.getName()); + Method read = bridge.getClass().getDeclaredMethod("procedureCalls", JsonObject.class, boolean.class); + read.setAccessible(true); + JsonObject output = (JsonObject) read.invoke(bridge, query, false); + require("non_call_instruction_reference_stays_partial", + output.getAsJsonObject("relationship_coverage").get("status").getAsString().equals("partial")); + query.addProperty("direction", "outgoing"); + Method edges = bridge.getClass().getDeclaredMethod("procedureReferences", JsonObject.class); + edges.setAccessible(true); + JsonObject known = (JsonObject) edges.invoke(bridge, query); + require("non_call_known_reference_is_not_unresolved", known.getAsJsonArray("unresolved_calls").isEmpty()); + references.delete(added); + Address unknown = currentProgram.getMemory().getMaxAddress().add(0x1000); + added = references.addMemoryReference(source.getAddress(), unknown, + RefType.UNCONDITIONAL_CALL, SourceType.USER_DEFINED, -1); + JsonObject unresolved = (JsonObject) edges.invoke(bridge, query); + require("non_call_unknown_reference_keeps_site", unresolved.getAsJsonArray("unresolved_calls").size() == 1); + } finally { + references.delete(added); + } + } + @Override public void run() throws Exception { String[] args = getScriptArgs(); @@ -164,6 +277,8 @@ public class ReaNoReturnProbe extends GhidraScript { Object output = pseudocode.invoke(bridge, query); require("recovered_pseudocode", output instanceof JsonObject value && value.get("value").getAsString().contains("rea_after_call")); + nonCallInstructionReference(bridge); + knownInteriorCall(bridge); } finally { ((ghidra.app.decompiler.DecompInterface) field.get(bridge)).dispose(); } diff --git a/tests/conformance/providers/ghidra/windowsCapabilities.test.ts b/tests/conformance/providers/ghidra/windowsCapabilities.test.ts index 86fe3ec3b..abc96587d 100644 --- a/tests/conformance/providers/ghidra/windowsCapabilities.test.ts +++ b/tests/conformance/providers/ghidra/windowsCapabilities.test.ts @@ -17,16 +17,25 @@ const dependencies = ( available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, private_runtime_dacl: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, reparse_safe_path_admission: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, }), probeSymlinkCreation: () => @@ -91,16 +100,25 @@ describe("Windows host capability report", () => { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, private_runtime_dacl: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, reparse_safe_path_admission: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, }, }); diff --git a/tests/fixtures/fakeGhidra.mjs b/tests/fixtures/fakeGhidra.mjs index 0a130c744..a7f29ba12 100644 --- a/tests/fixtures/fakeGhidra.mjs +++ b/tests/fixtures/fakeGhidra.mjs @@ -32,7 +32,15 @@ if (mode === "silent") setInterval(() => undefined, 1_000); else { await rm(endpointPath, { force: true }); const sockets = new Set(); + let rejectedConnections = 0; const server = createServer((socket) => { + if ( + mode === "reject_all_connections" || + (mode === "reject_first_connection" && rejectedConnections++ === 0) + ) { + socket.destroy(); + return; + } sockets.add(socket); socket.once("close", () => sockets.delete(socket)); socket.setEncoding("utf8"); diff --git a/tests/fixtures/fakeHopper.mjs b/tests/fixtures/fakeHopper.mjs index 192d69851..2491988a4 100644 --- a/tests/fixtures/fakeHopper.mjs +++ b/tests/fixtures/fakeHopper.mjs @@ -217,6 +217,7 @@ const server = createServer((socket) => { name: "REA Hopper bridge", version: "1.0.0", run_id: runId, + process_id: process.pid, }, }, true, diff --git a/tests/fixtures/ghidraHandshakeProbe.py b/tests/fixtures/ghidraHandshakeProbe.py new file mode 100644 index 000000000..1c363e51a --- /dev/null +++ b/tests/fixtures/ghidraHandshakeProbe.py @@ -0,0 +1,107 @@ +"""JDK-only TCP transport regression using the production admission methods. + +Run: python3 tests/fixtures/ghidraHandshakeProbe.py /path/to/java +Ghidra request parsing/dispatch are fixture seams; no engine is required. +""" +import pathlib +import subprocess +import sys +import tempfile + +root = pathlib.Path(__file__).resolve().parents[2] +bridge = (root / "bridge/ghidra/ReaGhidraBridge.java").read_text() +start = bridge.index(" private static String readLoopbackHandshake(") +end = bridge.index(" private void acceptClient(", start) +methods = bridge[start:end] +harness = r''' +import java.io.*; +import java.net.*; +import java.nio.channels.*; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.*; + +public class ReaHandshakeProbe { + record SessionDescriptor(String token) {} + record Request(String method) {} + private Request parseRequest(String line, String token) { + if (!line.equals(token + ":ping")) throw new IllegalArgumentException(); + return new Request("ping"); + } + private void serveClient(SessionDescriptor descriptor, BufferedReader reader, + BufferedWriter writer, Request first) throws Exception { + if (!first.method().equals("ping")) throw new AssertionError("first request lost"); + String next = reader.readLine(); + if (next == null || next.length() != 200_000) + throw new AssertionError("pipelined authenticated request lost or capped"); + writer.write("accepted\n"); + writer.flush(); + } + static Socket connect(int port) throws Exception { + Socket peer = new Socket("127.0.0.1", port); + peer.setSoTimeout(3_000); + return peer; + } + static void closed(Socket peer) throws Exception { + if (peer.getInputStream().read() != -1) throw new AssertionError("peer not rejected"); + } + public static void main(String[] args) throws Exception { + try (ServerSocketChannel server = ServerSocketChannel.open(); + ExecutorService worker = Executors.newSingleThreadExecutor()) { + server.bind(new InetSocketAddress("127.0.0.1", 0)); + int port = ((InetSocketAddress) server.getLocalAddress()).getPort(); + Future serving = worker.submit(() -> { + try { new ReaHandshakeProbe().acceptAuthenticatedLoopbackClient( + server, new SessionDescriptor("secret")); } + catch (Exception error) { throw new RuntimeException(error); } + }); + try (Socket invalid = connect(port)) { + invalid.getOutputStream().write("bad-token:ping\n".getBytes(StandardCharsets.UTF_8)); + closed(invalid); + } + // EOF before a newline is not an authenticated request. + try (Socket eof = connect(port)) { + eof.getOutputStream().write("secret:ping".getBytes(StandardCharsets.UTF_8)); + eof.shutdownOutput(); + closed(eof); + } + try (Socket oversized = connect(port)) { + oversized.getOutputStream().write(new byte[64 * 1024 + 1]); + closed(oversized); + } + try (Socket idle = connect(port)) { + long start = System.nanoTime(); + closed(idle); + long elapsed = (System.nanoTime() - start) / 1_000_000; + if (elapsed < 700 || elapsed > 2_500) throw new AssertionError("idle deadline " + elapsed); + } + try (Socket trickle = connect(port)) { + long start = System.nanoTime(); + for (int i = 0; i < 4; i++) { + trickle.getOutputStream().write('x'); + Thread.sleep(250); + } + closed(trickle); + if ((System.nanoTime() - start) / 1_000_000 > 2_000) + throw new AssertionError("slow input extended deadline"); + } + // Same write contains the first and the following frame. The first + // reader must consume exactly one line without swallowing lookahead. + try (Socket valid = connect(port)) { + valid.getOutputStream().write(("secret:ping\n" + "x".repeat(200_000) + "\n") + .getBytes(StandardCharsets.UTF_8)); + String reply = new BufferedReader(new InputStreamReader(valid.getInputStream(), + StandardCharsets.UTF_8)).readLine(); + if (!"accepted".equals(reply)) throw new AssertionError("legitimate client failed"); + } + serving.get(3, TimeUnit.SECONDS); + } + System.out.println("PASS: rejection, EOF, frame ceiling, idle/trickle deadlines, retry, pipeline, authenticated large request"); + } +__METHODS__ +} +'''.replace("__METHODS__", methods) +with tempfile.TemporaryDirectory(prefix="rea-ghidra-handshake-") as directory: + source = pathlib.Path(directory) / "ReaHandshakeProbe.java" + source.write_text(harness) + subprocess.run([sys.argv[1] if len(sys.argv) > 1 else "java", str(source)], + check=True, timeout=30) From d82ad1faf0b2774c56a10d8a81b09defacd99c7b Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:28:23 +0800 Subject: [PATCH 05/26] test(android): preserve malformed DEX prototype rejection (#2917) Preserve malformed DEX parameter/return rejection with tagged producer errors and exact issue paths. The equivalent portable descriptor fix is already on main via #2910. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- tests/boundary/android/JadxAnalysis.test.ts | 62 +++++++++++++-------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/tests/boundary/android/JadxAnalysis.test.ts b/tests/boundary/android/JadxAnalysis.test.ts index 36b0ab9d9..ad7d0d614 100644 --- a/tests/boundary/android/JadxAnalysis.test.ts +++ b/tests/boundary/android/JadxAnalysis.test.ts @@ -279,26 +279,44 @@ it("preserves bounded instruction coverage separately from using-node coverage", }); }); -it("rejects malformed method descriptors at the Android adapter boundary", async () => { - const invalidSummary = { - ...summary, - methods: summary.methods.map((method) => ({ - ...method, - dex_descriptor: "not-a-dex-descriptor", - })), - }; - await expect( - analyzeJadxRequest( - { - json: () => Promise.resolve(invalidSummary), - text: () => Promise.reject(new Error("unexpected text call")), +it.each([ + "not-a-dex-descriptor", + "(Lbad(name;)V", + "(Lbad)name;)V", + "()Lbad(name;", + "()Lbad)name;", +])( + "rejects malformed DEX prototype %s at the adapter boundary", + async (descriptor) => { + const invalidSummary = { + ...summary, + methods: summary.methods.map((method) => ({ + ...method, + dex_descriptor: descriptor, + })), + }; + await expect( + analyzeJadxRequest( + { + json: () => Promise.resolve(invalidSummary), + text: () => Promise.reject(new Error("unexpected text call")), + }, + { + operation: "inspect_android_class", + input: { path: "/fixture.apk", class_name: "fixture.Target" }, + }, + {} as never, + engine, + ), + ).rejects.toMatchObject({ + _tag: "AnalysisOutputError", + cause: { + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["methods", 0, "dex_descriptor"], + }), + ]), }, - { - operation: "inspect_android_class", - input: { path: "/fixture.apk", class_name: "fixture.Target" }, - }, - {} as never, - engine, - ), - ).rejects.toThrow(); -}); + }); + }, +); From 55dd21a5e2172187dfe7167df684f85ecb536f5e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:29:16 +0800 Subject: [PATCH 06/26] fix(native): retain partial format facts and explicit coverage Preserve XML and Interface Builder identity at their parsing boundary, retain usable Objective-C slots with precise coverage, and keep native trace failures distinct from absence. Add caller-selected Interface Builder summaries backed by complete retained Evidence, and preserve comparison limitations. Reviewed fixes: #1969, #2088, #2108, #2513, #2639, #2684, #2731, #2759, #2875, #2888, #2889, #2970, #2971, #3002. Validation: relevant native/domain focused lane passed 142 tests; earlier Interface Builder/MCP checks passed 24 tests. Real native UI and Ghidra producer coverage was not repeated. --- docs/native-investigation.md | 15 +- .../EnhancedNativeInvestigation.test.ts | 20 + src/application/native/NativeUiActionTrace.ts | 5 + .../native/NativeValueTrace.test.ts | 356 +++++++++++++++--- src/application/native/NativeValueTrace.ts | 94 +++-- src/contracts/analysisViewToolContracts.ts | 4 +- src/contracts/artifactToolContracts.ts | 8 +- src/contracts/investigationExamples.ts | 1 + src/contracts/native/nativeToolContracts.ts | 16 +- src/contracts/toolOutputSchemaGroups.ts | 32 +- src/domain/analysisView/androidPackageView.ts | 19 +- src/domain/analysisView/dartAotView.ts | 19 +- src/domain/analysisView/evmInterfaceView.ts | 19 +- .../analysisView/interfaceBuilderView.test.ts | 168 +++++++++ .../analysisView/interfaceBuilderView.ts | 248 ++++++++++++ src/domain/analysisView/machoView.ts | 19 +- .../analysisView/managedArtifactView.ts | 22 +- src/domain/analysisView/wasmArtifactView.ts | 19 +- src/domain/callPath.test.ts | 36 ++ src/domain/callPath.ts | 4 +- src/domain/changedBehavior.ts | 20 +- src/domain/comparisonSourceEvidence.ts | 12 + src/domain/functionCallCoverage.test.ts | 40 +- src/domain/functionCallCoverage.ts | 37 +- src/domain/functionComparison.test.ts | 61 +++ src/domain/functionComparisonDimensions.ts | 9 +- src/domain/hopperValues.ts | 8 +- src/domain/propertyListKeys.test.ts | 55 ++- src/domain/propertyListKeys.ts | 132 ++++++- ...uctionVerification.functionSources.test.ts | 4 +- src/domain/reconstructionVerification.test.ts | 87 ++++- src/domain/reconstructionVerification.ts | 57 +-- src/domain/staticRuntimeCorrelation.test.ts | 72 +++- src/domain/staticRuntimeCorrelation.ts | 32 +- src/go/GoBinaryContainer.ts | 6 +- src/go/GoBinaryImage.ts | 35 +- src/go/GoElfImage.ts | 6 +- src/go/GoMachoImage.ts | 2 +- src/go/GoPeImage.ts | 2 +- src/native/AppleDispatchCoverage.ts | 33 +- src/native/AppleDispatchMetadata.test.ts | 276 ++++++++++++++ src/native/AppleDispatchMetadata.ts | 27 +- src/native/AppleObjcDispatchFacets.ts | 87 ++++- src/native/AppleSwiftVtables.test.ts | 4 + src/native/parsers/demangle.test.ts | 90 +++++ src/native/parsers/demangle.ts | 8 +- src/server/registerArtifactTools.ts | 48 +-- src/server/registerManagedTools.ts | 13 +- src/server/registerNativeTools.ts | 47 +-- src/server/retainedSummaryDelivery.ts | 62 +++ tests/boundary/go/goBinaryImage.test.ts | 102 ++++- .../mcp/interfaceBuilderSummaryMcp.test.ts | 152 ++++++++ .../mcp/keyedArchiveNumberMetadataMcp.test.ts | 26 +- .../mcp/keyedArchivePrototypeKeyMcp.test.ts | 26 +- .../mcp/keyedArchiveXmlExtensionMcp.test.ts | 86 +++++ tests/boundary/mcp/nativeDemangleMcp.test.ts | 24 +- .../mcp/nativeMachoSummaryMcp.test.ts | 23 +- .../artifactProvider.directoryProbes.test.ts | 138 +++++++ .../artifactProvider.failureDetail.test.ts | 28 +- 59 files changed, 2602 insertions(+), 499 deletions(-) create mode 100644 src/domain/analysisView/interfaceBuilderView.test.ts create mode 100644 src/domain/analysisView/interfaceBuilderView.ts create mode 100644 src/native/parsers/demangle.test.ts create mode 100644 src/server/retainedSummaryDelivery.ts create mode 100644 tests/boundary/mcp/interfaceBuilderSummaryMcp.test.ts create mode 100644 tests/boundary/mcp/keyedArchiveXmlExtensionMcp.test.ts create mode 100644 tests/boundary/providers/native/artifactProvider.directoryProbes.test.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index f963fd4d8..be1380ef0 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -227,6 +227,10 @@ nodes and edges. Node payloads are bounded to 8 MiB and serialized results to 32 MiB. Pagination returns edges whose source is on the node page and preserves stable IDs for endpoints outside that page. +Within one value trace, each exact procedure selector is attempted at most once, +including failed analyses. Failures remain unknown observations and do not +establish canonical function identity. A new trace can retry those selectors. + The raw Java `is_dead` flag can remain set after Ghidra decodes a live block; `block_membership` reports actual syntax-tree membership separately. @@ -319,7 +323,16 @@ complete observation. digest are preserved; key interpretation follows the pinned plist decoder. - `decode_interface_builder` / `rea decode-interface-builder ` reads compiled storyboard and nib resources into objects, outlets, actions, - connections and controller/class names. `verify:interface-builder` compiles + connections and controller/class names. A full app graph is often several + megabytes (XLD.app: 6.7 MB, about 2M tokens, which also exceeds the default + MCP response budget), so MCP callers should pass `detail: "summary"`. The + summary keeps every decoded document and node/edge/coverage counts and + retains the complete Evidence; page rows with `inspect_analysis_view` + `view: {kind: "interface-builder", collection: "nodes", document: + "Contents/Resources/Base.lproj/MainMenu.nib", offset: 0, limit: 64}`. + `document` selects rows whose evidence names that exact + `documents[].relative_path` (nodes and edges only); `source_truncated` keeps + graph incompleteness visible on every page. `verify:interface-builder` compiles a source-owned AppKit XIB with `ibtool`. Storyboards require an installed iOS platform; unsupported archive forms remain explicit. XML plist archives accept UTF-8 and BOM-marked UTF-16 in either byte order. diff --git a/src/application/native/EnhancedNativeInvestigation.test.ts b/src/application/native/EnhancedNativeInvestigation.test.ts index 1e1b517b6..0a28f681c 100644 --- a/src/application/native/EnhancedNativeInvestigation.test.ts +++ b/src/application/native/EnhancedNativeInvestigation.test.ts @@ -12,6 +12,7 @@ import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, + AnalysisOutputError, } from "../../domain/analysisErrorCore.js"; import { EnhancedTools } from "../EnhancedTools.js"; import { @@ -615,6 +616,25 @@ describe("native UI query outcomes", () => { }); }); + it("returns native seed resolution failures instead of reporting not found", async () => { + const result = await new EnhancedTools( + testAnalysis({ + procedure_address: () => + err( + new AnalysisOutputError( + "procedure_address", + "invalid address response", + ), + ), + }), + ).execute("trace_native_ui_action", { action: "missing-selector" }); + + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError", operation: "procedure_address" }, + }); + }); + it("returns cancellation instead of a successful trace during callee expansion", async () => { const result = await new EnhancedTools( testAnalysis({ diff --git a/src/application/native/NativeUiActionTrace.ts b/src/application/native/NativeUiActionTrace.ts index 8a6affdf7..cac49875d 100644 --- a/src/application/native/NativeUiActionTrace.ts +++ b/src/application/native/NativeUiActionTrace.ts @@ -125,6 +125,11 @@ export const traceNativeUiAction = async ( ); if (!native.ok && native.error._tag === "AnalysisCancelledError") return err(native.error); + if ( + !native.ok && + native.error._tag !== "AnalysisCapabilityUnavailableError" + ) + return err(native.error); if (native.ok) { const address = z.string().safeParse(native.value.result); if (!address.success || native.value.subject?.sha256 !== targetSha256) diff --git a/src/application/native/NativeValueTrace.test.ts b/src/application/native/NativeValueTrace.test.ts index cdc663d54..56ed3761d 100644 --- a/src/application/native/NativeValueTrace.test.ts +++ b/src/application/native/NativeValueTrace.test.ts @@ -8,7 +8,10 @@ import { ghidraFunctionDossier } from "../../domain/ghidraValues.fixture.js"; import { functionDossierSchema } from "../../domain/hopperValues.js"; import { jsonValueSchema } from "../../domain/jsonValue.js"; import { nativeValueTraceSchema } from "../../domain/native/nativeValueTrace.js"; -import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; import { ok, err } from "../../domain/result.js"; const subject = { @@ -181,52 +184,71 @@ describe("native value parameter-use node membership", () => { }); }); - it("preserves operation IDs that collide with generated parameter IDs", async () => { - const base = functionDossierSchema.parse(ghidraFunctionDossier()); - const address = base.procedure.address; - const execution = createAnalysisExecution( - { - ...base, - native_value_flow: { - available: true, - provenance: "ghidra-high-pcode", - operations: [ - node("parameter:0", address, "COPY"), - node("use", address, "COPY"), - ], - def_use: [], - effects: [], - parameters: [{ ordinal: 0, name: "value", data_type: "/int" }], - parameter_uses: [{ ordinal: 0, use: "use", input_index: 1 }], - truncated: false, - omitted_operations_lower_bound: 0, - known_omitted_inputs: 0, - known_omitted_edges: 0, - limitations: [], - }, - }, - provider, - { subject }, - ); - const result = await traceNativeValues( - { execute: async () => ok(execution) }, - { procedure: address, max_nodes: 2 }, - ); - - if (!result.ok) throw result.error; - const parsedResult = nativeValueTraceSchema.parse(result.value); - expect(parsedResult).toMatchObject({ - total_nodes: 2, - total_edges: 1, - edges: [ + it.each([3, 4])( + "keeps parameter identity separate from colliding operation IDs with max_nodes=%i", + async (max_nodes) => { + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const address = base.procedure.address; + const execution = createAnalysisExecution( { - source: `${address}/parameter:0`, - target: `${address}/use`, - kind: "parameter-use", + ...base, + native_value_flow: { + available: true, + provenance: "ghidra-high-pcode", + operations: [ + node("parameter:0", address, "COPY"), + node("parameter:0#1", address, "COPY"), + node("use", address, "COPY"), + ], + def_use: [], + effects: [], + parameters: [{ ordinal: 0, name: "value", data_type: "/int" }], + parameter_uses: [{ ordinal: 0, use: "use", input_index: 1 }], + truncated: false, + omitted_operations_lower_bound: 0, + known_omitted_inputs: 0, + known_omitted_edges: 0, + limitations: [], + }, }, - ], - }); - }); + provider, + { subject }, + ); + const result = await traceNativeValues( + { execute: async () => ok(execution) }, + { procedure: address, max_nodes }, + ); + + if (!result.ok) throw result.error; + const parsedResult = nativeValueTraceSchema.parse(result.value); + expect(parsedResult).toMatchObject({ + total_nodes: max_nodes, + total_edges: max_nodes === 3 ? 0 : 1, + edges: + max_nodes === 3 + ? [] + : [ + { + source: `${address}/parameter:0#2`, + target: `${address}/use`, + kind: "parameter-use", + }, + ], + }); + expect( + parsedResult.nodes + .filter(({ kind }) => kind === "operation") + .map(({ id }) => id), + ).toEqual([ + `${address}/parameter:0`, + `${address}/parameter:0#1`, + `${address}/use`, + ]); + expect(new Set(parsedResult.nodes.map(({ id }) => id)).size).toBe( + max_nodes, + ); + }, + ); }); const nativeFlowDossier = ( @@ -316,6 +338,97 @@ const callBindingAnalysis = ( }, }); +describe("native value failed-callee traversal", () => { + it("attempts a repeated failed selector once and still analyzes a later callee", async () => { + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const caller = nativeFlowDossier(base, { + address: "0x1000", + name: "caller", + operations: [ + node("bad-1", "0x1100", "CALL"), + node("bad-2", "0x1104", "CALL"), + node("good", "0x1108", "CALL"), + ], + parameters: [], + }); + const good = nativeFlowDossier(base, { + address: "0x3000", + name: "good", + operations: [], + parameters: [], + }); + const attempts: string[] = []; + const port: AnalysisOperationPort = { + execute: async (operation, parameters) => { + if (operation === "resolve_native_call_targets") { + const destination = + parameters.address === "0x1108" ? "0x3000" : "0x2000"; + return ok( + createAnalysisExecution( + { + call_site: String(parameters.address), + procedure: "0x1000", + status: "direct", + mechanism: "direct", + targets: [ + { + address: destination, + procedure: destination, + status: "direct", + basis: "provider-reference", + references: [], + }, + ], + limitations: [], + }, + provider, + { subject }, + ), + ); + } + expect(operation).toBe("analyze_function"); + const selector = String(parameters.procedure); + attempts.push(selector); + if (selector === "0x2000") + return err( + new AnalysisOutputError("analyze_function", "Callee unavailable"), + ); + return ok( + createAnalysisExecution( + selector === "0x1000" ? caller : good, + provider, + { subject }, + ), + ); + }, + }; + // A new invocation retries the failed selector: failures are not cached + // across traces, and successful procedure identity remains separate. + for (let run = 0; run < 2; run++) { + attempts.length = 0; + const result = await traceNativeValues(port, { + procedure: "0x1000", + max_functions: 3, + }); + if (!result.ok) throw result.error; + const trace = nativeValueTraceSchema.parse(result.value); + expect(attempts).toEqual(["0x1000", "0x2000", "0x3000"]); + expect(trace.decompilations).toBe(3); + expect(trace.procedures.map(({ address }) => address)).toEqual([ + "0x1000", + "0x3000", + ]); + expect(trace.unknowns).toContainEqual({ + procedure: "0x2000", + address: null, + reason: expect.stringContaining("Callee unavailable"), + }); + expect(trace.edges.filter(({ kind }) => kind === "call")).toHaveLength(3); + expect(trace.truncated).toBe(false); + } + }); +}); + describe("native value call-binding fan-out", () => { it("preserves binding order and edge-budget behavior for repeated callees", async () => { const callCount = 128; @@ -558,3 +671,156 @@ describe("native value trace selector origin", () => { ]); }); }); + +describe("native value node byte budgets", () => { + it("stops admitting parameters after an operation exceeds the byte budget", async () => { + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const value = nativeFlowDossier(base, { + address: "0x1000", + name: "caller", + operations: [node("use", "0x1000", "a".repeat(9 * 1024 * 1024))], + parameters: [{ ordinal: 0, name: "value", data_type: "/int" }], + }); + const result = await traceNativeValues( + { + execute: async () => + ok(createAnalysisExecution(value, provider, { subject })), + }, + { procedure: "0x1000" }, + ); + if (!result.ok) throw result.error; + expect(nativeValueTraceSchema.parse(result.value)).toMatchObject({ + truncated: true, + total_nodes: 0, + total_edges: 0, + nodes: [], + edges: [], + }); + }); + + it.each([ + { label: "plain text", name: "a".repeat(9 * 1024 * 1024) }, + { label: "JSON escapes", name: "\0".repeat(2 * 1024 * 1024) }, + ])("rejects an oversized parameter encoded with $label", async ({ name }) => { + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const value = nativeFlowDossier(base, { + address: "0x1000", + name: "caller", + operations: [ + node("parameter:0", "0x1000", "COPY"), + node("use", "0x1000", "MULTIEQUAL"), + ], + parameters: [{ ordinal: 0, name, data_type: "/int" }], + }); + const parsed = functionDossierSchema.parse(value); + if (!parsed.native_value_flow?.available) + throw new Error("Invalid fixture"); + parsed.native_value_flow.parameter_uses = [ + { ordinal: 0, use: "use", input_index: 0 }, + ]; + const result = await traceNativeValues( + { + execute: async () => + ok(createAnalysisExecution(parsed, provider, { subject })), + }, + { procedure: "0x1000" }, + ); + if (!result.ok) throw result.error; + const trace = nativeValueTraceSchema.parse(result.value); + expect(trace).toMatchObject({ + truncated: true, + total_nodes: 2, + total_edges: 0, + nodes: [ + expect.objectContaining({ kind: "operation" }), + expect.objectContaining({ kind: "operation" }), + ], + }); + }); + + it("shares the byte budget across operation and parameter nodes", async () => { + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const value = nativeFlowDossier(base, { + address: "0x1000", + name: "caller", + operations: [node("use", "0x1000", "MULTIEQUAL")], + parameters: [0, 1].map((ordinal) => ({ + ordinal, + name: "a".repeat(5 * 1024 * 1024), + data_type: "/int", + })), + }); + const parsed = functionDossierSchema.parse(value); + if (!parsed.native_value_flow?.available) + throw new Error("Invalid fixture"); + parsed.native_value_flow.parameter_uses = [0, 1].map((ordinal) => ({ + ordinal, + use: "use", + input_index: ordinal, + })); + const result = await traceNativeValues( + { + execute: async () => + ok(createAnalysisExecution(parsed, provider, { subject })), + }, + { procedure: "0x1000" }, + ); + if (!result.ok) throw result.error; + const trace = nativeValueTraceSchema.parse(result.value); + expect(trace).toMatchObject({ + truncated: true, + total_nodes: 2, + total_edges: 1, + }); + expect(trace.nodes.map(({ id }) => id)).toEqual([ + "0x1000/use", + "0x1000/parameter:0", + ]); + expect(trace.edges[0]?.source).toBe("0x1000/parameter:0"); + const retainedBytes = trace.nodes.reduce( + (total, value) => total + Buffer.byteLength(JSON.stringify(value)), + 0, + ); + expect(retainedBytes).toBeLessThanOrEqual(8 * 1024 * 1024); + }); +}); + +describe("native value byte-bound call bindings", () => { + it("omits argument bindings to rejected callee parameter nodes", async () => { + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const caller = nativeFlowDossier(base, { + address: "0x1000", + name: "caller", + operations: [node("call", "0x1000", "CALL")], + parameters: [], + }); + const callee = nativeFlowDossier(base, { + address: "0x2000", + name: "callee", + operations: [node("return", "0x2000", "RETURN")], + parameters: [ + { ordinal: 0, name: "a".repeat(9 * 1024 * 1024), data_type: "/int" }, + ], + }); + const result = await traceNativeValues( + callBindingAnalysis("0x1000", "0x2000", caller, callee), + { procedure: "0x1000" }, + ); + if (!result.ok) throw result.error; + const trace = nativeValueTraceSchema.parse(result.value); + expect(trace.truncated).toBe(true); + expect(trace.nodes.map(({ kind }) => kind)).toEqual([ + "operation", + "operation", + ]); + expect(trace.edges.map(({ kind }) => kind)).toEqual([ + "call", + "return-binding", + ]); + expect(trace.unknowns).toContainEqual( + expect.objectContaining({ + reason: "Parameter bindings for 0x2000 are unavailable or omitted", + }), + ); + }); +}); diff --git a/src/application/native/NativeValueTrace.ts b/src/application/native/NativeValueTrace.ts index 71c602396..fc304201d 100644 --- a/src/application/native/NativeValueTrace.ts +++ b/src/application/native/NativeValueTrace.ts @@ -12,6 +12,9 @@ import { import { err, ok } from "../../domain/result.js"; import type { AnalysisOperationPort } from "../AnalysisProvider.js"; import type { EnhancedResult } from "../EnhancedToolTypes.js"; +import { jsonParts } from "../../domain/jsonSerialization.js"; + +const NODE_PAYLOAD_BYTES = 8 * 1024 * 1024; /** Compose bounded high-p-code def-use graphs across exactly referenced call destinations. */ export const traceNativeValues = async ( @@ -21,7 +24,6 @@ export const traceNativeValues = async ( ): EnhancedResult => { const input = nativeValueTraceInputSchema.parse(parameters); const nodes: typeof nativeValueTraceSchema._output.nodes = []; - const nodeIds = new Set(); const edges: typeof nativeValueTraceSchema._output.edges = []; const procedures: typeof nativeValueTraceSchema._output.procedures = []; const unknowns: typeof nativeValueTraceSchema._output.unknowns = []; @@ -34,17 +36,46 @@ export const traceNativeValues = async ( evidence_id: string; }[] = []; const pending = [{ procedure: input.procedure, depth: 0 }]; + const attempted = new Set(); const visited = new Set(); let targetSha256: string | undefined; let truncated = false; let decompilations = 0; let callSites = 0; let bytes = 0; + let nodeBudgetReached = false; + const retainNode = (value: (typeof nodes)[number]): boolean => { + if (nodeBudgetReached) return false; + if (nodes.length >= input.max_nodes) { + truncated = true; + nodeBudgetReached = true; + return false; + } + let size = 0; + // Reuse bounded JSON encoding to count escapes and UTF-8 expansion + // before allocating a complete serialized node that might not fit. + for (const part of jsonParts(value)) { + size += Buffer.byteLength(part); + if (size > NODE_PAYLOAD_BYTES - bytes) { + truncated = true; + nodeBudgetReached = true; + return false; + } + } + bytes += size; + nodes.push(value); + return true; + }; while (pending.length > 0) { if (signal?.aborted) return err(new AnalysisCancelledError("trace_native_values")); const next = pending.shift(); - if (next === undefined || visited.has(next.procedure)) continue; + if ( + next === undefined || + attempted.has(next.procedure) || + visited.has(next.procedure) + ) + continue; if (decompilations >= input.max_functions) { truncated = true; unknowns.push({ @@ -54,6 +85,9 @@ export const traceNativeValues = async ( }); break; } + // An exact selector is attempted once per trace, even on failure. Only + // successful dossiers establish canonical procedure identity in visited. + attempted.add(next.procedure); const execution = await analysis.execute( "analyze_function", { procedure: next.procedure }, @@ -135,6 +169,10 @@ export const traceNativeValues = async ( }); } const ids = new Set(); + // Keep generated parameter identity independent of which operations fit + // the budget, while preserving every provider-supplied operation ID. + const operationIds = new Set(flow.operations.map(({ id }) => id)); + const parameterIds = new Map(); const qualified = (id: string) => `${address}/${id}`; for (const operation of flow.operations) { if ( @@ -150,44 +188,37 @@ export const traceNativeValues = async ( kind: "operation" as const, parameter: null, }; - const size = Buffer.byteLength(JSON.stringify(node)); - if (nodes.length >= input.max_nodes || bytes + size > 8 * 1024 * 1024) { - truncated = true; - break; - } - bytes += size; + if (!retainNode(node)) break; ids.add(operation.id); - nodes.push(node); - nodeIds.add(node.id); } for (const parameter of flow.parameters) { - if (nodes.length >= input.max_nodes) { - truncated = true; + const parameterId = `parameter:${parameter.ordinal}`; + let localId = parameterId; + for (let suffix = 1; operationIds.has(localId); suffix++) + localId = `${parameterId}#${suffix}`; + const id = qualified(localId); + if ( + !retainNode({ + id, + procedure: address, + evidence_id: evidenceId, + kind: "parameter", + operation: null, + parameter, + }) + ) break; - } - const id = `${address}/parameter:${parameter.ordinal}`; - nodes.push({ - id, - procedure: address, - evidence_id: evidenceId, - kind: "parameter", - operation: null, - parameter, - }); - nodeIds.add(id); + parameterIds.set(parameter.ordinal, id); } for (const use of flow.parameter_uses) { - if ( - !ids.has(use.use) || - !nodeIds.has(`${address}/parameter:${use.ordinal}`) - ) - continue; + const source = parameterIds.get(use.ordinal); + if (!ids.has(use.use) || source === undefined) continue; if (edges.length >= input.max_edges) { truncated = true; break; } edges.push({ - source: `${address}/parameter:${use.ordinal}`, + source, target: qualified(use.use), input_index: use.input_index, kind: "parameter-use", @@ -295,9 +326,10 @@ export const traceNativeValues = async ( } if ( truncated && - (nodes.length >= input.max_nodes || + (nodeBudgetReached || + nodes.length >= input.max_nodes || edges.length >= input.max_edges || - bytes >= 8 * 1024 * 1024) + bytes >= NODE_PAYLOAD_BYTES) ) break; } diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 7d023752d..189757295 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -9,14 +9,14 @@ import { evidenceResultOf } from "./toolOutputSchemaPrimitives.js"; const RETAINED_EXAMPLE_ID = `ev_${"a".repeat(64)}`; -/** Selected views of completed layout, JavaScript, native, inventory, WASM, Mach-O and web network Evidence. */ +/** Selected views of completed layout, JavaScript, native, inventory, WASM, Mach-O, Interface Builder and web network Evidence. */ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ { name: "inspect_analysis_view", ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, inspect_evm_interface, inspect_managed_artifact, inspect_android_package, inspect_dart_aot or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. For layout, JavaScript, inventory, WASM, Mach-O, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions; occurrence logical_paths matches exact recorded strings (omitted means all available, [] means none). source_complete distinguishes complete inventories from partial source records; page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_android_package, inspect_dart_aot or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions; occurrence logical_paths matches exact recorded strings (omitted means all available, [] means none). source_complete distinguishes complete inventories from partial source records; page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/contracts/artifactToolContracts.ts b/src/contracts/artifactToolContracts.ts index 287b41214..3d63488a4 100644 --- a/src/contracts/artifactToolContracts.ts +++ b/src/contracts/artifactToolContracts.ts @@ -89,11 +89,17 @@ export const ARTIFACT_TOOL_CONTRACTS = [ ), artifact( "decode_interface_builder", - "Decode compiled storyboard and nib archives inside the active Apple app bundle, including keyed property lists and NIBArchive object tables, into a bounded object, hierarchy, and connection graph. This is static archive parsing; unrecognized archive classes and unresolved code handlers remain explicit unknowns.", + "Decode compiled storyboard and nib archives inside the active Apple app bundle, including keyed property lists and NIBArchive object tables, into a bounded object, hierarchy, and connection graph. This is static archive parsing; unrecognized archive classes and unresolved code handlers remain explicit unknowns. A full app graph is often several megabytes; detail summary retains it and returns documents with node, edge and coverage counts, then page rows, optionally one document's, with inspect_analysis_view kind interface-builder.", z.strictObject({ max_documents: z.number().int().min(1).max(64).default(64), max_objects: z.number().int().min(1).max(20_000).default(20_000), max_connections: z.number().int().min(1).max(40_000).default(40_000), + detail: analysisDetailSchema + .unwrap() + .optional() + .describe( + "Omitted or complete returns the complete decoded graph Evidence. summary retains that complete Evidence in this session and returns its inspect_analysis_view summary; use normalized_result.parent_evidence_id for interface-builder pages.", + ), }), ), artifact( diff --git a/src/contracts/investigationExamples.ts b/src/contracts/investigationExamples.ts index 8218ee609..96258225c 100644 --- a/src/contracts/investigationExamples.ts +++ b/src/contracts/investigationExamples.ts @@ -28,6 +28,7 @@ export const FUNCTION_COMPARISON_EVIDENCE = createEvidence( result: jsonValueSchema.parse(comparison), confidence: "derived", authority: "analyst-inference", + limitations: comparison.limitations, evidenceLinks: [ FUNCTION_COMPARISON_EXAMPLE.left.evidence_id, FUNCTION_COMPARISON_EXAMPLE.right.evidence_id, diff --git a/src/contracts/native/nativeToolContracts.ts b/src/contracts/native/nativeToolContracts.ts index d45b62cc6..05838bfb3 100644 --- a/src/contracts/native/nativeToolContracts.ts +++ b/src/contracts/native/nativeToolContracts.ts @@ -66,15 +66,13 @@ const native = ( }; /** Ordered single-line symbols; `swift-demangle` ends each result with LF. */ -export const swiftSymbolsSchema = z - .array( - z - .string() - .min(1) - .regex(/^[^\n]*$/u, "Each Swift symbol must be one line.") - .regex(/^[^\x00]*$/u, "Swift symbols cannot contain NUL."), - ) - .min(1); +export const swiftSymbolsSchema = z.array( + z + .string() + .min(1) + .regex(/^[^\n]*$/u, "Each Swift symbol must be one line.") + .regex(/^[^\x00]*$/u, "Swift symbols cannot contain NUL."), +); /** Provider-neutral semantic operations backed initially by macOS utilities. */ export const NATIVE_TOOL_CONTRACTS = [ diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 1a36c0896..bbadef367 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { byteSearchResultSchema } from "../domain/native/bytePattern.js"; import { artifactInventorySummaryViewResultSchema, + interfaceBuilderSummaryViewResultSchema, machoSummaryViewResultSchema, managedArtifactSummaryViewResultSchema, } from "../domain/analysisView/analysisView.js"; @@ -360,17 +361,19 @@ export const enhancedOutputSchemas: Readonly> = { z.object({ document: z.string(), image_base: z.string().min(1).optional(), - segments: z.array( - z.object({ - name: z.string(), - start: z.string(), - end: z.string(), - length: z.number().min(0), - image_base: z.string().exactOptional(), - address_space: z.string().exactOptional(), - }), - ), - segment_count: z.number().int().min(0), + segments: z + .array( + z.object({ + name: z.string(), + start: z.string(), + end: z.string(), + length: z.number().min(0), + image_base: z.string().exactOptional(), + address_space: z.string().exactOptional(), + }), + ) + .optional(), + segment_count: z.number().int().min(0).optional(), procedure_count: z.number().int().min(0), string_count: z.number().int().min(0), }), @@ -404,7 +407,12 @@ export const artifactOutputSchemas: Readonly> = { ]), ), extract_artifact: evidenceResultOf(artifactExtractionResultSchema), - decode_interface_builder: evidenceResultOf(interfaceBuilderAnalysisSchema), + decode_interface_builder: evidenceResultOf( + z.union([ + interfaceBuilderAnalysisSchema, + interfaceBuilderSummaryViewResultSchema, + ]), + ), inspect_keyed_archive: evidenceResultOf(keyedArchiveResultSchema), inspect_asset_catalog: evidenceResultOf(appleAssetCatalogResultSchema), trace_dylib_resolution: evidenceResultOf(dylibResolutionResultSchema), diff --git a/src/domain/analysisView/androidPackageView.ts b/src/domain/analysisView/androidPackageView.ts index caf948c10..519153db1 100644 --- a/src/domain/analysisView/androidPackageView.ts +++ b/src/domain/analysisView/androidPackageView.ts @@ -1,8 +1,8 @@ import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { androidResultSchemas } from "../android/androidAnalysis.js"; -import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { err, ok, type Result } from "../result.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; import { analysisViewInputError, type AnalysisViewParent, @@ -53,19 +53,8 @@ export const projectAndroidPackageView = ( }, ]), ); - const parsed = packageSchema.safeParse(parent.normalizedResult); - if (!parsed.success) - return err( - analysisInputErrorFromIssues( - "inspect_analysis_view", - parsed.error.issues.map((issue) => ({ - ...issue, - path: ["source", "normalized_result", ...issue.path], - })), - { source: { normalized_result: parent.normalizedResult } }, - { cause: parsed.error }, - ), - ); + const parsed = parseAnalysisViewResult(packageSchema, parent); + if (!parsed.ok) return parsed; // Engine artifact_sha256 identifies the JAR, never the APK. A missing // authenticated APK subject cannot be repaired using the engine identity. if (parent.subject == null) @@ -79,7 +68,7 @@ export const projectAndroidPackageView = ( }, ]), ); - const report = parsed.data; + const report = parsed.value; const shared = { parent_evidence_id: parent.evidenceId, parent_operation: "inspect_android_package" as const, diff --git a/src/domain/analysisView/dartAotView.ts b/src/domain/analysisView/dartAotView.ts index 4d66aa190..76242973c 100644 --- a/src/domain/analysisView/dartAotView.ts +++ b/src/domain/analysisView/dartAotView.ts @@ -1,8 +1,8 @@ import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { flutterResultSchemas } from "../flutter/flutterBuildAnalysis.js"; -import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { err, ok, type Result } from "../result.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; import { analysisViewInputError, type AnalysisViewParent, @@ -59,20 +59,9 @@ export const projectDartAotView = ( }, ]), ); - const parsed = aotSchema.safeParse(parent.normalizedResult); - if (!parsed.success) - return err( - analysisInputErrorFromIssues( - "inspect_analysis_view", - parsed.error.issues.map((issue) => ({ - ...issue, - path: ["source", "normalized_result", ...issue.path], - })), - { source: { normalized_result: parent.normalizedResult } }, - { cause: parsed.error }, - ), - ); - const report = parsed.data; + const parsed = parseAnalysisViewResult(aotSchema, parent); + if (!parsed.ok) return parsed; + const report = parsed.value; const { package_uris, dart_uris, class_like_tokens, ...pool } = report.string_pool; const string_pool = { diff --git a/src/domain/analysisView/evmInterfaceView.ts b/src/domain/analysisView/evmInterfaceView.ts index d9f3994d4..f931f21c7 100644 --- a/src/domain/analysisView/evmInterfaceView.ts +++ b/src/domain/analysisView/evmInterfaceView.ts @@ -2,8 +2,8 @@ import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { uniqueSorted } from "../canonicalOrdering.js"; import { evmInterfaceSchema } from "../evm/evmInterface.js"; -import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { err, ok, type Result } from "../result.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; import { analysisViewInputError, type AnalysisViewParent, @@ -60,20 +60,9 @@ export const projectEvmInterfaceView = ( }, ]), ); - const parsed = evmInterfaceSchema.safeParse(parent.normalizedResult); - if (!parsed.success) - return err( - analysisInputErrorFromIssues( - "inspect_analysis_view", - parsed.error.issues.map((issue) => ({ - ...issue, - path: ["source", "normalized_result", ...issue.path], - })), - { source: { normalized_result: parent.normalizedResult } }, - { cause: parsed.error }, - ), - ); - const report = parsed.data; + const parsed = parseAnalysisViewResult(evmInterfaceSchema, parent); + if (!parsed.ok) return parsed; + const report = parsed.value; if ( parent.subject == null || parent.subject.format !== "file" || diff --git a/src/domain/analysisView/interfaceBuilderView.test.ts b/src/domain/analysisView/interfaceBuilderView.test.ts new file mode 100644 index 000000000..a591d3194 --- /dev/null +++ b/src/domain/analysisView/interfaceBuilderView.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it } from "vitest"; + +import { + projectAnalysisView, + type AnalysisViewParent, +} from "./analysisView.js"; + +const target = "a".repeat(64); +const evidence = (artifact_path: string) => [ + { + kind: "interface_builder_resource" as const, + description: "compiled document", + location: { address: null, file_offset: null }, + artifact_path, + artifact_sha256: "b".repeat(64), + }, +]; +const node = (id: string, kind: string, document: string) => ({ + id, + kind, + name: id, + location: null, + attributes: {}, + evidence: evidence(document), +}); +const main = "Contents/Resources/Main.nib"; +const second = "Contents/Resources/Second.nib"; +const document = (relative_path: string) => ({ + relative_path, + archive_sha256: "b".repeat(64), + document_kind: "nib" as const, + object_count: 2, + connection_count: 1, + hierarchy_complete: true, +}); +const report = (truncated = false) => ({ + target_sha256: target, + documents: [document(main), document(second)], + graph: { + target_sha256: target, + provider: { id: "rea-ib", version: "1", tool_version: "1" }, + nodes: [ + node("main:root", "scene", main), + node("main:button", "control", main), + node("second:root", "scene", second), + ], + edges: [ + { + id: "main:edge", + from: "main:root", + to: "main:button", + relation: "contains", + resolution: "observed", + evidence: evidence(main), + limitations: [], + }, + ], + coverage: [ + { + facet: `objects:${main}`, + status: "complete", + reason: null, + examined: 2, + omitted: 0, + }, + ], + truncated, + }, + limitations: ["Recognized archive fields only."], +}); +const parent = ( + normalizedResult: unknown = report(), + sha256 = target, +): AnalysisViewParent => ({ + evidenceId: `ev_${"c".repeat(64)}`, + operation: "decode_interface_builder", + normalizedResult, + subject: { + name: "Fixture", + digest: { sha256 }, + format: "mach-o", + architecture: "arm64", + local_path: "/Applications/Fixture.app/Contents/MacOS/Fixture", + }, + limitations: [], +}); + +describe("Interface Builder analysis views", () => { + it("summarizes documents and graph counts without rows", () => { + const projected = projectAnalysisView(parent(), { kind: "summary" }); + if (!projected.ok) throw projected.error; + expect(projected.value).toMatchObject({ + kind: "summary", + parent_operation: "decode_interface_builder", + artifact: { sha256: target }, + summary: { + documents: report().documents, + available_counts: { nodes: 3, edges: 1, coverage: 1 }, + node_kinds: { control: 1, scene: 2 }, + edge_relations: { contains: 1 }, + coverage_statuses: { complete: 1 }, + truncated: false, + }, + coverage: { examined: 7, total: 7 }, + }); + expect(JSON.stringify(projected.value)).not.toContain("main:button"); + }); + + it("pages one document's nodes and keeps decoder truncation visible", () => { + const projected = projectAnalysisView(parent(), { + kind: "interface-builder", + collection: "nodes", + document: main, + offset: 1, + limit: 5, + }); + if (!projected.ok) throw projected.error; + expect(projected.value).toMatchObject({ + kind: "interface-builder", + items: [node("main:button", "control", main)], + source_truncated: false, + coverage: { status: "page", examined: 1, total: 2, exhausted: true }, + }); + const truncated = projectAnalysisView(parent(report(true)), { + kind: "interface-builder", + collection: "coverage", + offset: 0, + limit: 64, + }); + if (!truncated.ok) throw truncated.error; + expect(truncated.value).toMatchObject({ + items: report().graph.coverage, + source_truncated: true, + unknowns: expect.arrayContaining([ + expect.stringContaining("The decoded graph is incomplete"), + ]), + }); + }); + + it("refuses unknown documents, other views and mismatched targets", () => { + const unknown = projectAnalysisView(parent(), { + kind: "interface-builder", + collection: "edges", + document: "Contents/Resources/Missing.nib", + offset: 0, + limit: 64, + }); + if (unknown.ok) throw new Error("expected an unknown-document refusal"); + expect(unknown.error).toMatchObject({ + issues: [{ path: ["view", "document"] }], + }); + const wrongKind = projectAnalysisView(parent(), { + kind: "macho", + facet: "imports", + offset: 0, + limit: 64, + }); + if (wrongKind.ok) throw new Error("expected a view-kind refusal"); + expect(wrongKind.error).toMatchObject({ issues: [{ path: ["view"] }] }); + const mismatched = projectAnalysisView(parent(report(), "d".repeat(64)), { + kind: "summary", + }); + if (mismatched.ok) throw new Error("expected a subject refusal"); + expect(mismatched.error).toMatchObject({ + issues: [{ path: ["source", "subject"] }], + }); + }); +}); diff --git a/src/domain/analysisView/interfaceBuilderView.ts b/src/domain/analysisView/interfaceBuilderView.ts new file mode 100644 index 000000000..8710795a1 --- /dev/null +++ b/src/domain/analysisView/interfaceBuilderView.ts @@ -0,0 +1,248 @@ +import { z } from "zod"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { uniqueSorted } from "../canonicalOrdering.js"; +import { + interfaceBuilderAnalysisSchema, + interfaceBuilderDocumentSchema, +} from "../apple/interfaceBuilderGraph.js"; +import { + nativeInvestigationCoverageSchema, + nativeInvestigationEdgeSchema, + nativeInvestigationNodeSchema, +} from "../native/nativeInvestigationGraph.js"; +import { err, ok, type Result } from "../result.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; +import { + analysisViewInputError, + type AnalysisViewParent, + type AnalysisViewRequest, + type UnsignedAnalysisView, +} from "./analysisView.js"; +import { + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; + +const page = { + kind: z.literal("interface-builder"), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +} as const; + +/** Original decoded rows; graph rows can be narrowed to one decoded document. */ +export const interfaceBuilderViewRequestSchema = z.discriminatedUnion( + "collection", + [ + z.strictObject({ + ...page, + collection: z.enum(["nodes", "edges"]), + document: z + .string() + .min(1) + .optional() + .describe( + "Exact documents[].relative_path; selects rows whose evidence names that archive.", + ), + }), + z.strictObject({ + ...page, + collection: z.enum(["documents", "coverage"]), + }), + ], +); + +export const interfaceBuilderViewItemsSchema = z.union([ + z.array(interfaceBuilderDocumentSchema), + z.array(nativeInvestigationNodeSchema), + z.array(nativeInvestigationEdgeSchema), + z.array(nativeInvestigationCoverageSchema), +]); + +/** The decoder's own truncation, independent of page exhaustion. */ +export const interfaceBuilderViewContextSchema = z.strictObject({ + source_truncated: z.boolean(), +}); + +const countsSchema = z.record(z.string(), z.number().int().positive()); + +/** Every decoded document with graph counts; rows and evidence stay retained. */ +export const interfaceBuilderSummarySchema = z.strictObject({ + target_sha256: interfaceBuilderAnalysisSchema.shape.target_sha256, + provider: interfaceBuilderAnalysisSchema.shape.graph.shape.provider, + documents: interfaceBuilderAnalysisSchema.shape.documents, + available_counts: z.strictObject({ + nodes: z.number().int().nonnegative(), + edges: z.number().int().nonnegative(), + coverage: z.number().int().nonnegative(), + }), + node_kinds: countsSchema, + edge_relations: countsSchema, + coverage_statuses: countsSchema, + truncated: z.boolean(), + limitation_count: z.number().int().nonnegative(), +}); + +const tally = (values: readonly string[]): Record => { + const counts = new Map(); + for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); + return Object.fromEntries( + [...counts].sort(([left], [right]) => (left < right ? -1 : 1)), + ); +}; + +type InterfaceBuilderReport = z.output; + +/** Admit a well-formed report bound to its recorded target, if one exists. */ +const parseInterfaceBuilderReport = ( + parent: AnalysisViewParent, +): Result => { + const parsed = parseAnalysisViewResult( + interfaceBuilderAnalysisSchema, + parent, + ); + if (!parsed.ok) return parsed; + const report = parsed.value; + if ( + report.graph.target_sha256 !== report.target_sha256 || + (parent.subject != null && + parent.subject.digest.sha256 !== report.target_sha256) + ) + return err( + analysisViewInputError([ + { + path: ["source", "subject"], + reason: "invalid_value", + message: + "Interface Builder parent Evidence subject and graph must identify the report's target SHA-256.", + }, + ]), + ); + return ok(report); +}; + +/** Every decoded document with graph tallies; rows stay in the retained parent. */ +const summarizeInterfaceBuilder = ( + report: InterfaceBuilderReport, + limitationCount: number, +) => { + const { graph } = report; + const total = + report.documents.length + + graph.nodes.length + + graph.edges.length + + graph.coverage.length; + return { + summary: { + target_sha256: report.target_sha256, + provider: graph.provider, + documents: report.documents, + available_counts: { + nodes: graph.nodes.length, + edges: graph.edges.length, + coverage: graph.coverage.length, + }, + node_kinds: tally(graph.nodes.map((node) => node.kind)), + edge_relations: tally(graph.edges.map((edge) => edge.relation)), + coverage_statuses: tally(graph.coverage.map((row) => row.status)), + truncated: graph.truncated, + limitation_count: limitationCount, + }, + coverage: completeWithinViewCoverage(total, total), + }; +}; + +/** Project retained decode_interface_builder Evidence without reopening archives. */ +export const projectInterfaceBuilderView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if (view.kind !== "summary" && view.kind !== "interface-builder") + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Interface Builder decoding supports summary and interface-builder views.", + }, + ]), + ); + const parsed = parseInterfaceBuilderReport(parent); + if (!parsed.ok) return parsed; + const report = parsed.value; + const { graph } = report; + const limitations = uniqueSorted([ + ...parent.limitations, + ...report.limitations, + ]); + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: "decode_interface_builder" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: { + path: parent.subject?.local_path ?? "", + sha256: report.target_sha256, + }, + limitations, + unknowns: [ + "Rows describe recognized archive objects and connections only; unrecognized classes, fields and runtime-created UI remain unknown.", + ...(graph.truncated + ? [ + "The decoded graph is incomplete (see coverage and limitations for the cause); page exhaustion does not establish absence of undecoded objects or connections.", + ] + : []), + ], + }; + if (view.kind === "summary") + return ok({ + kind: "summary", + view, + ...shared, + ...summarizeInterfaceBuilder(report, limitations.length), + }); + const document = + view.collection === "nodes" || view.collection === "edges" + ? view.document + : undefined; + if ( + document !== undefined && + !report.documents.some((row) => row.relative_path === document) + ) + return err( + analysisViewInputError([ + { + path: ["view", "document"], + reason: "invalid_value", + message: + "Select an exact documents[].relative_path from this decoded report.", + }, + ]), + ); + const inDocument = < + Row extends { readonly evidence: readonly { artifact_path: unknown }[] }, + >( + candidates: readonly Row[], + ): readonly Row[] => + document === undefined + ? candidates + : candidates.filter((row) => + row.evidence.some((evidence) => evidence.artifact_path === document), + ); + const rows = + view.collection === "documents" + ? report.documents + : view.collection === "coverage" + ? graph.coverage + : view.collection === "nodes" + ? inDocument(graph.nodes) + : inDocument(graph.edges); + const items = rows.slice(view.offset, view.offset + view.limit); + return ok({ + kind: "interface-builder", + view, + items, + ...shared, + source_truncated: graph.truncated, + coverage: pageViewCoverage(view.offset, items.length, rows.length), + }); +}; diff --git a/src/domain/analysisView/machoView.ts b/src/domain/analysisView/machoView.ts index 1a64d23aa..fe39d6326 100644 --- a/src/domain/analysisView/machoView.ts +++ b/src/domain/analysisView/machoView.ts @@ -2,9 +2,9 @@ import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { uniqueSorted } from "../canonicalOrdering.js"; import { isMachOFormat } from "../binaryTargetTypes.js"; -import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { inspectMachoSchema } from "../native/nativeInspection.js"; import { err, ok, type Result } from "../result.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; import { analysisViewInputError, type AnalysisViewParent, @@ -93,19 +93,8 @@ type MachoReport = z.output; const parseMachoReport = ( parent: AnalysisViewParent, ): Result => { - const parsed = inspectMachoSchema.safeParse(parent.normalizedResult); - if (!parsed.success) - return err( - analysisInputErrorFromIssues( - "inspect_analysis_view", - parsed.error.issues.map((issue) => ({ - ...issue, - path: ["source", "normalized_result", ...issue.path], - })), - { source: { normalized_result: parent.normalizedResult } }, - { cause: parsed.error }, - ), - ); + const parsed = parseAnalysisViewResult(inspectMachoSchema, parent); + if (!parsed.ok) return parsed; if (parent.subject != null && !isMachOFormat(parent.subject.format)) return err( analysisViewInputError([ @@ -117,7 +106,7 @@ const parseMachoReport = ( }, ]), ); - return ok(parsed.data); + return ok(parsed.value); }; /** Counts replace rows; every collection caveat stays in the limitations. */ diff --git a/src/domain/analysisView/managedArtifactView.ts b/src/domain/analysisView/managedArtifactView.ts index 49d16fa80..3dd0033d7 100644 --- a/src/domain/analysisView/managedArtifactView.ts +++ b/src/domain/analysisView/managedArtifactView.ts @@ -1,12 +1,12 @@ import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { uniqueSorted } from "../canonicalOrdering.js"; -import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { managedArtifactInspectionSchema, managedMemberInspectionSchema, } from "../managed/managedArtifact.js"; import { err, ok, type Result } from "../result.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; import { analysisViewInputError, type AnalysisViewParent, @@ -67,22 +67,12 @@ export const projectManagedArtifactView = ( }, ]), ); - const parsed = managedArtifactInspectionSchema.safeParse( - parent.normalizedResult, + const parsed = parseAnalysisViewResult( + managedArtifactInspectionSchema, + parent, ); - if (!parsed.success) - return err( - analysisInputErrorFromIssues( - "inspect_analysis_view", - parsed.error.issues.map((issue) => ({ - ...issue, - path: ["source", "normalized_result", ...issue.path], - })), - { source: { normalized_result: parent.normalizedResult } }, - { cause: parsed.error }, - ), - ); - const report = parsed.data; + if (!parsed.ok) return parsed; + const report = parsed.value; const identity_scope = { token_identity: "build-local" as const, requires_artifact_sha256: report.artifact.sha256, diff --git a/src/domain/analysisView/wasmArtifactView.ts b/src/domain/analysisView/wasmArtifactView.ts index dc3036b26..52a97632a 100644 --- a/src/domain/analysisView/wasmArtifactView.ts +++ b/src/domain/analysisView/wasmArtifactView.ts @@ -2,11 +2,11 @@ import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { uniqueSorted } from "../canonicalOrdering.js"; import { err, ok, type Result } from "../result.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; import { wasmArtifactSchema, wasmArtifactMatchesSubject, } from "../wasm/wasmArtifact.js"; -import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { analysisViewInputError, type AnalysisViewParent, @@ -81,20 +81,9 @@ export const projectWasmArtifactView = ( }, ]), ); - const parsed = wasmArtifactSchema.safeParse(parent.normalizedResult); - if (!parsed.success) - return err( - analysisInputErrorFromIssues( - "inspect_analysis_view", - parsed.error.issues.map((issue) => ({ - ...issue, - path: ["source", "normalized_result", ...issue.path], - })), - { source: { normalized_result: parent.normalizedResult } }, - { cause: parsed.error }, - ), - ); - const report = parsed.data; + const parsed = parseAnalysisViewResult(wasmArtifactSchema, parent); + if (!parsed.ok) return parsed; + const report = parsed.value; if ( !wasmArtifactMatchesSubject( report.artifact, diff --git a/src/domain/callPath.test.ts b/src/domain/callPath.test.ts index 80a260ac3..048cb63b4 100644 --- a/src/domain/callPath.test.ts +++ b/src/domain/callPath.test.ts @@ -337,6 +337,42 @@ describe("call path topology and capacity", () => { }); }); +describe("directional call relationship coverage", () => { + it.each([ + { + directions: ["callers"] as ["callers"], + status: "not_found", + exhaustive: true, + }, + { + directions: ["callees"] as ["callees"], + status: "unknown", + exhaustive: false, + }, + { + directions: ["callers", "callees"] as ["callers", "callees"], + status: "unknown", + exhaustive: false, + }, + ])( + "uses $directions coverage for the forward search", + ({ directions, status, exhaustive }) => { + const limitation = "The inventory omitted a reference."; + const relationshipCoverage: RelationshipCoverage = { + status: "partial", + limitations: [limitation], + limited_directions: directions, + }; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { relationshipCoverage }), + observe("0x2000", [], "rea-workflow", { relationshipCoverage }), + ]); + expect(result).toMatchObject({ status, search_scope: { exhaustive } }); + expect(result.limitations).toContain(limitation); + }, + ); +}); + describe("explicit call relationship coverage", () => { it("accumulates 150,000 partial reasons without argument-stack exhaustion", () => { const result = run([ diff --git a/src/domain/callPath.ts b/src/domain/callPath.ts index 3ffbc2356..65524fbc2 100644 --- a/src/domain/callPath.ts +++ b/src/domain/callPath.ts @@ -10,7 +10,7 @@ import { parseFunctionEvidence, type FunctionSnapshot, } from "./functionDossierEvidence.js"; -import { isCallerOnlyUnknownCoverage } from "./functionCallCoverage.js"; +import { isCallerOnlyIncompleteCoverage } from "./functionCallCoverage.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; /** Minimal directed caller-to-callee adjacency for call-path search. */ @@ -268,7 +268,7 @@ const inspectSearch = ({ coverage === null ? snapshot.limitations.length > 0 : coverage.status !== "complete" && - !isCallerOnlyUnknownCoverage(coverage) + !isCallerOnlyIncompleteCoverage(coverage) ) incompleteExpandedCoverage = true; } else if (node === goal) { diff --git a/src/domain/changedBehavior.ts b/src/domain/changedBehavior.ts index 56e9b6ad9..3d4062e83 100644 --- a/src/domain/changedBehavior.ts +++ b/src/domain/changedBehavior.ts @@ -6,7 +6,10 @@ import { compositeKey, compareUnicodeCodePoints, } from "./unicodeCodePointOrder.js"; -import { comparisonSourceEvidenceIds } from "./comparisonSourceEvidence.js"; +import { + assertComparisonFindingLinks, + comparisonSourceEvidenceIds, +} from "./comparisonSourceEvidence.js"; import { evidenceSchema, parseEvidence, type Evidence } from "./evidence.js"; import { functionComparisonResultSchema } from "./functionComparisonSchemas.js"; import { @@ -260,7 +263,7 @@ const parseResult = ( const result = functionComparisonResultSchema.parse( evidence.normalized_result, ); - assertNestedLinks( + assertComparisonFindingLinks( evidence, result.dimensions.flatMap(({ evidence_links: links }) => links), ); @@ -269,24 +272,13 @@ const parseResult = ( const result = artifactComparisonResultSchema.parse( evidence.normalized_result, ); - assertNestedLinks( + assertComparisonFindingLinks( evidence, result.changes.flatMap(({ evidence_links: links }) => links), ); return { operation, evidence, result }; }; -const assertNestedLinks = ( - evidence: Evidence, - nestedLinks: readonly string[], -): void => { - const topLevel = new Set(evidence.evidence_links); - if (nestedLinks.some((evidenceId) => !topLevel.has(evidenceId))) - throw new TypeError( - "Comparison finding cites Evidence outside its top-level closure", - ); -}; - const findingsFor = (comparison: ParsedComparison): Finding[] => { if (comparison.operation === "compare_process_captures") return processFindings(comparison); diff --git a/src/domain/comparisonSourceEvidence.ts b/src/domain/comparisonSourceEvidence.ts index 420428b48..096e76548 100644 --- a/src/domain/comparisonSourceEvidence.ts +++ b/src/domain/comparisonSourceEvidence.ts @@ -55,3 +55,15 @@ export const comparisonSourceEvidenceIds = (evidence: Evidence): string[] => { const sides = comparisonSourceEvidenceSides(evidence); return uniqueSorted([...sides.left, ...sides.right]); }; + +/** Preserve the source closure when consuming nested comparison findings. */ +export const assertComparisonFindingLinks = ( + evidence: Pick, + nestedLinks: readonly string[], +): void => { + const closure = new Set(evidence.evidence_links); + if (nestedLinks.some((id) => !closure.has(id))) + throw new TypeError( + "Comparison finding cites Evidence outside its top-level closure", + ); +}; diff --git a/src/domain/functionCallCoverage.test.ts b/src/domain/functionCallCoverage.test.ts index 6fd3fb045..543f6ca92 100644 --- a/src/domain/functionCallCoverage.test.ts +++ b/src/domain/functionCallCoverage.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { combineFunctionRelationshipCoverage, - isCallerOnlyUnknownCoverage, + isCallerOnlyIncompleteCoverage, } from "./functionCallCoverage.js"; import type { RelationshipCoverage } from "./relationshipCoverage.js"; @@ -22,14 +22,14 @@ const cappedCallees: RelationshipCoverage = { }; const complete: RelationshipCoverage = { status: "complete" }; -describe("isCallerOnlyUnknownCoverage", () => { +describe("isCallerOnlyIncompleteCoverage", () => { it("detects unknown callers beside complete callees", () => { const combined = combineFunctionRelationshipCoverage( unknownCallers, complete, ); expect(combined.status).toBe("unknown"); - expect(isCallerOnlyUnknownCoverage(combined)).toBe(true); + expect(isCallerOnlyIncompleteCoverage(combined)).toBe(true); }); it("rejects unknown callees beside complete callers", () => { @@ -37,7 +37,7 @@ describe("isCallerOnlyUnknownCoverage", () => { complete, unknownCallees, ); - expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + expect(isCallerOnlyIncompleteCoverage(combined)).toBe(false); }); it("rejects unknown coverage on both sides", () => { @@ -45,7 +45,7 @@ describe("isCallerOnlyUnknownCoverage", () => { unknownCallers, unknownCallees, ); - expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + expect(isCallerOnlyIncompleteCoverage(combined)).toBe(false); }); it("rejects unknown callers beside IDA-capped callees", () => { @@ -54,11 +54,37 @@ describe("isCallerOnlyUnknownCoverage", () => { cappedCallees, ); expect(combined.status).toBe("unknown"); - expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + expect(isCallerOnlyIncompleteCoverage(combined)).toBe(false); }); it("rejects complete coverage", () => { const combined = combineFunctionRelationshipCoverage(complete, complete); - expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + expect(isCallerOnlyIncompleteCoverage(combined)).toBe(false); + }); + + it("retains caller-only partial coverage without relying on limitation wording", () => { + const combined = combineFunctionRelationshipCoverage( + { status: "partial", limitations: ["Missing incoming references."] }, + complete, + ); + expect(combined).toMatchObject({ limited_directions: ["callers"] }); + expect(isCallerOnlyIncompleteCoverage(combined)).toBe(true); + }); + + it("uses explicit directions even when explanation text contains direction labels", () => { + expect( + isCallerOnlyIncompleteCoverage({ + status: "unknown", + reason: "Callers: unavailable. Callees: diagnostic text.", + limited_directions: ["callers"], + }), + ).toBe(true); + expect( + isCallerOnlyIncompleteCoverage({ + status: "unknown", + reason: "Callers: unavailable.", + limited_directions: ["callees"], + }), + ).toBe(false); }); }); diff --git a/src/domain/functionCallCoverage.ts b/src/domain/functionCallCoverage.ts index b64e78cd6..ce3798c4f 100644 --- a/src/domain/functionCallCoverage.ts +++ b/src/domain/functionCallCoverage.ts @@ -22,6 +22,16 @@ export const combineFunctionRelationshipCoverage = ( ), ); const details = diagnostics.length > 0 ? { diagnostics } : {}; + const callersLimited = callers?.status !== "complete"; + const calleesLimited = callees?.status !== "complete"; + const limited_directions: Extract< + RelationshipCoverage, + { status: "partial" } + >["limited_directions"] = callersLimited + ? calleesLimited + ? ["callers", "callees"] + : ["callers"] + : ["callees"]; const reasons = reads.flatMap(({ direction, coverage }) => { const label = direction === "callers" ? callerCoveragePrefix : calleeCoveragePrefix; @@ -43,30 +53,37 @@ export const combineFunctionRelationshipCoverage = ( return { status: "unknown", reason: uniqueSorted(reasons).join(" "), + limited_directions, ...details, }; if (reasons.length > 0) return { status: "partial", limitations: uniqueSorted(reasons), + limited_directions, ...details, }; return { status: "complete", ...details }; }; /** - * Whether combined relationship coverage is unknown only on the caller side. - * The combine helper above prefixes each direction's reasons, so a reason - * that carries the caller prefix and no callee prefix records observed - * callees with unobservable callers. Forward (callee-edge) searches stay - * exhaustive under that caveat; the caller reason remains a limitation. + * Forward searches depend on callee coverage. Preserve the historical + * unknown-reason encoding when reading observations without explicit directions. */ -export const isCallerOnlyUnknownCoverage = ( +export const isCallerOnlyIncompleteCoverage = ( coverage: RelationshipCoverage, -): boolean => - coverage.status === "unknown" && - coverage.reason.startsWith(callerCoveragePrefix) && - !coverage.reason.includes(calleeCoveragePrefix); +): boolean => { + if (coverage.status === "complete") return false; + if (coverage.limited_directions !== undefined) + return !coverage.limited_directions.some( + (direction) => direction === "callees", + ); + return ( + coverage.status === "unknown" && + coverage.reason.startsWith(callerCoveragePrefix) && + !coverage.reason.includes(calleeCoveragePrefix) + ); +}; /** Explain why two resolved call sets cannot establish call-set equality. */ export const functionCallCoverageLimitations = ( diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index 16f8282c9..e49b0a892 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -625,6 +625,40 @@ describe("function comparison CFG address normalization", () => { }); describe("function comparison unresolved text", () => { + it("qualifies exact text changes caused by relocated provider labels", () => { + const left = observe( + "2", + functionDossierSchema.parse({ + ...dossier("return FUN_00001010(DAT_00001020) % 3;", "0x1000"), + assembly: ["0x1000 call FUN_00001010", "0x1004 ret"], + }), + ); + const right = observe( + "3", + functionDossierSchema.parse({ + ...dossier("return FUN_00002010(DAT_00002020) % 3;", "0x2000"), + assembly: ["0x2000 call FUN_00002010", "0x2004 ret"], + }), + ); + const comparison = compareFunctions(left, right); + for (const name of ["pseudocode", "assembly"]) { + const dimension = comparison.dimensions.find( + ({ dimension }) => dimension === name, + ); + expect(dimension).toMatchObject({ + status: "changed", + limitations: expect.arrayContaining([ + expect.stringContaining( + "layout-only differences do not establish a behavior change", + ), + ]), + }); + } + expect( + comparison.dimensions.find(({ dimension }) => dimension === "cfg"), + ).toMatchObject({ status: "unchanged" }); + }); + it("counts Unicode text consistently for unresolved provider comparisons", () => { const left = observe("b", dossier("return '😀';", "0x1000"), "provider-a"); const right = observe("c", dossier("return '😀';", "0x2000"), "provider-b"); @@ -886,6 +920,31 @@ describe("function comparison Swift string literals", () => { }); describe("function comparison partial Swift literal coverage", () => { + it("keeps literal comparisons unknown at unresolved jump candidates", () => { + const unresolved = functionDossierSchema.parse({ + ...withLiterals("0x2000", []), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "unresolved_jump", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension( + "8", + withLiterals("0x1000", ["SleepDisabled"]), + "9", + unresolved, + ), + ).toMatchObject({ status: "unknown" }); + }); + it("does not equate a partial literal record with a complete one", () => { const partial = functionDossierSchema.parse({ ...withLiterals("0x2000", ["SleepDisabled"]), @@ -930,7 +989,9 @@ describe("function comparison partial Swift literal coverage", () => { stringsDimension("d", withLiterals("0x1000", []), "e", partial), ).toMatchObject({ status: "changed" }); }); +}); +describe("function comparison partial Swift literal omission interpretation", () => { it("does not interpret an unexamined path as a missing literal at a retained site", () => { const partial = functionDossierSchema.parse({ ...withLiterals("0x2000", ["A"]), diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index f91057a6b..4a36e1c27 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -121,12 +121,14 @@ const compareText = ( right: string, context: ComparisonContext, ): FunctionDimension => { - const limitations = - dimension === "assembly" + const limitations = [ + "Exact provider text is compared without address or generated-name normalization; layout-only differences do not establish a behavior change.", + ...(dimension === "assembly" ? [ "Assembly is opaque provider text; relocation normalization is unavailable.", ] - : []; + : []), + ]; if (context.providersDiffer) return unresolvedDimension({ dimension, @@ -149,6 +151,7 @@ const compareText = ( leftCount: [...left].length, rightCount: [...right].length, limitations: [ + ...limitations, "Bounded line diff exceeded its time or edit-distance limit.", ], }); diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 258b1a82f..6596d0d7b 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -319,15 +319,15 @@ const swiftStringLiteralOmissionFields = { source_address: swiftStringLiteralFields.source_address, byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), reason: z - .literal("capacity") + .enum(["capacity", "unresolved_jump"]) .describe( - "The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.", + "Why this candidate is omitted: capacity stopped decoding, or an unresolved jump left its use unknown.", ), registers: swiftStringLiteralFields.registers, words: swiftStringLiteralFields.words, }; -/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */ +/** A Swift literal candidate omitted because decoding capacity or control flow left it unknown. */ export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( "representation", [ @@ -410,7 +410,7 @@ export const functionDossierSchema = z .array(swiftStringLiteralOmissionSchema) .exactOptional() .describe( - "The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.", + "Literal candidates omitted for capacity or unresolved control flow. Present exactly when swift_string_literals is present; empty when all examined use sites were classified.", ), basic_blocks: z.array( z diff --git a/src/domain/propertyListKeys.test.ts b/src/domain/propertyListKeys.test.ts index ded36daff..d95f64857 100644 --- a/src/domain/propertyListKeys.test.ts +++ b/src/domain/propertyListKeys.test.ts @@ -1,4 +1,5 @@ -import { expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; +import { parse } from "plist"; import { omitPrototypeKeys, parseXmlPropertyList } from "./propertyListKeys.js"; @@ -234,3 +235,55 @@ it("locates sparse omitted keys across newline-dense source", () => { omittedPrototypeKeys: 2, }); }); + +describe("OpenStep plist nesting preflight", () => { + const arrays = (depth: number, leaf = "x") => + "(".repeat(depth) + leaf + ")".repeat(depth); + + it.each([129, 3000])( + "rejects %s nested OpenStep values with the shared depth limit", + (depth) => { + expect(() => parseXmlPropertyList(arrays(depth))).toThrow( + "Plist exceeds 128 nesting levels", + ); + }, + ); + + it.each(["{}", "()"])( + "preserves the first %s root despite deeply nested trailing text", + (root) => { + expect(parseXmlPropertyList(root + arrays(3000))).toEqual({ + value: root === "{}" ? {} : [], + omittedPrototypeKeys: 0, + }); + }, + ); + + it("accepts the deepest supported OpenStep value and empty container", () => { + expect(() => parseXmlPropertyList(arrays(128))).not.toThrow(); + expect(() => parseXmlPropertyList(arrays(129, ""))).not.toThrow(); + }); + + it.each([ + "(x, y,)", + "{ key = x next = (y,); }", + "{ (compound, key) = x; { nested = key; } = y; }", + "{ key = (); next = {}; }", + ])("preserves pinned separator and compound-key semantics for %s", (text) => { + expect(parseXmlPropertyList(text)).toEqual({ + value: parse(text), + omittedPrototypeKeys: 0, + }); + }); + + it("ignores brackets in quoted strings, escaped quotes, comments, and data", () => { + expect( + parseXmlPropertyList( + '{ quote = "\\\"( )"; /* ((( */ data = <2829>; // (((\n values = (x); }', + ), + ).toEqual({ + value: { quote: '"( )', data: new Uint8Array([40, 41]), values: ["x"] }, + omittedPrototypeKeys: 0, + }); + }); +}); diff --git a/src/domain/propertyListKeys.ts b/src/domain/propertyListKeys.ts index 35aa0889d..2dcab952c 100644 --- a/src/domain/propertyListKeys.ts +++ b/src/domain/propertyListKeys.ts @@ -94,7 +94,10 @@ export const parseXmlPropertyList = ( // Preserve the pinned decoder's existing OpenStep autodetection, also used // by app Info.plist resolution. Only XML needs the key-element substitution. const first = text.trimStart()[0]; - if (first === "{" || first === "(") return omitPrototypeKeys(parse(text)); + if (first === "{" || first === "(") { + assertOpenStepDepthWithinLimit(text); + return omitPrototypeKeys(parse(text)); + } // Normalize only the literal line endings the pinned decoder normalizes. // Preserve character references in the source: serializing a DOM would turn // e.g. an encoded carriage return into a literal newline on the second parse. @@ -111,6 +114,133 @@ export const parseXmlPropertyList = ( return withoutKey(parse(substituted), placeholder); }; +interface OpenStepContainer { + end: string; + next: "key" | "equals" | "value" | "separator" | "after-value"; +} + +/** Consume only the punctuation expected between the pinned decoder's values. */ +const consumeOpenStepSeparator = ( + parent: OpenStepContainer | undefined, + character: string, +): "consumed" | "invalid" | "none" => { + if (parent?.next === "after-value") { + parent.next = "key"; + return character === ";" ? "consumed" : "none"; + } + if (parent?.next !== "equals" && parent?.next !== "separator") return "none"; + if (character !== (parent.next === "equals" ? "=" : ",")) return "invalid"; + parent.next = "value"; + return "consumed"; +}; + +/** An array may close after a comma; a dictionary still needs its assigned value. */ +const admitsOpenStepCloser = ( + parent: OpenStepContainer | undefined, + character: string, +): boolean => + parent?.end === character && + parent.next !== "equals" && + !(parent.end === "}" && parent.next === "value"); + +/** Bound OpenStep value nesting before the pinned decoder recursively reads it. */ +const assertOpenStepDepthWithinLimit = (text: string): void => { + const containers: OpenStepContainer[] = []; + const completeValue = (): void => { + const parent = containers.at(-1); + if (parent !== undefined) + parent.next = + parent.next === "key" + ? "equals" + : parent.end === "}" + ? "after-value" + : "separator"; + }; + const assertDepth = (): void => { + if (containers.length > MAX_PLIST_DEPTH) + throw new RangeError( + `Plist exceeds ${String(MAX_PLIST_DEPTH)} nesting levels`, + ); + }; + let cursor = 0; + while (cursor < text.length) { + const afterTrivia = skipOpenStepTrivia(text, cursor); + if (afterTrivia !== cursor) { + cursor = afterTrivia; + continue; + } + const character = text[cursor] ?? ""; + const parent = containers.at(-1); + // Stop at the first impossible token: only the pinned decoder owns syntax + // diagnostics. A later deep suffix cannot change that failure into a limit. + if (character === "}" || character === ")") { + if (!admitsOpenStepCloser(parent, character)) return; + containers.pop(); + // parseOpenStep returns the first root value without reading trailing text. + if (containers.length === 0) return; + completeValue(); + cursor += 1; + continue; + } + const separator = consumeOpenStepSeparator(parent, character); + if (separator === "invalid") return; + if (separator === "consumed") { + cursor += 1; + continue; + } + if (character === "{" || character === "(") { + assertDepth(); + containers.push({ + end: character === "{" ? "}" : ")", + next: character === "{" ? "key" : "value", + }); + cursor += 1; + continue; + } + const end = skipOpenStepScalar(text, cursor); + if (end === cursor) return; + if (parent?.next !== "key") assertDepth(); + cursor = end; + completeValue(); + } +}; + +/** Skip grammar trivia without interpreting delimiter examples inside comments. */ +const skipOpenStepTrivia = (text: string, cursor: number): number => { + if (/\s/u.test(text[cursor] ?? "")) return cursor + 1; + if (!text.startsWith("//", cursor) && !text.startsWith("/*", cursor)) + return cursor; + const line = text[cursor + 1] === "/"; + const end = text.indexOf(line ? "\n" : "*/", cursor + 2); + return end === -1 ? text.length : end + (line ? 1 : 2); +}; + +/** Read the extent of one opaque scalar; the pinned decoder still owns its value. */ +const skipOpenStepScalar = (text: string, start: number): number => { + let cursor = start; + const character = text[cursor]; + if (character === '"') { + cursor += 1; + while (cursor < text.length) { + const next = text[cursor]; + cursor += next === "\\" ? 2 : 1; + if (next === '"') break; + } + } else if (character === "<") { + const end = text.indexOf(">", cursor + 1); + cursor = end === -1 ? text.length : end + 1; + } else { + // Match the pinned OpenStep decoder's unquoted-string alphabet. Other + // syntax remains for the decoder to reject; preflight only owns depth. + while ( + cursor < text.length && + /[a-zA-Z0-9._/$:-]/u.test(text[cursor] ?? "") + ) + cursor += 1; + } + return cursor; +}; + /** Elements whose children the pinned plist value decoder visits recursively. */ const PLIST_CONTAINER_ELEMENTS = new Set(["plist", "dict", "array"]); diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.ts index 8f25ccb36..7ca6961be 100644 --- a/src/domain/reconstructionVerification.functionSources.test.ts +++ b/src/domain/reconstructionVerification.functionSources.test.ts @@ -662,7 +662,7 @@ describe("function reconstruction admission controls", () => { ]); }); - it("rejects selected-dimension citations outside comparison closure", () => { + it("rejects any comparison finding link outside the top-level closure", () => { const evidence = comparison(singular, { result: jsonValueSchema.parse({ ...baselineResult, @@ -677,7 +677,7 @@ describe("function reconstruction admission controls", () => { }), }); expect(admittedVerification(evidence, [left, right, extra])).toThrow( - /dimension cites Evidence outside comparison closure/u, + /finding cites Evidence outside its top-level closure/u, ); }); }); diff --git a/src/domain/reconstructionVerification.test.ts b/src/domain/reconstructionVerification.test.ts index 664438d85..159a81729 100644 --- a/src/domain/reconstructionVerification.test.ts +++ b/src/domain/reconstructionVerification.test.ts @@ -21,7 +21,13 @@ const source = ( digit: string, authority: "controlled-replay" | "shipped-artifact", confidence: "observed" | "derived" = "observed", - captureResult: JsonValue = EMPTY_PROCESS_CAPTURE_EXAMPLE, + { + captureResult = EMPTY_PROCESS_CAPTURE_EXAMPLE, + replayEnvironment = environment, + }: { + readonly captureResult?: JsonValue; + readonly replayEnvironment?: typeof environment | null; + } = {}, ): Evidence => createEvidence( { @@ -43,7 +49,7 @@ const source = ( result: authority === "controlled-replay" ? captureResult : { digit }, confidence, authority, - environment: authority === "controlled-replay" ? environment : null, + environment: authority === "controlled-replay" ? replayEnvironment : null, }, ); @@ -113,6 +119,31 @@ const processComparison = ( }, ); +const multiSourceProcessComparison = ( + left: readonly Evidence[], + right: readonly Evidence[], +): Evidence => + createEvidence( + undefined, + { + id: "rea-process", + name: "REA deterministic process harness", + version: "3", + }, + { + predicateType: "rea.process-comparison", + operation: "compare_process_captures", + parameters: { + left_evidence_ids: left.map(({ evidence_id: id }) => id), + right_evidence_ids: right.map(({ evidence_id: id }) => id), + }, + result: processResult(), + confidence: "derived", + authority: "analyst-inference", + evidenceLinks: [...left, ...right].map(({ evidence_id: id }) => id), + }, + ); + const behavioralSpec = (comparison: Evidence) => ({ name: "CLI compatibility", claims: [ @@ -288,12 +319,9 @@ describe("reconstruction verification", () => { describe("reconstruction process capture input", () => { it("rejects saved captures without explicit executable identity", () => { - const left = source( - "1", - "controlled-replay", - "observed", - legacyV3Capture(), - ); + const left = source("1", "controlled-replay", "observed", { + captureResult: legacyV3Capture(), + }); const right = source("2", "controlled-replay"); const comparison = processComparison(left, right, processResult()); expect(() => @@ -306,6 +334,49 @@ describe("reconstruction process capture input", () => { }); describe("reconstruction verification outcomes", () => { + it("requires every selected behavioral source to share a replay environment", () => { + const replay = (digit: string, id: string | null) => + source(digit, "controlled-replay", "observed", { + replayEnvironment: id === null ? null : { ...environment, id }, + }); + const verify = (left: readonly Evidence[], right: readonly Evidence[]) => { + const comparison = multiSourceProcessComparison(left, right); + return verifyReconstruction( + behavioralSpec(comparison), + createEvidenceBundle([...left, ...right, comparison]), + ); + }; + const limitation = + "Controlled replay environments are unavailable or incompatible."; + const differentOnLeft = verify( + [replay("1", "env-a"), replay("2", "env-b")], + [replay("3", "env-a"), replay("4", "env-a")], + ); + expect(differentOnLeft.status).toBe("unknown"); + expect(differentOnLeft.claims.items[0]?.limitations).toContain(limitation); + + const differentOnRight = verify( + [replay("5", "env-a"), replay("6", "env-a")], + [replay("7", "env-a"), replay("8", "env-b")], + ); + expect(differentOnRight.status).toBe("unknown"); + expect(differentOnRight.claims.items[0]?.limitations).toContain(limitation); + + const unavailable = verify( + [replay("9", "env-a"), replay("a", "env-a")], + [replay("b", "env-a"), replay("c", null)], + ); + expect(unavailable.status).toBe("unknown"); + expect(unavailable.claims.items[0]?.limitations).toContain(limitation); + + expect( + verify( + [replay("d", "env-a"), replay("e", "env-a")], + [replay("f", "env-a"), replay("0", "env-a")], + ).status, + ).toBe("pass"); + }); + it("fails an authoritative observed disagreement", () => { const left = source("1", "controlled-replay"); const right = source("2", "controlled-replay"); diff --git a/src/domain/reconstructionVerification.ts b/src/domain/reconstructionVerification.ts index 6f693a158..ee7b73b9b 100644 --- a/src/domain/reconstructionVerification.ts +++ b/src/domain/reconstructionVerification.ts @@ -7,7 +7,10 @@ import { artifactComparisonResultSchema } from "./artifactComparison.js"; import { artifactInventoryResultSchema } from "./artifactGraph.js"; import { uniqueSorted } from "./canonicalOrdering.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; -import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js"; +import { + comparisonSourceEvidenceSides, + assertComparisonFindingLinks, +} from "./comparisonSourceEvidence.js"; import { parseEvidence, type Evidence } from "./evidence.js"; import { parseEvidenceBundle } from "./evidenceBundle.js"; import { functionComparisonResultSchema } from "./functionComparisonSchemas.js"; @@ -176,6 +179,7 @@ const evaluateClaim = ( unknown_ids: displayedUnknowns.map(({ unknown_id: id }) => id), limitations: uniqueSorted([ ...comparison.limitations, + ...observation.limitations, ...limitations, ...displayedUnknowns.map( ({ question }) => `Residual unknown: ${question}`, @@ -245,6 +249,7 @@ const comparisonObservation = ( evidence: Evidence, ): { readonly status: ObservedStatus; + readonly limitations: readonly string[]; readonly hasIndependentFunctionChange?: boolean; } => { if (claim.kind === "behavioral") { @@ -255,16 +260,26 @@ const comparisonObservation = ( return { status: claim.dimension === "overall" ? result.status : result[claim.dimension], + limitations: [], }; } if (claim.kind === "structural-function") { const result = functionComparisonResultSchema.parse( evidence.normalized_result, ); + assertComparisonFindingLinks( + evidence, + result.dimensions.flatMap(({ evidence_links }) => evidence_links), + ); if (claim.dimension === "overall") return { status: result.status, - // Historical overall changes may come solely from incomplete call sets. + limitations: result.dimensions.flatMap( + ({ limitations }) => limitations, + ), + // Historical overall changes can be caused only by incomplete call + // sets. Ignore that aggregate change when independently checking the + // source call-coverage limitations. hasIndependentFunctionChange: result.function_match.status === "mismatched" || (result.function_match.status === "matched" && @@ -279,28 +294,16 @@ const comparisonObservation = ( ); if (dimension === undefined) throw new TypeError("Function comparison dimension missing"); - if ( - dimension.evidence_links.some( - (id) => !evidence.evidence_links.includes(id), - ) - ) - throw new TypeError( - "Function dimension cites Evidence outside comparison closure", - ); - return { status: dimension.status }; + return { status: dimension.status, limitations: dimension.limitations }; } const result = artifactComparisonResultSchema.parse( evidence.normalized_result, ); - if ( - result.changes.some((item) => - item.evidence_links.some((id) => !evidence.evidence_links.includes(id)), - ) - ) - throw new TypeError( - "Artifact change cites Evidence outside comparison closure", - ); - return { status: result.status }; + assertComparisonFindingLinks( + evidence, + result.changes.flatMap(({ evidence_links }) => evidence_links), + ); + return { status: result.status, limitations: [] }; }; const validateProcessStatus = ( @@ -346,13 +349,15 @@ const authorityLimitations = ( ) limitations.push("Structural source artifact identity is unavailable."); if (claim.kind === "behavioral") { - const byId = new Map(sources.map((source) => [source.evidence_id, source])); - const left = byId.get(sides.left[0] ?? ""); - const right = byId.get(sides.right[0] ?? ""); + const referenceEnvironment = sources[0]?.environment; if ( - left?.environment === null || - right?.environment === null || - JSON.stringify(left?.environment) !== JSON.stringify(right?.environment) + referenceEnvironment === null || + referenceEnvironment === undefined || + sources.some( + ({ environment }) => + environment === null || + JSON.stringify(environment) !== JSON.stringify(referenceEnvironment), + ) ) limitations.push( "Controlled replay environments are unavailable or incompatible.", diff --git a/src/domain/staticRuntimeCorrelation.test.ts b/src/domain/staticRuntimeCorrelation.test.ts index e2d4fff14..83051af7e 100644 --- a/src/domain/staticRuntimeCorrelation.test.ts +++ b/src/domain/staticRuntimeCorrelation.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from "vitest"; import { INVESTIGATION_EXAMPLES } from "../contracts/investigationExamples.js"; import { createEvidence, type Evidence } from "./evidence.js"; -import type { JsonValue } from "./jsonValue.js"; +import { jsonValueSchema, type JsonValue } from "./jsonValue.js"; +import { FUNCTION_COMPARISON_EXAMPLE } from "../contracts/functionComparisonExample.js"; +import { functionDossierSchema } from "./hopperValues.js"; +import { compareFunctions } from "./functionComparison.js"; import { correlateStaticAndRuntime, staticRuntimeCorrelationInputSchema, @@ -113,6 +116,70 @@ const mapping = ( ], }); +it("keeps text, assembly and unavailable CFG caveats on their selected correlations", () => { + const source = FUNCTION_COMPARISON_EXAMPLE.left; + const left = createEvidence( + { path: "/tmp/unavailable-cfg", sha256: "a".repeat(64), format: "mach-o" }, + source.provider, + { + operation: source.operation, + parameters: source.parameters, + result: jsonValueSchema.parse({ + ...functionDossierSchema.parse(source.normalized_result), + basic_blocks: [{ start: "0x1000", end: "0x1000", successors: [] }], + }), + confidence: source.confidence, + authority: source.authority, + }, + ); + const right = FUNCTION_COMPARISON_EXAMPLE.right; + const computed = compareFunctions(left, right); + const comparison = createEvidence(undefined, staticComparison.provider, { + predicateType: "rea.function-comparison", + operation: "compare_functions", + parameters: { + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + }, + result: jsonValueSchema.parse(computed), + confidence: "derived", + authority: "analyst-inference", + limitations: computed.limitations, + evidenceLinks: [left.evidence_id, right.evidence_id], + }); + const input = mapping(processComparison("unchanged"), "static_only"); + const result = correlateStaticAndRuntime({ + ...input, + static_comparisons: [comparison], + mappings: ["pseudocode", "assembly", "cfg"].map((dimension) => ({ + ...input.mappings[0], + static: { + comparison_evidence_id: comparison.evidence_id, + selector: { kind: "function", dimension }, + }, + })), + }); + expect(result.status).toBe("unknown"); + const caveats = new Map( + result.correlations.items.map((item) => [ + item.static.selector.kind === "function" + ? item.static.selector.dimension + : "artifact", + item.limitations.join(" "), + ]), + ); + expect(caveats.get("pseudocode")).toMatch(/generated-name normalization/u); + expect(caveats.get("pseudocode")).not.toMatch( + /Assembly is opaque|CFG addresses/u, + ); + expect(caveats.get("assembly")).toMatch(/Assembly is opaque/u); + expect(caveats.get("assembly")).not.toMatch(/CFG addresses/u); + expect(caveats.get("cfg")).toMatch(/CFG addresses/u); + expect(caveats.get("cfg")).not.toMatch( + /generated-name normalization|Assembly is opaque/u, + ); +}); + describe("static/runtime correlation", () => { it("accepts real runtime dimensions and rejects removed protocol dimensions", () => { const runtime = processComparison("unchanged"); @@ -154,6 +221,9 @@ describe("static/runtime correlation", () => { expect(result.limitations.join(" ")).toMatch( /does not establish causality/u, ); + expect(result.correlations.items[0]?.limitations.join(" ")).toMatch( + /generated-name normalization/u, + ); }); it("classifies an explicit expectation mismatch as contradiction", () => { diff --git a/src/domain/staticRuntimeCorrelation.ts b/src/domain/staticRuntimeCorrelation.ts index b00f5a3d5..0362569a7 100644 --- a/src/domain/staticRuntimeCorrelation.ts +++ b/src/domain/staticRuntimeCorrelation.ts @@ -6,7 +6,10 @@ import { z } from "zod"; import { artifactComparisonResultSchema } from "./artifactComparison.js"; import { uniqueSorted } from "./canonicalOrdering.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; -import { comparisonSourceEvidenceIds } from "./comparisonSourceEvidence.js"; +import { + comparisonSourceEvidenceIds, + assertComparisonFindingLinks, +} from "./comparisonSourceEvidence.js"; import { evidenceSchema, parseEvidence, type Evidence } from "./evidence.js"; import { functionComparisonResultSchema } from "./functionComparisonSchemas.js"; import { @@ -236,7 +239,10 @@ const correlate = ( mapping.runtime.comparison_evidence_id, "runtime comparison", ); - const selectedStatic = selectStatic(staticEvidence, mapping.static.selector); + const { limitations: selectedLimitations, ...selectedStatic } = selectStatic( + staticEvidence, + mapping.static.selector, + ); const process = processCaptureComparisonSchema.parse( runtimeEvidence.normalized_result, ); @@ -270,6 +276,7 @@ const correlate = ( ]), limitations: uniqueSorted([ ...staticEvidence.limitations, + ...selectedLimitations, ...runtimeEvidence.limitations, ...process.limitations, "This correlation tests an explicit hypothesis and does not establish causality.", @@ -281,7 +288,9 @@ const correlate = ( const selectStatic = ( evidence: Evidence, selector: z.infer, -): Pick => { +): Pick & { + readonly limitations: readonly string[]; +} => { if (selector.kind === "function") { if (evidence.operation !== "compare_functions") throw new TypeError( @@ -299,6 +308,7 @@ const selectStatic = ( return { status: match.status, conclusion_kind: match.conclusion_kind, + limitations: match.limitations, }; } if (evidence.operation !== "compare_artifacts") @@ -319,6 +329,7 @@ const selectStatic = ( const status = match.classification; return { status, + limitations: [], conclusion_kind: status === "contradiction" ? "contradiction" @@ -342,7 +353,7 @@ const parseStatic = (input: unknown): Evidence => { const result = artifactComparisonResultSchema.parse( evidence.normalized_result, ); - assertNestedLinks( + assertComparisonFindingLinks( evidence, result.changes.flatMap(({ evidence_links: links }) => links), ); @@ -350,7 +361,7 @@ const parseStatic = (input: unknown): Evidence => { const result = functionComparisonResultSchema.parse( evidence.normalized_result, ); - assertNestedLinks( + assertComparisonFindingLinks( evidence, result.dimensions.flatMap(({ evidence_links: links }) => links), ); @@ -358,17 +369,6 @@ const parseStatic = (input: unknown): Evidence => { return evidence; }; -const assertNestedLinks = ( - evidence: Evidence, - nestedLinks: readonly string[], -): void => { - const closure = new Set(evidence.evidence_links); - if (nestedLinks.some((evidenceId) => !closure.has(evidenceId))) - throw new TypeError( - "Comparison finding cites Evidence outside its top-level closure", - ); -}; - const parseRuntime = (input: unknown): Evidence => { const evidence = parseEvidence(input); if (evidence.operation !== "compare_process_captures") diff --git a/src/go/GoBinaryContainer.ts b/src/go/GoBinaryContainer.ts index 96b4cb146..11a3c3ada 100644 --- a/src/go/GoBinaryContainer.ts +++ b/src/go/GoBinaryContainer.ts @@ -12,7 +12,7 @@ export class GoBinaryFormatFailure extends Error { /** Resource boundary exceeded while decoding metadata or native container structure. */ export class GoBinaryResourceFailure extends Error { constructor( - readonly boundary: "build-info" | "structure", + readonly boundary: "build-info" | "structure" | "search", readonly maximum_bytes: number, message: string, ) { @@ -44,7 +44,7 @@ export interface GoImageSection { readonly compressed: boolean; } -/** Native container facts and its producer-defined build-info search area. */ +/** Native container facts and its ordered producer-defined build-info search ranges. */ export interface GoBinaryContainer { readonly format: "elf" | "pe" | "macho"; readonly architecture: string; @@ -53,7 +53,7 @@ export interface GoBinaryContainer { readonly mappings: readonly GoFileMapping[]; readonly zero_fills: readonly GoZeroFillMapping[]; readonly sections: readonly GoImageSection[]; - readonly search: GoFileMapping | null; + readonly search: readonly GoFileMapping[]; } /** Reader that checks every structural range before decoding numeric fields. */ diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index e94a80ab6..c9e2366c8 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -25,6 +25,7 @@ const MAGIC = Buffer.from("ff20476f206275696c64696e663a", "hex"); const MODULE_START = Buffer.from("3077af0c9274080241e1c107e6d618e6", "hex"); const MODULE_END = Buffer.from("f932433186182072008242104116d8f2", "hex"); const MAX_BUILD_INFO_BYTES = 1024 * 1024; +const MAX_HEADER_SEARCH_BYTES = 256 * 1024 * 1024; /** Exact file span of one embedded string's contents, excluding its length/header. */ export interface GoStringLocation { @@ -57,10 +58,31 @@ export interface GoBinaryImage { export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { const container = readGoBinaryContainer(bytes); const { format, architecture, bits, byte_order } = container; - const search = container.search; - const header = findHeader(bytes, search); - if (header === null || search === null) + let selected: { search: GoFileMapping; header: number } | null = null; + const searched = new Set(); + let remainingSearchBytes = MAX_HEADER_SEARCH_BYTES; + for (const search of container.search) { + // Identical file spans and alignment inspect the same candidate bytes. + // Partially overlapping spans still consume a cumulative work budget. + const identity = `${search.offset}:${search.size}:${search.address % 16n}`; + if (searched.has(identity)) continue; + searched.add(identity); + if (search.size > remainingSearchBytes) + throw new GoBinaryResourceFailure( + "search", + MAX_HEADER_SEARCH_BYTES, + "Go build-info discovery exceeds the cumulative 256 MiB header-search budget", + ); + remainingSearchBytes -= search.size; + const header = findHeader(bytes, search); + if (header !== null) { + selected = { search, header }; + break; + } + } + if (selected === null) return { format, architecture, bits, byte_order, build_info: null }; + const { search, header } = selected; const verifyMapping = (size: number): void => { const address = search.address + BigInt(header - search.offset); if ( @@ -222,11 +244,8 @@ export const readGoBinaryContainer = (bytes: Buffer): GoBinaryContainer => { ); }; -const findHeader = ( - bytes: Buffer, - search: GoFileMapping | null, -): number | null => { - if (search === null || search.size === 0) return null; +const findHeader = (bytes: Buffer, search: GoFileMapping): number | null => { + if (search.size === 0) return null; const end = search.offset + search.size; const selected = bytes.subarray(search.offset, end); let offset = Number((16n - (search.address % 16n)) % 16n); diff --git a/src/go/GoElfImage.ts b/src/go/GoElfImage.ts index de4388114..1ba0c3058 100644 --- a/src/go/GoElfImage.ts +++ b/src/go/GoElfImage.ts @@ -103,7 +103,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { const mappings: GoFileMapping[] = []; const zeroFills: GoZeroFillMapping[] = []; const sections: GoImageSection[] = []; - let fallback: GoFileMapping | null = null; + const fallback: GoFileMapping[] = []; for (let index = 0; index < phCount; index++) { const offset = phStart + index * phWidth; if (reader.u32(offset) !== 1) continue; @@ -126,7 +126,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { if (mapping.size !== 0) mappings.push(mapping); if (memorySize > size) zeroFills.push({ address: address + size, size: memorySize - size }); - if (fallback === null && (flags & 3) === 2) fallback = mapping; + if (mapping.size !== 0 && (flags & 3) === 2) fallback.push(mapping); } let search: GoFileMapping | null = null; if (shCount !== 0) { @@ -226,7 +226,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { mappings, zero_fills: zeroFills, sections, - search: search ?? fallback, + search: search === null ? fallback : [search], }; }; diff --git a/src/go/GoMachoImage.ts b/src/go/GoMachoImage.ts index 5184a5829..cba13bf2d 100644 --- a/src/go/GoMachoImage.ts +++ b/src/go/GoMachoImage.ts @@ -193,7 +193,7 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { mappings, zero_fills: zeroFills, sections, - search: named[0] ?? fallback, + search: named.length > 0 ? named : fallback === null ? [] : [fallback], }; }; diff --git a/src/go/GoPeImage.ts b/src/go/GoPeImage.ts index 0787bc93d..304a70397 100644 --- a/src/go/GoPeImage.ts +++ b/src/go/GoPeImage.ts @@ -157,7 +157,7 @@ export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { mappings, zero_fills: zeroFills, sections, - search, + search: search === null ? [] : [search], }; }; diff --git a/src/native/AppleDispatchCoverage.ts b/src/native/AppleDispatchCoverage.ts index a6c68ca51..44c6b9f36 100644 --- a/src/native/AppleDispatchCoverage.ts +++ b/src/native/AppleDispatchCoverage.ts @@ -27,6 +27,8 @@ export const pushDispatchCoverage = (input: { readonly examined: number; readonly categoriesExamined: number; readonly truncated: boolean; + readonly objcListScanTruncated: boolean; + readonly objcClassListScanTruncated: boolean; readonly fixups: PointerFixups; readonly vtables: FacetDecodeFacts; readonly swift: FacetDecodeFacts; @@ -38,6 +40,8 @@ export const pushDispatchCoverage = (input: { examined, categoriesExamined, truncated, + objcListScanTruncated, + objcClassListScanTruncated, fixups, vtables, swift, @@ -78,7 +82,7 @@ export const pushDispatchCoverage = (input: { const classReasons = [ ...failures, ...classRecords, - ...(truncated ? ["max_records_reached"] : []), + ...(truncated || objcListScanTruncated ? ["max_records_reached"] : []), ]; const categoryMethods = result.objc_dispatch_implementations.filter( ({ category }) => category !== undefined, @@ -99,20 +103,18 @@ export const pushDispatchCoverage = (input: { ({ decode }) => decode?.status === "decoded", ).length, }); + const relativePointerReasons = [ + ...(truncated || objcClassListScanTruncated ? ["max_records_reached"] : []), + ...(result.relative_pointers.some( + (item) => item.decode.status !== "decoded", + ) + ? ["relative_pointer_targets_unresolved"] + : []), + ]; result.coverage.push({ facet: "binary_relative_pointers", - status: - truncated || - result.relative_pointers.some((item) => item.decode.status !== "decoded") - ? "partial" - : "complete", - reason: truncated - ? "max_records_reached" - : result.relative_pointers.some( - (item) => item.decode.status !== "decoded", - ) - ? "relative_pointer_targets_unresolved" - : null, + status: relativePointerReasons.length > 0 ? "partial" : "complete", + reason: relativePointerReasons.join("; ") || null, examined: result.relative_pointers.length, decoded: result.relative_pointers.filter( (item) => item.decode.status === "decoded", @@ -126,6 +128,7 @@ export const pushDispatchCoverage = (input: { facet: "objc_properties_categories", status: truncated || + objcClassListScanTruncated || categoryFailures.length > 0 || result.objc_categories.some( ({ decode }) => decode.status !== "decoded", @@ -137,7 +140,9 @@ export const pushDispatchCoverage = (input: { [ ...categoryFailures, ...categoryMethodReasons, - ...(truncated ? ["max_records_reached"] : []), + ...(truncated || objcClassListScanTruncated + ? ["max_records_reached"] + : []), ].join("; ") || null, examined: categoriesExamined, decoded: result.objc_categories.filter( diff --git a/src/native/AppleDispatchMetadata.test.ts b/src/native/AppleDispatchMetadata.test.ts index 244a35aeb..f715c3c4e 100644 --- a/src/native/AppleDispatchMetadata.test.ts +++ b/src/native/AppleDispatchMetadata.test.ts @@ -115,6 +115,50 @@ const fixture = ( return bytes; }; const provenance = { path: "/fixture/app", sha256: "a".repeat(64) }; + +const partialSiblingsFixture = ( + options: { + readonly emptyMiddleClass?: boolean; + } = {}, +) => { + const bytes = Buffer.alloc(4096); + const imageBase = 0x100000000n; + bytes.writeUInt32LE(0xfeedfacf, 0); + bytes.writeUInt32LE(0x0100000c, 4); + bytes.writeUInt32LE(2, 12); + bytes.writeUInt32LE(1, 16); + bytes.writeUInt32LE(232, 20); + bytes.writeUInt32LE(0x19, 32); + bytes.writeUInt32LE(232, 36); + bytes.write("__DATA", 40); + bytes.writeBigUInt64LE(imageBase, 56); + bytes.writeBigUInt64LE(4096n, 64); + bytes.writeBigUInt64LE(4096n, 80); + bytes.writeUInt32LE(5, 92); + bytes.writeUInt32LE(2, 96); + bytes.write("__objc_classlist", 104); + bytes.writeBigUInt64LE(imageBase + 0x200n, 136); + bytes.writeBigUInt64LE(24n, 144); + bytes.write("__objc_protolist", 184); + bytes.writeBigUInt64LE(imageBase + 0x300n, 216); + bytes.writeBigUInt64LE(8n, 224); + for (const [slot, record, ro, name] of [ + [0x200, 0x400, 0x500, 0x600], + [0x208, 0x440, 0x540, 0x620], + [0x210, 0x480, 0x580, 0x640], + ] as const) { + bytes.writeBigUInt64LE(imageBase + BigInt(record), slot); + bytes.writeBigUInt64LE(imageBase + BigInt(ro), record + 32); + bytes.writeUInt32LE(2, ro); + bytes.writeUInt32LE(16, ro + 8); + bytes.writeBigUInt64LE(imageBase + BigInt(name), ro + 24); + } + bytes.write("KeptClass\0", 0x600); + if (!options.emptyMiddleClass) bytes.write("OtherClass\0", 0x620); + bytes.write("LaterClass\0", 0x640); + return bytes; +}; + describe("Apple dispatch pointer fixup metadata", () => { it("keeps decoded bindings and their coverage when an independent stream fails", () => { const bytes = fixture(); @@ -367,6 +411,23 @@ describe("Apple dispatch coverage ownership", () => { }); describe("malformed Apple dispatch metadata", () => { + it("retains valid class siblings when one class has an empty name", () => { + const result = decodeAppleDispatchMetadata( + partialSiblingsFixture({ emptyMiddleClass: true }), + 100, + provenance, + ); + expect(result.objc_classes.map(({ name }) => name)).toEqual([ + "KeptClass", + "LaterClass", + ]); + expect( + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ), + ).toMatchObject({ status: "partial", examined: 3 }); + }); + it("rejects malformed command and section boundaries", () => { const bytes = fixture(); bytes.writeUInt32LE(7, 36); @@ -530,3 +591,218 @@ describe("byte-swapped universal Apple dispatch metadata", () => { ); }); }); + +const unmappedProtocolSectionBase = 0x100000000n; +const unmappedProtocolImage = (bad: "protocol" | "class" | "neither") => { + const bytes = Buffer.alloc(4096); + bytes.writeUInt32LE(0xfeedfacf, 0); + bytes.writeUInt32LE(0x0100000c, 4); + bytes.writeUInt32LE(2, 12); + bytes.writeUInt32LE(1, 16); + bytes.writeUInt32LE(232, 20); + bytes.writeUInt32LE(0x19, 32); + bytes.writeUInt32LE(232, 36); + bytes.write("__DATA", 40); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase, 56); + bytes.writeBigUInt64LE(4096n, 64); + bytes.writeBigUInt64LE(4096n, 80); + bytes.writeUInt32LE(5, 92); + bytes.writeUInt32LE(2, 96); + bytes.write("__objc_classlist", 104); + bytes.writeBigUInt64LE( + unmappedProtocolSectionBase + (bad === "class" ? 0x9000n : 0x200n), + 136, + ); + bytes.writeBigUInt64LE(8n, 144); + bytes.write("__objc_protolist", 184); + bytes.writeBigUInt64LE( + unmappedProtocolSectionBase + (bad === "protocol" ? 0x9000n : 0x300n), + 216, + ); + bytes.writeBigUInt64LE(8n, 224); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x400n, 0x200); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x500n, 0x420); + bytes.writeUInt32LE(2, 0x500); + bytes.writeUInt32LE(16, 0x508); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x600n, 0x518); + bytes.write("RetainedClass\0", 0x600); + return bytes; +}; +const unmappedProtocolClassCoverage = (result: { + coverage: { facet: string; status: string; reason: string | null }[]; +}) => + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ); + +describe("Apple dispatch unmapped protocol-list slot", () => { + it("returns the class when every list slot is mapped", () => { + const result = decodeAppleDispatchMetadata( + unmappedProtocolImage("neither"), + 100, + provenance, + ); + expect(result.objc_classes).toMatchObject([{ name: "RetainedClass" }]); + expect(unmappedProtocolClassCoverage(result)).toMatchObject({ + status: "complete", + }); + }); + + it("records partial coverage when the class-list slot is unmapped", () => { + const result = decodeAppleDispatchMetadata( + unmappedProtocolImage("class"), + 100, + provenance, + ); + expect(unmappedProtocolClassCoverage(result)).toMatchObject({ + status: "partial", + reason: expect.stringContaining("0x100009000"), + }); + }); + + it("retains the class and partial coverage when the protocol-list slot is unmapped", () => { + const result = decodeAppleDispatchMetadata( + unmappedProtocolImage("protocol"), + 100, + provenance, + ); + expect(result.objc_classes).toMatchObject([{ name: "RetainedClass" }]); + expect(unmappedProtocolClassCoverage(result)).toMatchObject({ + status: "partial", + reason: expect.stringContaining("0x100009000"), + }); + }); +}); + +describe("Apple dispatch list scan budget", () => { + it("bounds failed protocol-list slots by the record budget", () => { + // The section size is file-declared, so a crafted image can claim an + // arbitrarily large unmapped protocol list. Failed slots must spend the + // record budget like decoded ones, otherwise the loop runs unbounded. + const bytes = unmappedProtocolImage("protocol"); + bytes.writeBigUInt64LE(80000n, 224); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + const coverage = unmappedProtocolClassCoverage(result); + expect(coverage).toMatchObject({ status: "partial" }); + expect(coverage?.reason).toContain("max_records_reached"); + const failures = + coverage?.reason?.match(/Unmapped or ambiguous metadata pointer/g) ?? []; + expect(failures.length).toBeLessThanOrEqual(100); + }); + + it("keeps independent facets complete when only zero-pointer list scanning is capped", () => { + const bytes = unmappedProtocolImage("neither"); + bytes.writeBigUInt64LE(0n, 144); + bytes.writeBigUInt64LE(16n, 224); + const result = decodeAppleDispatchMetadata(bytes, 1, provenance); + expect(unmappedProtocolClassCoverage(result)).toMatchObject({ + status: "partial", + reason: expect.stringContaining("max_records_reached"), + }); + for (const facet of [ + "objc_properties_categories", + "binary_relative_pointers", + ]) + expect( + result.coverage.find((item) => item.facet === facet), + ).toMatchObject({ + status: "complete", + reason: null, + }); + }); + + it("marks class-dependent coverage partial when zero slots hide a later class", () => { + const bytes = unmappedProtocolImage("neither"); + bytes.writeBigUInt64LE(0n, 224); + bytes.writeBigUInt64LE(16n, 144); + bytes.writeBigUInt64LE(0n, 0x200); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x400n, 0x208); + const result = decodeAppleDispatchMetadata(bytes, 1, provenance); + expect(result.objc_classes).toEqual([]); + for (const facet of [ + "objc_class_method_ivar_metadata", + "objc_properties_categories", + "binary_relative_pointers", + ]) + expect( + result.coverage.find((item) => item.facet === facet), + ).toMatchObject({ + status: "partial", + reason: expect.stringContaining("max_records_reached"), + }); + for (const facet of [ + "swift_class_vtable_descriptors", + "swift_conformances_static_witness_slots", + ]) + expect( + result.coverage.find((item) => item.facet === facet), + ).toMatchObject({ status: "complete" }); + }); + + it("bounds failed class-list slots by the record budget", () => { + // Same file-declared-size hazard as the protocol list: a crafted image + // can declare an arbitrarily large unmapped class list. Failed class + // slots must spend the record budget like decoded ones, otherwise the + // loop runs unbounded (each failed read produced an unbounded failure + // string with no budget spend). + const bytes = unmappedProtocolImage("class"); + bytes.writeBigUInt64LE(80000n, 144); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + const coverage = unmappedProtocolClassCoverage(result); + expect(coverage).toMatchObject({ status: "partial" }); + expect(coverage?.reason).toContain("max_records_reached"); + const failures = + coverage?.reason?.match(/Unmapped or ambiguous metadata pointer/g) ?? []; + expect(failures.length).toBeLessThanOrEqual(100); + }); + + it("reports the real failure for a mapped-but-malformed class at the budget boundary", () => { + // The class-list slot itself is mapped, so readClass admits the record + // budget and only then throws on the unmapped class_ro pointer. The + // failure must be recorded as-is: it already spent one budget unit, so + // admitting again would break before the reason is appended and report + // only max_records_reached. + const bytes = unmappedProtocolImage("neither"); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x700n, 0x200); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x9000n, 0x720); + const result = decodeAppleDispatchMetadata(bytes, 1, provenance); + const coverage = unmappedProtocolClassCoverage(result); + expect(coverage).toMatchObject({ status: "partial" }); + expect(coverage?.reason).toContain( + "Unmapped or ambiguous metadata pointer 0x100009018", + ); + expect(coverage?.reason).not.toContain("max_records_reached"); + }); + + it("does not skip a valid class after a mapped-but-malformed class", () => { + // With two slots and a budget of two, the corrupt first slot must spend + // exactly one unit: admitting twice would exhaust the budget and skip + // the valid class behind it. + const bytes = unmappedProtocolImage("neither"); + bytes.writeBigUInt64LE(16n, 144); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x700n, 0x200); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x9000n, 0x720); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x400n, 0x208); + const result = decodeAppleDispatchMetadata(bytes, 2, provenance); + expect(result.objc_classes).toMatchObject([{ name: "RetainedClass" }]); + expect(unmappedProtocolClassCoverage(result)?.reason).toContain( + "Unmapped or ambiguous metadata pointer 0x100009018", + ); + }); + + it("reports the real failure for a mapped-but-truncated protocol slot", () => { + // The protocol slot is mapped, so protocolReader.record admits the + // budget itself before throwing on the truncated declaration. The loop + // must not admit a second time; the recorded reason stays the + // declaration failure, not max_records_reached. + const bytes = unmappedProtocolImage("neither"); + bytes.writeBigUInt64LE(unmappedProtocolSectionBase + 0x700n, 0x300); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + const coverage = unmappedProtocolClassCoverage(result); + expect(coverage).toMatchObject({ status: "partial" }); + expect(coverage?.reason).toContain( + "Truncated Objective-C protocol declaration", + ); + expect(coverage?.reason).not.toContain("max_records_reached"); + }); +}); diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index 26ff8caaf..dfc5aec6a 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -105,15 +105,22 @@ export const decodeAppleDispatchMetadata = ( evidence, offset, }; - const { failures, categoryIssues, examined, categoriesExamined } = - decodeObjcDispatchFacets({ - bytes, - sections, - segments, - readers, - budget, - result, - }); + const { + failures, + categoryIssues, + examined, + categoriesExamined, + objcListScanTruncated, + objcClassListScanTruncated, + } = decodeObjcDispatchFacets({ + bytes, + sections, + segments, + readers, + budget, + maxListSlots: maxRecords, + result, + }); const swiftRelative = createSwiftRelativeReader({ bytes, readers, @@ -163,6 +170,8 @@ export const decodeAppleDispatchMetadata = ( categoryIssues, examined, categoriesExamined, + objcListScanTruncated, + objcClassListScanTruncated, truncated: budget.truncated, fixups, vtables, diff --git a/src/native/AppleObjcDispatchFacets.ts b/src/native/AppleObjcDispatchFacets.ts index b52545840..955fb956e 100644 --- a/src/native/AppleObjcDispatchFacets.ts +++ b/src/native/AppleObjcDispatchFacets.ts @@ -1,3 +1,4 @@ +import { err, ok, type Result } from "../domain/result.js"; import type { ObjcSwiftMetadata } from "../domain/native/objcSwiftMetadata.js"; import type { Segment, Section } from "./AppleMachoSelection.js"; import { boundClassName, decodeObjcCategories } from "./AppleObjcCategories.js"; @@ -67,14 +68,25 @@ export const decodeObjcDispatchFacets = (input: { segments: readonly Segment[]; readers: ObjcDispatchReaders; budget: DispatchRecordBudget; + maxListSlots: number; result: ObjcSwiftMetadata; }): { failures: string[]; categoryIssues: DecodeIssue[]; examined: number; categoriesExamined: number; + objcListScanTruncated: boolean; + objcClassListScanTruncated: boolean; } => { - const { bytes, sections, segments, readers: read, budget, result } = input; + const { + bytes, + sections, + segments, + readers: read, + budget, + maxListSlots, + result, + } = input; const failures: string[] = []; const categoryIssues: DecodeIssue[] = []; let examined = 0; @@ -190,6 +202,33 @@ export const decodeObjcDispatchFacets = (input: { result, failures, }); + // Each list scan has its own slot bound: zero and duplicate pointers do + // not produce records, but still consume pointer-reading work. The bound + // is per list so one list's slots cannot starve the other list's scan. + // Record decoders retain their existing admission ownership. + let objcListScanTruncated = false; + let objcClassListScanTruncated = false; + const createListSlotReader = () => { + let listSlotsExamined = 0; + return (field: bigint): Result => { + if (listSlotsExamined >= maxListSlots) { + objcListScanTruncated = true; + return err("exhausted"); + } + listSlotsExamined++; + try { + return ok(read.pointer(field)); + } catch (cause: unknown) { + const admitted = budget.admit(); + failures.push( + `${hex(field)}: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + return err(admitted ? "unmapped" : "exhausted"); + } + }; + }; + const readProtocolListSlot = createListSlotReader(); + const readClassListSlot = createListSlotReader(); for (const section of sections.filter( ({ name }) => name === "__objc_protolist", )) { @@ -200,7 +239,21 @@ export const decodeObjcDispatchFacets = (input: { budget.markTruncated(); break; } - protocolReader.record(read.pointer(section.address + BigInt(index * 8))); + const field = section.address + BigInt(index * 8); + const pointer = readProtocolListSlot(field); + if (!pointer.ok) { + if (pointer.error === "exhausted") break; + continue; + } + try { + protocolReader.record(pointer.value); + } catch (cause) { + // record() admits the slot itself before decoding, so admitting + // again here would count one protocol twice. + failures.push( + `${hex(field)}: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } } } const visited = new Set(); @@ -209,6 +262,10 @@ export const decodeObjcDispatchFacets = (input: { visited.add(hex(address)); const ro = read.pointer(address + 32n) & ~7n; const name = read.string(read.pointer(ro + 24n)); + if (name.length === 0) + throw new TypeError( + `Objective-C class at ${hex(address)} has an empty name`, + ); const root = (read.u32(ro) & 2) !== 0; const superclass = resolveSuperclass(read, address, name, failures); if (superclass.name === null && superclass.address === null && !root) @@ -320,11 +377,24 @@ export const decodeObjcDispatchFacets = (input: { budget.markTruncated(); break; } - examined++; const field = section.address + BigInt(index * 8); + const pointer = readClassListSlot(field); + if (pointer.ok || pointer.error === "unmapped") examined++; + if (!pointer.ok) { + if (pointer.error === "exhausted") { + objcClassListScanTruncated = true; + break; + } + continue; + } try { - readClass(read.pointer(field)); + readClass(pointer.value); } catch (cause) { + // The slot was already admitted inside readClass before anything + // could throw (its admit runs before the first pointer read), so + // admitting again here would count one class twice: at the budget + // boundary it would break before appending the failure reason, and + // with larger limits malformed classes would starve valid ones. failures.push( `${hex(field)}: ${cause instanceof Error ? cause.message : String(cause)}`, ); @@ -341,5 +411,12 @@ export const decodeObjcDispatchFacets = (input: { protocols: (list) => protocolReader.list(list), localClassName: (address) => localClassName(read, address), }); - return { failures, categoryIssues, examined, categoriesExamined }; + return { + failures, + categoryIssues, + examined, + categoriesExamined, + objcListScanTruncated, + objcClassListScanTruncated, + }; }; diff --git a/src/native/AppleSwiftVtables.test.ts b/src/native/AppleSwiftVtables.test.ts index fb1615820..fe79b955e 100644 --- a/src/native/AppleSwiftVtables.test.ts +++ b/src/native/AppleSwiftVtables.test.ts @@ -60,6 +60,8 @@ describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => { examined: 0, categoriesExamined: 0, truncated: false, + objcListScanTruncated: false, + objcClassListScanTruncated: false, fixups: { kind: "none", formats: [], @@ -165,6 +167,8 @@ describe("Swift vtable coverage facts: unsupported pointer fixup coverage", () = examined: 0, categoriesExamined: 0, truncated: false, + objcListScanTruncated: false, + objcClassListScanTruncated: false, fixups: { kind: "chained", formats: ["format-99"], diff --git a/src/native/parsers/demangle.test.ts b/src/native/parsers/demangle.test.ts new file mode 100644 index 000000000..5b15229bd --- /dev/null +++ b/src/native/parsers/demangle.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; + +import { parseDemangledSymbols } from "./demangle.js"; +import { swiftSymbolsSchema } from "../../contracts/native/nativeToolContracts.js"; + +describe("parseDemangledSymbols", () => { + it("returns an empty array when given an empty input list", () => { + expect(swiftSymbolsSchema.safeParse([]).success).toBe(true); + expect(parseDemangledSymbols([], "")).toEqual([]); + expect(parseDemangledSymbols([], "\n")).toEqual([]); + }); + + it("rejects demangler output when the input batch is empty", () => { + expect(() => parseDemangledSymbols([], "unexpected\n")).toThrow( + "swift-demangle emitted output for an empty input", + ); + }); + + it("parses demangled symbols successfully", () => { + const inputs = ["$s4main4testyyF"]; + const output = "main.test() -> ()\n"; + expect(parseDemangledSymbols(inputs, output)).toEqual([ + { + input: "$s4main4testyyF", + output: "main.test() -> ()", + status: "demangled", + }, + ]); + }); + + it("identifies unchanged non-Swift symbols", () => { + const inputs = ["_printf"]; + const output = "_printf\n"; + expect(parseDemangledSymbols(inputs, output)).toEqual([ + { + input: "_printf", + output: "_printf", + status: "unchanged", + }, + ]); + }); + + it("identifies invalid Swift symbols that fail demangling", () => { + const inputs = ["$sInvalidMangle", "_$sAlsoInvalid"]; + const output = "$sInvalidMangle\n_$sAlsoInvalid\n"; + expect(parseDemangledSymbols(inputs, output)).toEqual([ + { + input: "$sInvalidMangle", + output: "$sInvalidMangle", + status: "invalid", + }, + { + input: "_$sAlsoInvalid", + output: "_$sAlsoInvalid", + status: "invalid", + }, + ]); + }); + + it("parses mixed batches preserving ordering", () => { + const inputs = ["$s4main4testyyF", "_printf", "$sInvalidMangle"]; + const output = "main.test() -> ()\n_printf\n$sInvalidMangle\n"; + expect(parseDemangledSymbols(inputs, output)).toEqual([ + { + input: "$s4main4testyyF", + output: "main.test() -> ()", + status: "demangled", + }, + { + input: "_printf", + output: "_printf", + status: "unchanged", + }, + { + input: "$sInvalidMangle", + output: "$sInvalidMangle", + status: "invalid", + }, + ]); + }); + + it("throws TypeError when output count does not match input count", () => { + expect(() => + parseDemangledSymbols(["sym1", "sym2"], "demangled1\n"), + ).toThrow(TypeError); + expect(() => + parseDemangledSymbols(["sym1"], "demangled1\ndemangled2\n"), + ).toThrow("swift-demangle output count does not match input"); + }); +}); diff --git a/src/native/parsers/demangle.ts b/src/native/parsers/demangle.ts index ba960be56..afcb30b62 100644 --- a/src/native/parsers/demangle.ts +++ b/src/native/parsers/demangle.ts @@ -6,6 +6,8 @@ const resultSchema = z.object({ status: z.enum(["demangled", "unchanged", "invalid"]), }); +export type DemangledSymbol = z.infer; + /** * Preserve input ordering while parsing `swift-demangle --compact` lines. * The demangler ends each record with LF, so a carriage return belongs to the @@ -14,7 +16,11 @@ const resultSchema = z.object({ export const parseDemangledSymbols = ( inputs: readonly string[], output: string, -) => { +): DemangledSymbol[] => { + if (inputs.length === 0) { + if (output === "" || output === "\n") return []; + throw new TypeError("swift-demangle emitted output for an empty input"); + } const lines = output.replace(/\n$/u, "").split("\n"); if (lines.length !== inputs.length) throw new TypeError("swift-demangle output count does not match input"); diff --git a/src/server/registerArtifactTools.ts b/src/server/registerArtifactTools.ts index 1217664e6..d2df1da9e 100644 --- a/src/server/registerArtifactTools.ts +++ b/src/server/registerArtifactTools.ts @@ -1,39 +1,36 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; -import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import { ARTIFACT_TOOL_CONTRACTS, artifactInspectionInputSchema, } from "../contracts/artifactToolContracts.js"; -import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; -import { - requireAnalysisViewRetention, - summarizeRetainedAnalysis, -} from "../application/analysisView/AnalysisViewService.js"; import { createArtifactExtractionDestination } from "../application/artifacts/ArtifactExtractionDestination.js"; -import { ok } from "../domain/result.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; -import type { Logger } from "pino"; import { registerEvidenceTools } from "./registerEvidenceTools.js"; import { artifactInspectionResultSchema } from "../domain/artifactInspection.js"; +import { + retainedSummaryDelivery, + withoutDeliveryDetail, + type EvidenceToolFamilyOptions, +} from "./retainedSummaryDelivery.js"; + +/** Operations whose MCP delivery may be a retained summary view. */ +const SUMMARY_DELIVERY: ReadonlySet = new Set([ + "inspect_artifact", + "decode_interface_builder", +]); +/** inspect_artifact keeps an explicit detail in its Evidence parameters. */ +const decodeWithoutDelivery = withoutDeliveryDetail( + new Set(["decode_interface_builder"]), +); /** Register deterministic artifact inventory and safe extraction operations. */ export const registerArtifactTools = ( server: EvidenceMcpServer, - options: { - readonly logger: Logger; - readonly activeTarget: (() => BinaryTarget | undefined) | undefined; - readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; - readonly evidenceById?: EvidenceLookup | undefined; - readonly withAdmittedAnalysis: import("./analysisAdmission.js").WithAdmittedAnalysis; - }, + options: EvidenceToolFamilyOptions, ): void => { - const retention = { - recordEvidence: options.recordEvidence, - evidenceById: options.evidenceById, - }; registerEvidenceTools(server, ARTIFACT_TOOL_CONTRACTS, { ...options, + ...retainedSummaryDelivery(SUMMARY_DELIVERY, options), parametersForExecution: (operation, parameters) => { if (operation === "inspect_artifact") { const { detail: _detail, ...execution } = @@ -42,16 +39,9 @@ export const registerArtifactTools = ( } return operation === "extract_artifact" ? { ...parameters, output_root: createArtifactExtractionDestination() } - : parameters; + : decodeWithoutDelivery(operation, parameters); }, - preflight: (operation, parameters) => - operation === "inspect_artifact" && parameters.detail === "summary" - ? requireAnalysisViewRetention(operation, retention) - : ok(null), - projectEvidence: (operation, parameters, evidence) => - operation === "inspect_artifact" && parameters.detail === "summary" - ? summarizeRetainedAnalysis(evidence, retention) - : ok(evidence), + parametersForEvidence: decodeWithoutDelivery, sourceEvidence: (operation, result) => operation === "inspect_artifact" ? artifactInspectionResultSchema diff --git a/src/server/registerManagedTools.ts b/src/server/registerManagedTools.ts index 5ff513d67..2f00d16ce 100644 --- a/src/server/registerManagedTools.ts +++ b/src/server/registerManagedTools.ts @@ -1,6 +1,4 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; -import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; -import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; import { requireAnalysisViewRetention, summarizeRetainedAnalysis, @@ -8,24 +6,17 @@ import { import type { AnalysisOperationPort } from "../application/AnalysisProvider.js"; import { MANAGED_TOOL_CONTRACTS } from "../contracts/managed/managedToolContracts.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; import { err, ok } from "../domain/result.js"; -import type { Logger } from "pino"; import { registerEvidenceTools } from "./registerEvidenceTools.js"; +import type { EvidenceToolFamilyOptions } from "./retainedSummaryDelivery.js"; import { runManagedProviderExecution } from "../composition/directAnalysis.js"; import { isManagedToolName } from "../contracts/managed/managedToolContracts.js"; /** Register execution-free managed PE/CLI inspection. */ export const registerManagedTools = ( server: EvidenceMcpServer, - options: { - readonly logger: Logger; - readonly activeTarget: (() => BinaryTarget | undefined) | undefined; - readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; - readonly evidenceById?: EvidenceLookup | undefined; - readonly withAdmittedAnalysis: import("./analysisAdmission.js").WithAdmittedAnalysis; - }, + options: EvidenceToolFamilyOptions, ): void => { let selectedManagedPath: string | undefined; const analysisForAdmission = ( diff --git a/src/server/registerNativeTools.ts b/src/server/registerNativeTools.ts index 036389d16..62668daa1 100644 --- a/src/server/registerNativeTools.ts +++ b/src/server/registerNativeTools.ts @@ -1,54 +1,25 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; -import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import { NATIVE_TOOL_CONTRACTS } from "../contracts/native/nativeToolContracts.js"; -import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; -import { - requireAnalysisViewRetention, - summarizeRetainedAnalysis, -} from "../application/analysisView/AnalysisViewService.js"; -import type { JsonValue } from "../domain/jsonValue.js"; -import { ok } from "../domain/result.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; -import type { Logger } from "pino"; import { registerEvidenceTools } from "./registerEvidenceTools.js"; +import { + retainedSummaryDelivery, + withoutDeliveryDetail, + type EvidenceToolFamilyOptions, +} from "./retainedSummaryDelivery.js"; -/** Delivery detail selects a response shape; it never reaches the tools or Evidence identity. */ -const withoutDelivery = ( - operation: string, - parameters: Readonly>, -): Readonly> => { - if (operation !== "inspect_macho") return parameters; - const { detail: _detail, ...selected } = parameters; - return selected; -}; +const SUMMARY_DELIVERY: ReadonlySet = new Set(["inspect_macho"]); /** Register macOS native contracts; Mach-O inspection can deliver a retained summary. */ export const registerNativeTools = ( server: EvidenceMcpServer, - options: { - readonly logger: Logger; - readonly activeTarget: (() => BinaryTarget | undefined) | undefined; - readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; - readonly evidenceById?: EvidenceLookup | undefined; - readonly withAdmittedAnalysis: import("./analysisAdmission.js").WithAdmittedAnalysis; - }, + options: EvidenceToolFamilyOptions, ): void => { - const retention = { - recordEvidence: options.recordEvidence, - evidenceById: options.evidenceById, - }; + const withoutDelivery = withoutDeliveryDetail(SUMMARY_DELIVERY); registerEvidenceTools(server, NATIVE_TOOL_CONTRACTS, { ...options, + ...retainedSummaryDelivery(SUMMARY_DELIVERY, options), parametersForExecution: withoutDelivery, parametersForEvidence: withoutDelivery, - preflight: (operation, parameters) => - operation === "inspect_macho" && parameters.detail === "summary" - ? requireAnalysisViewRetention(operation, retention) - : ok(null), - projectEvidence: (operation, parameters, evidence) => - operation === "inspect_macho" && parameters.detail === "summary" - ? summarizeRetainedAnalysis(evidence, retention) - : ok(evidence), }); }; diff --git a/src/server/retainedSummaryDelivery.ts b/src/server/retainedSummaryDelivery.ts new file mode 100644 index 000000000..0bc703a7c --- /dev/null +++ b/src/server/retainedSummaryDelivery.ts @@ -0,0 +1,62 @@ +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; +import { + requireAnalysisViewRetention, + summarizeRetainedAnalysis, +} from "../application/analysisView/AnalysisViewService.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import type { Evidence } from "../domain/evidence.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { ok } from "../domain/result.js"; +import type { Logger } from "pino"; +import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; + +/** Registration inputs shared by provider-backed Evidence tool families. */ +export interface EvidenceToolFamilyOptions { + readonly logger: Logger; + readonly activeTarget: (() => BinaryTarget | undefined) | undefined; + readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; + readonly evidenceById?: EvidenceLookup | undefined; + readonly withAdmittedAnalysis: WithAdmittedAnalysis; +} + +type Parameters = Readonly>; + +/** + * Summary delivery for the named operations: refuse it before analysis when + * this server retains no Evidence, then return the retained summary view. + */ +export const retainedSummaryDelivery = ( + operations: ReadonlySet, + options: Pick, +) => { + const retention = { + recordEvidence: options.recordEvidence, + evidenceById: options.evidenceById, + }; + const selected = (operation: string, parameters: Parameters): boolean => + operations.has(operation) && parameters.detail === "summary"; + return { + preflight: (operation: string, parameters: Parameters) => + selected(operation, parameters) + ? requireAnalysisViewRetention(operation, retention) + : ok(null), + projectEvidence: ( + operation: string, + parameters: Parameters, + evidence: Evidence, + ) => + selected(operation, parameters) + ? summarizeRetainedAnalysis(evidence, retention) + : ok(evidence), + }; +}; + +/** Delivery detail selects a response shape; it never reaches tools or Evidence identity. */ +export const withoutDeliveryDetail = + (operations: ReadonlySet) => + (operation: string, parameters: Parameters): Parameters => { + if (!operations.has(operation)) return parameters; + const { detail: _detail, ...selected } = parameters; + return selected; + }; diff --git a/tests/boundary/go/goBinaryImage.test.ts b/tests/boundary/go/goBinaryImage.test.ts index c8069be98..9bc91c185 100644 --- a/tests/boundary/go/goBinaryImage.test.ts +++ b/tests/boundary/go/goBinaryImage.test.ts @@ -5,13 +5,113 @@ import { fatImage, machoImage, } from "../../../src/artifacts/apple/MachoImage.fixture.js"; -import { readGoBinaryImage } from "../../../src/go/GoBinaryImage.js"; +import { + readGoBinaryImage, + GoBinaryResourceFailure, +} from "../../../src/go/GoBinaryImage.js"; import { readGoMachoImage } from "../../../src/go/GoMachoImage.js"; import { createGoBinaryFixture, + createGoSplitElfFixture, + createGoOverlappingElfFixture, GO_MODULE_TEXT, } from "../../fixtures/go/image.js"; +it("bounds cumulative header searches across overlapping ELF file spans", () => { + const bytes = createGoOverlappingElfFixture(); + expect(() => readGoBinaryImage(bytes)).toThrow(GoBinaryResourceFailure); + expect(() => readGoBinaryImage(bytes)).toThrow(/256 MiB.*search/); +}); + +it("does not charge duplicate ELF search spans repeatedly", () => { + expect( + readGoBinaryImage(createGoOverlappingElfFixture(true)).build_info, + ).toBeNull(); +}); + +it("accepts a search exactly at the cumulative byte limit", () => { + const bytes = createGoOverlappingElfFixture(); + bytes.writeUInt16LE(256, 56); + expect(readGoBinaryImage(bytes).build_info).toBeNull(); +}); + +it("keeps different virtual alignments distinct when file spans match", () => { + const fixture = createGoBinaryFixture({ sectionless: true }); + fixture.bytes.copy(fixture.bytes, 184, 128, 184); + fixture.bytes.writeUInt16LE(2, 56); + fixture.bytes.writeBigUInt64LE(0x200001n, 128 + 16); + expect(readGoBinaryImage(fixture.bytes).build_info?.header_offset).toBe( + fixture.headerOffset, + ); +}); + +it.each([ + [32, "little", "inline"], + [32, "little", "pointer"], + [32, "big", "inline"], + [32, "big", "pointer"], + [64, "little", "inline"], + [64, "little", "pointer"], + [64, "big", "inline"], + [64, "big", "pointer"], +] as const)( + "retains later-load ELF metadata without sections (%d %s %s)", + (bits, byteOrder, encoding) => { + const original = createGoSplitElfFixture({ bits, byteOrder, encoding }); + const sectionless = createGoSplitElfFixture({ + bits, + byteOrder, + encoding, + sectionless: true, + }); + const expected = readGoBinaryImage(original.bytes).build_info; + expect(expected).not.toBeNull(); + expect(readGoBinaryImage(sectionless.bytes).build_info).toEqual(expected); + }, +); + +it("continues past an empty file-backed ELF data load", () => { + const fixture = createGoSplitElfFixture({ sectionless: true }); + const expected = readGoBinaryImage(fixture.bytes).build_info; + fixture.bytes.writeBigUInt64LE(0n, 128 + 32); + expect(readGoBinaryImage(fixture.bytes).build_info).toEqual(expected); +}); + +it.each([4, 5, 7])( + "does not search a later non-data ELF load with flags %d", + (flags) => { + const fixture = createGoSplitElfFixture({ sectionless: true }); + fixture.bytes.writeUInt32LE(flags, 184 + 4); + expect(readGoBinaryImage(fixture.bytes).build_info).toBeNull(); + }, +); + +it("prefers the named ELF section to an earlier writable-load marker", () => { + const fixture = createGoSplitElfFixture(); + const expected = readGoBinaryImage(fixture.bytes).build_info; + fixture.bytes.copy( + fixture.bytes, + 512, + fixture.headerOffset, + fixture.headerOffset + 32, + ); + expect(readGoBinaryImage(fixture.bytes).build_info).toEqual(expected); +}); + +it("does not skip malformed earlier metadata to report a later valid record", () => { + const fixture = createGoSplitElfFixture({ sectionless: true }); + fixture.bytes.copy( + fixture.bytes, + 512, + fixture.headerOffset, + fixture.headerOffset + 32, + ); + fixture.bytes[512 + 15] = 0xff; + expect(() => readGoBinaryImage(fixture.bytes)).toThrow( + /unsupported format bit/, + ); +}); + it.each([ ["elf", 32, "little"], ["elf", 32, "big"], diff --git a/tests/boundary/mcp/interfaceBuilderSummaryMcp.test.ts b/tests/boundary/mcp/interfaceBuilderSummaryMcp.test.ts new file mode 100644 index 000000000..b62fcdf53 --- /dev/null +++ b/tests/boundary/mcp/interfaceBuilderSummaryMcp.test.ts @@ -0,0 +1,152 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; +import { z } from "zod"; + +import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interfaceBuilderGraph.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { openProductionTarget } from "../../support/mcp/productionSession.js"; + +const evidenceOf = (schema: Schema) => + z.object({ + evidence_id: z.string(), + parameters: z.record(z.string(), z.unknown()), + normalized_result: schema, + }); +const summarySchema = z.object({ + kind: z.literal("summary"), + parent_evidence_id: z.string(), + summary: z.object({ + documents: z.array(z.unknown()), + available_counts: z.object({ + nodes: z.number(), + edges: z.number(), + coverage: z.number(), + }), + }), +}); +const pageSchema = z.object({ + kind: z.literal("interface-builder"), + items: z.array(z.unknown()), + coverage: z.object({ next_offset: z.number().nullable() }), +}); + +/** A two-document app; each nib is a real rooted UIKit outlet archive. */ +const fixtureApp = async (): Promise => { + const app = join( + await createTestTempDirectory("rea-ib-summary-mcp-"), + "Fixture.app", + ); + const contents = join(app, "Contents"); + const resources = join(contents, "Resources"); + await mkdir(resources, { recursive: true }); + await mkdir(join(contents, "MacOS")); + await writeFile( + join(contents, "MacOS", "Fixture"), + thinMach(0xfeedfacf, 0x0100000c), + ); + await writeFile( + join(contents, "Info.plist"), + "CFBundleExecutableFixture", + ); + const nib = await readFile( + join(process.cwd(), "tests/fixtures/interface-builder/UIKitOutlet.xml"), + ); + await writeFile(join(resources, "Main.nib"), nib); + await writeFile(join(resources, "Second.nib"), nib); + return app; +}; + +mcpTest( + "retains complete Interface Builder Evidence and pages one document's rows", + async ({ mcp, onTestFinished }) => { + const app = await fixtureApp(); + const client = await openProductionTarget(mcp, onTestFinished, app); + + // Summary first: its own retained parent must serve every later page. + const summarized = await client.callTool({ + name: "decode_interface_builder", + arguments: { detail: "summary" }, + }); + expect(summarized.isError, JSON.stringify(summarized.content)).not.toBe( + true, + ); + const summary = evidenceOf(summarySchema).parse( + summarized.structuredContent, + ).normalized_result; + const source = { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }; + const readAll = async (view: Record) => { + const rows: unknown[] = []; + let offset: number | null = 0; + while (offset !== null) { + const paged = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source, + view: { kind: "interface-builder", ...view, offset, limit: 2 }, + }, + }); + expect(paged.isError, JSON.stringify(paged.content)).not.toBe(true); + const page = evidenceOf(pageSchema).parse( + paged.structuredContent, + ).normalized_result; + rows.push(...page.items); + offset = page.coverage.next_offset; + } + return rows; + }; + const main = "Contents/Resources/Main.nib"; + const edges = await readAll({ collection: "edges" }); + const mainNodes = await readAll({ collection: "nodes", document: main }); + + // Only then compare with a complete call made for reference. + const complete = await client.callTool({ + name: "decode_interface_builder", + arguments: {}, + }); + expect(complete.isError, JSON.stringify(complete.content)).not.toBe(true); + const full = evidenceOf(interfaceBuilderAnalysisSchema).parse( + complete.structuredContent, + ); + expect(full.parameters).not.toHaveProperty("detail"); + const report = full.normalized_result; + expect(report.documents).toHaveLength(2); + // Delivery detail never reaches the decoder or the source Evidence ID. + expect(summary.parent_evidence_id).toBe(full.evidence_id); + expect(summary.summary.documents).toEqual(report.documents); + expect(summary.summary.available_counts).toEqual({ + nodes: report.graph.nodes.length, + edges: report.graph.edges.length, + coverage: report.graph.coverage.length, + }); + expect(edges).toEqual(report.graph.edges); + const expectedMainNodes = report.graph.nodes.filter(({ evidence }) => + evidence.some(({ artifact_path }) => artifact_path === main), + ); + expect(expectedMainNodes.length).toBeGreaterThan(0); + expect(expectedMainNodes.length).toBeLessThan(report.graph.nodes.length); + expect(mainNodes).toEqual(expectedMainNodes); + + const unknownDocument = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source, + view: { + kind: "interface-builder", + collection: "nodes", + document: "Contents/Resources/Missing.nib", + }, + }, + }); + expect(parseMcpToolError(unknownDocument)).toMatchObject({ + error: { code: "invalid_request" }, + }); + }, +); diff --git a/tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts b/tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts index 4a40a3064..485c058dd 100644 --- a/tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts +++ b/tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts @@ -1,40 +1,24 @@ -import { silentLogger } from "../../../src/logger.js"; import { expect } from "vitest"; import { z } from "zod"; -import { createBinarySession } from "../../../src/composition/binary.js"; -import { parseConfig } from "../../../src/config/parseConfig.js"; import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; -import { createServer } from "../../../src/server/createServer.js"; import { archiveNumberMetadataCases, expectKeyedArchiveNumberMetadata, keyedArchiveNumberMetadataFixture, } from "../../fixtures/keyedArchiveNumberMetadata.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { openProductionTarget } from "../../support/mcp/productionSession.js"; mcpTest.for(archiveNumberMetadataCases)( "MCP preserves numeric meaning for $name", async (item, { mcp, onTestFinished }) => { const fixture = await keyedArchiveNumberMetadataFixture(item); - const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); - if (!configured.ok) throw configured.error; - const session = createBinarySession( - configured.value, - silentLogger, - process.env, + const client = await openProductionTarget( + mcp, + onTestFinished, + fixture.path, ); - onTestFinished(async () => { - await session.close(); - }); - const client = await mcp.connect( - createServer({ kind: "session", session }), - ); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: fixture.path }, - }); - expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); const result = await client.callTool({ name: "inspect_keyed_archive", arguments: {}, diff --git a/tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts b/tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts index b7803f0bd..17d536003 100644 --- a/tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts +++ b/tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts @@ -1,39 +1,23 @@ import { expect } from "vitest"; import { z } from "zod"; -import { createBinarySession } from "../../../src/composition/binary.js"; -import { parseConfig } from "../../../src/config/parseConfig.js"; import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; -import { silentLogger } from "../../../src/logger.js"; -import { createServer } from "../../../src/server/createServer.js"; import { expectKeyedArchivePrototypeKeys, keyedArchivePrototypeKeyFixture, } from "../../fixtures/keyedArchivePrototypeKeys.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { openProductionTarget } from "../../support/mcp/productionSession.js"; mcpTest( "MCP retains ordinary evidence with formatted XML prototype keys", async ({ mcp, onTestFinished }) => { const fixture = await keyedArchivePrototypeKeyFixture(); - const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); - if (!configured.ok) throw configured.error; - const session = createBinarySession( - configured.value, - silentLogger, - process.env, + const client = await openProductionTarget( + mcp, + onTestFinished, + fixture.path, ); - onTestFinished(async () => { - await session.close(); - }); - const client = await mcp.connect( - createServer({ kind: "session", session }), - ); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: fixture.path }, - }); - expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); const result = await client.callTool({ name: "inspect_keyed_archive", arguments: {}, diff --git a/tests/boundary/mcp/keyedArchiveXmlExtensionMcp.test.ts b/tests/boundary/mcp/keyedArchiveXmlExtensionMcp.test.ts new file mode 100644 index 000000000..3b5e8bb5e --- /dev/null +++ b/tests/boundary/mcp/keyedArchiveXmlExtensionMcp.test.ts @@ -0,0 +1,86 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { build } from "plist"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { createBinarySession } from "../../../src/composition/binary.js"; +import { parseConfig } from "../../../src/config/parseConfig.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; + +mcpTest( + "MCP inspects an XML keyed archive with a non-plist suffix", + async ({ mcp, onTestFinished }) => { + const root = await createTestTempDirectory("rea-keyed-xml-extension-mcp-"); + const path = join(root, "Model.archive"); + await writeFile( + path, + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "leaf"], + $top: { root: { UID: 1 } }, + }).replace("?>", `?>${" ".repeat(8192)}`), + ); + const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); + if (!configured.ok) throw configured.error; + const session = createBinarySession( + configured.value, + silentLogger, + process.env, + ); + onTestFinished(async () => { + await session.close(); + }); + const client = await mcp.connect( + createServer({ kind: "session", session }), + ); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + const inventoryReply = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect( + inventoryReply.isError, + JSON.stringify(inventoryReply.content), + ).not.toBe(true); + const inspectionEvidence = parseEvidence(inventoryReply.structuredContent); + const inspection = artifactInspectionResultSchema.parse( + inspectionEvidence.normalized_result, + ); + const inventory = artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ); + expect(inventory.manifest.root_format).toBe("plist"); + expect(inventory.occurrences[0]?.artifact_format).toBe("plist"); + + const result = await client.callTool({ + name: "inspect_keyed_archive", + arguments: {}, + }); + expect(result.isError, JSON.stringify(result.content)).not.toBe(true); + expect( + z + .object({ normalized_result: keyedArchiveResultSchema }) + .parse(result.structuredContent).normalized_result, + ).toMatchObject({ + archive_path: "Model.archive", + archive_format: "xml-plist", + objects: expect.arrayContaining([ + expect.objectContaining({ value: "leaf" }), + ]), + }); + }, +); diff --git a/tests/boundary/mcp/nativeDemangleMcp.test.ts b/tests/boundary/mcp/nativeDemangleMcp.test.ts index 71f7732a5..8e66a27fa 100644 --- a/tests/boundary/mcp/nativeDemangleMcp.test.ts +++ b/tests/boundary/mcp/nativeDemangleMcp.test.ts @@ -1,15 +1,12 @@ -import { silentLogger } from "../../../src/logger.js"; import { copyFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect } from "vitest"; import { z } from "zod"; -import { createBinarySession } from "../../../src/composition/binary.js"; -import { parseConfig } from "../../../src/config/parseConfig.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { openProductionTarget } from "../../support/mcp/productionSession.js"; const demangledSchema = z.object({ normalized_result: z.object({ @@ -30,24 +27,7 @@ describe.skipIf(process.platform !== "darwin")( "tool", ); await copyFile("/usr/bin/true", target); - const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); - if (!configured.ok) throw configured.error; - const session = createBinarySession( - configured.value, - silentLogger, - process.env, - ); - onTestFinished(async () => { - await session.close(); - }); - const client = await mcp.connect( - createServer({ kind: "session", session }), - ); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: target }, - }); - expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + const client = await openProductionTarget(mcp, onTestFinished, target); const called = await client.callTool({ name: "demangle_swift", diff --git a/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts index e492263fa..1a2f29902 100644 --- a/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts +++ b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts @@ -1,12 +1,9 @@ -import { silentLogger } from "../../../src/logger.js"; import { copyFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect } from "vitest"; import { z } from "zod"; -import { createBinarySession } from "../../../src/composition/binary.js"; -import { parseConfig } from "../../../src/config/parseConfig.js"; import { inspectMachoSchema } from "../../../src/domain/native/nativeInspection.js"; import { createServer } from "../../../src/server/createServer.js"; import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; @@ -16,6 +13,7 @@ import { ok } from "../../../src/domain/result.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { openProductionTarget } from "../../support/mcp/productionSession.js"; const evidenceOf = (schema: Schema) => z.object({ @@ -54,24 +52,7 @@ describe.skipIf(process.platform !== "darwin")( "ls", ); await copyFile("/bin/ls", target); - const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); - if (!configured.ok) throw configured.error; - const session = createBinarySession( - configured.value, - silentLogger, - process.env, - ); - onTestFinished(async () => { - await session.close(); - }); - const client = await mcp.connect( - createServer({ kind: "session", session }), - ); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: target }, - }); - expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + const client = await openProductionTarget(mcp, onTestFinished, target); const complete = await client.callTool({ name: "inspect_macho", diff --git a/tests/boundary/providers/native/artifactProvider.directoryProbes.test.ts b/tests/boundary/providers/native/artifactProvider.directoryProbes.test.ts new file mode 100644 index 000000000..fc4fc4519 --- /dev/null +++ b/tests/boundary/providers/native/artifactProvider.directoryProbes.test.ts @@ -0,0 +1,138 @@ +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import { createPackageWithOptions } from "@electron/asar"; +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { createDirectAnalysis } from "../../../../src/composition/directAnalysis.js"; +import { artifactInspectionResultSchema } from "../../../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../../../src/domain/evidence.js"; + +const { runProviderAnalysis } = createDirectAnalysis({}); + +// mach_header_64 for an arm64 MH_EXECUTE image with no load commands. +const MACH_O_64 = Buffer.alloc(32); +MACH_O_64.writeUInt32LE(0xfeedfacf, 0); +MACH_O_64.writeUInt32LE(0x0100000c, 4); +MACH_O_64.writeUInt32LE(2, 12); +const INFO_PLIST = ` +CFBundleExecutableFixture +`; + +const writeTree = async ( + root: string, + files: Readonly>, +): Promise => { + for (const [path, contents] of Object.entries(files)) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), contents); + } +}; + +const probeOperations = async (path: string): Promise => { + const evidence = parseEvidence( + await runProviderAnalysis(path, "inspect_artifact", {}), + ); + const inspection = artifactInspectionResultSchema.parse( + evidence.normalized_result, + ); + expect(inspection.subject.root_format).toBe("directory"); + expect( + inspection.unexplored_branches.map(({ next_probe }) => next_probe), + ).toEqual(inspection.next_probes); + return inspection.next_probes.map(({ operation }) => operation); +}; + +describe("directory inspection probes", () => { + it("proposes native and plist probes for an application bundle without JavaScript", async () => { + const root = await createTestTempDirectory("rea-artifact-bundle-"); + const bundle = join(root, "Fixture.app"); + await writeTree(bundle, { + "Contents/MacOS/Fixture": MACH_O_64, + "Contents/Info.plist": INFO_PLIST, + }); + + expect(await probeOperations(bundle)).toEqual([ + "open_binary", + "inspect_plist", + ]); + }); + + it("keeps JavaScript analysis first for an Electron-style bundle", async () => { + const root = await createTestTempDirectory("rea-artifact-electron-"); + const source = join(root, "source"); + const bundle = join(root, "Fixture.app"); + await writeTree(source, { "main.js": "console.log('ok');\n" }); + await writeTree(bundle, { + "Contents/MacOS/Fixture": MACH_O_64, + "Contents/Info.plist": INFO_PLIST, + }); + await createPackageWithOptions( + source, + join(bundle, "Contents/Resources/app.asar"), + {}, + ); + + expect(await probeOperations(bundle)).toEqual([ + "analyze_javascript_application", + "open_binary", + "inspect_plist", + ]); + }); + + it.each(["cjs", "cts", "mjs", "mts", "js", "jsx", "ts", "tsx"])( + "proposes JavaScript analysis for .%s sources the analyzer reads", + async (extension) => { + const root = await createTestTempDirectory("rea-artifact-typescript-"); + const bundle = join(root, "Fixture.app"); + await writeTree(bundle, { + "Contents/MacOS/Fixture": MACH_O_64, + "Contents/Info.plist": INFO_PLIST, + [`Contents/Resources/app/main.${extension}`]: + "export const ok = true;\n", + }); + + expect(await probeOperations(bundle)).toEqual([ + "analyze_javascript_application", + "open_binary", + "inspect_plist", + ]); + }, + ); + + it("preserves plist inspection for a readable Info.plist symlink", async () => { + const root = await createTestTempDirectory("rea-artifact-plist-link-"); + const bundle = join(root, "Fixture.app"); + await writeTree(bundle, { "Contents/MacOS/Fixture": MACH_O_64 }); + await writeFile(join(root, "bundle-metadata"), INFO_PLIST); + await symlink( + join(root, "bundle-metadata"), + join(bundle, "Contents/Info.plist"), + ); + expect(await probeOperations(bundle)).toEqual([ + "open_binary", + "inspect_plist", + ]); + }); + + it("does not propose JavaScript analysis for a symlink the analyzers never follow", async () => { + const root = await createTestTempDirectory("rea-artifact-symlink-"); + const bundle = join(root, "Fixture.app"); + await writeTree(bundle, { + "Contents/MacOS/Fixture": MACH_O_64, + "Contents/Info.plist": INFO_PLIST, + }); + await writeTree(root, { "outside.js": "console.log('outside');\n" }); + await mkdir(join(bundle, "Contents/Resources")); + await symlink( + join(root, "outside.js"), + join(bundle, "Contents/Resources/main.js"), + ); + + expect(await probeOperations(bundle)).toEqual([ + "open_binary", + "inspect_plist", + ]); + }); +}); diff --git a/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts b/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts index e500d2c0c..5b042e6e2 100644 --- a/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts +++ b/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts @@ -41,10 +41,32 @@ it("keeps logical names that differ only in case and records the limitation", as } }); +it("keeps literal backslash ZIP names distinct from slash paths", async () => { + const root = await createTestTempDirectory("rea-artifact-detail-"); + const path = join(root, "fixture.zip"); + await writeOrderedZip(path, ["a\\b.js", "a/b.js"]); + const target = await parseBinaryTarget(path); + if (!target.ok) throw new Error("expected a ZIP target"); + const result = await new ArtifactProvider(process.env) + .createClient(target.value) + .execute("inventory_artifact", {}); + if (!result.ok) throw result.error; + const inventory = artifactInventoryResultSchema.parse(result.value.result); + const literal = inventory.occurrences.find( + ({ logical_path }) => logical_path === "a\\b.js", + ); + const slash = inventory.occurrences.find( + ({ logical_path }) => logical_path === "a/b.js", + ); + expect(literal).toBeDefined(); + expect(slash).toBeDefined(); + expect(literal?.occurrence_id).not.toBe(slash?.occurrence_id); +}); + it("reports an unsafe path as a hostile artifact path", async () => { const root = await createTestTempDirectory("rea-artifact-detail-"); const path = join(root, "fixture.zip"); - await writeOrderedZip(path, ["a\\b.js"]); + await writeOrderedZip(path, ["../outside.js"]); const target = await parseBinaryTarget(path); if (!target.ok) throw new Error("expected a ZIP target"); const result = await new ArtifactProvider(process.env) @@ -53,11 +75,11 @@ it("reports an unsafe path as a hostile artifact path", async () => { if (result.ok) throw new Error("expected the path constraint to fail"); expect(projectAnalysisError(result.error)).toMatchObject({ code: "artifact_operation_failed", - message: 'Artifact path is absolute or unsafe: "a\\\\b.js"', + message: 'Artifact path is not normalized: "../outside.js"', details: { operation: "inventory_artifact", reason: "path", - detail: 'Artifact path is absolute or unsafe: "a\\\\b.js"', + detail: 'Artifact path is not normalized: "../outside.js"', }, }); }); From ec271121e4996f89f666a5a2ed1cd2f3229411b4 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:31:03 +0800 Subject: [PATCH 07/26] fix(artifacts): preserve path identity and exclusion applicability Normalize path facts once at the owning filesystem boundary, preserve caller spelling and unavailable inventory members, and retain typed read failures. Record exclusion entry kinds so only proven directories can establish descendant symlink targets; omitted legacy kinds stay unknown. Reviewed fixes include #1969, #2642, #2759, and #2994. Validation: 26 reference graph source tests passed; prior artifact checks passed 275 tests with one Windows-only skip. Updated compiled import workflows are included in final validation after regeneration. --- src/application/BinaryTargetResolver.ts | 45 +++- src/application/ReferenceSourceImport.ts | 8 +- src/application/ReferenceSourceVcsAdapter.ts | 17 +- src/artifacts/ArtifactHash.ts | 23 +- src/artifacts/ArtifactPaths.ts | 11 +- src/artifacts/DirectoryArtifactReader.ts | 34 ++- src/artifacts/SafeOutputTree.ts | 5 + .../InterfaceBuilderDecodeBudget.test.ts | 106 +++++++++ .../apple/InterfaceBuilderDecodeBudget.ts | 73 ++++-- .../ArtifactGraphConstruction.test.ts | 22 ++ .../inventory/ArtifactGraphConstruction.ts | 17 +- src/artifacts/inventory/classify.ts | 35 ++- .../javascript/JavaScriptArtifactFiles.ts | 121 +++++++++- src/domain/android/androidAnalysis.ts | 4 +- src/domain/android/androidApplication.ts | 13 +- src/domain/apktool/apktoolResourceAnalysis.ts | 35 ++- src/domain/apple/appleArchiveFormat.ts | 206 +++++++++++++++- .../apple/interfaceBuilderGraph.test.ts | 44 +++- src/domain/apple/interfaceBuilderGraph.ts | 2 +- src/domain/artifactGraph.ts | 1 - src/domain/artifactIdentity.ts | 4 +- src/domain/artifactInspection.ts | 48 +++- src/domain/referenceSourceClassification.ts | 56 +++-- src/domain/referenceSourceGraph.test.ts | 101 ++++++++ src/domain/referenceSourceGraph.ts | 70 ++++-- src/domain/referenceSourcePathIdentity.ts | 19 +- src/javascript/recovery/RecoveryFiles.ts | 5 + src/reference/ReferenceSourceReaderErrors.ts | 3 + tests/boundary/cli/jsonInputSize.test.ts | 26 +- .../cli/referenceSourceExclusions.test.ts | 61 ++++- .../boundary/cli/referenceSourceUtf8.test.ts | 148 +++++++++++- .../filesystem/appleXmlDeclarations.test.ts | 28 +++ .../artifactLiteralBackslash.test.ts | 222 ++++++++++++++++++ .../filesystem/asarHeaderAdmission.test.ts | 115 +++++++++ .../boundary/filesystem/binaryTarget.test.ts | 41 ++++ .../directoryArtifactReaderOwnership.test.ts | 26 +- .../filesystem/keyedArchiveXmlNesting.test.ts | 74 ++++++ .../filesystem/recoveryInputPaths.test.ts | 72 ++++++ .../referenceSourceFailureDetails.test.ts | 60 +++++ .../filesystem/referenceSourceImport.test.ts | 117 +++++++++ .../filesystem/referenceSourceVcs.test.ts | 26 +- 41 files changed, 2022 insertions(+), 122 deletions(-) create mode 100644 src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts create mode 100644 tests/boundary/filesystem/artifactLiteralBackslash.test.ts create mode 100644 tests/boundary/filesystem/asarHeaderAdmission.test.ts create mode 100644 tests/boundary/filesystem/recoveryInputPaths.test.ts diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 348454e72..7189ed7de 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -32,7 +32,11 @@ import { } from "../domain/rawBinary.js"; import type { BinaryArchitecture } from "../domain/binaryTargetTypes.js"; import { observeMzWindowsHeader } from "../domain/dosMz.js"; -import { hasBinaryPlistSignature } from "../domain/apple/appleArchiveFormat.js"; +import { + createXmlPlistSignatureProbe, + hasBinaryPlistSignature, + MAX_XML_PLIST_PROBE_BYTES, +} from "../domain/apple/appleArchiveFormat.js"; import { hasZipSignature, isZipPackageFormat, @@ -49,6 +53,8 @@ import { } from "../domain/binaryTarget.js"; import type { PeCodeViewRecord } from "../domain/binaryTargetTypes.js"; +const XML_PLIST_PROBE_CHUNK_BYTES = 64 * 1024; + /** Options for resolving a local binary target and its stable identity. */ export interface BinaryTargetResolverOptions { readonly cwd?: string; @@ -288,12 +294,12 @@ const detectArtifactFormat = async ( > => { throwIfTargetResolutionCancelled(signal); const lower = path.toLowerCase(); - const magic = Buffer.alloc(8); + const magic = Buffer.alloc(4096); const observed = await handle.read(magic, 0, magic.length, 0); throwIfTargetResolutionCancelled(signal); if ( - observed.bytesRead === 8 && - magic.equals(Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0])) + observed.bytesRead >= 8 && + magic.subarray(0, 8).equals(Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0])) ) throw new BinaryTargetError( path, @@ -329,6 +335,37 @@ const detectArtifactFormat = async ( return "dmg"; } } + // XML property lists can use archive-specific suffixes such as .archive. + // The named-format check above keeps .js, .map, and .asar classifications. + const xmlProbe = createXmlPlistSignatureProbe(); + let xmlState = xmlProbe.write(magic.subarray(0, observed.bytesRead)); + if (xmlState === "matched") return "plist"; + const chunk = Buffer.alloc(XML_PLIST_PROBE_CHUNK_BYTES); + let bytesRead = observed.bytesRead; + while (xmlState === "pending" && bytesRead < MAX_XML_PLIST_PROBE_BYTES) { + throwIfTargetResolutionCancelled(signal); + const read = await handle.read( + chunk, + 0, + Math.min(chunk.length, MAX_XML_PLIST_PROBE_BYTES - bytesRead), + bytesRead, + ); + throwIfTargetResolutionCancelled(signal); + if (read.bytesRead === 0) break; + bytesRead += read.bytesRead; + xmlState = xmlProbe.write(chunk.subarray(0, read.bytesRead)); + if (xmlState === "matched") return "plist"; + } + if (bytesRead === MAX_XML_PLIST_PROBE_BYTES) { + const { size } = await handle.stat(); + throwIfTargetResolutionCancelled(signal); + if (size > bytesRead) + throw new BinaryTargetError( + path, + "XML plist prolog was not found within the 1 MiB format probe limit", + { constraint: "file_format_rejected" }, + ); + } return undefined; }; diff --git a/src/application/ReferenceSourceImport.ts b/src/application/ReferenceSourceImport.ts index 769f8bb8c..e07a38146 100644 --- a/src/application/ReferenceSourceImport.ts +++ b/src/application/ReferenceSourceImport.ts @@ -151,6 +151,7 @@ const createShouldExclude = throw new Error(`Ignored secret path has no matching rule: ${path}`); exclusions.push({ path, + entry_kind: kind, reason: "configured-secret", pattern: secretMatch.rule.pattern, }); @@ -167,7 +168,12 @@ const createShouldExclude = reason !== "caller-excluded" ) throw new Error(`Ignored path has unknown rule origin: ${path}`); - exclusions.push({ path, reason, pattern: match.rule.pattern }); + exclusions.push({ + path, + entry_kind: kind, + reason, + pattern: match.rule.pattern, + }); return true; }; diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.ts index 346ae6ebe..ac05ce0a4 100644 --- a/src/application/ReferenceSourceVcsAdapter.ts +++ b/src/application/ReferenceSourceVcsAdapter.ts @@ -162,14 +162,21 @@ const resolveSourceHead = async ( // The library probes readFile() without a path to detect promise APIs. // Reject that invocation without attributing it to a source read. if (path === undefined) - throw new TypeError("Git filesystem read requires a path"); + throw new TypeError("Git metadata path is required"); try { const bytes = await readMetadataBytes(path); const encoding = - typeof options === "string" ? options : options?.encoding; - return encoding === undefined || encoding === null - ? bytes - : bytes.toString(encoding); + typeof options === "string" + ? options + : typeof options === "object" && + options !== null && + "encoding" in options + ? options.encoding + : undefined; + if (encoding === undefined || encoding === null) return bytes; + if (typeof encoding !== "string" || !Buffer.isEncoding(encoding)) + throw new TypeError("Unsupported Git metadata text encoding"); + return bytes.toString(encoding); } catch (cause: unknown) { if (cause instanceof ArtifactReaderFailure) { cleanupFailures.push(cause); diff --git a/src/artifacts/ArtifactHash.ts b/src/artifacts/ArtifactHash.ts index b3bf43bfa..c1234f847 100644 --- a/src/artifacts/ArtifactHash.ts +++ b/src/artifacts/ArtifactHash.ts @@ -3,6 +3,10 @@ import type { Readable } from "node:stream"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { streamChunkToBuffer } from "./StreamBytes.js"; +import { + createXmlPlistSignatureProbe, + MAX_XML_PLIST_PROBE_BYTES, +} from "../domain/apple/appleArchiveFormat.js"; /** Full stream digest and byte count with bounded classification evidence. */ export type HashResult = { @@ -31,19 +35,32 @@ export const hashReadable = async ( const hash = createHash("sha256"); const prefixes: Buffer[] = []; let prefixBytes = 0; + const xmlProbe = createXmlPlistSignatureProbe(); let bytes = 0; for await (const raw of stream) { abortIfNeeded(signal); const chunk = streamChunkToBuffer(raw); bytes += chunk.length; hash.update(chunk); - if (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES) { + let chunkOffset = 0; + while ( + chunkOffset < chunk.length && + (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES || + (xmlProbe.state === "pending" && + prefixBytes < MAX_XML_PLIST_PROBE_BYTES)) + ) { + const limit = + prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES + ? ARTIFACT_CLASSIFICATION_PREFIX_BYTES + : MAX_XML_PLIST_PROBE_BYTES; const selected = chunk.subarray( - 0, - ARTIFACT_CLASSIFICATION_PREFIX_BYTES - prefixBytes, + chunkOffset, + Math.min(chunkOffset + limit - prefixBytes, chunk.length), ); prefixes.push(selected); prefixBytes += selected.length; + chunkOffset += selected.length; + xmlProbe.write(selected); } } return { sha256: hash.digest("hex"), bytes, prefix: Buffer.concat(prefixes) }; diff --git a/src/artifacts/ArtifactPaths.ts b/src/artifacts/ArtifactPaths.ts index ea81ea41f..b83165907 100644 --- a/src/artifacts/ArtifactPaths.ts +++ b/src/artifacts/ArtifactPaths.ts @@ -4,7 +4,7 @@ import { logicalPathNfc } from "../domain/artifactIdentity.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; -const DRIVE_OR_UNC = /^(?:[A-Za-z]:|\\|\/\/)/u; +const DRIVE_PREFIX = /^[A-Za-z]:/u; /** * English Unicode case folding used only to describe logical names a @@ -20,14 +20,9 @@ export const ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION = `Logical archive names export const DESTINATION_CASE_COLLISION_PREFIX = "Destination filesystem cannot store both "; -/** Normalize an untrusted logical archive path without touching filesystem. */ +/** Normalize slash-separated logical names; backslashes are literal components. */ export const normalizeArtifactPath = (input: string): string => { - if ( - input.includes("\0") || - input.includes("\\") || - input.startsWith("/") || - DRIVE_OR_UNC.test(input) - ) + if (input.includes("\0") || input.startsWith("/") || DRIVE_PREFIX.test(input)) throw new ArtifactReaderFailure( "path", `Artifact path is absolute or unsafe: ${JSON.stringify(input)}`, diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.ts index 399bf979b..56ef4ccc7 100644 --- a/src/artifacts/DirectoryArtifactReader.ts +++ b/src/artifacts/DirectoryArtifactReader.ts @@ -30,6 +30,7 @@ interface DirectoryTraversalOptions extends ArtifactTraversalOptions { export class DirectoryArtifactReader implements ArtifactReader { readonly format = "directory" as const; #rootPromise: Promise | undefined; + #rootIdentity: { readonly dev: bigint; readonly ino: bigint } | undefined; readonly #rootPath: string; readonly #directories = new Set(); readonly #files = new Map(); @@ -52,7 +53,7 @@ export class DirectoryArtifactReader implements ArtifactReader { const directory = pending.pop(); if (directory === undefined) break; await withDirectoryIoContext("inspect directory", directory, () => - assertContainedDirectory(root, directory), + assertContainedDirectory(root, directory, this.#rootIdentity), ); for await (const entry of this.#readDirectory( directory, @@ -62,21 +63,26 @@ export class DirectoryArtifactReader implements ArtifactReader { )) yield entry; await withDirectoryIoContext("inspect directory", directory, () => - assertContainedDirectory(root, directory), + assertContainedDirectory(root, directory, this.#rootIdentity), ); } } async #resolveRoot(): Promise { - return (this.#rootPromise ??= realpath(this.#rootPath)).catch( - (cause: unknown) => { - throw directoryIoFailure( - "resolve artifact root", - this.#rootPath, - cause, + return (this.#rootPromise ??= (async () => { + const root = await realpath(this.#rootPath); + const metadata = await lstat(root, { bigint: true }); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) + throw new ArtifactReaderFailure( + "path", + "Artifact root changed during traversal", ); - }, - ); + this.#rootIdentity = { dev: metadata.dev, ino: metadata.ino }; + return root; + })().catch((cause: unknown) => { + if (cause instanceof ArtifactReaderFailure) throw cause; + throw directoryIoFailure("resolve artifact root", this.#rootPath, cause); + })); } async *#readDirectory( @@ -100,6 +106,9 @@ export class DirectoryArtifactReader implements ArtifactReader { }; try { for (;;) { + await withDirectoryIoContext("inspect directory", directory, () => + assertContainedDirectory(root, directory, this.#rootIdentity), + ); const child = await handle.handle.read(); if (child === null) break; abortIfNeeded(signal); @@ -440,11 +449,16 @@ const abortIfNeeded = (signal?: AbortSignal): void => { const assertContainedDirectory = async ( root: string, directory: string, + rootIdentity: { readonly dev: bigint; readonly ino: bigint } | undefined, ): Promise => { const canonical = await realpath(directory); const fromRoot = relative(root, canonical); const metadata = await lstat(directory); + const currentRoot = await lstat(root, { bigint: true }); if ( + rootIdentity === undefined || + currentRoot.dev !== rootIdentity.dev || + currentRoot.ino !== rootIdentity.ino || metadata.isSymbolicLink() || !metadata.isDirectory() || fromRoot === ".." || diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.ts index 73885ad05..0cf4dab98 100644 --- a/src/artifacts/SafeOutputTree.ts +++ b/src/artifacts/SafeOutputTree.ts @@ -580,6 +580,11 @@ const normalizeDestinationPath = ( platform: NodeJS.Platform, ): string => { const path = normalizeArtifactPath(relativePath); + if (platform === "win32" && path.includes("\\")) + throw new ArtifactReaderFailure( + "path", + `Windows destination cannot materialize artifact path: ${JSON.stringify(path)}; backslashes denote path separators.`, + ); if (platform === "win32" && path.includes(":")) throw new ArtifactReaderFailure( "path", diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts new file mode 100644 index 000000000..b02aee745 --- /dev/null +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts @@ -0,0 +1,106 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { analyzeInterfaceBuilderBundle } from "./InterfaceBuilderAnalysis.js"; +import { preflightPropertyListDecode } from "./InterfaceBuilderDecodeBudget.js"; +import { parseXmlPropertyList } from "../../domain/propertyListKeys.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; + +const example = ""; +const archive = (content: string, comment = "") => + `${comment}$archiverNSKeyedArchiver$objects$null$classCF$UID2title${content}$classnameNSButton$classesNSButtonNSObject$toprootCF$UID1`; + +const inertDeclarations = [ + { + kind: "CDATA text", + xml: archive(``), + title: example, + }, + { + kind: "XML comment", + xml: archive("Build", ``), + title: "Build", + }, +] as const; + +describe("inert XML declaration examples in Interface Builder archives", () => { + it.each(inertDeclarations)( + "preflights $kind without treating it as entity expansion", + ({ xml }) => { + expect(() => parseXmlPropertyList(xml)).not.toThrow(); + expect( + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).toMatchObject({ + binaryPrototypeKeyObserved: false, + }); + }, + ); + + it.each(inertDeclarations)( + "retains $kind and archive provenance through bundle decoding", + async ({ xml, title }) => { + const root = await createTestTempDirectory("rea-ib-inert-declaration-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle); + const bytes = Buffer.from(xml); + await writeFile(join(bundle, "Panel.nib"), bytes); + + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "a".repeat(64), + }); + + expect(result.documents).toContainEqual( + expect.objectContaining({ + relative_path: "Panel.nib", + archive_sha256: createHash("sha256").update(bytes).digest("hex"), + }), + ); + expect(result.graph.nodes.map(({ name }) => name)).toContain(title); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "complete", + }), + ); + }, + ); + + it("continues to refuse an actual internal entity declaration before decoding", () => { + const xml = archive("&harmless;").replace( + "]> + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).toThrow( + "XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget", + ); + }); + + it("does not mistake a bracket in a quoted external identifier for an internal subset", () => { + const xml = archive("Build").replace( + " parseXmlPropertyList(xml)).not.toThrow(); + expect(() => + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).not.toThrow(); + }); + + it.each([ + '[edition].dtd">', + "[edition].dtd'>", + " ?>", + ])("consumes quoted or opaque declaration text once: %s", (declaration) => { + const xml = archive("Build").replace(" parseXmlPropertyList(xml)).not.toThrow(); + expect( + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).toMatchObject({ binaryPrototypeKeyObserved: false }); + }); +}); diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts index 3593fa43e..5d143f52a 100644 --- a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts @@ -115,28 +115,23 @@ const estimateXmlPlistExpansion = ( xmlText?: string, ): number => { const xml = xmlText ?? decodeXmlPlistText(bytes); - if (/]*\[/iu.test(xml)) - throw new InterfaceBuilderDecodeBudgetExceeded( - "aggregate_decode_budget_exhausted", - "XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget", - ); - const elementPattern = - /||]*>|<\?[^]*?\?>|<\/?[A-Za-z][^>]*>/giu; const decodedSourceBytes = xml.length * 2; let elementCount = 0; let xmlDepth = 0; - for (const match of xml.matchAll(elementPattern)) { - const token = match[0] ?? ""; - if ( - token.startsWith("" + : xml.startsWith("" + : xml.startsWith("" + : undefined; + if (opaqueEnd !== undefined) { + const end = xml.indexOf(opaqueEnd, start + 2); + if (end < 0) return; + cursor = end + opaqueEnd.length; + continue; + } + const doctype = /^") break; + else if (doctype && character === "[") internalSubset = true; + } + if (end === xml.length) return; + cursor = end + 1; + if (doctype || element) + yield { + doctype, + internalSubset, + closing: xml[start + 1] === "/", + selfClosing: xml[end - 1] === "/", + }; + } +} + /** Resource a binary plist preflight found over its budget, or a malformed reference cycle. */ export type BinaryPlistExpansionFailure = "cycle" | "depth" | "expansion"; diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index 925d3543a..6531d4557 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -101,3 +101,25 @@ describe("bounded FAT inventory classification", () => { }, ); }); + +it("preserves path-selected inventory roles for XML content", () => { + const xml = Buffer.from( + '', + ); + expect(classifyArtifactContent("state.archive", xml)).toEqual({ + kind: "plist", + format: "plist", + }); + expect(classifyArtifactContent("state.js", xml)).toEqual({ + kind: "javascript", + format: "javascript-bundle", + }); + expect(classifyArtifactContent("state.map", xml)).toEqual({ + kind: "source-map", + format: "source-map", + }); + expect(classifyArtifactContent("state.asar", xml)).toEqual({ + kind: "resource", + format: "file", + }); +}); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index b7b194744..f2564d254 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -1,6 +1,10 @@ import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; +import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js"; import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; -import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; +import { + hasBinaryPlistSignature, + hasXmlPlistSignature, +} from "../../domain/apple/appleArchiveFormat.js"; import { artifactEdgeId, artifactIdForContent, @@ -365,7 +369,7 @@ export const classifyArtifactPath = ( const lower = path.toLowerCase(); if (lower.endsWith(".map")) return { kind: "source-map", format: "source-map" }; - if (/\.(?:m?js|cjs)$/u.test(lower)) + if (isJavaScriptSourcePath(path)) return { kind: "javascript", format: "javascript-bundle" }; if (/\.framework(?:\/|$)/u.test(lower)) return { kind: "framework", format: "file" }; @@ -390,6 +394,14 @@ export const classifyArtifactContent = ( readonly format: ArtifactOccurrence["artifact_format"]; } => { const format = classifyArtifactBytes(prefix, fileSize); + const byPath = classifyArtifactPath(path); + if ( + format === "plist" && + !hasBinaryPlistSignature(prefix) && + (["javascript", "source-map", "entitlements"].includes(byPath.kind) || + path.toLowerCase().endsWith(".asar")) + ) + return byPath; if ( format === "file" && path.toLowerCase().endsWith(".asar") && @@ -442,6 +454,7 @@ export const classifyArtifactBytes = ( ): ArtifactNode["format"] => { if (hasZipSignature(prefix)) return "zip"; if (hasBinaryPlistSignature(prefix)) return "plist"; + if (hasXmlPlistSignature(prefix)) return "plist"; const fatFormat = classifyFatArtifact(prefix, fileSize); if (fatFormat !== null) return fatFormat; if (prefix.length >= 4) { diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 6322d6028..73cc6d760 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -5,7 +5,14 @@ import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.j import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; -import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js"; +import { + abortIfNeeded, + ARTIFACT_CLASSIFICATION_PREFIX_BYTES, +} from "../ArtifactHash.js"; +import { + createXmlPlistSignatureProbe, + MAX_XML_PLIST_PROBE_BYTES, +} from "../../domain/apple/appleArchiveFormat.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import { hasZipSignature, @@ -104,7 +111,7 @@ export const classifyAndHashRootForInventory = async ( "integrity", `Root artifact changed before inventory: ${path}`, ); - const format = await classifyRootFormat(path, handle); + const format = await classifyRootFormat(path, handle, options.signal); const digest = await hashStableRootArtifactHandle( path, handle, @@ -144,10 +151,11 @@ export const classifyAndHashRootForInventory = async ( const classifyRootFormat = async ( path: string, handle: FileHandle, + signal?: AbortSignal, ): Promise => { const magic = Buffer.alloc(ARTIFACT_CLASSIFICATION_PREFIX_BYTES); const observed = await handle.read(magic, 0, magic.length, 0); - const prefix = magic.subarray(0, observed.bytesRead); + let prefix = magic.subarray(0, observed.bytesRead); const lower = path.toLowerCase(); if (hasZipSignature(prefix)) return zipRootPackageFormat(lower); if ( @@ -156,6 +164,27 @@ const classifyRootFormat = async ( ) return "pkg"; if (lower.endsWith(".dmg") && (await hasKolyTrailer(handle))) return "dmg"; + const xmlProbe = createXmlPlistSignatureProbe(); + xmlProbe.write(prefix); + const chunks = [prefix]; + let prefixBytes = prefix.length; + while ( + xmlProbe.state === "pending" && + prefixBytes < MAX_XML_PLIST_PROBE_BYTES + ) { + abortIfNeeded(signal); + const chunk = Buffer.alloc( + Math.min(64 * 1024, MAX_XML_PLIST_PROBE_BYTES - prefixBytes), + ); + const read = await handle.read(chunk, 0, chunk.length, prefixBytes); + abortIfNeeded(signal); + if (read.bytesRead === 0) break; + const selected = chunk.subarray(0, read.bytesRead); + chunks.push(selected); + prefixBytes += selected.length; + xmlProbe.write(selected); + } + prefix = Buffer.concat(chunks); // ASAR is chosen from its header pickle; any other suffix is a role hint. return classifyArtifactContent(path, prefix, (await handle.stat()).size) .format; diff --git a/src/artifacts/javascript/JavaScriptArtifactFiles.ts b/src/artifacts/javascript/JavaScriptArtifactFiles.ts index 78d8cf750..d4845feac 100644 --- a/src/artifacts/javascript/JavaScriptArtifactFiles.ts +++ b/src/artifacts/javascript/JavaScriptArtifactFiles.ts @@ -19,11 +19,14 @@ import { hashReadable } from "../ArtifactHash.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; +import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js"; import type { JavaScriptArtifactFileKind, JavaScriptArtifactFile, JavaScriptArtifactContainer, + JavaScriptUnanalyzedArtifactFile, + JavaScriptUnanalyzedArtifactLookup, JavaScriptArtifactFileSet, } from "../../domain/javascript/javascriptArtifactFiles.js"; @@ -140,6 +143,10 @@ export const readJavaScriptArtifactFiles = async ( assertExpectedFilesWereVisited(expected, files); return { files, + unanalyzed_files: createUnanalyzedArtifactLookup( + snapshot, + context.containers, + ), containers: context.containers.sort((left, right) => compareUnicodeCodePoints(left.path, right.path), ), @@ -438,6 +445,118 @@ const expectedInventory = ( return { files, containers, unavailableFiles }; }; +/** Canonical snapshot arrays are already ordered by code-point identity. */ +const lowerBound = ( + rows: readonly T[], + key: string, + identity: (row: T) => string, +): number => { + let low = 0; + let high = rows.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + const row = rows[middle]; + if (row !== undefined && compareUnicodeCodePoints(identity(row), key) < 0) + low = middle + 1; + else high = middle; + } + return low; +}; + +const createUnanalyzedArtifactLookup = ( + snapshot: ArtifactInventorySnapshot, + containers: readonly JavaScriptArtifactContainer[], +): JavaScriptUnanalyzedArtifactLookup => { + const requested = new Map< + string, + JavaScriptUnanalyzedArtifactFile | undefined + >(); + const directories = new Map>(); + const containerIds = new Map( + containers.map((container) => [container.path, container.sha256]), + ); + const containerFor = (path: string): string => { + let prefix = path; + for (;;) { + const slash = prefix.lastIndexOf("/"); + if (slash < 0) return snapshot.manifest.root_sha256; + prefix = prefix.slice(0, slash); + const sha256 = containerIds.get(prefix); + if (sha256 !== undefined) return sha256; + } + }; + const isOpaque = (occurrence: ArtifactOccurrence): boolean => + occurrence.entry_kind === "file" && + relevantKind(occurrence.logical_path) === undefined; + return { + get(path) { + if (requested.has(path)) return requested.get(path); + const occurrence = + snapshot.occurrences[ + lowerBound(snapshot.occurrences, path, (entry) => entry.logical_path) + ]; + const id = + occurrence?.logical_path === path && isOpaque(occurrence) + ? occurrence.artifact_id + : undefined; + const candidate = + id == null + ? undefined + : snapshot.nodes[ + lowerBound(snapshot.nodes, id, (entry) => entry.artifact_id) + ]; + const node = candidate?.artifact_id === id ? candidate : undefined; + const file = + occurrence?.logical_path !== path || !isOpaque(occurrence) + ? undefined + : { + path, + container_sha256: containerFor(path), + sha256: node?.sha256 ?? null, + bytes: node?.size ?? occurrence.declared_size, + inventory_artifact_id: occurrence.artifact_id, + limitations: occurrence.limitations, + }; + requested.set(path, file); + return file; + }, + hasDirectory(path, containerSha256) { + let results = directories.get(path); + const previous = results?.get(containerSha256); + if (previous !== undefined) return previous; + const prefix = `${path}/`; + let present = false; + for ( + let index = lowerBound( + snapshot.occurrences, + prefix, + (entry) => entry.logical_path, + ); + index < snapshot.occurrences.length; + index += 1 + ) { + const entry = snapshot.occurrences[index]; + if (entry === undefined || !entry.logical_path.startsWith(prefix)) + break; + if ( + isOpaque(entry) && + (containerSha256 === undefined || + containerFor(entry.logical_path) === containerSha256) + ) { + present = true; + break; + } + } + if (results === undefined) { + results = new Map(); + directories.set(path, results); + } + results.set(containerSha256, present); + return present; + }, + }; +}; + const readAll = async ( stream: Readable, signal?: AbortSignal, @@ -471,7 +590,7 @@ const relevantKind = (path: string): JavaScriptArtifactFileKind | undefined => { if (lower === "package.json" || lower.endsWith("/package.json")) return "package-json"; if (lower.endsWith(".json")) return "json"; - if (/\.(?:cjs|cts|mjs|mts|js|jsx|ts|tsx)$/u.test(lower)) return "javascript"; + if (isJavaScriptSourcePath(path)) return "javascript"; if (/\.html?$/u.test(lower)) return "html"; if (lower.endsWith(".map")) return "source-map"; if (lower.endsWith(".node")) return "native-addon"; diff --git a/src/domain/android/androidAnalysis.ts b/src/domain/android/androidAnalysis.ts index eaf521a50..83439518e 100644 --- a/src/domain/android/androidAnalysis.ts +++ b/src/domain/android/androidAnalysis.ts @@ -187,8 +187,8 @@ export const androidResultSchemas = { top_class: z.string(), source_line: z.null(), dex_offset: z.null(), - /** false means transport-bounded instruction evidence; null/omitted is unknown. */ - direct_invokes_exhaustive: z.boolean().nullable().default(null), + /** false means transport-bounded instruction evidence; null is unknown. */ + direct_invokes_exhaustive: z.boolean().nullable(), /** Raw direct invoke instructions inside this using node, when available. */ direct_invokes: z .array( diff --git a/src/domain/android/androidApplication.ts b/src/domain/android/androidApplication.ts index 99b20c321..6425c5bac 100644 --- a/src/domain/android/androidApplication.ts +++ b/src/domain/android/androidApplication.ts @@ -18,6 +18,8 @@ import { projectionCoverage, projectionDigest, projectionEvidenceIdSchema, + unavailableComponentSchema, + unavailableComponents, } from "../mobileApplicationGraphProjection.js"; const evidenceIdSchema = projectionEvidenceIdSchema; @@ -42,6 +44,7 @@ export const androidApplicationProjectionResultSchema = z.strictObject({ javascript: z.array(componentSchema), signing: z.array(componentSchema), }), + unavailable_components: z.array(unavailableComponentSchema), runtime_families: z.array( z.enum([ "dalvik-art", @@ -92,6 +95,7 @@ export const projectAndroidApplication = ( `Android application projection requires APK inventory Evidence; the inventory root format is ${inventory.manifest.root_format}. APK is chosen for a ZIP archive whose name ends in .apk.`, ); const all = projectedComponents(inventory); + const unavailable = unavailableComponents(inventory); const classified = classify(all); const bridgeProjection = bridgeCandidates( [...classified.dex, ...classified.jvm_classes], @@ -118,6 +122,11 @@ export const projectAndroidApplication = ( `Bridge candidate pairs exceeded the projection safety budget; ${bridgeProjection.coverage.omitted_candidates} hypotheses are omitted. Component arrays still include every component from the supplied inventory pages.`, ] : []), + ...(unavailable.length > 0 + ? [ + "Some inventoried occurrences have no authenticated content identity. Their paths and limitations are retained in unavailable_components; component absence is unknown.", + ] + : []), ]; const withoutId = { root_sha256: inventory.manifest.root_sha256, @@ -126,12 +135,14 @@ export const projectAndroidApplication = ( .map(({ evidence_id: id }) => id) .sort(compareProjectionStrings), components, + unavailable_components: unavailable, runtime_families: runtimeFamilies(all), bridge_candidates: bridgeProjection.candidates, bridge_candidate_coverage: bridgeProjection.coverage, coverage: projectionCoverage( inventory, - bridgeProjection.coverage.status === "complete", + bridgeProjection.coverage.status === "complete" && + unavailable.length === 0, ), limitations, }; diff --git a/src/domain/apktool/apktoolResourceAnalysis.ts b/src/domain/apktool/apktoolResourceAnalysis.ts index df217d040..4cbca0745 100644 --- a/src/domain/apktool/apktoolResourceAnalysis.ts +++ b/src/domain/apktool/apktoolResourceAnalysis.ts @@ -3,6 +3,32 @@ import { localPathStringSchema } from "../localPath.js"; import { digestSchema } from "../digests.js"; const MAX_STRINGS = 2_000; +const ANDROID_NON_LOCALE_QUALIFIERS = new Set([ + "car", + "desk", + "television", + "appliance", + "watch", + "vrheadset", + "night", + "notnight", +]); + +const localeQualifierSchema = z + .string() + .min(2) + .max(16) + .regex( + /^(?:[a-z]{2,3}(?:-(?:r?[A-Z]{2}|r?[0-9]{3}))?|b\+[a-z]{2,3}(?:\+[A-Za-z0-9]{2,8})*)$/u, + "Locale must look like de, pt-BR, pt-rBR, or b+zh+Hans+CN", + ) + .refine( + (value) => !ANDROID_NON_LOCALE_QUALIFIERS.has(value.toLowerCase()), + "Android resource configuration qualifiers are not locales", + ); + +/** Locale language tags admitted by Android resource table selection. */ +export const androidLocaleQualifierSchema = localeQualifierSchema; /** * Caller intent for Apktool-backed resource decoding. Apktool is @@ -19,14 +45,7 @@ export const apktoolInputSchemas = { .describe( `Project res/values/strings.xml entries (bounded to ${String(MAX_STRINGS)} entries)`, ), - locale: z - .string() - .min(2) - .max(16) - .regex( - /^(?:[a-z]{2,3}(?:-(?:r?[A-Z]{2}|r?[0-9]{3}))?|b\+[a-z]{2,3}(?:\+[A-Za-z0-9]{2,8})*)$/u, - "Locale must look like de, pt-BR, pt-rBR, or b+zh+Hans+CN", - ) + locale: androidLocaleQualifierSchema .optional() .describe( "Project strings from res/values-/strings.xml instead of the default table", diff --git a/src/domain/apple/appleArchiveFormat.ts b/src/domain/apple/appleArchiveFormat.ts index e831266f1..204cc19f7 100644 --- a/src/domain/apple/appleArchiveFormat.ts +++ b/src/domain/apple/appleArchiveFormat.ts @@ -3,10 +3,214 @@ const NIB_ARCHIVE_MAGIC = [ 0x4e, 0x49, 0x42, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, ]; -/** Recognize the exact binary plist version supported by the decoder, without validating its body. */ +/** Shared format-probe bound; this does not bound the plist document itself. */ +export const MAX_XML_PLIST_PROBE_BYTES = 1024 * 1024; + +/** Recognize the binary plist version supported by the decoder, without validating its body. */ export const hasBinaryPlistSignature = (bytes: Uint8Array): boolean => BINARY_PLIST_MAGIC.every((byte, index) => bytes[index] === byte); +export type XmlPlistProbeState = "pending" | "matched" | "rejected"; + +/** Incremental, bounded XML signature scanner. Consumed prolog text is discarded. */ +export const createXmlPlistSignatureProbe = (): { + write(bytes: Uint8Array): XmlPlistProbeState; + readonly state: XmlPlistProbeState; +} => { + let state: XmlPlistProbeState = "pending"; + let preview: TextDecoder | undefined; + let validator: TextDecoder | undefined; + let utf16 = false; + let bomBytes = 0; + let fedBytes = 0; + let decodedBytes = 0; + let preamble: Uint8Array = new Uint8Array(); + const scanner = createXmlPlistPrologScanner(); + const encodedLength = (text: string): number => + utf16 ? text.length * 2 : new TextEncoder().encode(text).length; + return { + get state() { + return state; + }, + write(bytes) { + if (state !== "pending" || bytes.length === 0) return state; + let selected = bytes.subarray( + 0, + MAX_XML_PLIST_PROBE_BYTES - fedBytes - preamble.length, + ); + if (preview === undefined) { + if (preamble.length !== 0 || selected.length < 4) { + const joined = new Uint8Array(preamble.length + selected.length); + joined.set(preamble); + joined.set(selected, preamble.length); + selected = joined; + if (selected.length < 4) { + preamble = selected; + return state; + } + preamble = new Uint8Array(); + } + const little = selected[0] === 0xff && selected[1] === 0xfe; + const big = selected[0] === 0xfe && selected[1] === 0xff; + utf16 = little || big; + bomBytes = utf16 + ? 2 + : selected[0] === 0xef && selected[1] === 0xbb && selected[2] === 0xbf + ? 3 + : 0; + const encoding = little ? "utf-16le" : big ? "utf-16be" : "utf-8"; + preview = new TextDecoder(encoding); + validator = new TextDecoder(encoding, { fatal: true }); + } + const text = preview.decode(selected, { stream: true }); + const result = scanner(text); + // Validate only through the observed signature. Invalid document bytes + // after that signature cannot undo the format observation. + const through = + result.state === "matched" + ? bomBytes + + decodedBytes + + encodedLength(text.slice(0, result.consumed)) - + fedBytes + : selected.length; + try { + validator!.decode(selected.subarray(0, through), { stream: true }); + state = result.state; + } catch { + state = "rejected"; + } + decodedBytes += encodedLength(text); + fedBytes += selected.length; + if (fedBytes === MAX_XML_PLIST_PROBE_BYTES && state === "pending") + state = "rejected"; + return state; + }, + }; +}; + +/** Recognize a plist prolog without claiming that its XML document is valid. */ +export const hasXmlPlistSignature = (bytes: Uint8Array): boolean => + createXmlPlistSignatureProbe().write(bytes) === "matched"; + +const createXmlPlistPrologScanner = (): ((chunk: string) => { + state: XmlPlistProbeState; + consumed: number; +}) => { + let text = ""; + let totalCharacters = 0; + let pending: "?>" | "-->" | undefined; + let doctypeName = false; + return (chunk) => { + const before = totalCharacters; + totalCharacters += chunk.length; + text += chunk; + let cursor = 0; + const result = (state: XmlPlistProbeState, through = cursor) => ({ + state, + consumed: totalCharacters - text.length + through - before, + }); + while (true) { + if (pending !== undefined) { + const end = text.indexOf(pending, cursor); + if (end < 0) { + text = text.slice(Math.max(cursor, text.length - pending.length + 1)); + return result("pending"); + } + cursor = end + pending.length; + pending = undefined; + } + while (cursor < text.length && isXmlWhitespace(text[cursor]!)) cursor++; + if (cursor === text.length) { + text = ""; + return result("pending"); + } + if (doctypeName) { + const token = tokenAt(text, cursor, "plist"); + if ( + token === "partial" || + (token === "match" && cursor + 5 === text.length) + ) { + text = text.slice(cursor); + return result("pending"); + } + const delimiter = text[cursor + 5]; + return result( + token === "match" && + (delimiter === ">" || + delimiter === "[" || + (delimiter !== undefined && isXmlWhitespace(delimiter))) + ? "matched" + : "rejected", + cursor + 6, + ); + } + for (const [token, terminator] of [ + [""], + [""], + ] as const) { + const match = tokenAt(text, cursor, token); + if (match === "partial") { + text = text.slice(cursor); + return result("pending"); + } + if (match === "match") { + pending = terminator; + cursor += token.length; + break; + } + } + if (pending !== undefined) continue; + const doctype = tokenAt(text, cursor, "" || + (delimiter !== undefined && isXmlWhitespace(delimiter))) + ? "matched" + : "rejected", + cursor + 7, + ); + } + }; +}; + +type TokenMatch = "match" | "partial" | "none"; +const tokenAt = (text: string, offset: number, token: string): TokenMatch => { + const available = text.slice(offset, offset + token.length); + return available === token + ? "match" + : token.startsWith(available) + ? "partial" + : "none"; +}; +const isXmlWhitespace = (character: string): boolean => + character === " " || + character === "\t" || + character === "\r" || + character === "\n"; + /** Recognize the serialized signature without claiming a valid NIB object table. */ export const hasNibArchiveSignature = (bytes: Uint8Array): boolean => NIB_ARCHIVE_MAGIC.every((byte, index) => bytes[index] === byte); diff --git a/src/domain/apple/interfaceBuilderGraph.test.ts b/src/domain/apple/interfaceBuilderGraph.test.ts index 96c6b4915..3105e1c5c 100644 --- a/src/domain/apple/interfaceBuilderGraph.test.ts +++ b/src/domain/apple/interfaceBuilderGraph.test.ts @@ -195,7 +195,7 @@ describe("Interface Builder parser bounds", () => { }); }); -describe("keyed archive decoding and graph limits", () => { +describe("keyed archive identity decoding", () => { it("preserves keyed archive UID identities, arrays, and control actions", () => { const parsed = parseInterfaceBuilderRecords({ $archiver: "NSKeyedArchiver", @@ -245,7 +245,9 @@ describe("keyed archive decoding and graph limits", () => { }), ); }); +}); +describe("Interface Builder graph limits", () => { it("reports records omitted by the parser's hard object ceiling", () => { const objects = Object.fromEntries( Array.from({ length: 20_001 }, (_, index) => [ @@ -316,6 +318,46 @@ describe("keyed archive decoding and graph limits", () => { }), ); }); + + it("keeps a later document complete when an earlier hierarchy hits the limit", () => { + const document = ( + relativePath: string, + objectId: string, + references: number, + ) => ({ + relativePath, + archiveSha256: hash, + documentKind: "nib" as const, + raw: { + "com.apple.ibtool.document.objects": { + [objectId]: { class: "UIView" }, + }, + "com.apple.ibtool.document.hierarchy": Array.from( + { length: references }, + () => ({ objectID: objectId }), + ), + }, + }); + const result = buildInterfaceBuilderAnalysis({ + targetSha256: hash, + toolVersion: "test", + documents: [ + document("First.nib", "first", 5), + document("Second.nib", "second", 1), + ], + limits: interfaceBuilderLimitsSchema.parse({ max_objects: 4 }), + }); + + expect( + result.documents.map(({ hierarchy_complete }) => hierarchy_complete), + ).toEqual([false, true]); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "hierarchy:Second.nib", + status: "complete", + }), + ); + }); }); describe("keyed archive object references", () => { diff --git a/src/domain/apple/interfaceBuilderGraph.ts b/src/domain/apple/interfaceBuilderGraph.ts index b3e5216dc..75bc6b735 100644 --- a/src/domain/apple/interfaceBuilderGraph.ts +++ b/src/domain/apple/interfaceBuilderGraph.ts @@ -432,7 +432,7 @@ export const buildInterfaceBuilderAnalysis = (input: { parsed.unresolvedConnections, hierarchy_complete: parsed.hierarchy.length > 0 && - !truncated && + !hierarchyTruncated && parserOmittedHierarchyReferences === 0, }); coverage.push( diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 5a804c8d6..0d12f3812 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -14,7 +14,6 @@ const boundedRelativePathSchema = z .refine( (path) => !path.startsWith("/") && - !path.includes("\\") && path.split("/").every((part) => part !== "" && part !== ".."), "Expected a normalized relative POSIX path without traversal", ); diff --git a/src/domain/artifactIdentity.ts b/src/domain/artifactIdentity.ts index 3107d9af0..8321129a9 100644 --- a/src/domain/artifactIdentity.ts +++ b/src/domain/artifactIdentity.ts @@ -19,9 +19,7 @@ export const logicalPathNfc = (value: string): string => value.normalize("NFC"); */ export const normalizeJoinedLogicalPath = (value: string): string => { if (value.includes("\0")) return value; - return logicalPathNfc( - posix.normalize(value.replaceAll("\\", "/")).replace(/^\.\//u, ""), - ); + return logicalPathNfc(posix.normalize(value).replace(/^\.\//u, "")); }; /** Whether a joined logical path leaves the archive root. */ diff --git a/src/domain/artifactInspection.ts b/src/domain/artifactInspection.ts index ae0e3aa6e..cb9c5ba48 100644 --- a/src/domain/artifactInspection.ts +++ b/src/domain/artifactInspection.ts @@ -299,7 +299,53 @@ const probesFor = ( inventory: ArtifactInventoryResult, ): ArtifactInspectionResult["next_probes"] => { const format = inventory.manifest.root_format; - if (["asar", "javascript-bundle", "directory"].includes(format)) + return format === "directory" + ? directoryProbes(inventory) + : probesForFormat(format); +}; + +// An application bundle directory is only a container, so its probes follow +// the formats it holds: a bundle of Mach-O executables is not a JavaScript tree. +// Formats are ordered so a repeated operation keeps its most general rationale. +const DIRECTORY_MEMBER_FORMATS: readonly ArtifactInventoryResult["manifest"]["root_format"][] = + [ + "asar", + "javascript-bundle", + "mach-o-universal", + "mach-o", + "elf", + "pe", + "plist", + ]; + +const directoryProbes = ( + inventory: ArtifactInventoryResult, +): ArtifactInspectionResult["next_probes"] => { + // JavaScript acquisition does not follow symlinks. Other probes retain + // their own applicability rules; plist inspection follows readable links. + const present = new Set( + inventory.occurrences + .filter( + ({ entry_kind, artifact_format }) => + entry_kind !== "symlink" || artifact_format !== "javascript-bundle", + ) + .map(({ artifact_format }) => artifact_format), + ); + const members = DIRECTORY_MEMBER_FORMATS.filter((memberFormat) => + present.has(memberFormat), + ); + const probes = members.flatMap(probesForFormat); + return probes.filter( + (probe, index) => + probes.findIndex(({ operation }) => operation === probe.operation) === + index, + ); +}; + +const probesForFormat = ( + format: ArtifactInventoryResult["manifest"]["root_format"], +): ArtifactInspectionResult["next_probes"] => { + if (["asar", "javascript-bundle"].includes(format)) return [ { operation: "analyze_javascript_application", diff --git a/src/domain/referenceSourceClassification.ts b/src/domain/referenceSourceClassification.ts index ea53a18a4..0c82cf6d6 100644 --- a/src/domain/referenceSourceClassification.ts +++ b/src/domain/referenceSourceClassification.ts @@ -213,7 +213,6 @@ const GENERATED_DIRECTORY_NAMES = new Set([ ".svelte-kit", ".astro", "storybook-static", - "docs/api", ".codex", ".devin", ".factory", @@ -225,6 +224,8 @@ const GENERATED_DIRECTORY_NAMES = new Set([ ".nyc_output", ]); +const GENERATED_DIRECTORY_PATHS = [["docs", "api"]] as const; + const VENDOR_DIRECTORY_NAMES = new Set([ "node_modules", "vendor", @@ -315,13 +316,18 @@ const basenameMatch = (filename: string, set: ReadonlySet): boolean => { return set.has(lower); }; -const isPathUnderAny = (path: string, names: ReadonlySet): boolean => { - for (const segment of path.split("/")) { - if (segment === "") continue; - if (names.has(segment.toLowerCase())) return true; - } - return false; -}; +const isPathUnderAny = ( + parts: readonly string[], + names: ReadonlySet, +): boolean => parts.some((segment) => names.has(segment)); + +const containsDirectoryPath = ( + parts: readonly string[], + directory: readonly string[], +): boolean => + parts.some((_, start) => + directory.every((segment, offset) => parts[start + offset] === segment), + ); /** Detect a source language from a POSIX path. */ export const detectReferenceSourceLanguage = (path: string): string | null => { @@ -344,33 +350,34 @@ export const classifyReferenceSourcePath = ( path: string, ): ReferenceSourceClassification[] => { const classifications: Set = new Set(); - const parts = path.split("/").filter((segment) => segment !== ""); + const parts = path + .toLowerCase() + .split("/") + .filter((segment) => segment !== ""); const filename = parts[parts.length - 1] ?? ""; - const lower = filename.toLowerCase(); - if (parts.length === 0) { classifications.add("source"); return [...classifications].sort(); } - addDirectoryClassifications(path, classifications); + addDirectoryClassifications(parts, classifications); addFilenameClassifications(filename, classifications); if ( - lower.endsWith(".json") || - lower.endsWith(".yaml") || - lower.endsWith(".yml") + filename.endsWith(".json") || + filename.endsWith(".yaml") || + filename.endsWith(".yml") ) { if (!classifications.has("manifest") && !classifications.has("config")) classifications.add("config"); } - if (isPathUnderAny(path, SOURCE_DIRECTORY_NAMES)) { + if (isPathUnderAny(parts, SOURCE_DIRECTORY_NAMES)) { if (!classifications.has("vendor") && !classifications.has("generated")) classifications.add("source"); } if (classifications.size === 0) { - if (lower === "license" || lower === "copying") + if (filename === "license" || filename === "copying") classifications.add("documentation"); else classifications.add("unknown"); } @@ -379,15 +386,20 @@ export const classifyReferenceSourcePath = ( }; const addDirectoryClassifications = ( - path: string, + parts: readonly string[], classifications: Set, ): void => { - if (isPathUnderAny(path, TEST_DIRECTORY_NAMES)) classifications.add("test"); - if (isPathUnderAny(path, DOCUMENTATION_DIRECTORY_NAMES)) + if (isPathUnderAny(parts, TEST_DIRECTORY_NAMES)) classifications.add("test"); + if (isPathUnderAny(parts, DOCUMENTATION_DIRECTORY_NAMES)) classifications.add("documentation"); - if (isPathUnderAny(path, VENDOR_DIRECTORY_NAMES)) + if (isPathUnderAny(parts, VENDOR_DIRECTORY_NAMES)) classifications.add("vendor"); - if (isPathUnderAny(path, GENERATED_DIRECTORY_NAMES)) + if ( + isPathUnderAny(parts, GENERATED_DIRECTORY_NAMES) || + GENERATED_DIRECTORY_PATHS.some((directory) => + containsDirectoryPath(parts, directory), + ) + ) classifications.add("generated"); }; diff --git a/src/domain/referenceSourceGraph.test.ts b/src/domain/referenceSourceGraph.test.ts index 1286cdde6..272c2f62c 100644 --- a/src/domain/referenceSourceGraph.test.ts +++ b/src/domain/referenceSourceGraph.test.ts @@ -434,6 +434,56 @@ describe("historical source graph validation", () => { }); }); +describe("historical source graph excluded symlink targets", () => { + it("retains internal symlinks into excluded trees without accepting sibling prefixes", () => { + const input = { ...graphInput(), inventory_state: "partial" as const }; + input.exclusions = [ + { + path: "dist", + entry_kind: "directory", + reason: "caller-excluded", + pattern: "dist/", + }, + ]; + input.entries[2] = symlinkEntry({ target: "../dist/generated.js" }); + const graph = createHistoricalSourceGraph(input); + expect(graph.entries[2]).toMatchObject({ + target: "../dist/generated.js", + target_state: "internal", + }); + + input.entries[2] = symlinkEntry({ target: "../dist-old/generated.js" }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /recorded exclusion/u, + ); + }); + + it("does not treat an excluded file or unknown-kind ancestor as a directory", () => { + const input = { ...graphInput(), inventory_state: "partial" as const }; + input.exclusions = [ + { + path: "secret.txt", + entry_kind: "file", + reason: "configured-secret", + pattern: "secret.txt", + }, + ]; + input.entries[2] = symlinkEntry({ target: "../secret.txt/child" }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /recorded exclusion/u, + ); + + input.exclusions[0] = { + path: "secret.txt", + reason: "configured-secret", + pattern: "secret.txt", + }; + expect(() => createHistoricalSourceGraph(input)).toThrow( + /recorded exclusion/u, + ); + }); +}); + describe("historical source graph symlink path identity", () => { it("joins internal symlink targets with the inventory NFC normalizer", () => { // "café" spelled NFC, as the inventory produces it, and the same @@ -486,3 +536,54 @@ describe("historical source graph symlink path identity", () => { ); }); }); + +it.each([ + "\0", + "/absolute.js", + "../outside.js", + "a/../b.js", + "a//b.js", + "./a.js", + "a/", +])("rejects unsafe historical logical entry spelling %j", (path) => { + const input = graphInput(); + input.entries = [fileEntry({ path })]; + input.relationships = []; + expect(() => createHistoricalSourceGraph(input)).toThrow(); +}); + +it("rejects NUL in observed symlink targets before graph identity", () => { + const input = graphInput(); + input.entries[2] = symlinkEntry({ target: "literal\0name.js" }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /normalized POSIX symlink target/u, + ); +}); + +it.each(["C:\\outside\\file.js", "\\\\server\\share.js"])( + "preserves unowned recorded Windows absolute target policy %j", + (target) => { + const external = graphInput(); + external.inventory_state = "partial"; + external.entries[2] = symlinkEntry({ target, target_state: "external" }); + expect(createHistoricalSourceGraph(external).entries).toContainEqual( + expect.objectContaining({ target, target_state: "external" }), + ); + const internal = graphInput(); + internal.entries[2] = symlinkEntry({ target, target_state: "internal" }); + expect(() => createHistoricalSourceGraph(internal)).toThrow( + /target path does not match its target state/u, + ); + }, +); + +it("refuses an absolute POSIX internal target even when lexical joining would name an inventoried file", () => { + const input = graphInput(); + input.entries[2] = symlinkEntry({ + target: "/main.ts", + target_state: "internal", + }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /target path does not match its target state/u, + ); +}); diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts index 197e473bf..0c1c7caaf 100644 --- a/src/domain/referenceSourceGraph.ts +++ b/src/domain/referenceSourceGraph.ts @@ -18,7 +18,7 @@ const relativePathSchema = z .refine( (path) => !path.startsWith("/") && - !path.includes("\\") && + !path.includes("\0") && path .split("/") .every((part) => part !== "" && part !== "." && part !== ".."), @@ -85,10 +85,9 @@ const symlinkTargetSchema = z .min(1) .refine( (target) => - isPortableAbsoluteReferenceTarget(target) || - (!target.startsWith("/") && - !target.includes("\\") && - posix.normalize(target) === target), + !target.includes("\0") && + (isPortableAbsoluteReferenceTarget(target) || + (!target.startsWith("/") && posix.normalize(target) === target)), "Expected a normalized POSIX symlink target", ); @@ -130,6 +129,9 @@ const parseFailureSchema = z.strictObject({ const exclusionSchema = z.discriminatedUnion("reason", [ z.strictObject({ path: relativePathSchema, + entry_kind: z + .enum(["file", "directory", "symlink", "other"]) + .exactOptional(), reason: z.enum([ "configured-secret", "project-ignored", @@ -140,6 +142,9 @@ const exclusionSchema = z.discriminatedUnion("reason", [ }), z.strictObject({ path: relativePathSchema, + entry_kind: z + .enum(["file", "directory", "symlink", "other"]) + .exactOptional(), reason: z.enum(["symlink-escape", "unreadable"]), }), ]); @@ -320,10 +325,50 @@ const checkSymlinks = ( context: z.RefinementCtx, ): void => { const lookupPath = createReferenceSourcePathLookup(paths); + const excludedPaths = createReferenceSourcePathLookup( + graph.exclusions.map(({ path }) => path), + ); + const excludedKinds = new Map< + string, + Set<"file" | "directory" | "symlink" | "other" | undefined> + >(); + for (const { path, entry_kind } of graph.exclusions) { + const kinds = excludedKinds.get(path) ?? new Set(); + kinds.add(entry_kind); + excludedKinds.set(path, kinds); + } for (const [index, entry] of graph.entries.entries()) { if (entry.kind !== "symlink") continue; if (entry.target === null) continue; - const absolute = isPortableAbsoluteReferenceTarget(entry.target); + const resolved = posix.normalize( + posix.join(posix.dirname(entry.path), entry.target), + ); + const escapesRoot = resolved.startsWith("../") || resolved === ".."; + // The root is implicit; every other internal target needs an exact or + // uniquely Unicode-equivalent recorded inventory identity. + const inventoried = + resolved === "." || lookupPath.resolve(resolved) !== undefined; + const exactExcluded = excludedPaths.resolve(resolved); + let excluded = exactExcluded !== undefined; + for ( + let candidate = posix.dirname(resolved); + candidate !== "." && candidate !== "/"; + candidate = posix.dirname(candidate) + ) { + const excludedAncestor = excludedPaths.resolve(candidate); + if (excludedAncestor === undefined) continue; + const kinds = excludedKinds.get(excludedAncestor); + if (kinds?.size !== 1 || !kinds.has("directory")) { + excluded = false; + break; + } + excluded = true; + } + const ownedRelativeTarget = + !entry.target.startsWith("/") && !escapesRoot && inventoried; + const absolute = + isPortableAbsoluteReferenceTarget(entry.target) && + !(entry.target_state === "internal" && ownedRelativeTarget); if ( (entry.target_state === "external" && !absolute) || (entry.target_state === "internal" && absolute) @@ -334,18 +379,11 @@ const checkSymlinks = ( path: ["entries", index, "target"], }); if (entry.target_state !== "internal") continue; - const resolved = posix.normalize( - posix.join(posix.dirname(entry.path), entry.target), - ); - if ( - resolved.startsWith("../") || - resolved === ".." || - // The inventory root is implicit rather than an entry with path ".". - (resolved !== "." && lookupPath.resolve(resolved) === undefined) - ) + if (escapesRoot || (!inventoried && !excluded)) context.addIssue({ code: "custom", - message: "Internal symlink target must resolve to an inventoried entry", + message: + "Internal symlink target must resolve to an inventoried entry or a recorded exclusion", path: ["entries", index, "target"], }); } diff --git a/src/domain/referenceSourcePathIdentity.ts b/src/domain/referenceSourcePathIdentity.ts index cde64c71a..4ac6963cf 100644 --- a/src/domain/referenceSourcePathIdentity.ts +++ b/src/domain/referenceSourcePathIdentity.ts @@ -1,6 +1,10 @@ +import { posix } from "node:path"; + export interface ReferenceSourcePathLookup { readonly resolve: (path: string) => string | undefined; readonly resolveCandidates: (paths: readonly string[]) => string | undefined; + /** Resolve a path to the closest recorded parent identity. */ + readonly resolveAncestor: (path: string) => string | undefined; } /** @@ -18,11 +22,24 @@ export const createReferenceSourcePathLookup = ( } const equivalent = (path: string): string | undefined => canonical.get(path.normalize("NFC")) ?? undefined; + const resolve = (path: string): string | undefined => + exact.has(path) ? path : equivalent(path); return { - resolve: (path) => (exact.has(path) ? path : equivalent(path)), + resolve, resolveCandidates: (candidates) => candidates.find((path) => exact.has(path)) ?? candidates.map(equivalent).find((path) => path !== undefined), + resolveAncestor: (path) => { + for ( + let candidate = path; + candidate !== "." && candidate !== "/"; + candidate = posix.dirname(candidate) + ) { + const found = resolve(candidate); + if (found !== undefined) return found; + } + return undefined; + }, }; }; diff --git a/src/javascript/recovery/RecoveryFiles.ts b/src/javascript/recovery/RecoveryFiles.ts index c64f011b4..cd1dcb666 100644 --- a/src/javascript/recovery/RecoveryFiles.ts +++ b/src/javascript/recovery/RecoveryFiles.ts @@ -167,6 +167,11 @@ export const snapshotRecoveryInputs = async ( ? "bundle.js" : relative(commonDirectory, resolve(path)).split(sep).join("/"); try { + // Wakaru's multi-input public paths interpret backslashes as separators. + if (relativePath.includes("\\")) + throw new TypeError( + "Wakaru cannot preserve literal backslashes in input paths", + ); if (normalizeArtifactPath(relativePath) !== relativePath) throw new TypeError( "Input names must already use NFC Unicode spelling", diff --git a/src/reference/ReferenceSourceReaderErrors.ts b/src/reference/ReferenceSourceReaderErrors.ts index 6ddd02a3f..4444b43ba 100644 --- a/src/reference/ReferenceSourceReaderErrors.ts +++ b/src/reference/ReferenceSourceReaderErrors.ts @@ -29,6 +29,9 @@ const FILESYSTEM_ERROR_CODES = new Set([ "EISDIR", "ELOOP", "EMFILE", + // A traversed child can exceed the platform's absolute path limit even + // though the selected root and every component are valid. + "ENAMETOOLONG", "ENFILE", "ENOENT", "ENOTDIR", diff --git a/tests/boundary/cli/jsonInputSize.test.ts b/tests/boundary/cli/jsonInputSize.test.ts index 0a4aad8ee..33996b42f 100644 --- a/tests/boundary/cli/jsonInputSize.test.ts +++ b/tests/boundary/cli/jsonInputSize.test.ts @@ -8,14 +8,14 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; cliTest( - "rejects malformed oversized JSON through both CLI and Evidence readers", + "rejects malformed oversized input through CLI and Evidence readers", async ({ cli, processes }) => { const root = await createTestTempDirectory("rea-json-string-limit-"); const input = join(root, "oversized.json"); const file = await open(input, "wx"); try { - // Sparse zero bytes are valid UTF-8 but invalid JSON. Both readers - // stream the input and reject the syntax before assembling a value. + // Sparse zero bytes are valid UTF-8. Both streaming readers reject + // their invalid JSON syntax without assembling a whole-document string. await file.truncate(constants.MAX_STRING_LENGTH + 1); } finally { await file.close(); @@ -57,7 +57,7 @@ cliTest( cliTest( "traces valid Evidence beyond the whole-document string limit", - async ({ cli }) => { + async ({ cli, processes }) => { const root = await createTestTempDirectory("rea-json-streamed-evidence-"); const source = join(root, "application"); await mkdir(source); @@ -175,7 +175,7 @@ cliTest( cliTest( "returns a typed string assembly constraint under a bounded heap", - async ({ cli }) => { + async ({ cli, processes }) => { const root = await createTestTempDirectory("rea-json-string-headroom-"); const input = join(root, "long-string.json"); const file = await open(input, "wx"); @@ -207,5 +207,21 @@ cliTest( action: expect.stringContaining("re-analyze a smaller selection"), }, }); + const evidence = await processes.run(process.execPath, [ + "--max-old-space-size=768", + "--input-type=module", + "-e", + `import { readJsonFile } from "./dist/application/JsonFiles.js"; + import { projectAnalysisError } from "./dist/domain/analysisErrorProjection.js"; + const result = await readJsonFile(process.argv[1]); + console.log(JSON.stringify(result.ok ? result : projectAnalysisError(result.error)));`, + input, + ]); + expect(JSON.parse(evidence.stdout)).toMatchObject({ + code: "resource_constraint", + details: { operation: "read_evidence_file", resource: "memory" }, + }); + expect(evidence.stderr).toBe(""); + expect(evidence.exitCode).toBe(0); }, ); diff --git a/tests/boundary/cli/referenceSourceExclusions.test.ts b/tests/boundary/cli/referenceSourceExclusions.test.ts index cbc063285..87b854116 100644 --- a/tests/boundary/cli/referenceSourceExclusions.test.ts +++ b/tests/boundary/cli/referenceSourceExclusions.test.ts @@ -1,4 +1,10 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { + mkdir, + readFile, + readlink, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { describe, expect } from "vitest"; @@ -10,6 +16,56 @@ import { cliTest } from "../../support/cli/cliFixture.js"; describe.skipIf(process.platform === "win32")( "compiled reference-source exclusion provenance", () => { + cliTest( + "imports source beside a link into a default-excluded directory", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-link-exclusion-", + ); + await mkdir(join(root, "dist")); + const source = "export const value = 1;\n"; + await writeFile(join(root, "source.ts"), source); + await symlink("dist", join(root, "alias")); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + cwd: root, + environment: { + HOME: root, + USERPROFILE: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }, + }); + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.inventory_state).toBe("partial"); + expect(graph.exclusions).toContainEqual({ + path: "dist", + entry_kind: "directory", + reason: "default-ignored", + pattern: "dist/", + }); + expect(graph.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "alias", + kind: "symlink", + target: "dist", + target_state: "internal", + }), + expect.objectContaining({ + path: "source.ts", + kind: "file", + content_state: "hashed", + }), + ]), + ); + expect(graph.entries.some(({ path }) => path === "dist")).toBe(false); + expect(await readFile(join(root, "source.ts"), "utf8")).toBe(source); + expect(await readlink(join(root, "alias"))).toBe("dist"); + }, + ); + cliTest( "reports project, default, and explicit secret exclusions", async ({ cli }) => { @@ -42,16 +98,19 @@ describe.skipIf(process.platform === "win32")( expect.arrayContaining([ { path: "project.tmp", + entry_kind: "file", reason: "project-ignored", pattern: "project.tmp", }, { path: "logs/application.log", + entry_kind: "file", reason: "default-ignored", pattern: "*.log", }, { path: "private.txt", + entry_kind: "file", reason: "configured-secret", pattern: "private.txt", }, diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.ts index 06093f427..0a1243a91 100644 --- a/tests/boundary/cli/referenceSourceUtf8.test.ts +++ b/tests/boundary/cli/referenceSourceUtf8.test.ts @@ -1,5 +1,11 @@ import { createHash } from "node:crypto"; -import { writeFile } from "node:fs/promises"; +import { + mkdir, + readFile, + readlink, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; @@ -39,7 +45,9 @@ cliTest( arguments: ["import-reference-source", root, "--json"], environment: { HOME: root, XDG_CONFIG_HOME: root, XDG_CACHE_HOME: root }, }); - expect(result.exitCode).toBe(0); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); const graph = historicalSourceGraphSchema.parse(result.json); expect(graph.parse_failures).toEqual([ { @@ -75,3 +83,139 @@ cliTest( ); }, ); + +cliTest.skipIf(process.platform === "win32")( + "preserves historical literal-backslash entries and their exact owning import base", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-literal-paths-"); + const config = await createTestTempDirectory( + "rea-reference-literal-config-", + ); + await mkdir(join(root, "literal")); + await mkdir(join(root, "literal\\dir")); + const sources = new Map([ + ["literal\\name.js", "export const value = 'literal';\n"], + ["literal/name.js", "export const value = 'slash';\n"], + ["literal\\dir/main.js", 'import "./dep.js";\n'], + ["literal\\dir/dep.js", "export const value = 'owned';\n"], + ]); + for (const [path, bytes] of sources) + await writeFile(join(root, path), bytes); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + for (const [path, bytes] of sources) + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + content_state: "hashed", + }), + ); + for (const [path, bytes] of sources) + expect(await readFile(join(root, path), "utf8")).toBe(bytes); + expect(graph.relationships).toContainEqual({ + from_path: "literal\\dir/main.js", + to: "literal\\dir/dep.js", + kind: "imports", + resolution: "internal", + parse_state: "parsed", + }); + }, +); + +cliTest.skipIf(process.platform === "win32")( + "preserves an observed historical symlink target with a literal backslash", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-literal-link-"); + const config = await createTestTempDirectory( + "rea-reference-literal-link-config-", + ); + await writeFile( + join(root, "literal\\name.js"), + "export const value = 42;\n", + ); + await symlink("literal\\name.js", join(root, "link.js")); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(await readlink(join(root, "link.js"))).toBe("literal\\name.js"); + expect(await readFile(join(root, "literal\\name.js"), "utf8")).toBe( + "export const value = 42;\n", + ); + expect(graph.entries).toContainEqual( + expect.objectContaining({ + kind: "symlink", + path: "link.js", + target: "literal\\name.js", + target_state: "internal", + }), + ); + }, +); + +cliTest.skipIf(process.platform === "win32")( + "imports an inventory-owned POSIX UNC-looking symlink target", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-unc-literal-"); + const config = await createTestTempDirectory("rea-reference-unc-config-"); + const literal = "\\\\server\\share.js"; + const literalBytes = "export const value = 'literal';\n"; + const slashBytes = "export const value = 'slash';\n"; + await mkdir(join(root, "server")); + await writeFile(join(root, literal), literalBytes); + await writeFile(join(root, "server", "share.js"), slashBytes); + await symlink(literal, join(root, "link.js")); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.entries).toContainEqual( + expect.objectContaining({ + kind: "symlink", + path: "link.js", + target: literal, + target_state: "internal", + }), + ); + for (const [path, bytes] of [ + [literal, literalBytes], + ["server/share.js", slashBytes], + ]) { + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path, + sha256: createHash("sha256").update(bytes!).digest("hex"), + }), + ); + expect(await readFile(join(root, path!), "utf8")).toBe(bytes); + } + expect(await readlink(join(root, "link.js"))).toBe(literal); + }, +); diff --git a/tests/boundary/filesystem/appleXmlDeclarations.test.ts b/tests/boundary/filesystem/appleXmlDeclarations.test.ts index e066655eb..56763d8b0 100644 --- a/tests/boundary/filesystem/appleXmlDeclarations.test.ts +++ b/tests/boundary/filesystem/appleXmlDeclarations.test.ts @@ -1,5 +1,7 @@ import { writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; import { join } from "node:path"; +import { promisify } from "node:util"; import { build } from "plist"; import { expect, it } from "vitest"; @@ -14,6 +16,32 @@ const archive = build({ $top: { root: { CF$UID: 1 } }, }); +it("preflights repeated unterminated quoted declarations without suffix rescans", async () => { + const preflightUrl = new URL( + "../../../dist/artifacts/apple/InterfaceBuilderDecodeBudget.js", + import.meta.url, + ).href; + const { stdout } = await promisify(execFile)( + process.execPath, + [ + "--input-type=module", + "-e", + ` + import { preflightPropertyListDecode } from ${JSON.stringify(preflightUrl)}; + const xml = 'x" '.repeat(70000); + const observed = preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024); + process.stdout.write(JSON.stringify({ observed, expected: xml.length * 4 })); + `, + ], + { timeout: 5000 }, + ); + const result = JSON.parse(stdout) as { + observed: { estimatedBytes: number }; + expected: number; + }; + expect(result.observed.estimatedBytes).toBe(result.expected); +}, 10000); + const fixture = async (declaration: string) => { const bundlePath = await createTestTempDirectory("rea-xml-declaration-"); await writeFile( diff --git a/tests/boundary/filesystem/artifactLiteralBackslash.test.ts b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts new file mode 100644 index 000000000..02105ad60 --- /dev/null +++ b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts @@ -0,0 +1,222 @@ +import { createHash } from "node:crypto"; +import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { Readable } from "node:stream"; + +import { createPackage } from "@electron/asar"; +import { expect, it } from "vitest"; + +import { extractArtifact } from "../../../src/artifacts/extraction/ArtifactExtraction.js"; +import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js"; +import { + inventoryArtifact, + withArtifactResourceScope, +} from "../../fixtures/artifactInventory.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; + +const NAMES = [ + "literal\\name.txt", + "\\leading.txt", + "\\\\server\\share.txt", + "ordinary.txt", + "literal/name.txt", + "nested/literal\\name.txt", +]; +const CONTENT = Buffer.from("exact member bytes\n"); +const DIGEST = createHash("sha256").update(CONTENT).digest("hex"); + +it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( + "inventories and extracts literal POSIX backslashes from %s without changing names", + async (format) => { + const root = await createTestTempDirectory("rea-literal-backslash-"); + const source = join(root, "source"); + await mkdir(source); + for (const name of NAMES) { + await mkdir(dirname(join(source, name)), { recursive: true }); + await writeFile(join(source, name), CONTENT); + } + const input = format === "directory" ? source : join(root, "input.asar"); + if (format === "asar") await createPackage(source, input); + + const inventory = await inventoryArtifact(input); + for (const name of NAMES) { + const occurrence = inventory.occurrences.find( + ({ logical_path }) => logical_path === name, + ); + expect(occurrence).toMatchObject({ + entry_kind: "file", + hash_status: "verified", + }); + expect( + inventory.nodes.find( + ({ artifact_id }) => artifact_id === occurrence?.artifact_id, + ), + ).toMatchObject({ sha256: DIGEST, size: CONTENT.length }); + } + + const output = join(root, "output"); + const extracted = await withArtifactResourceScope((resourceScope) => + extractArtifact({ + inputPath: input, + // The owner checks lstat first and selects the directory reader; + // BinaryTarget formats have no separate directory tag. + inputFormat: "asar", + outputRoot: output, + environment: {}, + integrityPolicy: "fail", + resourceScope, + }), + ); + expect( + extracted.artifacts.map(({ relative_path }) => relative_path).sort(), + ).toEqual([...NAMES].sort()); + expect((await readdir(output)).sort()).toEqual( + [...new Set(NAMES.map((name) => name.split("/")[0]))].sort(), + ); + for (const name of NAMES) + expect(await readFile(join(output, name))).toEqual(CONTENT); + }, +); + +it.each(NAMES.filter((name) => name.includes("\\")))( + "rejects Windows separator interpretation before materializing %j", + async (name) => { + const root = await createTestTempDirectory("rea-backslash-output-"); + const output = join(root, "output"); + const tree = await SafeOutputTree.create(output, "win32"); + const stream = Readable.from(CONTENT); + try { + await expect( + tree.write(name, stream, { sha256: DIGEST, bytes: CONTENT.length }), + ).rejects.toMatchObject({ + reason: "path", + message: expect.stringContaining( + "Windows destination cannot materialize artifact path", + ), + }); + expect(stream.destroyed).toBe(true); + expect(await readdir(output)).toEqual([]); + } finally { + expect(await tree.rollback()).toMatchObject({ status: "complete" }); + } + await expect(access(output)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +it.each([ + "../outside", + "safe/../outside", + "/outside", + "//server/share", + "C:\\outside", + "C:/outside", + "safe/\0bad", +])("keeps output containment rejection for %j", async (name) => { + const root = await createTestTempDirectory("rea-backslash-containment-"); + const output = join(root, "output"); + const tree = await SafeOutputTree.create(output); + try { + await expect( + tree.write(name, Readable.from(CONTENT), { + sha256: DIGEST, + bytes: CONTENT.length, + }), + ).rejects.toMatchObject({ reason: "path" }); + expect(await readdir(output)).toEqual([]); + } finally { + expect(await tree.rollback()).toMatchObject({ status: "complete" }); + } +}); + +it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( + "retains literal POSIX JavaScript names in application and semantic graphs from %s", + async (format) => { + const root = await createTestTempDirectory("rea-backslash-js-graph-"); + const source = join(root, "source"); + await mkdir(source); + const names = [ + "literal\\name.js", + "\\leading.js", + "\\\\server\\share.js", + "nested/literal\\name.js", + ]; + for (const name of names) { + await mkdir(dirname(join(source, name)), { recursive: true }); + await writeFile(join(source, name), "export const marker = 42;\n"); + } + const input = format === "directory" ? source : join(root, "input.asar"); + if (format === "asar") await createPackage(source, input); + const result = await analyzeJavaScriptApplication({ + input_path: input, + format, + }); + if (!result.ok) throw result.error; + const parsed = javascriptApplicationAnalysisResultSchema.parse( + result.value.normalized_result, + ); + const observed = parsed.graph.nodes.flatMap(({ observations }) => + observations.map(({ properties }) => properties.path), + ); + expect(observed).toEqual(expect.arrayContaining(names)); + const semantic = parsed.semantic_graph?.nodes.map( + ({ identity }) => identity.module_path, + ); + expect(semantic).toEqual(expect.arrayContaining(names)); + }, +); + +it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( + "resolves ordinary relative imports inside a literal POSIX directory from %s", + async (format) => { + const root = await createTestTempDirectory("rea-backslash-import-base-"); + const source = join(root, "source"); + const literal = "literal\\dir"; + await mkdir(join(source, literal), { recursive: true }); + await mkdir(join(source, "literal", "dir"), { recursive: true }); + const body = "export const marker = 'literal';\n"; + await writeFile( + join(source, literal, "a.js"), + "import { marker } from './b.js'; export { marker };\n", + ); + await writeFile(join(source, literal, "b.js"), body); + await writeFile( + join(source, "literal", "dir", "b.js"), + "export const marker = 'decoy';\n", + ); + const input = format === "directory" ? source : join(root, "input.asar"); + if (format === "asar") await createPackage(source, input); + const result = await analyzeJavaScriptApplication({ + input_path: input, + format, + }); + if (!result.ok) throw result.error; + const parsed = javascriptApplicationAnalysisResultSchema.parse( + result.value.normalized_result, + ); + const imports = parsed.graph.edges.filter( + ({ properties }) => properties.specifier === "./b.js", + ); + expect(imports.length).toBeGreaterThan(0); + expect(imports.map(({ properties }) => properties.resolved_path)).toEqual( + imports.map(() => `${literal}/b.js`), + ); + expect(parsed.graph.nodes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + identity: { + strategy: "content-digest", + stability: "global-exact", + sha256: createHash("sha256").update(body).digest("hex"), + }, + observations: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ path: `${literal}/b.js` }), + }), + ]), + }), + ]), + ); + }, +); diff --git a/tests/boundary/filesystem/asarHeaderAdmission.test.ts b/tests/boundary/filesystem/asarHeaderAdmission.test.ts new file mode 100644 index 000000000..e16e2ea2d --- /dev/null +++ b/tests/boundary/filesystem/asarHeaderAdmission.test.ts @@ -0,0 +1,115 @@ +import { mkdir, open, readFile, truncate, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createPackage } from "@electron/asar"; +import { expect, it, vi } from "vitest"; + +import { + admitAsarHeader, + MAX_ASAR_HEADER_BYTES, +} from "../../../src/artifacts/AsarHeader.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const createFixture = async () => { + const root = await createTestTempDirectory("rea-asar-header-admission-"); + const source = join(root, "source"); + const archive = join(root, "fixture.asar"); + await mkdir(source); + await writeFile(join(source, "main.js"), "export const answer = 42;\n"); + await createPackage(source, archive); + return { archive, original: await readFile(archive) }; +}; + +const admit = async (path: string) => { + const handle = await open(path, "r"); + const read = vi.spyOn(handle, "read"); + try { + return await admitAsarHeader(handle); + } finally { + // Admission must inspect only the fixed prefix, even for forged extents. + const calls = [...read.mock.calls]; + read.mockRestore(); + await handle.close(); + expect(handle.fd).toBe(-1); + expect(calls).toEqual([[expect.any(Buffer), 0, 16, 0]]); + } +}; + +it("admits a header produced by the upstream ASAR serializer", async () => { + const { archive, original } = await createFixture(); + await expect(admit(archive)).resolves.toEqual({ + archiveBytes: original.length, + headerBytes: original.readUInt32LE(4), + jsonBytes: original.readUInt32LE(12), + }); + expect(await readFile(archive)).toEqual(original); +}); + +it("rejects every truncated prefix of a real serialized header", async () => { + const { archive, original } = await createFixture(); + const headerEnd = 8 + original.readUInt32LE(4); + for (let length = 0; length < headerEnd; length++) { + await writeFile(archive, original.subarray(0, length)); + await expect( + admit(archive), + `truncated at byte ${length}`, + ).rejects.toMatchObject({ + reason: "format", + }); + } + // Member data is outside header admission; an intact header still passes. + await writeFile(archive, original.subarray(0, headerEnd)); + await expect(admit(archive)).resolves.toMatchObject({ + archiveBytes: headerEnd, + }); +}); + +it("rejects contradictory and unsigned-boundary extents before decoding", async () => { + const { archive, original } = await createFixture(); + const mutations = [ + { offset: 0, values: [0, 1, 3, 5, 0xffffffff] }, + { offset: 4, values: [0, 4, 7, original.length, 0x80000000, 0xffffffff] }, + { offset: 8, values: [0, 1, original.readUInt32LE(8) + 4, 0xffffffff] }, + { offset: 12, values: [original.readUInt32LE(8), 0x80000000, 0xffffffff] }, + ]; + for (const { offset, values } of mutations) { + for (const value of values) { + const bytes = Buffer.from(original); + bytes.writeUInt32LE(value, offset); + await writeFile(archive, bytes); + await expect( + admit(archive), + `uint32 at ${offset} = ${value}`, + ).rejects.toMatchObject({ + reason: "format", + }); + } + } +}); + +it("separates the header decode budget from container validity at its exact boundary", async () => { + const root = await createTestTempDirectory("rea-asar-header-budget-"); + const archive = join(root, "budget.asar"); + expect(MAX_ASAR_HEADER_BYTES).toBe(16 * 1024 * 1024); + for (const headerBytes of [ + MAX_ASAR_HEADER_BYTES, + MAX_ASAR_HEADER_BYTES + 4, + ]) { + const prefix = Buffer.alloc(16); + prefix.writeUInt32LE(4, 0); + prefix.writeUInt32LE(headerBytes, 4); + prefix.writeUInt32LE(headerBytes - 4, 8); + prefix.writeUInt32LE(2, 12); + await writeFile(archive, prefix); + // Supply the declared extent without allocating a header-sized JS buffer. + // This tests extent admission only; the padding is not valid archive JSON. + await truncate(archive, headerBytes + 8); + if (headerBytes === MAX_ASAR_HEADER_BYTES) + await expect(admit(archive)).resolves.toEqual({ + archiveBytes: headerBytes + 8, + headerBytes, + jsonBytes: 2, + }); + else + await expect(admit(archive)).rejects.toMatchObject({ reason: "limit" }); + } +}); diff --git a/tests/boundary/filesystem/binaryTarget.test.ts b/tests/boundary/filesystem/binaryTarget.test.ts index eb5977a54..93ef57afa 100644 --- a/tests/boundary/filesystem/binaryTarget.test.ts +++ b/tests/boundary/filesystem/binaryTarget.test.ts @@ -26,6 +26,7 @@ import { thinMach, } from "../../../src/domain/binaryTarget.fixture.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { hasXmlPlistSignature } from "../../../src/domain/apple/appleArchiveFormat.js"; import { parseExecutableHeader } from "../../../src/domain/binaryTarget.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -258,6 +259,46 @@ describe("binary target I/O: package classification", () => { }); }); + it("classifies XML plists by content while preserving named formats", async () => { + const directory = await createTestTempDirectory("rea-xml-plist-target-"); + const xml = Buffer.from( + '$archiverNSKeyedArchiver', + ); + const archivePath = join(directory, "Model.archive"); + const javascriptPath = join(directory, "Model.js"); + await writeFile(archivePath, xml); + await writeFile(javascriptPath, xml); + + const archive = await parseBinaryTarget(archivePath); + const javascript = await parseBinaryTarget(javascriptPath); + expect(archive.ok && archive.value).toMatchObject({ + kind: "artifact", + format: "plist", + }); + expect(javascript.ok && javascript.value).toMatchObject({ + kind: "artifact", + format: "javascript", + }); + }); + + it("continues a bounded XML plist probe beyond the initial file window", async () => { + const directory = await createTestTempDirectory("rea-xml-long-prolog-"); + const path = join(directory, "Model.archive"); + await writeFile( + path, + `${" ".repeat(8192)}`, + ); + + const result = await parseBinaryTarget(path); + expect(result.ok && result.value).toMatchObject({ + kind: "artifact", + format: "plist", + }); + expect(hasXmlPlistSignature(Buffer.from(``))).toBe( + false, + ); + }); + it("does not trust a ZIP-family extension without ZIP magic", async () => { const directory = await createTestTempDirectory("rea-fake-archive-"); const path = join(directory, "fake.zip"); diff --git a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts index 4145e7bca..0ca2e791a 100644 --- a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts +++ b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts @@ -1,4 +1,4 @@ -import { writeFile, type FileHandle } from "node:fs/promises"; +import { mkdir, rename, writeFile, type FileHandle } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -47,6 +47,30 @@ afterEach(() => { }); describe("DirectoryArtifactReader resource ownership", () => { + it("rejects descendants after the selected root directory is replaced", async () => { + const { root, reader } = await makeFixture(); + const subdirectory = join(root, "sub"); + await mkdir(subdirectory); + await writeFile(join(subdirectory, "old.txt"), "original tree\n"); + const movedRoot = `${root}-original`; + try { + const entries = reader.entries()[Symbol.asyncIterator](); + expect((await entries.next()).value?.path).toBe("sub"); + expect((await entries.next()).value?.path).toBe("sample.txt"); + + await rename(root, movedRoot); + await mkdir(root); + await mkdir(subdirectory); + await writeFile(join(subdirectory, "new.txt"), "replacement tree\n"); + await expect(entries.next()).rejects.toMatchObject({ + reason: "path", + message: "Directory traversal escaped or changed its artifact root", + }); + } finally { + await reader.close(); + } + }); + it("closes the exact directory owner on early iterator return", async () => { const { reader } = await makeFixture(); let closeCalls = 0; diff --git a/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts b/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts index e2dbe084d..c1a2790e1 100644 --- a/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts +++ b/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts @@ -2,6 +2,7 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; +import { parse } from "plist"; import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -27,3 +28,76 @@ it.each([129, 3000])( }); }, ); + +it.each(["(}", "{)", "(@", "{key @", "(x @", "(x ", "{key ", "{key = x,"])( + "preserves the pinned OpenStep syntax diagnosis before a deep suffix after %s", + async (prefix) => { + const text = prefix + "(".repeat(130); + let pinned: unknown; + try { + parse(text); + } catch (cause) { + pinned = cause; + } + if (!(pinned instanceof Error) || pinned instanceof RangeError) + throw new Error( + "Malformed control must fail in the pinned syntax decoder", + ); + const root = await createTestTempDirectory("rea-keyed-openstep-malformed-"); + await writeFile(join(root, "Malformed.plist"), text); + const rejected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: "a".repeat(64), + parameters: { path: "Malformed.plist" }, + platform: "linux", + }).catch((cause: unknown) => cause); + expect(rejected).toMatchObject({ + reason: "format", + message: `Cannot decode selected Foundation keyed archive: ${pinned.message}`, + }); + }, +); + +it("retains the limit classification for syntactically valid deep OpenStep values", async () => { + const root = await createTestTempDirectory("rea-keyed-openstep-limit-"); + await writeFile( + join(root, "Deep.plist"), + "(".repeat(129) + "x" + ")".repeat(129), + ); + const rejected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: "a".repeat(64), + parameters: { path: "Deep.plist" }, + platform: "linux", + }).catch((cause: unknown) => cause); + expect(rejected).toMatchObject({ + reason: "limit", + message: + "Cannot decode selected Foundation keyed archive: Plist exceeds 128 nesting levels", + }); +}); + +it.each([ + ["array sibling", (deep: string) => `((x),${deep})`], + ["compound dictionary key", (deep: string) => `{(key)=${deep};}`], +] as const)( + "retains the named OpenStep depth limit after a completed %s", + async (_name, document) => { + const root = await createTestTempDirectory( + "rea-keyed-openstep-sibling-depth-", + ); + const deep = "(".repeat(30000) + "x" + ")".repeat(30000); + await writeFile(join(root, "Deep.plist"), document(deep)); + const rejected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: "a".repeat(64), + parameters: { path: "Deep.plist" }, + platform: "linux", + }).catch((cause: unknown) => cause); + expect(rejected).toMatchObject({ + reason: "limit", + message: + "Cannot decode selected Foundation keyed archive: Plist exceeds 128 nesting levels", + }); + }, +); diff --git a/tests/boundary/filesystem/recoveryInputPaths.test.ts b/tests/boundary/filesystem/recoveryInputPaths.test.ts new file mode 100644 index 000000000..6fb6afa21 --- /dev/null +++ b/tests/boundary/filesystem/recoveryInputPaths.test.ts @@ -0,0 +1,72 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { snapshotRecoveryInputs } from "../../../src/javascript/recovery/RecoveryFiles.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it + .skipIf(process.platform === "win32") + .each(["cafe\u0301.js", "chunk\\name.js"])( + "refuses an unsupported Wakaru multi-input spelling %s before snapshotting", + async (name) => { + const root = await createTestTempDirectory("rea-recovery-input-path-"); + const selected = join(root, name); + const ordinary = join(root, "main.js"); + const staging = join(root, "snapshots"); + await writeFile(selected, "export const value = 'literal';"); + await writeFile(ordinary, "export const value = 'ordinary';"); + const slashDecoy = join(root, "chunk", "name.js"); + await mkdir(join(root, "chunk")); + await writeFile(slashDecoy, "export const value = 'slash decoy';"); + await expect( + snapshotRecoveryInputs([selected, ordinary], staging), + ).rejects.toBeInstanceOf(AnalysisInputError); + await expect(access(staging)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(selected, "utf8")).toBe( + "export const value = 'literal';", + ); + expect(await readFile(ordinary, "utf8")).toBe( + "export const value = 'ordinary';", + ); + expect(await readFile(slashDecoy, "utf8")).toBe( + "export const value = 'slash decoy';", + ); + }, +); + +it.skipIf(process.platform === "win32")( + "keeps ordinary slash layout and safely renames a single literal input", + async () => { + const root = await createTestTempDirectory("rea-recovery-input-path-"); + const literal = join(root, "chunk\\name.js"); + const slash = join(root, "chunk", "name.js"); + const main = join(root, "main.js"); + await mkdir(join(root, "chunk")); + await writeFile(literal, "export const value = 'literal';"); + await writeFile(slash, "export const value = 'slash';"); + await writeFile(main, "export const value = 'main';"); + const single = await snapshotRecoveryInputs( + [literal], + join(root, "single"), + ); + expect(single[0].relative_path).toBe("bundle.js"); + expect(await readFile(single[0].snapshot_path, "utf8")).toBe( + "export const value = 'literal';", + ); + const multiple = await snapshotRecoveryInputs( + [main, slash], + join(root, "multiple"), + ); + expect(multiple.map(({ relative_path }) => relative_path)).toEqual([ + "main.js", + "chunk/name.js", + ]); + expect(await readFile(multiple[1]?.snapshot_path ?? "", "utf8")).toBe( + "export const value = 'slash';", + ); + expect(await readFile(literal, "utf8")).toBe( + "export const value = 'literal';", + ); + }, +); diff --git a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts index dd87ea92a..d379f01c6 100644 --- a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts +++ b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts @@ -117,6 +117,7 @@ describe("reference import native entry failure details", () => { ).toBeUndefined(); expect(imported.value.exclusions).toContainEqual({ path: "secret-events.pipe", + entry_kind: "other", reason: "configured-secret", pattern: "*secret*", }); @@ -133,3 +134,62 @@ describe("reference import native entry failure details", () => { ).toMatchObject({ kind: "file", content_state: "hashed", limitations: [] }); }); }); + +describe("reference import paths beyond the platform limit", () => { + it.skipIf(process.platform === "win32")( + "reports an entry beyond the absolute path limit as an I/O failure", + async () => { + const root = await createTestTempDirectory("rea-import-deep-path-"); + await writeFile(join(root, "kept.txt"), "k"); + // Relative creation lets the tree exceed PATH_MAX (1024 on macOS, 4096 + // on Linux) while each 200-byte component stays below NAME_MAX. + const deepTree = (action: "create" | "remove") => + promisify(execFile)(process.execPath, [ + "-e", + `const fs = require("node:fs"); + const names = Array.from({ length: 24 }, (_, i) => "d" + String(i).padStart(2, "0") + "x".repeat(197)); + process.chdir(${JSON.stringify(root)}); + if (${JSON.stringify(action)} === "create") { + for (const name of names) { fs.mkdirSync(name); process.chdir(name); } + fs.writeFileSync("deep.txt", "d"); + } else { + for (const name of names) process.chdir(name); + fs.rmSync("deep.txt"); + for (const name of [...names].reverse()) { process.chdir(".."); fs.rmdirSync(name); } + }`, + ]); + await deepTree("create"); + try { + const read = await readReferenceSource(root); + if (!read.ok) throw new Error(read.error.message); + expect(read.value.entries).toContainEqual( + expect.objectContaining({ path: "kept.txt", status: "read" }), + ); + const failed = read.value.entries.find( + (entry) => entry.status === "failed", + ); + expect(failed).toMatchObject({ + status: "failed", + code: "io", + message: expect.stringContaining("ENAMETOOLONG"), + }); + const imported = await importTree(root); + if (!imported.ok) throw new Error(imported.error.message); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ + path: "kept.txt", + content_state: "hashed", + }), + ); + const deep = imported.value.entries.find( + (entry) => entry.path === failed?.path, + ); + expect(deep?.limitations.join(" ")).toContain( + failed?.status === "failed" ? failed.message : "missing failure", + ); + } finally { + await deepTree("remove"); + } + }, + ); +}); diff --git a/tests/boundary/filesystem/referenceSourceImport.test.ts b/tests/boundary/filesystem/referenceSourceImport.test.ts index 8443dfbdd..f49ef6ba4 100644 --- a/tests/boundary/filesystem/referenceSourceImport.test.ts +++ b/tests/boundary/filesystem/referenceSourceImport.test.ts @@ -89,6 +89,90 @@ describe("reference source import projections", () => { }); describe("reference source symlink import", () => { + it.each([ + { target: "dist", reason: "default-ignored", pattern: "dist/" }, + { target: "generated", reason: "project-ignored", pattern: "generated/" }, + { target: "cache", reason: "caller-excluded", pattern: "cache/" }, + ])( + "retains links into a $reason directory without importing it", + async ({ target, reason, pattern }) => { + const root = await createTestTempDirectory( + "rea-reference-excluded-links-", + ); + await mkdir(join(root, target, "nested"), { recursive: true }); + await writeFile( + join(root, target, "nested", "output.js"), + "export const generated = true;\n", + ); + await writeFile(join(root, "source.ts"), "export const value = 1;\n"); + if (reason === "project-ignored") + await writeFile(join(root, ".gitignore"), `${pattern}\n`); + await symlink(target, join(root, "alias")); + await symlink(`${target}/nested/output.js`, join(root, "output-link")); + + const result = await importReferenceSource({ + root, + caller: "reference-import-test", + policy: { secretPatterns: [] }, + ...(reason === "caller-excluded" ? { excludePaths: [pattern] } : {}), + }); + if (!result.ok) throw result.error; + expect(result.value.inventory_state).toBe("partial"); + expect(result.value.exclusions).toContainEqual({ + path: target, + entry_kind: "directory", + reason, + pattern, + }); + expect(result.value.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "source.ts", + kind: "file", + content_state: "hashed", + }), + expect.objectContaining({ + path: "alias", + kind: "symlink", + target, + target_state: "internal", + }), + expect.objectContaining({ + path: "output-link", + kind: "symlink", + target: `${target}/nested/output.js`, + target_state: "internal", + }), + ]), + ); + expect( + result.value.entries.some( + ({ path }) => path === target || path.startsWith(`${target}/`), + ), + ).toBe(false); + }, + ); + + it("does not treat an excluded file as an excluded directory for a child link", async () => { + const root = await createTestTempDirectory( + "rea-reference-excluded-file-link-", + ); + await writeFile(join(root, "secret.txt"), "secret\n"); + await writeFile(join(root, "source.ts"), "export const value = 1;\n"); + await symlink("secret.txt/child", join(root, "alias")); + + const result = await importReferenceSource({ + root, + caller: "reference-import-test", + policy: { secretPatterns: [] }, + excludePaths: ["secret.txt"], + }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.code).toBe("parse"); + expect(result.error.message).toMatch(/recorded exclusion/u); + }); + it("preserves an unreadable target as unknown instead of a synthetic path", () => { const read: ReferenceSourceRead = { root: "/reference", @@ -160,6 +244,24 @@ describe("reference source manifest inventory", () => { ["out/widget_spec.js", "JavaScript", ["generated", "source", "test"]], ["vendor/widget_test.go", "Go", ["source", "test", "vendor"]], ["settings_spec.json", "JSON", ["config", "test"]], + [ + "docs/api/client.ts", + "TypeScript", + ["documentation", "generated", "source"], + ], + [ + "packages/widget/Docs/API/client.ts", + "TypeScript", + ["documentation", "generated", "source"], + ], + ["docs/apis/client.ts", "TypeScript", ["documentation", "source"]], + ["mydocs/api/client.ts", "TypeScript", ["source"]], + ["docs/manual/api/client.ts", "TypeScript", ["documentation", "source"]], + ["docs/api.ts", "TypeScript", ["documentation", "source"]], + ["public/client.ts", "TypeScript", ["generated", "source"]], + [".github", null, ["config"]], + [".husky", null, ["config"]], + [".gitignore", null, ["config"]], ] as const; await Promise.all( files.map(async ([path]) => { @@ -184,6 +286,11 @@ describe("reference source manifest inventory", () => { language, classifications, }); + for (const path of ["docs/api", "packages/widget/Docs/API"]) + expect(entries.get(path), path).toMatchObject({ + kind: "directory", + classifications: ["documentation", "generated"], + }); }); }); @@ -305,6 +412,7 @@ describe("reference source import behavior", () => { expect(left.value.parse_failures).toHaveLength(1); expect(left.value.exclusions).toContainEqual({ path: ".env", + entry_kind: "file", reason: "configured-secret", pattern: ".env", }); @@ -505,46 +613,55 @@ describe.skipIf(process.platform === "win32")( expect.arrayContaining([ { path: "project.tmp", + entry_kind: "file", reason: "project-ignored", pattern: "project.tmp", }, { path: "project.log", + entry_kind: "file", reason: "configured-secret", pattern: "project.log", }, { path: "keep.log", + entry_kind: "file", reason: "default-ignored", pattern: "*.log", }, { path: "dist", + entry_kind: "directory", reason: "default-ignored", pattern: "dist/", }, { path: "node_modules", + entry_kind: "directory", reason: "default-ignored", pattern: "node_modules/", }, { path: "empty-ignored", + entry_kind: "directory", reason: "project-ignored", pattern: "empty-ignored/", }, { path: "project-parent", + entry_kind: "directory", reason: "project-ignored", pattern: "project-parent/", }, { path: "caller-parent", + entry_kind: "directory", reason: "caller-excluded", pattern: "caller-parent/", }, { path: "secret-dir", + entry_kind: "directory", reason: "configured-secret", pattern: "secret-dir/", }, diff --git a/tests/boundary/filesystem/referenceSourceVcs.test.ts b/tests/boundary/filesystem/referenceSourceVcs.test.ts index dbfa9ff2b..3e46be0ce 100644 --- a/tests/boundary/filesystem/referenceSourceVcs.test.ts +++ b/tests/boundary/filesystem/referenceSourceVcs.test.ts @@ -1,4 +1,4 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { chmod, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -108,3 +108,27 @@ it("reads a linked worktree's own branch and detached HEAD through shared Git re dirty: null, }); }); + +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "keeps unreadable loose refs unknown instead of falling back to packed refs", + async () => { + const root = await createTestTempDirectory("rea-reference-loose-ref-"); + const gitdir = join(root, ".git"); + const looseRef = join(gitdir, "refs", "heads", "main"); + await mkdir(join(gitdir, "refs", "heads"), { recursive: true }); + const head = "1234567890abcdef1234567890abcdef12345678"; + await writeFile(join(gitdir, "HEAD"), "ref: refs/heads/main\n"); + await writeFile(join(gitdir, "packed-refs"), `${head} refs/heads/main\n`); + await writeFile(looseRef, `${head}\n`); + await chmod(looseRef, 0); + try { + expect(await readVcs(root)).toEqual({ + kind: "unknown", + head: null, + dirty: null, + }); + } finally { + await chmod(looseRef, 0o600); + } + }, +); From a6c3d3a365fae5fd41455f1a6eaad041caddfa50 Mon Sep 17 00:00:00 2001 From: Danish Rafie Ekaputra <155859586+0xRafie@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:31:16 +0700 Subject: [PATCH 08/26] docs(skill): route browser actions to scenario capture Route user browser actions to passive observation and agent actions to declared scenarios. Clarify requested artifacts, step outcomes, connect navigation and cleanup, and the limits of listener and request-initiator evidence. Distinguish discarded raw headers from retained response metadata. Update the bundled skill to version 42 with matching package metadata. Validation on the synchronized revision: docs (5 tests), typecheck/lint, formatting, 61 focused tests, and real Linux Chromium launch/connect and passive CLI probes. Verified declared steps, requested artifacts, digests, external page preservation, temporary form reset and cleanup. Native Windows/macOS and separate follow-up tools were not rerun. Package install/update and CLI/MCP verification passed earlier on 738d02a, before this main sync. Co-authored-by: 0xRafie <155859586+0xRafie@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../skills/reverse-engineer-anything/SKILL.md | 9 +++- .../references/runtime-observation.md | 45 ++++++++++++++++--- src/generatedPackageMetadata.ts | 2 +- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index f7e44e17a..b0b5b8b33 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "41" + version: "42" --- # REA @@ -104,6 +104,10 @@ explicit path or endpoint and do not need it. consult the NativeAOT workflow in `references/native-and-artifacts.md` for optional metadata recovery and its supported host/layout boundary. - User-owned browser page already open: `list_browser_targets`. +- Browser action the agent must perform (click, type, navigate): + `capture_browser_scenario` with declared actions. See + [runtime observation](references/runtime-observation.md) for connect/launch + and the focused follow-up browser tools. - Fiddler Everywhere traffic or a native `.saz` capture: read the [Fiddler workflow](references/runtime-observation.md#fiddler-everywhere-traffic-debugging). Inspect saved SAZ with `inspect_web_network_capture` and `format: "saz"`; @@ -323,7 +327,8 @@ as complete while required questions remain open. [references/javascript-applications.md](references/javascript-applications.md) - Android APK declarations, classes, methods, and static references: [references/android-applications.md](references/android-applications.md) -- Passive browser/Electron observation and static/runtime reconciliation: +- Browser/Electron observation (passive and scenario) and static/runtime + reconciliation: [references/runtime-observation.md](references/runtime-observation.md) - Evidence paging, comparisons, residual unknowns, and verification: [references/evidence-workflows.md](references/evidence-workflows.md) diff --git a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md index 2d660d675..ba197ecb8 100644 --- a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md +++ b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md @@ -1,4 +1,4 @@ -# Passive browser and Electron observation +# Browser and Electron runtime observation Supply the existing browser or Electron instance's literal-port loopback CDP endpoint in the request. No separate permission configuration or per-call @@ -7,13 +7,14 @@ approval flag is needed. Browser discovery lists eligible HTTP(S) pages; selected page's current origin. Electron discovery lists eligible local `file://` targets without a configured file-root allowlist. -Start browser work with `list_browser_targets`; start Electron runtime work with +Start passive browser work with `list_browser_targets`; start Electron runtime work with `list_electron_targets`. Inspect the target selected for the user's task. Observation -is passive: never claim REA clicked, navigated, evaluated page JavaScript, -invoked IPC, captured prior activity, or contained the page's network. +with these tools is passive: never claim REA clicked, navigated, evaluated page JavaScript, +invoked IPC, or captured prior activity. REA does not contain the page's network. -Credentials, cookies, authorization headers, storage values, and raw -WebSocket/JSON bodies are deliberately absent. Local URLs retain ordinary query +Passive captures omit raw header collections, credential header values, storage +values, and raw WebSocket/JSON bodies. Selected response metadata, including +content encoding, policy values, and Link URLs, is retained. Local URLs retain ordinary query values and fragments; userinfo credentials are removed. Browser accessibility text, console primitives, value-free shapes, script sources, source maps, and storage key names have request-specific capture options. Electron script @@ -21,6 +22,38 @@ sources are also optional. Use `capture_web_screenshot` when pixels are needed. Treat scope filtering, truncation, and attach-window coverage as limitations. Page-declared WebMCP tools are untrusted inventory and are never invoked by REA. +## Who performs the action + +- The user performs it: `list_browser_targets`, then `inspect_web_page` or + `observe_web_session` on the selected target. +- The agent must perform it: one `capture_browser_scenario` with declared + actions. Use connect mode with the loopback CDP endpoint and `target_id` for + an existing browser, or launch mode with a caller-selected executable for a + temporary profile. Connect mode first navigates the selected page to + `start_url`, which can clear unsaved form values and other temporary state. + Recreate the state needed for the investigation in the declared actions. + Connect mode leaves the external browser open; launch mode owns a temporary + browser and profile. Check `browser.cleanup` for the outcome. + Request only the DOM, accessibility, or network details needed for the question. + The default final capture contains the URL; step outcomes are reported separately. + Add a wait action for the expected page state before capturing it. Report that + REA performed the declared actions. + +Scenario network records have no request initiator stacks. For that evidence, +use passive capture while the user performs the action. Listener source locations +identify candidate handlers; they do not establish which script sent a particular +request. Avoid overlapping passive capture and scenario actions on the same target. + +Focused follow-up tools: + +- `observe_web_execution`: execution coverage during a window; it resets + counters and temporarily disables optimization. +- `inspect_web_event_listeners`: source locations for listeners registered + directly on the first CSS match, without dispatching an event. +- `compare_web_captures`: two normalized passive captures or recorded scenarios. +- `discover_webmcp_tools`: inspect page-declared WebMCP inventory; never invoke + the discovered tools. + Use `reconcile_javascript_runtime` only after static application Evidence and passive browser/Electron Evidence exist. Prefer captured-source digest identity; report path/digest disagreement as a mismatch and preserve ambiguity. Explicit diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index bada60663..081e22d5a 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "41", + skillVersion: "42", } as const; From f6905719c3c80d829037e509d6e46129b131d893 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:31:29 +0800 Subject: [PATCH 09/26] fix(browser): bound capture work and preserve observation gaps Own service-worker CDP registrations and retry failed teardown. Admit redaction and network metadata against remaining capture capacity, retain explicit omission reasons, and preserve source URL, redirect ancestry, caller-selected content, and historical capture values. Forward CLI cancellation through provider cleanup. Reviewed fixes: #2263, #2637, #2767, #2845, #2859, #2863, #2872, #2874, #2876, #2879, #2955, #2957, #2961, #3033. Validation: 25 current event/network source tests passed; earlier focused browser lanes passed 201 source tests and 176 boundary tests with two skips, plus 35 Python tests. Real browser/Windows lanes were not rerun. Playwright allocation before admission remains unbounded by these retention limits. --- bridge/mitmproxy/capture.py | 118 ++- bridge/mitmproxy/test_capture.py | 174 +++++ docs/browser-observation.md | 32 +- docs/browser-scenario-contract.md | 41 +- .../browser-capture-metadata-budget-e2e.mjs | 4 +- ...wser-scenario-service-worker-setup-e2e.mjs | 260 +++++++ .../browser-scenario-service-workers-e2e.mjs | 605 ++++++++++++++++ scripts/verify-real-browser.mjs | 7 + src/application/BrowserObservationService.ts | 17 +- src/browser/BrowserScenarioSecrets.ts | 22 + src/browser/CdpBrowserProvider.ts | 32 +- .../CdpCaptureDocuments.baseUrl.test.ts | 36 + src/browser/CdpCaptureDocuments.ts | 3 +- src/browser/CdpWebMcpDiscovery.ts | 28 +- .../PlaywrightElectronActiveProvider.test.ts | 21 + .../PlaywrightElectronActiveProvider.ts | 20 +- src/browser/PlaywrightScenarioEvents.test.ts | 156 +++- src/browser/PlaywrightScenarioEvents.ts | 104 ++- .../PlaywrightScenarioServiceWorkers.ts | 307 ++++++++ src/browser/PlaywrightScenarioSession.test.ts | 311 +++++++- src/browser/PlaywrightScenarioSession.ts | 107 ++- src/browser/WebSourceMapFetcher.ts | 16 +- src/browser/assets/LocalWebScriptArtifacts.ts | 5 + src/browser/history/CaptureRedaction.ts | 17 + src/browser/history/HarCapture.test.ts | 33 + src/browser/history/HarCapture.ts | 9 +- src/browser/history/QlogCapture.test.ts | 25 + src/browser/history/QlogCapture.ts | 20 +- src/browser/history/SazMetadata.test.ts | 24 + src/browser/history/SazMetadata.ts | 10 +- .../network/PlaywrightNetworkContent.ts | 43 +- .../network/PlaywrightScenarioNetwork.test.ts | 117 ++- .../network/PlaywrightScenarioNetwork.ts | 239 ++++-- src/captures/TsharkCaptureProvider.ts | 1 + src/cli/browserAdvancedCommands.ts | 78 +- src/cli/browserCommands.ts | 191 ++--- src/cliJavaScriptRuntimeCommands.ts | 103 +-- .../analysisView/pcapCaptureView.test.ts | 99 +++ src/domain/analysisView/pcapCaptureView.ts | 13 +- .../webNetworkCaptureView.test.ts | 683 ++++++++++++++++++ .../analysisView/webNetworkCaptureView.ts | 306 +++++++- src/domain/browserScenarioCapture.test.ts | 8 + src/domain/browserScenarioCaptureValues.ts | 196 ++--- src/domain/browserScenarioDiff.test.ts | 304 ++++++++ src/domain/browserScenarioDiff.ts | 197 ++++- src/domain/browserScenarioDiffEvidence.ts | 81 ++- src/domain/browserScenarioDiffValues.ts | 15 +- .../browserScenarioNetworkComparison.ts | 222 ++++++ src/domain/browserScenarioNormalization.ts | 38 +- src/domain/webCaptureDiff.ts | 58 +- src/domain/webMcpDiscovery.test.ts | 128 ++++ src/domain/webMcpDiscovery.ts | 3 + ...cdpBrowserProvider.document_script.test.ts | 126 ++-- .../browser/historicalWebCapture.test.ts | 62 +- tests/boundary/browser/webCaptureDiff.test.ts | 59 ++ .../browser/webSourceMapFetcher.test.ts | 28 + tests/boundary/cli/browserCli.test.ts | 9 +- tests/fixtures/fakeCdpBrowserEvents.ts | 283 ++++---- tests/fixtures/fakeCdpBrowserTypes.ts | 1 + tests/fixtures/webScriptCapture.ts | 1 + 60 files changed, 5539 insertions(+), 717 deletions(-) create mode 100644 bridge/mitmproxy/test_capture.py create mode 100644 scripts/lib/browser-scenario-service-worker-setup-e2e.mjs create mode 100644 scripts/lib/browser-scenario-service-workers-e2e.mjs create mode 100644 src/browser/PlaywrightScenarioServiceWorkers.ts create mode 100644 src/browser/history/SazMetadata.test.ts create mode 100644 src/domain/analysisView/pcapCaptureView.test.ts create mode 100644 src/domain/browserScenarioNetworkComparison.ts create mode 100644 src/domain/webMcpDiscovery.test.ts diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.py index dcd90e745..412306012 100644 --- a/bridge/mitmproxy/capture.py +++ b/bridge/mitmproxy/capture.py @@ -41,6 +41,87 @@ class BoundedReader: return self.handle.read(count) +SPECIAL_SCHEMES = {"ftp", "file", "http", "https", "ws", "wss"} +SCHEME_PREFIX = r"^[\x00-\x20]*([a-z][a-z0-9+.-]*):" +NETWORK_PREFIX = r"^[\x00-\x20]*[/\\]{2}" +URL_IGNORABLE_CONTROLS = "\t\n\r" +URL_CONTROL_TRANSLATION = {ord(char): None for char in URL_IGNORABLE_CONTROLS} + + +def authority_start(text, base_scheme=None): + """Return where an authority begins and whether it is special, or None. + + Follows WHATWG URL parsing as src/domain/recordedTransportUrl.ts does: + special schemes (and scheme-relative references) skip every leading "/" + or "\\" before the authority, so `https://\\user:pass@host` still has + userinfo. File URLs carry no userinfo and keep their "//" boundary. + """ + scheme = re.match(SCHEME_PREFIX, text, flags=re.I) + name = scheme.group(1).lower() if scheme is not None else None + if name is not None and name in SPECIAL_SCHEMES and name != "file": + # A same-scheme Location without two slash-equivalent separators is + # relative to the recorded request, so its @ belongs to the path. + following = text[scheme.end():scheme.end() + 2] + if name == base_scheme and (len(following) < 2 or any(char not in "/\\" for char in following)): + return None + start = scheme.end() + else: + if scheme is not None: + if text[scheme.end():scheme.end() + 2] != "//": + return None + return scheme.end() + 2, name in SPECIAL_SCHEMES + # A non-special base treats backslashes as authority data and only + # recognizes two forward slashes as a network reference. + if base_scheme is not None and base_scheme not in SPECIAL_SCHEMES: + prefix = re.match(r"^[\x00-\x20]*//", text) + return None if prefix is None else (prefix.end(), False) + prefix = re.match(NETWORK_PREFIX, text) + if prefix is None: + return None + start = prefix.end() + if base_scheme == "file": + return start, True + while start < len(text) and text[start] in "/\\": + start += 1 + return start, True + + +def without_userinfo(value, base_scheme=None): + """Remove an authority's userinfo using the WHATWG authority boundary. + + Special schemes and scheme-relative references also end the authority + at a backslash, so later path text containing "@" is not userinfo. + """ + binary = isinstance(value, bytes) + text = value.decode("latin-1") if binary else value + # WHATWG removes these controls before interpreting scheme/separators. + # Keep that parse view separate from the exact source spelling and bytes. + parsed_text = text.translate(URL_CONTROL_TRANSLATION) + located = authority_start(parsed_text, base_scheme) + if located is None: + return value + start, special = located + terminators = "/?#\\" if special else "/?#" + end = start + while end < len(parsed_text) and parsed_text[end] not in terminators: + end += 1 + at = parsed_text.rfind("@", start, end) + if at < start: + return value + parsed_offset = 0 + raw_start = 0 + for raw_offset, char in enumerate(text): + if char in URL_IGNORABLE_CONTROLS: + continue + if parsed_offset == start: + raw_start = raw_offset + if parsed_offset == at: + safe = text[:raw_start] + text[raw_offset + 1 :] + return safe.encode("latin-1") if binary else safe + parsed_offset += 1 + raise CaptureFailure("decoder", "Normalized URL authority could not be bound to its source.") + + def project_record(state, secrets): binaries, numbers, redactions = [], [], [] byte_secrets = [] @@ -58,16 +139,23 @@ def project_record(state, secrets): return None return text - def transport_value(value, path, transport_url): + def transport_value(value, path, transport_url, base_scheme): if not (transport_url or re.fullmatch(r"/(?:backup/)*request/(?:path|authority)", path)): return value, False binary = isinstance(value, bytes) - pattern = rb"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" if binary else r"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" - replacement = rb"\1" if binary else r"\1" separator = b"@" if binary else "@" - safe_url = re.sub(pattern, replacement, value, flags=re.I) - if path.endswith("/authority") and separator in safe_url: - safe_url = safe_url.rsplit(separator, 1)[1] + if transport_url: + safe_url = without_userinfo(value, base_scheme) + elif path.endswith("/authority"): + safe_url = value.rsplit(separator, 1)[-1] + else: + # Native request.path is origin-form unless it explicitly carries + # a scheme (absolute-form). Leading separators are path evidence, + # not a header URL reference's network authority. + text = value.decode("latin-1") if binary else value + safe_url = without_userinfo(value) if re.match( + SCHEME_PREFIX, text.translate(URL_CONTROL_TRANSLATION), flags=re.I + ) else value changed = safe_url != value if changed: redactions.append({"pointer": path, "reason": "transport-credential"}) @@ -80,7 +168,7 @@ def project_record(state, secrets): del numbers[before[1]:] del redactions[before[2]:] - def visit(value, path, depth=0, credential=False, transport_url=False): + def visit(value, path, depth=0, credential=False, transport_url=False, base_scheme=None): if depth > 64: raise CaptureFailure("input-limit", "Native capture exceeds the 64-level evidence nesting budget.", path) if credential: @@ -90,7 +178,7 @@ def project_record(state, secrets): binaries.append({"pointer": path, "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None}) return None if isinstance(value, bytes): - safe_bytes, credential_url = transport_value(value, path, transport_url) + safe_bytes, credential_url = transport_value(value, path, transport_url, base_scheme) content_base64 = base64.b64encode(value).decode("ascii") digest = hashlib.sha256(value).hexdigest() hidden = any(secret in value or secret in safe_bytes for secret in byte_secrets) or any(secret in content_base64 or secret in digest or secret in str(len(value)) for secret in secrets) @@ -107,7 +195,7 @@ def project_record(state, secrets): return None return safe_text if isinstance(value, str): - safe_text, _ = transport_value(value, path, transport_url) + safe_text, _ = transport_value(value, path, transport_url, base_scheme) return redact(safe_text, path, value) if isinstance(value, bool) or value is None: return value @@ -125,10 +213,16 @@ def project_record(state, secrets): for index, field in enumerate(value): if not isinstance(field, (list, tuple)) or len(field) != 2 or not isinstance(field[0], bytes): raise CaptureFailure("format", "Native header/trailer is not an ordered byte pair.", pointer(path, index)) - result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"})]) + result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"}, base_scheme if field[0].lower() == b"location" else None)]) return result - return [visit(item, pointer(path, index), depth + 1) for index, item in enumerate(value)] + return [visit(item, pointer(path, index), depth + 1, base_scheme=base_scheme) for index, item in enumerate(value)] if isinstance(value, dict): + if path == "" or re.fullmatch(r"/backup(?:/backup)*", path): + request = value.get("request") + scheme = request.get("scheme") if isinstance(request, dict) else None + if isinstance(scheme, bytes): + scheme = scheme.decode("latin-1") + base_scheme = scheme.lower() if isinstance(scheme, str) and re.fullmatch(r"[a-z][a-z0-9+.-]*", scheme, flags=re.I) else None result = {} for key, item in value.items(): if not isinstance(key, str): @@ -139,7 +233,7 @@ def project_record(state, secrets): validate_omitted(item, pointer(path, key), depth + 1) redactions.append({"pointer": path, "reason": "explicit-sensitive-value", "scope": "property-name"}) continue - result[key] = visit(item, pointer(path, key), depth + 1) + result[key] = visit(item, pointer(path, key), depth + 1, base_scheme=base_scheme) return result raise CaptureFailure("decoder", "Native parser returned an unexpected value type: " + type(value).__name__ + ".", path) diff --git a/bridge/mitmproxy/test_capture.py b/bridge/mitmproxy/test_capture.py new file mode 100644 index 000000000..f0a1106b1 --- /dev/null +++ b/bridge/mitmproxy/test_capture.py @@ -0,0 +1,174 @@ +"""Source-owned tests for the mitmproxy adapter's record projection.""" + +import importlib.util +import os +import sys +import types +import unittest +from unittest import mock + + +def load_capture(): + """Load the adapter with fresh mitmproxy stand-ins scoped to its import.""" + fake_mitmproxy = types.ModuleType("mitmproxy") + fake_mitmproxy.ctx = types.SimpleNamespace() + fake_mitmproxy.version = types.SimpleNamespace(VERSION="12.2.3") + fake_io = types.ModuleType("mitmproxy.io") + fake_io.tnetstring = types.ModuleType("mitmproxy.io.tnetstring") + stand_ins = { + "mitmproxy": fake_mitmproxy, + "mitmproxy.io": fake_io, + "mitmproxy.io.tnetstring": fake_io.tnetstring, + } + spec = importlib.util.spec_from_file_location( + "rea_mitmproxy_capture", os.path.join(os.path.dirname(__file__), "capture.py") + ) + module = importlib.util.module_from_spec(spec) + with mock.patch.dict(sys.modules, stand_ins): + spec.loader.exec_module(module) + return module + + +capture = load_capture() + + +def location(value, scheme=None): + state = {"response": {"headers": [[b"Location", value]]}} + if scheme is not None: + state["request"] = {"scheme": scheme} + record = capture.project_record(state, []) + sidecar = next( + field for field in record["binary_fields"] if field["pointer"] == "/response/headers/0/1" + ) + return record["reported"]["response"]["headers"][0][1], sidecar, record["redactions"] + + +class TransportUserinfoTests(unittest.TestCase): + def test_ignorable_controls_cannot_hide_authority_credentials(self): + for control in [b"\t", b"\n", b"\r"]: + for value in [b"https://" + control + b"/user:pass@host/path", b"ht" + control + b"tps://user:" + control + b"pass@host/path"]: + with self.subTest(value=value): + text, sidecar, redactions = location(value, b"https") + self.assertNotIn("user", text) + self.assertNotIn("pass", text) + self.assertEqual(sidecar["state"], "redacted") + self.assertEqual(len(redactions), 1) + + def test_mixed_network_separators_remove_credentials(self): + for prefix in [b"\\\\", b"/\\", b"\\/"]: + with self.subTest(prefix=prefix): + text, sidecar, redactions = location(prefix + b"user:pass@host/path", b"https") + self.assertEqual(text, prefix.decode("ascii") + "host/path") + self.assertEqual(sidecar["state"], "redacted") + self.assertEqual(len(redactions), 1) + + def test_non_special_base_keeps_backslashes_inside_network_authorities(self): + text, sidecar, redactions = location(b"//user\\name:pass@host/x", b"foo") + self.assertEqual(text, "//host/x") + self.assertEqual(sidecar["state"], "redacted") + self.assertEqual(len(redactions), 1) + text, sidecar, redactions = location(b"/\\invoice@customer", b"foo") + self.assertEqual(text, "/\\invoice@customer") + self.assertEqual(sidecar["state"], "retained") + self.assertEqual(redactions, []) + + def test_native_request_paths_preserve_origin_form_and_redact_absolute_form(self): + for value in [b"/\\invoice@customer", b"//invoice@customer", b"\\\\invoice@customer"]: + with self.subTest(value=value): + record = capture.project_record({"request": {"path": value}}, []) + self.assertEqual(record["reported"]["request"]["path"], value.decode("ascii")) + self.assertEqual(record["binary_fields"][0]["state"], "retained") + self.assertEqual(record["redactions"], []) + record = capture.project_record({"request": {"path": b"https://user:pass@host/x"}}, []) + self.assertEqual(record["reported"]["request"]["path"], "https://host/x") + self.assertEqual(record["binary_fields"][0]["state"], "redacted") + + def test_same_scheme_locations_preserve_relative_path_text_and_bytes(self): + for value in [b"https:invoice@customer", b"https:/invoice@customer", b"https:\\invoice@customer"]: + with self.subTest(value=value): + text, sidecar, redactions = location(value, b"https") + self.assertEqual(text, value.decode("ascii")) + self.assertEqual(sidecar["state"], "retained") + self.assertEqual(redactions, []) + + def test_location_authorities_still_remove_credentials_with_a_base_scheme(self): + for value, scheme, expected in [ + (b"https://user:pass@host/path", b"https", "https://host/path"), + (b"https:\\\\user:pass@host/path", b"https", "https:\\\\host/path"), + (b"https:user:pass@host/path", b"http", "https:host/path"), + ]: + with self.subTest(value=value, scheme=scheme): + text, sidecar, redactions = location(value, scheme) + self.assertEqual(text, expected) + self.assertEqual(sidecar["state"], "redacted") + self.assertEqual(len(redactions), 1) + + def test_backup_location_uses_its_own_request_scheme(self): + record = capture.project_record({ + "request": {"scheme": b"http"}, + "backup": { + "request": {"scheme": b"https"}, + "response": {"headers": [[b"Location", b"https:invoice@customer"]]}, + }, + }, []) + self.assertEqual(record["reported"]["backup"]["response"]["headers"][0][1], "https:invoice@customer") + self.assertEqual(record["redactions"], []) + + def test_backslash_ends_a_special_scheme_authority(self): + text, sidecar, redactions = location(b"https://example.test\\part@tail") + self.assertEqual(text, "https://example.test\\part@tail") + self.assertEqual(sidecar["state"], "retained") + self.assertIsNotNone(sidecar["content_base64"]) + self.assertEqual(redactions, []) + + def test_userinfo_before_a_backslash_is_still_removed(self): + text, sidecar, redactions = location(b"https://user:pw@example.test\\path") + self.assertEqual(text, "https://example.test\\path") + self.assertEqual(sidecar["state"], "redacted") + self.assertEqual( + redactions, + [{"pointer": "/response/headers/0/1", "reason": "transport-credential"}], + ) + + def test_ordinary_userinfo_is_removed(self): + text, _sidecar, _redactions = location(b"https://user:pw@example.test/a@b") + self.assertEqual(text, "https://example.test/a@b") + + def test_empty_userinfo_marker_is_removed(self): + self.assertEqual( + capture.without_userinfo("https://@host"), "https://host" + ) + + def test_special_schemes_skip_leading_slashes_before_userinfo(self): + for value, expected in [ + (b"https://\\user:pass@host/path", b"https://\\host/path"), + (b"https:\\\\user:pass@host/path", b"https:\\\\host/path"), + (b"https:user:pass@host", b"https:host"), + (b"//\\user:pass@host", b"//\\host"), + ]: + with self.subTest(value=value): + self.assertEqual(capture.without_userinfo(value), expected) + text, sidecar, redactions = location(b"https://\\user:pass@host/path") + self.assertEqual(text, "https://\\host/path") + self.assertEqual(sidecar["state"], "redacted") + self.assertEqual(len(redactions), 1) + + def test_file_base_keeps_paths_after_the_fixed_network_boundary(self): + for value in [b"///u@h", b"////u@h"]: + with self.subTest(value=value): + text, sidecar, redactions = location(value, b"file") + self.assertEqual(text, value.decode("ascii")) + self.assertEqual(sidecar["state"], "retained") + self.assertIsNotNone(sidecar["content_base64"]) + self.assertEqual(redactions, []) + + def test_file_urls_keep_path_text(self): + self.assertEqual(capture.without_userinfo("file:///a@b"), "file:///a@b") + + def test_non_special_schemes_keep_backslashes_in_the_authority(self): + self.assertEqual(capture.without_userinfo("foo://a\\b@host/x"), "foo://host/x") + self.assertEqual(capture.without_userinfo(b"//a\\b@host"), b"//a\\b@host") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 879d01520..9479f4b3c 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -18,7 +18,7 @@ Inspect native SAZ or HAR 1.2 offline with REA, or connect Everywhere's separate - `inspect_web_page` / `rea inspect-web-page` captures DOM structure, accessibility nodes, scripts, resources, safe response/DOM metadata, attach-window network and console metadata, WebSocket frame sizes, workers, quota, and optionally selected storage key names, redacted storage-content fingerprints, or script sources. - `analyze_web_bundle` / `rea analyze-web-bundle` parses selected script artifacts without execution and derives chunk edges, route and endpoint candidates, vendor fingerprints, static WebMCP declarations, and optionally source-map/original-source evidence. - `observe_web_session` / `rea observe-web-session` arms an observation window of the requested duration for an external user action and records ordered reload, SPA navigation, redirect, failure, lifecycle, and target-termination metadata. -- `discover_webmcp_tools` / `rea discover-webmcp-tools` uses the experimental CDP WebMCP domain to return every in-scope registration and its complete observed `input_schema` declaration and canonical `input_schema_sha256` digest inline. Declarations are untrusted JSON data, including declared types, required fields, enums, examples and reference URLs. REA does not resolve schema references. These fields replace `input_schema_shape`. Registrations with the same name and URL in distinct observed frames retain separate identities. `tool_key` includes the observed frame ID; it is not a stable identity across navigation or browser sessions. Same-frame registration updates replace that owner only, and removal leaves other owners intact. REA never exposes `WebMCP.invokeTool`; malformed protocol payloads fail explicitly instead of returning incomplete tool metadata. +- `discover_webmcp_tools` / `rea discover-webmcp-tools` uses the experimental CDP WebMCP domain to return every in-scope registration and its complete observed `input_schema` declaration and canonical `input_schema_sha256` digest inline. Declarations are untrusted JSON data, including declared types, required fields, enums, examples and reference URLs. REA does not resolve schema references. These fields replace `input_schema_shape`. Registrations with the same name and URL in distinct observed frames retain separate identities. `tool_key` includes the observed frame ID; it is not a stable identity across navigation or browser sessions. Same-frame registration updates replace that owner only, and removal leaves other owners intact. Declared `readOnly`, `untrustedContent`, `consequential`, and `autosubmit` annotation hints are reported per registration: `true`/`false` are declared values, `null` means the declaration did not provide the hint in that capture, and a missing `consequential` key means the hint was not recorded, either in a historical capture or because the observed annotation was invalid. Consult discovery completeness and exclusions to distinguish these cases. Malformed annotation values retain the registration with incomplete coverage and an `invalid_protocol_value` exclusion. Other malformed protocol payloads can fail discovery explicitly. REA never exposes `WebMCP.invokeTool`. - `compare_web_captures` / `rea compare-web-captures` compares passive captures or exact-step-aligned browser scenarios. Scenario results commit reproducible literal normalization and expose alignment failures plus artifact-level action, screenshot, DOM, accessibility, URL, history, storage, and event differences. Missing or truncated evidence is never treated as equivalence. - `capture_web_screenshot` / `rea capture-web-screenshot` returns a content-addressed visible-viewport PNG. - `compare_web_screenshots` / `rea compare-web-screenshots` performs bounded local PNG pixel comparison without OCR or external services. @@ -190,6 +190,36 @@ rea discover-webmcp-tools http://127.0.0.1:9222 TARGET_ID \ --json ``` +### WebMCP discovery prerequisites + +`discover_webmcp_tools` reports only what the attached browser's experimental +`WebMCP` CDP domain exposes. When a browser rejects the unknown domain, the +result reports `status: "unavailable"`. Whether a page can register tools at +all depends on several conditions that are not interchangeable: + +- A site's own origin-trial token can enable the API for that origin + **provided the trial is still active, the browser version and platform + support it, and user settings permit it** — token presence alone is not a + guarantee. The trial is announced for Chrome 149 and later. +- The `WebMCPTesting` feature — `--enable-features=WebMCPTesting` at launch, or + `chrome://flags/#enable-webmcp-testing` in a normal profile — enables the + API page-side for that browser instance, independent of per-site tokens. + In one pilot run, a site whose registration code was present but whose + served token was scoped for third-party use registered 0 tools under a + default launch and 4 under the flag; that the token did not enroll the + first-party document is consistent with this but is an inference, not a + measured Chrome behavior. + +In Chrome 154 specifically (one tested version), `WebMCP.enable` answered +under a default launch — the debug domain was not flag-gated in that run. +Other versions may differ, so an unattributed zero-tool result should be +read as registration state only after domain availability is confirmed. + +Discovery is attach-only: the command needs a reachable loopback CDP endpoint +and a target ID from `list_browser_targets`, and it never navigates, reloads, +or evaluates the page. Observe the page in the state you want to measure; +registered tools reflect the document as loaded. + Accessibility names/descriptions, console primitive text, JSON body shapes, WebSocket shapes, script content, and storage key names are selected independently: ```bash diff --git a/docs/browser-scenario-contract.md b/docs/browser-scenario-contract.md index 403865be8..a581129b7 100644 --- a/docs/browser-scenario-contract.md +++ b/docs/browser-scenario-contract.md @@ -15,6 +15,12 @@ there is no separate origin allowlist or request-interception layer. Environment settings (including service workers blocked by default), empty storage, and a final sanitized URL capture are supplied by default. Set `environment.service_workers` to `allow` when the application requires them. +For attached browsers, `block` rejects captures when the selected page or its +frames are controlled by a service worker or already have a registration, +including after navigation. Existing caller registrations are never removed. +Navigation can run destination code before a newly discovered worker is rejected; +that rejection does not roll back browser effects. REA removes its owned +future-document scripts, restores live frame methods, then disconnects. Add secret declarations, storage seeds, and additional artifact or event capture only when needed. Action, secret, and storage counts are not capped. Duration, action, and navigation @@ -249,8 +255,12 @@ Each observed request object receives a capture-scoped `transaction_id`. Requests sharing a URL remain distinct. Redirects link to the preceding transaction only when it was observed; a null predecessor does not establish that no earlier hop existed. Request events require both transaction fields; -an absent ID is malformed capture data, while a null redirect predecessor is -an explicit unknown. +an absent ID is malformed capture data. `redirected_from_transaction_state` +records `root` when Playwright observed no predecessor and `unknown` when a +predecessor exists but was outside the capture. A linked predecessor is carried +by `redirected_from_transaction_id` with no state field. Older captures without +the state field retain the conservative meaning that a null predecessor is +unknown. `scenario.network_content` records the resolved content selection. Comparing captures with different selections reports network evidence as unknown with a @@ -284,8 +294,10 @@ browser cleanup, with a provider-owned five-second timeout per read and cancellation support. Failed or unexposed reads retain per-content states and make selected event coverage incomplete. Body events can arrive during later steps or final drain; event bounds describe receipt during the step, while the source event -reference establishes the transaction phase. Comparison attributes content to -its source event's step and excludes the sequence reference from normalization. +reference establishes the transaction phase. Comparison attributes content, like +every other event of its transaction, to the step that sent the request (the +receipt step when that request was not observed) and excludes the sequence +reference from normalization. Playwright scenario capture does not supply initiator stacks; passive CDP observation retains its own initiator evidence. @@ -300,7 +312,22 @@ an unchanged claim; observed differences may still prove a changed result. The comparison records its full normalization commitment and SHA-256 digest. Built-in rules exclude step elapsed time and event sequence/index fields, compare screenshots by content digest, and compare DOM/accessibility captures -by normalized text. Optional caller rules are bounded exact-literal +by normalized text. Network events are grouped by transaction without their +capture-local transaction IDs, attributed to the step that sent the request, +and compared as an unordered collection after caller rules apply, because +concurrent responses arrive in any order and can land in a later step; a +redirect retains the complete predecessor ancestry, including each hop's own +request, response outcome, and selected network content. Each transaction's +capture-local IDs, sequence/index fields, and content source-sequence anchor +are removed. Caller rules normalize retained strings before hashing. The +`preserve_redirect_transaction_ancestry` built-in rule fingerprints each hop +as canonical SHA-256 of `{ events, redirected_from }`: `events` is the sorted +list of its normalized event SHA-256 digests, and `redirected_from` is its +predecessor fingerprint or null for a root. Selected transaction fingerprints +are sorted for the step's network comparison. Temporary numeric graph indices +are lookup references only and do not enter the final digest. Missing or cyclic +predecessor evidence yields unknown comparability instead of an unchanged claim. +Other events keep receipt order. Optional caller rules are bounded exact-literal replacements over named artifact kinds. Rules and artifact-kind lists are sorted canonically before application, making the same policy reproducible regardless of input order. Rules operate only on already-redacted durable @@ -318,3 +345,7 @@ selection. Captures with different selections report event evidence as unknown with a recovery instruction, even when no selected family emitted an event. Captures written before this field existed omit it, so their event evidence is also unknown. + +Network normalization also commits the removal of capture-local transaction IDs, +attribution to the requesting step, and unordered transaction comparison. These +built-in rule identities participate in the normalization digest. diff --git a/scripts/lib/browser-capture-metadata-budget-e2e.mjs b/scripts/lib/browser-capture-metadata-budget-e2e.mjs index 26b695371..26e339b25 100644 --- a/scripts/lib/browser-capture-metadata-budget-e2e.mjs +++ b/scripts/lib/browser-capture-metadata-budget-e2e.mjs @@ -54,10 +54,12 @@ export async function verifyBrowserCaptureMetadataBudget(endpoint) { if (address === null || typeof address === "string") throw new Error("Browser metadata fixture did not bind a TCP port"); fixtureOrigin = `http://127.0.0.1:${String(address.port)}`; + // Each independent capture opens a new Debugger session. Keep each script + // reachable so V8 cannot collect it before the later sessions attach. const annotations = Array.from( { length: SCRIPT_COUNT }, (_, index) => - ``, + ``, ).join(""); page = `${annotations}`; const fixtureUrl = `${fixtureOrigin}/capture-metadata-budget`; diff --git a/scripts/lib/browser-scenario-service-worker-setup-e2e.mjs b/scripts/lib/browser-scenario-service-worker-setup-e2e.mjs new file mode 100644 index 000000000..ace6fc567 --- /dev/null +++ b/scripts/lib/browser-scenario-service-worker-setup-e2e.mjs @@ -0,0 +1,260 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { createServer, request } from "node:http"; +import { connect } from "node:net"; + +import { createBrowserScenarioProvider } from "../../dist/composition/browserScenario.js"; +import { browserScenarioSchema } from "../../dist/domain/browserScenario.js"; +import { PlaywrightScenarioServiceWorkers } from "../../dist/browser/PlaywrightScenarioServiceWorkers.js"; + +const beforeDeadline = async (promise, label, timeoutMs = 2_000) => { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} did not settle`)), + timeoutMs, + ); + }), + ]); + } finally { + clearTimeout(timer); + } +}; + +// Real loopback proxy: Playwright's first transport is forwarded to Chrome, +// then only the policy owner's discovery/handshake is deliberately withheld. +const startProxy = async (endpoint, mode, ownerOnly = false) => { + const upstream = new URL(endpoint); + const sockets = new Set(); + let versions = 0; + let upgrades = 0; + let enabled = true; + let entered; + const stalled = new Promise((resolve) => { + entered = resolve; + }); + let closed; + let stalledClose; + const stall = (socket) => { + socket.resume(); + socket.once("end", () => socket.end()); + stalledClose = new Promise((resolve) => { + closed = resolve; + }); + socket.once("close", () => closed()); + entered(); + }; + const server = createServer((incoming, response) => { + const versionRequest = incoming.url.replace(/\/$/, "") === "/json/version"; + if (versionRequest) versions += 1; + if ( + enabled && + mode === "http-discovery" && + versions === 2 && + versionRequest + ) { + stall(incoming.socket); + return; + } + const outgoing = request(new URL(incoming.url, upstream), (received) => { + const chunks = []; + received.on("data", (chunk) => chunks.push(chunk)); + received.on("end", () => { + const content = Buffer.concat(chunks).toString("utf8"); + response.writeHead(received.statusCode, { + "content-type": "application/json", + }); + response.end( + content.replaceAll( + `127.0.0.1:${upstream.port}`, + `127.0.0.1:${server.address().port}`, + ), + ); + }); + }); + outgoing.on("error", () => response.destroy()); + response.on("close", () => outgoing.destroy()); + outgoing.end(); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + }); + server.on("upgrade", (incoming, socket, head) => { + upgrades += 1; + if ( + enabled && + mode !== "http-discovery" && + upgrades === (ownerOnly ? 1 : 2) + ) { + stall(socket); + return; + } + const outgoing = connect(Number(upstream.port), "127.0.0.1", () => { + const headers = incoming.rawHeaders.flatMap((value, index, all) => + index % 2 === 0 ? [`${value}: ${all[index + 1]}`] : [], + ); + outgoing.write( + `${incoming.method} ${incoming.url} HTTP/${incoming.httpVersion}\r\n${headers.join("\r\n")}\r\n\r\n`, + ); + if (head.length > 0) outgoing.write(head); + outgoing.pipe(socket); + socket.pipe(outgoing); + }); + outgoing.on("error", () => socket.destroy()); + socket.on("error", () => outgoing.destroy()); + socket.on("close", () => outgoing.destroy()); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const origin = `http://127.0.0.1:${server.address().port}`; + let proxyEndpoint = origin; + if (mode === "browser-websocket") { + const version = await (await fetch(`${endpoint}/json/version`)).json(); + proxyEndpoint = version.webSocketDebuggerUrl.replace( + `127.0.0.1:${upstream.port}`, + `127.0.0.1:${server.address().port}`, + ); + } + return { + endpoint: proxyEndpoint, + stalled, + waitClosed: () => stalledClose, + resume: () => { + enabled = false; + }, + close: async () => { + const done = once(server, "close"); + server.close(); + for (const socket of sockets) socket.destroy(); + await done; + }, + }; +}; + +export const verifyScenarioPolicySetup = async ( + endpoint, + targetId, + origin, + page, +) => { + const result = {}; + for (const mode of ["http-websocket", "http-discovery", "deadline"]) { + const proxy = await startProxy(endpoint, mode); + const provider = createBrowserScenarioProvider(process.env); + const scenario = browserScenarioSchema.parse({ + browser: { + mode: "connect", + cdp_endpoint: proxy.endpoint, + target_id: targetId, + }, + start_url: { url: origin }, + environment: { service_workers: "block" }, + actions: [ + { step_id: "settle", action: "wait_for_timeout", duration_ms: 1 }, + ], + capture: { events: [], after_each_step: ["dom"], at_end: ["dom"] }, + }); + const cancellation = new AbortController(); + let failure; + try { + const pending = provider.captureScenario(scenario, { + signal: cancellation.signal, + }); + await beforeDeadline(proxy.stalled, `${mode} setup`); + if (mode !== "deadline") cancellation.abort(); + const cancelled = await beforeDeadline( + pending, + `${mode} termination`, + mode === "deadline" ? 12_000 : 2_000, + ); + assert.equal(cancelled.ok, false); + assert.equal(cancelled.error.cleanupIncomplete, false); + if (mode === "deadline") assert.equal(cancelled.error.reason, "timeout"); + await beforeDeadline(proxy.waitClosed(), `${mode} socket release`); + assert.equal( + await page.evaluate(() => + navigator.serviceWorker.register.toString().includes("[native code]"), + ), + true, + ); + proxy.resume(); + const next = await beforeDeadline( + provider.captureScenario(scenario), + `${mode} target reacquisition`, + ); + if (!next.ok) throw next.error; + assert.equal(next.value.browser.cleanup, "disconnected-external"); + result[mode] = { + ...(mode === "deadline" ? { timed_out: true } : { cancelled: true }), + socket_closed: true, + target_reused: true, + }; + } catch (cause) { + failure = cause; + throw cause; + } finally { + cancellation.abort(); + try { + await proxy.close(); + } finally { + try { + await provider.close(); + } catch (cleanupFailure) { + if (failure !== undefined) + throw new AggregateError([failure, cleanupFailure]); + throw cleanupFailure; + } + } + } + } + const proxy = await startProxy(endpoint, "browser-websocket", true); + const policy = new PlaywrightScenarioServiceWorkers( + page, + proxy.endpoint, + targetId, + ); + try { + const rejected = assert.rejects( + policy.initialize(), + (error) => error._tag === "AnalysisCancelledError", + ); + await beforeDeadline(proxy.stalled, "browser websocket approval handshake"); + await beforeDeadline(policy.close(), "browser websocket cleanup"); + await rejected; + await beforeDeadline( + proxy.waitClosed(), + "browser websocket socket release", + ); + proxy.resume(); + const next = new PlaywrightScenarioServiceWorkers( + page, + proxy.endpoint, + targetId, + ); + try { + await beforeDeadline( + next.initialize(), + "browser websocket reacquisition", + ); + } finally { + await next.close(); + } + result["browser-websocket"] = { + cancelled: true, + socket_closed: true, + target_reused: true, + direct_owner: true, + }; + } finally { + try { + await policy.close(); + } finally { + await proxy.close(); + } + } + return result; +}; diff --git a/scripts/lib/browser-scenario-service-workers-e2e.mjs b/scripts/lib/browser-scenario-service-workers-e2e.mjs new file mode 100644 index 000000000..3af2f4f8d --- /dev/null +++ b/scripts/lib/browser-scenario-service-workers-e2e.mjs @@ -0,0 +1,605 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import { setTimeout as delay } from "node:timers/promises"; + +import { chromium } from "playwright-core"; + +import { PlaywrightScenarioSession } from "../../dist/browser/PlaywrightScenarioSession.js"; +import { PlaywrightScenarioServiceWorkers } from "../../dist/browser/PlaywrightScenarioServiceWorkers.js"; +import { createBrowserScenarioProvider } from "../../dist/composition/browserScenario.js"; +import { browserScenarioSchema } from "../../dist/domain/browserScenario.js"; +import { runScenarioCli } from "./browser-scenario-verifier.mjs"; +import { verifyScenarioPolicySetup } from "./browser-scenario-service-worker-setup-e2e.mjs"; + +const startSite = async () => { + let otherOrigin; + const server = createServer((request, response) => { + const path = new URL(request.url, "http://localhost").pathname; + if (path === "/sw.js") { + response.setHeader("content-type", "application/javascript"); + response.end(`self.addEventListener("install", () => self.skipWaiting()); + self.addEventListener("activate", event => event.waitUntil(clients.claim())); + self.addEventListener("fetch", event => { + if (new URL(event.request.url).pathname === "/probe") + event.respondWith(new Response("worker")); + });`); + } else if (path === "/probe") { + response.end("network"); + } else if (path === "/failed-navigation") { + response.destroy(); + } else if (path === "/stalled-image.png") { + response.writeHead(200, { "content-type": "image/png" }); + response.flushHeaders(); + } else if (path === "/pending-image" || path === "/pending-child") { + response.setHeader("content-type", "text/html"); + response.end( + '', + ); + } else if (path === "/pending-frame") { + response.setHeader("content-type", "text/html"); + response.end( + '', + ); + } else { + response.setHeader("content-type", "text/html"); + response.end(`

ready

+ Navigate + ${path === "/frame" ? `` : ""} + ${ + path === "/register" + ? `` + : "" + }`); + } + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + return { + origin: `http://127.0.0.1:${server.address().port}`, + setOtherOrigin: (origin) => { + otherOrigin = origin; + }, + close: async () => { + const closed = once(server, "close"); + server.close(); + server.closeAllConnections(); + await closed; + }, + }; +}; + +/** Real attached-page policy checks, with caller-owned workers on a second origin. */ +export async function verifyScenarioServiceWorkers(endpoint, targetId) { + const first = await startSite(); + const second = await startSite(); + first.setOtherOrigin(second.origin); + second.setOtherOrigin(first.origin); + const observer = await chromium.connectOverCDP(endpoint); + let provider = createBrowserScenarioProvider(process.env); + let session; + let failure; + try { + let page; + for (const context of observer.contexts()) { + for (const candidate of context.pages()) { + const cdp = await context.newCDPSession(candidate); + const { targetInfo } = await cdp.send("Target.getTargetInfo"); + await cdp.detach(); + if (targetInfo.targetId === targetId) page = candidate; + } + } + assert.ok(page, "External fixture page disappeared"); + const scenarioFor = (url, actions = [], policy = "block") => + browserScenarioSchema.parse({ + browser: { + mode: "connect", + cdp_endpoint: endpoint, + target_id: targetId, + }, + start_url: { url }, + environment: { service_workers: policy }, + actions: + actions.length === 0 + ? [ + { + step_id: "settle", + action: "wait_for_timeout", + duration_ms: 1, + }, + ] + : actions, + capture: { events: [], after_each_step: ["dom"], at_end: ["dom"] }, + }); + const assertRestored = async () => { + assert.equal(page.isClosed(), false); + assert.equal( + await page.evaluate(() => + ServiceWorkerContainer.prototype.register + .toString() + .includes("[native code]"), + ), + true, + ); + }; + const assertRejected = async (scenario) => { + const result = await provider.captureScenario(scenario); + assert.equal( + result.ok, + false, + "Worker-influenced capture unexpectedly succeeded", + ); + assert.equal(result.error.reason, "target_not_allowed"); + assert.equal( + result.error.partialObservation.capture.browser.cleanup, + "disconnected-external", + ); + await assertRestored(); + }; + + // A fresh attached origin blocks registration, then restores both the + // current document and future documents when its transport disconnects. + const clean = scenarioFor(`${first.origin}/register`, [ + { + step_id: "registration-rejected", + action: "wait_for", + locator: { kind: "css", selector: "#state[data-blocked='true']" }, + state: "visible", + }, + ]); + const cleanCapture = (await runScenarioCli(clean)).normalized_result; + assert.ok(cleanCapture.steps.every(({ status }) => status === "completed")); + assert.equal(cleanCapture.browser.cleanup, "disconnected-external"); + await assertRestored(); + await page.reload(); + await page.getByText("registered", { exact: true }).waitFor(); + await page.evaluate(async () => { + for (const registration of await navigator.serviceWorker.getRegistrations()) + await registration.unregister(); + }); + await page.goto("about:blank"); + + // The caller installs a real worker at a different origin, then moves the + // selected target away before REA attaches. Its registration must survive. + await page.goto(second.origin); + await page.evaluate(async () => { + await navigator.serviceWorker.register("/sw.js"); + await navigator.serviceWorker.ready; + }); + await page.reload(); + assert.equal( + await page.evaluate(async () => (await fetch("/probe")).text()), + "worker", + ); + await page.goto("about:blank"); + await assertRejected(scenarioFor(second.origin)); + await page.goto("about:blank"); + await assertRejected( + scenarioFor(first.origin, [ + { + step_id: "cross-origin", + action: "goto", + destination: { url: second.origin }, + wait_until: "load", + }, + ]), + ); + await page.goto("about:blank"); + await assertRejected( + scenarioFor(first.origin, [ + { + step_id: "link-navigation", + action: "click", + locator: { kind: "css", selector: "#navigate" }, + }, + { step_id: "loaded", action: "wait_for_timeout", duration_ms: 100 }, + ]), + ); + await page.goto("about:blank"); + await assertRejected(scenarioFor(`${first.origin}/frame`)); + + const allowed = await provider.captureScenario( + scenarioFor(second.origin, [], "allow"), + ); + if (!allowed.ok) throw allowed.error; + assert.ok( + allowed.value.steps.every(({ status }) => status === "completed"), + ); + assert.equal( + await page.evaluate(async () => (await fetch("/probe")).text()), + "worker", + ); + assert.equal( + await page.evaluate( + async () => (await navigator.serviceWorker.getRegistrations()).length, + ), + 1, + ); + + // Restoration also runs after navigation failure and active cancellation. + await page.goto("about:blank"); + const failed = await provider.captureScenario( + scenarioFor(`${first.origin}/failed-navigation`), + ); + assert.equal(failed.ok, false); + // Chromium completes its error-document transition after goto rejects. + await page.waitForSelector("#main-frame-error", { state: "attached" }); + // A context replacement can defer restoration. Retained cleanup must + // restore the replacement document before releasing the external page. + await provider.close(); + provider = createBrowserScenarioProvider(process.env); + await page.goto(first.origin); + await assertRestored(); + const cancellation = new AbortController(); + const pending = provider.captureScenario( + scenarioFor(`${first.origin}/register`, [ + { + step_id: "wait", + action: "wait_for", + locator: { kind: "css", selector: "#absent" }, + state: "visible", + timeout_ms: 30_000, + }, + ]), + { signal: cancellation.signal }, + ); + try { + const deadline = Date.now() + 10_000; + while ((await page.locator("#state").textContent()) !== "blocked") { + assert.ok( + Date.now() < deadline, + "Scenario never applied registration policy", + ); + await delay(25); + } + } finally { + cancellation.abort(); + } + const cancelled = await pending; + assert.ok( + !cancelled.ok || + cancelled.value.steps.some(({ status }) => status === "cancelled"), + ); + await assertRestored(); + + // Caller changes made after installation belong to the caller, too. + session = await PlaywrightScenarioSession.open( + scenarioFor(first.origin), + process.env, + {}, + ); + await page.evaluate(() => { + ServiceWorkerContainer.prototype.register = + function callerReplacement() {}; + }); + await session.close(); + session = undefined; + assert.equal( + await page.evaluate(() => ServiceWorkerContainer.prototype.register.name), + "callerReplacement", + ); + await page.reload(); + await assertRestored(); + await page.evaluate(() => { + delete ServiceWorkerContainer.prototype.register; + }); + const policyBrowser = await chromium.connectOverCDP(endpoint); + try { + let policyPage; + for (const context of policyBrowser.contexts()) { + for (const candidate of context.pages()) { + const cdp = await context.newCDPSession(candidate); + const { targetInfo } = await cdp.send("Target.getTargetInfo"); + await cdp.detach(); + if (targetInfo.targetId === targetId) policyPage = candidate; + } + } + assert.ok(policyPage); + const policy = new PlaywrightScenarioServiceWorkers( + policyPage, + endpoint, + targetId, + ); + await policy.initialize(); + await policy.close(); + } finally { + await policyBrowser.close(); + } + assert.equal( + await page.evaluate(() => + Object.hasOwn(ServiceWorkerContainer.prototype, "register"), + ), + false, + ); + await page.reload(); + await assertRestored(); + const policyCleanup = await verifyPolicyCleanup({ + page, + endpoint, + targetId, + provider, + scenarioFor, + origin: first.origin, + }); + return { + mocked: false, + cli: true, + production_provider: true, + initial_navigation: true, + action_navigation: true, + cross_origin_frame: true, + allow_preserves_existing_worker: true, + registration_blocked: true, + live_document_and_reload_restored: true, + failure_and_cancellation_cleanup: true, + caller_replacement_preserved: true, + caller_original_absence_preserved: true, + policyCleanup, + external_target_preserved: true, + }; + } catch (cause) { + failure = cause; + throw cause; + } finally { + await session?.close(); + try { + await provider.close(); + } catch (cleanupFailure) { + if (failure !== undefined) + throw new AggregateError([failure, cleanupFailure]); + throw cleanupFailure; + } finally { + await observer.close(); + await first.close(); + await second.close(); + } + } +} + +const verifyPolicyCleanup = async ({ + page, + endpoint, + targetId, + provider, + scenarioFor, + origin, +}) => { + const nativeRegister = async () => + page.evaluate(() => + navigator.serviceWorker.register.toString().includes("[native code]"), + ); + for (const path of ["/pending-image", "/pending-frame"]) { + const cancellation = new AbortController(); + const pending = provider.captureScenario(scenarioFor(`${origin}${path}`), { + signal: cancellation.signal, + }); + try { + await page + .locator( + path === "/pending-image" ? "#pending-image" : "#pending-frame", + ) + .waitFor({ state: "attached" }); + if (path === "/pending-frame") { + const child = page + .frames() + .find((frame) => frame.parentFrame() !== null); + assert.ok(child); + await child.waitForURL(`${origin}/pending-child`, { + waitUntil: "domcontentloaded", + }); + } + } finally { + cancellation.abort(); + } + const result = await pending; + assert.equal(result.ok, false); + assert.equal( + result.error.partialObservation.capture.browser.cleanup, + "disconnected-external", + ); + assert.equal(await nativeRegister(), true); + const next = await provider.captureScenario(scenarioFor(origin)); + if (!next.ok) throw next.error; + assert.equal(next.value.browser.cleanup, "disconnected-external"); + } + + await page.addInitScript("globalThis.reaCallerInitScript = 'preserved'"); + const policy = new PlaywrightScenarioServiceWorkers(page, endpoint, targetId); + await policy.initialize(); + const exposure = await page.evaluate(async () => { + const register = navigator.serviceWorker.register; + const originals = Object.getOwnPropertySymbols(register).map( + (symbol) => register[symbol], + ); + const restore = Object.getOwnPropertyNames(globalThis).find((name) => + name.startsWith("rea-restore-"), + ); + const denied = globalThis[restore]("not-the-session-secret"); + let blocked = false; + try { + await navigator.serviceWorker.register("/sw.js"); + } catch { + blocked = true; + } + return { + exposesOriginal: originals.some( + (value) => typeof value?.value === "function", + ), + denied, + blocked, + registrations: (await navigator.serviceWorker.getRegistrations()).length, + }; + }); + assert.deepEqual(exposure, { + exposesOriginal: false, + denied: false, + blocked: true, + registrations: 0, + }); + + // Crossing the renderer boundary must not layer a wrapper over an earlier + // wrapper in a surviving realm, or leave an OOPIF unprotected. + const crossSite = `${origin.replace("127.0.0.1", "localhost")}/register`; + await page.evaluate((url) => { + const child = document.createElement("iframe"); + child.id = "cross-site"; + child.src = url; + document.body.append(child); + }, crossSite); + const child = page.frames().find((frame) => frame.parentFrame() !== null); + assert.ok(child); + await child.waitForURL(crossSite, { waitUntil: "load" }); + assert.equal(await child.locator("#state").textContent(), "blocked"); + await page.locator("#cross-site").evaluate((frame, url) => { + frame.src = url; + }, `${origin}/register`); + await child.waitForURL(`${origin}/register`, { waitUntil: "load" }); + assert.equal(await child.locator("#state").textContent(), "blocked"); + + // Gate one real frame evaluation to force a new main document between live + // frame restorations. Browser replies and navigation remain real. + const originalEvaluate = child.evaluate.bind(child); + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + let entered; + const started = new Promise((resolve) => { + entered = resolve; + }); + child.evaluate = async (...args) => { + entered(); + await gate; + return originalEvaluate(...args); + }; + const closing = policy.close(); + void closing.catch(() => undefined); + try { + await started; + await page.goto(origin); + } finally { + release(); + child.evaluate = originalEvaluate; + } + await closing; + assert.equal(await nativeRegister(), true); + await page.reload(); + assert.equal(await nativeRegister(), true); + assert.equal( + await page.evaluate(() => globalThis.reaCallerInitScript), + "preserved", + ); + assert.equal( + await page.evaluate(() => + Object.getOwnPropertyNames(globalThis).some((name) => + name.startsWith("rea-restore-"), + ), + ), + false, + ); + await verifyRejectedPolicyInstallation(page.context(), endpoint, origin); + const setup = await verifyScenarioPolicySetup( + endpoint, + targetId, + origin, + page, + ); + return { + stalled_image_cancelled: true, + stalled_frame_cancelled: true, + target_reused_after_cancel: true, + private_original_descriptor: true, + wrong_restore_token_rejected: true, + renderer_transition: true, + navigation_during_restoration: true, + caller_init_script_preserved: true, + failed_main_installation_rejected: true, + failed_accessor_installation_rejected: true, + failed_iframe_installation_rejected: true, + replaced_iframe_policy_rejected: true, + setup, + }; +}; + +const verifyRejectedPolicyInstallation = async (context, endpoint, origin) => { + for (const realm of ["main", "accessor", "existing-iframe", "new-iframe"]) { + const page = await context.newPage(); + let policy; + try { + await page.goto(origin); + const cdp = await context.newCDPSession(page); + const { targetInfo } = await cdp.send("Target.getTargetInfo"); + await cdp.detach(); + policy = new PlaywrightScenarioServiceWorkers( + page, + endpoint, + targetInfo.targetId, + ); + if (realm === "new-iframe") await policy.initialize(); + let frame = page.mainFrame(); + if (realm === "existing-iframe" || realm === "new-iframe") { + const crossSite = origin.replace("127.0.0.1", "localhost"); + await page.evaluate((url) => { + const child = document.createElement("iframe"); + child.src = url; + document.body.append(child); + }, crossSite); + frame = page + .frames() + .find((candidate) => candidate.parentFrame() !== null); + assert.ok(frame); + await frame.waitForURL(crossSite, { waitUntil: "load" }); + } + await frame.evaluate((kind) => { + const prototype = ServiceWorkerContainer.prototype; + const descriptor = Object.getOwnPropertyDescriptor( + prototype, + "register", + ); + if (kind === "accessor") { + const original = descriptor.value; + Object.defineProperty(prototype, "register", { + configurable: true, + enumerable: descriptor.enumerable, + get: () => original, + }); + return; + } + Object.defineProperty(prototype, "register", { + ...descriptor, + configurable: false, + ...(kind === "new-iframe" + ? { value: function callerReplacement() {} } + : {}), + }); + }, realm); + await assert.rejects( + realm === "new-iframe" ? policy.assertBlocked() : policy.initialize(), + (error) => error.reason === "target_not_allowed", + ); + await policy.close(); + assert.equal( + await frame.evaluate((replaced) => { + const register = navigator.serviceWorker.register; + return replaced + ? register.name === "callerReplacement" + : register.toString().includes("[native code]"); + }, realm === "new-iframe"), + true, + ); + } finally { + try { + await policy?.close(); + } finally { + await page.close(); + } + } + } +}; diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index 158877bca..3f793d82c 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -50,6 +50,7 @@ import { verifyBrowserCaptureMetadataBudget } from "./lib/browser-capture-metada import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs"; import { verifyScenarioEnvironment } from "./lib/browser-scenario-environment-e2e.mjs"; import { verifyScenarioStorage } from "./lib/browser-scenario-storage-e2e.mjs"; +import { verifyScenarioServiceWorkers } from "./lib/browser-scenario-service-workers-e2e.mjs"; const SCENARIO_SECRET_VALUE = "rea-browser-verifier-secret"; const SCENARIO_URL_SECRET_VALUE = "rea-browser-url-verifier-secret"; @@ -299,6 +300,11 @@ try { endpoint, targetId: target, }); + process.stderr.write("Browser verifier: scenario service workers\n"); + const scenarioServiceWorkers = await verifyScenarioServiceWorkers( + endpoint, + target, + ); const launchedScenario = await createBrowserScenarioProvider( process.env, ).captureScenario( @@ -400,6 +406,7 @@ try { scenarioFailure, scenarioEnvironment, scenarioStorage, + scenarioServiceWorkers, verified: true, }; } finally { diff --git a/src/application/BrowserObservationService.ts b/src/application/BrowserObservationService.ts index a080f1ef8..0637774c4 100644 --- a/src/application/BrowserObservationService.ts +++ b/src/application/BrowserObservationService.ts @@ -11,7 +11,10 @@ import type { CaptureWebScreenshotInput, CompareWebScreenshotsInput, } from "../domain/webScreenshot.js"; -import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, +} from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; import type { ExecutionOptions } from "./AnalysisProvider.js"; @@ -27,6 +30,8 @@ export const listBrowserTargets = async ( const ready = requireBrowser(browser, "list_browser_targets"); if (!ready.ok) return ready; const result = await ready.value.listTargets(input, options); + if (result.ok && options.signal?.aborted === true) + return err(new AnalysisCancelledError("list_browser_targets")); return result.ok ? ok( createBrowserEvidence( @@ -48,6 +53,8 @@ export const inspectWebPage = async ( const ready = requireBrowser(browser, "inspect_web_page"); if (!ready.ok) return ready; const result = await ready.value.inspectPage(input, options); + if (result.ok && options.signal?.aborted === true) + return err(new AnalysisCancelledError("inspect_web_page")); return result.ok ? ok( createBrowserEvidence( @@ -69,6 +76,8 @@ export const analyzeWebBundle = async ( const ready = requireBrowser(browser, "analyze_web_bundle"); if (!ready.ok) return ready; const analyzed = await ready.value.analyzeBundle(input, options); + if (analyzed.ok && options.signal?.aborted === true) + return err(new AnalysisCancelledError("analyze_web_bundle")); if (!analyzed.ok) return analyzed; return ok( createBrowserEvidence( @@ -89,6 +98,8 @@ export const observeWebSession = async ( const ready = requireBrowser(browser, "observe_web_session"); if (!ready.ok) return ready; const observed = await ready.value.observeSession(input, options); + if (observed.ok && options.signal?.aborted === true) + return err(new AnalysisCancelledError("observe_web_session")); return observed.ok ? ok( createBrowserEvidence( @@ -110,6 +121,8 @@ export const discoverWebMcpTools = async ( const ready = requireBrowser(browser, "discover_webmcp_tools"); if (!ready.ok) return ready; const discovered = await ready.value.discoverWebMcpTools(input, options); + if (discovered.ok && options.signal?.aborted === true) + return err(new AnalysisCancelledError("discover_webmcp_tools")); return discovered.ok ? ok( createBrowserEvidence( @@ -151,6 +164,8 @@ export const captureWebScreenshot = async ( const ready = requireBrowser(browser, "capture_web_screenshot"); if (!ready.ok) return ready; const captured = await ready.value.captureScreenshot(input, options); + if (captured.ok && options.signal?.aborted === true) + return err(new AnalysisCancelledError("capture_web_screenshot")); return captured.ok ? ok( createBrowserEvidence( diff --git a/src/browser/BrowserScenarioSecrets.ts b/src/browser/BrowserScenarioSecrets.ts index aa583c016..5ed7eed16 100644 --- a/src/browser/BrowserScenarioSecrets.ts +++ b/src/browser/BrowserScenarioSecrets.ts @@ -230,6 +230,7 @@ export class BrowserScenarioSecrets { private readonly urlReplacements: readonly SecretReplacement[]; private readonly textReplacements: readonly SecretReplacement[]; private readonly textNeedles: readonly SecretNeedle[]; + private readonly redactionCostPerCharacter: number; private constructor(private readonly values: ReadonlyMap) { this.urlPatterns = declaredPatterns(values, "url"); @@ -237,6 +238,19 @@ export class BrowserScenarioSecrets { this.urlReplacements = secretReplacements(values, "url"); this.textReplacements = secretReplacements(values, "text"); this.textNeedles = secretNeedles(this.textReplacements, this.textPatterns); + this.redactionCostPerCharacter = 8; + for (const [replacements, patterns] of [ + [this.textReplacements, this.textPatterns], + [this.urlReplacements, this.urlPatterns], + ] as const) + for (const replacement of replacements) + this.redactionCostPerCharacter = Math.max( + this.redactionCostPerCharacter, + 8 + + (Buffer.byteLength(redactionMarker(replacement.id, patterns)) * 8 + + 128) / + replacement.text.length, + ); } static resolve( @@ -271,6 +285,14 @@ export class BrowserScenarioSecrets { return url.href; } + /** Conservative preflight for output expansion and per-match part arrays. + * Callers measuring untrusted observations must reserve this cost before + * invoking redaction. A one-byte secret can otherwise expand every byte. + */ + redactionAllocationCost(length: number): number { + return Math.ceil(length * this.redactionCostPerCharacter); + } + redact(value: string): string { const patterns = this.textPatterns; if (patterns.length === 0) return value; diff --git a/src/browser/CdpBrowserProvider.ts b/src/browser/CdpBrowserProvider.ts index 744a82983..14757a9a6 100644 --- a/src/browser/CdpBrowserProvider.ts +++ b/src/browser/CdpBrowserProvider.ts @@ -43,7 +43,10 @@ import { type WebScreenshotDiff, } from "../domain/webScreenshot.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; -import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisInputError, +} from "../domain/analysisErrorCore.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js"; @@ -119,7 +122,7 @@ export class CdpBrowserProvider implements BrowserObservationPort { }), ); } catch (cause: unknown) { - return err(providerError(cause, "list_browser_targets")); + return err(providerError(cause, "list_browser_targets", options.signal)); } } @@ -131,7 +134,7 @@ export class CdpBrowserProvider implements BrowserObservationPort { const captured = await this.#capture(input, options, "inspect_web_page"); return ok(webPageInspectionSchema.parse(captured.inspection)); } catch (cause: unknown) { - return err(providerError(cause, "inspect_web_page")); + return err(providerError(cause, "inspect_web_page", options.signal)); } } @@ -164,7 +167,7 @@ export class CdpBrowserProvider implements BrowserObservationPort { ), ); } catch (cause: unknown) { - return err(providerError(cause, "analyze_web_bundle")); + return err(providerError(cause, "analyze_web_bundle", options.signal)); } } @@ -184,7 +187,7 @@ export class CdpBrowserProvider implements BrowserObservationPort { ), ); } catch (cause: unknown) { - return err(providerError(cause, "observe_web_session")); + return err(providerError(cause, "observe_web_session", options.signal)); } } @@ -204,7 +207,7 @@ export class CdpBrowserProvider implements BrowserObservationPort { ), ); } catch (cause: unknown) { - return err(providerError(cause, "discover_webmcp_tools")); + return err(providerError(cause, "discover_webmcp_tools", options.signal)); } } @@ -236,7 +239,9 @@ export class CdpBrowserProvider implements BrowserObservationPort { ), ); } catch (cause: unknown) { - return err(providerError(cause, "capture_web_screenshot")); + return err( + providerError(cause, "capture_web_screenshot", options.signal), + ); } } @@ -413,11 +418,18 @@ const scopeToTargetOrigin = ( const providerError = ( cause: unknown, operation: BrowserObservationOperation, + signal?: AbortSignal, ): AnalysisError => cause instanceof BrowserObservationError && cause.operation !== operation ? new BrowserObservationError(operation, cause.reason, { cause }) : cause instanceof AnalysisError ? cause - : new ProviderAdapterError(CDP_BROWSER_PROVIDER_IDENTITY.id, operation, { - cause, - }); + : signal?.aborted === true + ? new AnalysisCancelledError(operation, { cause }) + : new ProviderAdapterError( + CDP_BROWSER_PROVIDER_IDENTITY.id, + operation, + { + cause, + }, + ); diff --git a/src/browser/CdpCaptureDocuments.baseUrl.test.ts b/src/browser/CdpCaptureDocuments.baseUrl.test.ts index e3201eade..26dbbeb76 100644 --- a/src/browser/CdpCaptureDocuments.baseUrl.test.ts +++ b/src/browser/CdpCaptureDocuments.baseUrl.test.ts @@ -59,6 +59,42 @@ const capture = (value: ReturnType) => { }; describe("DOM metadata document base URLs", () => { + it("splits link relations only on HTML ASCII whitespace", () => { + const strings = [ + documentUrl, + "#document", + "LINK", + "", + "href", + "/valid", + "rel", + "alternate\u00a0mcp alternate\u000bmodel-context alternate\u2003ai-plugin alternate\ufeffservice-desc mcp\tservice-desc", + ]; + const result = captureDom( + { + strings, + documents: [ + { + documentURL: 0, + nodes: { + nodeType: [9, 1], + nodeName: [1, 2], + nodeValue: [3, 3], + parentIndex: [-1, 0], + attributes: [[], [4, 5, 6, 7]], + }, + }, + ], + }, + new Set([origin]), + input, + ).next(); + if (!result.done) throw new Error("Unexpected SVG identity query"); + expect( + result.value.agentHints.map(({ declaration }) => declaration), + ).toEqual(["mcp", "service-desc"]); + }); + it("resolves links, actions and agent hints against the captured document base", () => { const result = capture(snapshot(`${origin}/screens/assets/`)); expect(result.urls.map(({ url }) => url)).toEqual([ diff --git a/src/browser/CdpCaptureDocuments.ts b/src/browser/CdpCaptureDocuments.ts index f7eb4fe31..f95904fc0 100644 --- a/src/browser/CdpCaptureDocuments.ts +++ b/src/browser/CdpCaptureDocuments.ts @@ -556,7 +556,8 @@ const domMetadata = ( } const rel = (pairs.get("rel") ?? "") .toLowerCase() - .split(/\s+/u) + // HTML link type tokens are separated by ASCII whitespace only. + .split(/[\t\n\f\r ]+/u) .filter((value) => agentRelValues.includes(value)); const href = urls.find(({ attribute }) => attribute === "href")?.url ?? null; const agentHints = diff --git a/src/browser/CdpWebMcpDiscovery.ts b/src/browser/CdpWebMcpDiscovery.ts index b414bdd89..51bfafc71 100644 --- a/src/browser/CdpWebMcpDiscovery.ts +++ b/src/browser/CdpWebMcpDiscovery.ts @@ -446,6 +446,24 @@ const normalizeTool = ( } const description = cdpStringValue(value.description) ?? ""; const annotations = recordValue(value.annotations); + const malformedAnnotations = + Object.hasOwn(value, "annotations") && annotations === undefined; + if (malformedAnnotations) + completeness.exclude("webmcp_tools", "invalid_protocol_value"); + const annotation = (key: string): boolean | null => { + if (annotations === undefined || !Object.hasOwn(annotations, key)) + return null; + const hint = annotations[key]; + if (typeof hint === "boolean") return hint; + completeness.exclude("webmcp_tools", "invalid_protocol_value"); + return null; + }; + const consequential = annotation("consequential"); + const unknownConsequential = + malformedAnnotations || + (annotations !== undefined && + Object.hasOwn(annotations, "consequential") && + typeof annotations.consequential !== "boolean"); const declaration = jsonValueSchema.safeParse(value.inputSchema); if (!declaration.success) completeness.exclude("webmcp_tools", "invalid_protocol_value"); @@ -466,9 +484,10 @@ const normalizeTool = ( ? digestCanonicalValue(schema) : null, annotations: { - read_only: booleanOrNull(annotations?.readOnly), - untrusted_content: booleanOrNull(annotations?.untrustedContent), - autosubmit: booleanOrNull(annotations?.autosubmit), + read_only: annotation("readOnly"), + untrusted_content: annotation("untrustedContent"), + ...(unknownConsequential ? {} : { consequential }), + autosubmit: annotation("autosubmit"), }, registration_source: registrationSource( recordValue(value.stackTrace), @@ -540,9 +559,6 @@ const toolKey = (frameUrl: string, frameId: string, name: string): string => .update(compositeKey([frameUrl, frameId, name])) .digest("hex")}`; -const booleanOrNull = (value: unknown): boolean | null => - typeof value === "boolean" ? value : null; - const integerOrNull = (value: unknown): number | null => { const number = numberValue(value); return number === undefined ? null : Math.max(0, Math.trunc(number)); diff --git a/src/browser/PlaywrightElectronActiveProvider.test.ts b/src/browser/PlaywrightElectronActiveProvider.test.ts index cdb2698ee..50f9905d5 100644 --- a/src/browser/PlaywrightElectronActiveProvider.test.ts +++ b/src/browser/PlaywrightElectronActiveProvider.test.ts @@ -8,6 +8,7 @@ import type { OwnedProcessGroup } from "../process/ProcessOwnership.js"; import { cleanupElectronProcesses, PlaywrightElectronActiveProvider, + safeElectronEnvironment, } from "./PlaywrightElectronActiveProvider.js"; afterEach(() => { @@ -45,6 +46,26 @@ it("passes the selected environment and ownership token to Electron launch", asy }); }); +it("passes required Windows runtime variables without inheriting unrelated values", () => { + expect( + safeElectronEnvironment( + { + SYSTEMROOT: "C:\\Windows", + LOCALAPPDATA: "C:\\Users\\analyst\\AppData\\Local", + USERPROFILE: "C:\\Users\\analyst", + APP_SECRET: "must-not-pass", + }, + "run-id", + "win32", + ), + ).toEqual({ + SYSTEMROOT: "C:\\Windows", + LOCALAPPDATA: "C:\\Users\\analyst\\AppData\\Local", + USERPROFILE: "C:\\Users\\analyst", + REA_PROCESS_RUN_ID: "run-id", + }); +}); + const ownership = { runId: "run", leaderPid: 4242, diff --git a/src/browser/PlaywrightElectronActiveProvider.ts b/src/browser/PlaywrightElectronActiveProvider.ts index 14a98aab4..816f86489 100644 --- a/src/browser/PlaywrightElectronActiveProvider.ts +++ b/src/browser/PlaywrightElectronActiveProvider.ts @@ -358,9 +358,10 @@ export const cleanupElectronProcesses = async ( }; }; -const safeElectronEnvironment = ( +export const safeElectronEnvironment = ( environment: Readonly>, runId: string, + platform = process.platform, ): Record => Object.fromEntries([ ...[ @@ -372,6 +373,23 @@ const safeElectronEnvironment = ( "USER", "DISPLAY", "WAYLAND_DISPLAY", + ...(platform === "win32" + ? [ + "SYSTEMROOT", + "TEMP", + "APPDATA", + "LOCALAPPDATA", + "USERPROFILE", + "COMSPEC", + "PATHEXT", + "PROGRAMDATA", + "ALLUSERSPROFILE", + "USERNAME", + "SYSTEMDRIVE", + "HOMEDRIVE", + "HOMEPATH", + ] + : []), ] .map((name) => [name, environment[name]] as const) .filter( diff --git a/src/browser/PlaywrightScenarioEvents.test.ts b/src/browser/PlaywrightScenarioEvents.test.ts index 4e7841365..4c0e21d30 100644 --- a/src/browser/PlaywrightScenarioEvents.test.ts +++ b/src/browser/PlaywrightScenarioEvents.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { Page } from "playwright-core"; import { browserScenarioSchema } from "../domain/browserScenario.js"; @@ -166,43 +166,127 @@ describe("PlaywrightScenarioEvents", () => { stack: "s".repeat(300_000), }); }); +}); - it("retains every observed event without a collection count ceiling", () => { - const listeners = new Map void)[]>(); - const page = { - url: () => "https://app.example.test/", - on: (name: string, listener: (value: Error) => void) => { - const registered = listeners.get(name) ?? []; - registered.push(listener); - listeners.set(name, registered); - }, - } as unknown as Page; - const scenario = browserScenarioSchema.parse({ - browser: { - mode: "launch", - executable_path: "/opt/chromium", - }, - start_url: { url: "https://app.example.test/" }, - actions: [ - { step_id: "wait", action: "wait_for_timeout", duration_ms: 1 }, - ], - capture: { events: ["page-errors"] }, +it("retains ordinary event volumes and reports event-count exhaustion", () => { + const listeners = new Map void)[]>(); + const page = { + url: () => "https://app.example.test/", + on: (name: string, listener: (value: Error) => void) => { + const registered = listeners.get(name) ?? []; + registered.push(listener); + listeners.set(name, registered); + }, + } as unknown as Page; + const scenario = browserScenarioSchema.parse({ + browser: { + mode: "launch", + executable_path: "/opt/chromium", + }, + start_url: { url: "https://app.example.test/" }, + actions: [{ step_id: "wait", action: "wait_for_timeout", duration_ms: 1 }], + capture: { events: ["page-errors"] }, + }); + const secrets = BrowserScenarioSecrets.resolve(scenario, {}); + if (secrets === undefined) throw new Error("Expected resolved secrets"); + const events = new PlaywrightScenarioEvents({ + page, + enabled: new Set(["page-errors"]), + secrets, + }); + for (let index = 0; index < 2_001; index += 1) + listeners.get("pageerror")?.forEach((listener) => { + const error = new Error(`failure-${index}`); + error.stack = ""; + listener(error); }); - const secrets = BrowserScenarioSecrets.resolve(scenario, {}); - if (secrets === undefined) throw new Error("Expected resolved secrets"); - const events = new PlaywrightScenarioEvents({ - page, - enabled: new Set(["page-errors"]), - secrets, - }); - for (let index = 0; index < 2_001; index += 1) - listeners - .get("pageerror") - ?.forEach((listener) => listener(new Error(`failure-${index}`))); - expect(events.result()).toMatchObject({ retained: 2_001, dropped: 0 }); - expect(events.result().items.at(-1)).toMatchObject({ - kind: "page-error", - message: "failure-2000", + expect(events.result()).toMatchObject({ retained: 2_001, dropped: 0 }); + expect(events.result().items.at(-1)).toMatchObject({ + kind: "page-error", + message: "failure-2000", + }); + // Omit whole oversized events, preserving ordinary messages rather than + // inventing truncated observations. This also tests admission before redact. + const redact = vi.spyOn(secrets, "redact"); + listeners.get("pageerror")?.forEach((listener) => { + const error = new Error("x".repeat(1_000_000)); + error.stack = ""; + listener(error); + }); + expect(redact).not.toHaveBeenCalled(); + for (let index = 0; index < 10_000; index += 1) + listeners.get("pageerror")?.forEach((listener) => { + const error = new Error(`short-${index}`); + error.stack = ""; + listener(error); }); + expect(events.result()).toMatchObject({ retained: 10_000, dropped: 2_002 }); + expect(events.limitations().join(" ")).toContain("resource budget"); + const aggregate = new PlaywrightScenarioEvents({ + page, + enabled: new Set(["page-errors"]), + secrets, + }); + const aggregateListener = listeners.get("pageerror")?.at(-1); + const aggregateRedact = vi.spyOn(secrets, "redact"); + let refused = false; + for (let index = 0; index < 64 && !refused; index += 1) { + aggregateRedact.mockClear(); + const droppedBefore = aggregate.result().dropped; + const error = new Error("x".repeat(400_000)); + error.stack = ""; + aggregateListener?.(error); + refused = aggregate.result().dropped > droppedBefore; + if (refused) expect(aggregateRedact.mock.calls.length).toBe(0); + } + expect(refused).toBe(true); + + const small = new Error("small event remains eligible"); + small.stack = ""; + aggregateRedact.mockClear(); + aggregateListener?.(small); + expect(aggregateRedact.mock.calls.length).toBeGreaterThan(0); + expect(aggregate.result().retained).toBeGreaterThan(0); + expect(aggregate.result().items.at(-1)).toMatchObject({ + kind: "page-error", + message: "small event remains eligible", }); }); + +it("caps live descendant listeners and reuses capacity after closure", () => { + const listeners = new Map void>(); + const page = { + on: (name: string, listener: (value: unknown) => void) => + listeners.set(name, listener), + off: vi.fn(), + } as unknown as Page; + const scenario = browserScenarioSchema.parse({ + browser: { mode: "launch", executable_path: "/fixture/chrome" }, + start_url: { url: "https://example.test" }, + actions: [{ step_id: "wait", action: "wait_for_timeout", duration_ms: 1 }], + }); + const secrets = BrowserScenarioSecrets.resolve(scenario, {}); + if (secrets === undefined) throw new Error("Expected resolved secrets"); + const events = new PlaywrightScenarioEvents({ + page, + enabled: new Set(["websockets"]), + secrets, + }); + const sockets = Array.from({ length: 2049 }, () => ({ + url: () => "wss://example.test/", + on: vi.fn<(name: string, listener: () => void) => void>(), + off: vi.fn(), + })); + for (const socket of sockets.slice(0, 2048)) + listeners.get("websocket")?.(socket); + expect(sockets[2046]?.on).toHaveBeenCalledTimes(3); + expect(sockets[2047]?.on).not.toHaveBeenCalled(); + expect(events.result().dropped).toBe(1); + const close = sockets[0]?.on.mock.calls.find( + ([name]) => name === "close", + )?.[1]; + close?.(); + expect(sockets[0]?.off).toHaveBeenCalledTimes(3); + listeners.get("websocket")?.(sockets[2048]); + expect(sockets[2048]?.on).toHaveBeenCalledTimes(3); +}); diff --git a/src/browser/PlaywrightScenarioEvents.ts b/src/browser/PlaywrightScenarioEvents.ts index aee9db630..807d1ae70 100644 --- a/src/browser/PlaywrightScenarioEvents.ts +++ b/src/browser/PlaywrightScenarioEvents.ts @@ -46,9 +46,30 @@ interface EventCaptureOptions { readonly signal?: AbortSignal; } +// Conservative allocation accounting includes UTF-16 storage and worst-case +// JSON escaping. Reserve before schema cloning; never serialize to measure. +const MAX_EVENT_COST = 4 * 1024 * 1024; +const MAX_CAPTURE_COST = 16 * 1024 * 1024; +const MAX_EVENTS = 10_000; +const MAX_LISTENER_OWNERS = 2048; +const allocationCost = (value: unknown, remaining: number): number => { + if (typeof value === "string") return value.length * 8 + 2; + if (value === null || typeof value !== "object") return 32; + let cost = 2; + for (const key in value) { + if (!Object.hasOwn(value, key)) continue; + const item = (value as Record)[key]; + cost += key.length * 8 + 4 + allocationCost(item, remaining - cost); + if (cost > remaining) break; + } + return cost; +}; + /** Arrival-ordered Playwright event capture with current-step attribution. */ export class PlaywrightScenarioEvents { private readonly items: BrowserScenarioEvent[] = []; + private retainedCost = 0; + private dropped = 0; private stepIndex = 0; private sequence = 0; private readonly enabled: ReadonlySet; @@ -85,6 +106,7 @@ export class PlaywrightScenarioEvents { selection: network, secrets: options.secrets, emit: (event) => this.push(event), + admitText: (values) => this.admitProducerText(values), ...(options.signal === undefined ? {} : { signal: options.signal }), }); this.network = collector; @@ -157,6 +179,10 @@ export class PlaywrightScenarioEvents { `Popup ${family} events before Page discovery are unavailable; later events are retained.`, ); const gaps = [...popupGaps, ...(this.network?.limitations() ?? [])]; + if (this.dropped > 0) + gaps.push( + "Browser event collection reached its resource budget; omitted events and descendant observations are unknown.", + ); return this.enabled.has("network") && !this.ownsContext ? [ "Shared-context network capture excludes requests without a known root or descendant Page frame.", @@ -172,14 +198,24 @@ export class PlaywrightScenarioEvents { } { return { retained: this.items.length, - dropped: 0, + dropped: this.dropped, items: this.items, }; } private push(event: UnindexedEvent): number { - if (this.sealed) return this.sequence; + if (this.sealed) return 0; this.sequence += 1; + if (this.items.length >= MAX_EVENTS) { + this.dropped += 1; + return 0; + } + const cost = allocationCost(event, MAX_EVENT_COST) + 128; + if (cost > MAX_EVENT_COST || this.retainedCost + cost > MAX_CAPTURE_COST) { + this.dropped += 1; + return 0; + } + this.retainedCost += cost; const parsed = browserScenarioEventSchema.parse({ ...event, sequence: this.sequence, @@ -189,6 +225,44 @@ export class PlaywrightScenarioEvents { return this.sequence; } + private admitText(...values: readonly string[]): boolean { + return this.admitProducerText(values); + } + + /** Admit conservative output cost before running redaction or cloning. */ + private admitProducerText(values: Iterable): boolean { + if (this.sealed) return false; + let cost = 4096; + for (const value of values) { + cost += this.secrets.redactionAllocationCost(value.length); + if ( + cost > MAX_EVENT_COST || + cost > MAX_CAPTURE_COST - this.retainedCost + ) { + this.sequence += 1; + this.dropped += 1; + return false; + } + } + if ( + this.items.length >= MAX_EVENTS || + cost > MAX_EVENT_COST || + cost > MAX_CAPTURE_COST - this.retainedCost + ) { + this.sequence += 1; + this.dropped += 1; + return false; + } + return true; + } + + private admitOwner(): boolean { + if (this.sealed) return false; + if (this.stopListeners.size < MAX_LISTENER_OWNERS) return true; + this.dropped += 1; + return false; + } + private retainListener(owner: object, stop: () => void): void { let stops = this.stopListeners.get(owner); if (stops === undefined) { @@ -220,6 +294,7 @@ export class PlaywrightScenarioEvents { } if (this.enabled.has("page-errors")) { const onError = (error: Error) => { + if (!this.admitText(error.message, error.stack ?? "")) return; this.push({ kind: "page-error", message: this.secrets.redact(error.message), @@ -271,16 +346,20 @@ export class PlaywrightScenarioEvents { } private console(message: ConsoleMessage): void { + const rawText = message.text(); const location = message.location(); + if (!this.admitText(rawText, location.url, message.type())) return; this.push({ kind: "console", level: this.secrets.redact(message.type()), - text: this.secrets.redact(message.text()), + text: this.secrets.redact(rawText), url: location.url === "" ? null : this.safeUrl(location.url), }); } private webSocket(socket: WebSocket): void { + if (!this.admitOwner()) return; + if (!this.admitText(socket.url())) return; const url = this.safeUrl(socket.url()); this.push({ kind: "websocket-opened", url }); const onSent = ({ payload }: { payload: string | Buffer }) => @@ -317,6 +396,7 @@ export class PlaywrightScenarioEvents { }); return; } + if (!this.admitText(payload)) return; const text = this.secrets.redact(payload); this.push({ kind, @@ -332,6 +412,7 @@ export class PlaywrightScenarioEvents { kind: "frame-attached" | "frame-detached" | "frame-navigated", frame: Frame, ): void { + if (!this.admitText(frame.url(), frame.name())) return; this.push({ kind, url: frame.url() === "" ? null : this.safeUrl(frame.url()), @@ -340,6 +421,7 @@ export class PlaywrightScenarioEvents { } private worker(worker: Worker): void { + if (!this.admitOwner() || !this.admitText(worker.url())) return; const details = { url: this.safeUrl(worker.url()), name: null, @@ -355,6 +437,7 @@ export class PlaywrightScenarioEvents { private popup(page: Page): void { if (this.pages.has(page)) return; + if (!this.admitOwner() || !this.admitText(page.url())) return; this.pages.add(page); for (const family of [ "frames", @@ -369,12 +452,14 @@ export class PlaywrightScenarioEvents { }; if (this.enabled.has("popups")) { this.push({ kind: "popup-opened", ...opened }); - const onClose = () => + const onClose = () => { + if (!this.admitText(page.url())) return; this.push({ kind: "popup-closed", url: page.url() === "" ? null : this.safeUrl(page.url()), name: null, }); + }; page.on("close", onClose); this.retainListener(page, () => page.off("close", onClose)); } @@ -382,11 +467,12 @@ export class PlaywrightScenarioEvents { } private download(download: Download): void { - this.push({ - kind: "download-cancelled", - suggested_filename: this.secrets.redact(download.suggestedFilename()), - url: this.safeUrl(download.url()), - }); + if (this.admitText(download.suggestedFilename(), download.url())) + this.push({ + kind: "download-cancelled", + suggested_filename: this.secrets.redact(download.suggestedFilename()), + url: this.safeUrl(download.url()), + }); void download.cancel().catch((cause: unknown) => { // best-effort cleanup: download cancellation must not reject unhandled. void cause; diff --git a/src/browser/PlaywrightScenarioServiceWorkers.ts b/src/browser/PlaywrightScenarioServiceWorkers.ts new file mode 100644 index 000000000..4963fbf5d --- /dev/null +++ b/src/browser/PlaywrightScenarioServiceWorkers.ts @@ -0,0 +1,307 @@ +import { randomUUID } from "node:crypto"; + +import { z } from "zod"; +import type { Page } from "playwright-core"; + +import { browserObservationEndpointSchema } from "../domain/browserObservation.js"; +import { BrowserObservationError } from "../domain/browserObservationError.js"; +import { CdpConnection } from "./CdpConnection.js"; +import { withCdpDiscoveryDeadline } from "./CdpEndpoint.js"; +import { + attachCdpTargetSession, + openCdpEndpoint, + type CdpTargetSession, +} from "./CdpTargetSession.js"; +import { withPlaywrightExecutionBoundary } from "./PlaywrightExecutionBoundary.js"; + +const OPERATION = "capture_browser_scenario" as const; +const scriptResponse = z.object({ identifier: z.string() }); +const attachedEvent = z.object({ + sessionId: z.string(), + targetInfo: z.object({ type: z.string() }), +}); +const detachedEvent = z.object({ sessionId: z.string() }); +type PolicyTarget = { + readonly id: string; + ready: Promise; + identifier?: string; +}; + +/** Owns only its frame scripts and registration wrappers, never caller-owned workers. */ +export class PlaywrightScenarioServiceWorkers { + private readonly marker = JSON.stringify( + `rea-service-workers-${randomUUID()}`, + ); + private readonly restoreKey = JSON.stringify(`rea-restore-${randomUUID()}`); + private readonly secret = JSON.stringify(randomUUID()); + #initialization: Promise | undefined; + #restoration: Promise | undefined; + #session: CdpTargetSession | undefined; + #targets = new Map(); + #closed = false; + #released = false; + #detached = false; + #removeListener: (() => void) | undefined; + #setup = new AbortController(); + + constructor( + private readonly page: Page, + private readonly endpoint: string, + private readonly targetId: string, + ) {} + + initialize(): Promise { + return (this.#initialization ??= this.install()); + } + + private async install(): Promise { + this.#session = await withCdpDiscoveryDeadline( + OPERATION, + this.#setup.signal, + async (signal) => { + await withPlaywrightExecutionBoundary( + () => this.assertBlocked(), + undefined, + signal, + ); + const opened = await openCdpEndpoint( + browserObservationEndpointSchema.parse(this.endpoint), + OPERATION, + signal, + ); + const target = opened.discovery.targets.find( + ({ id }) => id === this.targetId, + ); + if (target === undefined) { + await opened.connection?.close(); + throw new BrowserObservationError(OPERATION, "target_not_found"); + } + const connection = + opened.connection ?? + (await CdpConnection.connect( + opened.discovery.webSocket.url, + OPERATION, + signal, + )); + return await attachCdpTargetSession( + connection, + target, + OPERATION, + signal, + ); + }, + ); + const connection = this.#session.connection; + this.#removeListener = connection.onEvent((event) => { + if ( + event.method === "Target.attachedToTarget" && + this.#targets.has(event.sessionId ?? "") + ) { + const { sessionId, targetInfo } = attachedEvent.parse(event.params); + if (targetInfo.type === "iframe") this.addTarget(sessionId); + } else if (event.method === "Target.detachedFromTarget") { + const id = detachedEvent.parse(event.params).sessionId; + this.#targets.delete(id); + if (id === this.#session?.sessionId) this.#detached = true; + } + }); + const id = this.#session.sessionId; + if (id === undefined) + throw new BrowserObservationError(OPERATION, "protocol_error"); + this.addTarget(id); + await this.settle(); + // CDP returns an identifier even when an init script throws in a realm. + await withPlaywrightExecutionBoundary( + () => this.assertBlocked(), + undefined, + this.#setup.signal, + ); + } + + private addTarget(id: string): void { + const target: PolicyTarget = { id, ready: Promise.resolve() }; + this.#targets.set(id, target); + const ready = this.initializeTarget(target); + target.ready = ready; + void ready.catch(() => undefined); + } + + private async initializeTarget(target: PolicyTarget): Promise { + try { + if (this.#closed) return; + await this.send(target.id, "Page.enable"); + if (this.#closed) return; + const added = scriptResponse.parse( + await this.send(target.id, "Page.addScriptToEvaluateOnNewDocument", { + runImmediately: true, + source: `(() => { + if (!("serviceWorker" in navigator)) return; + const prototype = ServiceWorkerContainer.prototype; + const descriptor = Object.getOwnPropertyDescriptor(prototype, "register"); + if (typeof descriptor?.value === "function" && Object.hasOwn(descriptor.value, Symbol.for(${this.marker}))) return; + const register = () => Promise.reject(new Error("service workers are blocked by scenario policy")); + Object.defineProperty(register, Symbol.for(${this.marker}), { value: true }); + const secret = ${this.secret}; + Object.defineProperty(globalThis, ${this.restoreKey}, { configurable: true, value: (provided) => { + if (provided !== secret) return false; + if (Object.getOwnPropertyDescriptor(prototype, "register")?.value === register) { + if (descriptor === undefined) delete prototype.register; + else Object.defineProperty(prototype, "register", descriptor); + } + delete globalThis[${this.restoreKey}]; + return true; + }}); + Object.defineProperty(prototype, "register", { ...descriptor, configurable: true, value: register }); + })()`, + }), + ); + target.identifier = added.identifier; + if (!this.#closed) + await this.send(target.id, "Target.setAutoAttach", { + autoAttach: true, + waitForDebuggerOnStart: true, + flatten: true, + filter: [{ type: "iframe", exclude: false }, { exclude: true }], + }); + } finally { + // Only this page's iframe targets are attached; unrelated workers/popups + // are not acquired. Resume a child even if installation or cleanup fails. + if ( + this.#targets.has(target.id) && + target.id !== this.#session?.sessionId + ) + await this.send(target.id, "Runtime.runIfWaitingForDebugger"); + } + } + + private send( + id: string, + method: string, + params: Readonly> = {}, + ): Promise { + if (this.#session === undefined) + throw new BrowserObservationError(OPERATION, "disconnected"); + return this.#session.connection.send(method, params, id); + } + + private async settle(): Promise { + for (;;) { + const targets = [...this.#targets.values()]; + await Promise.all( + targets.map(async (target) => { + try { + await target.ready; + } catch (cause: unknown) { + if (this.#targets.get(target.id) === target) throw cause; + } + }), + ); + if ( + targets.length === this.#targets.size && + targets.every((target) => this.#targets.get(target.id) === target) + ) + return; + } + } + + async assertBlocked(): Promise { + if (this.#session !== undefined) await this.settle(); + for (const frame of this.page.frames()) { + let violatesPolicy: unknown; + try { + violatesPolicy = await frame.evaluate(`(async () => { + if (!("serviceWorker" in navigator)) return false; + const serviceWorker = navigator.serviceWorker; + if (${this.#session !== undefined} && + (!Object.hasOwn(serviceWorker.register, Symbol.for(${this.marker})) || + typeof globalThis[${this.restoreKey}] !== "function")) return true; + if (serviceWorker.controller !== null) return true; + return (await serviceWorker.getRegistrations()).length > 0; + })()`); + } catch (cause: unknown) { + if (frame.isDetached()) continue; + throw cause; + } + if (violatesPolicy) + throw new BrowserObservationError(OPERATION, "target_not_allowed"); + } + } + + close(stopLoading = false): Promise { + this.#closed = true; + // Abort discovery/handshakes and reads, never an already-sent page mutation. + this.#setup.abort(); + const restoration = (this.#restoration ??= this.restore(stopLoading)); + void restoration.then( + () => { + if (this.#restoration === restoration) this.#restoration = undefined; + }, + () => { + if (this.#restoration === restoration) this.#restoration = undefined; + }, + ); + return withPlaywrightExecutionBoundary(() => restoration, 1_000); + } + + private async restore(stopLoading: boolean): Promise { + await this.#initialization?.catch(() => undefined); + if (this.#released) return; + const session = this.#session; + if (session === undefined) return; + if (!this.page.isClosed()) { + // This owner exists even with empty storage seeds. Stopped navigation is + // independent of storage and restoration never waits for a load event. + if (stopLoading && !this.#detached && session.sessionId !== undefined) + await this.send(session.sessionId, "Page.stopLoading"); + await Promise.all( + [...this.#targets.values()].map(({ ready }) => + ready.catch(() => undefined), + ), + ); + for (const target of this.#targets.values()) { + if (!this.#detached && target.identifier !== undefined) { + try { + await this.send( + target.id, + "Page.removeScriptToEvaluateOnNewDocument", + { identifier: target.identifier }, + ); + } catch (cause: unknown) { + if (this.#targets.get(target.id) === target) throw cause; + } + delete target.identifier; + } + } + // Future documents cannot install our wrapper after the removals above. + // A realm lost to navigation is not assumed restored: retain and retry. + for (const frame of this.page.frames()) { + if (frame.isDetached()) continue; + try { + await frame.evaluate(`(() => { + const restore = globalThis[${this.restoreKey}]; + if (typeof restore === "function" && restore(${this.secret}) === true) return; + const register = globalThis.ServiceWorkerContainer?.prototype.register; + if (typeof register === "function" && Object.hasOwn(register, Symbol.for(${this.marker}))) + throw new Error("service-worker policy restoration could not be confirmed"); + })()`); + } catch (cause: unknown) { + if (!frame.isDetached()) throw cause; + } + } + } + // Detaching the exact owned session also releases its iframe auto-attach. + if ( + !this.page.isClosed() && + !this.#detached && + session.sessionId !== undefined + ) { + await session.connection.send("Target.detachFromTarget", { + sessionId: session.sessionId, + }); + this.#detached = true; + } + this.#removeListener?.(); + await session.connection.close(); + this.#released = true; + } +} diff --git a/src/browser/PlaywrightScenarioSession.test.ts b/src/browser/PlaywrightScenarioSession.test.ts index dbc170880..d27920773 100644 --- a/src/browser/PlaywrightScenarioSession.test.ts +++ b/src/browser/PlaywrightScenarioSession.test.ts @@ -12,6 +12,29 @@ const mocks = vi.hoisted(() => ({ } | undefined, finishEvents: vi.fn(), + policySend: vi.fn(), + policyTransportClose: vi.fn(), + policyEvent: undefined as + | ((event: { method: string; params: unknown }) => void) + | undefined, +})); + +vi.mock("./CdpTargetSession.js", () => ({ + openCdpEndpoint: async () => ({ + discovery: { targets: [{ id: "target" }] }, + connection: { + send: mocks.policySend, + onEvent: (listener: typeof mocks.policyEvent) => { + mocks.policyEvent = listener; + return () => undefined; + }, + close: mocks.policyTransportClose, + }, + }), + attachCdpTargetSession: async (connection: unknown) => ({ + connection, + sessionId: "policy-session", + }), })); vi.mock("./PlaywrightScenarioBrowser.js", () => ({ @@ -85,6 +108,10 @@ const fixture = () => { evaluate: vi.fn(async (_script: unknown) => false), addInitScript: vi.fn(async (_script: unknown) => undefined), context: () => context, + frames: () => [page], + isDetached: () => false, + isClosed: (): boolean => false, + waitForLoadState: vi.fn(async () => undefined), }); const context = Object.assign(new EventEmitter(), { setDefaultTimeout: vi.fn(), @@ -123,8 +150,154 @@ const fixture = () => { beforeEach(() => { vi.clearAllMocks(); + mocks.policySend.mockReset(); + mocks.policyTransportClose.mockReset(); + mocks.policyEvent = undefined; + mocks.policySend.mockImplementation(async (method: string) => + method === "Page.addScriptToEvaluateOnNewDocument" + ? { identifier: "owned-policy-script" } + : {}, + ); + mocks.policyTransportClose.mockResolvedValue(undefined); }); +it("stops attached navigation without storage and retries rejected policy cleanup", async () => { + const { cleanup } = fixture(); + mocks.storage = undefined; + const attached = browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + storage: {}, + }); + const session = await PlaywrightScenarioSession.open(attached, {}, {}); + const failure = new Error("stopLoading rejected"); + mocks.policySend.mockRejectedValueOnce(failure); + await expect(session.close(true)).rejects.toBe(failure); + expect(cleanup.close).not.toHaveBeenCalled(); + await expect(session.close(true)).resolves.toBe("disconnected-external"); + expect( + mocks.policySend.mock.calls.filter( + ([method]) => method === "Page.stopLoading", + ), + ).toHaveLength(2); + expect(cleanup.close).toHaveBeenCalledTimes(1); +}); + +it("rejects an init script accepted by CDP when the live realm is unprotected", async () => { + const { page, cleanup } = fixture(); + page.evaluate.mockResolvedValueOnce(false).mockResolvedValueOnce(true); + const attached = browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + }); + await expect( + PlaywrightScenarioSession.open(attached, {}, {}), + ).rejects.toMatchObject({ + reason: "target_not_allowed", + }); + expect( + mocks.policySend.mock.calls.some( + ([method]) => method === "Page.addScriptToEvaluateOnNewDocument", + ), + ).toBe(true); + expect( + mocks.policySend.mock.calls.some( + ([method]) => method === "Page.removeScriptToEvaluateOnNewDocument", + ), + ).toBe(true); + expect(cleanup.close).toHaveBeenCalledTimes(1); +}); + +it("retains an owned script identifier after rejected removal", async () => { + const { cleanup } = fixture(); + const attached = browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + }); + const session = await PlaywrightScenarioSession.open(attached, {}, {}); + const failure = new Error("removeScript rejected"); + mocks.policySend.mockRejectedValueOnce(failure); + await expect(session.close()).rejects.toBe(failure); + expect(cleanup.close).not.toHaveBeenCalled(); + await expect(session.close()).resolves.toBe("disconnected-external"); + const removals = mocks.policySend.mock.calls.filter( + ([method]) => method === "Page.removeScriptToEvaluateOnNewDocument", + ); + expect(removals).toHaveLength(2); + expect( + removals.every(([, params]) => params.identifier === "owned-policy-script"), + ).toBe(true); +}); + +it("retries transport close without detaching an already released policy session", async () => { + fixture(); + const attached = browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + }); + const session = await PlaywrightScenarioSession.open(attached, {}, {}); + const failure = new Error("transport close rejected"); + mocks.policyTransportClose.mockRejectedValueOnce(failure); + await expect(session.close()).rejects.toBe(failure); + await expect(session.close()).resolves.toBe("disconnected-external"); + expect( + mocks.policySend.mock.calls.filter( + ([method]) => method === "Target.detachFromTarget", + ), + ).toHaveLength(1); + expect(mocks.policyTransportClose).toHaveBeenCalledTimes(2); +}); + +it.each(["closed page", "detached target"])( + "releases its policy transport after a %s", + async (terminal) => { + const { page } = fixture(); + const attached = browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + }); + const session = await PlaywrightScenarioSession.open(attached, {}, {}); + if (terminal === "closed page") page.isClosed = () => true; + else + mocks.policyEvent?.({ + method: "Target.detachedFromTarget", + params: { sessionId: "policy-session" }, + }); + await expect(session.close()).resolves.toBe("disconnected-external"); + expect( + mocks.policySend.mock.calls.some( + ([method]) => method === "Target.detachFromTarget", + ), + ).toBe(false); + expect(mocks.policyTransportClose).toHaveBeenCalledTimes(1); + }, +); + it("uses browser disconnect as terminal storage release after detach failure", async () => { const { cleanup, storage } = fixture(); const detachFailure = new Error("CDP detach rejected"); @@ -176,18 +349,136 @@ it("blocks new attached service-worker registration without removing existing re }); const session = await PlaywrightScenarioSession.open(attached, {}, {}); - expect(page.evaluate).toHaveBeenCalledTimes(2); - expect(page.addInitScript).toHaveBeenCalledTimes(1); - expect(page.addInitScript).toHaveBeenCalledWith( - expect.stringContaining("service workers are blocked by scenario policy"), - ); - expect(page.addInitScript).toHaveBeenCalledWith( - expect.not.stringContaining("getRegistrations"), - ); - expect(page.addInitScript).toHaveBeenCalledWith( - expect.not.stringContaining("unregister"), + expect(page.evaluate).toHaveBeenCalledTimes(3); + const injection = mocks.policySend.mock.calls.find( + ([method]) => method === "Page.addScriptToEvaluateOnNewDocument", + )?.[1]; + expect(injection.source).toContain( + "service workers are blocked by scenario policy", ); + expect(injection.source).not.toContain("getRegistrations"); + expect(injection.source).not.toContain("unregister"); await expect(session.close()).resolves.toBe("disconnected-external"); + expect(mocks.policySend).toHaveBeenCalledWith( + "Page.removeScriptToEvaluateOnNewDocument", + { identifier: "owned-policy-script" }, + "policy-session", + ); + expect(cleanup.close).toHaveBeenCalledTimes(1); +}); + +it("retains the attached connection when registration restoration fails", async () => { + const { cleanup, page } = fixture(); + const attached = browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + }); + const session = await PlaywrightScenarioSession.open(attached, {}, {}); + const failure = new Error("registration restoration failed"); + page.evaluate.mockRejectedValueOnce(failure); + await expect(session.close()).rejects.toBe(failure); + expect(cleanup.close).not.toHaveBeenCalled(); + await expect(session.close()).resolves.toBe("disconnected-external"); + expect(cleanup.close).toHaveBeenCalledTimes(1); + await expect(session.close()).resolves.toBe("disconnected-external"); +}); + +it("cancels a stalled policy preflight without allowing late setup", async () => { + const { cleanup, page } = fixture(); + let release: () => void = () => undefined; + let entered: () => void = () => undefined; + const started = new Promise((resolve) => { + entered = resolve; + }); + page.evaluate.mockImplementationOnce(async () => { + entered(); + await new Promise((resolve) => { + release = resolve; + }); + return false; + }); + const cancellation = new AbortController(); + const opening = PlaywrightScenarioSession.open( + browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + }), + {}, + { signal: cancellation.signal }, + ); + const outcome = expect(opening).rejects.toMatchObject({ + _tag: "AnalysisCancelledError", + }); + await started; + cancellation.abort(); + await outcome; + expect(cleanup.close).toHaveBeenCalledTimes(1); + expect(cleanup.close.mock.calls[0]).toHaveLength(1); + release(); + await new Promise((resolve) => setImmediate(resolve)); + expect(mocks.policySend).not.toHaveBeenCalled(); +}); + +it("waits for a pending registration installation before cancelled cleanup", async () => { + const { cleanup, page } = fixture(); + let releaseInstallation: () => void = () => undefined; + let installationStarted: () => void = () => undefined; + const installed = new Promise((resolve) => { + releaseInstallation = resolve; + }); + const started = new Promise((resolve) => { + installationStarted = resolve; + }); + mocks.policySend.mockImplementation(async (method: string) => { + if (method === "Page.addScriptToEvaluateOnNewDocument") { + installationStarted(); + await installed; + return { identifier: "owned-policy-script" }; + } + return {}; + }); + const cancellation = new AbortController(); + const opening = PlaywrightScenarioSession.open( + browserScenarioSchema.parse({ + ...scenario, + browser: { + mode: "connect", + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }, + environment: { service_workers: "block" }, + }), + {}, + { signal: cancellation.signal }, + ); + const outcome = expect(opening).rejects.toMatchObject({ + _tag: "AnalysisCancelledError", + }); + await started; + cancellation.abort(); + await new Promise((resolve) => setImmediate(resolve)); + expect(cleanup.close).not.toHaveBeenCalled(); + expect(page.evaluate).toHaveBeenCalledTimes(1); + releaseInstallation(); + await outcome; + expect(page.evaluate.mock.calls.at(-1)?.[0]).toContain( + "policy restoration could not be confirmed", + ); + expect(mocks.policySend).toHaveBeenCalledWith( + "Page.stopLoading", + {}, + "policy-session", + ); expect(cleanup.close).toHaveBeenCalledTimes(1); }); diff --git a/src/browser/PlaywrightScenarioSession.ts b/src/browser/PlaywrightScenarioSession.ts index 949193b7c..56a043732 100644 --- a/src/browser/PlaywrightScenarioSession.ts +++ b/src/browser/PlaywrightScenarioSession.ts @@ -1,4 +1,4 @@ -import type { BrowserContext, Page } from "playwright-core"; +import type { BrowserContext } from "playwright-core"; import type { BrowserScenario } from "../domain/browserScenario.js"; import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js"; @@ -14,6 +14,7 @@ import { performPlaywrightScenarioAction } from "./PlaywrightScenarioActions.js" import { capturePlaywrightStepArtifacts } from "./PlaywrightScenarioArtifacts.js"; import { PlaywrightScenarioEvents } from "./PlaywrightScenarioEvents.js"; import { PlaywrightScenarioStorage } from "./PlaywrightScenarioStorage.js"; +import { PlaywrightScenarioServiceWorkers } from "./PlaywrightScenarioServiceWorkers.js"; import { withPlaywrightExecutionBoundary } from "./PlaywrightExecutionBoundary.js"; import { browserScenarioCaptureData, @@ -50,41 +51,13 @@ const installCookies = async ( ); }; -const blockAttachedServiceWorkers = async (page: Page): Promise => { - const hasServiceWorker = await page.evaluate(`(async () => { - if (!("serviceWorker" in navigator)) return false; - const serviceWorker = navigator.serviceWorker; - if (serviceWorker.controller !== null) return true; - return (await serviceWorker.getRegistrations()).length > 0; - })()`); - // An attached context belongs to the caller. Reject a block request when it - // already has a worker instead of unregistering that caller-owned resource. - if (hasServiceWorker) - throw new BrowserObservationError(OPERATION, "target_not_allowed"); - const blockRegistration = `(() => { - if (!("serviceWorker" in navigator)) return; - Object.defineProperty(ServiceWorkerContainer.prototype, "register", { - configurable: false, - value: () => Promise.reject(new Error("service workers are blocked by scenario policy")) - }); - })()`; - await page.addInitScript(blockRegistration); - await page.evaluate(blockRegistration); -}; - const initializePage = async ( context: BrowserContext, - page: Page, scenario: BrowserScenario, secrets: BrowserScenarioSecrets, ): Promise => { context.setDefaultTimeout(0); context.setDefaultNavigationTimeout(0); - if ( - scenario.browser.mode === "connect" && - scenario.environment.service_workers === "block" - ) - await blockAttachedServiceWorkers(page); await installCookies(context, scenario, secrets); }; @@ -97,6 +70,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { private readonly secrets: BrowserScenarioSecrets; private readonly eventCapture: PlaywrightScenarioEvents; private readonly storage: PlaywrightScenarioStorage | undefined; + private readonly serviceWorkers: PlaywrightScenarioServiceWorkers | undefined; private readonly signal: AbortSignal | undefined; #storageReleased = false; @@ -107,6 +81,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { readonly secrets: BrowserScenarioSecrets; readonly eventCapture: PlaywrightScenarioEvents; readonly storage: PlaywrightScenarioStorage | undefined; + readonly serviceWorkers: PlaywrightScenarioServiceWorkers | undefined; readonly signal: AbortSignal | undefined; }, ) { @@ -116,6 +91,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { this.secrets = options.secrets; this.eventCapture = options.eventCapture; this.storage = options.storage; + this.serviceWorkers = options.serviceWorkers; this.signal = options.signal; this.version = opened.browser.version(); this.initialUrl = opened.page.url(); @@ -178,6 +154,11 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { undefined, options.signal, ); + await withPlaywrightExecutionBoundary( + () => session.serviceWorkers?.assertBlocked() ?? Promise.resolve(), + undefined, + options.signal, + ); await withPlaywrightExecutionBoundary( () => session.storage?.settle() ?? Promise.resolve(), undefined, @@ -228,9 +209,21 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { ): Promise { let events: PlaywrightScenarioEvents | undefined; let storage: PlaywrightScenarioStorage | undefined; + const serviceWorkers = + scenario.browser.mode === "connect" && + scenario.environment.service_workers === "block" + ? new PlaywrightScenarioServiceWorkers( + opened.page, + scenario.browser.cdp_endpoint, + scenario.browser.target_id, + ) + : undefined; try { await withPlaywrightExecutionBoundary( - () => initializePage(opened.context, opened.page, scenario, secrets), + async () => { + await serviceWorkers?.initialize(); + await initializePage(opened.context, scenario, secrets); + }, undefined, options.signal, ); @@ -261,12 +254,14 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { secrets, eventCapture: events, storage, + serviceWorkers, signal: options.signal, }); } catch (cause: unknown) { let storageReleased = false; const cleanup = async () => { let storageFailure: { readonly cause: unknown } | undefined; + let policyFailure: { readonly cause: unknown } | undefined; let browserFailure: { readonly cause: unknown } | undefined; if (!storageReleased) { try { @@ -276,17 +271,30 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { } } try { - await opened.cleanup.close( - events === undefined - ? undefined - : () => events?.finish() ?? Promise.resolve(), - options.signal, - ); + if (!opened.cleanup.browserClosed) await serviceWorkers?.close(true); } catch (failure: unknown) { - browserFailure = { cause: failure }; + policyFailure = { cause: failure }; + } + if (policyFailure === undefined) { + try { + await opened.cleanup.close( + events === undefined + ? undefined + : () => events?.finish() ?? Promise.resolve(), + ); + } catch (failure: unknown) { + browserFailure = { cause: failure }; + } + } else { + try { + await events?.finish(); + } catch (failure: unknown) { + browserFailure = { cause: failure }; + } } if (opened.cleanup.browserClosed) storageReleased = true; throwCleanupFailures([ + ...(policyFailure === undefined ? [] : [policyFailure.cause]), ...(storageReleased || storageFailure === undefined ? [] : [storageFailure.cause]), @@ -358,6 +366,11 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { >, signal?: AbortSignal, ) { + await withPlaywrightExecutionBoundary( + () => this.serviceWorkers?.assertBlocked() ?? Promise.resolve(), + undefined, + signal, + ); await withPlaywrightExecutionBoundary( () => this.storage?.settle() ?? Promise.resolve(), undefined, @@ -378,6 +391,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { async close(stopLoading = this.signal?.aborted === true) { let storageFailure: { readonly cause: unknown } | undefined; + let policyFailure: { readonly cause: unknown } | undefined; let browserFailure: { readonly cause: unknown } | undefined; if (!this.#storageReleased) { try { @@ -387,12 +401,27 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { } } try { - await this.opened.cleanup.close(() => this.eventCapture.finish()); + if (!this.opened.cleanup.browserClosed) + await this.serviceWorkers?.close(stopLoading); } catch (failure: unknown) { - browserFailure = { cause: failure }; + policyFailure = { cause: failure }; + } + if (policyFailure === undefined) { + try { + await this.opened.cleanup.close(() => this.eventCapture.finish()); + } catch (failure: unknown) { + browserFailure = { cause: failure }; + } + } else { + try { + await this.eventCapture.finish(); + } catch (failure: unknown) { + browserFailure = { cause: failure }; + } } if (this.opened.cleanup.browserClosed) this.#storageReleased = true; throwCleanupFailures([ + ...(policyFailure === undefined ? [] : [policyFailure.cause]), ...(this.#storageReleased || storageFailure === undefined ? [] : [storageFailure.cause]), diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 5dbbe2ad1..4ab805f43 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -729,7 +729,10 @@ const originalModuleEdges = ( ); continue; } - const parsed = parseJavaScriptSource(source.artifact.text); + const parsed = parseJavaScriptSource( + source.artifact.text, + sourceParserPath(sourceName), + ); check(); if (parsed === null) { incomplete.push(sourceName); @@ -783,6 +786,17 @@ const originalModuleEdges = ( return { edges, incomplete }; }; +/** Select parser semantics from the source URL path, excluding query/fragment. */ +const sourceParserPath = (source: string): string => { + try { + return new URL(source).pathname; + } catch (cause: unknown) { + // Unresolved source identities can still carry a useful relative path. + void cause; + return source; + } +}; + const checkOperation = ( callerSignal: AbortSignal | undefined, operationSignal: AbortSignal | undefined, diff --git a/src/browser/assets/LocalWebScriptArtifacts.ts b/src/browser/assets/LocalWebScriptArtifacts.ts index 51fc14c45..13391bff8 100644 --- a/src/browser/assets/LocalWebScriptArtifacts.ts +++ b/src/browser/assets/LocalWebScriptArtifacts.ts @@ -84,6 +84,11 @@ export class LocalWebScriptArtifacts implements WebScriptArtifactPort { "path", `Selected source path changes during normalization: ${selected.content.relative_path}`, ); + if (process.platform === "win32" && portable.includes("\\")) + throw new ArtifactReaderFailure( + "path", + `Windows source path cannot contain backslashes as literal name characters: ${JSON.stringify(portable)}`, + ); const root = resolve(dirname(input.manifest_path)); const sourcePath = join(root, "files", portable); targetPath = sourcePath; diff --git a/src/browser/history/CaptureRedaction.ts b/src/browser/history/CaptureRedaction.ts index 450b1bb05..63feb4899 100644 --- a/src/browser/history/CaptureRedaction.ts +++ b/src/browser/history/CaptureRedaction.ts @@ -1,3 +1,4 @@ +import { isSafeNumber, type LosslessNumber } from "lossless-json"; import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js"; import { parseRecordedTransportUrl } from "../../domain/recordedTransportUrl.js"; @@ -50,6 +51,22 @@ export class CaptureRedaction { return null; } + /** Check both retained spellings before projecting a producer JSON number. */ + number( + value: LosslessNumber, + pointer: string, + redact = true, + ): { reported: number | null; literal: string } | null { + const reported = isSafeNumber(value.value) ? Number(value.value) : null; + if ( + redact && + (this.text(value.value, pointer) === null || + (reported !== null && this.text(String(reported), pointer) === null)) + ) + return null; + return { reported, literal: value.value }; + } + /** Known HAR credential surfaces; unknown extensions are ordinary caller-selected evidence. */ harCredential( pointer: string, diff --git a/src/browser/history/HarCapture.test.ts b/src/browser/history/HarCapture.test.ts index b931fda06..60cf99e1b 100644 --- a/src/browser/history/HarCapture.test.ts +++ b/src/browser/history/HarCapture.test.ts @@ -257,6 +257,39 @@ it.each(["9007199254740993", "4740993", "99"])( }, ); +it.each([ + { encoded: "1e3", sensitive: "1000" }, + { encoded: "123.4500", sensitive: "123.45" }, + { encoded: "0.0000001", sensitive: "1e-7" }, +])( + "excludes sensitive numbers introduced by JSON normalization: $encoded", + ({ encoded, sensitive }) => { + const text = JSON.stringify(historicalHar()) + .replace('"log":{', `"_number":${encoded},"log":{`) + .replace('"cache":{}', `"cache":{},"_number":${encoded}`); + const result = decodeHarCapture(text, [sensitive]); + for (const projection of [result.container, ...result.records]) { + expect(projection.reported).toMatchObject({ _number: null }); + expect(projection.numeric_literals).not.toContainEqual( + expect.objectContaining({ pointer: "/_number" }), + ); + expect(projection.redactions).toContainEqual({ + pointer: "/_number", + reason: "explicit-sensitive-value", + }); + } + const unmarked = decodeHarCapture(text, []); + for (const projection of [unmarked.container, ...unmarked.records]) { + expect(projection.reported).toMatchObject({ _number: Number(encoded) }); + expect(projection.numeric_literals).toContainEqual({ + pointer: "/_number", + producer_type: "json-number", + literal: encoded, + }); + } + }, +); + it("keeps unsafe extension numeric lexemes and rejects unsafe mandatory schema numbers", () => { const text = JSON.stringify(historicalHar()); const result = decodeHarCapture( diff --git a/src/browser/history/HarCapture.ts b/src/browser/history/HarCapture.ts index 81c5ffb15..d23083b6b 100644 --- a/src/browser/history/HarCapture.ts +++ b/src/browser/history/HarCapture.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { isSafeNumber, LosslessNumber } from "lossless-json"; +import { LosslessNumber } from "lossless-json"; import { parseCaptureJson } from "./CaptureJson.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { @@ -121,13 +121,14 @@ const projectHar = ( ) return redactor.credential(pointer); if (item instanceof LosslessNumber) { - if (redact && redactor.text(item.value, pointer) === null) return null; + const number = redactor.number(item, pointer, redact); + if (number === null) return null; numeric_literals.push({ pointer, producer_type: "json-number", - literal: item.value, + literal: number.literal, }); - return isSafeNumber(item.value) ? Number(item.value) : null; + return number.reported; } if (typeof item === "string") { if ( diff --git a/src/browser/history/QlogCapture.test.ts b/src/browser/history/QlogCapture.test.ts index fb4e10072..23c776565 100644 --- a/src/browser/history/QlogCapture.test.ts +++ b/src/browser/history/QlogCapture.test.ts @@ -186,6 +186,31 @@ it("excludes structural auth material and explicit normalized values without gue expect(traces[0]?.incomplete_fact_records).toContain(2); }); +it.each([ + { hidden: "sent", incomplete: [0], streams: 0, losses: 1 }, + { hidden: "lost", incomplete: [1], streams: 1, losses: 0 }, + { hidden: "name", incomplete: [0, 1], streams: 0, losses: 0 }, +])( + "lists records whose event name a declaration of $hidden hides as incomplete", + ({ hidden, incomplete, streams, losses }) => { + const { traces } = facts( + capture([ + event([frame(0, 10, true)]), + { + name: "recovery:packet_lost", + time: 1, + data: { packet_number: 5, type: "1RTT" }, + }, + { name: "extension:custom", time: 2, data: { body: "ordinary" } }, + ]), + [hidden], + ); + expect(traces[0]?.incomplete_fact_records).toEqual(incomplete); + expect(traces[0]?.streams).toHaveLength(streams); + expect(traces[0]?.loss_declarations).toHaveLength(losses); + }, +); + it("omits a hidden derived total for a stream with more contributions than call arguments allow", () => { const events = Array.from({ length: 130 }, (_, index) => event( diff --git a/src/browser/history/QlogCapture.ts b/src/browser/history/QlogCapture.ts index 8d19fc64f..245ec1fd8 100644 --- a/src/browser/history/QlogCapture.ts +++ b/src/browser/history/QlogCapture.ts @@ -84,7 +84,12 @@ export const decodeQlogCapture = ( "Record ordinal is derived from trace/event source traversal, not global time order. Qlog lengths and FIN do not expose original HTTP body bytes or establish network completeness.", ], }); - collectTransportFacts(projection, ordinal, { fact, streams, excluded }); + collectTransportFacts(projection, ordinal, { + fact, + streams, + excluded, + name: object(event) && typeof event.name === "string" ? event.name : "", + }); } for (const stream of streams.values()) { summarizeStream(stream); @@ -223,14 +228,25 @@ const collectTransportFacts = ( fact: Trace; streams: Map; excluded: (value: string) => boolean; + /** Validated source event name, before declarations are applied. */ + name: string; }, ): void => { - const { fact, streams, excluded } = state; + const { fact, streams, excluded, name } = state; if (!object(projection.reported) || !object(projection.reported.data)) { fact.incomplete_fact_records.push(ordinal); return; } const visible = projection.reported; + // Facts are selected by event name, so a declaration that hides the name + // of a loss or packet event omits its facts like any other hidden field. + if ( + (name === "recovery:packet_lost" || packetEvent(name)) && + visible.name !== name + ) { + fact.incomplete_fact_records.push(ordinal); + return; + } const data = visible.data; if (!object(data)) return; const literals = new Map( diff --git a/src/browser/history/SazMetadata.test.ts b/src/browser/history/SazMetadata.test.ts new file mode 100644 index 000000000..a5dbbf783 --- /dev/null +++ b/src/browser/history/SazMetadata.test.ts @@ -0,0 +1,24 @@ +import { expect, it } from "vitest"; +import { decodeSazMetadata } from "./SazMetadata.js"; + +it("retains U+FFFD in well-formed metadata as an ordinary XML character", () => { + const decoded = JSON.stringify( + decodeSazMetadata( + Buffer.from( + '�', + ), + ), + ); + expect(decoded).toContain('{"name":"V","value":"caf�"}'); + expect(decoded).toContain('"value":"�"'); +}); + +it.each([ + "�", + '', + "�", +])("still rejects malformed metadata that also contains U+FFFD: %j", (xml) => { + expect(() => decodeSazMetadata(Buffer.from(xml))).toThrow( + "SAZ metadata is not well-formed XML.", + ); +}); diff --git a/src/browser/history/SazMetadata.ts b/src/browser/history/SazMetadata.ts index 6e311f8e6..f9d1b6733 100644 --- a/src/browser/history/SazMetadata.ts +++ b/src/browser/history/SazMetadata.ts @@ -25,7 +25,15 @@ export const decodeSazMetadata = (bytes: Buffer): JsonValue => { try { document = new DOMParser({ normalizeLineEndings: (source) => source, - onError: () => { + onError: (level, message) => { + // The parser warns whenever the source contains U+FFFD. It is a legal + // XML Char, and the fatal decoder above already rejected invalid + // source bytes, so only that warning is not a well-formedness failure. + if ( + level === "warning" && + message.startsWith("Unicode replacement character detected") + ) + return; parseFailed = true; }, }).parseFromString(xml, "application/xml"); diff --git a/src/browser/network/PlaywrightNetworkContent.ts b/src/browser/network/PlaywrightNetworkContent.ts index 9a35d179f..390831030 100644 --- a/src/browser/network/PlaywrightNetworkContent.ts +++ b/src/browser/network/PlaywrightNetworkContent.ts @@ -33,12 +33,15 @@ const credentialHeaders = new Set([ const errorMessage = ( cause: unknown, secrets: BrowserScenarioSecrets, -): string => - secrets.redact( +): string => { + const message = cause instanceof Error && cause.message !== "" ? cause.message - : "Network read failed", - ); + : "Network read failed"; + return secrets.redactionAllocationCost(message.length) <= 1024 * 1024 + ? secrets.redact(message) + : "Network read failed; diagnostic exceeds the allocation budget."; +}; /** Read actual producer headers without flattening duplicates or exposing credentials. */ export const captureNetworkHeaders = async ( @@ -52,6 +55,15 @@ export const captureNetworkHeaders = async ( options.timeoutMs, options.signal, ); + let cost = 0; + for (const { name, value } of headers) { + cost += secrets.redactionAllocationCost(name.length + value.length) + 128; + if (cost > 1024 * 1024) + return { + state: "unavailable", + message: "Headers exceed the 1 MiB allocation budget.", + }; + } return { state: "captured", items: headers.map(({ name, value }) => { @@ -87,8 +99,31 @@ export const captureNetworkBody = async ( options.signal, ); if (observed === null) return { state: "not_exposed" }; + // This bounds subsequent redaction, base64 and validation copies, but the + // Playwright API itself has already materialized the body (see PR limits). + if (observed.length > 64 * 1024) + return { + state: "unavailable", + reason: "body-read-failed", + message: "Body exceeds the 64 KiB retained-content budget.", + }; + if (secrets.redactionAllocationCost(observed.length) > 1024 * 1024) + return { + state: "unavailable", + reason: "body-read-failed", + message: "Body redaction exceeds the allocation budget.", + }; const retained = secrets.redactBytes(observed); const contentType = source.headers()["content-type"]; + if ( + contentType !== undefined && + secrets.redactionAllocationCost(contentType.length) > 64 * 1024 + ) + return { + state: "unavailable", + reason: "body-read-failed", + message: "Content type exceeds the retained metadata budget.", + }; return { state: "captured", representation: diff --git a/src/browser/network/PlaywrightScenarioNetwork.test.ts b/src/browser/network/PlaywrightScenarioNetwork.test.ts index 0c67c7c3a..97e95a296 100644 --- a/src/browser/network/PlaywrightScenarioNetwork.test.ts +++ b/src/browser/network/PlaywrightScenarioNetwork.test.ts @@ -27,7 +27,7 @@ class RequestFixture implements ScenarioNetworkRequest { resourceType() { return "fetch"; } - headers() { + headers(): Record { return { "content-type": "text/plain", authorization: "Bearer private-transport", @@ -127,7 +127,7 @@ const harness = ( return sequence; }, }); - return { collector, events }; + return { collector, events, secrets }; }; const bodyText = (event: BrowserScenarioEvent) => { @@ -444,10 +444,13 @@ it("links observed redirect hops and leaves unseen predecessors unknown", async collector.request(unseen); await collector.finish(); const requests = events.filter((event) => event.kind === "request"); + expect(requests[0]?.redirected_from_transaction_state).toBe("root"); expect(requests[1]?.redirected_from_transaction_id).toBe( requests[0]?.transaction_id, ); + expect(requests[1]?.redirected_from_transaction_state).toBeUndefined(); expect(requests[2]?.redirected_from_transaction_id).toBeNull(); + expect(requests[2]?.redirected_from_transaction_state).toBe("unknown"); }); it("preserves read errors and does not turn unexposed or empty bytes into the same state", async () => { @@ -533,3 +536,113 @@ it("redacts declared encodings in body media metadata", async () => { }), ); }); + +it("bounds admitted transactions before invoking content producers", async () => { + const { collector, events } = harness(); + const admitted: RequestFixture[] = []; + for (let index = 0; index < 1024; index += 1) { + const request = new RequestFixture(`body-${index}`); + admitted.push(request); + collector.request(request); + collector.request(request); // repeated producer delivery cannot add jobs + } + const rejected = new RequestFixture("rejected"); + const response = new ResponseFixture(rejected, "rejected"); + collector.request(rejected); + collector.response(response); + collector.finished(rejected); + await collector.finish(); + expect(admitted[0]?.bodyReads).toBe(1); + expect( + admitted + .slice(16) + .every((request) => request.bodyReads === 0 && request.headerReads === 0), + ).toBe(true); + expect(rejected.bodyReads).toBe(0); + expect(rejected.headerReads).toBe(0); + expect(response.bodyReads).toBe(0); + expect(response.headerReads).toBe(0); + expect( + events.filter((event) => event.kind === "network-content"), + ).toHaveLength(1024); + expect(collector.limitations().join(" ")).toContain("resource budget"); +}); + +it("keeps header names and selected body when unselected values are oversized", async () => { + const { collector, events } = harness({ + request_body: true, + response_body: false, + header_values: false, + }); + const request = new RequestFixture("retained body"); + request.headers = () => ({ + "x-large-unselected": "x".repeat(1024 * 1024 + 1), + }); + collector.request(request); + await collector.finish(); + + expect(events).toContainEqual( + expect.objectContaining({ + kind: "request", + header_names: ["x-large-unselected"], + }), + ); + expect( + events.find((event) => event.kind === "network-content"), + ).toMatchObject({ + phase: "request", + headers: { state: "not_requested" }, + body: { state: "captured" }, + }); + expect(request.bodyReads).toBe(1); +}); + +it("omits oversized body copies and retains ordinary content", async () => { + const { collector, events } = harness(); + collector.request(new RequestFixture("x".repeat(64 * 1024 + 1))); + collector.request(new RequestFixture("ordinary")); + await collector.finish(); + const content = events.filter((event) => event.kind === "network-content"); + expect(content[0]?.body).toMatchObject({ + state: "unavailable", + message: expect.stringContaining("budget"), + }); + expect(bodyText(content[1]!)).toBe("ordinary"); +}); + +it("reserves redaction expansion before creating replacement copies", async () => { + const { collector, events, secrets } = harness(selection, undefined, "a"); + const redact = vi.spyOn(secrets, "redactBytes"); + collector.request(new RequestFixture("a".repeat(64 * 1024))); + await collector.finish(); + expect( + events.find((event) => event.kind === "network-content")?.body, + ).toMatchObject({ + state: "unavailable", + message: expect.stringContaining("redaction"), + }); + // Ordinary header values can still pass through the byte redactor. + expect(redact.mock.calls.every(([value]) => value.length < 64 * 1024)).toBe( + true, + ); +}); + +it("does not read content when phase metadata was omitted", async () => { + const { secrets } = harness(); + const redact = vi.spyOn(secrets, "redact"); + const collector = new PlaywrightScenarioNetwork({ + selection, + secrets, + admitText: () => false, + emit: () => 0, + }); + const request = new RequestFixture("ordinary"); + collector.request(request); + await collector.finish(); + expect(redact).not.toHaveBeenCalled(); + expect(request.bodyReads + request.headerReads).toBe(0); + expect(collector.limitations().join(" ")).toContain( + "aggregate browser event budget", + ); + expect(collector.limitations().join(" ")).toContain("incomplete"); +}); diff --git a/src/browser/network/PlaywrightScenarioNetwork.ts b/src/browser/network/PlaywrightScenarioNetwork.ts index 45129cb95..24479ef37 100644 --- a/src/browser/network/PlaywrightScenarioNetwork.ts +++ b/src/browser/network/PlaywrightScenarioNetwork.ts @@ -46,18 +46,22 @@ type UnindexedNetworkEvent = BrowserScenarioEvent extends infer Event interface Transaction { readonly id: string; readonly redirectedFrom: string | null; + readonly redirectedFromState: "root" | "unknown" | undefined; response?: { readonly source: ScenarioNetworkResponse; readonly sequence: number; }; terminal?: "finished" | "failed"; responseScheduled: boolean; + requestObserved: boolean; } interface NetworkCaptureOptions { readonly selection: BrowserNetworkContentSelection; readonly secrets: BrowserScenarioSecrets; readonly emit: (event: UnindexedNetworkEvent) => number; + /** Check aggregate event capacity before redacting producer-owned metadata. */ + readonly admitText?: (values: Iterable) => boolean; readonly signal?: AbortSignal; /** Production uses 5s; a fixture can supply its own bounded-read clock. */ readonly readTimeoutMs?: number; @@ -72,6 +76,7 @@ export class PlaywrightScenarioNetwork { private readonly pending: Promise[] = []; private readonly gaps = new Set(); private accepting = true; + private activeReads = 0; private readonly readOptions: NetworkReadOptions; constructor(private readonly options: NetworkCaptureOptions) { @@ -85,21 +90,21 @@ export class PlaywrightScenarioNetwork { request(request: ScenarioNetworkRequest): void { if (!this.accepting) return; const transaction = this.transaction(request); + if (transaction === undefined || transaction.requestObserved) return; + if (!this.admitMetadata(request)) return; + transaction.requestObserved = true; const sequence = this.options.emit({ ...this.metadata(request, transaction), kind: "request", status: null, failure: null, }); + if (sequence === 0) return; if ( this.options.selection.request_body || this.options.selection.header_values ) - this.collect( - transaction, - sequence, - "request", - request, + this.collect(transaction, sequence, "request", request, () => this.options.selection.request_body ? captureNetworkBody( async () => request.postDataBuffer(), @@ -117,6 +122,16 @@ export class PlaywrightScenarioNetwork { if (!this.accepting) return; const request = response.request(); const transaction = this.transaction(request); + if ( + transaction === undefined || + !this.admitMetadata( + request, + undefined, + [], + [{ source: response, url: response.url() }], + ) + ) + return; const sequence = this.options.emit({ ...this.metadata(request, transaction), kind: "response", @@ -125,6 +140,7 @@ export class PlaywrightScenarioNetwork { header_names: this.headerNames(response), failure: null, }); + if (sequence === 0) return; transaction.response = { source: response, sequence }; } @@ -132,13 +148,18 @@ export class PlaywrightScenarioNetwork { finished(request: ScenarioNetworkRequest): void { if (!this.accepting) return; const transaction = this.transaction(request); + if (transaction === undefined || !this.admitMetadata(request)) return; transaction.terminal = "finished"; - this.options.emit({ + const sequence = this.options.emit({ ...this.metadata(request, transaction), kind: "request-finished", status: null, failure: null, }); + if (sequence === 0) { + transaction.responseScheduled = true; + return; + } this.responseContent(transaction); } @@ -146,16 +167,21 @@ export class PlaywrightScenarioNetwork { failed(request: ScenarioNetworkRequest): void { if (!this.accepting) return; const transaction = this.transaction(request); + if (transaction === undefined) return; transaction.terminal = "failed"; - const message = this.options.secrets.redact( - request.failure()?.errorText || "Unknown request failure", - ); - this.options.emit({ + const failure = request.failure()?.errorText || "Unknown request failure"; + if (!this.admitMetadata(request, undefined, [failure])) return; + const message = this.options.secrets.redact(failure); + const sequence = this.options.emit({ ...this.metadata(request, transaction), kind: "request-failed", status: null, failure: message, }); + if (sequence === 0) { + transaction.responseScheduled = true; + return; + } this.responseContent(transaction, { state: "unavailable", reason: "request-failed", @@ -172,10 +198,11 @@ export class PlaywrightScenarioNetwork { this.accepting = false; for (const [request, transaction] of this.transactions) { if (transaction.terminal === undefined) { + if (!this.admitMetadata(request)) continue; this.gaps.add( "Some network requests had not completed when capture ended; later activity is unknown.", ); - this.options.emit({ + const sequence = this.options.emit({ ...this.metadata(request, transaction), kind: "request-unfinished", status: null, @@ -185,6 +212,10 @@ export class PlaywrightScenarioNetwork { ? "cancelled" : "capture-ended", }); + if (sequence === 0) { + transaction.responseScheduled = true; + continue; + } this.responseContent(transaction, { state: "unavailable", reason: @@ -204,22 +235,110 @@ export class PlaywrightScenarioNetwork { return [...this.gaps]; } - private transaction(request: ScenarioNetworkRequest): Transaction { + private transaction( + request: ScenarioNetworkRequest, + ): Transaction | undefined { const known = this.transactions.get(request); if (known !== undefined) return known; + if (this.transactions.size >= 1024) { + this.gaps.add( + "Network collection reached its 1024-transaction resource budget; further transactions were omitted.", + ); + return undefined; + } const predecessor = request.redirectedFrom(); + const predecessorTransaction = + predecessor === null ? undefined : this.transactions.get(predecessor); const transaction: Transaction = { id: `request-${this.transactions.size + 1}`, - redirectedFrom: + redirectedFrom: predecessorTransaction?.id ?? null, + redirectedFromState: predecessor === null - ? null - : (this.transactions.get(predecessor)?.id ?? null), + ? "root" + : predecessorTransaction === undefined + ? "unknown" + : undefined, responseScheduled: false, + requestObserved: false, }; this.transactions.set(request, transaction); return transaction; } + private admitProducerStrings(values: Iterable): boolean { + const collected: string[] = []; + let cost = 0; + for (const value of values) { + cost += this.options.secrets.redactionAllocationCost(value.length) + 128; + if (cost > 1024 * 1024) { + this.gaps.add( + "Some network metadata exceeded its allocation budget and was omitted.", + ); + return false; + } + collected.push(value); + } + if (this.options.admitText?.(collected) ?? true) return true; + this.gaps.add( + "Some network metadata was omitted because the aggregate browser event budget was reached; affected transactions are incomplete.", + ); + return false; + } + + private admitMetadata( + source: NetworkHeadersSource, + url?: string, + additionalStrings: readonly string[] = [], + additionalSources: readonly { + source: NetworkHeadersSource; + url?: string; + }[] = [], + ): boolean { + return this.admitProducerStrings( + this.metadataStrings(source, url, additionalStrings, additionalSources), + ); + } + + private *metadataStrings( + source: NetworkHeadersSource, + url: string | undefined, + additionalStrings: readonly string[], + additionalSources: readonly { + source: NetworkHeadersSource; + url?: string; + }[], + ): Generator { + yield* this.retainedMetadataStrings(source, source.headers(), url); + yield* additionalStrings; + for (const extra of additionalSources) + yield* this.retainedMetadataStrings( + extra.source, + extra.source.headers(), + extra.url, + ); + } + + private *retainedMetadataStrings( + source: NetworkHeadersSource, + headers: Readonly>, + url?: string, + ): Generator { + if (url !== undefined) yield url; + for (const name in headers) { + if (!Object.hasOwn(headers, name)) continue; + yield name; + // Names are always retained in metadata. Values are retained only in + // selected header-value content; do not charge discarded values. + if (this.options.selection.header_values) yield headers[name]!; + } + if ("method" in source) { + const request = source as ScenarioNetworkRequest; + yield request.method(); + yield request.url(); + yield request.resourceType(); + } + } + private headerNames(source: NetworkHeadersSource): string[] { return Object.keys(source.headers()) .map((name) => this.options.secrets.redact(name)) @@ -230,6 +349,11 @@ export class PlaywrightScenarioNetwork { return { transaction_id: transaction.id, redirected_from_transaction_id: transaction.redirectedFrom, + ...(transaction.redirectedFromState === undefined + ? {} + : { + redirected_from_transaction_state: transaction.redirectedFromState, + }), method: this.options.secrets.redact(request.method()), url: this.options.secrets.sanitizeUrl(request.url()), resource_type: this.options.secrets.redact(request.resourceType()), @@ -255,17 +379,18 @@ export class PlaywrightScenarioNetwork { response.sequence, "response", response.source, - this.options.selection.response_body - ? unavailable === undefined - ? captureNetworkBody( - () => response.source.body(), - "response", - response.source, - this.options.secrets, - this.readOptions, - ) - : Promise.resolve(unavailable) - : Promise.resolve({ state: "not_requested" }), + () => + this.options.selection.response_body + ? unavailable === undefined + ? captureNetworkBody( + () => response.source.body(), + "response", + response.source, + this.options.secrets, + this.readOptions, + ) + : Promise.resolve(unavailable) + : Promise.resolve({ state: "not_requested" }), ); } @@ -274,30 +399,54 @@ export class PlaywrightScenarioNetwork { sequence: number, phase: "request" | "response", source: NetworkHeadersSource, - body: Promise, + body: () => Promise, ): void { + if (this.activeReads >= 16) { + const message = "Network content exceeds the 16 concurrent-read budget."; + this.gaps.add(message); + this.options.emit({ + kind: "network-content", + transaction_id: transaction.id, + source_event_sequence: sequence, + phase, + headers: this.options.selection.header_values + ? { state: "unavailable", message } + : { state: "not_requested" }, + body: this.options.selection[ + phase === "request" ? "request_body" : "response_body" + ] + ? { state: "unavailable", reason: "body-read-failed", message } + : { state: "not_requested" }, + }); + return; + } + this.activeReads += 1; const headers = this.options.selection.header_values ? captureNetworkHeaders(source, this.options.secrets, this.readOptions) : Promise.resolve({ state: "not_requested" } as const); this.pending.push( - Promise.all([headers, body]).then(([retainedHeaders, retainedBody]) => { - if ( - retainedHeaders.state === "unavailable" || - retainedBody.state === "unavailable" || - retainedBody.state === "not_exposed" - ) - this.gaps.add( - "Some selected network content was unavailable; per-transaction content states retain the reason.", - ); - this.options.emit({ - kind: "network-content", - transaction_id: transaction.id, - source_event_sequence: sequence, - phase, - headers: retainedHeaders, - body: retainedBody, - }); - }), + Promise.all([headers, body()]) + .then(([retainedHeaders, retainedBody]) => { + if ( + retainedHeaders.state === "unavailable" || + retainedBody.state === "unavailable" || + retainedBody.state === "not_exposed" + ) + this.gaps.add( + "Some selected network content was unavailable; per-transaction content states retain the reason.", + ); + this.options.emit({ + kind: "network-content", + transaction_id: transaction.id, + source_event_sequence: sequence, + phase, + headers: retainedHeaders, + body: retainedBody, + }); + }) + .finally(() => { + this.activeReads -= 1; + }), ); } } diff --git a/src/captures/TsharkCaptureProvider.ts b/src/captures/TsharkCaptureProvider.ts index c4c89b88e..5103ac40b 100644 --- a/src/captures/TsharkCaptureProvider.ts +++ b/src/captures/TsharkCaptureProvider.ts @@ -166,6 +166,7 @@ export class TsharkCaptureProvider implements PacketCapturePort { const configurationPath = join(root.path, "configuration"); await mkdir(configurationPath, { mode: 0o700 }); const execute = async (args: readonly string[]) => { + capturedOutput = undefined; const output = await runOwnedCommand( { command: limiter, diff --git a/src/cli/browserAdvancedCommands.ts b/src/cli/browserAdvancedCommands.ts index dc0c268d1..90f7b35c9 100644 --- a/src/cli/browserAdvancedCommands.ts +++ b/src/cli/browserAdvancedCommands.ts @@ -27,6 +27,7 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { parseCliJsonInput } from "../cliJsonInput.js"; import type { Logger } from "pino"; import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { withCommandCancellation } from "./commandCancellation.js"; /** Register WebMCP, capture-diff, and screenshot CLI equivalents. */ export const registerAdvancedBrowserCommands = ( @@ -61,23 +62,29 @@ const registerWebMcp = ( .describe("Observation duration in milliseconds"), }), run: ({ args, options }) => - logCliCommand(logger, "discover-webmcp-tools", async () => { - const context = browserContext(); - const input = { - cdp_endpoint: args.endpoint, - allowed_origins: options.allowedOrigins, - target_id: args.targetId, - observation_ms: options.observationMs, - }; - const parsed = discoverWebMcpToolsInputSchema.safeParse(input); - if (!parsed.success) - return inputError( - "discover_webmcp_tools", - projectInputIssues(parsed.error.issues, input), + withCommandCancellation((signal) => + logCliCommand(logger, "discover-webmcp-tools", async () => { + const context = browserContext(); + const input = { + cdp_endpoint: args.endpoint, + allowed_origins: options.allowedOrigins, + target_id: args.targetId, + observation_ms: options.observationMs, + }; + const parsed = discoverWebMcpToolsInputSchema.safeParse(input); + if (!parsed.success) + return inputError( + "discover_webmcp_tools", + projectInputIssues(parsed.error.issues, input), + ); + const result = await discoverWebMcpTools( + context.provider, + parsed.data, + { signal }, ); - const result = await discoverWebMcpTools(context.provider, parsed.data); - return result.ok ? result.value : browserCliError(result.error); - }), + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); }; @@ -161,25 +168,28 @@ const registerScreenshot = ( ...browserScopeOptions, }), run: ({ args, options }) => - logCliCommand(logger, "capture-web-screenshot", async () => { - const context = browserContext(); - const input = { - cdp_endpoint: args.endpoint, - allowed_origins: options.allowedOrigins, - target_id: args.targetId, - }; - const parsed = captureWebScreenshotInputSchema.safeParse(input); - if (!parsed.success) - return inputError( - "capture_web_screenshot", - projectInputIssues(parsed.error.issues, input), + withCommandCancellation((signal) => + logCliCommand(logger, "capture-web-screenshot", async () => { + const context = browserContext(); + const input = { + cdp_endpoint: args.endpoint, + allowed_origins: options.allowedOrigins, + target_id: args.targetId, + }; + const parsed = captureWebScreenshotInputSchema.safeParse(input); + if (!parsed.success) + return inputError( + "capture_web_screenshot", + projectInputIssues(parsed.error.issues, input), + ); + const result = await captureWebScreenshot( + context.provider, + parsed.data, + { signal }, ); - const result = await captureWebScreenshot( - context.provider, - parsed.data, - ); - return result.ok ? result.value : browserCliError(result.error); - }), + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); }; diff --git a/src/cli/browserCommands.ts b/src/cli/browserCommands.ts index e9043ec80..d04d3fb55 100644 --- a/src/cli/browserCommands.ts +++ b/src/cli/browserCommands.ts @@ -22,6 +22,7 @@ import { browserScopeOptions, observationDuration, } from "../cliObservationOptions.js"; +import { withCommandCancellation } from "./commandCancellation.js"; /** Register CLI equivalents of the passive browser MCP tools. */ export const registerBrowserCommands = ( @@ -48,24 +49,30 @@ const registerTargetList = ( }), options: z.object({ ...browserScopeOptions }), run: ({ args, options }) => - logCliCommand(logger, "list-browser-targets", async () => { - const context = browserContext(); - const input = { - cdp_endpoint: args.endpoint, - allowed_origins: options.allowedOrigins, - }; - const parsed = listBrowserTargetsInputSchema.safeParse(input); - if (!parsed.success) - return browserCliError( - analysisInputErrorFromIssues( - "list_browser_targets", - parsed.error.issues, - input, - ), + withCommandCancellation((signal) => + logCliCommand(logger, "list-browser-targets", async () => { + const context = browserContext(); + const input = { + cdp_endpoint: args.endpoint, + allowed_origins: options.allowedOrigins, + }; + const parsed = listBrowserTargetsInputSchema.safeParse(input); + if (!parsed.success) + return browserCliError( + analysisInputErrorFromIssues( + "list_browser_targets", + parsed.error.issues, + input, + ), + ); + const result = await listBrowserTargets( + context.provider, + parsed.data, + { signal }, ); - const result = await listBrowserTargets(context.provider, parsed.data); - return result.ok ? result.value : browserCliError(result.error); - }), + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); }; @@ -84,33 +91,37 @@ const registerPageInspection = ( }), options: browserPageInspectionOptions, run: ({ args, options }) => - logCliCommand(logger, "inspect-web-page", async () => { - const context = browserContext(); - const input = { - cdp_endpoint: args.endpoint, - allowed_origins: options.allowedOrigins, - target_id: args.targetId, - observation_ms: options.observationMs, - include_accessibility_text: options.includeAccessibilityText, - include_console_text: options.includeConsoleText, - include_json_body_shapes: options.includeJsonBodyShapes, - include_websocket_shapes: options.includeWebsocketShapes, - include_script_sources: options.includeScriptSources, - include_storage_keys: options.includeStorageKeys, - include_storage_fingerprints: options.includeStorageFingerprints, - }; - const parsed = inspectWebPageInputSchema.safeParse(input); - if (!parsed.success) - return browserCliError( - analysisInputErrorFromIssues( - "inspect_web_page", - parsed.error.issues, - input, - ), - ); - const result = await inspectWebPage(context.provider, parsed.data); - return result.ok ? result.value : browserCliError(result.error); - }), + withCommandCancellation((signal) => + logCliCommand(logger, "inspect-web-page", async () => { + const context = browserContext(); + const input = { + cdp_endpoint: args.endpoint, + allowed_origins: options.allowedOrigins, + target_id: args.targetId, + observation_ms: options.observationMs, + include_accessibility_text: options.includeAccessibilityText, + include_console_text: options.includeConsoleText, + include_json_body_shapes: options.includeJsonBodyShapes, + include_websocket_shapes: options.includeWebsocketShapes, + include_script_sources: options.includeScriptSources, + include_storage_keys: options.includeStorageKeys, + include_storage_fingerprints: options.includeStorageFingerprints, + }; + const parsed = inspectWebPageInputSchema.safeParse(input); + if (!parsed.success) + return browserCliError( + analysisInputErrorFromIssues( + "inspect_web_page", + parsed.error.issues, + input, + ), + ); + const result = await inspectWebPage(context.provider, parsed.data, { + signal, + }); + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); }; @@ -136,28 +147,32 @@ const registerBundleAnalysis = ( .describe("Fetch source maps referenced by captured scripts"), }), run: ({ args, options }) => - logCliCommand(logger, "analyze-web-bundle", async () => { - const context = browserContext(); - const input = { - cdp_endpoint: args.endpoint, - allowed_origins: options.allowedOrigins, - target_id: args.targetId, - observation_ms: options.observationMs, - include_script_sources: true, - fetch_source_maps: options.fetchSourceMaps, - }; - const parsed = analyzeWebBundleInputSchema.safeParse(input); - if (!parsed.success) - return browserCliError( - analysisInputErrorFromIssues( - "analyze_web_bundle", - parsed.error.issues, - input, - ), - ); - const result = await analyzeWebBundle(context.provider, parsed.data); - return result.ok ? result.value : browserCliError(result.error); - }), + withCommandCancellation((signal) => + logCliCommand(logger, "analyze-web-bundle", async () => { + const context = browserContext(); + const input = { + cdp_endpoint: args.endpoint, + allowed_origins: options.allowedOrigins, + target_id: args.targetId, + observation_ms: options.observationMs, + include_script_sources: true, + fetch_source_maps: options.fetchSourceMaps, + }; + const parsed = analyzeWebBundleInputSchema.safeParse(input); + if (!parsed.success) + return browserCliError( + analysisInputErrorFromIssues( + "analyze_web_bundle", + parsed.error.issues, + input, + ), + ); + const result = await analyzeWebBundle(context.provider, parsed.data, { + signal, + }); + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); }; @@ -178,25 +193,31 @@ const registerObservationSession = ( observationMs: observationDuration(10_000, 1), }), run: ({ args, options }) => - logCliCommand(logger, "observe-web-session", async () => { - const context = browserContext(); - const input = { - cdp_endpoint: args.endpoint, - allowed_origins: options.allowedOrigins, - target_id: args.targetId, - observation_ms: options.observationMs, - }; - const parsed = observeWebSessionInputSchema.safeParse(input); - if (!parsed.success) - return browserCliError( - analysisInputErrorFromIssues( - "observe_web_session", - parsed.error.issues, - input, - ), + withCommandCancellation((signal) => + logCliCommand(logger, "observe-web-session", async () => { + const context = browserContext(); + const input = { + cdp_endpoint: args.endpoint, + allowed_origins: options.allowedOrigins, + target_id: args.targetId, + observation_ms: options.observationMs, + }; + const parsed = observeWebSessionInputSchema.safeParse(input); + if (!parsed.success) + return browserCliError( + analysisInputErrorFromIssues( + "observe_web_session", + parsed.error.issues, + input, + ), + ); + const result = await observeWebSession( + context.provider, + parsed.data, + { signal }, ); - const result = await observeWebSession(context.provider, parsed.data); - return result.ok ? result.value : browserCliError(result.error); - }), + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); }; diff --git a/src/cliJavaScriptRuntimeCommands.ts b/src/cliJavaScriptRuntimeCommands.ts index 23ff1e40e..ee8d50442 100644 --- a/src/cliJavaScriptRuntimeCommands.ts +++ b/src/cliJavaScriptRuntimeCommands.ts @@ -16,6 +16,7 @@ import { analysisInputErrorFromIssues } from "./domain/inputIssueProjection.js"; import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; import type { JsonValue } from "./domain/jsonValue.js"; import type { Logger } from "pino"; +import { withCommandCancellation } from "./cli/commandCancellation.js"; const observeOptionsSchema = z.object({ runtimeKind: javascriptRuntimeKindSchema @@ -42,30 +43,33 @@ export const registerJavaScriptRuntimeObservationCommands = ( endpoint: z.string().describe("Literal-loopback Inspector endpoint"), }), run: ({ args }) => - logCliCommand( - logger, - CLI_COMMANDS.listJavaScriptRuntimeTargets, - async () => { - const context = runtimeContext(); - const input = { - inspector_endpoint: args.endpoint, - }; - const parsed = - listJavaScriptRuntimeTargetsInputSchema.safeParse(input); - if (!parsed.success) - return cliError( - analysisInputErrorFromIssues( - "list_javascript_runtime_targets", - parsed.error.issues, - input, - ), + withCommandCancellation((signal) => + logCliCommand( + logger, + CLI_COMMANDS.listJavaScriptRuntimeTargets, + async () => { + const context = runtimeContext(); + const input = { + inspector_endpoint: args.endpoint, + }; + const parsed = + listJavaScriptRuntimeTargetsInputSchema.safeParse(input); + if (!parsed.success) + return cliError( + analysisInputErrorFromIssues( + "list_javascript_runtime_targets", + parsed.error.issues, + input, + ), + ); + const result = await listJavaScriptRuntimeTargets( + context.provider, + parsed.data, + { signal }, ); - const result = await listJavaScriptRuntimeTargets( - context.provider, - parsed.data, - ); - return result.ok ? result.value : cliError(result.error); - }, + return result.ok ? result.value : cliError(result.error); + }, + ), ), }); @@ -79,29 +83,36 @@ export const registerJavaScriptRuntimeObservationCommands = ( }), options: observeOptionsSchema, run: ({ args, options }) => - logCliCommand(logger, CLI_COMMANDS.observeJavaScriptRuntime, async () => { - const context = runtimeContext(); - const input = { - inspector_endpoint: args.endpoint, - target_id: args.targetId, - runtime_kind: options.runtimeKind, - observation_ms: options.observationMs, - }; - const parsed = observeJavaScriptRuntimeInputSchema.safeParse(input); - if (!parsed.success) - return cliError( - analysisInputErrorFromIssues( - "observe_javascript_runtime", - parsed.error.issues, - input, - ), - ); - const result = await observeJavaScriptRuntime( - context.provider, - parsed.data, - ); - return result.ok ? result.value : cliError(result.error); - }), + withCommandCancellation((signal) => + logCliCommand( + logger, + CLI_COMMANDS.observeJavaScriptRuntime, + async () => { + const context = runtimeContext(); + const input = { + inspector_endpoint: args.endpoint, + target_id: args.targetId, + runtime_kind: options.runtimeKind, + observation_ms: options.observationMs, + }; + const parsed = observeJavaScriptRuntimeInputSchema.safeParse(input); + if (!parsed.success) + return cliError( + analysisInputErrorFromIssues( + "observe_javascript_runtime", + parsed.error.issues, + input, + ), + ); + const result = await observeJavaScriptRuntime( + context.provider, + parsed.data, + { signal }, + ); + return result.ok ? result.value : cliError(result.error); + }, + ), + ), }); }; diff --git a/src/domain/analysisView/pcapCaptureView.test.ts b/src/domain/analysisView/pcapCaptureView.test.ts new file mode 100644 index 000000000..6e56ea3e6 --- /dev/null +++ b/src/domain/analysisView/pcapCaptureView.test.ts @@ -0,0 +1,99 @@ +import { expect, it } from "vitest"; +import { createEvidence } from "../evidence.js"; +import { + projectAnalysisView, + type AnalysisViewParent, +} from "./analysisView.js"; + +const capture = (tsharkStderr: string): AnalysisViewParent => { + const target = { + path: "/recordings/input.pcap", + sha256: "c".repeat(64), + bytes: 24, + }; + const report = { + client: { + tshark_command: "/usr/bin/tshark", + tshark_source: "path", + tshark_version: "4.6.6", + capinfos_command: "/usr/bin/capinfos", + capinfos_source: "path", + capinfos_version: "4.6.6", + }, + target, + file_type: "pcap", + encapsulation: "Ethernet", + packet_count: 0, + capture_start: null, + capture_end: null, + capture_duration_seconds: null, + snapshot_length: null, + endpoints: [], + endpoint_count: 0, + dns_queries: [], + dns_query_count: 0, + tls_sni_values: [], + tls_sni_count: 0, + protocol_distribution: [], + protocol_count: 0, + http_request_uris: [], + http_request_uri_count: 0, + http_uri_redaction_count: 0, + unattributed_dns_addresses: [], + unattributed_dns_address_count: 0, + unattributed_dns_packet_count: 0, + coverage: "complete", + diagnostics: { capinfos_stderr: "", tshark_stderr: tsharkStderr }, + }; + const evidence = createEvidence( + { ...target, format: "file" }, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "inspect_pcap_capture", + parameters: { path: target.path }, + result: report, + confidence: "derived", + authority: "historical-reference", + }, + ); + return { + evidenceId: evidence.evidence_id, + operation: evidence.operation, + normalizedResult: report, + subject: evidence.subject, + limitations: evidence.limitations, + }; +}; + +const diagnostics = (offset: number, limit: number) => + ({ + kind: "pcap", + facet: "diagnostics", + stream: "tshark", + offset, + limit, + }) as const; + +it("pages diagnostics without splitting a surrogate pair", () => { + const parent = capture("ab😀cd"); + expect(projectAnalysisView(parent, diagnostics(3, 2))).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + const first = projectAnalysisView(parent, diagnostics(0, 3)); + if (!first.ok) throw first.error; + expect(first.value).toMatchObject({ + page: { facet: "diagnostics", text: "ab", status: "retained" }, + coverage: { examined: 2, total: 6, next_offset: 2 }, + }); + let text = ""; + for (let offset: number | null = 0; offset !== null;) { + const page = projectAnalysisView(parent, diagnostics(offset, 3)); + if (!page.ok) throw page.error; + if (page.value.kind !== "pcap" || page.value.page.facet !== "diagnostics") + throw new Error("Expected a diagnostics page."); + text += page.value.page.text; + offset = page.value.coverage.next_offset; + } + expect(text).toBe("ab😀cd"); +}); diff --git a/src/domain/analysisView/pcapCaptureView.ts b/src/domain/analysisView/pcapCaptureView.ts index a28398d44..c388f7989 100644 --- a/src/domain/analysisView/pcapCaptureView.ts +++ b/src/domain/analysisView/pcapCaptureView.ts @@ -13,6 +13,7 @@ import { completeWithinViewCoverage, pageViewCoverage, } from "./analysisViewCoverage.js"; +import { analysisTextPage } from "./analysisViewText.js"; const captureSchema = pcapResultSchemas.inspect_pcap_capture; const fields = captureSchema.shape; @@ -288,7 +289,13 @@ export const projectPcapCaptureView = ( view.stream === "tshark" ? capture.diagnostics?.tshark_stderr : capture.diagnostics?.capinfos_stderr; - const text = value?.slice(view.offset, view.offset + view.limit) ?? ""; + const text = analysisTextPage( + value ?? "", + view.offset, + view.limit, + "Diagnostics", + ); + if (!text.ok) return text; return ok({ kind: "pcap", view, @@ -296,10 +303,10 @@ export const projectPcapCaptureView = ( source_coverage: capture.coverage, page: { facet: "diagnostics", - text, + text: text.value.item.text, status: value === undefined ? "unknown" : "retained", }, - coverage: pageViewCoverage(view.offset, text.length, value?.length ?? 0), + coverage: text.value.coverage, unknowns: value === undefined ? [ diff --git a/src/domain/analysisView/webNetworkCaptureView.test.ts b/src/domain/analysisView/webNetworkCaptureView.test.ts index ff73b29a6..7035c2b95 100644 --- a/src/domain/analysisView/webNetworkCaptureView.test.ts +++ b/src/domain/analysisView/webNetworkCaptureView.test.ts @@ -174,3 +174,686 @@ it("rejects invalid ordinal coverage in a borrowed parent", () => { ); expect(result.ok).toBe(false); }); + +const nativePreview = ( + format: "mitmproxy" | "har", + request: Readonly>, + { + response = { + http_version: "HTTP/1.1", + status_code: 404, + reason: "Not Found", + content: "omitted body", + }, + producerType = "integer", + sidecars, + flowType = "http", + redactions = [], + binaryFields = [], + }: { + readonly response?: Readonly>; + readonly producerType?: "integer" | "float"; + readonly flowType?: string | null; + readonly binaryFields?: readonly Record[]; + readonly redactions?: readonly { + pointer: string; + reason: string; + scope?: string; + }[]; + readonly sidecars?: readonly { + pointer: string; + producer_type: string; + literal: string; + }[]; + } = {}, +) => { + // The bridge records a numeric sidecar for every native number. + const literals = + sidecars ?? + Object.entries({ request, response }).flatMap(([side, fields]) => + Object.entries(fields) + .filter(([, value]) => typeof value === "number") + .map(([field, value]) => ({ + pointer: `/${side}/${field}`, + producer_type: producerType, + literal: String(value), + })), + ); + const artifact = { + path: "/recordings/flows.mitm", + sha256: "c".repeat(64), + bytes: 100, + }; + const decoder = { id: "fixture", name: "Fixture", version: "1" }; + const report = webNetworkCaptureSchema.parse({ + artifact, + decoder, + format, + runtime_attribution: "unknown", + limitations: [], + container: { + reported: null, + numeric_literals: [], + redactions: [], + records_pointer: null, + }, + total_records: 1, + records: [ + { + ordinal: 0, + location: { kind: "byte-range", offset: 0, bytes: 10 }, + reported: { + ...(flowType === null ? {} : { type: flowType }), + request: { ...request, headers: [["Accept", "*/*"]] }, + response, + }, + binary_fields: binaryFields, + numeric_literals: literals, + redactions, + limitations: [], + }, + ], + }); + const evidence = createEvidence({ ...artifact, format: "file" }, decoder, { + operation: "inspect_web_network_capture", + parameters: {}, + result: report, + confidence: "observed", + authority: "historical-reference", + }); + const preview = projectAnalysisView( + { + evidenceId: evidence.evidence_id, + operation: evidence.operation, + normalizedResult: report, + subject: evidence.subject, + limitations: evidence.limitations, + }, + { kind: "capture", detail: "preview", offset: 0, limit: 1 }, + ); + if (!preview.ok || preview.value.kind !== "capture") + throw new Error("Expected a capture preview page."); + return preview.value; +}; + +it("previews the mitmproxy request target and status code as headline fields", () => { + const preview = nativePreview("mitmproxy", { + host: "example.test", + port: 443, + method: "GET", + scheme: "https", + authority: "", + path: "/users/42", + http_version: "HTTP/1.1", + }); + expect(preview).toMatchObject({ + items: [ + { + reported: { + type: "http", + request: { + method: "GET", + scheme: "https", + host: "example.test", + port: 443, + authority: "", + path: "/users/42", + http_version: "HTTP/1.1", + }, + response: { + http_version: "HTTP/1.1", + status_code: 404, + reason: "Not Found", + }, + }, + numeric_literals: [ + { + pointer: "/request/port", + producer_type: "integer", + literal: "443", + }, + { + pointer: "/response/status_code", + producer_type: "integer", + literal: "404", + }, + ], + }, + ], + }); + expect(JSON.stringify(preview)).not.toContain("Accept"); + expect(JSON.stringify(preview)).not.toContain("omitted body"); +}); + +it("previews the asterisk-form OPTIONS target", () => { + expect( + nativePreview("mitmproxy", { method: "OPTIONS", path: "*" }).items[0] + ?.reported, + ).toMatchObject({ request: { method: "OPTIONS", path: "*" } }); +}); + +it.each([ + [ + "an ambiguous absolute path", + "mitmproxy", + "https:user:password@example.test/a", + ], + ["a network-path reference", "mitmproxy", "//user:password@example.test/a"], + [ + "a slash-backslash reference", + "mitmproxy", + "/\\user:password@example.test/a", + ], + [ + "a slash-backslash reference with an unparseable port", + "mitmproxy", + "/\\user:password@example.test:65536/a", + ], + ["a path with an at-sign", "mitmproxy", "/users/password@example"], + ["a slash-tab reference", "mitmproxy", "/\t/user:password@example.test/a"], + ["an oversized path", "mitmproxy", `/${"password".repeat(1024)}`], + ["a nested path value", "mitmproxy", { content: "password ".repeat(64) }], + ["a path in a non-mitmproxy record", "har", "/users/password"], +] as const)("keeps %s out of the preview", (_label, format, path) => { + const preview = nativePreview(format, { method: "GET", path }); + const [item] = preview.items; + expect(item?.reported).toMatchObject({ request: { method: "GET" } }); + expect(JSON.stringify(item?.reported)).not.toContain("password"); + if (format === "har") + expect(JSON.stringify(item?.reported)).not.toContain("status_code"); +}); + +it("bounds the serialized mitmproxy headline of each preview record", () => { + const control = "\u0001".repeat(8 * 1024); + const malformed = nativePreview("mitmproxy", { + method: "GET", + scheme: control, + host: control, + port: control, + authority: control, + path: `/${control}`, + }); + expect(malformed.items[0]?.reported).toMatchObject({ + request: { method: "GET" }, + }); + expect( + Buffer.byteLength(JSON.stringify(malformed.items[0]?.reported)), + ).toBeLessThan(1024); + const escaped = nativePreview("mitmproxy", { + method: "GET", + path: `/${'"'.repeat(2 * 1024 - 1)}`, + }); + expect(escaped.items[0]?.reported).toMatchObject({ + request: { path: `/${'"'.repeat(2 * 1024 - 1)}` }, + }); + expect( + Buffer.byteLength(JSON.stringify(escaped.items[0]?.reported)), + ).toBeLessThan(5 * 1024); +}); + +it("keeps producer floats out of integer mitmproxy headline slots", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", port: 443, path: "/a" }, + { + response: { + http_version: "HTTP/1.1", + status_code: 404, + reason: "Not Found", + }, + producerType: "float", + }, + ).items[0]?.reported; + expect(reported).toMatchObject({ + request: { method: "GET", path: "/a" }, + response: { reason: "Not Found" }, + }); + expect(reported).not.toMatchObject({ request: { port: 443 } }); + expect(reported).not.toMatchObject({ response: { status_code: 404 } }); +}); + +it.each([ + [ + "an oversized literal", + [ + { + pointer: "/request/port", + producer_type: "integer", + literal: "4".repeat(100_000), + }, + ], + ], + [ + "a literal that does not represent the value", + [{ pointer: "/request/port", producer_type: "integer", literal: "444" }], + ], + [ + "duplicate sidecars", + [ + { pointer: "/request/port", producer_type: "integer", literal: "443" }, + { pointer: "/request/port", producer_type: "integer", literal: "443" }, + ], + ], +])( + "keeps a port whose sidecar has %s out of the preview", + (_label, sidecars) => { + const [item] = nativePreview( + "mitmproxy", + { method: "GET", port: 443, path: "/a" }, + { response: { http_version: "HTTP/1.1", reason: "OK" }, sidecars }, + ).items; + expect(item?.reported).toEqual({ + type: "http", + request: { method: "GET", path: "/a" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); + expect(item?.numeric_literals).toEqual([]); + }, +); + +it.each(["tcp", null])( + "adds no mitmproxy headline fields to a %s flow", + (flowType) => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a", port: 443 }, + { flowType }, + ).items[0]?.reported; + expect(reported).toEqual({ + ...(flowType === null ? {} : { type: flowType }), + request: { method: "GET" }, + response: { http_version: "HTTP/1.1", reason: "Not Found" }, + }); + }, +); + +it("keeps a string target that carries a numeric sidecar out of the preview", () => { + const [item] = nativePreview( + "mitmproxy", + { method: "GET", path: "/a" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + sidecars: [ + { + pointer: "/request/path", + producer_type: "integer", + literal: "9".repeat(100_000), + }, + ], + }, + ).items; + expect(item?.reported).toEqual({ + type: "http", + request: { method: "GET" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); + expect(item?.numeric_literals).toEqual([]); +}); + +it("keeps a mitmproxy headline the record marks as redacted out of the preview", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", host: "secret", path: "/a" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + redactions: [ + { pointer: "/request/host", reason: "explicit-sensitive-value" }, + ], + }, + ).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: { method: "GET", path: "/a" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); +}); + +it("rejects a non-number integer headline without throwing", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a", port: { toString: null } }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + sidecars: [ + { pointer: "/request/port", producer_type: "integer", literal: "443" }, + ], + }, + ).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: { method: "GET", path: "/a" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); +}); + +it.each([ + ["the request", "/request", { method: "GET" }], + ["the record root", "", {}], +] as const)( + "keeps mitmproxy headlines under a redacted %s out of the preview", + (_label, pointer, request) => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", host: "secret", path: "/secret" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + redactions: [{ pointer, reason: "explicit-sensitive-value" }], + }, + ).items[0]?.reported; + expect(JSON.stringify(reported)).not.toContain("secret"); + expect(reported).toMatchObject({ request }); + }, +); + +it("keeps a mitmproxy headline with a redacted byte descriptor out of the preview", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", host: "example.test", path: "/secret" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + binaryFields: [ + { + pointer: "/request/path", + representation: "producer-bytes", + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + }, + ], + }, + ).items[0]?.reported; + expect(reported).toMatchObject({ + request: { method: "GET", host: "example.test" }, + }); + expect(JSON.stringify(reported)).not.toContain("secret"); +}); + +it("adds no mitmproxy headline fields when the flow type is redacted", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + redactions: [{ pointer: "/type", reason: "explicit-sensitive-value" }], + }, + ).items[0]?.reported; + expect(reported).toMatchObject({ request: { method: "GET" } }); + expect(reported).not.toMatchObject({ request: { path: "/a" } }); +}); + +it.each([ + ["the flow type", "/type"], + ["the record root", ""], +])( + "adds no mitmproxy headline fields when %s carries a numeric sidecar", + (_label, pointer) => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + sidecars: [{ pointer, producer_type: "integer", literal: "1" }], + }, + ).items[0]?.reported; + expect(reported).toMatchObject({ request: { method: "GET" } }); + expect(reported).not.toMatchObject({ request: { path: "/a" } }); + }, +); + +const retainedBytes = (pointer: string, text: string) => { + const bytes = Buffer.from(text); + return { + pointer, + representation: "producer-bytes", + state: "retained", + content_base64: bytes.toString("base64"), + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +}; + +it("adds no mitmproxy headline fields when the flow type was producer bytes", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + binaryFields: [retainedBytes("/type", "http")], + }, + ).items[0]?.reported; + expect(reported).toMatchObject({ type: "http", request: { method: "GET" } }); + expect(reported).not.toMatchObject({ request: { path: "/a" } }); +}); + +it("keeps mitmproxy headlines under a byte-valued side out of the preview", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a", port: 443 }, + { + response: { http_version: "HTTP/1.1", reason: "OK", status_code: 200 }, + binaryFields: [ + retainedBytes("/request", "GET /a"), + retainedBytes("/response", "200"), + ], + }, + ).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: { method: "GET" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); +}); + +it("previews a byte-backed path but not a byte-backed port", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a", port: 443 }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + binaryFields: [ + retainedBytes("/request/path", "/a"), + retainedBytes("/request/port", "443"), + ], + }, + ).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: { method: "GET", path: "/a" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); +}); + +it("adds no mitmproxy headline fields when the flow type has a child facet", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + sidecars: [ + { pointer: "/type/child", producer_type: "integer", literal: "1" }, + ], + }, + ).items[0]?.reported; + expect(reported).toMatchObject({ request: { method: "GET" } }); + expect(reported).not.toMatchObject({ request: { path: "/a" } }); +}); + +it.each([ + [ + "a child sidecar", + { + sidecars: [ + { pointer: "/request/path/0", producer_type: "integer", literal: "1" }, + ], + }, + ], + [ + "a child redaction", + { + redactions: [ + { pointer: "/request/path/0", reason: "explicit-sensitive-value" }, + ], + }, + ], + [ + "HAR content bytes", + { + binaryFields: [ + { + ...retainedBytes("/request/path", "/a"), + representation: "har-base64-content", + }, + ], + }, + ], + [ + "bytes that are not its text", + { binaryFields: [retainedBytes("/request/path", "/b")] }, + ], + [ + "a byte count that is not its text", + { binaryFields: [{ ...retainedBytes("/request/path", "/a"), bytes: 3 }] }, + ], + [ + "a digest that is not its text", + { + binaryFields: [ + { ...retainedBytes("/request/path", "/a"), sha256: "0".repeat(64) }, + ], + }, + ], +] as const)( + "keeps a mitmproxy path with %s out of the preview", + (_label, facets) => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", host: "example.test", path: "/a" }, + { response: { http_version: "HTTP/1.1", reason: "OK" }, ...facets }, + ).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: { method: "GET", host: "example.test" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); + }, +); + +it.each([ + ["path", { method: "GET", path: "/\ud800" }, { method: "GET" }], + [ + "host", + { method: "GET", host: "a\udc00.test", path: "/a" }, + { method: "GET", path: "/a" }, + ], +])( + "keeps a mitmproxy %s with an unpaired surrogate out of the preview", + (_field, request, expected) => { + const reported = nativePreview("mitmproxy", request, { + response: { http_version: "HTTP/1.1", reason: "OK" }, + }).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: expected, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); + }, +); + +it("keeps an unpaired-surrogate path out despite replacement-character bytes", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/\ud800" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + binaryFields: [retainedBytes("/request/path", "/�")], + }, + ).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: { method: "GET" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); +}); + +it("keeps mitmproxy headlines under a numeric sidecar out of the preview", () => { + const [item] = nativePreview( + "mitmproxy", + { method: "GET", path: "/a", port: 443 }, + { + response: { http_version: "HTTP/1.1", reason: "OK", status_code: 200 }, + sidecars: [ + { pointer: "/request", producer_type: "integer", literal: "7" }, + { pointer: "/request/port", producer_type: "integer", literal: "443" }, + { + pointer: "/response/status_code", + producer_type: "integer", + literal: "200", + }, + ], + }, + ).items; + expect(item?.reported).toEqual({ + type: "http", + request: { method: "GET" }, + response: { http_version: "HTTP/1.1", reason: "OK", status_code: 200 }, + }); + expect(item?.numeric_literals).toEqual([ + { + pointer: "/response/status_code", + producer_type: "integer", + literal: "200", + }, + ]); +}); + +it("keeps sibling mitmproxy headlines next to an omitted property name", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", host: "example.test", path: "/a" }, + { + response: { http_version: "HTTP/1.1", reason: "OK" }, + redactions: [ + { + pointer: "/request", + reason: "explicit-sensitive-value", + scope: "property-name", + }, + ], + }, + ).items[0]?.reported; + expect(reported).toMatchObject({ + request: { method: "GET", host: "example.test", path: "/a" }, + }); +}); + +it("selects mitmproxy target fields from the request and the status from the response only", () => { + const reported = nativePreview( + "mitmproxy", + { method: "GET", path: "/a", status_code: 200 }, + { + response: { http_version: "HTTP/1.1", reason: "OK", path: "/extension" }, + }, + ).items[0]?.reported; + expect(reported).toEqual({ + type: "http", + request: { method: "GET", path: "/a" }, + response: { http_version: "HTTP/1.1", reason: "OK" }, + }); +}); + +it.each(["scheme", "host", "port", "authority"])( + "keeps a userinfo-bearing mitmproxy %s out of the preview", + (field) => { + const preview = nativePreview("mitmproxy", { + method: "GET", + path: "/users/42", + [field]: "user:password@example.test", + }); + const reported = preview.items[0]?.reported; + expect(reported).toMatchObject({ + request: { method: "GET", path: "/users/42" }, + }); + expect(JSON.stringify(reported)).not.toContain("password"); + }, +); diff --git a/src/domain/analysisView/webNetworkCaptureView.ts b/src/domain/analysisView/webNetworkCaptureView.ts index b335e9241..f33bdb0df 100644 --- a/src/domain/analysisView/webNetworkCaptureView.ts +++ b/src/domain/analysisView/webNetworkCaptureView.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; @@ -62,12 +63,290 @@ const object = ( ? value : null; +const HEADLINE_FIELDS = [ + "method", + "url", + "httpVersion", + "http_version", + "status", + "statusText", + "reason", +]; +// Native mitmproxy flows report the request target in parts and the status +// as status_code. They are previewed only for mitmproxy records and only when +// each value has its field's well-formed shape, which also bounds the +// serialized size of every record (at most ~4 KiB for an escape-heavy path). +// The path must be origin or asterisk form without control characters and +// without any "@": native redaction misses absolute and ambiguous spellings, +// and URL parsing can fail before it recognises their userinfo, so a preview +// fails closed on the marker every userinfo needs. Host and authority never +// carry a userinfo marker either. +// Target fields come only from the request and the status only from the +// response, and integer slots require the producer's own integer type. +// Excluded values stay in complete record pages. +const MITMPROXY_HEADLINE_FIELDS: Readonly> = { + request: ["scheme", "host", "port", "authority", "path"], + response: ["status_code"], +}; +const MITMPROXY_PATH_MAX_LENGTH = 2 * 1024; +const integerIn = (value: JsonValue, min: number, max: number): boolean => + typeof value === "number" && + Number.isInteger(value) && + value >= min && + value <= max; +/** True when a code unit is C0, DEL or (optionally) a space. */ +const hasControl = (value: string, includeSpace: boolean): boolean => { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code < 0x20 || code === 0x7f || (includeSpace && code === 0x20)) + return true; + } + return false; +}; +// An unpaired surrogate has no UTF-8 form, so no producer reported it as text. +const LONE_SURROGATE = /\p{Cs}/u; +const hostLike = (value: JsonValue, maxLength: number): boolean => + typeof value === "string" && + value.length <= maxLength && + !hasControl(value, true) && + !/[@/\\]/u.test(value) && + !LONE_SURROGATE.test(value); +type NumericSidecar = WebNetworkCaptureRecord["numeric_literals"][number]; +type BinaryField = WebNetworkCaptureRecord["binary_fields"][number]; +const MITMPROXY_INTEGER_FIELDS = new Set(["port", "status_code"]); +/** Retained native producer bytes whose content, length and digest are all the text's UTF-8. */ +const describesText = (descriptor: BinaryField, text: string): boolean => { + if ( + descriptor.representation !== "producer-bytes" || + descriptor.state !== "retained" + ) + return false; + // An unpaired surrogate has no UTF-8 form; Buffer.from would invent + // replacement-character bytes that no producer value could have had. + const utf8 = Buffer.from(text, "utf8"); + return ( + utf8.toString("utf8") === text && + descriptor.bytes === utf8.length && + descriptor.content_base64 === utf8.toString("base64") && + descriptor.sha256 === createHash("sha256").update(utf8).digest("hex") + ); +}; +/** + * Integer slots need exactly one integer sidecar whose literal is the value's + * canonical decimal, and no byte descriptor. String slots need no sidecar and + * at most one byte descriptor, which must describe the value's own UTF-8. + * Selecting a field also selects its sidecars, so this keeps every copied + * literal small and consistent. + */ +const representationMatches = ( + field: string, + value: JsonValue, + sidecars: readonly NumericSidecar[], + bytes: readonly BinaryField[], +): boolean => { + if (MITMPROXY_INTEGER_FIELDS.has(field)) + return ( + bytes.length === 0 && + typeof value === "number" && + sidecars.length === 1 && + sidecars[0]?.producer_type === "integer" && + sidecars[0].literal === String(value) + ); + if (sidecars.length !== 0 || bytes.length > 1) return false; + const [descriptor] = bytes; + return ( + descriptor === undefined || + (typeof value === "string" && describesText(descriptor, value)) + ); +}; +const headlineShape = (field: string, value: JsonValue): boolean => { + switch (field) { + case "scheme": + return ( + typeof value === "string" && + /^[A-Za-z][A-Za-z0-9+.-]{0,31}$/u.test(value) + ); + case "host": + return hostLike(value, 255); + case "authority": + return hostLike(value, 262); + case "port": + return integerIn(value, 0, 65_535); + case "status_code": + return integerIn(value, 100, 999); + case "path": + return ( + typeof value === "string" && + value.length <= MITMPROXY_PATH_MAX_LENGTH && + !hasControl(value, false) && + !value.includes("@") && + !LONE_SURROGATE.test(value) && + (value === "*" || + (value.startsWith("/") && + !value.startsWith("//") && + !value.startsWith("/\\"))) + ); + default: + return false; + } +}; +/** Strict ancestors of a JSON pointer, from the root down. */ +const ancestorsOf = (pointer: string): string[] => { + const ancestors: string[] = []; + for ( + let at = pointer.indexOf("/"); + at !== -1; + at = pointer.indexOf("/", at + 1) + ) + ancestors.push(pointer.slice(0, at)); + return ancestors; +}; +/** Add a pointer's strict ancestors, stopping at one already present (its own ancestors are too). */ +const addAncestors = (pointer: string, into: Set): void => { + for ( + let at = pointer.lastIndexOf("/"); + at !== -1; + at = at === 0 ? -1 : pointer.lastIndexOf("/", at - 1) + ) { + const ancestor = pointer.slice(0, at); + if (into.has(ancestor)) return; + into.add(ancestor); + } +}; +const groupByPointer = ( + facets: readonly T[], +): Map => { + const groups = new Map(); + for (const facet of facets) { + const group = groups.get(facet.pointer); + if (group === undefined) groups.set(facet.pointer, [facet]); + else group.push(facet); + } + return groups; +}; + +/** A record's facet coordinates, indexed once for every headline check. */ +type FacetIndex = { + readonly sidecars: ReadonlyMap; + readonly bytes: ReadonlyMap; + /** The producer held a number or bytes here. */ + readonly scalars: ReadonlySet; + /** A facet lies beneath: the producer held a container here. */ + readonly containers: ReadonlySet; + /** + * Withheld by a value redaction or a redacted byte descriptor; either facet + * withholds the value even when the other one is missing. + */ + readonly withheld: ReadonlySet; + /** The parent of an omitted property name; it hides no sibling values. */ + readonly omittedNames: ReadonlySet; +}; +const indexFacets = (record: WebNetworkCaptureRecord): FacetIndex => { + const containers = new Set(); + const withheld = new Set(); + const omittedNames = new Set(); + for (const { pointer } of record.numeric_literals) + addAncestors(pointer, containers); + for (const { pointer, state } of record.binary_fields) { + addAncestors(pointer, containers); + if (state === "redacted") withheld.add(pointer); + } + for (const redaction of record.redactions) { + addAncestors(redaction.pointer, containers); + ("scope" in redaction ? omittedNames : withheld).add(redaction.pointer); + } + const sidecars = groupByPointer(record.numeric_literals); + const bytes = groupByPointer(record.binary_fields); + return { + sidecars, + bytes, + scalars: new Set([...sidecars.keys(), ...bytes.keys()]), + containers, + withheld, + omittedNames, + }; +}; +/** + * A value the record marks as redacted, or that lies under a redacted value, + * is never promoted even if it is still present (imported or caller-supplied + * Evidence). A scalar is also inconsistent when a numeric sidecar or byte + * descriptor sits on an ancestor (the producer held a scalar there) or any + * facet sits beneath it (the producer held a container at it). + */ +const usableScalar = (index: FacetIndex, pointer: string): boolean => { + const ancestors = ancestorsOf(pointer); + return ( + !index.omittedNames.has(pointer) && + !index.containers.has(pointer) && + !index.withheld.has(pointer) && + !ancestors.some( + (ancestor) => index.withheld.has(ancestor) || index.scalars.has(ancestor), + ) + ); +}; +/** + * The container format alone does not make a flow HTTP; its own type does, + * when the producer supplied that type as text rather than a number or bytes. + */ +const textualFlowType = (index: FacetIndex): boolean => + usableScalar(index, "/type") && !index.scalars.has("/type"); +/** Shape checks run first, so the representation check only sees bounded values. */ +const mitmproxyHeadline = ( + index: FacetIndex, + pointer: string, + field: string, + value: JsonValue, +): boolean => + usableScalar(index, pointer) && + headlineShape(field, value) && + representationMatches( + field, + value, + index.sidecars.get(pointer) ?? [], + index.bytes.get(pointer) ?? [], + ); + +/** Copy one message side's headlines; mitmproxy fields need the record's facet index. */ +const selectHeadlines = ( + fields: Readonly>, + key: string, + facets: FacetIndex | null, + selectedPointers: Set, +): Record => { + const selected: Record = {}; + const candidates = [ + ...HEADLINE_FIELDS.map((field) => [field, null] as const), + ...(facets === null ? [] : (MITMPROXY_HEADLINE_FIELDS[key] ?? [])).map( + (field) => [field, facets] as const, + ), + ]; + for (const [field, index] of candidates) { + const value = fields[field]; + if ( + !Object.hasOwn(fields, field) || + value === undefined || + (index !== null && + !mitmproxyHeadline(index, `/${key}/${field}`, field, value)) + ) + continue; + selected[field] = value; + selectedPointers.add(`/${key}/${field}`); + } + return selected; +}; + /** Select already parsed producer facts without interpreting HTTP or payload bytes again. */ -const preview = (record: WebNetworkCaptureRecord) => { +const preview = (record: WebNetworkCaptureRecord, format: string) => { const source = object(record.reported); const reported: Record = {}; const selectedPointers = new Set(); if (source !== null) { + const index = + format === "mitmproxy" && source.type === "http" + ? indexFacets(record) + : null; + const mitmproxyFacets = + index !== null && textualFlowType(index) ? index : null; for (const key of ["session_id", "name", "time", "startedDateTime", "type"]) if (Object.hasOwn(source, key) && source[key] !== undefined) { reported[key] = source[key]; @@ -82,21 +361,12 @@ const preview = (record: WebNetworkCaptureRecord) => { reported[key] = value; continue; } - const selected: Record = {}; - for (const field of [ - "method", - "url", - "httpVersion", - "http_version", - "status", - "statusText", - "reason", - ]) - if (Object.hasOwn(fields, field) && fields[field] !== undefined) { - selected[field] = fields[field]; - selectedPointers.add(`/${key}/${field}`); - } - reported[key] = selected; + reported[key] = selectHeadlines( + fields, + key, + mitmproxyFacets, + selectedPointers, + ); } } return { @@ -245,7 +515,9 @@ export const projectWebNetworkCaptureView = ( view, ...shared, items: captureViewItemsSchema.parse( - view.detail === "complete" ? records : records.map(preview), + view.detail === "complete" + ? records + : records.map((record) => preview(record, report.format)), ), coverage: pageViewCoverage( view.offset, diff --git a/src/domain/browserScenarioCapture.test.ts b/src/domain/browserScenarioCapture.test.ts index 8a7b39283..bc075967d 100644 --- a/src/domain/browserScenarioCapture.test.ts +++ b/src/domain/browserScenarioCapture.test.ts @@ -15,6 +15,7 @@ const networkEvent = { step_index: 0, transaction_id: "request-1", redirected_from_transaction_id: null, + redirected_from_transaction_state: "root", method: "GET", url: sanitizeBrowserUrl("https://example.test/resource"), resource_type: "document", @@ -40,6 +41,13 @@ it("requires transaction IDs and explicit unknown redirect predecessors", () => redirected_from_transaction_id: undefined, }).success, ).toBe(false); + expect( + browserScenarioEventSchema.safeParse({ + ...request, + redirected_from_transaction_id: "request-0", + redirected_from_transaction_state: "unknown", + }).success, + ).toBe(false); }); it("parses each network outcome without contradictory status or failure data", () => { diff --git a/src/domain/browserScenarioCaptureValues.ts b/src/domain/browserScenarioCaptureValues.ts index 26c1559bb..aa00f222a 100644 --- a/src/domain/browserScenarioCaptureValues.ts +++ b/src/domain/browserScenarioCaptureValues.ts @@ -144,6 +144,8 @@ const networkEventShape = { ...eventBase, transaction_id: z.string().min(1), redirected_from_transaction_id: z.string().min(1).nullable(), + /** Distinguishes a captured root from an unobserved predecessor for null links. */ + redirected_from_transaction_state: z.enum(["root", "unknown"]).optional(), method: z.string().min(1), url: sanitizedBrowserUrlSchema, resource_type: z.string().min(1), @@ -157,96 +159,110 @@ const webSocketFrameShape = { truncated: z.boolean(), }; -export const browserScenarioEventSchema = z.union([ - z.strictObject({ - ...eventBase, - kind: z.literal("console"), - level: z.string().min(1), - text: z.string(), - url: eventUrl, - }), - z.strictObject({ - ...eventBase, - kind: z.literal("page-error"), - message: z.string(), - stack: z.string().nullable(), - }), - z.strictObject({ - ...networkEventShape, - kind: z.literal("request"), - status: z.null(), - failure: z.null(), - }), - z.strictObject({ - ...networkEventShape, - kind: z.literal("response"), - status: z.number().int().min(100).max(599), - failure: z.null(), - }), - z.strictObject({ - ...networkEventShape, - kind: z.literal("request-failed"), - status: z.null(), - failure: z.string().min(1), - }), - z.strictObject({ - ...networkEventShape, - kind: z.literal("request-finished"), - status: z.null(), - failure: z.null(), - }), - z.strictObject({ - ...networkEventShape, - kind: z.literal("request-unfinished"), - status: z.null(), - failure: z.null(), - reason: z.enum(["capture-ended", "cancelled"]), - }), - z.strictObject({ - ...eventBase, - kind: z.literal("network-content"), - transaction_id: z.string().min(1), - source_event_sequence: z.number().int().min(1), - phase: z.enum(["request", "response"]), - headers: browserNetworkHeadersSchema, - body: browserNetworkBodySchema, - }), - z.strictObject({ - ...eventBase, - kind: z.enum(["websocket-opened", "websocket-closed"]), - url: sanitizedBrowserUrlSchema, - }), - z.strictObject({ - ...webSocketFrameShape, - payload_type: z.literal("text"), - payload_text: z.string(), - }), - z.strictObject({ - ...webSocketFrameShape, - payload_type: z.literal("binary"), - payload_text: z.null(), - }), - z.strictObject({ - ...eventBase, - kind: z.enum([ - "frame-attached", - "frame-detached", - "frame-navigated", - "worker-created", - "worker-closed", - "popup-opened", - "popup-closed", - ]), - url: eventUrl, - name: z.string().nullable(), - }), - z.strictObject({ - ...eventBase, - kind: z.literal("download-cancelled"), - suggested_filename: z.string(), - url: sanitizedBrowserUrlSchema, - }), -]); +export const browserScenarioEventSchema = z + .union([ + z.strictObject({ + ...eventBase, + kind: z.literal("console"), + level: z.string().min(1), + text: z.string(), + url: eventUrl, + }), + z.strictObject({ + ...eventBase, + kind: z.literal("page-error"), + message: z.string(), + stack: z.string().nullable(), + }), + z.strictObject({ + ...networkEventShape, + kind: z.literal("request"), + status: z.null(), + failure: z.null(), + }), + z.strictObject({ + ...networkEventShape, + kind: z.literal("response"), + status: z.number().int().min(100).max(599), + failure: z.null(), + }), + z.strictObject({ + ...networkEventShape, + kind: z.literal("request-failed"), + status: z.null(), + failure: z.string().min(1), + }), + z.strictObject({ + ...networkEventShape, + kind: z.literal("request-finished"), + status: z.null(), + failure: z.null(), + }), + z.strictObject({ + ...networkEventShape, + kind: z.literal("request-unfinished"), + status: z.null(), + failure: z.null(), + reason: z.enum(["capture-ended", "cancelled"]), + }), + z.strictObject({ + ...eventBase, + kind: z.literal("network-content"), + transaction_id: z.string().min(1), + source_event_sequence: z.number().int().min(1), + phase: z.enum(["request", "response"]), + headers: browserNetworkHeadersSchema, + body: browserNetworkBodySchema, + }), + z.strictObject({ + ...eventBase, + kind: z.enum(["websocket-opened", "websocket-closed"]), + url: sanitizedBrowserUrlSchema, + }), + z.strictObject({ + ...webSocketFrameShape, + payload_type: z.literal("text"), + payload_text: z.string(), + }), + z.strictObject({ + ...webSocketFrameShape, + payload_type: z.literal("binary"), + payload_text: z.null(), + }), + z.strictObject({ + ...eventBase, + kind: z.enum([ + "frame-attached", + "frame-detached", + "frame-navigated", + "worker-created", + "worker-closed", + "popup-opened", + "popup-closed", + ]), + url: eventUrl, + name: z.string().nullable(), + }), + z.strictObject({ + ...eventBase, + kind: z.literal("download-cancelled"), + suggested_filename: z.string(), + url: sanitizedBrowserUrlSchema, + }), + ]) + .superRefine((event, context) => { + if ( + "redirected_from_transaction_id" in event && + event.redirected_from_transaction_id !== null && + event.redirected_from_transaction_state !== undefined + ) + context.addIssue({ + code: "custom", + message: + "Redirect state is valid only when the predecessor transaction ID is null", + path: ["redirected_from_transaction_state"], + }); + }); export type BrowserScenarioEvent = z.infer; const completenessSectionSchema = z.enum([ diff --git a/src/domain/browserScenarioDiff.test.ts b/src/domain/browserScenarioDiff.test.ts index f15bd9da0..aaf3ab9d8 100644 --- a/src/domain/browserScenarioDiff.test.ts +++ b/src/domain/browserScenarioDiff.test.ts @@ -29,6 +29,7 @@ const networkCapture = (sourceSequence: number, receiptStep: number) => { resource_type: "fetch", transaction_id: "request-1", redirected_from_transaction_id: null, + redirected_from_transaction_state: "root", header_names: [], status: null, failure: null, @@ -235,6 +236,309 @@ it("reports event-family mismatch and legacy selection as unknown", () => { ); }); +const concurrentCapture = ( + requestOrder: readonly ("a" | "b")[], + statuses: Readonly>, + responseStep = 0, + responseOrder: readonly ("a" | "b")[] = requestOrder, + redirect = false, +) => { + const base = scenarioCapture({}); + const network = (path: "a" | "b") => ({ + step_index: 0, + transaction_id: `request-${String(requestOrder.indexOf(path) + 1)}`, + redirected_from_transaction_id: + redirect && path === "b" + ? `request-${String(requestOrder.indexOf("a") + 1)}` + : null, + ...(!redirect || path !== "b" + ? { redirected_from_transaction_state: "root" as const } + : {}), + method: "GET", + url: { + url: `https://app.example.test/${path}.js`, + origin: "https://app.example.test", + query_parameter_names: [], + redacted: false, + }, + resource_type: "script", + header_names: [], + failure: null, + }); + const request = (path: "a" | "b") => ({ + ...network(path), + kind: "request", + status: null, + }); + const response = (path: "a" | "b") => ({ + ...network(path), + step_index: responseStep, + kind: "response", + status: statuses[path], + }); + const events = ( + redirect + ? requestOrder.flatMap((path) => [request(path), response(path)]) + : [...requestOrder.map(request), ...responseOrder.map(response)] + ).map((event, index) => ({ ...event, sequence: index + 1 })); + return browserScenarioCaptureSchema.parse({ + ...base, + events: { items: events, retained: events.length, dropped: 0 }, + }); +}; + +const redirectChainCapture = (swapLeaves = false, reverseReceipt = false) => { + const base = scenarioCapture({}); + const nodes = [ + { id: "root-a", parent: null, path: "first-a", status: 301 }, + { id: "root-b", parent: null, path: "first-b", status: 302 }, + { id: "middle-a", parent: "root-a", path: "same-middle", status: 307 }, + { id: "middle-b", parent: "root-b", path: "same-middle", status: 307 }, + { + id: "leaf-a", + parent: swapLeaves ? "middle-b" : "middle-a", + path: "last-a", + status: 200, + }, + { + id: "leaf-b", + parent: swapLeaves ? "middle-a" : "middle-b", + path: "last-b", + status: 200, + }, + ]; + const events = (reverseReceipt ? [...nodes].reverse() : nodes) + .flatMap((node) => { + const network = { + step_index: 0, + transaction_id: (reverseReceipt ? "renamed-" : "") + node.id, + redirected_from_transaction_id: + node.parent === null + ? null + : (reverseReceipt ? "renamed-" : "") + node.parent, + ...(node.parent === null + ? { redirected_from_transaction_state: "root" as const } + : {}), + method: "GET", + url: { + url: `https://app.example.test/${node.path}`, + origin: "https://app.example.test", + query_parameter_names: [], + redacted: false, + }, + resource_type: "document", + header_names: [], + failure: null, + }; + return [ + { ...network, kind: "request", status: null }, + { ...network, kind: "response", status: node.status }, + ]; + }) + .map((event, index) => ({ ...event, sequence: index + 1 })); + return browserScenarioCaptureSchema.parse({ + ...base, + events: { items: events, retained: events.length, dropped: 0 }, + }); +}; + +describe("browser scenario network event order", () => { + it("retains multi-hop redirect ancestry while ignoring IDs and concurrent receipt order", () => { + const compare = (after: BrowserScenarioCapture) => + compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: redirectChainCapture(), + after_scenario: after, + }), + ); + expect(compare(redirectChainCapture(false, true)).overall_status).toBe( + "unchanged", + ); + expect(compare(redirectChainCapture(true)).overall_status).toBe("changed"); + }); + + it("keeps missing and cyclic redirect ancestry incomparable", () => { + for (const predecessor of ["missing", "middle-a", null]) { + const capture = redirectChainCapture(); + const events = capture.events.items.map((event) => + "transaction_id" in event && event.transaction_id === "root-a" + ? { + ...event, + redirected_from_transaction_id: predecessor, + redirected_from_transaction_state: undefined, + } + : event, + ); + const broken = browserScenarioCaptureSchema.parse({ + ...capture, + events: { ...capture.events, items: events }, + }); + const compared = compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: broken, + after_scenario: broken, + }), + ); + expect(compared.overall_status).toBe("unknown"); + expect( + compared.steps.flatMap((step) => step.artifact_diffs), + ).toContainEqual( + expect.objectContaining({ + artifact: "events", + status: "unknown", + reason: expect.stringContaining("Redirect"), + }), + ); + } + }); + + it("keeps contradictory phase ancestry unknown", () => { + const capture = concurrentCapture(["a", "b"], { a: 200, b: 404 }); + const events = capture.events.items.map((event) => + event.kind === "response" && event.transaction_id === "request-2" + ? { + ...event, + redirected_from_transaction_id: "request-1", + redirected_from_transaction_state: undefined, + } + : event, + ); + const malformed = browserScenarioCaptureSchema.parse({ + ...capture, + events: { ...capture.events, items: events }, + }); + const compared = compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: malformed, + after_scenario: malformed, + }), + ); + expect(compared.overall_status).toBe("unknown"); + expect( + compared.steps.flatMap((step) => step.artifact_diffs), + ).toContainEqual( + expect.objectContaining({ + artifact: "events", + status: "unknown", + reason: expect.stringContaining("Redirect predecessor"), + }), + ); + }); +}); + +describe("browser scenario network comparison stability", () => { + it("keeps normalization away from generated network fingerprints", () => { + const rules = [..."0123456789abcdef"].map((match, index) => ({ + rule_id: `remove-fingerprint-hex-${String(index)}`, + artifacts: ["events"] as const, + match, + replacement: "", + })); + const compared = compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: concurrentCapture(["a", "b"], { + a: 200, + b: 404, + }), + after_scenario: concurrentCapture(["a", "b"], { + a: 201, + b: 404, + }), + normalization: { rules }, + }), + ); + expect(compared.overall_status).toBe("changed"); + }); + + it("reports changed ordinary events despite incomplete redirect ancestry", () => { + const withGap = (text: string) => { + const base = scenarioCapture({}); + const consoleEvent = base.events.items[0]; + if (consoleEvent?.kind !== "console") + throw new Error("Expected console fixture"); + return browserScenarioCaptureSchema.parse({ + ...base, + scenario: { + ...base.scenario, + event_families: ["console", "network"], + }, + events: { + retained: 2, + dropped: 0, + items: [ + { ...consoleEvent, text }, + { + sequence: 2, + step_index: 0, + kind: "request", + transaction_id: "orphan-request", + redirected_from_transaction_id: null, + method: "GET", + url: { + url: "https://app.example.test/orphan", + origin: "https://app.example.test", + query_parameter_names: [], + redacted: false, + }, + resource_type: "fetch", + header_names: [], + status: null, + failure: null, + }, + ], + }, + }); + }; + const compared = compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: withGap("settings-ready"), + after_scenario: withGap("settings-failed"), + }), + ); + expect(compared.overall_status).toBe("changed"); + }); + + it("compares concurrent network transactions without receipt order, receipt step or capture-local IDs", () => { + const compare = ( + before: BrowserScenarioCapture, + after: BrowserScenarioCapture, + ) => + compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: before, + after_scenario: after, + }), + ).overall_status; + const statuses = { a: 200, b: 404 }; + + expect( + compare( + concurrentCapture(["a", "b"], statuses), + concurrentCapture(["b", "a"], statuses, 1, ["a", "b"]), + ), + ).toBe("unchanged"); + expect( + compare( + concurrentCapture(["a", "b"], statuses), + concurrentCapture(["b", "a"], { a: 404, b: 200 }), + ), + ).toBe("changed"); + expect( + compare( + concurrentCapture(["a", "b"], statuses, 0, ["a", "b"]), + concurrentCapture(["a", "b"], statuses, 0, ["b", "a"]), + ), + ).toBe("unchanged"); + // Redirect linkage is distinct from independent concurrent transactions. + expect( + compare( + concurrentCapture(["a", "b"], statuses), + concurrentCapture(["a", "b"], statuses, 0, ["a", "b"], true), + ), + ).toBe("changed"); + }); +}); + describe("browser scenario comparison", () => { it("records canonical literal normalization and ignores declared volatile fields", () => { const before = scenarioCapture({ diff --git a/src/domain/browserScenarioDiff.ts b/src/domain/browserScenarioDiff.ts index 2cb761a76..55c8901bc 100644 --- a/src/domain/browserScenarioDiff.ts +++ b/src/domain/browserScenarioDiff.ts @@ -1,12 +1,14 @@ import type { BrowserScenarioCapture } from "./browserScenarioCapture.js"; import type { BrowserScenarioStep } from "./browserScenarioCaptureValues.js"; +import { z } from "zod"; +import { digestCanonicalValue } from "./canonicalDigest.js"; import { actionState, artifactEvidenceState, captured, eventEvidenceState, eventFamilyCoverageReason, - eventsFor, + prepareScenarioEvents, networkContentCoverageReason, screenshotProjection, type CaptureState, @@ -24,8 +26,14 @@ import { import { commitBrowserScenarioNormalization, digestNormalizedScenarioValue, + normalizeScenarioValue, } from "./browserScenarioNormalization.js"; +const scenarioEventsValueSchema = z.strictObject({ + events: z.array(z.unknown()), + network_transactions: z.array(z.string()), +}); + type ComparableArtifact = { readonly artifact: BrowserScenarioArtifactKind; } & ( @@ -59,6 +67,14 @@ export const compareBrowserScenarios = ( input: CompareBrowserScenariosInput, ): BrowserScenarioDiff => { const normalization = commitBrowserScenarioNormalization(input.normalization); + const beforeEvents = prepareScenarioEvents( + input.before_scenario, + normalization.rules, + ); + const afterEvents = prepareScenarioEvents( + input.after_scenario, + normalization.rules, + ); const alignment = alignScenarioSteps( input.before_scenario, input.after_scenario, @@ -68,6 +84,8 @@ export const compareBrowserScenarios = ( afterCapture: input.after_scenario, alignment, rules: normalization.rules, + beforeEvents, + afterEvents, }); const failures = [...alignment.failures, ...compared.failures]; const alignmentStatus = @@ -105,7 +123,7 @@ export const compareBrowserScenarios = ( limitations: [ "Steps align only by exact, unique step_id; missing, duplicate, or action-mismatched steps are reported and prevent an unchanged claim.", "Only declared literal normalization rules are applied, in canonical rule_id order, to already-redacted durable capture fields.", - "Elapsed time and event sequence/index values are excluded as volatile; screenshot bytes are compared by content digest.", + "Elapsed time, event sequence/index values and capture-local transaction IDs are excluded as volatile; network transactions are compared as an unordered collection in the step that sent their request; screenshot bytes are compared by content digest.", "A changed status proves an observed normalized difference. Missing or truncated evidence, incompatible capture context, or alignment failure prevents an unchanged claim.", ], }); @@ -160,6 +178,8 @@ interface CompareAlignedStepsInput { readonly afterCapture: BrowserScenarioCapture; readonly alignment: ScenarioAlignment; readonly rules: readonly BrowserScenarioNormalizationRule[]; + readonly beforeEvents: ReturnType; + readonly afterEvents: ReturnType; } const compareAlignedSteps = ( @@ -184,6 +204,8 @@ const compareAlignedSteps = ( afterCapture: input.afterCapture, afterStep, rules: input.rules, + beforeEvents: input.beforeEvents, + afterEvents: input.afterEvents, }); const reportable = artifacts.flatMap(({ diff }) => diff === null ? [] : [diff], @@ -266,6 +288,8 @@ interface CompareStepArtifactsInput { readonly afterCapture: BrowserScenarioCapture; readonly afterStep: BrowserScenarioStep; readonly rules: readonly BrowserScenarioNormalizationRule[]; + readonly beforeEvents: ReturnType; + readonly afterEvents: ReturnType; } const compareStepArtifacts = ( @@ -322,32 +346,156 @@ const compareStepArtifacts = ( before: input.beforeStep.artifacts.storage, after: input.afterStep.artifacts.storage, }), - compareCapturedValues({ + compareScenarioEvents({ ...common, artifact: "events", - before: captured( - eventsFor(input.beforeCapture, input.beforeStep.step_index), - ), - after: captured( - eventsFor(input.afterCapture, input.afterStep.step_index), - ), + before: captured(input.beforeEvents.forStep(input.beforeStep.step_index)), + after: captured(input.afterEvents.forStep(input.afterStep.step_index)), beforeState: eventEvidenceState(input.beforeCapture, input.beforeStep), afterState: eventEvidenceState(input.afterCapture, input.afterStep), incomparableReason: eventCoverageReason( input.beforeCapture, input.afterCapture, + input.beforeEvents.coverageReason, + input.afterEvents.coverageReason, + ), + familyIncomparableReason: eventFamilyCoverageReason( + input.beforeCapture, + input.afterCapture, ), }), ]; }; +const compareScenarioEvents = ( + input: CompareCapturedValuesInput<{ + readonly events: readonly unknown[]; + readonly network_transactions: readonly string[]; + }>, +): ComparableArtifact => { + const beforeState = + input.beforeState ?? + artifactEvidenceState(input.artifact, input.before, input.beforeStep); + const afterState = + input.afterState ?? + artifactEvidenceState(input.artifact, input.after, input.afterStep); + if (beforeState === "not_requested" && afterState === "not_requested") + return { artifact: input.artifact, status: "not_compared", diff: null }; + const beforeValue = + input.before.state === "captured" ? input.before.value : undefined; + const afterValue = + input.after.state === "captured" ? input.after.value : undefined; + const isCaptured = + beforeState === "captured" && + afterState === "captured" && + beforeValue !== undefined && + afterValue !== undefined; + const ordinaryBeforeDigest = + beforeValue === undefined + ? null + : digestCanonicalValue( + normalizeScenarioValue(beforeValue.events, "events", input.rules), + "Browser scenario", + ); + const ordinaryAfterDigest = + afterValue === undefined + ? null + : digestCanonicalValue( + normalizeScenarioValue(afterValue.events, "events", input.rules), + "Browser scenario", + ); + const beforeSha256 = + beforeState === "captured" && beforeValue !== undefined + ? digestScenarioEvents(beforeValue, ordinaryBeforeDigest) + : null; + const afterSha256 = + afterState === "captured" && afterValue !== undefined + ? digestScenarioEvents(afterValue, ordinaryAfterDigest) + : null; + const changedOrdinaryEvents = + isCaptured && ordinaryBeforeDigest !== ordinaryAfterDigest; + if (changedOrdinaryEvents && input.familyIncomparableReason === undefined) + return { + artifact: input.artifact, + status: "changed", + diff: { + artifact: input.artifact, + status: "changed", + before_state: beforeState, + after_state: afterState, + before_sha256: beforeSha256, + after_sha256: afterSha256, + reason: null, + }, + }; + if ( + !isCaptured || + input.incomparableReason !== undefined || + input.familyIncomparableReason !== undefined + ) + return { + artifact: input.artifact, + status: "unknown", + diff: { + artifact: input.artifact, + status: "unknown", + before_state: beforeState, + after_state: afterState, + before_sha256: beforeSha256, + after_sha256: afterSha256, + reason: + joinIncomparableReasons( + input.familyIncomparableReason, + input.incomparableReason, + ) ?? + `Artifact coverage is not comparable (${beforeState} before, ${afterState} after).`, + }, + }; + if (beforeSha256 === null || afterSha256 === null) + throw new TypeError("Captured event digest refinement failed"); + if (beforeSha256 === afterSha256) + return { artifact: input.artifact, status: "unchanged", diff: null }; + return { + artifact: input.artifact, + status: "changed", + diff: { + artifact: input.artifact, + status: "changed", + before_state: beforeState, + after_state: afterState, + before_sha256: beforeSha256, + after_sha256: afterSha256, + reason: null, + }, + }; +}; + +const digestScenarioEvents = ( + value: unknown, + ordinaryEventsDigest: string | null, +): string => { + const parsed = scenarioEventsValueSchema.parse(value); + return digestCanonicalValue( + { + ordinary_events_sha256: ordinaryEventsDigest, + // These are already normalized SHA-256 transaction fingerprints. + network_transactions: parsed.network_transactions, + }, + "Browser scenario", + ); +}; + const eventCoverageReason = ( before: BrowserScenarioCapture, after: BrowserScenarioCapture, + beforeNetworkReason: string | undefined, + afterNetworkReason: string | undefined, ): string | undefined => { const reasons = [ eventFamilyCoverageReason(before, after), networkContentCoverageReason(before, after), + beforeNetworkReason, + afterNetworkReason, ].filter((reason) => reason !== undefined); return reasons.length === 0 ? undefined : reasons.join(" "); }; @@ -360,9 +508,11 @@ interface CompareCapturedValuesInput { readonly afterStep: BrowserScenarioStep; readonly rules: readonly BrowserScenarioNormalizationRule[]; readonly project?: (value: Value) => unknown; + readonly order?: (normalized: unknown) => unknown; readonly beforeState?: BrowserScenarioEvidenceState; readonly afterState?: BrowserScenarioEvidenceState; readonly incomparableReason?: string | undefined; + readonly familyIncomparableReason?: string | undefined; } const compareCapturedValues = ( @@ -376,6 +526,7 @@ const compareCapturedValues = ( afterStep, rules, project = (value) => value, + order, } = input; const beforeState = input.beforeState ?? artifactEvidenceState(artifact, before, beforeStep); @@ -385,11 +536,21 @@ const compareCapturedValues = ( return { artifact, status: "not_compared", diff: null }; const beforeSha256 = beforeState === "captured" && before.state === "captured" - ? digestNormalizedScenarioValue(project(before.value), artifact, rules) + ? digestNormalizedScenarioValue( + project(before.value), + artifact, + rules, + order, + ) : null; const afterSha256 = afterState === "captured" && after.state === "captured" - ? digestNormalizedScenarioValue(project(after.value), artifact, rules) + ? digestNormalizedScenarioValue( + project(after.value), + artifact, + rules, + order, + ) : null; if ( beforeState !== "captured" || @@ -409,7 +570,10 @@ const compareCapturedValues = ( before_sha256: beforeSha256, after_sha256: afterSha256, reason: - input.incomparableReason ?? + joinIncomparableReasons( + input.familyIncomparableReason, + input.incomparableReason, + ) ?? `Artifact coverage is not comparable (${beforeState} before, ${afterState} after).`, }, }; @@ -431,3 +595,12 @@ const compareCapturedValues = ( }, }; }; + +const joinIncomparableReasons = ( + ...reasons: (string | undefined)[] +): string | undefined => { + const present = reasons.filter( + (reason): reason is string => reason !== undefined, + ); + return present.length === 0 ? undefined : [...new Set(present)].join(" "); +}; diff --git a/src/domain/browserScenarioDiffEvidence.ts b/src/domain/browserScenarioDiffEvidence.ts index f7f4b45e3..a6b5a0b5d 100644 --- a/src/domain/browserScenarioDiffEvidence.ts +++ b/src/domain/browserScenarioDiffEvidence.ts @@ -5,9 +5,16 @@ import type { BrowserStepArtifacts, } from "./browserScenarioCaptureValues.js"; import type { BrowserScenarioCapture } from "./browserScenarioCapture.js"; +import { z } from "zod"; +import { + fingerprintScenarioNetworkTransactions, + scenarioNetworkTransactions, + scenarioNetworkCoverageReason, +} from "./browserScenarioNetworkComparison.js"; import type { BrowserScenarioArtifactKind, BrowserScenarioEvidenceState, + BrowserScenarioNormalizationRule, } from "./browserScenarioDiffValues.js"; /** Capture-state input consumed by browser scenario artifact comparison. */ @@ -94,26 +101,62 @@ export const actionState = (step: BrowserScenarioStep) => ({ error: step.error, }); -/** Select events attributed to one scenario step without sequence metadata. */ -export const eventsFor = ( +/** One step's ordered events and a single indexed network graph. */ +export interface ScenarioStepEvents { + readonly events: readonly unknown[]; + readonly network_transactions: readonly string[]; +} + +/** Build one graph and fingerprint index per capture, then select in linear total work. */ +export const prepareScenarioEvents = ( capture: BrowserScenarioCapture, - stepIndex: number, -): readonly unknown[] => { - const sourceSteps = new Map( - capture.events.items.map(({ sequence, step_index }) => [ - sequence, - step_index, - ]), - ); - return capture.events.items.flatMap( - ({ sequence: _sequence, step_index: eventStepIndex, ...event }) => { - if (event.kind === "network-content") { - const { source_event_sequence: sourceSequence, ...content } = event; - return sourceSteps.get(sourceSequence) === stepIndex ? [content] : []; - } - return eventStepIndex === stepIndex ? [event] : []; - }, - ); + rules: readonly BrowserScenarioNormalizationRule[], +): { + readonly coverageReason: string | undefined; + readonly forStep: (stepIndex: number) => ScenarioStepEvents; +} => { + const eventsByStep = new Map(); + for (const { sequence: _sequence, step_index, ...event } of capture.events + .items) { + if ("transaction_id" in event) continue; + const events = eventsByStep.get(step_index) ?? []; + events.push(event); + eventsByStep.set(step_index, events); + } + const nodes = scenarioNetworkTransactions(capture); + const fingerprints = fingerprintScenarioNetworkTransactions(nodes, rules); + const networkByStep = new Map(); + for (const [index, node] of nodes.entries()) { + const step = networkByStep.get(node.step_index) ?? []; + const fingerprint = fingerprints[index]; + if (fingerprint === undefined) + throw new Error("Network comparison node has no fingerprint"); + step.push(fingerprint); + networkByStep.set(node.step_index, step); + } + for (const fingerprintsForStep of networkByStep.values()) + fingerprintsForStep.sort(); + return { + coverageReason: scenarioNetworkCoverageReason(capture, nodes), + forStep: (stepIndex) => ({ + events: eventsByStep.get(stepIndex) ?? [], + network_transactions: networkByStep.get(stepIndex) ?? [], + }), + }; +}; + +const scenarioStepEventsSchema = z.strictObject({ + events: z.array(z.unknown()), + network_transactions: z.array(z.string()), +}); + +/** Network ancestry is already fingerprinted per capture; keep the collection unordered. */ +export const orderNetworkTransactions = (value: unknown): unknown => { + const events = scenarioStepEventsSchema.parse(value); + return { + events: events.events, + network_transactions: events.network_transactions, + }; }; /** Different content selection is missing comparability, not an observed website change. */ diff --git a/src/domain/browserScenarioDiffValues.ts b/src/domain/browserScenarioDiffValues.ts index ff1c51f2d..0a5206188 100644 --- a/src/domain/browserScenarioDiffValues.ts +++ b/src/domain/browserScenarioDiffValues.ts @@ -123,20 +123,29 @@ const alignmentFailureSchema = z.strictObject({ step_id: z.string().min(1).nullable(), reason: z.string().min(1), }); -const builtInNormalizationSchema = z.enum([ +export const BROWSER_SCENARIO_BUILT_IN_NORMALIZATION_RULES = [ "ignore_step_elapsed_ms", "ignore_event_sequence", "ignore_event_step_index", "compare_screenshot_content_digest", "compare_normalized_text_content", -]); + "ignore_capture_local_network_transaction_ids", + "attribute_network_transactions_to_request_step", + "compare_network_transactions_as_unordered_collection", + "preserve_redirect_transaction_ancestry", +] as const; +const builtInNormalizationSchema = z.enum( + BROWSER_SCENARIO_BUILT_IN_NORMALIZATION_RULES, +); /** Step alignment, recorded normalization, and inline artifact differences. */ export const browserScenarioDiffSchema = z.strictObject({ comparison_kind: z.literal("browser_scenario"), overall_status: z.enum(["changed", "unchanged", "unknown"]), normalization: z.strictObject({ - built_in_rules: z.array(builtInNormalizationSchema).length(5), + built_in_rules: z + .array(builtInNormalizationSchema) + .length(BROWSER_SCENARIO_BUILT_IN_NORMALIZATION_RULES.length), rules: z.array(normalizationRuleSchema), sha256: z.string().regex(/^[a-f0-9]{64}$/u), }), diff --git a/src/domain/browserScenarioNetworkComparison.ts b/src/domain/browserScenarioNetworkComparison.ts new file mode 100644 index 000000000..56af2fa7c --- /dev/null +++ b/src/domain/browserScenarioNetworkComparison.ts @@ -0,0 +1,222 @@ +import { z } from "zod"; +import type { BrowserScenarioCapture } from "./browserScenarioCapture.js"; +import type { BrowserScenarioNormalizationRule } from "./browserScenarioDiffValues.js"; +import { digestCanonicalValue } from "./canonicalDigest.js"; +import { normalizeScenarioValue } from "./browserScenarioNormalization.js"; + +/** Numeric links are temporary local indices, never durable transaction identity. */ +export const scenarioNetworkTransactionSchema = z.strictObject({ + events: z.array(z.unknown()), + predecessor: z.number().int().min(-2), + step_index: z.number().int().min(0), +}); +export type ScenarioNetworkTransaction = z.infer< + typeof scenarioNetworkTransactionSchema +>; + +/** Project each transaction once, retaining ancestry as references rather than nested copies. */ +export const scenarioNetworkTransactions = ( + capture: BrowserScenarioCapture, +): ScenarioNetworkTransaction[] => { + const sourceSteps = new Map( + capture.events.items.map((event) => [event.sequence, event.step_index]), + ); + const requests = new Map( + capture.events.items.flatMap((event) => + event.kind === "request" ? [[event.transaction_id, event]] : [], + ), + ); + const grouped = new Map< + string, + { + events: unknown[]; + predecessor: string | null; + predecessorState: "root" | "unknown" | undefined; + predecessorConflict: boolean; + stepIndex: number; + } + >(); + for (const { sequence: _sequence, step_index: step, ...event } of capture + .events.items) { + if (!("transaction_id" in event)) continue; + const id = event.transaction_id; + const request = requests.get(id); + const receiptStep = + event.kind === "network-content" + ? sourceSteps.get(event.source_event_sequence) + : step; + const requestStep = request?.step_index ?? receiptStep; + let evidence: unknown; + let predecessor: string | null; + let predecessorState: "root" | "unknown" | undefined; + if (event.kind === "network-content") { + const { + transaction_id: _id, + source_event_sequence: _source, + ...content + } = event; + evidence = content; + predecessor = request?.redirected_from_transaction_id ?? null; + predecessorState = request?.redirected_from_transaction_state; + } else { + const { + transaction_id: _id, + redirected_from_transaction_id: parent, + redirected_from_transaction_state: _state, + ...network + } = event; + evidence = network; + predecessor = + request === undefined ? parent : request.redirected_from_transaction_id; + predecessorState = + request === undefined + ? event.redirected_from_transaction_state + : request.redirected_from_transaction_state; + } + const previous = grouped.get(id); + if (previous === undefined) + grouped.set(id, { + events: [evidence], + predecessor, + predecessorState, + predecessorConflict: + event.kind !== "network-content" && + request !== undefined && + ((event.redirected_from_transaction_id !== null && + event.redirected_from_transaction_id !== + request.redirected_from_transaction_id) || + (event.redirected_from_transaction_state === "root" && + request.redirected_from_transaction_id !== null)), + stepIndex: requestStep ?? step, + }); + else { + previous.events.push(evidence); + if (event.kind !== "network-content") { + const parent = event.redirected_from_transaction_id; + const state = event.redirected_from_transaction_state; + const knownParent = previous.predecessor; + if ( + parent !== null && + (knownParent !== null + ? parent !== knownParent + : previous.predecessorState === "root") + ) + previous.predecessorConflict = true; + else if (parent !== null && knownParent === null) { + previous.predecessor = parent; + previous.predecessorState = state; + } else if (parent === null && state === "root" && knownParent !== null) + previous.predecessorConflict = true; + else if ( + parent === null && + state === "root" && + previous.predecessorState !== "root" + ) + previous.predecessorState = "root"; + } + } + } + const indices = new Map([...grouped.keys()].map((id, index) => [id, index])); + return [...grouped.values()].map((node) => ({ + events: node.events, + predecessor: node.predecessorConflict + ? -2 + : node.predecessor === null + ? node.predecessorState === "root" + ? -1 + : -2 + : (indices.get(node.predecessor) ?? -2), + step_index: node.stepIndex, + })); +}; + +/** Missing or cyclic ancestry cannot support an unchanged claim. */ +export const scenarioNetworkCoverageReason = ( + capture: BrowserScenarioCapture, + nodes = scenarioNetworkTransactions(capture), +): string | undefined => { + const finished = new Set(); + for (let start = 0; start < nodes.length; start += 1) { + const path = new Set(); + let index = start; + while (index >= 0 && !finished.has(index)) { + if (path.has(index)) + return "Redirect ancestry contains a cycle; network transaction identity remains unknown."; + path.add(index); + const node = nodes[index]; + if (node === undefined) + return "Redirect predecessor evidence is missing; network transaction identity remains unknown."; + index = node.predecessor; + } + if (index === -2) + return "Redirect predecessor evidence is missing; network transaction identity remains unknown."; + for (const item of path) finished.add(item); + } + return undefined; +}; + +/** Hash normalized own evidence and its complete parent fingerprint in linear graph space. */ +export const fingerprintScenarioNetworkTransactions = ( + nodes: readonly ScenarioNetworkTransaction[], + rules: readonly BrowserScenarioNormalizationRule[] = [], +): string[] => { + const own = nodes.map((node) => + node.events + .map((event) => + digestCanonicalValue( + normalizeScenarioValue(event, "events", rules), + "Browser scenario", + ), + ) + .sort(), + ); + const fingerprints = new Map(); + for (let start = 0; start < nodes.length; start += 1) { + const path: number[] = []; + const positions = new Map(); + let index = start; + while ( + index >= 0 && + nodes[index] !== undefined && + !fingerprints.has(index) + ) { + const position = positions.get(index); + if (position !== undefined) { + // These captures are incomparable; retain an explicit cycle observation. + for (const member of path.slice(position)) + fingerprints.set( + member, + digestCanonicalValue( + { events: own[member], redirected_from: { state: "cycle" } }, + "Browser scenario", + ), + ); + break; + } + positions.set(index, path.length); + path.push(index); + index = nodes[index]?.predecessor ?? -2; + } + for (const member of path.reverse()) { + if (fingerprints.has(member)) continue; + const predecessor = nodes[member]?.predecessor ?? -2; + const parent = + predecessor === -1 + ? null + : (fingerprints.get(predecessor) ?? { state: "missing" }); + fingerprints.set( + member, + digestCanonicalValue( + { events: own[member], redirected_from: parent }, + "Browser scenario", + ), + ); + } + } + return nodes.map((_, index) => { + const fingerprint = fingerprints.get(index); + if (fingerprint === undefined) + throw new Error("Network comparison node has no fingerprint"); + return fingerprint; + }); +}; diff --git a/src/domain/browserScenarioNormalization.ts b/src/domain/browserScenarioNormalization.ts index 6af5fb2ac..5f02514bc 100644 --- a/src/domain/browserScenarioNormalization.ts +++ b/src/domain/browserScenarioNormalization.ts @@ -1,20 +1,13 @@ import { digestCanonicalValue } from "./canonicalDigest.js"; -import type { - BrowserScenarioArtifactKind, - BrowserScenarioDiff, - BrowserScenarioNormalization, - BrowserScenarioNormalizationRule, +import { + BROWSER_SCENARIO_BUILT_IN_NORMALIZATION_RULES, + type BrowserScenarioArtifactKind, + type BrowserScenarioDiff, + type BrowserScenarioNormalization, + type BrowserScenarioNormalizationRule, } from "./browserScenarioDiffValues.js"; -const BUILT_IN_RULES = [ - "ignore_step_elapsed_ms", - "ignore_event_sequence", - "ignore_event_step_index", - "compare_screenshot_content_digest", - "compare_normalized_text_content", -] as const; - /** Canonically order and commit a parsed browser scenario normalization policy. */ export const commitBrowserScenarioNormalization = ( input: BrowserScenarioNormalization, @@ -27,22 +20,29 @@ export const commitBrowserScenarioNormalization = ( .sort((left, right) => left.rule_id < right.rule_id ? -1 : left.rule_id > right.rule_id ? 1 : 0, ); - const commitment = { built_in_rules: [...BUILT_IN_RULES], rules }; + const commitment = { + built_in_rules: [...BROWSER_SCENARIO_BUILT_IN_NORMALIZATION_RULES], + rules, + }; return { ...commitment, sha256: digestCanonicalValue(commitment, "Browser scenario"), }; }; -/** Digest one artifact after applying committed exact-literal rules. */ +/** Digest one artifact after applying committed exact-literal rules, then any artifact ordering. */ export const digestNormalizedScenarioValue = ( value: unknown, artifact: BrowserScenarioArtifactKind, rules: readonly BrowserScenarioNormalizationRule[], + order: (normalized: unknown) => unknown = (normalized) => normalized, ): string => - digestCanonicalValue(normalize(value, artifact, rules), "Browser scenario"); + digestCanonicalValue( + order(normalizeScenarioValue(value, artifact, rules)), + "Browser scenario", + ); -const normalize = ( +export const normalizeScenarioValue = ( value: unknown, artifact: BrowserScenarioArtifactKind, rules: readonly BrowserScenarioNormalizationRule[], @@ -57,12 +57,12 @@ const normalize = ( value, ); if (Array.isArray(value)) - return value.map((item) => normalize(item, artifact, rules)); + return value.map((item) => normalizeScenarioValue(item, artifact, rules)); if (typeof value !== "object" || value === null) return value; return Object.fromEntries( Object.entries(value).map(([key, item]) => [ key, - normalize(item, artifact, rules), + normalizeScenarioValue(item, artifact, rules), ]), ); }; diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index d8ce6d9e2..ba23228b1 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -207,12 +207,7 @@ const compareWebCaptureDimensions = ( sectionsComplete(after, ["metadata"]), "Safe metadata capture was incomplete in at least one observation.", ), - webmcp: compareDimension( - webMcpMap(input.before.webmcp), - webMcpMap(input.after.webmcp), - webMcpComplete(input.before.webmcp) && webMcpComplete(input.after.webmcp), - "WebMCP discovery was unavailable or incomplete in at least one capture.", - ), + webmcp: webMcpDimension(input.before.webmcp, input.after.webmcp), accessibility: accessibilityDimension(before, after), storage: storageDimension(before, after), }; @@ -532,12 +527,51 @@ const webSocketPayloadMaps = ( return [left, right]; }; -const webMcpMap = ( - discovery: WebMcpDiscovery | null, -): ReadonlyMap => - discovery === null - ? new Map() - : keyed(discovery.tools.items, (tool) => tool.tool_key); +const webMcpDimension = ( + before: WebMcpDiscovery | null, + after: WebMcpDiscovery | null, +): Dimension => { + const afterTools = new Map( + after?.tools.items.map((tool) => [tool.tool_key, tool]), + ); + const unknownHints = new Set( + (before?.tools.items ?? []) + .filter((tool) => { + const other = afterTools.get(tool.tool_key); + return ( + other !== undefined && + (tool.annotations.consequential === undefined || + other.annotations.consequential === undefined) + ); + }) + .map((tool) => tool.tool_key), + ); + const project = ( + discovery: WebMcpDiscovery | null, + ): ReadonlyMap => + keyed( + (discovery?.tools.items ?? []).map((tool) => { + if (!unknownHints.has(tool.tool_key)) return tool; + const { consequential: _unknown, ...annotations } = tool.annotations; + return { ...tool, annotations }; + }), + (tool) => tool.tool_key, + ); + const result = compareDimension( + project(before), + project(after), + webMcpComplete(before) && webMcpComplete(after), + "WebMCP discovery was unavailable or incomplete in at least one capture.", + ); + return result.status === "unchanged" && unknownHints.size > 0 + ? { + ...result, + status: "unknown", + reason: + "Consequential annotations were not recorded or were invalid in at least one capture.", + } + : result; +}; const webMcpComplete = (discovery: WebMcpDiscovery | null): boolean => discovery !== null && diff --git a/src/domain/webMcpDiscovery.test.ts b/src/domain/webMcpDiscovery.test.ts new file mode 100644 index 000000000..452038d6b --- /dev/null +++ b/src/domain/webMcpDiscovery.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; + +import { createEvidence } from "./evidence.js"; +import { + createEvidenceBundle, + parseEvidenceBundle, + serializeEvidenceBundle, +} from "./evidenceBundle.js"; +import { jsonValueSchema } from "./jsonValue.js"; +import { + webMcpDiscoverySchema, + type WebMcpDiscovery, +} from "./webMcpDiscovery.js"; + +const PROVIDER = { id: "fixture", name: "Fixture provider", version: "1" }; + +const discovery = (annotations: Record): WebMcpDiscovery => + webMcpDiscoverySchema.parse({ + browser: { + product: "Chrome", + protocol_version: "1.3", + revision: "test", + user_agent: "test", + js_version: "test", + }, + target: { + target_id: "page-1", + url: "http://127.0.0.1:3000/", + origin: "http://127.0.0.1:3000", + }, + status: "available", + tools: { + total: 1, + items: [ + { + tool_key: `webmcp_${"a".repeat(64)}`, + name: "book_flight", + description: "Book a flight", + frame_id: "frame-main", + frame_url: "http://127.0.0.1:3000/", + owner_origin: "http://127.0.0.1:3000", + declaration_kind: "imperative", + input_schema: null, + input_schema_sha256: null, + annotations, + registration_source: null, + trust: "page-declared-untrusted", + }, + ], + }, + completeness: { + status: "complete_within_window", + conditions: ["complete_within_window"], + policy_filtered_sections: [], + attach_limited_sections: [], + truncated_sections: [], + unavailable_sections: [], + excluded: [], + dropped_events: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + total: 0, + }, + }, + limitations: [], + }); + +describe("WebMCP discovery consequential annotation", () => { + it.each([true, false, null])( + "preserves a declared consequential value of %s", + (consequential) => { + const parsed = discovery({ + read_only: null, + untrusted_content: null, + consequential, + autosubmit: null, + }); + expect(parsed.tools.items[0]?.annotations.consequential).toBe( + consequential, + ); + }, + ); + + it("admits historical captures recorded before consequential was surfaced", () => { + const parsed = discovery({ + read_only: true, + untrusted_content: null, + autosubmit: false, + }); + expect(parsed.tools.items[0]?.annotations.consequential).toBeUndefined(); + }); + + it("survives the Evidence bundle export and re-read round-trip", () => { + const result = discovery({ + read_only: null, + untrusted_content: null, + consequential: true, + autosubmit: null, + }); + const evidence = createEvidence(undefined, PROVIDER, { + predicateType: "rea.webmcp-discovery", + operation: "discover_webmcp_tools", + parameters: {}, + result: jsonValueSchema.parse(result), + }); + const bundle = createEvidenceBundle([evidence]); + const reread = parseEvidenceBundle( + JSON.parse(serializeEvidenceBundle(bundle)), + ); + const record = reread.records[0]; + expect(record?.normalized_result).toMatchObject({ + tools: { + items: [ + { + annotations: { + consequential: true, + }, + }, + ], + }, + }); + }); +}); diff --git a/src/domain/webMcpDiscovery.ts b/src/domain/webMcpDiscovery.ts index 0924f5b78..0edf81dd3 100644 --- a/src/domain/webMcpDiscovery.ts +++ b/src/domain/webMcpDiscovery.ts @@ -33,6 +33,9 @@ const webMcpToolSchema = z.object({ annotations: z.object({ read_only: z.boolean().nullable(), untrusted_content: z.boolean().nullable(), + // Absent when this annotation was not recorded or its protocol value was invalid; null + // distinguishes a declaration that omitted it from a declared false. + consequential: z.boolean().nullable().optional(), autosubmit: z.boolean().nullable(), }), registration_source: z diff --git a/tests/boundary/browser/cdpBrowserProvider.document_script.test.ts b/tests/boundary/browser/cdpBrowserProvider.document_script.test.ts index 5fd1e061b..5342dc9ee 100644 --- a/tests/boundary/browser/cdpBrowserProvider.document_script.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.document_script.test.ts @@ -256,63 +256,70 @@ describe("CdpBrowserProvider: transient WebMCP frames", () => { }); describe("CdpBrowserProvider: WebMCP discovery and completeness", () => { - it("discovers untrusted WebMCP declarations without registering or invoking them", async () => { - const browser = await startFakeCdpBrowser({ webMcpTools: true }); - trackBrowser(browser); - const result = await new CdpBrowserProvider().discoverWebMcpTools( - discoverWebMcpToolsInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 0, - }), - ); + it.each([true, false])( + "discovers untrusted WebMCP declarations with consequential=%s without invoking them", + async (consequential) => { + const browser = await startFakeCdpBrowser({ + webMcpTools: true, + webMcpConsequential: consequential, + }); + trackBrowser(browser); + const result = await new CdpBrowserProvider().discoverWebMcpTools( + discoverWebMcpToolsInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + }), + ); - if (!result.ok) throw result.error; - expect(result.value).toMatchObject({ - status: "available", - tools: { - total: 1, - items: [ - { - name: "search_orders", - description: "Search orders; authorization=Bearer tool-secret", - declaration_kind: "declarative", - owner_origin: browser.allowedOrigin, - annotations: { - read_only: true, - untrusted_content: true, - autosubmit: false, + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + status: "available", + tools: { + total: 1, + items: [ + { + name: "search_orders", + description: "Search orders; authorization=Bearer tool-secret", + declaration_kind: "declarative", + owner_origin: browser.allowedOrigin, + annotations: { + read_only: true, + untrusted_content: true, + consequential, + autosubmit: false, + }, + trust: "page-declared-untrusted", + registration_source: { + url: `${browser.allowedOrigin}/app.js?token=tool-source-secret`, + line: 12, + column: 4, + }, }, - trust: "page-declared-untrusted", - registration_source: { - url: `${browser.allowedOrigin}/app.js?token=tool-source-secret`, - line: 12, - column: 4, - }, - }, - ], - }, - }); - expect(result.value.completeness.excluded).toContainEqual( - expect.objectContaining({ - section: "webmcp_tools", - reason: "out_of_target_scope", - }), - ); - expect(result.value.completeness.unavailable_sections).not.toContain( - "webmcp_tools", - ); - const serialized = JSON.stringify(result.value); - expect(serialized).toContain("tool-secret"); - expect(serialized).toContain("tool-source-secret"); - expect(serialized).toContain("schema-secret"); - expect(serialized).not.toContain("private-tool-secret"); - const methods = browser.commands.map(({ method }) => method); - expect(methods).toContain("WebMCP.enable"); - expect(methods).not.toContain("WebMCP.invokeTool"); - expect(methods).not.toContain("Runtime.evaluate"); - }); + ], + }, + }); + expect(result.value.completeness.excluded).toContainEqual( + expect.objectContaining({ + section: "webmcp_tools", + reason: "out_of_target_scope", + }), + ); + expect(result.value.completeness.unavailable_sections).not.toContain( + "webmcp_tools", + ); + const serialized = JSON.stringify(result.value); + expect(serialized).toContain("tool-secret"); + expect(serialized).toContain("tool-source-secret"); + expect(serialized).toContain("schema-secret"); + expect(serialized).not.toContain("private-tool-secret"); + const methods = browser.commands.map(({ method }) => method); + expect(methods).toContain("WebMCP.enable"); + expect(methods).not.toContain("WebMCP.invokeTool"); + expect(methods).not.toContain("Runtime.evaluate"); + }, + ); it("reports WebMCP unavailable when the experimental domain is absent", async () => { const browser = await startFakeCdpBrowser({ @@ -756,6 +763,15 @@ describe("CdpBrowserProvider: WebMCP registration owners", () => { const tools = result.value.tools.items; expect(tools).toHaveLength(mode === "retain" ? 2 : 1); expect(result.value.tools.total).toBe(tools.length); + // Declarations without annotations report absent hints as null, never + // collapsed to a declared false. + for (const tool of tools) + expect(tool.annotations).toEqual({ + read_only: null, + untrusted_content: null, + consequential: null, + autosubmit: null, + }); expect(tools).toContainEqual( expect.objectContaining({ name: "read_item", diff --git a/tests/boundary/browser/historicalWebCapture.test.ts b/tests/boundary/browser/historicalWebCapture.test.ts index 781172cd3..815810a9e 100644 --- a/tests/boundary/browser/historicalWebCapture.test.ts +++ b/tests/boundary/browser/historicalWebCapture.test.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { mkdtemp, writeFile, rm, readFile } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { expect, it, onTestFinished } from "vitest"; @@ -44,6 +44,66 @@ const cli = async ( }; import { historicalHar } from "../../fixtures/historicalHar.js"; +it.each([ + { encoded: "1e3", literal: "1000" }, + { encoded: "0.0000001", literal: "1e-7" }, +])( + "excludes normalized HAR numbers through CLI and MCP: $encoded", + async ({ encoded, literal }) => { + const root = await createTestTempDirectory("rea-har-normalized-number-"); + const path = join(root, "capture.har"); + const original = JSON.stringify(historicalHar()) + .replace('"log":{', `"_number":${encoded},"log":{`) + .replace('"cache":{}', `"cache":{},"_number":${encoded}`); + await writeFile(path, original); + const { call } = await connectLocalToolsMcp(); + for (const marked of [false, true]) { + const fromCli = await cli( + path, + "har", + marked ? ["--sensitive-value", literal] : [], + ); + expect(fromCli.ok, fromCli.stdout).toBe(true); + const evidence = parseEvidence(JSON.parse(fromCli.stdout)); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + ...(marked ? { sensitive_values: [literal] } : {}), + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcp = parseEvidence(response.structuredContent); + expect(mcp.evidence_id).toBe(evidence.evidence_id); + expect(mcp.normalized_result).toEqual(evidence.normalized_result); + const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); + for (const projection of [capture.container, ...capture.records]) { + expect(projection.reported).toMatchObject({ + _number: marked ? null : Number(encoded), + }); + const numbers = projection.numeric_literals.filter( + (number) => number.pointer === "/_number", + ); + expect(numbers).toEqual( + marked + ? [] + : [ + { + pointer: "/_number", + producer_type: "json-number", + literal: encoded, + }, + ], + ); + if (marked) + expect(projection.redactions).toContainEqual({ + pointer: "/_number", + reason: "explicit-sensitive-value", + }); + } + } + expect(await readFile(path, "utf8")).toBe(original); + }, +); + it("excludes declared binary lengths through CLI, MCP and retained capture pages", async () => { const root = await createTestTempDirectory("rea-har-byte-length-"); const path = join(root, "capture.har"); diff --git a/tests/boundary/browser/webCaptureDiff.test.ts b/tests/boundary/browser/webCaptureDiff.test.ts index 99f347c81..4d372650e 100644 --- a/tests/boundary/browser/webCaptureDiff.test.ts +++ b/tests/boundary/browser/webCaptureDiff.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js"; import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js"; +import { discoverWebMcpToolsInputSchema } from "../../../src/domain/webMcpDiscovery.js"; import { compareWebCaptures } from "../../../src/domain/webCaptureDiff.js"; import { compareWebCapturesInputSchema, @@ -202,6 +203,64 @@ afterEach(async () => { }); describe("web capture diff", () => { + it.each([ + { missing: "before", changed: false, expected: "unknown" }, + { missing: "after", changed: false, expected: "unknown" }, + { missing: "before", changed: true, expected: "changed" }, + { missing: "after", changed: true, expected: "changed" }, + { missing: "neither", changed: false, expected: "changed" }, + ] as const)( + "preserves consequential uncertainty: $missing, other change $changed", + async ({ missing, changed, expected }) => { + const browser = await startFakeCdpBrowser({ webMcpTools: true }); + browsers.push(browser); + const provider = new CdpBrowserProvider(); + const request = { + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + }; + const inspection = await provider.inspectPage( + inspectWebPageInputSchema.parse(request), + ); + const discovery = await provider.discoverWebMcpTools( + discoverWebMcpToolsInputSchema.parse(request), + ); + if (!inspection.ok) throw inspection.error; + if (!discovery.ok) throw discovery.error; + const before = structuredClone(discovery.value); + const after = structuredClone(discovery.value); + markSectionsComplete(before, ["webmcp_tools"]); + markSectionsComplete(after, ["webmcp_tools"]); + const left = before.tools.items[0]; + const right = after.tools.items[0]; + if (left === undefined || right === undefined) + throw new Error("missing WebMCP registration fixture"); + left.annotations.consequential = true; + right.annotations.consequential = false; + if (missing === "before") delete left.annotations.consequential; + if (missing === "after") delete right.annotations.consequential; + if (changed) right.description += " changed"; + const result = compareWebCaptures( + compareWebCapturesInputSchema.parse({ + before: { inspection: inspection.value, webmcp: before }, + after: { inspection: inspection.value, webmcp: after }, + }), + ); + expect(result.dimensions.webmcp).toMatchObject({ + status: expected, + total_changes: expected === "changed" ? 1 : 0, + }); + if (expected === "changed") + expect(result.dimensions.webmcp.changes[0]).toMatchObject({ + change: "modified", + }); + }, + ); +}); + +describe("web capture dimension uncertainty", () => { it("reports stable observed changes while preserving unknown dimensions", async () => { const browser = await startFakeCdpBrowser(); browsers.push(browser); diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index b5a15e120..84ef1506f 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -1115,6 +1115,34 @@ describe("source-map dependency coverage of hard-to-parse sources", () => { ).toEqual(expected); }); + it("selects parser mode from the original source URL pathname", async () => { + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => + Promise.resolve( + new Response( + JSON.stringify({ + version: 3, + names: [], + sources: ["main.ts?revision=2#view.tsx"], + sourcesContent: [`import "./dep.ts"; const value = 1;`], + mappings: "AAAA", + }), + ), + ), + }); + expect(result.items[0]?.status).toBe("included"); + expect(result.items[0]?.limitation).toBeNull(); + expect(result.items[0]?.original_sources[0]?.source).toBe( + `${origin}/assets/main.ts?revision=2#view.tsx`, + ); + expect(result.items[0]?.original_module_edges).toEqual([ + expect.objectContaining({ + specifier: "./dep.ts", + resolved_source: `${origin}/assets/dep.ts`, + }), + ]); + }); + it("keeps recovered edges from a partly recovered source and reports them as partial", async () => { const result = await fetchWithSources([ 'with (scope) { require("./c.js"); }\nrequire("./b.js");\nimport "./a.js";', diff --git a/tests/boundary/cli/browserCli.test.ts b/tests/boundary/cli/browserCli.test.ts index 351d07512..56daccafb 100644 --- a/tests/boundary/cli/browserCli.test.ts +++ b/tests/boundary/cli/browserCli.test.ts @@ -411,7 +411,14 @@ describe("browser CLI capture parity", () => { operation: "discover_webmcp_tools", normalized_result: { tools: { - items: [expect.objectContaining({ name: "search_orders" })], + items: [ + expect.objectContaining({ + name: "search_orders", + annotations: expect.objectContaining({ + consequential: true, + }), + }), + ], }, }, }); diff --git a/tests/fixtures/fakeCdpBrowserEvents.ts b/tests/fixtures/fakeCdpBrowserEvents.ts index 39a088172..294c2ad56 100644 --- a/tests/fixtures/fakeCdpBrowserEvents.ts +++ b/tests/fixtures/fakeCdpBrowserEvents.ts @@ -16,6 +16,13 @@ export const emitEvents = ( emitCaptureNavigation(socket, command, port, options); }; +const webMcpToolAnnotations = { + readOnly: true, + untrustedContent: true, + consequential: true, + autosubmit: false, +}; + const emitWebMcpEvents = ( socket: WebSocket, command: FakeCdpCommand, @@ -29,142 +36,160 @@ const emitWebMcpEvents = ( emitSameUrlWebMcpRegistrations(socket, command, options); return; } - if (command.method === "WebMCP.enable" && options.webMcpTools === true) { - const inputSchemaProperties = - options.webMcpSchemaPropertyCount === undefined - ? { - orderId: { type: "string", example: "schema-secret" }, - includeItems: { type: "boolean", default: true }, - } - : Object.fromEntries( - Array.from( - { length: options.webMcpSchemaPropertyCount }, - (_value, index) => [`field_${String(index)}`, { type: "string" }], - ), - ); + if (command.method !== "WebMCP.enable" || options.webMcpTools !== true) + return; + emitMainWebMcpTools(socket, command, port, options); + emitChildWebMcpNavigations(socket, command, port, options); +}; + +const emitMainWebMcpTools = ( + socket: WebSocket, + command: FakeCdpCommand, + port: number, + options: FakeOptions, +): void => { + const inputSchemaProperties = + options.webMcpSchemaPropertyCount === undefined + ? { + orderId: { type: "string", example: "schema-secret" }, + includeItems: { type: "boolean", default: true }, + } + : Object.fromEntries( + Array.from( + { length: options.webMcpSchemaPropertyCount }, + (_value, index) => [`field_${String(index)}`, { type: "string" }], + ), + ); + event(socket, "WebMCP.toolsAdded", command.sessionId, { + tools: [ + { + name: "search_orders", + description: "Search orders; authorization=Bearer tool-secret", + frameId: "frame-main", + backendNodeId: 42, + inputSchema: options.webMcpInputSchema ?? { + type: "object", + properties: inputSchemaProperties, + required: ["orderId"], + }, + annotations: { + ...webMcpToolAnnotations, + consequential: Object.hasOwn(options, "webMcpConsequential") + ? options.webMcpConsequential + : true, + }, + stackTrace: { + callFrames: [ + { + url: `http://127.0.0.1:${String(port)}/app.js?token=tool-source-secret`, + lineNumber: 12, + columnNumber: 4, + }, + ], + }, + }, + { + name: "private_tool", + description: "private-tool-secret", + frameId: "frame-private", + }, + ...(options.extraCollections === true + ? [ + { + name: "update_order", + description: "Update an order", + frameId: "frame-main", + inputSchema: { + type: "object", + properties: { orderId: { type: "string" } }, + }, + }, + ] + : []), + ...(options.webMcpFrameCount !== undefined && options.webMcpFrameCount > 0 + ? [ + { + name: "beyond_frame_limit_tool", + description: "Registered in an in-scope frame after index 999", + frameId: `webmcp-frame-${String(options.webMcpFrameCount - 1)}`, + }, + ] + : []), + ...(options.webMcpChildLeavesScope === true || + options.webMcpChildTransientBlank === true + ? [ + { + name: "child_tool", + description: "Must be removed after child navigation", + frameId: "frame-child", + }, + ] + : []), + ], + }); +}; + +const emitChildWebMcpNavigations = ( + socket: WebSocket, + command: FakeCdpCommand, + port: number, + options: FakeOptions, +): void => { + if (options.webMcpChildLeavesScope === true) { + event(socket, "Page.frameNavigated", command.sessionId, { + frame: { + id: "frame-child", + parentId: "frame-main", + url: "https://private.example.test/escaped", + }, + }); event(socket, "WebMCP.toolsAdded", command.sessionId, { tools: [ { - name: "search_orders", - description: "Search orders; authorization=Bearer tool-secret", - frameId: "frame-main", - backendNodeId: 42, - inputSchema: options.webMcpInputSchema ?? { - type: "object", - properties: inputSchemaProperties, - required: ["orderId"], - }, - annotations: { - readOnly: true, - untrustedContent: true, - autosubmit: false, - }, - stackTrace: { - callFrames: [ - { - url: `http://127.0.0.1:${String(port)}/app.js?token=tool-source-secret`, - lineNumber: 12, - columnNumber: 4, - }, - ], - }, + name: "escaped_child_tool", + description: "cross-origin-child-secret", + frameId: "frame-child", }, - { - name: "private_tool", - description: "private-tool-secret", - frameId: "frame-private", - }, - ...(options.extraCollections === true - ? [ - { - name: "update_order", - description: "Update an order", - frameId: "frame-main", - inputSchema: { - type: "object", - properties: { orderId: { type: "string" } }, - }, - }, - ] - : []), - ...(options.webMcpFrameCount !== undefined && - options.webMcpFrameCount > 0 - ? [ - { - name: "beyond_frame_limit_tool", - description: "Registered in an in-scope frame after index 999", - frameId: `webmcp-frame-${String(options.webMcpFrameCount - 1)}`, - }, - ] - : []), - ...(options.webMcpChildLeavesScope === true || - options.webMcpChildTransientBlank === true - ? [ - { - name: "child_tool", - description: "Must be removed after child navigation", - frameId: "frame-child", - }, - ] - : []), ], }); - if (options.webMcpChildLeavesScope === true) { - event(socket, "Page.frameNavigated", command.sessionId, { - frame: { - id: "frame-child", - parentId: "frame-main", - url: "https://private.example.test/escaped", - }, - }); - event(socket, "WebMCP.toolsAdded", command.sessionId, { - tools: [ - { - name: "escaped_child_tool", - description: "cross-origin-child-secret", - frameId: "frame-child", - }, - ], - }); - } - if (options.webMcpChildTransientBlank === true) - event(socket, "Page.frameNavigated", command.sessionId, { - frame: { - id: "frame-child", - parentId: "frame-main", - loaderId: "loader-transient", - url: "about:blank", - }, - }); - if (options.webMcpChildRecoversAfterTransient === true) { - event(socket, "Page.frameNavigated", command.sessionId, { - frame: { - id: "frame-child", - parentId: "frame-main", - loaderId: "loader-child-recovered", - url: `http://127.0.0.1:${String(port)}/child-recovered`, - }, - }); - event(socket, "WebMCP.toolsAdded", command.sessionId, { - tools: [ - { - name: "child_tool", - description: "Replayed from the current child document", - frameId: "frame-child", - }, - ], - }); - } - if (options.webMcpChildNavigatesAllowed === true) - event(socket, "Page.frameNavigated", command.sessionId, { - frame: { - id: "frame-child", - parentId: "frame-main", - loaderId: "loader-child-replacement", - url: `http://127.0.0.1:${String(port)}/child-replacement`, - }, - }); } + if (options.webMcpChildTransientBlank === true) + event(socket, "Page.frameNavigated", command.sessionId, { + frame: { + id: "frame-child", + parentId: "frame-main", + loaderId: "loader-transient", + url: "about:blank", + }, + }); + if (options.webMcpChildRecoversAfterTransient === true) { + event(socket, "Page.frameNavigated", command.sessionId, { + frame: { + id: "frame-child", + parentId: "frame-main", + loaderId: "loader-child-recovered", + url: `http://127.0.0.1:${String(port)}/child-recovered`, + }, + }); + event(socket, "WebMCP.toolsAdded", command.sessionId, { + tools: [ + { + name: "child_tool", + description: "Replayed from the current child document", + frameId: "frame-child", + }, + ], + }); + } + if (options.webMcpChildNavigatesAllowed === true) + event(socket, "Page.frameNavigated", command.sessionId, { + frame: { + id: "frame-child", + parentId: "frame-main", + loaderId: "loader-child-replacement", + url: `http://127.0.0.1:${String(port)}/child-replacement`, + }, + }); }; const emitSameUrlWebMcpRegistrations = ( diff --git a/tests/fixtures/fakeCdpBrowserTypes.ts b/tests/fixtures/fakeCdpBrowserTypes.ts index e75e0dc48..69a3c8ec7 100644 --- a/tests/fixtures/fakeCdpBrowserTypes.ts +++ b/tests/fixtures/fakeCdpBrowserTypes.ts @@ -107,6 +107,7 @@ export interface FakeOptions { readonly cachedResponseBody?: string; readonly invalidResponseBodyBase64?: boolean; readonly webMcpTools?: boolean; + readonly webMcpConsequential?: JsonValue; readonly webMcpSameUrlRegistrations?: "retain" | "remove-second"; readonly webMcpFrameCount?: number; readonly webMcpSchemaPropertyCount?: number; diff --git a/tests/fixtures/webScriptCapture.ts b/tests/fixtures/webScriptCapture.ts index 0c6318531..f7c021e66 100644 --- a/tests/fixtures/webScriptCapture.ts +++ b/tests/fixtures/webScriptCapture.ts @@ -19,6 +19,7 @@ export const scriptScenarioFixture = ( const request = { transaction_id: `request-${index + 1}`, redirected_from_transaction_id: null, + redirected_from_transaction_state: "root", method: "GET", url: sanitizeBrowserUrl(url), resource_type: "script", From 80de3145e458474319f160d2d2c09586e7a282a4 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:32:00 +0800 Subject: [PATCH 10/26] fix(mcp): align advertised contracts and retain workflow evidence Use one named input-schema projection for clients and catalog identity, preserve dialect declarations, and bound compact schema depth. Keep canonical SDK output validation when output advertisements are disabled. Release request-owned state on cancellation and carry upstream limitations through composed workflows. Preserve doctor full-output envelopes and validate navigation facts at their owner. Reviewed fixes: #2088, #2108, #2513, #2684, #2693, #2695, #2836, #2864, #2868, #2877, #2933, #2998, #3002, #3015. Validation: focused schema, SDK output-validation, context, and workflow tests passed; no schema dialect or canonical output validation was relaxed. --- docs/mcp-contracts.md | 49 ++- .../AnalysisContextQueries.test.ts | 47 +++ src/application/AnalysisContextQueries.ts | 26 +- src/application/DirectAnalysis.ts | 10 +- src/application/WorkflowEvidence.test.ts | 136 +++++++- src/application/WorkflowEvidence.ts | 44 ++- src/catalogIdentity.ts | 8 +- src/config.test.ts | 46 ++- src/config/environment.ts | 58 +++- src/config/mcpOutputSchemaProfile.ts | 34 ++ src/config/parseConfig.ts | 49 +-- src/contracts/adb/adbToolContracts.ts | 4 +- src/contracts/compactInputSchema.test.ts | 18 + src/contracts/compactInputSchema.ts | 40 ++- src/contracts/enhancedToolContracts.ts | 2 +- .../firmware/firmwareToolContracts.ts | 2 +- src/contracts/inputSchemaPresentation.ts | 56 +-- src/contracts/providerSelection.ts | 1 + src/contracts/toolEffects.test.ts | 21 +- src/contracts/toolEffects.ts | 2 +- src/contracts/toolInputReferences.test.ts | 50 ++- src/contracts/toolSchemaMetadata.ts | 8 + src/domain/analysisErrorCore.ts | 14 +- src/domain/analysisErrorPresentation.ts | 6 + src/domain/analysisErrorProjection.ts | 8 +- src/domain/analysisView/analysisView.ts | 32 +- src/domain/analysisView/analysisViewParent.ts | 25 ++ src/mcpDoctor.ts | 27 +- src/server/EvidenceMcpServer.test.ts | 24 ++ src/server/EvidenceMcpServer.ts | 107 ++++-- src/server/createServer.ts | 15 + src/server/registerEnhancedTools.ts | 17 +- src/server/toolResult.ts | 1 - tests/boundary/idaStartupDiagnostics.test.ts | 16 +- .../boundary/mcp/agentKnownAnswerMcp.test.ts | 328 ++++++++++++++++++ .../mcp/compactInputSchemaMcp.test.ts | 83 ++++- ...4-uses-complete-inventories-inline.test.ts | 148 +++++--- .../mcp/functionCallDirectionsMcp.test.ts | 138 ++++++++ .../mcp/functionComparisonMcp.test.ts | 261 +++++++------- tests/boundary/mcp/mcpDoctor.test.ts | 27 ++ .../boundary/mcp/outputSchemaProfile.test.ts | 115 ++++++ tests/boundary/mcp/serverIdentity.test.ts | 14 + 42 files changed, 1800 insertions(+), 317 deletions(-) create mode 100644 src/config/mcpOutputSchemaProfile.ts create mode 100644 src/domain/analysisView/analysisViewParent.ts create mode 100644 src/server/EvidenceMcpServer.test.ts create mode 100644 tests/boundary/mcp/functionCallDirectionsMcp.test.ts create mode 100644 tests/boundary/mcp/outputSchemaProfile.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 557c98ab4..02218bf51 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -193,7 +193,8 @@ after target content was read. ## Retained application Evidence inputs `analyze_javascript_application`, `inspect_binary_layout`, `inspect_artifact`, -`inspect_wasm_artifact`, and `inspect_macho` accept `detail`: +`inspect_wasm_artifact`, `inspect_macho`, and `decode_interface_builder` +accept `detail`: `complete` (default) returns the complete analysis Evidence; `summary` records that complete Evidence in the session and returns the derived `inspect_analysis_view` summary Evidence instead. Its @@ -276,6 +277,17 @@ symbols (Electron Framework: 2.5 MB of imports and exports, 3.3 KB as a summary), so select `detail: "summary"` on the first call. Delivery detail never reaches the native tools or changes the source Evidence ID. +Completed `decode_interface_builder` Evidence supports a summary and +`view: {kind: "interface-builder", collection: "nodes", offset: 0, limit: 64}`. +Collections are `documents`, `nodes`, `edges`, and `coverage`; for nodes and +edges an optional `document` keeps rows whose evidence names that exact decoded +`relative_path`, and an undecoded path is refused. The summary keeps every +decoded document, the provider, node-kind, edge-relation and coverage-status +counts, and whether the decoded graph is incomplete. Each page repeats +`source_truncated`. A complete app graph often exceeds the default MCP response +budget (XLD.app: 6.7 MB), so select `detail: "summary"` on the first call. +Delivery detail never reaches the decoder or changes the source Evidence ID. + `trace_application_feature`, `trace_javascript_semantics`, `compare_application_versions`, `compare_source_to_bundle`, and `compare_javascript_export_shapes` accept complete inline application Evidence @@ -671,7 +683,13 @@ advertised input schema within a 13,312-byte budget, measured to sit below Moonshot's observed cap (community measurements: a 13.7 KB schema is accepted while a ~15 KB one is rejected). The setting is read at startup; restart the server to apply changes, as with `REA_MCP_MAX_RESPONSE_BYTES`. The default -value `full` keeps the complete advertisement unchanged. +value `full` keeps the complete advertisement unchanged. Compact presentation +also reduces schemas whose expanded JSON tree exceeds ten levels, the limit +reported by strict-depth MCP clients in [#2813](https://github.com/morluto/rea/issues/2813). +WorkBuddy/CodeBuddy gateways have also rejected full REA catalogs with an +opaque `11133` error while accepting the compact profile +([#2999](https://github.com/morluto/rea/issues/2999)); compatibility depends on +the selected gateway and model. The compact profile renders the same tool set, output schemas, annotations, and canonical server-side validation. It changes only the advertised input @@ -696,6 +714,9 @@ JSON Schema: canonical schema still rejects malformed calls server-side, so callers following the reduced advertisement receive the canonical typed error instead of silent acceptance. +- Schemas deeper than ten nodes use the same reduced projection, with + root-property names, types, guidance, and required fields retained. The + server continues to enforce the full canonical input contract. Boundary tests pin every advertised compact schema to the budget, keep them valid JSON Schema that still accepts every canonical example. Client verification @@ -711,6 +732,30 @@ identity. Explicit schema IDs and relative external reference bases are preserved. This keeps every tool and validation rule in discovery and does not change the SDK's catalog invalidation or availability checks. +Some stdio hosts reject a complete `tools/list` frame before they load any +tools. The opt-in `REA_MCP_OUTPUT_SCHEMA_PROFILE=off` setting omits `outputSchema` +from the advertised catalog for hosts with a small frame limit (the measured +pi-web-ui failure and catalog breakdown are in [#2512](https://github.com/morluto/rea/issues/2512)). +The default `full` profile is unchanged. With `off`, REA validates successful +structured results against the same named canonical tool contract before +delivery; it still returns complete `structuredContent` and text results. +Only client-side schema discovery and validation are omitted, so hosts using +this profile should rely on REA's validation and retain their usual response +size limits. + +### Function comparison limitations + +`compare_functions` compares the `pseudocode` and `assembly` dimensions as +exact provider text. A `changed` status establishes a textual difference, not a +behavior change. Addresses and generated names are not normalized, so relocated +labels can change these dimensions while `cfg` remains unchanged. Both text +dimensions carry this limitation inline. Numeric constants remain part of the +exact comparison. Reconstruction verification and static/runtime correlation +retain limitations from the selected dimension; an overall function claim +includes all dimension limitations. Equal CFGs establish only equal observed +block sizes and successor relationships. Text from different provider +identities remains unknown. + ### Comparison input shapes `compare_web_captures` accepts exactly one of two input shapes: diff --git a/src/application/AnalysisContextQueries.test.ts b/src/application/AnalysisContextQueries.test.ts index 99288d7a3..ef384e935 100644 --- a/src/application/AnalysisContextQueries.test.ts +++ b/src/application/AnalysisContextQueries.test.ts @@ -10,6 +10,7 @@ import { } from "./AnalysisProvider.js"; import { AnalysisCancelledError, + AnalysisOutputError, AnalysisProtocolError, } from "../domain/analysisErrorCore.js"; import { err, ok } from "../domain/result.js"; @@ -17,6 +18,35 @@ import { err, ok } from "../domain/result.js"; const provider = { id: "fixture", name: "Fixture", version: "1" }; describe("analysis context queries: navigation", () => { + it.each([ + [null, "0x401000", "current_document"], + ["fixture", null, "current_address"], + ] as const)( + "rejects malformed %s / %s provider selection before returning success", + async (document, address, operation) => { + const analysis: AnalysisOperationPort = { + execute: async (name) => + ok( + createAnalysisExecution( + name === "current_document" ? document : address, + provider, + ), + ), + }; + const result = await getNavigationContext(analysis, {}); + expect(result).toEqual( + err( + new AnalysisOutputError( + operation, + operation === "current_document" + ? "expected a document identity string" + : "expected an address string", + ), + ), + ); + }, + ); + it("aggregates volatile navigation and treats a missing procedure as null", async () => { const calls: { operation: string; @@ -70,6 +100,23 @@ describe("analysis context queries: navigation", () => { }); }); + it("preserves empty string selections accepted by the output contract", async () => { + const analysis: AnalysisOperationPort = { + execute: async (operation) => + ok( + createAnalysisExecution( + operation === "resolve_containing_procedure" + ? { found: false, procedure: null, reason: "not_in_procedure" } + : "", + provider, + ), + ), + }; + await expect(getNavigationContext(analysis, {})).resolves.toEqual( + ok({ document: "", address: "", procedure: null }), + ); + }); + it("uses an explicit document without passing it to current_document and preserves procedure failures", async () => { const calls: string[] = []; const failure = new AnalysisProtocolError("provider response malformed"); diff --git a/src/application/AnalysisContextQueries.ts b/src/application/AnalysisContextQueries.ts index 379ebc322..bd6846233 100644 --- a/src/application/AnalysisContextQueries.ts +++ b/src/application/AnalysisContextQueries.ts @@ -4,10 +4,11 @@ import type { AnalysisOperationPort, } from "./AnalysisProvider.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { analysisBookmarkSchema } from "../domain/hopperValues.js"; import { resolveAnalysisDocument } from "./AnalysisDocument.js"; -import { ok, type Result } from "../domain/result.js"; +import { err, ok, type Result } from "../domain/result.js"; type Facet = | { readonly state: "available"; readonly value: JsonValue } @@ -42,8 +43,14 @@ export const getNavigationContext = async ( if (typeof document !== "string" && !document.ok) return document; const documentName = typeof document === "string" ? document : document.value.result; - const documentArgs = - typeof documentName === "string" ? parameters(documentName) : {}; + if (typeof documentName !== "string") + return err( + new AnalysisOutputError( + "current_document", + "expected a document identity string", + ), + ); + const documentArgs = parameters(documentName); const address = await analysis.execute( "current_address", documentArgs, @@ -51,17 +58,12 @@ export const getNavigationContext = async ( ); if (!address.ok) return address; if (typeof address.value.result !== "string") - return ok({ - document: documentName, - address: address.value.result, - procedure: null, - }); + return err( + new AnalysisOutputError("current_address", "expected an address string"), + ); const procedure = await analysis.execute( "resolve_containing_procedure", - parameters( - typeof documentName === "string" ? documentName : undefined, - address.value.result, - ), + parameters(documentName, address.value.result), executionOptions(signal), ); if (!procedure.ok) { diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index 8a7874598..5e3799b0d 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -42,6 +42,7 @@ import { workflowAnalysisProfile, } from "./InvestigationProviders.js"; import { + executeComposedWorkflow, createWorkflowEvidence, workflowSnapshotRecord, recordWorkflowUnknowns, @@ -401,18 +402,17 @@ const executeAnalysisTool = async (input: { tool === "trace_native_ui_action" ) { const { EnhancedTools } = await import("./EnhancedTools.js"); - const result = await new EnhancedTools(session).execute( - tool, - input.arguments, - signal, + const result = await executeComposedWorkflow(session, (observed) => + new EnhancedTools(observed).execute(tool, input.arguments, signal), ); if (!result.ok) return { output: cliError(result.error) }; const evidence = createWorkflowEvidence({ target: input.openedTarget, operation: tool, parameters: input.arguments, - result: result.value, + result: result.value.result, upstreamProfile: evidenceProfile, + upstreamLimitations: result.value.limitations, }); return { output: evidence, evidence }; } diff --git a/src/application/WorkflowEvidence.test.ts b/src/application/WorkflowEvidence.test.ts index dab61a79c..b374efa01 100644 --- a/src/application/WorkflowEvidence.test.ts +++ b/src/application/WorkflowEvidence.test.ts @@ -3,15 +3,145 @@ import { describe, expect, it } from "vitest"; import { recordWorkflowUnknowns, createWorkflowEvidence, + executeComposedWorkflow, } from "./WorkflowEvidence.js"; import { InvestigationRecords } from "./investigation/InvestigationRecords.js"; import { UnknownRegistryError } from "../domain/unknownRegistryError.js"; -import { err } from "../domain/result.js"; +import { err, ok } from "../domain/result.js"; +import { + createAnalysisExecution, + type AnalysisOperationPort, +} from "./AnalysisProvider.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js"; import { projectNativeApiInspection } from "./native/NativeApiInspection.js"; import { functionDossierSchema } from "../domain/hopperValues.js"; import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js"; +const provider = { id: "fixture", name: "Fixture", version: "1" }; + +it("keeps Evidence identity stable when dependent branches finish in opposite orders", async () => { + const run = async (order: readonly number[]) => { + const gate = () => { + let resolve: () => void = () => { + throw new Error("gate not initialized"); + }; + const promise = new Promise((release) => { + resolve = release; + }); + return { promise, resolve }; + }; + const parents = [gate(), gate()]; + const children = [gate(), gate()]; + const analysis: AnalysisOperationPort = { + execute: async (operation, parameters) => { + const index = Number(parameters.index); + if (operation === "procedure_address") { + await parents[index]!.promise; + return ok(createAnalysisExecution(String(index), provider)); + } + children[index]!.resolve(); + return ok( + createAnalysisExecution(String(index), provider, { + limitations: [ + index === 0 ? "first child caveat" : "second child caveat", + ], + }), + ); + }, + }; + const composed = await executeComposedWorkflow( + analysis, + async (observed) => { + const branches = Promise.all( + [0, 1].map(async (index) => { + await observed.execute("procedure_address", { index }); + const child = await observed.execute("address_name", { index }); + if (!child.ok) throw child.error; + return child.value.result; + }), + ); + for (const index of order) { + parents[index]!.resolve(); + await children[index]!.promise; + } + return ok(await branches); + }, + ); + if (!composed.ok) throw composed.error; + return createWorkflowEvidence({ + target: undefined, + operation: "batch_decompile", + parameters: {}, + result: composed.value.result, + upstreamProfile: undefined, + upstreamLimitations: composed.value.limitations, + }); + }; + const left = await run([0, 1]); + const right = await run([1, 0]); + expect(left.normalized_result).toEqual(right.normalized_result); + expect(left.limitations).toEqual(right.limitations); + expect(left.evidence_id).toBe(right.evidence_id); +}); + +it("retains unique upstream limits in canonical order across concurrent observations", async () => { + let release: () => void = () => { + throw new Error("gate not initialized"); + }; + const gate = new Promise((resolve) => { + release = resolve; + }); + let calls = 0; + const options = { signal: new AbortController().signal }; + const analysis: AnalysisOperationPort = { + execute: async (_, parameters, forwarded) => { + expect(parameters).toEqual({ marker: true }); + expect(forwarded).toBe(options); + const call = calls++; + if (call === 0) await gate; + else release(); + return ok( + createAnalysisExecution([], provider, { + limitations: call === 0 ? ["first", "shared"] : ["second", "shared"], + }), + ); + }, + }; + const result = await executeComposedWorkflow(analysis, async (observed) => { + await Promise.all([ + observed.execute("list_strings", { marker: true }, options), + observed.execute("list_procedures", { marker: true }, options), + ]); + return ok({ count: 0 }); + }); + expect(calls).toBe(2); + expect(result).toEqual( + ok({ result: { count: 0 }, limitations: ["first", "second", "shared"] }), + ); +}); + +it("preserves a failed composed operation without an extra provider read", async () => { + const failure = err(new AnalysisCancelledError("list_strings")); + let calls = 0; + const result = await executeComposedWorkflow( + { + execute: async () => { + calls++; + return failure; + }, + }, + async (observed) => { + const observedResult = await observed.execute("list_strings", {}); + return observedResult.ok + ? ok(observedResult.value.result) + : observedResult; + }, + ); + expect(result).toBe(failure); + expect(calls).toBe(1); +}); + describe("workflow residual question retention", () => { it("keeps questions linked to workflow Evidence and records repeats idempotently", () => { const records = new InvestigationRecords(); @@ -24,6 +154,7 @@ describe("workflow residual question retention", () => { parameters: { procedure: "0x1000" }, result, upstreamProfile: undefined, + upstreamLimitations: [], }); expect(records.recordEvidence(evidence).ok).toBe(true); const input = { @@ -61,6 +192,7 @@ describe("workflow residual question retention", () => { parameters: {}, result, upstreamProfile: undefined, + upstreamLimitations: [], }); expect(records.recordEvidence(evidence).ok).toBe(true); @@ -88,6 +220,7 @@ describe("workflow residual question retention", () => { parameters: {}, result, upstreamProfile: undefined, + upstreamLimitations: [], }); expect(records.recordEvidence(evidence).ok).toBe(true); @@ -139,6 +272,7 @@ it.each([ parameters: { procedure: "0x1000" }, result, upstreamProfile: undefined, + upstreamLimitations: [], }); const records = new InvestigationRecords(); expect(records.recordEvidence(evidence).ok).toBe(true); diff --git a/src/application/WorkflowEvidence.ts b/src/application/WorkflowEvidence.ts index bf2530080..5bbd75455 100644 --- a/src/application/WorkflowEvidence.ts +++ b/src/application/WorkflowEvidence.ts @@ -2,7 +2,12 @@ import { REA_WORKFLOW_PROVIDER, workflowAnalysisProfile, } from "./InvestigationProviders.js"; -import type { AnalysisOperation } from "./AnalysisProvider.js"; +import type { + AnalysisOperation, + AnalysisOperationPort, +} from "./AnalysisProvider.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { ok, type Result } from "../domain/result.js"; import type { AnalysisProfileCommitment } from "../domain/analysisProfile.js"; import type { WorkflowSnapshotRecordInput } from "./binary/BinarySessionRecords.js"; import { @@ -18,6 +23,37 @@ import { import type { JsonValue } from "../domain/jsonValue.js"; import type { UnknownRegistryPort } from "./investigation/InvestigationRecordPort.js"; import { UnknownRegistryError } from "../domain/unknownRegistryError.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; + +/** Retain the limits of exactly the observations used by one composed call. */ +export const executeComposedWorkflow = async ( + analysis: AnalysisOperationPort, + execute: ( + observed: AnalysisOperationPort, + ) => Promise>, +): Promise< + Result< + { readonly result: JsonValue; readonly limitations: readonly string[] }, + AnalysisError + > +> => { + const limitations = new Set(); + const result = await execute({ + execute: async (operation, parameters, options) => { + const execution = await analysis.execute(operation, parameters, options); + if (execution.ok) + execution.value.limitations.forEach((limitation) => + limitations.add(limitation), + ); + return execution; + }, + }); + if (!result.ok) return result; + return ok({ + result: result.value, + limitations: [...limitations].sort(compareUnicodeCodePoints), + }); +}; /** Build the shared Evidence record for a successful composed binary workflow. */ export const createWorkflowEvidence = (input: { @@ -26,6 +62,7 @@ export const createWorkflowEvidence = (input: { readonly parameters: Readonly>; readonly result: JsonValue; readonly upstreamProfile: AnalysisProfileCommitment | undefined; + readonly upstreamLimitations: readonly string[]; }): Evidence => createEvidence(input.target, REA_WORKFLOW_PROVIDER, { operation: input.operation, @@ -40,7 +77,10 @@ export const createWorkflowEvidence = (input: { ), }), confidence: "derived", - limitations: ["Derived by an REA composed workflow."], + limitations: [ + "Derived by an REA composed workflow.", + ...input.upstreamLimitations, + ], ...(input.operation === "batch_decompile" ? { locations: batchProcedureLocations(input.result) } : {}), diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index 250cef4d5..774cfc00f 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -8,6 +8,7 @@ import { TOOL_CONTRACTS } from "./contracts/toolContracts.js"; import type { ToolContract } from "./contracts/toolContractTypes.js"; import { CLI_COMMAND_NAMES } from "./cliCommandNames.js"; import { compareUnicodeCodePoints } from "./domain/unicodeCodePointOrder.js"; +import { advertisedInputJsonSchema } from "./contracts/toolSchemaMetadata.js"; const sortByName = ( contracts: readonly ToolContract[], @@ -33,12 +34,7 @@ const createToolCatalog = (contracts: readonly ToolContract[]) => effects: { ...contract.effects }, returns_target_content: contract.returnsTargetContent, annotations: contract.annotations, - input_schema: { - type: "object", - ...z.toJSONSchema(contract.inputSchema, { - unrepresentable: "any", - }), - }, + input_schema: advertisedInputJsonSchema(contract), output_schema: { type: "object", ...z.toJSONSchema(contract.outputSchema, { diff --git a/src/config.test.ts b/src/config.test.ts index efa1b48fc..0230d552b 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vitest"; -import { parseConfig } from "./config/parseConfig.js"; +import { + defaultHopperLauncherPath, + parseConfig, +} from "./config/parseConfig.js"; import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; it("reports a configuration failure for a NUL in Hopper loader arguments", () => { @@ -324,3 +327,44 @@ describe("configuration failure diagnostics", () => { }); }); }); + +describe("default Linux Hopper launcher", () => { + const home = "/fixture/home"; + const withExecutables = + (paths: readonly string[]) => + (path: string): boolean => + paths.includes(path); + + it.each([["/opt/hopper-v4/bin/hopper"], ["/usr/bin/hopper"]])( + "selects an existing system launcher %s", + (path) => { + expect( + defaultHopperLauncherPath("linux", home, withExecutables([path])), + ).toBe(path); + }, + ); + + it("prefers REA-managed and user-local launchers over system paths", () => { + expect( + defaultHopperLauncherPath( + "linux", + home, + withExecutables([ + "/opt/hopper/bin/Hopper", + "/opt/hopper-v4/bin/hopper", + "/usr/bin/hopper", + ]), + ), + ).toBe("/opt/hopper/bin/Hopper"); + expect( + defaultHopperLauncherPath( + "linux", + home, + withExecutables([ + `${home}/.local/share/rea/hopper/bin/Hopper`, + "/usr/bin/hopper", + ]), + ), + ).toBe(`${home}/.local/share/rea/hopper/bin/Hopper`); + }); +}); diff --git a/src/config/environment.ts b/src/config/environment.ts index 121d46614..f5cc0373d 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -17,6 +17,7 @@ import { } from "./providerStartupTimeout.js"; import { mcpInputSchemaProfileSchema } from "./mcpInputSchemaProfile.js"; import { mcpResponseBudgetSchema } from "./mcpResponseBudget.js"; +import { parseLoaderArgs, parseStringArray } from "./parsers.js"; const environmentSchema = z.object({ REA_ANALYSIS_PROVIDER: analysisProviderSelectorSchema.default("auto"), @@ -74,9 +75,16 @@ const environmentSchema = z.object({ REA_REFERENCE_SECRET_PATTERNS_JSON: z.string().default("[]"), }); -export type Environment = z.infer; +type Environment = Omit< + z.infer, + "HOPPER_LOADER_ARGS_JSON" | "REA_REFERENCE_SECRET_PATTERNS_JSON" +> & { + readonly loaderArgs: readonly string[]; + readonly referenceSecretPatterns: readonly string[]; +}; -export const parseEnvironment = ( +/** Parse individual settings before layered registration fragments are merged. */ +export const parseEnvironmentValues = ( environment: Readonly>, ): Result => { const parsed = environmentSchema.safeParse(environment); @@ -93,5 +101,49 @@ export const parseEnvironment = ( }), ); } - return ok(parsed.data); + const { + HOPPER_LOADER_ARGS_JSON: encodedLoaderArgs, + REA_REFERENCE_SECRET_PATTERNS_JSON: encodedSecretPatterns, + ...env + } = parsed.data; + const loaderArgs = parseLoaderArgs(encodedLoaderArgs); + if (!loaderArgs.ok) return loaderArgs; + const secretPatterns = parseStringArray( + encodedSecretPatterns, + "REA_REFERENCE_SECRET_PATTERNS_JSON", + ); + if (!secretPatterns.ok) return secretPatterns; + return ok({ + ...env, + loaderArgs: loaderArgs.value, + referenceSecretPatterns: secretPatterns.value, + }); +}; + +/** Validate complete runtime semantics without probing the host or persisting defaults. */ +export const parseEnvironment = ( + environment: Readonly>, + // Registration expressions are not evaluated; their absence is not proven. + unresolvedSettings: ReadonlySet = new Set(), +): Result => { + const parsed = parseEnvironmentValues(environment); + if (!parsed.ok) return parsed; + const env = parsed.value; + // Ghidra's analyzeHeadless accepts -cspec only together with -processor. + if ( + env.REA_GHIDRA_COMPILER_SPEC_ID !== undefined && + env.REA_GHIDRA_LANGUAGE_ID === undefined && + !unresolvedSettings.has("REA_GHIDRA_LANGUAGE_ID") + ) + return err( + new ConfigurationError("Invalid REA environment configuration", { + settings: [ + { + setting: "REA_GHIDRA_COMPILER_SPEC_ID", + constraint: "requires REA_GHIDRA_LANGUAGE_ID", + }, + ], + }), + ); + return parsed; }; diff --git a/src/config/mcpOutputSchemaProfile.ts b/src/config/mcpOutputSchemaProfile.ts new file mode 100644 index 000000000..e981ac941 --- /dev/null +++ b/src/config/mcpOutputSchemaProfile.ts @@ -0,0 +1,34 @@ +import { z } from "zod"; + +import { ConfigurationError } from "../domain/configurationErrors.js"; +import { err, ok, type Result } from "../domain/result.js"; + +/** Select whether tools/list carries the large output contracts. */ +export const mcpOutputSchemaProfileSchema = z.enum(["full", "off"]); + +export type McpOutputSchemaProfile = z.infer< + typeof mcpOutputSchemaProfileSchema +>; + +const profileConstraint = + 'REA_MCP_OUTPUT_SCHEMA_PROFILE must be "full" (default) or "off". The off profile omits output schemas from tools/list while REA validates every successful result against its canonical tool contract and returns complete structured content.'; + +/** Parse the output schema advertisement selection at server composition. */ +export const parseMcpOutputSchemaProfile = ( + value: string | undefined, +): Result => { + if (value === undefined) return ok(undefined); + const parsed = mcpOutputSchemaProfileSchema.safeParse(value); + return parsed.success + ? ok(parsed.data) + : err( + new ConfigurationError(profileConstraint, { + settings: [ + { + setting: "REA_MCP_OUTPUT_SCHEMA_PROFILE", + constraint: profileConstraint, + }, + ], + }), + ); +}; diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index 757e02af0..415af6b3d 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -1,17 +1,17 @@ import { accessSync, constants } from "node:fs"; -import { join } from "node:path"; -import { ConfigurationError } from "../domain/configurationErrors.js"; -import { err, ok, type Result } from "../domain/result.js"; +import type { ConfigurationError } from "../domain/configurationErrors.js"; +import { ok, type Result } from "../domain/result.js"; import { parseEnvironment } from "./environment.js"; import { homeDirectoryFromEnvironment } from "./homeDirectory.js"; -import { parseStringArray, parseLoaderArgs } from "./parsers.js"; import type { AppConfig } from "./types.js"; +import { + linuxHopperLauncherCandidates, + REA_MANAGED_LINUX_HOPPER_LAUNCHER, +} from "../application/LinuxHopper.js"; const DEFAULT_HOPPER_LAUNCHER_PATH = "/Applications/Hopper Disassembler.app/Contents/MacOS/hopper"; -const SYSTEM_LINUX_HOPPER = "/opt/hopper/bin/Hopper"; - const executableAvailable = (path: string): boolean => { try { accessSync(path, constants.X_OK); @@ -21,15 +21,16 @@ const executableAvailable = (path: string): boolean => { } }; -const defaultHopperLauncherPath = ( +export const defaultHopperLauncherPath = ( platform: NodeJS.Platform, homeDirectory: string, executable: (path: string) => boolean = executableAvailable, ): string => { if (platform !== "linux") return DEFAULT_HOPPER_LAUNCHER_PATH; - if (executable(SYSTEM_LINUX_HOPPER)) return SYSTEM_LINUX_HOPPER; - const userLocal = join(homeDirectory, ".local/share/rea/hopper/bin/Hopper"); - return executable(userLocal) ? userLocal : SYSTEM_LINUX_HOPPER; + return ( + linuxHopperLauncherCandidates(homeDirectory).find(executable) ?? + REA_MANAGED_LINUX_HOPPER_LAUNCHER + ); }; /** Parse provider configuration and runtime prerequisites at the composition root. */ @@ -39,28 +40,6 @@ export const parseConfig = ( const parsed = parseEnvironment(environment); if (!parsed.ok) return parsed; const env = parsed.value; - const loaderArgs = parseLoaderArgs(env.HOPPER_LOADER_ARGS_JSON); - if (!loaderArgs.ok) return loaderArgs; - const secretPatterns = parseStringArray( - env.REA_REFERENCE_SECRET_PATTERNS_JSON, - "REA_REFERENCE_SECRET_PATTERNS_JSON", - ); - if (!secretPatterns.ok) return secretPatterns; - // Ghidra's analyzeHeadless accepts -cspec only together with -processor. - if ( - env.REA_GHIDRA_COMPILER_SPEC_ID !== undefined && - env.REA_GHIDRA_LANGUAGE_ID === undefined - ) - return err( - new ConfigurationError("Invalid REA environment configuration", { - settings: [ - { - setting: "REA_GHIDRA_COMPILER_SPEC_ID", - constraint: "requires REA_GHIDRA_LANGUAGE_ID", - }, - ], - }), - ); return ok({ ...(env.REA_MCP_MAX_RESPONSE_BYTES === undefined ? {} @@ -104,8 +83,10 @@ export const parseConfig = ( hopperStartupTimeoutMs: env.REA_HOPPER_STARTUP_TIMEOUT_MS, hopperTargetPath: env.HOPPER_TARGET_PATH, hopperTargetKind: env.HOPPER_TARGET_KIND, - hopperLoaderArgs: loaderArgs.value, + hopperLoaderArgs: env.loaderArgs, logLevel: env.REA_LOG_LEVEL, - referenceSourcePolicy: { secretPatterns: secretPatterns.value }, + referenceSourcePolicy: { + secretPatterns: env.referenceSecretPatterns, + }, }); }; diff --git a/src/contracts/adb/adbToolContracts.ts b/src/contracts/adb/adbToolContracts.ts index e72b603d7..dd5859617 100644 --- a/src/contracts/adb/adbToolContracts.ts +++ b/src/contracts/adb/adbToolContracts.ts @@ -127,7 +127,7 @@ export const ADB_TOOL_CONTRACTS = [ name: "push_adb_file", ...toolContractMetadata("push_adb_file"), description: - "Push one local file to an absolute device path. A device-mutating tool: existing device files are refused unless overwrite is explicitly requested, and the transfer is verified by comparing the local digest with the device's own sha256sum when the device provides one.", + "Push one local file to an absolute device path. A device-mutating tool: checks for an existing device file before transfer and refuses it unless overwrite is explicitly requested; this check is not an atomic create and another process can create the path before adb writes. When the device provides sha256sum, the transfer is verified by comparing it with the local digest.", kind: "adb-provider", inputSchema: adbInputSchemas.push_adb_file, outputSchema: evidenceResultOf(adbResultSchemas.push_adb_file), @@ -224,7 +224,7 @@ export const ADB_TOOL_CONTRACTS = [ name: "inspect_adb_window", ...toolContractMetadata("inspect_adb_window"), description: - "Report the window manager's current focus declarations (`dumpsys window windows`: mCurrentFocus and mFocusedApp) — what is on screen at read time without any interaction. Read-only.", + "Report the window manager's current focus declarations (mCurrentFocus and mFocusedApp from `dumpsys window windows`, or from `dumpsys window displays` when the windows section declares neither, as on Android 16) — what is on screen at read time without any interaction. Read-only.", kind: "adb-provider", inputSchema: adbInputSchemas.inspect_adb_window, outputSchema: evidenceResultOf(adbResultSchemas.inspect_adb_window), diff --git a/src/contracts/compactInputSchema.test.ts b/src/contracts/compactInputSchema.test.ts index 3fcfc4265..212d3dbb4 100644 --- a/src/contracts/compactInputSchema.test.ts +++ b/src/contracts/compactInputSchema.test.ts @@ -162,6 +162,24 @@ describe("compact advertised input schema", () => { expect(compact(input)).toBe(canonical(input)); }); + it("does not reintroduce unsupported dialect keywords in reduced schemas", () => { + let nested: Record = { type: "string" }; + for (let level = 0; level < 12; level += 1) + nested = { type: "object", properties: { child: nested } }; + const { schema, presentation } = compactAdvertisedInputSchema( + { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: { deep: nested }, + dependentRequired: { detail: ["target"] }, + }, + 1_500, + ); + expect(presentation).toBe("reduced"); + expect(schema.$schema).toBe("https://json-schema.org/draft/2020-12/schema"); + expect(schema).not.toHaveProperty("dependentRequired"); + }); + it("keeps unconstrained JSON values explicit for client type inference", () => { const { schema } = compactAdvertisedInputSchema( { diff --git a/src/contracts/compactInputSchema.ts b/src/contracts/compactInputSchema.ts index 321ea8323..f756fdda4 100644 --- a/src/contracts/compactInputSchema.ts +++ b/src/contracts/compactInputSchema.ts @@ -37,6 +37,22 @@ const isSchemaObject = (value: unknown): value is Record => const serializedBytes = (value: unknown): number => Buffer.byteLength(JSON.stringify(value), "utf8"); +const schemaDepth = (value: unknown, depth = 0): number => { + if (Array.isArray(value)) + return Math.max( + depth, + ...value.map((child) => schemaDepth(child, depth + 1)), + ); + if (!isSchemaObject(value)) return depth; + return Math.max( + depth, + ...Object.values(value).map((child) => schemaDepth(child, depth + 1)), + ); +}; + +/** Strict clients commonly reject advertised schemas deeper than ten nodes. */ +const COMPACT_SCHEMA_MAX_DEPTH = 10; + /** Rewrite every schema-valued position; literal data stays opaque. */ const rebuildSchemaPositions = ( node: unknown, @@ -240,12 +256,24 @@ const reduceToShallowProperties = ( const properties = isSchemaObject(schema.properties) ? schema.properties : {}; const description = [ typeof schema.description === "string" ? schema.description : undefined, + ...Object.entries( + isSchemaObject(schema.dependentRequired) + ? schema.dependentRequired + : isSchemaObject(schema.dependencies) + ? schema.dependencies + : {}, + ).flatMap(([property, partners]) => + Array.isArray(partners) && + partners.every((partner) => typeof partner === "string") + ? [ + `When ${property} is provided, also provide ${partners.join(" and ")}.`, + ] + : [], + ), REDUCED_ANNOTATION, ] .filter((text) => text !== undefined) .join(" "); - const $schema = - typeof schema.$schema === "string" ? { $schema: schema.$schema } : {}; const shallow = Object.fromEntries( Object.entries(properties).map(([name, value]) => { const types = detectedTypes(value, schema); @@ -261,7 +289,7 @@ const reduceToShallowProperties = ( }), ); return { - ...$schema, + ...(typeof schema.$schema === "string" ? { $schema: schema.$schema } : {}), type: "object", description, properties: shallow, @@ -271,9 +299,6 @@ const reduceToShallowProperties = ( ...(typeof schema.minProperties === "number" ? { minProperties: schema.minProperties } : {}), - ...(isSchemaObject(schema.dependentRequired) - ? { dependentRequired: { ...schema.dependentRequired } } - : {}), ...(schema.additionalProperties === false ? { additionalProperties: false } : {}), @@ -292,7 +317,8 @@ export const compactAdvertisedInputSchema = ( budgetBytes: number, ): CompactedInputSchema => { const inline = inlineWithinBudget(stripAnnotations(schema), budgetBytes); - if (inline !== undefined) return { schema: inline, presentation: "inline" }; + if (inline !== undefined && schemaDepth(inline) <= COMPACT_SCHEMA_MAX_DEPTH) + return { schema: inline, presentation: "inline" }; const reduced = reduceToShallowProperties(schema); if (serializedBytes(reduced) <= budgetBytes) return { schema: reduced, presentation: "reduced" }; diff --git a/src/contracts/enhancedToolContracts.ts b/src/contracts/enhancedToolContracts.ts index be8c7dac9..9b124f818 100644 --- a/src/contracts/enhancedToolContracts.ts +++ b/src/contracts/enhancedToolContracts.ts @@ -62,7 +62,7 @@ export const ENHANCED_TOOL_CONTRACTS = [ ), enhanced( "binary_overview", - "Return native-binary metadata, every segment with its length, and exhaustive procedure/string counts. When every provider-reported segment agrees on an analysis image base, return it as image_base so callers can relate provider addresses to their loader coordinates. Use search_procedures, list_procedures, or analyze_function directly when you already know which procedure to inspect.", + "Return the bound document and exhaustive procedure/string counts, plus every segment and segment_count when the provider supports segment enumeration. If segment fields are omitted, segment coverage is unavailable rather than empty. When every observed segment agrees on an analysis image base, return it as image_base so callers can relate provider addresses to their loader coordinates. Use search_procedures, list_procedures, or analyze_function directly when you already know which procedure to inspect.", enhancedInputSchemas.binary_overview, ), ...FUNCTION_WORKFLOW_TOOL_CONTRACTS, diff --git a/src/contracts/firmware/firmwareToolContracts.ts b/src/contracts/firmware/firmwareToolContracts.ts index c93c8f903..00b1e373b 100644 --- a/src/contracts/firmware/firmwareToolContracts.ts +++ b/src/contracts/firmware/firmwareToolContracts.ts @@ -30,7 +30,7 @@ export const FIRMWARE_TOOL_CONTRACTS = [ ...toolContractMetadata("extract_firmware"), kind: "firmware-provider", description: - "Extract caller-selected firmware bytes with Unblob 26.6.x (verified with 26.6.4) into an absent output directory. Returns verified child paths/digests, task derivations, unknown chunks and extraction diagnostics inline. Preserves the default sandbox; publishes regular files only. Linux, caller-supplied tool, format-specific extractors and util-linux prlimit required. No mounts or extracted-code execution. Select a returned native file with open_binary for Ghidra analysis; decompressed offsets are not original-file addresses.", + "Extract caller-selected firmware bytes with Unblob 26.6.x (verified with 26.6.4) into an absent output directory. Returns verified child paths/digests, task derivations, unknown chunks, extraction diagnostics and explicit extraction_gaps rows naming unextracted regions, missing extractor dependencies and a continuation_id referencing shared extraction_continuations recovery instructions. Preserves the default sandbox; publishes regular files only. Linux, caller-supplied tool, format-specific extractors and util-linux prlimit required. No mounts or extracted-code execution. Select a returned native file with open_binary for Ghidra analysis; decompressed offsets are not original-file addresses.", inputSchema: firmwareInputSchemas.extract_firmware, outputSchema: evidenceResultOf(firmwareResultSchemas.extract_firmware), examples: [ diff --git a/src/contracts/inputSchemaPresentation.ts b/src/contracts/inputSchemaPresentation.ts index 22ad0ae30..3ed83907a 100644 --- a/src/contracts/inputSchemaPresentation.ts +++ b/src/contracts/inputSchemaPresentation.ts @@ -174,17 +174,6 @@ const flattenRootObjectUnion = ( const required = sharedRequiredProperties(branches); // A field unique to a group still needs that group's required partners. // Counting properties alone would let optional fields pad an incomplete pair. - const dependencies = Object.fromEntries( - [...variants.keys()].flatMap((name) => { - const owners = branches.filter((branch) => - Object.hasOwn(branch.properties ?? {}, name), - ); - const partners = sharedRequiredProperties(owners).filter( - (partner) => partner !== name && !required.includes(partner), - ); - return partners.length > 0 ? [[name, partners]] : []; - }), - ); const fewestRequired = Math.min( ...branches.map((branch) => requiredProperties(branch).length), ); @@ -196,8 +185,13 @@ const flattenRootObjectUnion = ( ? `Provide a complete input group: ${groups.join("; ")}.` : undefined, ].filter((text) => text !== undefined); + const { + dependencies: _dependencies, + dependentRequired: _dependentRequired, + ...presentation + } = rest; return { - ...rest, + ...presentation, type: "object", ...(description.length > 0 ? { description: description.join(" ") } : {}), properties: Object.fromEntries( @@ -208,13 +202,6 @@ const flattenRootObjectUnion = ( ), ...(required.length > 0 ? { required } : {}), ...(fewestRequired > 0 ? { minProperties: fewestRequired } : {}), - ...(Object.keys(dependencies).length > 0 - ? { - [root.$schema === "http://json-schema.org/draft-07/schema#" - ? "dependencies" - : "dependentRequired"]: dependencies, - } - : {}), ...(branches.every((branch) => branch.additionalProperties === false) ? { additionalProperties: false } : {}), @@ -228,7 +215,36 @@ export const presentInputJsonSchema = ( ): Record => { const flattened = flattenRootObjectUnion(root); const described = describeSchema(flattened, flattened, describeProperty); - return isObject(described) ? described : flattened; + if (!isObject(described)) return flattened; + const { + dependencies: _dependencies, + dependentRequired: _dependentRequired, + ...presentation + } = described; + const rootDependencies = isObject(described.dependentRequired) + ? described.dependentRequired + : isObject(described.dependencies) + ? described.dependencies + : undefined; + const dependencyGuidance = + rootDependencies === undefined + ? [] + : Object.entries(rootDependencies).flatMap(([property, partners]) => + Array.isArray(partners) && + partners.every((partner) => typeof partner === "string") + ? [ + `When ${property} is provided, also provide ${partners.join(" and ")}.`, + ] + : [], + ); + if (dependencyGuidance.length === 0) return presentation; + const description = [ + typeof presentation.description === "string" + ? presentation.description + : undefined, + ...dependencyGuidance, + ].filter((text) => text !== undefined); + return { ...presentation, description: description.join(" ") }; }; /** diff --git a/src/contracts/providerSelection.ts b/src/contracts/providerSelection.ts index d43ee5852..2a9e1bc0f 100644 --- a/src/contracts/providerSelection.ts +++ b/src/contracts/providerSelection.ts @@ -20,6 +20,7 @@ export const PROVIDER_REJECTION_CODES = [ "not_configured", "executable_missing", "runtime_missing", + "runtime_unavailable", "unsupported_host", "unsupported_version", "target_kind_unsupported", diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts index 7d23db92d..fbb92e2ab 100644 --- a/src/contracts/toolEffects.test.ts +++ b/src/contracts/toolEffects.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from "vitest"; import { TOOL_CONTRACTS } from "./toolContracts.js"; -import { TOOL_EFFECTS, annotationsFromEffects } from "./toolEffects.js"; +import { + TOOL_EFFECTS, + annotationsFromEffects, + type ToolEffects, +} from "./toolEffects.js"; /** * Descriptions that name running, launching, executing, starting, installing, @@ -159,8 +163,19 @@ describe("tool effects", () => { }); it("derives destructiveHint from each destructive effect alone", () => { - const base = TOOL_EFFECTS["binary_session"]; - if (base === undefined) throw new Error("binary_session is unaudited"); + // Keep annotation algebra independent of any operation's audited effects. + const base: ToolEffects = { + mutatesTarget: false, + mutatesSession: false, + writesFilesystem: false, + launchesProcess: false, + accessesNetwork: false, + changesUiState: false, + mayDiscardData: false, + executesForeignCode: false, + mutatesDevice: false, + idempotent: true, + }; expect(annotationsFromEffects(base)).toEqual({ readOnlyHint: true, destructiveHint: false, diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index c64edefca..a010467be 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -677,7 +677,7 @@ export const TOOL_EFFECTS: Readonly> = { mayDiscardData: true, idempotent: false, }), - binary_session: effects(), + binary_session: effects({ launchesProcess: true }), export_evidence_bundle: effects({ writesFilesystem: true, mayDiscardData: true, diff --git a/src/contracts/toolInputReferences.test.ts b/src/contracts/toolInputReferences.test.ts index f75a06531..24985ed21 100644 --- a/src/contracts/toolInputReferences.test.ts +++ b/src/contracts/toolInputReferences.test.ts @@ -40,7 +40,41 @@ it.each(["draft-2020-12", "draft-07"] as const)( }, ); -describe("advertised input references", () => { +describe("advertised object dependencies", () => { + it.each([ + ["draft-07", "dependencies"], + ["draft-2020-12", "dependentRequired"], + ] as const)( + "presents object-group dependencies as guidance without unsupported dialect keywords (%s)", + (dialect, dependencyKeyword) => { + const advertised = presentInputJsonSchema( + { + $schema: + dialect === "draft-07" + ? "http://json-schema.org/draft-07/schema#" + : "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: { url: { type: "string" }, body: { type: "string" } }, + required: ["body"], + [dependencyKeyword]: { body: ["url"] }, + }, + () => undefined, + ); + expect(advertised.$schema).toBe( + dialect === "draft-07" + ? "http://json-schema.org/draft-07/schema#" + : "https://json-schema.org/draft/2020-12/schema", + ); + expect(advertised).not.toHaveProperty("dependencies"); + expect(advertised).not.toHaveProperty("dependentRequired"); + expect(advertised.description).toContain( + "When body is provided, also provide url.", + ); + }, + ); +}); + +describe("advertised production selector contracts", () => { it.each(["draft-2020-12", "draft-07"] as const)( "preserves production native-selector constraints in %s", (target) => { @@ -89,7 +123,9 @@ describe("advertised input references", () => { expect(validate(unexpected)).toBe(false); }, ); +}); +describe("advertised schema metadata and validation", () => { it.each(["draft-2020-12", "draft-07"] as const)( "retains specialized descriptions, literal metadata and strict validation in %s", (target) => { @@ -339,12 +375,20 @@ describe("root input presentation", () => { { before: "first" }, { unexpected: true }, { before_scenario: 1, after_scenario: "wrong" }, - { before_scenario: 1, normalization: true }, - { after_scenario: 2, normalization: true }, ]) { expect(schema.safeParse(input).success).toBe(false); expect(validate(input)).toBe(false); } + // The vendor-safe advertisement uses group guidance instead of + // dependentRequired; the canonical runtime parser keeps exact unions. + expect( + schema.safeParse({ before_scenario: 1, normalization: true }).success, + ).toBe(false); + expect(validate({ before_scenario: 1, normalization: true })).toBe(true); + expect( + schema.safeParse({ after_scenario: 2, normalization: true }).success, + ).toBe(false); + expect(validate({ after_scenario: 2, normalization: true })).toBe(true); // Cross-group exclusion is enforced only by the canonical runtime parser. const mixed = { before: "first", diff --git a/src/contracts/toolSchemaMetadata.ts b/src/contracts/toolSchemaMetadata.ts index 3034b8354..093f4845c 100644 --- a/src/contracts/toolSchemaMetadata.ts +++ b/src/contracts/toolSchemaMetadata.ts @@ -94,6 +94,14 @@ export const toolInputSchemaWithMetadata = ( }, ); +/** Return the same complete input JSON Schema projection registered on MCP. */ +export const advertisedInputJsonSchema = ( + contract: Contract, +): Record => + toolInputSchemaWithMetadata(contract)["~standard"].jsonSchema.input({ + target: "draft-2020-12", + }) as Record; + /** Share output definitions and content-bound validator identities on the wire. */ export const toolOutputSchemaWithMetadata = ( contract: Contract, diff --git a/src/domain/analysisErrorCore.ts b/src/domain/analysisErrorCore.ts index 380b440a7..71a77a647 100644 --- a/src/domain/analysisErrorCore.ts +++ b/src/domain/analysisErrorCore.ts @@ -83,17 +83,29 @@ export interface AnalysisInputIssue { readonly maximum?: number; } +/** Refusal-specific context a producer projects with an output failure. */ +export interface AnalysisOutputErrorOptions extends AnalysisErrorOptions { + /** Caller-facing explanation for a deterministic, diagnosable refusal. */ + readonly userMessage?: string; + readonly remediationAction?: string; + readonly cleanupOnly?: boolean; +} + /** Provider output failed provider-neutral application parsing. */ export class AnalysisOutputError extends AnalysisError { readonly _tag = "AnalysisOutputError"; + override readonly userMessage: string | undefined; + readonly remediationAction: string | undefined; readonly cleanupOnly: boolean; constructor( readonly operation: string, readonly reason: string, - options?: AnalysisErrorOptions & { readonly cleanupOnly?: boolean }, + options?: AnalysisOutputErrorOptions, ) { super(`Invalid analysis output for ${operation}: ${reason}`, options); + this.userMessage = options?.userMessage; + this.remediationAction = options?.remediationAction; this.cleanupOnly = options?.cleanupOnly ?? false; } } diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 19c3a9002..54ad586c5 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -3,6 +3,7 @@ import { AnalysisArtifactChangedError, AnalysisCapabilityUnavailableError, AnalysisInputError, + AnalysisOutputError, AnalysisUnsupportedTargetError, AnalysisResourceConstraintError, } from "./analysisErrorCore.js"; @@ -107,6 +108,11 @@ export const analysisErrorRemediationAction = ( return `Correct ${[...new Set(error.settings.map(({ setting }) => setting))].join(", ")} in the environment that starts REA (your shell or the MCP client's registration), then rerun.`; if (error instanceof AnalysisInputError) return "Correct the listed arguments and retry."; + if ( + error instanceof AnalysisOutputError && + error.remediationAction !== undefined + ) + return error.remediationAction; if (error instanceof UnknownRegistryError && error.reason === "not-found") return "Check that the unknown_id belongs to this session, then retry."; return analysisErrorUserMessage(error); diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index fbd1b5386..2884bcfcb 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -273,7 +273,13 @@ const requestErrorDetails = ( boundary: "filesystem-read", }; if (error instanceof AnalysisOutputError) - return { operation: error.operation, reason: error.reason }; + return { + operation: error.operation, + reason: error.reason, + ...(error.diagnostics === undefined + ? {} + : { diagnostics: error.diagnostics }), + }; if (error instanceof AnalysisInputError && error.issues.length > 0) return { operation: error.operation, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index b08001e35..c15d15a12 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -62,6 +62,13 @@ import { machoViewRequestSchema, projectMachoView, } from "./machoView.js"; +import { + interfaceBuilderSummarySchema, + interfaceBuilderViewContextSchema, + interfaceBuilderViewItemsSchema, + interfaceBuilderViewRequestSchema, + projectInterfaceBuilderView, +} from "./interfaceBuilderView.js"; import { projectWebNetworkCaptureView, captureViewRequestSchema, @@ -206,6 +213,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ inventoryViewSchema, wasmViewRequestSchema, machoViewRequestSchema, + interfaceBuilderViewRequestSchema, evmViewRequestSchema, managedViewRequestSchema, androidViewRequestSchema, @@ -326,6 +334,7 @@ const viewResultBase = { "inventory_artifact", "inspect_wasm_artifact", "inspect_macho", + "decode_interface_builder", "inspect_evm_interface", "inspect_managed_artifact", "inspect_web_network_capture", @@ -382,6 +391,13 @@ export const machoSummaryViewResultSchema = z.strictObject({ summary: machoSummarySchema, }); +/** Decoded documents and graph counts for Interface Builder's selected delivery. */ +export const interfaceBuilderSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + parent_operation: z.literal("decode_interface_builder"), + summary: interfaceBuilderSummarySchema, +}); + /** Exact inferred-interface summary for EVM's caller-selected delivery. */ export const evmInterfaceSummaryViewResultSchema = z.strictObject({ ...summaryViewResultBase, @@ -444,6 +460,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ inventorySummarySchema, wasmSummarySchema, machoSummarySchema, + interfaceBuilderSummarySchema, evmSummarySchema, managedSummarySchema, captureSummarySchema, @@ -514,6 +531,14 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ ...machoViewContextSchema.shape, parent_operation: z.literal("inspect_macho"), }), + z.strictObject({ + kind: z.literal("interface-builder"), + view: interfaceBuilderViewRequestSchema, + items: interfaceBuilderViewItemsSchema, + ...viewResultBase, + ...interfaceBuilderViewContextSchema.shape, + parent_operation: z.literal("decode_interface_builder"), + }), z.strictObject({ kind: z.literal("evm"), view: evmViewRequestSchema, @@ -611,6 +636,10 @@ export type UnsignedAnalysisView = > | Omit, "view_digest"> | Omit, "view_digest"> + | Omit< + Extract, + "view_digest" + > | Omit, "view_digest"> | Omit, "view_digest"> | Omit, "view_digest"> @@ -674,7 +703,7 @@ const unsupportedParent = ( new AnalysisUnsupportedTargetError( INSPECT_ANALYSIS_VIEW_OPERATION, parentArtifactPath(parent), - `Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function, artifact inventory/inspection, WASM, Mach-O, EVM, managed artifact, Android package, Dart AOT, or web network capture Evidence.`, + `Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function, artifact inventory/inspection, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package, Dart AOT, or web network capture Evidence.`, ); const parentArtifactPath = (parent: AnalysisViewParent): string => { @@ -702,6 +731,7 @@ const parentArtifactPath = (parent: AnalysisViewParent): string => { const retainedViewProjectors = new Map([ ["inspect_wasm_artifact", projectWasmArtifactView], ["inspect_macho", projectMachoView], + ["decode_interface_builder", projectInterfaceBuilderView], ["inspect_evm_interface", projectEvmInterfaceView], ["inspect_managed_artifact", projectManagedArtifactView], ["inspect_android_package", projectAndroidPackageView], diff --git a/src/domain/analysisView/analysisViewParent.ts b/src/domain/analysisView/analysisViewParent.ts new file mode 100644 index 000000000..0a2b57180 --- /dev/null +++ b/src/domain/analysisView/analysisViewParent.ts @@ -0,0 +1,25 @@ +import type { z } from "zod"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisViewParent } from "./analysisView.js"; + +/** Parse a parent's result with its owning schema; issues point into the view source. */ +export const parseAnalysisViewResult = ( + schema: Schema, + parent: AnalysisViewParent, +): Result, AnalysisError> => { + const parsed = schema.safeParse(parent.normalizedResult); + if (parsed.success) return ok(parsed.data); + return err( + analysisInputErrorFromIssues( + "inspect_analysis_view", + parsed.error.issues.map((issue) => ({ + ...issue, + path: ["source", "normalized_result", ...issue.path], + })), + { source: { normalized_result: parent.normalizedResult } }, + { cause: parsed.error }, + ), + ); +}; diff --git a/src/mcpDoctor.ts b/src/mcpDoctor.ts index d7bf1e366..ee2529ff1 100644 --- a/src/mcpDoctor.ts +++ b/src/mcpDoctor.ts @@ -280,6 +280,7 @@ export const runProductionMcpDoctorCli = async ( arguments_: readonly string[], input: { readonly dispatcherPath: string; readonly packageRoot: string }, ): Promise<{ readonly output: string; readonly exitCode: number }> => { + const started = performance.now(); const parsed = parseOutputArguments(arguments_); if (!parsed.ok) { if (parsed.help) @@ -298,8 +299,18 @@ export const runProductionMcpDoctorCli = async ( cwd: input.packageRoot, environment: process.env, }); + const outputValue = parsed.fullOutput + ? { + ok: result.healthy, + ...(result.healthy ? { data: result } : { error: result }), + meta: { + command: MCP_DOCTOR_COMMAND, + duration: `${Math.round(performance.now() - started)}ms`, + }, + } + : result; return { - output: `${Formatter.format(result, parsed.format)}\n`, + output: `${Formatter.format(outputValue, parsed.format)}\n`, exitCode: result.healthy ? 0 : 1, }; }; @@ -369,7 +380,11 @@ const parseIdentityToolResult = ( * flag is an output selector declared in `MCP_DOCTOR_OPTIONS`. */ type McpDoctorArguments = - | { readonly ok: true; readonly format: OutputFormat } + | { + readonly ok: true; + readonly format: OutputFormat; + readonly fullOutput: boolean; + } | { readonly ok: false; readonly help: true } | { readonly ok: false; @@ -387,6 +402,7 @@ const parseOutputArguments = ( // masking the typo with usage. if (hasHelpInOptionPosition(arguments_)) return { ok: false, help: true }; let format: OutputFormat = "toon"; + let fullOutput = false; for (let index = 0; index < arguments_.length; index += 1) { const argument = arguments_[index]; if (argument === "--json") { @@ -409,7 +425,10 @@ const parseOutputArguments = ( format = candidate; continue; } - if (argument === "--full-output") continue; + if (argument === "--full-output") { + fullOutput = true; + continue; + } if (HELP_FLAGS.has(argument ?? "")) return { ok: false, help: true }; return { ok: false, @@ -418,7 +437,7 @@ const parseOutputArguments = ( message: `Unknown mcp doctor option: ${argument ?? "missing"}`, }; } - return { ok: true, format }; + return { ok: true, format, fullOutput }; }; /** diff --git a/src/server/EvidenceMcpServer.test.ts b/src/server/EvidenceMcpServer.test.ts new file mode 100644 index 000000000..dedbfe5ce --- /dev/null +++ b/src/server/EvidenceMcpServer.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; + +import { forgetPendingMcpRequest } from "./EvidenceMcpServer.js"; + +describe("MCP request bookkeeping", () => { + it("releases list, tool, and completion state when cancellation arrives", () => { + const completion = new AbortController(); + const pendingTools = new Map([[7, "inspect_artifact"]]); + const pendingToolLists = new Set([7]); + const pendingCompletions = new Map([[7, completion]]); + + forgetPendingMcpRequest( + pendingTools, + pendingToolLists, + pendingCompletions, + 7, + ); + + expect(pendingTools.has(7)).toBe(false); + expect(pendingToolLists.has(7)).toBe(false); + expect(completion.signal.aborted).toBe(true); + expect(pendingCompletions.has(7)).toBe(false); + }); +}); diff --git a/src/server/EvidenceMcpServer.ts b/src/server/EvidenceMcpServer.ts index f42fe9de6..28618adce 100644 --- a/src/server/EvidenceMcpServer.ts +++ b/src/server/EvidenceMcpServer.ts @@ -60,6 +60,47 @@ const hasAdvertisedInputJsonSchema = (value: unknown): boolean => { ); }; +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** Remove only client-facing output schemas; SDK registrations stay canonical. */ +const omitOutputSchemasFromToolsList = ( + message: JSONRPCMessage, + requestId: RequestId, +): JSONRPCMessage => { + if ( + !isJSONRPCResultResponse(message) || + message.id !== requestId || + !isRecord(message.result) || + !Array.isArray(message.result.tools) + ) + return message; + return { + ...message, + result: { + ...message.result, + tools: message.result.tools.map((tool: unknown) => { + if (!isRecord(tool)) return tool; + const { outputSchema: _outputSchema, ...advertisedTool } = tool; + return advertisedTool; + }), + }, + } as JSONRPCMessage; +}; + +/** Release every transport-owned reference for a cancelled/completed request. */ +export const forgetPendingMcpRequest = ( + pendingTools: Map, + pendingToolLists: Set, + pendingCompletions: Map, + id: RequestId, +): void => { + pendingTools.delete(id); + pendingToolLists.delete(id); + pendingCompletions.get(id)?.abort(); + pendingCompletions.delete(id); +}; + /** Bind oversized MCP error recovery to this server's Evidence ledger. */ export class EvidenceMcpServer extends McpServer { constructor( @@ -72,15 +113,15 @@ export class EvidenceMcpServer extends McpServer { private readonly compactPresentation: | CompactInputSchemaPresentation | undefined = undefined, + private readonly omitOutputSchemas = false, ) { super(info, options); } /** - * Compact the advertised input schema of every registered tool when the - * compact profile is active. The registered canonical schema, its parser, - * and output schemas are untouched; only the advertised JSON Schema - * projection is rendered within the provider's per-schema size budget. + * Compact advertised input schemas and optionally omit output-schema + * advertisements. Canonical parsers and contracts remain registered; the + * output-schema-off path validates successful structured results locally. * The signatures mirror the SDK's registrations so call sites keep their * inferred handler argument types. */ @@ -114,11 +155,11 @@ export class EvidenceMcpServer extends McpServer { compactAdvertisedInputSchema(projected, budget).schema, ) : undefined; + const registeredConfig = + inputSchema === undefined ? config : { ...config, inputSchema }; return super.registerTool( name, - (inputSchema === undefined - ? config - : { ...config, inputSchema }) as never, + registeredConfig as never, callback as never, ); } @@ -126,7 +167,12 @@ export class EvidenceMcpServer extends McpServer { /** Preserve the SDK transport lifecycle while retaining oversized errors. */ override connect(transport: Transport): Promise { return super.connect( - new ErrorEvidenceTransport(transport, this.recordEvidence, this.delivery), + new ErrorEvidenceTransport( + transport, + this.recordEvidence, + this.delivery, + this.omitOutputSchemas, + ), ); } } @@ -137,6 +183,7 @@ class ErrorEvidenceTransport implements Transport { onmessage: Transport["onmessage"]; readonly hasPerRequestStream?: boolean; private readonly pendingTools = new Map(); + private readonly pendingToolLists = new Set(); private readonly pendingCompletions = new Map(); constructor( @@ -145,6 +192,7 @@ class ErrorEvidenceTransport implements Transport { | EvidenceWriter["recordEvidence"] | undefined, private readonly delivery: ToolResultDelivery, + private readonly omitOutputSchemas: boolean, ) { if (transport.hasPerRequestStream !== undefined) this.hasPerRequestStream = transport.hasPerRequestStream; @@ -155,6 +203,8 @@ class ErrorEvidenceTransport implements Transport { typeof message.params?.["name"] === "string" ) this.pendingTools.set(message.id, message.params["name"]); + if (isJSONRPCRequest(message) && message.method === "tools/list") + this.pendingToolLists.add(message.id); if (isJSONRPCRequest(message) && message.method === "completion/complete") this.pendingCompletions.set(message.id, new AbortController()); // The SDK suppresses replies after cancellation, so send() cannot release @@ -164,11 +214,13 @@ class ErrorEvidenceTransport implements Transport { message.method === "notifications/cancelled" ) { const id = message.params?.["requestId"]; - if (typeof id === "string" || typeof id === "number") { - this.pendingTools.delete(id); - this.pendingCompletions.get(id)?.abort(); - this.pendingCompletions.delete(id); - } + if (typeof id === "string" || typeof id === "number") + forgetPendingMcpRequest( + this.pendingTools, + this.pendingToolLists, + this.pendingCompletions, + id, + ); } const completion = isJSONRPCRequest(message) && message.method === "completion/complete" @@ -183,6 +235,7 @@ class ErrorEvidenceTransport implements Transport { transport.onerror = (error) => this.onerror?.(error); transport.onclose = () => { this.pendingTools.clear(); + this.pendingToolLists.clear(); for (const completion of this.pendingCompletions.values()) completion.abort(); this.pendingCompletions.clear(); @@ -206,6 +259,7 @@ class ErrorEvidenceTransport implements Transport { await this.transport.close(); } finally { this.pendingTools.clear(); + this.pendingToolLists.clear(); for (const completion of this.pendingCompletions.values()) completion.abort(); this.pendingCompletions.clear(); @@ -230,17 +284,24 @@ class ErrorEvidenceTransport implements Transport { ? message.id : undefined; const tool = id === undefined ? undefined : this.pendingTools.get(id); + const omitToolSchemas = + this.omitOutputSchemas && + id !== undefined && + this.pendingToolLists.has(id); try { const recovered = this.recoverError(message, tool); + const advertised = omitToolSchemas + ? omitOutputSchemasFromToolsList(recovered, id) + : recovered; if ( - isJSONRPCResultResponse(recovered) && - isCallToolResult(recovered.result) && - recovered.result.isError === true + isJSONRPCResultResponse(advertised) && + isCallToolResult(advertised.result) && + advertised.result.isError === true ) { // Keep the complete typed JSON in content. Error projections are not // success outputSchema data; oversized recovery must consume their // private structured carrier before it is removed from the wire. - const { structuredContent: _diagnostic, ...result } = recovered.result; + const { structuredContent: _diagnostic, ...result } = advertised.result; // Errors the SDK builds itself (input validation, a thrown handler) // bypass ToolResultDelivery; mark them for target-content tools too. const marked = @@ -252,12 +313,16 @@ class ErrorEvidenceTransport implements Transport { _meta: { ...result._meta, ...TARGET_CONTENT_RESULT_META }, } : result; - await this.transport.send({ ...recovered, result: marked }, options); - } else await this.transport.send(recovered, options); + await this.transport.send({ ...advertised, result: marked }, options); + } else await this.transport.send(advertised, options); } finally { if (!isJSONRPCRequest(message) && id !== undefined) { - this.pendingTools.delete(id); - this.pendingCompletions.delete(id); + forgetPendingMcpRequest( + this.pendingTools, + this.pendingToolLists, + this.pendingCompletions, + id, + ); } } } diff --git a/src/server/createServer.ts b/src/server/createServer.ts index ad1c10677..e637f8d0d 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -39,6 +39,7 @@ import { type ToolResultDelivery, } from "./toolResult.js"; import { parseMcpResponseBudget } from "../config/mcpResponseBudget.js"; +import { parseMcpOutputSchemaProfile } from "../config/mcpOutputSchemaProfile.js"; import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES, parseMcpInputSchemaProfile, @@ -250,6 +251,7 @@ const createMcpServer = ( session: BinarySessionPort | undefined, delivery: ToolResultDelivery, compactPresentation: CompactInputSchemaPresentation | undefined, + omitOutputSchemas: boolean, ): EvidenceMcpServer => new EvidenceMcpServer( { @@ -269,6 +271,7 @@ const createMcpServer = ( : (evidence) => session.recordEvidence(evidence), delivery, compactPresentation, + omitOutputSchemas, ); /** @@ -283,6 +286,7 @@ export const createServer = ( const session = source.kind === "session" ? source.session : undefined; const analysis = source.kind === "session" ? source.session : source.analysis; const environment = snapshotEnvironment(options.environment ?? process.env); + const omitOutputSchemas = outputSchemasOmitted(environment); const providerEnvironment = captureProviderEnvironment( options.providerEnvironment, environment, @@ -330,6 +334,7 @@ export const createServer = ( session, delivery, selectCompactInputSchemaPresentation(environment), + omitOutputSchemas, ); const gdbSessions = new GdbSessionManager({ environment: providerEnvironment, @@ -587,6 +592,16 @@ const selectToolResultDelivery = ( return createToolResultDelivery(configured.value); }; +const outputSchemasOmitted = ( + environment: Readonly, +): boolean => { + const configured = parseMcpOutputSchemaProfile( + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE, + ); + if (!configured.ok) throw configured.error; + return configured.value === "off"; +}; + const selectCompactInputSchemaPresentation = ( environment: Readonly, ): CompactInputSchemaPresentation | undefined => { diff --git a/src/server/registerEnhancedTools.ts b/src/server/registerEnhancedTools.ts index e1e27c127..32cc4b6a1 100644 --- a/src/server/registerEnhancedTools.ts +++ b/src/server/registerEnhancedTools.ts @@ -14,6 +14,7 @@ import type { BinarySessionPort } from "../application/binary/BinarySessionPort. import { EnhancedTools } from "../application/EnhancedTools.js"; import type { ValidatedEnhancedCall } from "../application/EnhancedToolTypes.js"; import { + executeComposedWorkflow, createWorkflowEvidence, workflowSnapshotRecord, recordWorkflowUnknowns, @@ -231,15 +232,20 @@ const executeEnhancedTool = async ( context, parameters, }); - const services = new EnhancedTools({ + const analysis: AnalysisOperationPort = { execute: (operation, operationParameters, executionOptions) => admittedAnalysis.execute(operation, operationParameters, { ...executionOptions, progress, }), - }); + }; const result = await logToolExecution(registration.logger, name, () => - services.executeValidated(validatedCall, context.mcpReq.signal), + executeComposedWorkflow(analysis, (observed) => + new EnhancedTools(observed).executeValidated( + validatedCall, + context.mcpReq.signal, + ), + ), ); await progress.report({ phase: name, @@ -253,8 +259,9 @@ const executeEnhancedTool = async ( target: registration.activeTarget?.(), operation: name, parameters, - result: result.value, + result: result.value.result, upstreamProfile: registration.analysisProfile?.(), + upstreamLimitations: result.value.limitations, }); const recorded = registration.recordEvidence?.(evidence); if (recorded !== undefined && !recorded.ok) @@ -273,7 +280,7 @@ const executeEnhancedTool = async ( } const unknowns = recordWorkflowUnknowns({ name, - result: result.value, + result: result.value.result, evidenceId: evidence.evidence_id, recordUnknown: registration.recordUnknown, }); diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index 5d40afb5b..77e57acf0 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -48,7 +48,6 @@ const nativeEvidenceContinuation = ( /** Immutable encoding and recovery policy shared by one MCP server. */ export class ToolResultDelivery { readonly resultBudgetBytes: number; - constructor(maximumResponseBytes: number) { this.resultBudgetBytes = maximumResponseBytes - 1024; Object.freeze(this); diff --git a/tests/boundary/idaStartupDiagnostics.test.ts b/tests/boundary/idaStartupDiagnostics.test.ts index 8f36543b0..9776caf66 100644 --- a/tests/boundary/idaStartupDiagnostics.test.ts +++ b/tests/boundary/idaStartupDiagnostics.test.ts @@ -1,6 +1,14 @@ import { describe, expect, it } from "vitest"; import { createIdaMcpConnection } from "../../src/ida/IdaMcpConnection.js"; +const launcherEnvironment = (): Readonly> => { + if (process.platform !== "win32") return {}; + const systemRoot = process.env.SYSTEMROOT; + if (systemRoot === undefined || systemRoot.length === 0) + throw new Error("The Windows Node fixture requires its actual SYSTEMROOT."); + return { SYSTEMROOT: systemRoot }; +}; + describe("IDA startup diagnostics", () => { it("preserves the failing launcher's stderr instead of reporting only a closed transport", async () => { const connection = createIdaMcpConnection( @@ -10,7 +18,7 @@ describe("IDA startup diagnostics", () => { "-e", 'process.stderr.write("FATAL: IDA runtime initialization failed\\n", () => process.exit(2));', ], - env: {}, + env: launcherEnvironment(), mode: "headless", timeoutMs: 1000, }, @@ -37,7 +45,7 @@ describe("IDA startup diagnostics", () => { "-e", 'process.stderr.write("x".repeat(20000) + "FATAL-END", () => process.exit(2));', ], - env: {}, + env: launcherEnvironment(), mode: "headless", timeoutMs: 1000, }, @@ -66,7 +74,7 @@ it("drops a leading partial UTF-8 character from the bounded stderr tail", async "-e", 'process.stderr.write("€".repeat(6000), () => process.exit(2));', ], - env: {}, + env: launcherEnvironment(), mode: "headless", timeoutMs: 1000, }, @@ -93,7 +101,7 @@ it("retains an in-budget undecodable leading byte without claiming tail truncati "-e", "process.stderr.write(Buffer.from([0x80, 0x61, 0x62]), () => process.exit(2));", ], - env: {}, + env: launcherEnvironment(), mode: "headless", timeoutMs: 1000, }, diff --git a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts index b59dd80a8..11e8d365a 100644 --- a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts +++ b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts @@ -245,6 +245,334 @@ describe("source-produced parser comparison known answers", () => { }); }); +describe("Notes export known-answer evaluation", () => { + it("grades real analysis and traces while rejecting false values and unsupported citations", async () => { + const root = await createTestTempDirectory("rea-known-answer-notes-"); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("notes-export", targets); + if (claims === undefined) throw new Error("Missing Notes rubric"); + expect(claims.map(({ id }) => id)).toEqual([ + "renderer_api", + "renderer_ipc", + "main_ipc", + "preload", + "feature_modules", + "semantic_boundary", + ]); + const harness = await createApplicationMcpHarness(); + try { + const workflow = await notesWorkflow(harness, targets.notesElectron); + assertNotesAnswerFailures(workflow, claims); + assertNotesUnsupportedClaims(workflow, claims); + await assertNotesSeedBinding(harness, workflow, claims); + } finally { + await harness.close(); + } + }); + + it("binds Notes analysis and derived traces to the requested artifact path and bytes", async () => { + const root = await createTestTempDirectory( + "rea-known-answer-notes-identity-", + ); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("notes-export", targets); + if (claims === undefined) throw new Error("Missing Notes rubric"); + const foreign = join(root, "other-notes.asar"); + await cp(targets.notesElectron, foreign); + const harness = await createApplicationMcpHarness(); + try { + for (const path of [foreign, targets.notesElectron]) { + if (path === targets.notesElectron) { + await appendFile( + join(root, "notes-electron/renderer.js"), + "\n// changed shipped bytes\n", + ); + await createPackage(join(root, "notes-electron"), path); + } + const workflow = await notesWorkflow(harness, path); + const digest = notesEvidence(workflow, "analyze_javascript_application") + .subject?.digest.sha256; + expect(digest === targets.notesElectronSha256).toBe(path === foreign); + const assessment = grade( + workflow.events, + claimManifestByOperation(claims, workflow.evidence), + "analyze_javascript_application", + claims, + ).factualAssessment; + expect(assessment.status).toBe("failed"); + assertClaimFailure( + assessment, + "renderer_api", + "source_subject_mismatch", + ); + for (const id of ["feature_modules", "semantic_boundary"]) + assertClaimFailure(assessment, id, "source_links_mismatch"); + } + } finally { + await harness.close(); + } + }); +}); + +type NotesWorkflow = Awaited>; + +const notesEvidence = ( + workflow: NotesWorkflow, + operation: string, +): Evidence => { + const evidence = workflow.evidence.get(operation); + if (evidence === undefined) + throw new Error(`Missing Notes ${operation} Evidence`); + return evidence; +}; + +const assertClaimFailure = ( + assessment: ReturnType["factualAssessment"], + id: string, + failure: string, +) => { + expect(assessment.claims).toContainEqual( + expect.objectContaining({ + id, + failures: expect.arrayContaining([failure]), + }), + ); +}; + +const assertNotesAnswerFailures = ( + workflow: NotesWorkflow, + claims: readonly FixtureClaimExpectation[], +) => { + const manifest = claimManifestByOperation(claims, workflow.evidence); + const assess = (answer: unknown, events = workflow.events) => + grade(events, answer, "analyze_javascript_application", claims) + .factualAssessment; + expect(assess(manifest).status).toBe("passed"); + const cases = [ + { + id: "renderer_ipc", + patch: { + value: { + channel: "notes:delete", + operation: "send", + source: "preload.js", + }, + }, + failure: "value_mismatch", + }, + { + id: "semantic_boundary", + patch: { + value: { status: "found", coverage: "complete", relevant_unknowns: 0 }, + }, + failure: "value_mismatch", + }, + { + id: "renderer_api", + patch: { + evidence_id: notesEvidence(workflow, "trace_application_feature") + .evidence_id, + }, + failure: "source_operation_mismatch", + }, + ]; + for (const { id, patch, failure } of cases) { + const answer = { + claims: manifest.claims.map((claim) => + claim.id === id ? { ...claim, ...patch } : claim, + ), + }; + assertClaimFailure(assess(answer), id, failure); + } + const missingParent = assess( + manifest, + workflow.events.filter((event) => event.item.id !== "notes-analysis"), + ); + for (const id of ["feature_modules", "semantic_boundary"]) + assertClaimFailure(missingParent, id, "source_links_mismatch"); +}; + +const assertNotesUnsupportedClaims = ( + workflow: NotesWorkflow, + claims: readonly FixtureClaimExpectation[], +) => { + const manifest = claimManifestByOperation(claims, workflow.evidence); + const evidenceId = notesEvidence( + workflow, + "trace_application_feature", + ).evidence_id; + const unsupportedRuntime = { + claims: [ + ...manifest.claims, + { + id: "runtime_export", + value: "A click wrote notes.csv to the actual Downloads directory.", + evidence_id: evidenceId, + authority: "controlled-replay", + confidence: "observed", + }, + ], + }; + expect( + grade( + workflow.events, + unsupportedRuntime, + "analyze_javascript_application", + claims, + ).factualAssessment.manifestFailures, + ).toContain("unknown_claim:runtime_export"); + const prose = evaluateCodexEvents( + [ + ...workflow.events, + { + type: "item.completed", + item: { + type: "agent_message", + text: `Observed Evidence ${evidenceId}: notes:export, preload.js and main.js do not exist. The export certainly executed and wrote notes.csv; runtime behavior is known without further observation.`, + }, + }, + ], + "analyze_javascript_application", + { + requireEvidence: true, + requiredAnswerTermGroups: [["notes:export"], ["preload.js"], ["main.js"]], + fixtureClaims: claims, + }, + ); + expect(prose).toMatchObject({ + finalCitesEvidence: true, + answerTermCoverageMet: true, + factualCorrectness: "failed", + }); + expect(prose.factualAssessment.manifestFailures).toContain( + "invalid_final_claim_manifest", + ); +}; + +const assertNotesSeedBinding = async ( + harness: Awaited>, + workflow: NotesWorkflow, + claims: readonly FixtureClaimExpectation[], +) => { + const manifest = claimManifestByOperation(claims, workflow.evidence); + for (const match of ["contains", undefined] as const) { + const feature = await harness.client.callTool({ + name: "trace_application_feature", + arguments: { + application: notesEvidence(workflow, "analyze_javascript_application"), + seed: { + kind: "channel", + value: "notes:export", + ...(match === undefined ? {} : { match }), + case_sensitive: false, + }, + direction: "both", + }, + }); + expect(feature.isError).not.toBe(true); + const answer = { + claims: manifest.claims.map((claim) => + claim.id === "feature_modules" + ? { ...claim, evidence_id: resultEvidence(feature).evidence_id } + : claim, + ), + }; + const assessment = grade( + [ + ...workflow.events, + toolEvent( + `notes-feature-${match ?? "default"}`, + "trace_application_feature", + feature, + ), + ], + answer, + "analyze_javascript_application", + claims, + ).factualAssessment; + expect(assessment.status).toBe(match === undefined ? "passed" : "failed"); + if (match === "contains") { + assertClaimFailure( + assessment, + "feature_modules", + "source_assertion_mismatch", + ); + expect( + assessment.claims.find(({ id }) => id === "feature_modules")?.failures, + ).not.toContain("source_parameters_mismatch"); + } + } +}; + +const notesWorkflow = async ( + harness: Awaited>, + inputPath: string, +) => { + const analysis = await harness.client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: inputPath }, + }); + expect(analysis.isError).not.toBe(true); + const application = resultEvidence(analysis); + const feature = await harness.client.callTool({ + name: "trace_application_feature", + arguments: { + application, + seed: { + kind: "channel", + value: "notes:export", + match: "exact", + case_sensitive: false, + }, + direction: "both", + }, + }); + expect(feature.isError).not.toBe(true); + const semantic = await harness.client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application, + query: { + seed: { kind: "property", name: "invoke" }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: false, + expected: null, + }, + }, + }); + expect(semantic.isError).not.toBe(true); + return { + events: [ + toolEvent("notes-analysis", "analyze_javascript_application", analysis), + toolEvent("notes-feature", "trace_application_feature", feature), + toolEvent("notes-semantic", "trace_javascript_semantics", semantic), + ], + evidence: new Map([ + ["analyze_javascript_application", application], + ["trace_application_feature", resultEvidence(feature)], + ["trace_javascript_semantics", resultEvidence(semantic)], + ]), + }; +}; + +const claimManifestByOperation = ( + claims: readonly FixtureClaimExpectation[], + evidenceByOperation: ReadonlyMap, +) => ({ + claims: claims.map(({ id, expectedValue, source }) => { + const evidence = evidenceByOperation.get(source.operation); + if (evidence === undefined) + throw new Error(`Missing producer for claim ${id}`); + return { + id, + value: expectedValue, + evidence_id: evidence.evidence_id, + authority: evidence.authority, + confidence: evidence.confidence, + }; + }), +}); + const resultEvidence = (result: { readonly structuredContent?: unknown; }): Evidence => parseEvidence(result.structuredContent); diff --git a/tests/boundary/mcp/compactInputSchemaMcp.test.ts b/tests/boundary/mcp/compactInputSchemaMcp.test.ts index 5ca46c39b..8f8f7ee0c 100644 --- a/tests/boundary/mcp/compactInputSchemaMcp.test.ts +++ b/tests/boundary/mcp/compactInputSchemaMcp.test.ts @@ -4,6 +4,11 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import { describe, expect, it } from "vitest"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; +import { + schemaArrays, + schemaChildren, + schemaMaps, +} from "../../../src/contracts/inputSchemaPresentation.js"; import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES, parseMcpInputSchemaProfile, @@ -23,6 +28,45 @@ interface AdvertisedTool { const serializedBytes = (value: unknown): number => Buffer.byteLength(JSON.stringify(value), "utf8"); +const schemaDepth = (value: unknown, depth = 0): number => { + if (Array.isArray(value)) + return Math.max( + depth, + ...value.map((child) => schemaDepth(child, depth + 1)), + ); + if (typeof value !== "object" || value === null) return depth; + return Math.max( + depth, + ...Object.values(value).map((child) => schemaDepth(child, depth + 1)), + ); +}; + +const unsupportedDependencyKeywords = ( + schema: unknown, + path = "$", +): readonly string[] => { + if (Array.isArray(schema)) + return schema.flatMap((child, index) => + unsupportedDependencyKeywords(child, `${path}[${index}]`), + ); + if (typeof schema !== "object" || schema === null) return []; + return Object.entries(schema).flatMap(([key, value]) => { + const childPath = `${path}.${key}`; + if (key === "dependencies" || key === "dependentRequired") + return [childPath]; + if (["const", "enum", "default", "examples"].includes(key)) return []; + if (schemaMaps.has(key) && typeof value === "object" && value !== null) + return Object.entries(value).flatMap(([name, child]) => + unsupportedDependencyKeywords(child, `${childPath}.${name}`), + ); + if (schemaArrays.has(key) && Array.isArray(value)) + return unsupportedDependencyKeywords(value, childPath); + if (schemaChildren.has(key)) + return unsupportedDependencyKeywords(value, childPath); + return []; + }); +}; + const listToolsThroughTransport = async ( server: EvidenceMcpServer, ): Promise => { @@ -77,6 +121,12 @@ describe("MCP compact input schema profile", () => { serializedBytes(inputSchema) > COMPACT_INPUT_SCHEMA_BUDGET_BYTES, ); expect(oversized.map(({ name }) => name)).toEqual([]); + const tooDeep = tools.filter( + ({ inputSchema }) => schemaDepth(inputSchema) > 10, + ); + expect( + tooDeep.map(({ name, inputSchema }) => [name, schemaDepth(inputSchema)]), + ).toEqual([]); }); it("keeps output schemas and tool descriptions identical to the default profile", async () => { @@ -120,6 +170,31 @@ describe("MCP compact input schema profile", () => { } }); + it("omits Gemini-unsupported dependency keywords from both catalog profiles", async () => { + const full = fullServer(); + const compact = compactServer(); + registerCatalog(full); + registerCatalog(compact); + const [fullTools, compactTools] = await Promise.all([ + listToolsThroughTransport(full), + listToolsThroughTransport(compact), + ]); + for (const tools of [fullTools, compactTools]) + expect( + tools.flatMap(({ name, inputSchema }) => + unsupportedDependencyKeywords(inputSchema).map( + (path) => `${name}${path.slice(1)}`, + ), + ), + ).toEqual([]); + const sqlite = fullTools.find( + ({ name }) => name === "inspect_sqlite_database", + ); + expect(sqlite?.inputSchema.description).toContain( + "When row_limit is provided, also provide table.", + ); + }); + it("retains parameter types and avoids client-side reference expansion", async () => { const server = compactServer(); registerCatalog(server); @@ -136,13 +211,7 @@ describe("MCP compact input schema profile", () => { ({ name }) => name === "compare_managed_members", ); expect(comparison?.inputSchema.properties).toMatchObject({ - left: { - properties: { - normalized_result: { - type: ["null", "boolean", "object", "array", "number", "string"], - }, - }, - }, + left: { type: "object" }, }); const ledger = tools.find( ({ name }) => name === "build_reconstruction_obligation_ledger", diff --git a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts index a08d35a34..c2c0c52e9 100644 --- a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts +++ b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts @@ -82,60 +82,62 @@ it("discovers declarations without counting duplicates or compiler bookkeeping", }); }); +const completeOverviewPort: AnalysisOperationPort = { + execute: (name) => { + switch (name) { + case "list_segments": + return Promise.resolve( + ok([ + { + name: "__TEXT", + start: "0x1000", + end: "0x1800", + image_base: "0x1000", + readable: null, + writable: null, + executable: null, + }, + { + name: "__DATA", + start: "0x1800", + end: "0x2000", + image_base: "0x1000", + readable: null, + writable: null, + executable: null, + }, + ]), + ); + case "current_document": + return Promise.resolve(ok("fixture")); + case "list_documents": + return Promise.resolve(ok(["New Document", "fixture"])); + case "list_strings": + return Promise.resolve( + ok( + Array.from({ length: 700 }, (_, index) => ({ + address: `0x${(0x30 + index).toString(16)}`, + value: `string-${index}`, + })), + ), + ); + case "list_procedures": { + return Promise.resolve( + ok([ + { address: "0x1", value: "first" }, + { address: "0x2", value: "last" }, + ]), + ); + } + default: + throw new Error(`Unexpected analysis operation: ${name}`); + } + }, +}; + describe("enhanced MCP tools", () => { it("returns the complete overview inline with exhaustive totals", async () => { - const client = await connect({ - execute: (name) => { - switch (name) { - case "list_segments": - return Promise.resolve( - ok([ - { - name: "__TEXT", - start: "0x1000", - end: "0x1800", - image_base: "0x1000", - readable: null, - writable: null, - executable: null, - }, - { - name: "__DATA", - start: "0x1800", - end: "0x2000", - image_base: "0x1000", - readable: null, - writable: null, - executable: null, - }, - ]), - ); - case "current_document": - return Promise.resolve(ok("fixture")); - case "list_documents": - return Promise.resolve(ok(["New Document", "fixture"])); - case "list_strings": - return Promise.resolve( - ok( - Array.from({ length: 700 }, (_, index) => ({ - address: `0x${(0x30 + index).toString(16)}`, - value: `string-${index}`, - })), - ), - ); - case "list_procedures": { - return Promise.resolve( - ok([ - { address: "0x1", value: "first" }, - { address: "0x2", value: "last" }, - ]), - ); - } - default: - throw new Error(`Unexpected analysis operation: ${name}`); - } - }, - }); + const client = await connect(completeOverviewPort); const result = jsonResult( await client.callTool({ name: "binary_overview", @@ -167,6 +169,48 @@ describe("enhanced MCP tools", () => { }); }); + it("returns binary counts while omitting unavailable segment coverage", async () => { + const client = await connect({ + execute: (name) => { + if (name === "current_document") + return Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + "fixture", + name, + "the provider has one bound document", + ), + ), + ); + if (name === "list_documents") + return Promise.resolve(ok(["/tmp/fixture.bin"])); + if (name === "list_segments") + return Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + "fixture", + name, + "segment enumeration is unsupported", + ), + ), + ); + if (name === "list_procedures" || name === "list_strings") + return Promise.resolve(ok([])); + throw new Error(`Unexpected analysis operation: ${name}`); + }, + }); + + expect( + jsonResult( + await client.callTool({ name: "binary_overview", arguments: {} }), + ), + ).toEqual({ + document: "/tmp/fixture.bin", + procedure_count: 0, + string_count: 0, + }); + }); + it("rejects paginated provider output", async () => { const client = await connect({ execute: () => diff --git a/tests/boundary/mcp/functionCallDirectionsMcp.test.ts b/tests/boundary/mcp/functionCallDirectionsMcp.test.ts new file mode 100644 index 000000000..b4034d036 --- /dev/null +++ b/tests/boundary/mcp/functionCallDirectionsMcp.test.ts @@ -0,0 +1,138 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { expect, it } from "vitest"; + +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; +import type { RelationshipCoverage } from "../../../src/domain/relationshipCoverage.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +const observe = (relationshipCoverage: RelationshipCoverage) => { + const template = FUNCTION_COMPARISON_EXAMPLE.left; + if (template.subject === null) + throw new Error("Expected artifact-bound fixture"); + const dossier = functionDossierSchema.parse(template.normalized_result); + return createEvidence( + { + path: template.subject.local_path, + sha256: template.subject.digest.sha256, + format: template.subject.format, + }, + template.provider, + { + operation: "analyze_function", + parameters: template.parameters, + result: { ...dossier, callees: [], unresolved_calls: [] }, + confidence: template.confidence, + authority: template.authority, + relationshipCoverage, + }, + ); +}; + +it("preserves directional gaps through advertised schemas and public path reconstruction", async () => { + const session = createTestBinarySession(() => { + throw new Error("Offline call paths must not launch a provider"); + }); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "function-call-directions", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const advertised = (await client.listTools()).tools.find( + ({ name }) => name === "build_call_path", + ); + if (advertised === undefined) + throw new Error("Missing build_call_path contract"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + expect(ajv.validateSchema(advertised.inputSchema)).toBe(true); + const validate = ajv.compile(advertised.inputSchema); + for (const { directions, status, exhaustive } of [ + { + directions: ["callers"] as ["callers"], + status: "not_found", + exhaustive: true, + }, + { + directions: ["callees"] as ["callees"], + status: "unknown", + exhaustive: false, + }, + { + directions: ["callers", "callees"] as ["callers", "callees"], + status: "unknown", + exhaustive: false, + }, + ]) { + const limitation = + "An observed call reference is absent from the inventory."; + const coverage: RelationshipCoverage = { + status: "partial", + limitations: [limitation], + limited_directions: directions, + }; + const observation = observe(coverage); + expect(session.recordEvidence(observation).ok).toBe(true); + const input = { + functions: [observation], + start: { address: "0x1000" }, + goal: { address: "0x9999" }, + }; + expect(validate(input), JSON.stringify(validate.errors)).toBe(true); + for (const limited_directions of [ + [], + ["callers", "callers"], + ["callees", "callees"], + ["callers", "callees", "callers"], + ["incoming"], + ]) + expect( + validate({ + ...input, + functions: [ + { + ...observation, + relationship_coverage: { + ...coverage, + limited_directions, + }, + }, + ], + }), + ).toBe(false); + const result = await client.callTool({ + name: "build_call_path", + arguments: input, + }); + if (result.isError) throw new Error(JSON.stringify(result)); + expect(result.structuredContent).toMatchObject({ + normalized_result: { + status, + search_scope: { exhaustive }, + limitations: expect.arrayContaining([limitation]), + }, + }); + const comparison = await client.callTool({ + name: "compare_functions", + arguments: { left: observation, right: observation }, + }); + if (comparison.isError) throw new Error(JSON.stringify(comparison)); + expect(comparison.structuredContent).toMatchObject({ + normalized_result: { + dimensions: expect.arrayContaining([ + expect.objectContaining({ dimension: "calls", status: "unknown" }), + ]), + }, + }); + // Each scenario owns its Evidence and residual-unknown ledger. + const closed = await session.close(); + if (!closed.ok) throw closed.error; + } + } finally { + await Promise.allSettled([session.close(), client.close(), server.close()]); + } +}); diff --git a/tests/boundary/mcp/functionComparisonMcp.test.ts b/tests/boundary/mcp/functionComparisonMcp.test.ts index 97f07dc1e..89b3f14a6 100644 --- a/tests/boundary/mcp/functionComparisonMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonMcp.test.ts @@ -15,128 +15,157 @@ import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComp import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; +import { reconstructionVerificationResultSchema } from "../../../src/domain/reconstructionVerificationSchemas.js"; -describe("function comparison reconstruction MCP integration", () => { - it.each([ - { comparisonStatus: "unchanged", verificationStatus: "pass" }, - { comparisonStatus: "changed", verificationStatus: "fail" }, - ] as const)( - "verifies public $comparisonStatus function comparison as $verificationStatus", - async ({ comparisonStatus, verificationStatus }) => { - let providerLaunches = 0; - const session = createTestBinarySession(() => { - providerLaunches += 1; - throw new Error("Function verification must not launch a provider"); - }); - const left = FUNCTION_COMPARISON_EXAMPLE.left; - const right = - comparisonStatus === "changed" - ? FUNCTION_COMPARISON_EXAMPLE.right - : createEvidence( - { - path: "/tmp/function-unchanged", - sha256: "2".repeat(64), - format: "mach-o", - }, - left.provider, - { - operation: left.operation, - parameters: left.parameters, - result: left.normalized_result, - confidence: left.confidence, - authority: left.authority, - ...(left.relationship_coverage === null - ? {} - : { relationshipCoverage: left.relationship_coverage }), - }, - ); - const server = createServer({ kind: "session", session }); - const client = new Client({ - name: "function-verification-test", - version: "1", - }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const compared = await client.callTool({ - name: "compare_functions", - arguments: { left, right }, - }); - expect(compared.isError).not.toBe(true); - const comparison = parseEvidence(compared.structuredContent); - expect(comparison.normalized_result).toMatchObject({ - status: comparisonStatus, - }); - expect(comparison.parameters).toEqual({ - left_evidence_id: left.evidence_id, - right_evidence_id: right.evidence_id, - }); - expect(comparison.evidence_links).toEqual( - expect.arrayContaining([left.evidence_id, right.evidence_id]), - ); - - const verified = await client.callTool({ - name: "verify_reconstruction", - arguments: { - specification: { - name: "Function pseudocode comparison", - claims: [ - { - kind: "structural-function", - claim_id: "pseudocode", - title: "Pseudocode agrees", - comparison_evidence_id: comparison.evidence_id, - dimension: "pseudocode", - }, - ], +it.each([ + { comparisonStatus: "unchanged", verificationStatus: "pass" }, + { comparisonStatus: "changed", verificationStatus: "fail" }, +] as const)( + "verifies public $comparisonStatus function comparison as $verificationStatus", + async ({ comparisonStatus, verificationStatus }) => { + let providerLaunches = 0; + const session = createTestBinarySession(() => { + providerLaunches += 1; + throw new Error("Function verification must not launch a provider"); + }); + const left = FUNCTION_COMPARISON_EXAMPLE.left; + const right = + comparisonStatus === "changed" + ? FUNCTION_COMPARISON_EXAMPLE.right + : createEvidence( + { + path: "/tmp/function-unchanged", + sha256: "2".repeat(64), + format: "mach-o", }, - }, - }); - expect( - verified.isError, - JSON.stringify(verified.structuredContent), - ).not.toBe(true); - const verification = parseEvidence(verified.structuredContent); - expect(verification.normalized_result).toMatchObject({ - status: verificationStatus, - summary: { - total: 1, - passed: verificationStatus === "pass" ? 1 : 0, - failed: verificationStatus === "fail" ? 1 : 0, - unknown: 0, - }, - claims: { - items: [ - expect.objectContaining({ - status: verificationStatus, - observed_status: comparisonStatus, + left.provider, + { + operation: left.operation, + parameters: left.parameters, + result: left.normalized_result, + confidence: left.confidence, + authority: left.authority, + ...(left.relationship_coverage === null + ? {} + : { relationshipCoverage: left.relationship_coverage }), + }, + ); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "function-claims", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const compared = await client.callTool({ + name: "compare_functions", + arguments: { left, right }, + }); + expect(compared.isError).not.toBe(true); + const comparison = parseEvidence(compared.structuredContent); + expect(comparison.limitations).not.toContainEqual( + expect.stringContaining( + "without address or generated-name normalization", + ), + ); + expect(comparison.normalized_result).toMatchObject({ + status: comparisonStatus, + dimensions: expect.arrayContaining( + ["pseudocode", "assembly"].map((dimension) => + expect.objectContaining({ + dimension, + limitations: expect.arrayContaining([ + expect.stringContaining( + "without address or generated-name normalization", + ), + ]), + }), + ), + ), + }); + expect(comparison.parameters).toEqual({ + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + }); + expect(comparison.evidence_links).toEqual( + expect.arrayContaining([left.evidence_id, right.evidence_id]), + ); + + const verified = await client.callTool({ + name: "verify_reconstruction", + arguments: { + specification: { + name: "Function pseudocode comparison", + claims: ["pseudocode", "assembly", "cfg", "overall"].map( + (dimension) => ({ + kind: "structural-function", + claim_id: dimension, + title: `${dimension} agrees`, comparison_evidence_id: comparison.evidence_id, - left_evidence_ids: [left.evidence_id], - right_evidence_ids: [right.evidence_id], + dimension, }), - ], - }, - }); - expect( - session - .exportEvidenceBundle() - .records.find( - ({ evidence_id: id }) => id === comparison.evidence_id, ), - ).toEqual(comparison); - } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - expect(providerLaunches).toBe(0); + }, + }, + }); + expect( + verified.isError, + JSON.stringify(verified.structuredContent), + ).not.toBe(true); + const verification = parseEvidence(verified.structuredContent); + expect(verification.normalized_result).toMatchObject({ + status: verificationStatus, + summary: { + total: 4, + passed: verificationStatus === "pass" ? 4 : 2, + failed: verificationStatus === "fail" ? 2 : 0, + unknown: 0, + }, + claims: { + items: expect.arrayContaining([ + expect.objectContaining({ + dimension: "pseudocode", + status: verificationStatus, + observed_status: comparisonStatus, + comparison_evidence_id: comparison.evidence_id, + left_evidence_ids: [left.evidence_id], + right_evidence_ids: [right.evidence_id], + limitations: expect.arrayContaining([ + expect.stringContaining( + "without address or generated-name normalization", + ), + ]), + }), + ]), + }, + }); + const claims = reconstructionVerificationResultSchema.parse( + verification.normalized_result, + ).claims.items; + for (const claim of claims) { + const caveats = claim.limitations.join(" "); + if (claim.dimension === "cfg") + expect(caveats).not.toMatch(/generated-name normalization/u); + else expect(caveats).toMatch(/generated-name normalization/u); + if (claim.dimension === "assembly" || claim.dimension === "overall") + expect(caveats).toMatch(/Assembly is opaque/u); + else expect(caveats).not.toMatch(/Assembly is opaque/u); } - }, - ); -}); + expect( + session + .exportEvidenceBundle() + .records.find(({ evidence_id: id }) => id === comparison.evidence_id), + ).toEqual(comparison); + } finally { + await Promise.allSettled([ + client.close(), + server.close(), + session.close(), + ]); + expect(providerLaunches).toBe(0); + } + }, +); describe("function comparison MCP integration", () => { it("records linked comparison Evidence and its residual unknown", async () => { diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index e9b68b36f..acfb2bb0e 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -10,12 +10,35 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { compareInventory, runProductionMcpDoctor, + runProductionMcpDoctorCli, } from "../../../src/mcpDoctor.js"; import { CATALOG_IDENTITY } from "../../../src/catalogIdentity.js"; afterEach(() => vi.useRealTimers()); describe("production MCP doctor", () => { + it("preserves the full-output envelope for startup failures", async () => { + const result = await runProductionMcpDoctorCli( + ["--json", "--full-output"], + { + dispatcherPath: resolve("tests/fixtures/mcpDoctorExit.mjs"), + packageRoot: process.cwd(), + }, + ); + expect(result.exitCode).toBe(1); + expect(JSON.parse(result.output)).toMatchObject({ + ok: false, + error: { + healthy: false, + checks: [{ name: "transport", ok: false }], + }, + meta: { + command: "rea mcp doctor", + duration: expect.stringMatching(/^\d+ms$/u), + }, + }); + }); + it("connects to the production stdio child and verifies the canonical catalog", async () => { const result = await runProductionMcpDoctor({ command: process.execPath, @@ -44,7 +67,9 @@ describe("production MCP doctor", () => { CATALOG_IDENTITY.counts.mcp_tools, ); }, 30_000); +}); +describe("production MCP doctor probe validation", () => { it("rejects a schema-invalid probe while retaining the discovered inventory", async () => { const root = await createTestTempDirectory("rea-mcp-doctor-reply-"); const snapshotPath = join(root, "reply.json"); @@ -127,7 +152,9 @@ describe("production MCP doctor", () => { duplicates: ["c"], }); }); +}); +describe("production MCP doctor child lifecycle", () => { it("captures startup exit diagnostics", async () => { const result = await runProductionMcpDoctor({ command: process.execPath, diff --git a/tests/boundary/mcp/outputSchemaProfile.test.ts b/tests/boundary/mcp/outputSchemaProfile.test.ts new file mode 100644 index 000000000..a77dddf8c --- /dev/null +++ b/tests/boundary/mcp/outputSchemaProfile.test.ts @@ -0,0 +1,115 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { z } from "zod"; +import { expect, it } from "vitest"; + +import { + TOOL_CONTRACTS, + toolContract, +} from "../../../src/contracts/toolContracts.js"; +import { EvidenceMcpServer } from "../../../src/server/EvidenceMcpServer.js"; +import { toolRegistrationOptions } from "../../../src/server/toolRegistrationOptions.js"; +import { createToolResultDelivery } from "../../../src/server/toolResult.js"; +import type { ToolContract } from "../../../src/contracts/toolContractTypes.js"; + +const createServer = (omitOutputSchemas: boolean) => + new EvidenceMcpServer( + { name: "output-schema-fixture", version: "1" }, + { capabilities: {} }, + undefined, + createToolResultDelivery(10 * 1024 * 1024), + undefined, + omitOutputSchemas, + ); + +it("omits only output schema advertisements and keeps canonical MCP output validation", async () => { + const server = createServer(true); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "schema-profile-test", version: "1" }); + const contract = { + ...toolContract("comment"), + name: "output_schema_fixture", + inputSchema: z.strictObject({}), + outputSchema: z.strictObject({ value: z.string() }), + } satisfies ToolContract; + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async () => ({ + content: [{ type: "text" as const, text: "fixture" }], + structuredContent: { value: "complete" }, + }), + ); + const invalidContract = { + ...contract, + name: "invalid_output_schema_fixture", + } satisfies ToolContract; + server.registerTool( + invalidContract.name, + toolRegistrationOptions(invalidContract), + async () => ({ + content: [{ type: "text" as const, text: "invalid fixture" }], + structuredContent: { value: 42 }, + }), + ); + + await server.connect(serverTransport); + await client.connect(clientTransport); + try { + const tools = (await client.listTools()).tools; + expect(tools).toHaveLength(2); + expect(tools.every((tool) => tool.outputSchema === undefined)).toBe(true); + const success = await client.callTool({ + name: contract.name, + arguments: {}, + }); + expect(success.isError).not.toBe(true); + expect(success.structuredContent).toEqual({ value: "complete" }); + const invalid = await client.callTool({ + name: invalidContract.name, + arguments: {}, + }); + expect(invalid.isError).toBe(true); + expect(invalid.content).toEqual([ + expect.objectContaining({ + type: "text", + text: expect.stringContaining("Output validation error"), + }), + ]); + } finally { + await client.close(); + await server.close(); + } +}); + +it("keeps a full catalog below a measured one-megabyte frame when output schemas are off", async () => { + const measure = async (omitOutputSchemas: boolean): Promise => { + const server = createServer(omitOutputSchemas); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "catalog-size-test", version: "1" }); + for (const contract of TOOL_CONTRACTS) + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async () => ({ + content: [{ type: "text" as const, text: "unused" }], + }), + ); + await server.connect(serverTransport); + await client.connect(clientTransport); + try { + const listed = await client.listTools(); + return Buffer.byteLength(JSON.stringify(listed), "utf8"); + } finally { + await client.close(); + await server.close(); + } + }; + + const fullBytes = await measure(false); + const compactBytes = await measure(true); + expect(fullBytes).toBeGreaterThan(1024 * 1024); + expect(compactBytes).toBeLessThan(1024 * 1024); + expect(compactBytes).toBeLessThan(fullBytes); +}); diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index bd6f64471..934d0d18f 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -139,6 +139,20 @@ describe("server and catalog identity", () => { ); }); + it("changes catalog identity when advertised input examples change", () => { + const contract = TOOL_CONTRACTS.find((item) => item.examples.length > 0); + if (contract === undefined) throw new Error("expected an input example"); + const original = createCatalogDigests(); + const withoutExamples = createCatalogDigests( + TOOL_CONTRACTS.map((item) => + item === contract ? { ...item, examples: [] } : item, + ), + ); + expect(withoutExamples.tools_sha256).not.toBe(original.tools_sha256); + expect(withoutExamples.combined_sha256).not.toBe(original.combined_sha256); + expect(withoutExamples.prompts_sha256).toBe(original.prompts_sha256); + }); + it("reports unknown without a live comparison and distinguishes aligned from stale", () => { const unknown = createServerIdentity({ startedAt: "2026-07-13T00:00:00.000Z", From 5b7fe947108dcc0cc4a7e98f0c6d62080b5e7f8d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:32:15 +0800 Subject: [PATCH 11/26] fix(analysis): preserve extraction gaps and database failure reasons Derive firmware completeness from examined coverage, retain linked continuation evidence, and preserve reported extraction limits. Keep Android package attribution and Go image fixtures coherent, and classify SQLite busy failures without discarding descriptor cleanup facts. Reviewed fixes include firmware, ADB, SQLite, and recorded-transport bug reports and their focused PRs. Validation: firmware report source tests passed 21 cases; prior focused artifact/provider boundaries passed. Real Android devices and external firmware tools were not rerun. --- docs/firmware-analysis.md | 23 + .../AppBundleOpenStepDepth.test.ts | 60 +++ src/domain/firmware/firmwareAnalysis.ts | 31 ++ src/domain/harmony/harmonyApplication.ts | 28 +- .../mobileApplicationGraphProjection.ts | 27 + src/domain/recordedTransportUrl.test.ts | 11 + src/domain/relationshipCoverage.ts | 23 +- src/firmware/FirmwareProvider.ts | 2 + src/firmware/FirmwarePublication.ts | 313 +++++++++++ src/firmware/FirmwareReports.test.ts | 500 +++++++++++++++++- src/firmware/FirmwareReports.ts | 179 ++++++- src/signals/SigrokRecordingProvider.ts | 1 + src/sqlite/SqliteDatabaseFailures.ts | 25 + tests/acceptance/goBinaryCli.test.ts | 63 +++ tests/boundary/android/JadxAnalysis.test.ts | 108 +++- .../firmware/firmwareIntegration.test.ts | 303 +++++++++++ .../signals/recordingProvider.test.ts | 17 + .../sqlite/sqliteDatabaseBusy.test.ts | 159 ++++++ .../sqliteDatabaseDescriptorCleanup.test.ts | 19 +- .../sqlite/sqliteDatabaseProvider.test.ts | 47 +- tests/fixtures/firmware/producer.mjs | 289 +++++++++- tests/fixtures/go/image.ts | 51 ++ 22 files changed, 2179 insertions(+), 100 deletions(-) create mode 100644 src/application/AppBundleOpenStepDepth.test.ts create mode 100644 src/domain/recordedTransportUrl.test.ts create mode 100644 tests/boundary/sqlite/sqliteDatabaseBusy.test.ts diff --git a/docs/firmware-analysis.md b/docs/firmware-analysis.md index 1f4ff14d6..beab16400 100644 --- a/docs/firmware-analysis.md +++ b/docs/firmware-analysis.md @@ -103,6 +103,23 @@ build's source revision, so that field remains unknown. engine finished its configured extraction without those reported conditions; default skip rules and format coverage still apply. It does not establish complete firmware reconstruction. +- `extraction_gaps` makes those conditions recoverable: a row identifies an + unextracted region or a failed task/stage. A chunk retains its identity and + byte range, plus its original-file range when it belongs to the selected input. + A multi-file stage retains its reported ID and handler with unknown byte ranges; + a standalone task failure keeps unavailable stage identity null. Rows retain exact dependency names from + the producer's missing-dependency reports; an accepted report with no names + remains a failure with unknown dependencies; no executable identity or specific installation step is inferred. Each row's `continuation_id` + references one instruction in `extraction_continuations`, so many regions + share recovery prose without losing any region identity. Instructions use + already configured capabilities where one exists. Every extraction rerun + requires a fresh absent `output_directory`, preserving the prior evidence. + Nested unknown chunks use `inspect_firmware_regions` on the actual published + child rather than repeating the same extraction with unchanged handlers. + Inspection requires separately configured Binwalk (`REA_BINWALK_COMMAND`); + without it, the inspection step is unavailable and the region remains unexamined. A listed gap does not + mean the target is corrupt or that a retained region is empty; REA installs + nothing. - The destination must be absent, absolute, with an existing parent. Publication uses exclusive, symlink-resistant writes and durable digest readback. Existing destinations are never overwritten. This is not an atomic directory rename: @@ -167,6 +184,12 @@ records gzip offset/length and extracted file hashes. Real verification checks CLI/MCP, full-file and selected-range extraction, unknown chunks, depth limits, content parity and unchanged source bytes. No binary fixture is committed. +The optional missing-extractor lane asserts dependency names and the owning +handler in `extraction_gaps`. Those assertions must be run against the pinned +Unblob release to establish real-provider coverage; synthetic producer fixtures +alone do not verify Unblob's report nesting or stage identities. The gap ownership +claims remain unverified against real Unblob until that lane runs. + Additional lanes are explicit, so the base lane stays small: ```sh diff --git a/src/application/AppBundleOpenStepDepth.test.ts b/src/application/AppBundleOpenStepDepth.test.ts new file mode 100644 index 000000000..d4b86bd91 --- /dev/null +++ b/src/application/AppBundleOpenStepDepth.test.ts @@ -0,0 +1,60 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { resolveAppBundleExecutable } from "./AppBundleExecutable.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; + +describe("OpenStep Info.plist nesting during app target resolution", () => { + it.each([127, 128, 3000])( + "applies the shared bound to %s nested payload arrays", + async (depth) => { + const root = await createTestTempDirectory("rea-app-openstep-depth-"); + const app = join(root, "Example.app"); + const programs = join(app, "Contents", "MacOS"); + await mkdir(programs, { recursive: true }); + await writeFile(join(programs, "App"), "inert program-path fixture"); + await writeFile( + join(app, "Contents", "Info.plist"), + `{ CFBundleExecutable = App; payload = ${"(".repeat(depth)}x${")".repeat(depth)}; }`, + ); + + const result = await resolveAppBundleExecutable(app); + + if (depth === 127) { + expect(result).toMatchObject({ + ok: true, + value: { executable: join(programs, "App") }, + }); + } else { + expect(result).toMatchObject({ + ok: false, + error: { + path: join(app, "Contents", "Info.plist"), + cause: expect.objectContaining({ + message: "Plist exceeds 128 nesting levels", + }), + }, + }); + } + }, + ); + + it("resolves the first Info.plist root without decoding deeply nested trailing text", async () => { + const root = await createTestTempDirectory("rea-app-openstep-trailing-"); + const app = join(root, "Example.app"); + const programs = join(app, "Contents", "MacOS"); + await mkdir(programs, { recursive: true }); + await writeFile(join(programs, "App"), "inert program-path fixture"); + await writeFile( + join(app, "Contents", "Info.plist"), + `{ CFBundleExecutable = App; }${"(".repeat(3000)}x${")".repeat(3000)}`, + ); + + expect(await resolveAppBundleExecutable(app)).toMatchObject({ + ok: true, + value: { executable: join(programs, "App") }, + }); + }); +}); diff --git a/src/domain/firmware/firmwareAnalysis.ts b/src/domain/firmware/firmwareAnalysis.ts index 0ebbdc2e9..852f380cd 100644 --- a/src/domain/firmware/firmwareAnalysis.ts +++ b/src/domain/firmware/firmwareAnalysis.ts @@ -93,6 +93,32 @@ const chunk = z.strictObject({ handler: z.string().nullable(), encrypted: z.boolean().nullable(), root_file_range: range.nullable(), + /** The producer reported extraction problems for this chunk. */ + extraction_failed: z.boolean(), +}); +/** One unextracted region or failing task/stage, with a supported continuation + * using already configured capabilities when one exists. */ +export const extractionGap = z.strictObject({ + kind: z.enum([ + "missing-extractor-dependency", + "unknown-chunk-handler", + "encrypted-chunk", + "depth-limited", + "extractor-failure", + "unpublished-entry", + ]), + /** Logical task path from the producer report; `$output/…` for unpublished entries. */ + input_path: z.string(), + relative_path: z.string().nullable(), + handler: z.string().nullable(), + /** Reported chunk or multi-file stage identity; it does not imply a byte range. */ + provider_id: z.string().nullable(), + range: range.nullable(), + root_file_range: range.nullable(), + /** Extractor executables named by the producer; empty when it named none. */ + dependencies: z.array(z.string()), + /** Lookup key in the result's extraction_continuations table. */ + continuation_id: z.string(), }); /** Completed firmware result retained when private-runtime cleanup cannot be confirmed. */ @@ -158,6 +184,11 @@ export const firmwareResultSchemas = { ), depth_limited_paths: z.array(z.string()), diagnostics: z.array(jsonValueSchema), + extraction_gaps: z.array(extractionGap), + /** Each distinct recovery instruction is serialized once, shared by gap ID. */ + extraction_continuations: z.array( + z.strictObject({ id: z.string(), instruction: z.string() }), + ), published_bytes: bytes, }), } as const; diff --git a/src/domain/harmony/harmonyApplication.ts b/src/domain/harmony/harmonyApplication.ts index fff8429cb..219aed714 100644 --- a/src/domain/harmony/harmonyApplication.ts +++ b/src/domain/harmony/harmonyApplication.ts @@ -18,7 +18,8 @@ import { projectionCoverage, projectionDigest, projectionEvidenceIdSchema, - projectionPathSchema, + unavailableComponentSchema, + unavailableComponents, } from "../mobileApplicationGraphProjection.js"; const evidenceIdSchema = projectionEvidenceIdSchema; @@ -46,16 +47,7 @@ export const harmonyApplicationProjectionResultSchema = z.strictObject({ javascript: z.array(componentSchema), signing: z.array(componentSchema), }), - unavailable_components: z.array( - z.strictObject({ - path: projectionPathSchema, - artifact_id: z.null(), - sha256: z.null(), - format: z.string().min(1), - encrypted: z.boolean(), - limitations: z.array(z.string()), - }), - ), + unavailable_components: z.array(unavailableComponentSchema), app_pack_children: z.array(componentSchema), runtime_families: z.array( z.enum(["ark", "native", "javascript", "react-native", "flutter", "unity"]), @@ -93,19 +85,7 @@ export const projectHarmonyApplication = ( ); const rootFormat = harmonyRootFormat(inventory.manifest.root_format); const all = projectedComponents(inventory); - const unavailable = inventory.occurrences - .filter( - (occurrence) => - occurrence.logical_path !== "." && occurrence.artifact_id === null, - ) - .map((occurrence) => ({ - path: occurrence.logical_path, - artifact_id: null, - sha256: null, - format: occurrence.artifact_format, - encrypted: occurrence.encrypted, - limitations: occurrence.limitations, - })); + const unavailable = unavailableComponents(inventory); const classified = classify(all); const bridgeProjection = bridgeCandidates( classified.bytecode, diff --git a/src/domain/mobileApplicationGraphProjection.ts b/src/domain/mobileApplicationGraphProjection.ts index 13bbe9545..ba581a9b1 100644 --- a/src/domain/mobileApplicationGraphProjection.ts +++ b/src/domain/mobileApplicationGraphProjection.ts @@ -32,6 +32,16 @@ export const projectedComponentSchema = z.strictObject({ format: z.string().min(1), }); +/** Inventoried paths whose content identity is unavailable, retained as unknown. */ +export const unavailableComponentSchema = z.strictObject({ + path: projectionPathSchema, + artifact_id: z.null(), + sha256: z.null(), + format: z.string().min(1), + encrypted: z.boolean(), + limitations: z.array(z.string()), +}); + export type ProjectedComponent = z.infer; /** Path-pair hypotheses connecting managed content to a native library. */ @@ -90,6 +100,23 @@ export const projectedComponents = ( }); }; +export const unavailableComponents = ( + inventory: ParsedInventorySet["inventory"], +) => + inventory.occurrences + .filter( + (occurrence) => + occurrence.logical_path !== "." && occurrence.artifact_id === null, + ) + .map((occurrence) => ({ + path: occurrence.logical_path, + artifact_id: null, + sha256: null, + format: occurrence.artifact_format, + encrypted: occurrence.encrypted, + limitations: occurrence.limitations, + })); + export { bridgeCandidateCoverageSchema, projectCartesianCandidates }; /** Unicode code point ordering so projection arrays sort deterministically. */ diff --git a/src/domain/recordedTransportUrl.test.ts b/src/domain/recordedTransportUrl.test.ts new file mode 100644 index 000000000..9c7b78ea0 --- /dev/null +++ b/src/domain/recordedTransportUrl.test.ts @@ -0,0 +1,11 @@ +import { expect, it } from "vitest"; + +import { parseRecordedTransportUrl } from "./recordedTransportUrl.js"; + +it("recognizes an empty userinfo marker at the authority start", () => { + expect(parseRecordedTransportUrl("https://@host")).toMatchObject({ + origin: "https://host", + path: "/", + credentials: { start: 8, end: 9 }, + }); +}); diff --git a/src/domain/relationshipCoverage.ts b/src/domain/relationshipCoverage.ts index 1de973669..4dd4e0c40 100644 --- a/src/domain/relationshipCoverage.ts +++ b/src/domain/relationshipCoverage.ts @@ -1,24 +1,35 @@ import { z } from "zod"; -const diagnostics = z.array(z.string().min(1)).min(1); -const completeCoverage = z.strictObject({ status: z.literal("complete") }); +const diagnostics = z.array(z.string().min(1)).min(1).exactOptional(); +const completeCoverage = z.strictObject({ + status: z.literal("complete"), + diagnostics, +}); +const limitedDirections = z.union([ + z.tuple([z.literal("callers")]).check(z.minLength(1), z.maxLength(1)), + z.tuple([z.literal("callees")]).check(z.minLength(1), z.maxLength(1)), + z + .tuple([z.literal("callers"), z.literal("callees")]) + .check(z.minLength(2), z.maxLength(2)), +]); const partialCoverage = z.strictObject({ status: z.literal("partial"), limitations: z.array(z.string().min(1)).min(1), + limited_directions: limitedDirections.exactOptional(), + diagnostics, }); const unknownCoverage = z.strictObject({ status: z.literal("unknown"), reason: z.string().min(1), + limited_directions: limitedDirections.exactOptional(), + diagnostics, }); /** Explicit coverage state for a provider's observed caller/callee set. */ -export const relationshipCoverageSchema = z.union([ +export const relationshipCoverageSchema = z.discriminatedUnion("status", [ completeCoverage, - completeCoverage.extend({ diagnostics }), partialCoverage, - partialCoverage.extend({ diagnostics }), unknownCoverage, - unknownCoverage.extend({ diagnostics }), ]); export type RelationshipCoverage = z.infer; diff --git a/src/firmware/FirmwareProvider.ts b/src/firmware/FirmwareProvider.ts index 31fac4df1..b4c2badeb 100644 --- a/src/firmware/FirmwareProvider.ts +++ b/src/firmware/FirmwareProvider.ts @@ -97,6 +97,7 @@ const limitations = [ "Signature matches and reported sizes do not establish partition validity, encryption or runtime mappings. Binwalk may infer unknown lengths before emitting its report.", "Only Linux is supported by this integration. One worker, inherited per-process address-space/file-size/CPU limits and a deadline constrain execution; staging byte and entry budgets are polled and may transiently overshoot. Limits are not aggregate process-memory or filesystem quotas.", "Unblob's default sandbox and skip rules remain enabled. Missing external extractors, unknown chunks, encrypted chunks and depth limits yield partial coverage. Extracted code is never executed; only regular files are published, without executable permission bits.", + "extraction_gaps lists regions that were carved or left unextracted and names extractor dependencies the producer reported as missing. A listed gap does not mean the target is corrupt or that a retained region is empty.", "The actual launcher executable is hashed. Interpreter, shared libraries and installed Python package dependencies are not independently attested; source revision remains unknown for caller-supplied builds.", ] as const; @@ -381,6 +382,7 @@ export class FirmwareProvider implements FirmwareAnalysisPort { entries, normalized, selection: target.selection, + maxDepth: request.input.max_depth, engine, signal, retainCleanup: (tree) => { diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.ts index b2bc4ddd5..274a884b5 100644 --- a/src/firmware/FirmwarePublication.ts +++ b/src/firmware/FirmwarePublication.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { join } from "node:path"; import type { z } from "zod"; @@ -9,6 +10,74 @@ import { jsonValueSchema } from "../domain/jsonValue.js"; import { firmwareResultSchemas } from "../domain/firmware/firmwareAnalysis.js"; import { hashFirmwareFile, type FirmwareEntry } from "./FirmwareFiles.js"; import type { normalizeUnblobReport } from "./FirmwareReports.js"; +import type { extractionGap } from "../domain/firmware/firmwareAnalysis.js"; + +type ExtractionGap = z.infer; + +/** Static, truthful continuations that use already configured capabilities. */ +const GAP_CONTINUATIONS: Record = { + "missing-extractor-dependency": + "REA installs nothing. Add the named dependency to your own environment and rerun extract_firmware on the same immutable input with a fresh output_directory to re-evaluate the affected region; otherwise it stays unextracted.", + "unknown-chunk-handler": + "If Binwalk is configured with REA_BINWALK_COMMAND, run inspect_firmware_regions on the same input for signature hypotheses. Without configured Binwalk that inspection is unavailable. Alternatively, select a materially narrower range with extract_firmware and a fresh output_directory; the region is not decoded content.", + "encrypted-chunk": + "No static extraction continues past encryption; treat the region as unexamined rather than empty or corrupt.", + "extractor-failure": + "The producer retained this chunk's extraction problem in diagnostics. Review the reported failure, then rerun extract_firmware on the same immutable input with a fresh output_directory; the region is not decoded content.", + "depth-limited": + "Rerun extract_firmware with a higher max_depth (up to 10) and a fresh output_directory to descend into this path.", + "unpublished-entry": + "The entry is retained as metadata; only regular files are published, without executable permission bits.", +}; + +const UNNAMED_DEPENDENCY_CONTINUATION = + "The producer reported a missing extractor dependency without naming it; its identity is unavailable in this evidence. No specific installation or rerun can be prescribed from this report. Retain the gap as unextracted unless additional provider evidence establishes the dependency. REA installs nothing."; + +import { FIRMWARE_LIMITS } from "./FirmwareRelease.js"; + +/** The depth ceiling cannot be raised past the input schema maximum. */ +const MAX_EXTRACTION_DEPTH = 10; + +const depthContinuation = ( + maxDepth: number, + publishedChildPaths: readonly string[], +): string => + maxDepth < MAX_EXTRACTION_DEPTH + ? GAP_CONTINUATIONS["depth-limited"] + : publishedChildPaths.length > 0 + ? `This extraction already ran at the lane's maximum depth. To continue, rerun extract_firmware with path set to any eligible published child below this path and a fresh output directory; each child listed here restarts the depth budget: ${publishedChildPaths.map((path) => JSON.stringify(path)).join(", ")}.` + : `This extraction already ran at the lane's maximum depth, and no published child under this path is both nonempty and within the ${FIRMWARE_LIMITS.inputBytes}-byte firmware input limit, so the depth budget cannot be restarted from this output. Treat the region below this path as unexamined.`; + +// Use a different analyzer on the actual published child; repeating the same +// whole-child extraction with unchanged handlers would reproduce this gap. +const nestedUnknownContinuation = ( + publishedFile: { readonly path: string; readonly size: number } | undefined, +): string => { + const base = + "This unknown chunk was carved from a nested extraction, so its range is relative to the published child named by this gap's input_path, not to the original input."; + return publishedFile !== undefined && + publishedFile.size > 0 && + publishedFile.size <= FIRMWARE_LIMITS.inputBytes + ? `${base} Run inspect_firmware_regions with path set to that published child for signature hypotheses only if Binwalk is configured with REA_BINWALK_COMMAND. This inspection step is unavailable without configured Binwalk; retain the region as unexamined. Repeating the same whole-child extraction with unchanged handlers would not advance analysis.` + : `${base} That published child is not a nonempty file within the ${FIRMWARE_LIMITS.inputBytes}-byte firmware input limit, so inspect_firmware_regions cannot accept it as a fresh input; treat the region as unexamined.`; +}; + +const emptyGap = { + relative_path: null, + handler: null, + provider_id: null, + range: null, + root_file_range: null, + dependencies: [], +}; + +const gapSortKey = (gap: ExtractionGap): string => + [ + gap.input_path, + gap.kind, + gap.provider_id ?? "", + gap.relative_path ?? "", + ].join("\u0000"); /** Publish verified regular files, preserving links as metadata and rolling back failed writes. */ export const publishFirmwareExtraction = async (context: { @@ -16,6 +85,8 @@ export const publishFirmwareExtraction = async (context: { entries: readonly FirmwareEntry[]; normalized: ReturnType; selection: { offset: number; length: number; sha256: string }; + /** Caller-selected extraction depth, for boundary-aware continuations. */ + maxDepth: number; engine: z.infer["engine"]; signal: AbortSignal; retainCleanup: (tree: SafeOutputTree) => void; @@ -93,6 +164,12 @@ export const publishFirmwareExtraction = async (context: { derivations: context.normalized.derivations, depth_limited_paths: context.normalized.depth_limited_paths, diagnostics: context.normalized.diagnostics, + ...extractionGaps( + unpublished, + files, + context.normalized, + context.maxDepth, + ), published_bytes: files.reduce((sum, file) => sum + file.size, 0), }); context.signal.throwIfAborted(); @@ -135,3 +212,239 @@ export const publishFirmwareExtraction = async (context: { throw cause; } }; + +/** Derive explicit recovery rows from retained report facts; no new claims. */ +/** Gap rows for missing-dependency reports: chunk-nested rows keep their owning chunk. */ +const dependencyExtractionGaps = ( + normalized: ReturnType, + continuation: (instruction: string) => string, +): ExtractionGap[] => { + // Chunk-nested dependency reports name the chunk that could not extract; + // task-wide reports (standalone or multi-file) attach to the task. + // One index over the chunk list keeps dependency-row resolution linear. + const chunksByKey = new Map( + normalized.chunks.map((chunk) => [ + JSON.stringify([chunk.input_path, chunk.provider_id]), + chunk, + ]), + ); + return [ + ...normalized.chunk_missing_dependencies.flatMap( + ({ input_path, provider_id, dependencies }): ExtractionGap[] => { + const affected = chunksByKey.get( + JSON.stringify([input_path, provider_id]), + ); + return [ + { + kind: "missing-extractor-dependency", + input_path, + ...emptyGap, + ...(affected === undefined + ? {} + : { + handler: affected.handler, + provider_id, + range: affected.range, + root_file_range: affected.root_file_range, + }), + dependencies: [...dependencies], + continuation_id: continuation( + dependencies.length > 0 + ? GAP_CONTINUATIONS["missing-extractor-dependency"] + : UNNAMED_DEPENDENCY_CONTINUATION, + ), + }, + ]; + }, + ), + ...normalized.failed_extraction_stages + .filter(({ kind }) => kind === "missing-extractor-dependency") + .map( + ({ + input_path, + provider_id, + handler, + dependencies, + }): ExtractionGap => ({ + kind: "missing-extractor-dependency", + input_path, + ...emptyGap, + handler, + provider_id, + dependencies: [...dependencies], + continuation_id: continuation( + dependencies.length > 0 + ? GAP_CONTINUATIONS["missing-extractor-dependency"] + : UNNAMED_DEPENDENCY_CONTINUATION, + ), + }), + ), + // Standalone reports name only the task; do not borrow another stage's id. + ...normalized.missing_dependencies.map( + ({ input_path, dependencies }): ExtractionGap => ({ + kind: "missing-extractor-dependency", + input_path, + ...emptyGap, + dependencies: [...dependencies], + continuation_id: continuation( + dependencies.length > 0 + ? GAP_CONTINUATIONS["missing-extractor-dependency"] + : UNNAMED_DEPENDENCY_CONTINUATION, + ), + }), + ), + ]; +}; + +const extractionGaps = ( + unpublished: z.infer< + typeof firmwareResultSchemas.extract_firmware + >["unpublished_entries"], + files: readonly z.infer< + typeof firmwareResultSchemas.extract_firmware + >["files"][number][], + normalized: ReturnType, + maxDepth: number, +) => { + // Intern before row construction: N gaps retain N identities, while K unique + // recovery instructions consume prose only K times in the public result. + const instructions = new Map(); + const continuation = (instruction: string): string => { + const known = instructions.get(instruction); + if (known !== undefined) return known; + const id = createHash("sha256").update(instruction).digest("hex"); + instructions.set(instruction, id); + return id; + }; + const dependencyGaps = dependencyExtractionGaps(normalized, continuation); + const publishedFiles = new Map( + files.map((file) => [`$output/${file.relative_path}`, file]), + ); + const nestedContinuationId = (inputPath: string): string => { + return continuation( + nestedUnknownContinuation(publishedFiles.get(inputPath)), + ); + }; + const chunkGaps = ( + kind: ExtractionGap["kind"], + matches: (chunk: (typeof normalized.chunks)[number]) => boolean, + ): ExtractionGap[] => + normalized.chunks.filter(matches).map((chunk) => ({ + kind, + input_path: chunk.input_path, + ...emptyGap, + handler: chunk.handler, + provider_id: chunk.provider_id, + range: chunk.range, + root_file_range: chunk.root_file_range, + dependencies: [], + continuation_id: + kind === "unknown-chunk-handler" && + chunk.input_path.startsWith("$output/") + ? nestedContinuationId(chunk.input_path) + : continuation(GAP_CONTINUATIONS[kind]), + })); + // Chunks whose failure a dependency row already covers stay out of the + // generic extractor-failure emission. + const dependencyChunkKeys = new Set( + normalized.chunk_missing_dependencies.map(({ input_path, provider_id }) => + JSON.stringify([input_path, provider_id]), + ), + ); + // A depth ceiling can only be re-rooted through a published child that a + // fresh firmware input could actually accept; the index is only needed + // when a depth row will actually consult it. + const ceilingReached = + maxDepth >= MAX_EXTRACTION_DEPTH && + normalized.depth_limited_paths.length > 0; + const reRootablePaths = new Map>(); + if (ceilingReached) + for (const [filePath, file] of [...publishedFiles].sort( + ([left], [right]) => (left < right ? -1 : left > right ? 1 : 0), + )) { + const size = file.size; + if (size <= 0 || size > FIRMWARE_LIMITS.inputBytes) continue; + // The published file itself can re-root a fresh run; so can every + // directory containing it. + const segments = filePath.split("/"); + while (segments.length > 1) { + const ancestor = segments.join("/"); + const childPaths = reRootablePaths.get(ancestor) ?? new Set(); + childPaths.add(file.path); + reRootablePaths.set(ancestor, childPaths); + segments.pop(); + } + } + const depthContinuationIds = new Map(); + const depthContinuationId = (inputPath: string): string => { + const existing = depthContinuationIds.get(inputPath); + if (existing !== undefined) return existing; + const id = continuation( + depthContinuation( + maxDepth, + [...(reRootablePaths.get(inputPath) ?? [])].sort(), + ), + ); + depthContinuationIds.set(inputPath, id); + return id; + }; + const gaps: ExtractionGap[] = [ + ...dependencyGaps, + ...normalized.failed_extraction_stages + .filter(({ kind }) => kind === "extractor-failure") + .map(({ input_path, provider_id, handler }): ExtractionGap => ({ + kind: "extractor-failure", + input_path, + ...emptyGap, + handler, + provider_id, + dependencies: [], + continuation_id: continuation( + "The producer retained this extraction stage's failure in diagnostics. Review the stage report and correct the reported cause before rerunning extract_firmware on the same immutable input with a fresh output_directory; this report establishes no byte-region ownership.", + ), + })), + ...chunkGaps("unknown-chunk-handler", (chunk) => chunk.handler === null), + ...chunkGaps("encrypted-chunk", (chunk) => chunk.encrypted === true), + ...chunkGaps( + "extractor-failure", + (chunk) => + chunk.extraction_failed === true && + chunk.handler !== null && + chunk.encrypted !== true && + // The chunk's own dependency row already carries its recovery; a + // task-wide dependency names no region and replaces nothing. + !dependencyChunkKeys.has( + JSON.stringify([chunk.input_path, chunk.provider_id]), + ), + ), + ...normalized.depth_limited_paths.map((input_path): ExtractionGap => ({ + kind: "depth-limited", + input_path, + ...emptyGap, + dependencies: [], + continuation_id: depthContinuationId(input_path), + })), + ...unpublished.map((entry): ExtractionGap => ({ + kind: "unpublished-entry", + input_path: `$output/${entry.relative_path}`, + ...emptyGap, + relative_path: entry.relative_path, + dependencies: [], + continuation_id: continuation(GAP_CONTINUATIONS["unpublished-entry"]), + })), + ]; + gaps.sort((left, right) => + gapSortKey(left) < gapSortKey(right) + ? -1 + : gapSortKey(left) > gapSortKey(right) + ? 1 + : 0, + ); + return { + extraction_gaps: gaps, + extraction_continuations: [...instructions].map(([instruction, id]) => ({ + id, + instruction, + })), + }; +}; diff --git a/src/firmware/FirmwareReports.test.ts b/src/firmware/FirmwareReports.test.ts index b924180ba..1a491d507 100644 --- a/src/firmware/FirmwareReports.test.ts +++ b/src/firmware/FirmwareReports.test.ts @@ -42,7 +42,28 @@ const root = () => ({ subtasks: [], }); -describe("firmware producer boundaries", () => { +/** A chunk whose extraction failed on a missing dependency, when asked for it. */ +const nestedDependencyChunk = (id: string, offset: number) => ({ + __typename__: "ChunkReport", + id, + start_offset: offset, + end_offset: offset + 10, + size: 10, + handler_name: `handler-${id}`, + is_encrypted: false, + extraction_reports: + id === "gzip-id" + ? [ + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: ["sasquatch"], + }, + ] + : [], +}); + +describe("firmware producer chunk observations", () => { it("retains unknown chunks and their selected original-file range", () => { const normalized = normalizeUnblobReport([root()], context); expect(normalized.coverage).toBe("partial"); @@ -51,6 +72,412 @@ describe("firmware producer boundaries", () => { range: { offset: 0, length: 20 }, root_file_range: { offset: 64, length: 20 }, }); + expect(normalized.missing_dependencies).toEqual([]); + }); +}); + +describe("firmware producer dependency ownership", () => { + it("derives missing extractor dependencies from chunk and multi-file reports", () => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: record.reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + subtasks: [ + { + path: `${context.outputRoot}/rootfs`, + depth: 1, + blob_id: "sq-id", + is_multi_file: false, + }, + ], + }, + { + task: { + path: `${context.outputRoot}/rootfs`, + depth: 1, + blob_id: "sq-id", + is_multi_file: false, + }, + reports: [ + { + __typename__: "StatReport", + path: `${context.outputRoot}/rootfs`, + size: 0, + is_file: false, + is_dir: true, + is_link: false, + link_target: null, + }, + { + __typename__: "MultiFileReport", + id: "sq-id", + handler_name: "squashfs", + name: "rootfs", + paths: [], + extraction_reports: [ + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: ["sasquatch"], + }, + ], + }, + ], + subtasks: [], + }, + ], + context, + ); + expect(normalized.missing_dependencies).toEqual([]); + expect(normalized.failed_extraction_stages).toEqual([ + { + input_path: "$output/rootfs", + provider_id: "sq-id", + handler: "squashfs", + dependencies: ["sasquatch"], + kind: "missing-extractor-dependency", + }, + ]); + expect(normalized.chunk_missing_dependencies).toEqual([]); + }); + it("orders chunk dependencies totally across provider ids", () => { + const record = root(); + const chunkWithDependency = (id: string, offset: number) => ({ + __typename__: "ChunkReport", + id, + start_offset: offset, + end_offset: offset + 10, + size: 10, + handler_name: `handler-${id}`, + is_encrypted: false, + extraction_reports: [ + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: [`${id}-tool`], + }, + ], + }); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + chunkWithDependency("b-id", 0), + chunkWithDependency("a-id", 10), + ], + }, + ], + context, + ); + expect(normalized.chunk_missing_dependencies).toEqual([ + { + input_path: "$input", + provider_id: "a-id", + dependencies: ["a-id-tool"], + }, + { + input_path: "$input", + provider_id: "b-id", + dependencies: ["b-id-tool"], + }, + ]); + }); + it("keeps chunk-nested dependency reports bound to their owning chunk", () => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + nestedDependencyChunk("gzip-id", 0), + nestedDependencyChunk("other-id", 10), + ], + }, + ], + context, + ); + expect(normalized.missing_dependencies).toEqual([]); + expect(normalized.chunk_missing_dependencies).toEqual([ + { + input_path: "$input", + provider_id: "gzip-id", + dependencies: ["sasquatch"], + }, + ]); + }); +}); + +describe("firmware dependency failures with unavailable identities", () => { + it.each([{ dependencies: [] }, { dependencies: ["", ""] }])( + "retains unnamed chunk dependency ownership ($dependencies)", + ({ dependencies }) => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + { + ...nestedDependencyChunk("gzip-id", 0), + extraction_reports: [ + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies, + }, + ], + }, + ], + }, + ], + context, + ); + expect(normalized.missing_dependencies).toEqual([]); + expect(normalized.chunk_missing_dependencies).toEqual([ + { + input_path: "$input", + provider_id: "gzip-id", + dependencies: [], + }, + ]); + }, + ); + it("retains the missing-dependency reason on an unnamed multi-file stage", () => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports, + { + __typename__: "MultiFileReport", + id: "multi-id", + handler_name: "multipart-firmware", + name: "parts", + paths: [], + extraction_reports: [ + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: [], + }, + ], + }, + ], + }, + ], + context, + ); + expect(normalized.failed_extraction_stages).toEqual([ + { + input_path: "$input", + provider_id: "multi-id", + handler: "multipart-firmware", + dependencies: [], + kind: "missing-extractor-dependency", + }, + ]); + }); + it("retains named and unnamed dependency reports on one multi-file stage", () => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports, + { + __typename__: "MultiFileReport", + id: "multi-id", + handler_name: "multipart-firmware", + name: "parts", + paths: [], + extraction_reports: [ + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: ["sasquatch"], + }, + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: [], + }, + ], + }, + ], + }, + ], + context, + ); + expect(normalized.failed_extraction_stages).toEqual([ + { + input_path: "$input", + provider_id: "multi-id", + handler: "multipart-firmware", + dependencies: ["sasquatch"], + kind: "missing-extractor-dependency", + }, + { + input_path: "$input", + provider_id: "multi-id", + handler: "multipart-firmware", + dependencies: [], + kind: "missing-extractor-dependency", + }, + ]); + }); +}); + +describe("firmware producer chunk failures", () => { + it("marks chunks whose extraction reported problems", () => { + const record = root(); + const failing = { + __typename__: "ChunkReport", + id: "zip-id", + start_offset: 0, + end_offset: 20, + size: 20, + handler_name: "zip", + is_encrypted: false, + extraction_reports: [ + { __typename__: "SomeProducerFailure", severity: "ERROR" }, + ], + }; + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + failing, + ], + }, + ], + context, + ); + expect(normalized.chunks).toHaveLength(1); + expect(normalized.chunks[0]).toMatchObject({ + handler: "zip", + extraction_failed: true, + }); + expect(normalized.coverage).toBe("partial"); + }); +}); + +describe("firmware task dependency failures", () => { + it.each([{ dependencies: [] }, { dependencies: ["", ""] }])( + "retains unnamed dependency failures ($dependencies)", + ({ dependencies }) => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports, + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies, + }, + ], + }, + ], + context, + ); + expect(normalized.missing_dependencies).toEqual([ + { input_path: "$input", dependencies: [] }, + ]); + expect(normalized.diagnostics).toHaveLength(1); + }, + ); + it("retains named and unnamed task dependency reports together", () => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports, + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: ["sasquatch"], + }, + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: [], + }, + ], + }, + ], + context, + ); + expect(normalized.missing_dependencies).toEqual([ + { input_path: "$input", dependencies: ["sasquatch"] }, + { input_path: "$input", dependencies: [] }, + ]); + }); +}); + +describe("firmware chunk dependency failures", () => { + it("retains named and unnamed dependency reports on one chunk", () => { + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + { + ...nestedDependencyChunk("gzip-id", 0), + extraction_reports: [ + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: ["sasquatch"], + }, + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: [], + }, + ], + }, + ], + }, + ], + context, + ); + expect(normalized.chunk_missing_dependencies).toEqual([ + { + input_path: "$input", + provider_id: "gzip-id", + dependencies: ["sasquatch"], + }, + { input_path: "$input", provider_id: "gzip-id", dependencies: [] }, + ]); }); it("preserves unfamiliar report types as explicit diagnostics", () => { const record = root(); @@ -237,6 +664,77 @@ describe("Unblob task derivation identities", () => { report: multiFileReport, }, ]); + expect(normalized.failed_extraction_stages).toEqual([ + { + input_path: "$input", + provider_id: "multi-id", + handler: "multipart-firmware", + dependencies: ["sasquatch"], + kind: "missing-extractor-dependency", + }, + ]); + }); +}); + +describe("failed multi-file extraction stages", () => { + it("records a failed multi-file stage for a recovery row without chunks", () => { + const outputPath = `${context.outputRoot}/multi`; + const childTask = { + path: outputPath, + depth: 1, + blob_id: "multi-id", + is_multi_file: true, + }; + const stageFailure = { + __typename__: "ExtractorExecutionErrorReport", + severity: "ERROR", + handler: "multipart-firmware", + }; + const normalized = normalizeUnblobReport( + [ + { + ...root(), + reports: root().reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + subtasks: [childTask], + }, + { + task: childTask, + reports: [ + { + __typename__: "StatReport", + path: outputPath, + size: 0, + is_dir: true, + is_file: false, + is_link: false, + link_target: null, + }, + { + __typename__: "MultiFileReport", + id: "multi-id", + handler_name: "multipart-firmware", + name: "rootfs", + paths: [`${outputPath}/part-a`], + extraction_reports: [stageFailure], + }, + ], + subtasks: [], + }, + ], + context, + ); + expect(normalized.failed_extraction_stages).toEqual([ + { + input_path: "$output/multi", + provider_id: "multi-id", + handler: "multipart-firmware", + dependencies: [], + kind: "extractor-failure", + }, + ]); + expect(normalized.missing_dependencies).toEqual([]); }); }); diff --git a/src/firmware/FirmwareReports.ts b/src/firmware/FirmwareReports.ts index 90857de84..f06129aa2 100644 --- a/src/firmware/FirmwareReports.ts +++ b/src/firmware/FirmwareReports.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { digestSchema } from "../domain/digests.js"; +import { uniqueSorted } from "../domain/canonicalOrdering.js"; import { firmwareLogicalPath } from "./FirmwareFiles.js"; const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER); @@ -51,6 +52,15 @@ const multiFile = z.object({ extraction_reports: z.array(jsonValueSchema), }); const hashes = z.object({ sha256: digestSchema }); +/** + * The provider's structured missing-dependency report. Non-strict on purpose: + * unfamiliar extra fields stay in retained diagnostics, only the dependency + * names are derived here. + */ +const missingDependency = z.object({ + __typename__: z.literal("ExtractorDependencyNotFoundReport"), + dependencies: z.array(z.string()), +}); const region = z.object({ id: z.string(), start_offset: integer, @@ -79,7 +89,9 @@ const dependencyFailure = z.object({ /** Collect the named extractors from any dependency report nested in one value. */ const missingDependencies = (value: unknown): string[] => { - const found: string[] = []; + // Producer-controlled name volume must not make deduplication quadratic; + // a Set keeps collection linear and uniqueSorted restores canonical order. + const found = new Set(); const visit = (node: unknown): void => { if (Array.isArray(node)) { for (const item of node) visit(item); @@ -88,14 +100,13 @@ const missingDependencies = (value: unknown): string[] => { if (typeof node !== "object" || node === null) return; const failure = dependencyFailure.safeParse(node); if (failure.success) { - for (const dependency of failure.data.dependencies) - if (!found.includes(dependency)) found.push(dependency); + for (const dependency of failure.data.dependencies) found.add(dependency); return; } for (const child of Object.values(node)) visit(child); }; visit(value); - return found; + return [...found]; }; /** @@ -194,6 +205,13 @@ export const normalizeUnblobReport = ( >(); const sizes = new Map(); const depths: string[] = []; + const failedExtractionStages: { + input_path: string; + provider_id: string | null; + handler: string | null; + dependencies: readonly string[]; + kind: "missing-extractor-dependency" | "extractor-failure"; + }[] = []; const diagnostics: JsonValue[] = []; const blobOrigins = new Map(); const addBlobOrigin = ( @@ -216,8 +234,74 @@ export const normalizeUnblobReport = ( handler: string | null; encrypted: boolean | null; root_file_range: { offset: number; length: number } | null; + extraction_failed: boolean; }[] = []; const seen = new Set(); + const taskFailures = new Set(); + type DependencyFacts = { readonly names: Set; unnamed: boolean }; + const missingDependencies = new Map(); + /** Dependencies reported by one chunk's extraction, keyed with its chunk id. */ + const chunkMissingDependencies = new Map(); + const dependencyNames = ( + reports: readonly unknown[], + ): DependencyFacts | null => { + const names = new Set(); + let unnamed = false; + let reported = false; + for (const item of reports) { + const parsed = missingDependency.safeParse(item); + if (!parsed.success) continue; + reported = true; + if ( + parsed.data.dependencies.length === 0 || + parsed.data.dependencies.includes("") + ) + unnamed = true; + for (const name of parsed.data.dependencies) + if (name !== "") names.add(name); + } + return reported ? { names, unnamed } : null; + }; + const dependencyRows = ( + facts: DependencyFacts, + ): readonly { dependencies: readonly string[] }[] => [ + ...(facts.names.size > 0 + ? [{ dependencies: uniqueSorted([...facts.names]) }] + : []), + ...(facts.unnamed ? [{ dependencies: [] }] : []), + ]; + const noteMissingDependencies = ( + path: string, + reports: readonly unknown[], + ): void => { + const names = dependencyNames(reports); + // null means no accepted failure; [] means a definite dependency failure + // whose executable identity the producer left unavailable. + if (names === null) return; + const known = missingDependencies.get(path) ?? { + names: new Set(), + unnamed: false, + }; + for (const name of names.names) known.names.add(name); + known.unnamed ||= names.unnamed; + missingDependencies.set(path, known); + }; + const noteChunkMissingDependencies = ( + path: string, + chunkId: string, + reports: readonly unknown[], + ): void => { + const names = dependencyNames(reports); + if (names === null) return; + const key = JSON.stringify([path, chunkId]); + const known = chunkMissingDependencies.get(key) ?? { + names: new Set(), + unnamed: false, + }; + for (const name of names.names) known.names.add(name); + known.unnamed ||= names.unnamed; + chunkMissingDependencies.set(key, known); + }; for (const record of records) { const path = logical(record.task.path); if (seen.has(path)) @@ -273,6 +357,20 @@ export const normalizeUnblobReport = ( diagnostics.push( jsonValueSchema.parse({ input_path: path, report: item }), ); + noteMissingDependencies(path, [item]); + if ( + item.__typename__ === "ExtractorExecutionErrorReport" && + !taskFailures.has(path) + ) { + taskFailures.add(path); + failedExtractionStages.push({ + input_path: path, + provider_id: null, + handler: null, + dependencies: [], + kind: "extractor-failure", + }); + } if ( item.__typename__ === "ChunkReport" || item.__typename__ === "UnknownChunkReport" @@ -297,8 +395,10 @@ export const normalizeUnblobReport = ( }) .parse(item) : null; + const extractionFailed = + detail !== null && detail.extraction_reports.length > 0; if (detail !== null) - for (const problem of detail.extraction_reports) + for (const problem of detail.extraction_reports) { diagnostics.push( extractionDiagnostic( path, @@ -306,6 +406,9 @@ export const normalizeUnblobReport = ( jsonValueSchema.parse(problem), ), ); + // Nested under one chunk: keep the owning chunk, not the whole task. + noteChunkMissingDependencies(path, parsed.id, [problem]); + } addBlobOrigin(path, parsed.id, { handler: detail?.handler_name ?? null, range: { offset: parsed.start_offset, length: parsed.size }, @@ -316,6 +419,7 @@ export const normalizeUnblobReport = ( range: { offset: parsed.start_offset, length: parsed.size }, handler: detail?.handler_name ?? null, encrypted: detail?.is_encrypted ?? null, + extraction_failed: extractionFailed, root_file_range: path === "$input" ? { @@ -331,7 +435,7 @@ export const normalizeUnblobReport = ( handler: parsed.handler_name, range: null, }); - if (parsed.extraction_reports.length > 0) + if (parsed.extraction_reports.length > 0) { diagnostics.push( extractionDiagnostic( path, @@ -339,6 +443,28 @@ export const normalizeUnblobReport = ( jsonValueSchema.parse(item), ), ); + // The report owns its failure and dependencies. A produced subtask + // is a derivation, not a new owner for the parent's report. + const dependencies = dependencyNames(parsed.extraction_reports); + const rows = + dependencies === null + ? [{ dependencies: [] as readonly string[] }] + : dependencyRows(dependencies); + for (const row of rows) + failedExtractionStages.push({ + input_path: path, + provider_id: parsed.id, + handler: parsed.handler_name, + dependencies: + dependencies === null + ? [] + : uniqueSorted([...row.dependencies]), + kind: + dependencies === null + ? "extractor-failure" + : "missing-extractor-dependency", + }); + } } } } @@ -403,7 +529,48 @@ export const normalizeUnblobReport = ( chunks, derivations, depth_limited_paths: depths.sort(), + failed_extraction_stages: failedExtractionStages.sort((left, right) => + `${left.input_path}\u0000${left.provider_id ?? ""}` < + `${right.input_path}\u0000${right.provider_id ?? ""}` + ? -1 + : `${left.input_path}\u0000${left.provider_id ?? ""}` > + `${right.input_path}\u0000${right.provider_id ?? ""}` + ? 1 + : 0, + ), diagnostics, + missing_dependencies: [...missingDependencies.entries()] + .flatMap(([input_path, facts]) => + dependencyRows(facts).map(({ dependencies }) => ({ + input_path, + dependencies, + })), + ) + .sort((left, right) => + left.input_path < right.input_path + ? -1 + : left.input_path > right.input_path + ? 1 + : 0, + ), + chunk_missing_dependencies: [...chunkMissingDependencies.entries()] + .flatMap(([key, facts]) => { + const [input_path, provider_id] = JSON.parse(key) as [string, string]; + return dependencyRows(facts).map(({ dependencies }) => ({ + input_path, + provider_id, + dependencies, + })); + }) + .sort((left, right) => + `${left.input_path}\u0000${left.provider_id}` < + `${right.input_path}\u0000${right.provider_id}` + ? -1 + : `${left.input_path}\u0000${left.provider_id}` > + `${right.input_path}\u0000${right.provider_id}` + ? 1 + : 0, + ), coverage: depths.length > 0 || diagnostics.length > 0 || diff --git a/src/signals/SigrokRecordingProvider.ts b/src/signals/SigrokRecordingProvider.ts index 541138a6a..816556c62 100644 --- a/src/signals/SigrokRecordingProvider.ts +++ b/src/signals/SigrokRecordingProvider.ts @@ -162,6 +162,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { this.#resources.retain(root, undefined); const snapshotPath = join(root.path, "recording"); const execute = async (args: readonly string[]) => { + capturedOutput = undefined; const output = await runOwnedCommand( { command: limiter, diff --git a/src/sqlite/SqliteDatabaseFailures.ts b/src/sqlite/SqliteDatabaseFailures.ts index 06bf4e71e..09a1c0867 100644 --- a/src/sqlite/SqliteDatabaseFailures.ts +++ b/src/sqlite/SqliteDatabaseFailures.ts @@ -7,6 +7,7 @@ import { import { AnalysisAccessDeniedError, AnalysisArtifactChangedError, + AnalysisCapabilityUnavailableError, AnalysisCancelledError, AnalysisInputError, AnalysisOutputError, @@ -29,6 +30,15 @@ export const capturedSqliteOutput = ( truncated: snapshot.diagnosticTruncated === true, }); +/** The file-set member a filesystem error names, if it is the database, its WAL or its journal. */ +const busyMember = (cause: Error, path: string): string => { + const failed = "path" in cause ? cause.path : undefined; + return typeof failed === "string" && + [path, `${path}-wal`, `${path}-journal`].includes(failed) + ? failed + : path; +}; + /** Preserve artifact, caller, process, resource and output failure meanings. */ export const sqliteDatabaseFailure = ( cause: unknown, @@ -85,6 +95,21 @@ export const sqliteDatabaseFailure = ( return new AnalysisAccessDeniedError(operation, path, cause.code, { cause, }); + // Windows reports a sharing violation as EBUSY: another process holds the file without read + // sharing. The unchanged request succeeds once that process releases it. Name the member of the + // file set that was busy (the database, its WAL or its journal) when the error identifies it. + if (cause.code === "EBUSY") + return new AnalysisCapabilityUnavailableError( + SQLITE_PROVIDER_IDENTITY.id, + operation, + `Selected database file set is in use by another process (EBUSY): ${busyMember(cause, path)}`, + { + cause, + retryable: true, + userMessage: + "The selected SQLite database is held open by another process without read sharing. Close that application or inspect a copy, then retry.", + }, + ); if (cause.code === "ENOENT" || cause.code === "ENOTDIR") return new AnalysisInputError(operation, { cause }, [ { diff --git a/tests/acceptance/goBinaryCli.test.ts b/tests/acceptance/goBinaryCli.test.ts index fd6cd648e..22cb9fed0 100644 --- a/tests/acceptance/goBinaryCli.test.ts +++ b/tests/acceptance/goBinaryCli.test.ts @@ -7,11 +7,74 @@ import { createGoBinaryByteStringFixture, createGoBinaryDiagnosticFixtures, createGoBinaryFixture, + createGoSplitElfFixture, + createGoOverlappingElfFixture, createGoBinaryPathFailureFixture, GO_MODULE_TEXT, } from "../fixtures/go/image.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; + +cliTest( + "bounds overlapping ELF scans through CLI and keeps MCP responsive", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-go-overlap-"); + const path = join(root, "overlapping.elf"); + await writeFile(path, createGoOverlappingElfFixture()); + const result = await cli.run({ + arguments: ["inspect-go-binary", path, "--json"], + }); + expect(result.exitCode).toBe(1); + const expected = { + code: "resource_constraint", + details: { + operation: "inspect_go_binary", + resource: "cpu", + reported_limits: { + boundary: "search", + maximum_bytes: 256 * 1024 * 1024, + }, + }, + }; + expect(result.json).toMatchObject(expected); + const { client, call } = await connectLocalToolsMcp(); + const response = await call("inspect_go_binary", { path }); + expect(parseMcpToolError(response).error).toMatchObject(expected); + await client.ping(); + }, + 30000, +); + +cliTest( + "retains later-load sectionless ELF evidence through public CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-go-split-elf-"); + const { client, call } = await connectLocalToolsMcp(); + const infos = []; + for (const sectionless of [false, true]) { + const fixture = createGoSplitElfFixture({ sectionless }); + const path = join(root, sectionless ? "sectionless.elf" : "original.elf"); + await writeFile(path, fixture.bytes); + const result = await cli.run({ + arguments: ["inspect-go-binary", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_go_binary").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.build_info).not.toBeNull(); + infos.push(evidence.normalized_result.build_info); + const response = await call("inspect_go_binary", { path }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(evidence); + } + expect(infos[1]).toEqual(infos[0]); + await client.ping(); + }, + 30000, +); for (const expectedErrno of ["ELOOP", "ENAMETOOLONG"] as const) { cliTest.skipIf(expectedErrno === "ELOOP" && process.platform === "win32")( diff --git a/tests/boundary/android/JadxAnalysis.test.ts b/tests/boundary/android/JadxAnalysis.test.ts index 36b0ab9d9..10d451cad 100644 --- a/tests/boundary/android/JadxAnalysis.test.ts +++ b/tests/boundary/android/JadxAnalysis.test.ts @@ -4,7 +4,10 @@ import { analyzeJadxRequest, type JadxToolPort, } from "../../../src/android/JadxAnalysis.js"; -import { androidInputSchemas } from "../../../src/domain/android/androidAnalysis.js"; +import { + androidInputSchemas, + androidResultSchemas, +} from "../../../src/domain/android/androidAnalysis.js"; const summary = { full_name: "fixture.Target", @@ -41,6 +44,43 @@ const engine = { heap_limit_mib: null, }; +it("requires explicit direct invoke coverage state in result evidence", () => { + const result = { + engine, + class_name: "fixture.Target", + method_name: null, + overload_index: null, + reported_signature: null, + dex_descriptor: null, + total: 1, + references: [ + { + kind: "method", + name: "caller", + reported_full_name: "fixture.Other.caller", + containing_class: "fixture.Other", + top_class: "fixture.Other", + source_line: null, + dex_offset: null, + direct_invokes: null, + }, + ], + coverage: "complete", + reference_unit: "using_node", + }; + expect( + androidResultSchemas.trace_android_references.safeParse(result).success, + ).toBe(false); + expect( + androidResultSchemas.trace_android_references.safeParse({ + ...result, + references: [ + { ...result.references[0], direct_invokes_exhaustive: null }, + ], + }).success, + ).toBe(true); +}); + it("publishes exact DEX prototypes instead of discarding bridge metadata", async () => { const tools: JadxToolPort = { json: () => Promise.resolve(summary), @@ -246,7 +286,9 @@ it.each([ async (fields, expected) => { const result = await traceWithReference(fields as Record); expect(result).toMatchObject({ - references: [{ direct_invokes: expected }], + references: [ + { direct_invokes: expected, direct_invokes_exhaustive: null }, + ], }); }, ); @@ -279,26 +321,44 @@ it("preserves bounded instruction coverage separately from using-node coverage", }); }); -it("rejects malformed method descriptors at the Android adapter boundary", async () => { - const invalidSummary = { - ...summary, - methods: summary.methods.map((method) => ({ - ...method, - dex_descriptor: "not-a-dex-descriptor", - })), - }; - await expect( - analyzeJadxRequest( - { - json: () => Promise.resolve(invalidSummary), - text: () => Promise.reject(new Error("unexpected text call")), +it.each([ + "not-a-dex-descriptor", + "(Lbad(name;)V", + "(Lbad)name;)V", + "()Lbad(name;", + "()Lbad)name;", +])( + "rejects malformed DEX prototype %s at the adapter boundary", + async (descriptor) => { + const invalidSummary = { + ...summary, + methods: summary.methods.map((method) => ({ + ...method, + dex_descriptor: descriptor, + })), + }; + await expect( + analyzeJadxRequest( + { + json: () => Promise.resolve(invalidSummary), + text: () => Promise.reject(new Error("unexpected text call")), + }, + { + operation: "inspect_android_class", + input: { path: "/fixture.apk", class_name: "fixture.Target" }, + }, + {} as never, + engine, + ), + ).rejects.toMatchObject({ + _tag: "AnalysisOutputError", + cause: { + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["methods", 0, "dex_descriptor"], + }), + ]), }, - { - operation: "inspect_android_class", - input: { path: "/fixture.apk", class_name: "fixture.Target" }, - }, - {} as never, - engine, - ), - ).rejects.toThrow(); -}); + }); + }, +); diff --git a/tests/boundary/firmware/firmwareIntegration.test.ts b/tests/boundary/firmware/firmwareIntegration.test.ts index 4244212d3..6407739e7 100644 --- a/tests/boundary/firmware/firmwareIntegration.test.ts +++ b/tests/boundary/firmware/firmwareIntegration.test.ts @@ -15,6 +15,25 @@ import { ToolResultDelivery } from "../../../src/server/toolResult.js"; import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js"; import { SafeOutputTreeCreationFailure } from "../../../src/artifacts/SafeOutputTreeCreationFailure.js"; +// Assert recovery behavior through the public reference table, while keeping +// row identity expectations independent of the instruction's digest. +const gapsWithContinuations = ( + result: ReturnType, +) => { + const instructions = new Map( + result.extraction_continuations.map(({ id, instruction }) => [ + id, + instruction, + ]), + ); + return result.extraction_gaps.map(({ continuation_id, ...gap }) => { + const instruction = instructions.get(continuation_id); + if (instruction === undefined) + throw new Error("Missing recovery instruction"); + return { ...gap, continuation: instruction }; + }); +}; + const directoryCloseFailure = vi.hoisted(() => ({ mode: "off" as "off" | "transient" | "persistent", attempts: 0, @@ -270,6 +289,247 @@ it("retains the complete firmware result when published output cleanup stays unc ); }); +it("shares recovery prose across distinct unknown chunk identities", async () => { + const fixture = await firmwareFixture("many-unknown"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(normalized.extraction_gaps).toHaveLength(1000); + expect( + new Set(normalized.extraction_gaps.map(({ provider_id }) => provider_id)) + .size, + ).toBe(1000); + expect(normalized.extraction_continuations).toHaveLength(1); + const [instruction] = normalized.extraction_continuations; + expect(instruction?.instruction).toContain("inspect_firmware_regions"); + expect( + normalized.extraction_gaps.every( + (gap) => gap.continuation_id === instruction?.id, + ), + ).toBe(true); +}); + +it("retains a task dependency failure when the producer names no executable", async () => { + const fixture = await firmwareFixture("unnamed-dependency"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(gapsWithContinuations(normalized)).toEqual([ + { + kind: "missing-extractor-dependency", + input_path: "$input", + relative_path: null, + handler: null, + provider_id: null, + range: null, + root_file_range: null, + dependencies: [], + continuation: expect.stringContaining("identity is unavailable"), + }, + ]); +}); + +it("preserves an unnamed dependency failure on its owning chunk", async () => { + const fixture = await firmwareFixture("unnamed-chunk-dependency"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(gapsWithContinuations(normalized)).toEqual([ + { + kind: "missing-extractor-dependency", + input_path: "$input", + relative_path: null, + handler: "gzip", + provider_id: "gzip-id", + range: { offset: 0, length: 18 }, + root_file_range: { offset: 0, length: 18 }, + dependencies: [], + continuation: expect.stringContaining("identity is unavailable"), + }, + ]); +}); + +it("routes a nested unknown chunk through its published child", async () => { + const fixture = await firmwareFixture("nested-unknown"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(gapsWithContinuations(normalized)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "unknown-chunk-handler", + input_path: "$output/rootfs/config", + provider_id: "unknown-id", + range: { offset: 0, length: 14 }, + root_file_range: null, + continuation: expect.stringContaining( + "published child named by this gap's input_path", + ), + }), + ]), + ); + expect(gapsWithContinuations(normalized)[0]?.continuation).toContain( + "only if Binwalk is configured with REA_BINWALK_COMMAND", + ); + expect(gapsWithContinuations(normalized)[0]?.continuation).toContain( + "unavailable without configured Binwalk", + ); +}); + +it("retains every eligible child in a maximum-depth continuation", async () => { + const fixture = await firmwareFixture("depth-reentry"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + max_depth: 10, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + const gap = gapsWithContinuations(normalized).find( + ({ kind, input_path }) => + kind === "depth-limited" && input_path === "$output/rootfs/config", + ); + expect(gap?.continuation).toContain( + JSON.stringify(`${fixture.output}/rootfs/config/inner.gz`), + ); + expect(gap?.continuation).toContain( + JSON.stringify(`${fixture.output}/rootfs/config/second.gz`), + ); +}); + +it("withholds the nested rerun advice when the published child exceeds the input limit", async () => { + const fixture = await firmwareFixture("nested-unknown-oversized"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(gapsWithContinuations(normalized)).toEqual([ + { + kind: "unknown-chunk-handler", + input_path: "$output/rootfs/config", + relative_path: null, + handler: null, + provider_id: "unknown-id", + range: { offset: 0, length: 14 }, + root_file_range: null, + dependencies: [], + continuation: expect.stringContaining( + "cannot accept it as a fresh input", + ), + }, + ]); +}); + +it.each([ + ["depth", 10, "rerun extract_firmware"], + ["depth-orphan", 10, "cannot be restarted"], +] as const)( + "bounds the maximum-depth continuation by a usable published child (%s)", + async (mode, maxDepth, expected) => { + const fixture = await firmwareFixture(mode); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + max_depth: maxDepth, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(normalized.depth_limited_paths.length).toBeGreaterThan(0); + expect( + gapsWithContinuations(normalized).filter( + (gap) => gap.kind === "depth-limited", + ), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "depth-limited", + continuation: expect.stringContaining(expected), + }), + ]), + ); + }, +); + +it("reports a failed multi-file stage as an extractor-failure gap", async () => { + const fixture = await firmwareFixture("multi-file-failure"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(gapsWithContinuations(normalized)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "extractor-failure", + input_path: "$input", + handler: "tar", + provider_id: "mf-id", + range: null, + root_file_range: null, + }), + ]), + ); +}); + +it("carries the owning multi-file stage on a task-level dependency gap", async () => { + const fixture = await firmwareFixture("multi-file-dependency"); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (!result.ok) throw result.error; + const normalized = firmwareResultSchemas.extract_firmware.parse( + result.value.normalized_result, + ); + expect(gapsWithContinuations(normalized)).toEqual([ + { + kind: "missing-extractor-dependency", + input_path: "$input", + relative_path: null, + handler: "squashfs", + provider_id: "sq-id", + range: null, + root_file_range: null, + dependencies: ["sasquatch"], + continuation: expect.stringContaining("installs nothing"), + }, + ]); + expect(normalized.diagnostics).toEqual([ + expect.objectContaining({ stage: "squashfs" }), + ]); +}); + const injectPublishedDirectoryCloseFailure = (persistent: boolean) => { directoryCloseFailure.attempts = 0; const commit = SafeOutputTree.prototype.commit; @@ -304,6 +564,22 @@ it.each(["depth", "dependency", "link"])( expect(normalized.files).toHaveLength(1); if (mode === "depth") expect(normalized.depth_limited_paths).toEqual(["$output/rootfs/config"]); + if (mode === "depth") + expect(gapsWithContinuations(normalized)).toEqual([ + { + kind: "depth-limited", + input_path: "$output/rootfs/config", + relative_path: null, + handler: null, + provider_id: null, + range: null, + root_file_range: null, + dependencies: [], + continuation: expect.stringMatching( + /max_depth.*fresh output_directory/u, + ), + }, + ]); if (mode === "dependency") expect(normalized.diagnostics).toEqual([ { @@ -317,6 +593,20 @@ it.each(["depth", "dependency", "link"])( }, }, ]); + if (mode === "dependency") + expect(gapsWithContinuations(normalized)).toEqual([ + { + kind: "missing-extractor-dependency", + input_path: "$input", + relative_path: null, + handler: "gzip", + provider_id: "gzip-id", + range: { offset: 0, length: 18 }, + root_file_range: { offset: 0, length: 18 }, + dependencies: ["sasquatch"], + continuation: expect.stringContaining("installs nothing"), + }, + ]); if (mode === "link") { expect(normalized.unpublished_entries).toEqual([ { @@ -325,6 +615,19 @@ it.each(["depth", "dependency", "link"])( link_target: "/etc/passwd", }, ]); + expect(gapsWithContinuations(normalized)).toEqual([ + { + kind: "unpublished-entry", + input_path: "$output/escape", + relative_path: "escape", + handler: null, + provider_id: null, + range: null, + root_file_range: null, + dependencies: [], + continuation: expect.stringContaining("metadata"), + }, + ]); await expect(access(`${fixture.output}/escape`)).rejects.toMatchObject({ code: "ENOENT", }); diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 27bc6ff09..789a2010a 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -55,6 +55,8 @@ else { if(readFileSync('recording', 'utf8') !== 'recording bytes') throw Error(' }, async (input) => { calls++; + if (mode === "launch-failure" && !input.arguments.includes("--version")) + throw Object.assign(new Error("spawn E2BIG"), { code: "E2BIG" }); if (mode === "file-limit" && !input.arguments.includes("--version")) { const stdout = { text: "", bytes: 0, observedBytes: 0 }; const stderr = { @@ -243,6 +245,21 @@ it.each([ }, ); +it.each(["decode", "discovery"] as const)( + "does not attribute the version probe's output to a %s command that fails to launch", + async (operation) => { + const f = await fixture("launch-failure"); + const result = + operation === "decode" + ? await f.service.decode({ path: f.path, decoders: "uart" }) + : await f.service.inspectDecoders({}); + if (result.ok) throw new Error("Expected launch failure"); + expect(f.calls()).toBe(2); + expect(result.error.message).toContain("spawn E2BIG"); + expect(result.error.capturedOutput).toBeUndefined(); + }, +); + it("retains a completed observation and cleanup ownership for retry", async () => { const f = await fixture("ok", true); const result = await f.service.decode({ path: f.path, decoders: "uart" }); diff --git a/tests/boundary/sqlite/sqliteDatabaseBusy.test.ts b/tests/boundary/sqlite/sqliteDatabaseBusy.test.ts new file mode 100644 index 000000000..963ee9c83 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseBusy.test.ts @@ -0,0 +1,159 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { access } from "node:fs/promises"; +import { expect, it, onTestFinished, vi } from "vitest"; + +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { createSqliteDatabaseFixture } from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const busy = vi.hoisted(() => ({ path: "" })); + +// Node opens files with full sharing, so a test process cannot hold one without read sharing +// itself. This injects the error Windows returns for that sharing violation. +vi.mock("node:fs/promises", async (importOriginal) => { + const filesystem = await importOriginal(); + return { + ...filesystem, + open: async (...args: Parameters) => { + if (busy.path !== "" && args[0] === busy.path) + throw Object.assign( + new Error(`EBUSY: resource busy or locked, open '${busy.path}'`), + { code: "EBUSY", errno: -4082, syscall: "open", path: busy.path }, + ); + return filesystem.open(...args); + }, + }; +}); + +// Private roots go under the test's own workspace, so the cleanup check sees only this +// provider's root and not those of SQLite tests running in parallel. +const createProvider = (parent: string, roots: string[]) => + new SqliteDatabaseProvider(process.env, undefined, async () => { + const root = await PrivateRuntimeRoot.create({ parent }); + roots.push(root.path); + return root; + }); + +const expectRetryableBusy = ( + error: Parameters[0], + path: string, +) => + expect(projectAnalysisError(error)).toMatchObject({ + code: "capability_unavailable", + category: "unavailable", + retryable: true, + message: + "The selected SQLite database is held open by another process without read sharing. Close that application or inspect a copy, then retry.", + details: { + provider_id: "sqlite", + operation: "inspect_sqlite_database", + reason: `Selected database file set is in use by another process (EBUSY): ${path}`, + }, + }); + +const expectRootsRemoved = async (roots: string[]) => { + expect(roots).toHaveLength(1); + await expect(access(roots[0]!)).rejects.toMatchObject({ code: "ENOENT" }); +}; + +const stopHolder = async (holder: ChildProcess) => { + // A holder that never started has no pid and emits only "error", never "exit". + if ( + holder.pid === undefined || + holder.exitCode !== null || + holder.signalCode !== null + ) + return; + const exited = new Promise((resolve) => + holder.once("exit", () => resolve()), + ); + holder.kill(); + await exited; +}; + +it("reports a selected database held by another process as retryable and unavailable", async () => { + const workspace = await createTestWorkspace("rea-sqlite-busy-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const roots: string[] = []; + const provider = createProvider(workspace.root, roots); + onTestFinished(async () => { + busy.path = ""; + await provider.close(); + await removeTestWorkspace(workspace.root); + }); + busy.path = database.path; + + const result = await provider.inspect({ path: database.path }); + if (result.ok) throw new Error("Expected a busy database rejection"); + expectRetryableBusy(result.error, database.path); + await expectRootsRemoved(roots); +}); + +it("names the WAL when another process holds that member of the file set", async () => { + const workspace = await createTestWorkspace("rea-sqlite-busy-wal-"); + const database = createSqliteDatabaseFixture(workspace.root, true); + const roots: string[] = []; + const provider = createProvider(workspace.root, roots); + onTestFinished(async () => { + busy.path = ""; + await provider.close(); + database.close(); + await removeTestWorkspace(workspace.root); + }); + busy.path = database.walPath; + + const result = await provider.inspect({ path: database.path }); + if (result.ok) throw new Error("Expected a busy WAL rejection"); + expectRetryableBusy(result.error, database.walPath); + await expectRootsRemoved(roots); +}); + +it.runIf(process.platform === "win32")( + "reports a database that another Windows process holds without sharing as retryable", + async () => { + const workspace = await createTestWorkspace("rea-sqlite-busy-native-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const roots: string[] = []; + const provider = createProvider(workspace.root, roots); + const literal = database.path.replaceAll("'", "''"); + const holder = spawn( + "powershell.exe", + [ + "-NoProfile", + "-Command", + `$f = [System.IO.File]::Open('${literal}', 'Open', 'ReadWrite', 'None'); Write-Output ready; Start-Sleep -Seconds 60; $f.Close()`, + ], + { stdio: ["ignore", "pipe", "ignore"], windowsHide: true }, + ); + // One ordered cleanup: the holder must release the file before the workspace is removed. + onTestFinished(async () => { + await stopHolder(holder); + await provider.close(); + await removeTestWorkspace(workspace.root); + }); + await new Promise((resolve, reject) => { + holder.stdout?.once("data", () => resolve()); + holder.once("error", (error) => + reject(new Error("Holder did not start", { cause: error })), + ); + holder.once("exit", (code) => + reject( + new Error(`Holder exited before opening the file (${String(code)})`), + ), + ); + }); + + const result = await provider.inspect({ path: database.path }); + if (result.ok) throw new Error("Expected a busy database rejection"); + expectRetryableBusy(result.error, database.path); + await expectRootsRemoved(roots); + }, + 20_000, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts index f656c35f4..76ec81c4a 100644 --- a/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts @@ -1,7 +1,7 @@ import { Cli } from "incur"; import { access, writeFile } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it, onTestFinished, vi } from "vitest"; @@ -274,7 +274,7 @@ it("retains reply-read cancellation and retries the failed reply descriptor clos { signal: controller.signal }, ); if (result.ok) throw new Error("Expected cancelled reply read"); - expect(projectAnalysisError(result.error)).toMatchObject({ + const cancelled = { code: "cleanup_incomplete", details: { execution_failure: "cancelled", @@ -285,7 +285,20 @@ it("retains reply-read cancellation and retries the failed reply descriptor clos truncated: false, }, }, - }); + }; + // Windows cannot remove a private root while the reply descriptor is open; + // the retained cancellation is then reported as the root cleanup's cause. + expect(projectAnalysisError(result.error)).toMatchObject( + process.platform === "win32" + ? { + code: "cleanup_incomplete", + details: { + resources: [dirname(descriptorCleanup.path)], + diagnostics: { previous_error: cancelled }, + }, + } + : cancelled, + ); const handle = descriptorCleanup.handles[0]; if (handle === undefined) throw new Error("Expected owned reply descriptor"); expect(handle.fd).toBeGreaterThanOrEqual(0); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index 7805552c9..5ecc35672 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -626,27 +626,32 @@ nativeTest( const controller = new AbortController(); let ownedRoot = ""; let pid: number | undefined; - const provider = new SqliteDatabaseProvider({}, async (spawn) => { - ownedRoot = spawn.cwd ?? ""; - const requestPath = spawn.arguments[2]; - if (requestPath === undefined) - throw new Error("SQLite worker arguments missing"); - const launched = await spawnOwnedProviderProcess({ - ...spawn, - arguments: [ - "--input-type=module", - "-e", - `await import(${JSON.stringify(workerUrl.href)}); process.stdout.write('ready\\n'); setInterval(() => {}, 1000);`, - "sqlite-worker-test", - requestPath, - ], - }); - pid = launched.process.pid; - await waitForProviderProcessReady(launched.process); - await access(`${ownedRoot}/reply.json`); - setImmediate(() => controller.abort()); - return launched; - }); + const provider = new SqliteDatabaseProvider( + process.platform === "win32" && process.env.SystemRoot !== undefined + ? { SystemRoot: process.env.SystemRoot } + : {}, + async (spawn) => { + ownedRoot = spawn.cwd ?? ""; + const requestPath = spawn.arguments[2]; + if (requestPath === undefined) + throw new Error("SQLite worker arguments missing"); + const launched = await spawnOwnedProviderProcess({ + ...spawn, + arguments: [ + "--input-type=module", + "-e", + `await import(${JSON.stringify(workerUrl.href)}); process.stdout.write('ready\\n'); setInterval(() => {}, 1000);`, + "sqlite-worker-test", + requestPath, + ], + }); + pid = launched.process.pid; + await waitForProviderProcessReady(launched.process); + await access(`${ownedRoot}/reply.json`); + setImmediate(() => controller.abort()); + return launched; + }, + ); const result = await provider.inspect( { path: database.path, table: "records" }, { signal: controller.signal }, diff --git a/tests/fixtures/firmware/producer.mjs b/tests/fixtures/firmware/producer.mjs index 6654c4941..d7629073a 100644 --- a/tests/fixtures/firmware/producer.mjs +++ b/tests/fixtures/firmware/producer.mjs @@ -1,5 +1,12 @@ import { createHash } from "node:crypto"; -import { chmod, mkdir, readFile, writeFile, symlink } from "node:fs/promises"; +import { + chmod, + mkdir, + readFile, + truncate, + writeFile, + symlink, +} from "node:fs/promises"; import { join } from "node:path"; const args = process.argv.slice(2); const mode = process.env.REA_FIRMWARE_FIXTURE_MODE; @@ -61,18 +68,24 @@ if (args.includes("--version")) { : join(output, "rootfs", "config"); await mkdir(join(output, "rootfs")); const content = Buffer.from("firmware=true\n"); - if (mode !== "missing-file") + if ( + mode !== "missing-file" && + mode !== "depth-reentry" && + mode !== "depth-orphan" + ) await writeFile(child, mode === "budget" ? Buffer.alloc(10000) : content); if (mode === "link") await symlink("/etc/passwd", join(output, "escape")); const childHash = createHash("sha256").update(content).digest("hex"); const depth = - mode === "depth" ? Number(args[args.indexOf("--depth") + 1]) : 1; - const stat = (path, size) => ({ + mode === "depth" || mode === "depth-reentry" || mode === "depth-orphan" + ? Number(args[args.indexOf("--depth") + 1]) + : 1; + const stat = (path, size, isDir = false) => ({ __typename__: "StatReport", path, size, - is_file: true, - is_dir: false, + is_file: !isDir, + is_dir: isDir, is_link: false, link_target: null, }); @@ -84,6 +97,185 @@ if (args.includes("--version")) { }); const hash = (value) => ({ __typename__: "HashReport", sha256: value }); const childTask = task(child, depth, "gzip-id"); + if (mode === "nested-unknown") { + const nestedUnknownRecords = [ + { + task: childTask, + reports: [ + stat(child, content.length), + hash(childHash), + { + __typename__: "UnknownChunkReport", + id: "unknown-id", + start_offset: 0, + end_offset: content.length, + size: content.length, + }, + ], + subtasks: [], + }, + { + task: task(input, 0, ""), + reports: [ + stat(input, bytes.length), + hash(sha256), + { + __typename__: "ChunkReport", + id: "gzip-id", + start_offset: 0, + end_offset: bytes.length, + size: bytes.length, + handler_name: "gzip", + is_encrypted: false, + extraction_reports: [], + }, + ], + subtasks: [childTask], + }, + ]; + await writeFile(reportPath, JSON.stringify(nestedUnknownRecords)); + } + if (mode === "nested-unknown-oversized") { + // The child exceeds the firmware input limit, so a fresh run cannot + // accept it even though the unknown chunk's range is small. + const oversized = 128 * 1024 * 1024 + 1; + await truncate(child, oversized); + const oversizedHash = createHash("sha256") + .update(await readFile(child)) + .digest("hex"); + const nestedUnknownRecords = [ + { + task: childTask, + reports: [ + stat(child, oversized), + hash(oversizedHash), + { + __typename__: "UnknownChunkReport", + id: "unknown-id", + start_offset: 0, + end_offset: content.length, + size: content.length, + }, + ], + subtasks: [], + }, + { + task: task(input, 0, ""), + reports: [ + stat(input, bytes.length), + hash(sha256), + { + __typename__: "ChunkReport", + id: "gzip-id", + start_offset: 0, + end_offset: bytes.length, + size: bytes.length, + handler_name: "gzip", + is_encrypted: false, + extraction_reports: [], + }, + ], + subtasks: [childTask], + }, + ]; + await writeFile(reportPath, JSON.stringify(nestedUnknownRecords)); + } + if (mode === "depth-orphan") { + // A depth-limited directory with nothing published under it cannot + // re-root a fresh run. + const mid = join(output, "rootfs", "config"); + await mkdir(mid, { recursive: true }); + const midTask = task(mid, depth, "gzip-id"); + const orphanRecords = [ + { + task: midTask, + reports: [stat(mid, 0, true)], + subtasks: [], + }, + { + task: task(input, 0, ""), + reports: [ + stat(input, bytes.length), + hash(sha256), + { + __typename__: "ChunkReport", + id: "gzip-id", + start_offset: 0, + end_offset: bytes.length, + size: bytes.length, + handler_name: "gzip", + is_encrypted: false, + extraction_reports: [], + }, + ], + subtasks: [midTask], + }, + ]; + await writeFile(reportPath, JSON.stringify(orphanRecords)); + } + if (mode === "depth-reentry") { + // A depth-limited directory with two published children that can each + // re-root a fresh run, proving neither target is lost during indexing. + const mid = join(output, "rootfs", "config"); + const innerPath = join(mid, "inner.gz"); + const secondPath = join(mid, "second.gz"); + await mkdir(mid, { recursive: true }); + await writeFile(innerPath, content); + await writeFile(secondPath, content); + const innerTask = task(innerPath, depth + 1, "inner-id"); + const secondTask = task(secondPath, depth + 1, "second-id"); + const midTask = { + ...task(mid, depth, "gzip-id"), + is_multi_file: true, + }; + const depthReentryRecords = [ + { + task: innerTask, + reports: [stat(innerPath, content.length), hash(childHash)], + subtasks: [], + }, + { + task: secondTask, + reports: [stat(secondPath, content.length), hash(childHash)], + subtasks: [], + }, + { + task: midTask, + reports: [ + stat(mid, 0, true), + { + __typename__: "MultiFileReport", + id: "gzip-id", + handler_name: "gzip", + name: "config", + paths: [innerPath, secondPath], + extraction_reports: [], + }, + ], + subtasks: [innerTask, secondTask], + }, + { + task: task(input, 0, ""), + reports: [ + stat(input, bytes.length), + hash(sha256), + { + __typename__: "ChunkReport", + id: "gzip-id", + start_offset: 0, + end_offset: bytes.length, + size: bytes.length, + handler_name: "gzip", + is_encrypted: false, + extraction_reports: [], + }, + ], + subtasks: [midTask], + }, + ]; + await writeFile(reportPath, JSON.stringify(depthReentryRecords)); + process.exitCode = 0; + } const records = [ { task: childTask, @@ -100,6 +292,41 @@ if (args.includes("--version")) { reports: [ stat(input, bytes.length), hash(sha256), + ...(mode === "multi-file-failure" || mode === "multi-file-dependency" + ? [ + { + __typename__: "MultiFileReport", + id: mode === "multi-file-dependency" ? "sq-id" : "mf-id", + handler_name: + mode === "multi-file-dependency" ? "squashfs" : "tar", + name: + mode === "multi-file-dependency" + ? "rootfs.squashfs" + : "rootfs.tar", + paths: [ + join( + output, + mode === "multi-file-dependency" + ? "rootfs.squashfs" + : "rootfs.tar", + ), + ], + extraction_reports: [ + mode === "multi-file-dependency" + ? { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: ["sasquatch"], + } + : { + __typename__: "ExtractorExecutionErrorReport", + severity: "ERROR", + handler: "tar", + }, + ], + }, + ] + : []), { __typename__: "ChunkReport", id: "gzip-id", @@ -109,12 +336,15 @@ if (args.includes("--version")) { handler_name: "gzip", is_encrypted: false, extraction_reports: - mode === "dependency" + mode === "dependency" || mode === "unnamed-chunk-dependency" ? [ { __typename__: "ExtractorDependencyNotFoundReport", severity: "ERROR", - dependencies: ["sasquatch"], + dependencies: + mode === "unnamed-chunk-dependency" + ? [] + : ["sasquatch"], }, ] : [], @@ -123,13 +353,42 @@ if (args.includes("--version")) { subtasks: [childTask], }, ]; - await writeFile( - reportPath, - mode === "malformed" || mode === "workspace-cleanup-failure-malformed" - ? "[" - : JSON.stringify(records), - ); - if (mode === "dependency") process.exitCode = 1; + if (mode === "many-unknown") { + records[1].reports = [ + ...records[1].reports, + ...Array.from({ length: 1000 }, (_, index) => ({ + __typename__: "UnknownChunkReport", + id: `unknown-${index}`, + start_offset: 0, + end_offset: 1, + size: 1, + })), + ]; + } + if (mode === "unnamed-dependency") { + records[1].reports = [ + ...records[1].reports, + { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: [], + }, + ]; + } + if ( + mode !== "depth-reentry" && + mode !== "depth-orphan" && + mode !== "nested-unknown" && + mode !== "nested-unknown-oversized" + ) + await writeFile( + reportPath, + mode === "malformed" || mode === "workspace-cleanup-failure-malformed" + ? "[" + : JSON.stringify(records), + ); + if (mode === "dependency" || mode === "multi-file-dependency") + process.exitCode = 1; if ( mode === "workspace-cleanup-failure" || mode === "workspace-cleanup-failure-malformed" diff --git a/tests/fixtures/go/image.ts b/tests/fixtures/go/image.ts index 0c56050dc..9b930ff69 100644 --- a/tests/fixtures/go/image.ts +++ b/tests/fixtures/go/image.ts @@ -135,6 +135,57 @@ export const createGoBinaryFixture = ( return { bytes, headerOffset, versionOffset, moduleOffset, moduleBytes }; }; +/** Two writable ELF loads, as in a PIE whose build info is in the later load. */ +export const createGoSplitElfFixture = ( + options: Parameters[0] = {}, +) => { + const fixture = createGoBinaryFixture({ ...options, format: "elf" }); + const { bytes, headerOffset } = fixture; + const wide = (options.bits ?? 64) === 64; + const little = (options.byteOrder ?? "little") === "little"; + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const word = (offset: number, value: number) => { + if (wide) view.setBigUint64(offset, BigInt(value), little); + else view.setUint32(offset, value, little); + }; + const second = 128 + (wide ? 56 : 32); + bytes.copy(bytes, second, 128, second); + view.setUint16(wide ? 56 : 44, 2, little); + word(128 + (wide ? 32 : 16), headerOffset); + word(128 + (wide ? 40 : 20), headerOffset); + word(second + (wide ? 8 : 4), headerOffset); + word(second + (wide ? 16 : 8), 0x10000 + headerOffset); + word(second + (wide ? 32 : 16), bytes.length - headerOffset); + word(second + (wide ? 40 : 20), bytes.length - headerOffset); + return fixture; +}; + +/** Many file-backed ELF loads sharing a small input, without any metadata marker. */ +export const createGoOverlappingElfFixture = (duplicate = false): Buffer => { + const count = 257; + const size = 1024 * 1024; + const dataStart = 65536; + const bytes = Buffer.alloc(dataStart + size + count * 16); + createGoBinaryFixture({ sectionless: true, omitBuildInfo: true }).bytes.copy( + bytes, + 0, + 0, + 64, + ); + bytes.writeUInt16LE(count, 56); + for (let index = 0; index < count; index++) { + const header = 128 + index * 56; + const offset = dataStart + (duplicate ? 0 : index * 16); + bytes.writeUInt32LE(1, header); + bytes.writeUInt32LE(6, header + 4); + bytes.writeBigUInt64LE(BigInt(offset), header + 8); + bytes.writeBigUInt64LE(0x400000n + BigInt(offset), header + 16); + bytes.writeBigUInt64LE(BigInt(size), header + 32); + bytes.writeBigUInt64LE(BigInt(size), header + 40); + } + return bytes; +}; + /** Embedded invalid UTF-8 alongside valid records, with exact independent byte expectations. */ export const createGoBinaryByteStringFixture = () => { const version = Buffer.from("xgo1.26.0"); From 679658b28748729c9b881497e680c811a708f5f3 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:33:09 -0700 Subject: [PATCH 12/26] fix(reference): report entries beyond the path limit as I/O failures (#2716) * fix(reference): report entries beyond the path limit as I/O failures Reference-source traversal joins each child onto the root path. A valid tree whose nested entries exceed the platform's absolute path limit (PATH_MAX 1024 on macOS) makes lstat fail with ENAMETOOLONG. The reader's filesystem error allow-list lacked that code, so filesystemFailureDetail returned undefined and the native rejection escaped both readReferenceSource and importReferenceSource instead of becoming the existing failed `io` entry. Add ENAMETOOLONG to the allow-list, matching the JavaScript application reader's list. Readable siblings stay in the graph and the unreachable entry is recorded with its OS message. Fixes #2652 Co-Authored-By: Claude Opus 5.5 * test(reference): assert the imported graph keeps readable and failed entries Co-Authored-By: Claude Opus 5.5 * fix(process): report no remaining time after a provider deadline fires Node timers can fire while performance.now() is still a fraction of a millisecond short of the monotonic deadline. ProviderStartupDeadline then reported interruption "timeout" and an aborted signal while remainingMs() returned ceil(0.x) = 1, so polling loops guarded by remainingMs() > 0 (Hopper and Ghidra startup) could run past an elapsed deadline, and the real-timer test "aborts a bounded wait when its real monotonic deadline expires" failed intermittently in CI (expected 1 to be 0). Once the deadline's own timer has classified a timeout, report zero remaining time. A deterministic regression test fires the timer with the monotonic clock 0.4 ms short of the deadline. Co-Authored-By: Claude Opus 5.5 * fix(reference): preserve path limit entry uncertainty * fix(reference): preserve unknown entry failures and identity * fix(reference): type unknown-kind exclusions and keep them out of symlink counts The metadata-failure path offers "unknown" to exclusion predicates, but the predicate type only admitted concrete kinds, which broke the build. Directory-only secret and ignore rules now also match an unknown-kind entry (it may be a directory), and the summary counts unknown entries separately instead of as symlinks. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Codex --- src/application/ReferenceSourceImport.ts | 30 +++++-- .../ReferenceSourceImportEntries.ts | 9 ++ src/domain/referenceSourceGraph.test.ts | 24 ++++++ src/domain/referenceSourceGraph.ts | 12 ++- src/domain/referenceSourceSummary.test.ts | 20 +++++ src/domain/referenceSourceSummary.ts | 4 +- src/reference/ReferenceSourceReaderEntries.ts | 13 ++- src/reference/ReferenceSourceReaderErrors.ts | 5 ++ src/reference/ReferenceSourceReaderTypes.ts | 10 ++- .../referenceSourceFailureDetails.test.ts | 85 +++++++++++++++++++ 10 files changed, 197 insertions(+), 15 deletions(-) diff --git a/src/application/ReferenceSourceImport.ts b/src/application/ReferenceSourceImport.ts index 769f8bb8c..f917f1965 100644 --- a/src/application/ReferenceSourceImport.ts +++ b/src/application/ReferenceSourceImport.ts @@ -12,7 +12,7 @@ import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.j import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js"; import { readReferenceSource } from "../reference/ReferenceSourceReader.js"; import type { - ReferenceSourceEntryKind, + ReferenceSourceExclusionKind, ReferenceSourceRead, ReferenceSourceReaderError, } from "../reference/ReferenceSourceReaderTypes.js"; @@ -111,7 +111,8 @@ const deriveInventoryState = ( entry.limitations.length > 0 || (entry.kind === "file" && entry.content_state !== "hashed") || (entry.kind === "directory" && entry.tree_state !== "enumerated") || - (entry.kind === "symlink" && entry.target_state !== "internal"), + (entry.kind === "symlink" && entry.target_state !== "internal") || + entry.kind === "unknown", ); if (partialEntry) return "partial"; const partialRelationship = input.relationships.some( @@ -142,11 +143,24 @@ const createShouldExclude = exclusions: HistoricalSourceGraphInput["exclusions"], secrets: PreparedReferenceSourceImport["secrets"], ignored: PreparedReferenceSourceImport["ignored"], - ): ((path: string, kind: ReferenceSourceEntryKind) => boolean) => + ): ((path: string, kind: ReferenceSourceExclusionKind) => boolean) => (path, kind) => { - const patternPath = kind === "directory" ? `${path}/` : path; - const secretMatch = secrets.test(patternPath); - if (secretMatch.ignored) { + // An entry of unknown kind may be a directory, so directory-only rules + // must still exclude it; this fails closed for caller-marked secrets. + const patternPaths = + kind === "directory" + ? [`${path}/`] + : kind === "unknown" + ? [path, `${path}/`] + : [path]; + const firstMatch = (matcher: typeof secrets) => { + const results = patternPaths.map((patternPath) => + matcher.test(patternPath), + ); + return results.find(({ ignored }) => ignored) ?? results[0]; + }; + const secretMatch = firstMatch(secrets); + if (secretMatch?.ignored === true) { if (!secretMatch.rule) throw new Error(`Ignored secret path has no matching rule: ${path}`); exclusions.push({ @@ -156,8 +170,8 @@ const createShouldExclude = }); return true; } - const match = ignored.test(patternPath); - if (!match.ignored) return false; + const match = firstMatch(ignored); + if (match?.ignored !== true) return false; if (!match.rule) throw new Error(`Ignored path has no matching rule: ${path}`); const reason = match.rule.mark; diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 49c6f82f5..29dbef356 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -45,6 +45,8 @@ const entryRecoveryGuidance = ( return "This directory could not be read. Check its permissions, then try again."; if (entry.kind === "symlink") return "This symbolic link could not be read safely. Check the link and its permissions, then try again."; + if (entry.kind === "unknown") + return "This entry's type could not be determined. Check the parent directory and try again."; return "This file could not be read. Check its permissions, then try again."; }; @@ -73,6 +75,13 @@ const failedEntry = ( classifications, limitations: [limitation], }; + if (entry.kind === "unknown") + return { + path: entry.path, + kind: "unknown", + classifications: ["unknown"], + limitations: [limitation], + }; return { path: entry.path, kind: "file", diff --git a/src/domain/referenceSourceGraph.test.ts b/src/domain/referenceSourceGraph.test.ts index 1286cdde6..8fe868832 100644 --- a/src/domain/referenceSourceGraph.test.ts +++ b/src/domain/referenceSourceGraph.test.ts @@ -259,6 +259,30 @@ describe("historical source graph", () => { }); describe("partial historical source graphs", () => { + it("keeps unknown-entry diagnostics outside the root commitment", () => { + const base = graphInput(); + base.inventory_state = "partial"; + base.entries[2] = { + path: "src/package.json", + kind: "unknown", + classifications: ["unknown"], + limitations: ["Permission denied at /private/first"], + }; + const graph = createHistoricalSourceGraph(base); + const changedMessage = structuredClone(base); + changedMessage.entries[2] = { + path: "src/package.json", + kind: "unknown", + classifications: ["unknown"], + limitations: ["Permission denied at /private/second"], + }; + const revised = createHistoricalSourceGraph(changedMessage); + expect(revised.root_sha256).toBe(graph.root_sha256); + expect(computeHistoricalSourceGraphSha256(revised)).not.toBe( + computeHistoricalSourceGraphSha256(graph), + ); + }); + it("keeps an unreadable symlink target unknown and out of root identity", () => { const base = graphInput(); base.inventory_state = "partial"; diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts index 197e473bf..a830e0aa0 100644 --- a/src/domain/referenceSourceGraph.ts +++ b/src/domain/referenceSourceGraph.ts @@ -111,6 +111,10 @@ const sourceEntrySchema = z.union([ sourceFileSchema, sourceDirectorySchema, sourceSymlinkSchema, + z.strictObject({ + ...entryBaseShape, + kind: z.literal("unknown"), + }), ]); const sourceRelationshipSchema = z.strictObject({ @@ -361,7 +365,8 @@ const checkCompleteInventory = ( entry.limitations.length > 0 || (entry.kind === "file" && entry.content_state !== "hashed") || (entry.kind === "directory" && entry.tree_state !== "enumerated") || - (entry.kind === "symlink" && entry.target_state !== "internal"), + (entry.kind === "symlink" && entry.target_state !== "internal") || + entry.kind === "unknown", ); const partialRelationship = graph.relationships.some( ({ parse_state, resolution }) => @@ -478,6 +483,11 @@ const rootCommitment = ( target: entry.target, target_state: entry.target_state, }; + case "unknown": + return { + kind: entry.kind, + path: entry.path, + }; } }), exclusions, diff --git a/src/domain/referenceSourceSummary.test.ts b/src/domain/referenceSourceSummary.test.ts index a0d23b0e1..3f34a073f 100644 --- a/src/domain/referenceSourceSummary.test.ts +++ b/src/domain/referenceSourceSummary.test.ts @@ -81,6 +81,26 @@ describe("committed historical-source summaries", () => { expect(graph).toEqual(original); }); + it("counts entries of unknown kind separately from symlinks", () => { + const graph = createHistoricalSourceGraph( + input([ + { + path: "deep", + kind: "unknown", + classifications: ["unknown"], + limitations: ["lstat: ENAMETOOLONG"], + }, + ]), + ); + expect(createHistoricalSourceSummary(graph, 1).counts).toMatchObject({ + entries: 1, + files: 0, + directories: 0, + symlinks: 0, + unknown: 1, + }); + }); + it("sums known bytes losslessly beyond the JSON safe integer range", () => { const graph = createHistoricalSourceGraph( input( diff --git a/src/domain/referenceSourceSummary.ts b/src/domain/referenceSourceSummary.ts index e7e99e3fa..55d24025d 100644 --- a/src/domain/referenceSourceSummary.ts +++ b/src/domain/referenceSourceSummary.ts @@ -29,6 +29,7 @@ export const createHistoricalSourceSummary = ( files: 0, directories: 0, symlinks: 0, + unknown: 0, relationships: graph.relationships.length, parse_failures: graph.parse_failures.length, exclusions: graph.exclusions.length, @@ -59,7 +60,8 @@ export const createHistoricalSourceSummary = ( } else if (entry.kind === "directory") { counts.directories += 1; treeStates[entry.tree_state] += 1; - } else counts.symlinks += 1; + } else if (entry.kind === "symlink") counts.symlinks += 1; + else counts.unknown += 1; // Canonical entries already use Unicode code-point order; filtering preserves it. if (entry.path.split("/").length <= outlineDepth) outline.push(outlineEntry(entry)); diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts index e684a2b5c..8ec5e9d80 100644 --- a/src/reference/ReferenceSourceReaderEntries.ts +++ b/src/reference/ReferenceSourceReaderEntries.ts @@ -25,7 +25,7 @@ import { import type { PendingDirectory, ReferenceSourceEntry, - ReferenceSourceEntryKind, + ReferenceSourceExclusionKind, ReferenceSourceResult, TraversalState, } from "./ReferenceSourceReaderTypes.js"; @@ -201,6 +201,13 @@ const processEntry = async ( const metadata = await readMetadata(absolute); if (isAborted(state.signal)) return { ok: false, error: cancelled() }; if (!metadata.ok) { + const excluded = applyExclusion(state.shouldExclude, path, "unknown"); + if (!excluded.ok) + return { + ok: false, + error: failure("io", "Reference source exclusion check failed"), + }; + if (excluded.value) return { ok: true, value: undefined }; state.entries.push(entryFailure(path, "unknown", "io", metadata.message)); return { ok: true, value: undefined }; } @@ -371,10 +378,10 @@ const readSymlinkTarget = async ( const applyExclusion = ( shouldExclude: - | ((path: string, kind: ReferenceSourceEntryKind) => boolean) + | ((path: string, kind: ReferenceSourceExclusionKind) => boolean) | undefined, path: string, - kind: ReferenceSourceEntryKind, + kind: ReferenceSourceExclusionKind, ): { readonly ok: true; readonly value: boolean } | { readonly ok: false } => { try { return { ok: true, value: shouldExclude?.(path, kind) === true }; diff --git a/src/reference/ReferenceSourceReaderErrors.ts b/src/reference/ReferenceSourceReaderErrors.ts index 6ddd02a3f..2c826b4c9 100644 --- a/src/reference/ReferenceSourceReaderErrors.ts +++ b/src/reference/ReferenceSourceReaderErrors.ts @@ -29,6 +29,9 @@ const FILESYSTEM_ERROR_CODES = new Set([ "EISDIR", "ELOOP", "EMFILE", + // A traversed child can exceed the platform's absolute path limit even + // though the selected root and every component are valid. + "ENAMETOOLONG", "ENFILE", "ENOENT", "ENOTDIR", @@ -44,6 +47,8 @@ export const filesystemFailureDetail = ( ): string | undefined => { if (!(cause instanceof Error)) return undefined; const code: unknown = Reflect.get(cause, "code"); + if (code === "ENAMETOOLONG") + return `${operation}: ENAMETOOLONG (entry path exceeds the operating system path limit)`; return typeof code === "string" && FILESYSTEM_ERROR_CODES.has(code) ? `${operation}: ${cause.message}` : undefined; diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.ts index 3b0a368b5..1fec38313 100644 --- a/src/reference/ReferenceSourceReaderTypes.ts +++ b/src/reference/ReferenceSourceReaderTypes.ts @@ -10,11 +10,17 @@ export type ReferenceSourceEntryKind = | "symlink" | "other"; +/** + * Kind offered to exclusion predicates. `unknown` marks an entry whose + * metadata could not be read, so its concrete kind cannot be claimed. + */ +export type ReferenceSourceExclusionKind = ReferenceSourceEntryKind | "unknown"; + export interface ReferenceSourceReaderOptions { readonly signal?: AbortSignal; readonly shouldExclude?: ( path: string, - kind: ReferenceSourceEntryKind, + kind: ReferenceSourceExclusionKind, ) => boolean; } @@ -84,7 +90,7 @@ export type TraversalState = { readonly signal?: AbortSignal; readonly shouldExclude?: ( path: string, - kind: ReferenceSourceEntryKind, + kind: ReferenceSourceExclusionKind, ) => boolean; readonly entries: ReferenceSourceEntry[]; readonly pending: PendingDirectory[]; diff --git a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts index dd87ea92a..82476d6db 100644 --- a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts +++ b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts @@ -133,3 +133,88 @@ describe("reference import native entry failure details", () => { ).toMatchObject({ kind: "file", content_state: "hashed", limitations: [] }); }); }); + +describe("reference import paths beyond the platform limit", () => { + it.skipIf(process.platform === "win32")( + "reports an entry beyond the absolute path limit as an I/O failure", + async () => { + const root = await createTestTempDirectory("rea-import-deep-path-"); + await writeFile(join(root, "kept.txt"), "k"); + // Relative creation lets the tree exceed PATH_MAX (1024 on macOS, 4096 + // on Linux) while each 200-byte component stays below NAME_MAX. + const deepTree = (action: "create" | "remove") => + promisify(execFile)(process.execPath, [ + "-e", + `const fs = require("node:fs"); + const names = Array.from({ length: 24 }, (_, i) => "d" + String(i).padStart(2, "0") + "x".repeat(197)); + process.chdir(${JSON.stringify(root)}); + if (${JSON.stringify(action)} === "create") { + for (const name of names) { fs.mkdirSync(name); process.chdir(name); } + fs.writeFileSync("deep.txt", "d"); + } else { + for (const name of names) process.chdir(name); + fs.rmSync("deep.txt"); + for (const name of [...names].reverse()) { process.chdir(".."); fs.rmdirSync(name); } + }`, + ]); + await deepTree("create"); + try { + const read = await readReferenceSource(root); + if (!read.ok) throw new Error(read.error.message); + expect(read.value.entries).toContainEqual( + expect.objectContaining({ path: "kept.txt", status: "read" }), + ); + const failed = read.value.entries.find( + (entry) => entry.status === "failed", + ); + expect(failed).toMatchObject({ + status: "failed", + code: "io", + message: expect.stringContaining("ENAMETOOLONG"), + }); + const filteredRead = await readReferenceSource(root, { + shouldExclude: (path, kind) => + path === failed?.path && kind === "directory", + }); + if (!filteredRead.ok) throw new Error(filteredRead.error.message); + expect( + filteredRead.value.entries.find( + (entry) => entry.path === failed?.path, + ), + ).toMatchObject({ status: "failed", kind: "unknown", code: "io" }); + const imported = await importTree(root); + if (!imported.ok) throw new Error(imported.error.message); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ + path: "kept.txt", + content_state: "hashed", + }), + ); + const deep = imported.value.entries.find( + (entry) => entry.path === failed?.path, + ); + expect(deep?.limitations.join(" ")).toContain( + failed?.status === "failed" ? failed.message : "missing failure", + ); + // A directory-only secret rule still applies when the kind is unknown. + const secretPattern = `${failed?.path ?? "missing"}/`; + const secret = await importReferenceSource({ + root, + caller: "native-entry-failure-detail-test", + policy: { secretPatterns: [secretPattern] }, + }); + if (!secret.ok) throw new Error(secret.error.message); + expect( + secret.value.entries.some(({ path }) => path === failed?.path), + ).toBe(false); + expect(secret.value.exclusions).toContainEqual({ + path: failed?.path, + reason: "configured-secret", + pattern: secretPattern, + }); + } finally { + await deepTree("remove"); + } + }, + ); +}); From ff56f785946c609b09c1c60232b28d7c34ac89a9 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:33:26 -0700 Subject: [PATCH 13/26] fix(uninstall): remove the REA-owned ZCode fallback registration (#2783) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(uninstall): remove the REA-owned ZCode fallback registration ZCode loads the user-scope `~/.agents/mcp.json` `mcpServers` fallback whenever its native `mcp.servers` table is empty. Uninstall removed REA only from the native table, so a REA-owned fallback entry became eligible to load again once the last native server was gone, while uninstall reported ZCode as removed and complete. Also visit the ZCode fallback as a standard `mcpServers` JSON configuration in the uninstall host. It goes through the same ownership check, `.rea.backup`, readback and restore path as every other client configuration, so other fallback servers and settings, and a foreign `rea` entry, are preserved. Its result merges into the ZCode item. Fixes #2664 Co-Authored-By: Claude Opus 5.5 * fix(uninstall): preserve the strict ZCode fallback dialect Address review findings at their producing boundaries and preserve directly affected consumers. Source inspection and git diff --check completed. Tests and automated checks were not run at the user’s explicit request. * fix(comparison): recheck historical calls-only changes (#2223) * fix(comparison): preserve incomplete function call evidence Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty. Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls. * fix(comparison): recheck historical calls-only changes --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> * fix(ci): isolate Linux Hopper process fixture inventory * fix(android): make DEX descriptor pattern portable * fix: repair the static-check failures on the default branch (#2895) * fix: repair the static-check failures on the default branch The default branch fails its own required `Static checks` job three ways, so every pull request reports that job as failed regardless of its own change. - `npm run format:check` reports `src/domain/functionComparison.test.ts` and `src/ida/IdaOperationRunner.ts`. - `npm run lint` reports a duplicate `import { join } from "node:path"` in `tests/boundary/mcp/ghidraInventoryMcp.test.ts` (lines 4 and 17). - `npm run typecheck` reports `format: "devin"` missing from the client format union in `src/application/SupportedClients.ts`, and an unchecked index in `tests/boundary/cli/clientProfileLifecycle.test.ts`, whose pair array is inferred as `string[][]`. Apply the pinned formatter, drop the duplicate import, list the `"devin"` format that its own definition already uses, and mark the pair array `as const`. Validation: `npm run check:fast` and `npm run format:check` pass. * fix(setup): keep Devin on the JSON configuration dialect --------- Co-authored-by: 吴佳浩 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Lucien <164857476+dakjdakd@users.noreply.github.com> Co-authored-by: 吴佳浩Alben <35291372+XiaoBinGan@users.noreply.github.com> Co-authored-by: 吴佳浩 --- docs/client-configuration.md | 6 +- .../ClientConfigurationDocument.ts | 3 +- src/application/SupportedClients.ts | 1 + src/application/Uninstall.ts | 15 +++- .../zcodeClientConfiguration.test.ts | 90 ++++++++++++++++++- 5 files changed, 109 insertions(+), 6 deletions(-) diff --git a/docs/client-configuration.md b/docs/client-configuration.md index d03984ec8..346ff77be 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -294,7 +294,11 @@ the first native server into an empty table, it copies the fallback's `mcpServers` entries into the native table so they keep loading, and leaves the fallback file untouched. Existing REA environment overrides are preserved when refreshing its command. A malformed or unreadable fallback file fails setup instead of -being silently suppressed; repair or remove it and rerun. ZCode disables a +being silently suppressed; repair or remove it and rerun. Uninstall removes an +REA-owned `rea` entry from both the native table and the fallback, backing up +each changed file, so an emptied native table cannot load a leftover fallback +registration; other fallback servers, settings and a foreign `rea` entry are +kept. ZCode disables a server with `enable: false` on its entry; doctor reports such a registration as stale, and rerunning setup removes the flag to repair it. Native configuration uses strict JSON; setup refuses comments, trailing commas and a BOM rather than diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index f7ab8f0e2..896442835 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -167,7 +167,7 @@ export const parseZcodeFallbackConfiguration = ( ? {} : requireObject(document.mcpServers, "ZCode fallback mcpServers"), serversPath: ["mcpServers"], - dialect: "json", + dialect: "zcode_fallback", legacyServers: {}, }; }; @@ -183,6 +183,7 @@ export const parseClientConfiguration = ( if (format === undefined || format === "unsupported") throw new TypeError("client does not have a supported MCP config format"); // DeepSeek Harness stores a Cordis patch list, not a server table. + if (format === "zcode_fallback") return parseZcodeFallbackConfiguration(text); if (format === "dsh") return parseDshCordisDocument(text); const document = parseDocument(text, format); const serversKey = clientConfigurationServersKey(format); diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 20ab3ed29..c4da675e7 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -47,6 +47,7 @@ export interface SetupClient { | "pi" | "kimi" | "hermes" + | "zcode_fallback" | "zcode" | "dsh" | "unsupported"; diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index eac36c73d..d6447a8f6 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -231,12 +231,21 @@ export const systemUninstallHost = ( const clients = supportedClients(home, platform, environment); redirection = appDataRedirection(clients); return Promise.resolve( - clients.flatMap((client) => - uninstallConfigPaths(client).map((configPath) => ({ + clients.flatMap((client) => [ + ...uninstallConfigPaths(client).map((configPath) => ({ ...client, configPath, })), - ), + ...(client.zcodeFallbackPath === undefined + ? [] + : [ + { + ...client, + configPath: client.zcodeFallbackPath, + format: "zcode_fallback" as const, + }, + ]), + ]), ); }, inspectClient: async (client) => { diff --git a/tests/boundary/filesystem/zcodeClientConfiguration.test.ts b/tests/boundary/filesystem/zcodeClientConfiguration.test.ts index 4b00a2e01..268747356 100644 --- a/tests/boundary/filesystem/zcodeClientConfiguration.test.ts +++ b/tests/boundary/filesystem/zcodeClientConfiguration.test.ts @@ -13,7 +13,10 @@ import { } from "../../../src/application/SetupClientConfiguration.js"; import { skillDestinations } from "../../../src/application/SetupSkill.js"; import { supportedClients } from "../../../src/application/SupportedClients.js"; -import { systemUninstallHost } from "../../../src/application/Uninstall.js"; +import { + runUninstall, + systemUninstallHost, +} from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; @@ -257,6 +260,91 @@ describe("ZCode .agents fallback preservation", () => { }); }); +describe("ZCode fallback uninstall", () => { + const owned = { command: command[0], args: command.slice(1) }; + const writeFallback = async (home: string, servers: object) => { + await mkdir(join(home, ".agents"), { recursive: true }); + const text = `${JSON.stringify({ ui: { keep: true }, mcpServers: servers }, null, 2)}\n`; + await writeFile(fallbackPath(home), text); + return text; + }; + + it.each(["comment", "trailing-comma", "bom"])( + "fails closed on strict fallback syntax: %s", + async (mode) => { + const { home } = await zcodeClient(); + const valid = await writeFallback(home, { + rea: owned, + other: { command: "other-server" }, + }); + const original = + mode === "comment" + ? `// unsupported comment\n${valid}` + : mode === "bom" + ? `\uFEFF${valid}` + : valid.replace(/\}\s*$/u, ",}"); + await writeFile(fallbackPath(home), original); + const result = await runUninstall(false, systemUninstallHost(home)); + expect(result.items).toContainEqual( + expect.objectContaining({ name: "zcode", status: "failed" }), + ); + expect(await readFile(fallbackPath(home), "utf8")).toBe(original); + await expect( + readFile(`${fallbackPath(home)}.rea.backup`), + ).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it("removes the REA-owned fallback entry with the native registration", async () => { + const { home, client } = await zcodeClient(); + await writeFile( + client.configPath, + JSON.stringify({ mcp: { servers: { rea: owned } } }), + ); + const original = await writeFallback(home, { + rea: owned, + other: { command: "other-server" }, + }); + + const result = await runUninstall(false, systemUninstallHost(home)); + + expect(result.status).toBe("complete"); + expect(result.items.find(({ name }) => name === "zcode")?.status).toBe( + "removed", + ); + const native = JSON.parse(await readFile(client.configPath, "utf8")); + expect(native.mcp.servers).toEqual({}); + const fallback = JSON.parse(await readFile(fallbackPath(home), "utf8")); + expect(fallback).toEqual({ + ui: { keep: true }, + mcpServers: { other: { command: "other-server" } }, + }); + expect(await readFile(`${fallbackPath(home)}.rea.backup`, "utf8")).toBe( + original, + ); + }); + + it("removes a fallback-only REA entry and keeps a foreign rea entry", async () => { + for (const [servers, expected] of [ + [{ rea: owned }, {}], + [ + { rea: { command: "someone-else" } }, + { rea: { command: "someone-else" } }, + ], + ] as const) { + const { home } = await zcodeClient(); + await writeFallback(home, servers); + + expect( + (await runUninstall(false, systemUninstallHost(home))).status, + ).toBe("complete"); + expect( + JSON.parse(await readFile(fallbackPath(home), "utf8")).mcpServers, + ).toEqual(expected); + } + }); +}); + describe("ZCode fallback admission", () => { it.each([{ PORT: 123 }, { REA_MCP_INPUT_SCHEMA_PROFILE: "unknown" }])( "refuses malformed fallback REA environment %j before native writes", From 6a7f70fd5422489e724de54d1fd6052416b9732d Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:39:38 +0800 Subject: [PATCH 14/26] test(windows): verify retained inspection ownership after rename (#2902) Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- scripts/verify-windows-native-inspection.mjs | 66 ++++++++++++-------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/scripts/verify-windows-native-inspection.mjs b/scripts/verify-windows-native-inspection.mjs index 94adaf5a6..8480589a4 100644 --- a/scripts/verify-windows-native-inspection.mjs +++ b/scripts/verify-windows-native-inspection.mjs @@ -1,11 +1,12 @@ import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; +import { spawnSync } from "node:child_process"; import { + access, copyFile, mkdir, mkdtemp, + readFile, readdir, - rmdir, rm, stat, symlink, @@ -54,7 +55,7 @@ try { join(workspace, "dist", "windows", "WindowsNativeLoader.js"), ).href; const source = `import assert from 'node:assert/strict'; -import {existsSync,mkdirSync,readdirSync,renameSync,statSync} from 'node:fs'; +import {existsSync,mkdirSync,readFileSync,readdirSync,renameSync,statSync,writeFileSync} from 'node:fs'; import {join} from 'node:path'; import {systemWindowsNativeAuthority} from ${JSON.stringify(loader)}; const temp=process.env.TEMP; @@ -64,46 +65,61 @@ assert.equal(first.retryable,true); assert.match(first.reason,/Delete owned object by handle failed.*Win32 5/s); const [name,...extra]=readdirSync(temp);assert.ok(name);assert.equal(extra.length,0); const path=join(temp,name);const identity=statSync(path,{bigint:true}); -// Ownership follows the retained NTFS handle even when the directory moves. -// A replacement at its old pathname must remain caller-owned. -const moved=path+'-renamed';renameSync(path,moved); -assert.equal(statSync(moved,{bigint:true}).ino,identity.ino); +// Delete sharing permits rename; ownership follows the retained NTFS handle. +writeFileSync(join(path,'owned-child.txt'),'owned child'); +const renamed=path+'-renamed';renameSync(path,renamed); +const moved=statSync(renamed,{bigint:true}); +assert.equal(moved.dev,identity.dev);assert.equal(moved.ino,identity.ino); mkdirSync(path);const replacement=statSync(path,{bigint:true}); assert.notEqual(replacement.ino,identity.ino); +const sentinel=join(path,'unrelated.txt');writeFileSync(sentinel,'unrelated replacement'); if(process.argv[2]!=='teardown'){ const second=systemWindowsNativeAuthority();assert.equal(second.available,true); - assert.equal(existsSync(moved),false); + assert.equal(existsSync(renamed),false); assert.deepEqual(readdirSync(temp),[name]); + assert.equal(readFileSync(sentinel,'utf8'),'unrelated replacement'); assert.equal(statSync(path,{bigint:true}).ino,replacement.ino); assert.equal(systemWindowsNativeAuthority(),second); } -console.log(JSON.stringify({mode:process.argv[2],firstFailed:true,retainedOwner:true,inode:identity.ino.toString(),replacementName:name,replacementInode:replacement.ino.toString()})); +console.log(JSON.stringify({mode:process.argv[2],firstFailed:true,retainedOwner:true,name,renamed,device:identity.dev.toString(),inode:identity.ino.toString(),replacementInode:replacement.ino.toString()})); `; const fixture = join(workspace, "probe.mjs"); await writeFile(fixture, source); for (const mode of ["retry", "teardown"]) { const temp = join(workspace, mode); await mkdir(temp); - const observed = JSON.parse( - execFileSync(process.execPath, [fixture, mode], { - encoding: "utf8", - timeout: 20_000, - windowsHide: true, - env: { ...process.env, TEMP: temp, TMP: temp }, - }), - ); - assert.equal(observed.retainedOwner, true); - assert.deepEqual(await readdir(temp), [observed.replacementName]); - const replacement = join(temp, observed.replacementName); + const child = spawnSync(process.execPath, [fixture, mode], { + encoding: "utf8", + timeout: 20_000, + windowsHide: true, + env: { ...process.env, TEMP: temp, TMP: temp }, + }); + if (child.error !== undefined) throw child.error; + assert.equal(child.status, 0, child.stderr); assert.equal( - (await stat(replacement, { bigint: true })).ino.toString(), - observed.replacementInode, + child.stderr, + "", + "Native environment teardown reported a cleanup failure", + ); + const observed = JSON.parse(child.stdout); + assert.equal(observed.mode, mode); + assert.equal(observed.firstFailed, true); + assert.equal(observed.retainedOwner, true); + await assert.rejects(access(observed.renamed), { code: "ENOENT" }); + assert.deepEqual(await readdir(temp), [observed.name]); + const replacement = join(temp, observed.name); + const identity = await stat(replacement, { bigint: true }); + assert.equal(identity.dev.toString(), observed.device); + assert.equal(identity.ino.toString(), observed.replacementInode); + assert.notEqual(identity.ino.toString(), observed.inode); + assert.deepEqual(await readdir(replacement), ["unrelated.txt"]); + assert.equal( + await readFile(join(replacement, "unrelated.txt"), "utf8"), + "unrelated replacement", ); - await rmdir(replacement); - assert.deepEqual(await readdir(temp), []); } process.stdout.write( - `${JSON.stringify({ ok: true, controlledDispositionFailure: true, retainedOwner: true, renamedOwnerCleanup: true, replacementPreserved: true, retryRecovers: true, successfulAdmissionCached: true, environmentTeardownCleansProbe: true, issue1701HostCauseReproduced: false })}\n`, + `${JSON.stringify({ ok: true, controlledDispositionFailure: true, retainedOwner: true, renamedOwnerCleansChildren: true, unrelatedReplacementPreserved: true, retryRecovers: true, successfulAdmissionCached: true, environmentTeardownCleansProbe: true, issue1701HostCauseReproduced: false })}\n`, ); } finally { await rm(workspace, { recursive: true, force: true }); From 0f4c484be552a4d5e8e1d0fa072441bfa900c815 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:43:59 +0800 Subject: [PATCH 15/26] fix(reference): preserve missing symlink target facts Treat ENOTDIR while resolving a symlink target as missing target evidence, preserving its spelling and partial inventory. Keep graph-level excluded-ancestor rejection separate from import success. The compiled boundary regression passes. --- src/reference/ReferenceSourceReaderEntries.ts | 2 +- .../filesystem/referenceSourceImport.test.ts | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts index 8ec5e9d80..a7ae837e8 100644 --- a/src/reference/ReferenceSourceReaderEntries.ts +++ b/src/reference/ReferenceSourceReaderEntries.ts @@ -357,7 +357,7 @@ const readSymlinkTarget = async ( } catch (cause: unknown) { const code = cause instanceof Error ? Reflect.get(cause, "code") : undefined; - if (code !== "ENOENT") throw cause; + if (code !== "ENOENT" && code !== "ENOTDIR") throw cause; return { status: "read", kind: "symlink", diff --git a/tests/boundary/filesystem/referenceSourceImport.test.ts b/tests/boundary/filesystem/referenceSourceImport.test.ts index f49ef6ba4..65c6d4b22 100644 --- a/tests/boundary/filesystem/referenceSourceImport.test.ts +++ b/tests/boundary/filesystem/referenceSourceImport.test.ts @@ -167,10 +167,21 @@ describe("reference source symlink import", () => { policy: { secretPatterns: [] }, excludePaths: ["secret.txt"], }); - expect(result.ok).toBe(false); - if (result.ok) return; - expect(result.error.code).toBe("parse"); - expect(result.error.message).toMatch(/recorded exclusion/u); + if (!result.ok) throw new Error(result.error.message); + expect(result.value.inventory_state).toBe("partial"); + expect( + result.value.entries.find(({ path }) => path === "alias"), + ).toMatchObject({ + kind: "symlink", + target: "secret.txt/child", + target_state: "missing", + }); + expect(result.value.exclusions).toContainEqual({ + path: "secret.txt", + entry_kind: "file", + reason: "caller-excluded", + pattern: "secret.txt", + }); }); it("preserves an unreadable target as unknown instead of a synthetic path", () => { From 04672f2b917d7ae3d71424a6abbcc2da0f24fa0b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:44:52 +0800 Subject: [PATCH 16/26] fix(javascript): preserve builtin identity and share alias traversal Track module writes and escapes at their owner, preserve capture and reassignment ordering, and share CommonJS/default namespace identity without erasing parent-slot changes. Carry write sites into ambient alias resolution and use parsed write targets across consumers. Deduplicate returned-reference work with cycle context; retain unchanged temporal selection identity so provenance memoization stays effective. Reviewed fixes include #2041, #2465, #2830, #2881, and #2948. Remove superseded mutation representations and the ineffective per-markValue dedup layer. Validation: 380 focused JavaScript tests passed under a 384 MiB heap cap, including the previously aborting 30-level/128-call alias regression. Six public CLI/MCP builtin tests passed. Exact pdf.js completed-workflow performance remains unmeasured. --- .../JavaScriptSemanticGraphBuilder.test.ts | 14 + .../JavaScriptSemanticGraphResources.test.ts | 1 + .../electronBundledRequireIdentity.test.ts | 271 +++++++ src/domain/javascript/javascriptAstValues.ts | 35 + .../javascriptBootstrapIntegrity.ts | 527 +++++++++++++ .../javascriptBuiltinResourceMethods.ts | 11 + .../javascriptBundledRequireOrigin.ts | 225 ++++++ .../javascriptElectronMemberWrites.ts | 27 +- .../javascriptIntrinsicInvocation.test.ts | 54 ++ .../javascriptIntrinsicInvocation.ts | 12 +- .../javascriptSemanticAmbientWrites.ts | 22 +- ...tSemanticAnalysis.builtin_defaults.test.ts | 663 ++++++++++++++++ ...avascriptSemanticAnalysis.dataflow.test.ts | 89 +++ ...ascriptSemanticAnalysis.empty_keys.test.ts | 15 + ...ptSemanticAnalysis.native_bindings.test.ts | 36 + .../javascript/javascriptSemanticAnalysis.ts | 71 +- .../javascriptSemanticAsyncEffects.ts | 40 +- .../javascriptSemanticChildProcesses.ts | 79 +- .../javascriptSemanticDataEffectHelpers.ts | 719 +++++++++++++++++- .../javascriptSemanticDataEffects.ts | 27 +- .../javascriptSemanticGraphSchemas.ts | 1 - .../javascriptSemanticMemberMutations.ts | 191 +++-- ...scriptSemanticMutationInitializers.test.ts | 2 +- .../javascriptSemanticMutationInitializers.ts | 154 +++- .../javascript/javascriptSemanticObjects.ts | 28 +- .../javascriptSemanticProjection.ts | 42 +- .../javascriptSemanticRequireOrigin.ts | 311 +++++++- .../javascript/javascriptSemanticResources.ts | 76 +- .../javascriptSemanticReturnedReferences.ts | 25 + .../javascript/javascriptSemanticState.ts | 28 +- .../javascriptSemanticValues.memo.test.ts | 2 +- .../javascript/javascriptSemanticValues.ts | 57 +- .../javascriptBuiltinDefaultsCli.test.ts | 243 ++++++ tests/fixtures/javascriptSemanticGraph.ts | 1 + 34 files changed, 3778 insertions(+), 321 deletions(-) create mode 100644 src/domain/javascript/electronBundledRequireIdentity.test.ts create mode 100644 src/domain/javascript/javascriptBootstrapIntegrity.ts create mode 100644 src/domain/javascript/javascriptBuiltinResourceMethods.ts create mode 100644 src/domain/javascript/javascriptBundledRequireOrigin.ts create mode 100644 src/domain/javascript/javascriptIntrinsicInvocation.test.ts diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts index 4816be96c..c15437b08 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts @@ -65,6 +65,20 @@ it("projects closure and direct interprocedural flow without execution", () => { ); }); +it("projects a destructuring assignment member target as a property write", () => { + const graph = graphFor(` + const source = { count: 1 }; + [source.count] = [2]; + `); + const operations = graph.nodes.filter( + ({ properties }) => properties.operation_kind !== undefined, + ); + expect(operations.map(({ label }) => label)).toEqual(["write:/count"]); + expect(graph.relations).not.toContainEqual( + expect.objectContaining({ relation: "reads-property" }), + ); +}); + it("keeps dynamic calls explicit and returns complete deterministic results", () => { const graph = graphFor(` const handlers = { ready() { return 1; } }; diff --git a/src/application/javascript/JavaScriptSemanticGraphResources.test.ts b/src/application/javascript/JavaScriptSemanticGraphResources.test.ts index 62771c042..29e2ad64d 100644 --- a/src/application/javascript/JavaScriptSemanticGraphResources.test.ts +++ b/src/application/javascript/JavaScriptSemanticGraphResources.test.ts @@ -351,6 +351,7 @@ const emptyAnalysis = (): JavaScriptArtifactAnalysis => ({ packages: [], json_modules: [], html_scripts: [], + html_inline_scripts: [], source_maps: [], visited_ast_nodes: 0, findings: 0, diff --git a/src/domain/javascript/electronBundledRequireIdentity.test.ts b/src/domain/javascript/electronBundledRequireIdentity.test.ts new file mode 100644 index 000000000..6b963965a --- /dev/null +++ b/src/domain/javascript/electronBundledRequireIdentity.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +// Webpack 5 standalone bootstrap with an Electron external, as emitted for +// `target: "electron-main"`. +const bootstrap = (external: string, consumer: string, entry = "n(100);") => + `(()=>{var m={45840:${external},100:(e,t,n)=>{"use strict";${consumer}}},c={};` + + `function n(t){var r=c[t];if(void 0!==r)return r.exports;var o=c[t]={exports:{}};` + + `return m[t](o,o.exports,n),o.exports}${entry}})();`; + +const EXTERNAL = 'e=>{"use strict";e.exports=require("electron")}'; +const CONSUMER = + 'const s=n(45840);new s.BrowserWindow({webPreferences:{preload:"./preload.js",contextIsolation:!1}});' + + 's.ipcMain.handle("get-connections",async()=>[]);'; + +const electronOf = (source: string) => + analyzeJavaScriptStaticSource(source).electron; + +describe("Electron identity through Webpack externals", () => { + it("follows the loader passed to a module factory", () => { + const electron = electronOf(bootstrap(EXTERNAL, CONSUMER)); + expect(electron.browser_windows).toMatchObject([ + { preload_path: "./preload.js" }, + ]); + expect(electron.ipc).toMatchObject([ + { channel: "get-connections", side: "main" }, + ]); + }); + + it("follows a direct member read from the loader call", () => { + expect( + electronOf( + bootstrap( + EXTERNAL, + 'new(n(45840).BrowserWindow)({webPreferences:{preload:"./p.js"}});', + ), + ).browser_windows, + ).toMatchObject([{ preload_path: "./p.js" }]); + }); + + it("follows the runtime loader called from the bootstrap body", () => { + expect( + electronOf( + bootstrap( + EXTERNAL, + "", + 'const s=n(45840);new s.BrowserWindow({webPreferences:{preload:"./p.js"}});', + ), + ).browser_windows, + ).toMatchObject([{ preload_path: "./p.js" }]); + }); + + it.each([ + [ + "an external overwritten after the require", + bootstrap('e=>{e.exports=require("electron");e.exports=other}', CONSUMER), + ], + [ + "a local module table entry", + bootstrap("e=>{e.exports={BrowserWindow:class{}}}", CONSUMER), + ], + [ + "a shadowed require inside the external", + bootstrap( + 'e=>{const require=x=>x;e.exports=require("electron")}', + CONSUMER, + ), + ], + [ + "a reassigned loader parameter", + bootstrap(EXTERNAL, `n=other;${CONSUMER}`), + ], + [ + "a loader-shaped call outside a proven bootstrap", + `var m={45840:${EXTERNAL}};function n(t){return other(t)}${CONSUMER}`, + ], + [ + "a different table key", + bootstrap(EXTERNAL, CONSUMER.replace("n(45840)", "n(45841)")), + ], + ])("keeps %s unresolved", (_name, source) => { + const electron = electronOf(source); + expect(electron.browser_windows).toEqual([]); + // Only direct observations come from this identity; the separate + // wrapper-substitution path keeps its own resolution basis. + expect( + electron.ipc.filter( + ({ resolution_basis: basis }) => basis !== "wrapper-substitution", + ), + ).toEqual([]); + }); +}); + +describe("Webpack externals with a tampered runtime", () => { + it.each([ + [ + "a replaced bootstrap loader", + bootstrap( + EXTERNAL, + CONSUMER, + "const o=n;n=k=>k===100?o(k):local;n(100);", + ), + ], + [ + "a generator external", + bootstrap('function*(e){e.exports=require("electron")}', CONSUMER), + ], + [ + "an async external", + bootstrap('async e=>{e.exports=require("electron")}', CONSUMER), + ], + [ + "a loader that returns something else", + bootstrap(EXTERNAL, CONSUMER).replace( + "return m[t](o,o.exports,n),o.exports}", + "return m[t](o,o.exports,n),{BrowserWindow:class{}}}", + ), + ], + [ + "a table slot replaced before loading", + bootstrap( + EXTERNAL, + CONSUMER, + "m[45840]=e=>{e.exports={BrowserWindow:class{}}};n(100);", + ), + ], + [ + "a table slot replaced through an alias", + bootstrap(EXTERNAL, CONSUMER, "const a=m;a[45840]=local;n(100);"), + ], + [ + "a slot replaced through the runtime exposure", + bootstrap(EXTERNAL, CONSUMER, "n.m=m;n.m[45840]=local;n(100);"), + ], + [ + "a cache entry planted before loading", + bootstrap(EXTERNAL, CONSUMER, "c[45840]={exports:local};n(100);"), + ], + [ + "a computed table key that may collide", + bootstrap(EXTERNAL, CONSUMER).replace( + "c={};", + "c={};m={...m,[key]:e=>{e.exports=local}};", + ), + ], + [ + "an unresolved computed key in the table", + bootstrap(EXTERNAL, CONSUMER).replace( + `45840:${EXTERNAL},`, + `45840:${EXTERNAL},[key]:e=>{e.exports=local},`, + ), + ], + ])("keeps %s unresolved", (_name, source) => { + const electron = electronOf(source); + expect(electron.browser_windows).toEqual([]); + // Only direct observations come from this identity; the separate + // wrapper-substitution path keeps its own resolution basis. + expect( + electron.ipc.filter( + ({ resolution_basis: basis }) => basis !== "wrapper-substitution", + ), + ).toEqual([]); + }); +}); + +describe("Webpack externals with a tampered loader or module object", () => { + it.each([ + [ + "cached exports rewritten through the runtime exposure", + bootstrap( + EXTERNAL, + CONSUMER, + "n.c=c;n(45840);n.c[45840].exports=local;n(100);", + ), + ], + [ + "a pre-populated cache", + bootstrap(EXTERNAL, CONSUMER).replace( + "c={};", + "c={45840:{exports:local}};", + ), + ], + [ + "exports rewritten in the returned sequence", + bootstrap(EXTERNAL, CONSUMER).replace( + "return m[t](o,o.exports,n),o.exports}", + "return m[t](o,o.exports,n),o.exports=local,o.exports}", + ), + ], + [ + "exports rewritten before the return", + bootstrap(EXTERNAL, CONSUMER).replace( + "return m[t](o,o.exports,n),o.exports}", + "m[t](o,o.exports,n);o.exports=local;return o.exports}", + ), + ], + [ + "an alias of the exposed table", + bootstrap(EXTERNAL, CONSUMER, "n.m=m;const a=n.m;a[45840]=local;n(100);"), + ], + [ + "a rewritten module key", + bootstrap(EXTERNAL, CONSUMER).replace( + "function n(t){", + "function n(t){if(t===45840)t=100;", + ), + ], + [ + "a dispatch behind a guard", + bootstrap(EXTERNAL, CONSUMER).replace( + "return m[t](o,o.exports,n),o.exports}", + "if(t===100)m[t](o,o.exports,n);return o.exports}", + ), + ], + [ + "duplicate factory parameter names", + bootstrap('function(e,e,n){e.exports=require("electron")}', CONSUMER), + ], + [ + "an accessor for module exports", + bootstrap(EXTERNAL, CONSUMER).replace( + "var o=c[t]={exports:{}}", + "var o=c[t]={get exports(){return local},set exports(v){}}", + ), + ], + ])("keeps %s unresolved", (_name, source) => { + const electron = electronOf(source); + expect(electron.browser_windows).toEqual([]); + // Only direct observations come from this identity; the separate + // wrapper-substitution path keeps its own resolution basis. + expect( + electron.ipc.filter( + ({ resolution_basis: basis }) => basis !== "wrapper-substitution", + ), + ).toEqual([]); + }); + + it.each([ + [ + "a cache cleared when the factory throws", + bootstrap(EXTERNAL, CONSUMER).replace( + "return m[t](o,o.exports,n),o.exports}", + "var i=!0;try{m[t].call(o.exports,o,o.exports,n),i=!1}finally{i&&delete c[t]}return o.exports}", + ), + ], + [ + "the runtime exposing its cache", + bootstrap(EXTERNAL, CONSUMER, "n.c=c;n(100);"), + ], + [ + "chunk installation through the runtime table", + bootstrap( + EXTERNAL, + CONSUMER, + "n.m=m;n.i=e=>{for(var k in e)n.m[k]=e[k]};n(100);", + ), + ], + [ + "a loaded flag set before returning", + bootstrap(EXTERNAL, CONSUMER).replace( + "return m[t](o,o.exports,n),o.exports}", + "return m[t].call(o.exports,o,o.exports,n),o.loaded=!0,o.exports}", + ), + ], + ])("still resolves with %s", (_name, source) => { + expect(electronOf(source).browser_windows).toMatchObject([ + { preload_path: "./preload.js" }, + ]); + }); +}); diff --git a/src/domain/javascript/javascriptAstValues.ts b/src/domain/javascript/javascriptAstValues.ts index 26ff26a49..7a4386524 100644 --- a/src/domain/javascript/javascriptAstValues.ts +++ b/src/domain/javascript/javascriptAstValues.ts @@ -94,6 +94,41 @@ export const semanticObjectPatternKeys = ( return key === null ? [] : [key]; }); +/** Return the syntactic node whose target this construct writes. */ +export const semanticWriteSiteTarget = (node: t.Node): t.Node | null => { + if (t.isAssignmentExpression(node)) return node.left; + if (t.isUpdateExpression(node)) return node.argument; + if (t.isUnaryExpression(node, { operator: "delete" })) return node.argument; + if (t.isForOfStatement(node) || t.isForInStatement(node)) return node.left; + return null; +}; + +/** Whether a write site reads the previous target value before writing. */ +export const semanticWriteSiteReadsFirst = (site: t.Node): boolean => + (t.isAssignmentExpression(site) && site.operator !== "=") || + t.isUpdateExpression(site); + +/** Find the owning write site through inert TypeScript and parenthesis wrappers. */ +export const semanticWrappedWriteSite = ( + node: t.Node, + parent: t.Node, + ancestors: readonly t.Node[], +): t.Node | null => { + const chain = + ancestors.at(-1) === parent ? ancestors : [...ancestors, parent]; + for (let index = chain.length - 1; index >= 0; index -= 1) { + const candidate = chain[index]; + if (candidate === undefined) return null; + const target = semanticWriteSiteTarget(candidate); + if (target !== null) + return unwrapJavaScriptExpression(target).node === node + ? candidate + : null; + if (unwrapJavaScriptExpression(candidate).depth === 0) return null; + } + return null; +}; + /** Visit binding and member leaves in direct and destructuring write targets. */ export function* semanticWriteTargets( root: t.Node, diff --git a/src/domain/javascript/javascriptBootstrapIntegrity.ts b/src/domain/javascript/javascriptBootstrapIntegrity.ts new file mode 100644 index 000000000..1e748df41 --- /dev/null +++ b/src/domain/javascript/javascriptBootstrapIntegrity.ts @@ -0,0 +1,527 @@ +import * as t from "@babel/types"; + +import type { javascriptBootstrapAt } from "./javascriptBootstrap.js"; +import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; +import { + resolveSemanticBindingState, + semanticStableInitializer, + type JavaScriptSemanticAnalysisState, +} from "./javascriptSemanticState.js"; +import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { argumentValue } from "./javascriptStaticAnalysisHelpers.js"; + +type Bootstrap = NonNullable>; + +/** Slots a visible literal write may have replaced; the rest stay as declared. */ +export interface JavaScriptBootstrapIntegrity { + readonly replacedKeys: ReadonlySet; +} + +interface LoaderShape { + readonly parameterId: string; + /** The single unconditional `table[key](...)` call. */ + readonly dispatch: t.CallExpression; + readonly dispatches: ReadonlySet; + readonly moduleId: string; + readonly cacheId: string | undefined; + /** Cache reads, cache writes and deletes the loader itself performs. */ + readonly cacheSites: ReadonlySet; +} + +/** + * Prove that `loader(key)` returns the exports of the factory declared at + * `table[key]`: the loader returns only the dispatched module's exports or a + * cache entry it created for that module, the table has only literal keys, + * and the table and cache are referenced only by the loader or through the + * runtime's `loader.m`/`loader.c` exposure. Literal-key writes through that + * exposure replace their slot. Returns undefined when any of this is unproven. + */ +export const proveJavaScriptBootstrapIntegrity = ( + call: t.CallExpression, + bootstrap: Bootstrap, + loaderIds: ReadonlySet, + state: JavaScriptSemanticAnalysisState, +): JavaScriptBootstrapIntegrity | undefined => { + if ( + bootstrap.table.properties.some((property) => + t.isSpreadElement(property) + ? true + : semanticStaticPropertyKey(property.key, property.computed) === null, + ) + ) + return undefined; + const loader = bootstrap.body.body.find( + (statement): statement is t.FunctionDeclaration => + t.isFunctionDeclaration(statement) && + statement.id !== null && + statement.id !== undefined && + bindingOf(state, statement.id) === bootstrap.loaderBindingId, + ); + const shape = + loader === undefined + ? undefined + : loaderShape(loader, bootstrap.tableBindingId, state); + if (shape === undefined) return undefined; + return referencesAreContained(call, { bootstrap, shape, loaderIds }, state); +}; + +const loaderShape = ( + loader: t.FunctionDeclaration, + tableId: string, + state: JavaScriptSemanticAnalysisState, +): LoaderShape | undefined => { + const [parameter] = loader.params; + const parameterId = t.isIdentifier(parameter) + ? bindingOf(state, parameter) + : undefined; + // A rewritten key would dispatch a different slot than the caller named. + if ( + parameterId === undefined || + state.bindingsById.get(parameterId)?.definitions.length !== 1 || + loader.async || + loader.generator + ) + return undefined; + const dispatches = new Set(); + const calls: t.CallExpression[] = []; + const modules = new Set(); + const returns: (t.Expression | null | undefined)[] = []; + traverseJavaScriptAst(loader.body, { + skip: (node) => t.isFunction(node), + enter: (node) => { + if (t.isReturnStatement(node)) returns.push(node.argument); + if (!t.isCallExpression(node)) return; + const dispatch = tableDispatch(node, tableId, parameterId, state); + if (dispatch === undefined) return; + dispatches.add(dispatch.table); + calls.push(node); + if (dispatch.moduleId !== undefined) modules.add(dispatch.moduleId); + }, + }); + const [moduleId] = modules; + const [dispatch] = calls; + if ( + calls.length !== 1 || + dispatch === undefined || + modules.size !== 1 || + moduleId === undefined || + !runsUnconditionally(dispatch, loader, state) + ) + return undefined; + const cache = moduleCache(moduleId, parameterId, state); + if ( + cache === null || + !moduleObjectContained(loader, moduleId, dispatch, state) + ) + return undefined; + const shape: LoaderShape = { + parameterId, + dispatch, + dispatches, + moduleId, + cacheId: cache?.cacheId, + cacheSites: new Set(cache === undefined ? [] : [cache.site]), + }; + return returns.every((argument) => + returnsModuleExports(argument, shape, state), + ) + ? withCacheReads(loader, shape, state) + : undefined; +}; + +/** `table[key](module, ...)` or `table[key].call(thisArg, module, ...)`. */ +const tableDispatch = ( + call: t.CallExpression, + tableId: string, + parameterId: string, + state: JavaScriptSemanticAnalysisState, +): { readonly table: t.Node; readonly moduleId?: string } | undefined => { + const viaCall = + t.isMemberExpression(call.callee) && + semanticStaticPropertyKey(call.callee.property, call.callee.computed) === + "call"; + const slot = + viaCall && t.isMemberExpression(call.callee) + ? call.callee.object + : call.callee; + if ( + !t.isMemberExpression(slot) || + !slot.computed || + !t.isIdentifier(slot.object) || + !t.isIdentifier(slot.property) || + bindingOf(state, slot.object) !== tableId || + bindingOf(state, slot.property) !== parameterId + ) + return undefined; + const module = call.arguments[viaCall ? 1 : 0]; + const moduleId = t.isIdentifier(module) + ? bindingOf(state, module) + : undefined; + return moduleId === undefined + ? { table: slot.object } + : { table: slot.object, moduleId }; +}; + +/** + * The module object is a fresh literal, optionally stored as `cache[key]`. + * Undefined means no cache; null means the module binding is not proven. + */ +const moduleCache = ( + moduleId: string, + parameterId: string, + state: JavaScriptSemanticAnalysisState, +): { readonly cacheId: string; readonly site: t.Node } | undefined | null => { + const binding = state.bindingsById.get(moduleId); + const initializer = semanticStableInitializer(binding, state); + if ( + binding?.definitions.length !== 1 || + initializer === undefined || + initializer.projection.length !== 0 + ) + return null; + const value = initializer.node; + if (isPlainObject(value)) return undefined; + if ( + !t.isAssignmentExpression(value, { operator: "=" }) || + !isPlainObject(value.right) + ) + return null; + const cacheId = keyedSlot(value.left, parameterId, state); + return cacheId !== undefined && startsEmpty(cacheId, state) + ? { cacheId, site: value.left } + : null; +}; + +/** A pre-populated cache could answer the early return without a dispatch. */ +const startsEmpty = ( + cacheId: string, + state: JavaScriptSemanticAnalysisState, +): boolean => { + const binding = state.bindingsById.get(cacheId); + const initializer = semanticStableInitializer(binding, state); + return ( + binding?.definitions.length === 1 && + initializer?.projection.length === 0 && + t.isObjectExpression(initializer.node) && + initializer.node.properties.length === 0 + ); +}; + +/** An object literal of ordinary data properties, so `exports` stores plainly. */ +const isPlainObject = (node: t.Node): boolean => + t.isObjectExpression(node) && + node.properties.every( + (property) => + t.isObjectProperty(property) && + !property.computed && + semanticStaticPropertyKey(property.key, false) !== "__proto__", + ); + +/** Every ancestor up to the loader body executes whenever the loader runs. */ +const runsUnconditionally = ( + dispatch: t.Node, + loader: t.FunctionDeclaration, + state: JavaScriptSemanticAnalysisState, +): boolean => { + let child: t.Node = dispatch; + let parent = state.parentsByNode.get(child); + while (parent !== undefined && child !== loader.body) { + const ordered = + t.isSequenceExpression(parent) || + t.isExpressionStatement(parent) || + t.isReturnStatement(parent) || + t.isBlockStatement(parent) || + (t.isMemberExpression(parent) && parent.object === child) || + (t.isCallExpression(parent) && parent.callee === child) || + (t.isTryStatement(parent) && parent.block === child); + if (!ordered) return false; + child = parent; + parent = state.parentsByNode.get(parent); + } + return child === loader.body; +}; + +/** + * The module object reaches only the dispatch, `.exports` reads after it, and + * writes to members other than `exports`, so nothing swaps its exports. + */ +const moduleObjectContained = ( + loader: t.FunctionDeclaration, + moduleId: string, + dispatch: t.CallExpression, + state: JavaScriptSemanticAnalysisState, +): boolean => { + let contained = true; + traverseJavaScriptAst(loader.body, { + enter: (node, parent, readAncestors) => { + if (!contained || !t.isIdentifier(node) || parent === null) return; + const ancestors = readAncestors(); + if (!t.isReferenced(node, parent, ancestors.at(-2))) return; + if (bindingOf(state, node) !== moduleId) return; + contained = + !ancestors.some((ancestor) => t.isFunction(ancestor)) && + moduleUse(node, [parent, ancestors.at(-2)], dispatch); + }, + }); + return contained; +}; + +const moduleUse = ( + node: t.Identifier, + [parent, grandparent]: readonly [t.Node, t.Node | undefined], + dispatch: t.CallExpression, +): boolean => { + if (parent === dispatch) return dispatch.arguments.includes(node); + if (!t.isMemberExpression(parent) || parent.object !== node) return false; + const key = semanticStaticPropertyKey(parent.property, parent.computed); + const written = + (t.isAssignmentExpression(grandparent) && grandparent.left === parent) || + t.isUpdateExpression(grandparent) || + t.isUnaryExpression(grandparent, { operator: "delete" }); + if (written) return key !== null && key !== "exports"; + if (key !== "exports") return false; + // `module.exports` passed to the factory, or read after the dispatch ran. + return ( + grandparent === dispatch || + (parent.start ?? -1) > (dispatch.end ?? Number.POSITIVE_INFINITY) + ); +}; + +/** Record `cache[key]` reads that initialize the loader's early-return binding. */ +const withCacheReads = ( + loader: t.FunctionDeclaration, + shape: LoaderShape, + state: JavaScriptSemanticAnalysisState, +): LoaderShape | undefined => { + if (shape.cacheId === undefined) return shape; + const sites = new Set(shape.cacheSites); + let contained = true; + traverseJavaScriptAst(loader.body, { + skip: (node) => t.isFunction(node), + enter: (node, parent) => { + if ( + !t.isMemberExpression(node) || + keyedSlot(node, shape.parameterId, state) !== shape.cacheId + ) + return; + const read = t.isVariableDeclarator(parent) && parent.init === node; + const deleted = t.isUnaryExpression(parent, { operator: "delete" }); + if (read || deleted || sites.has(node)) sites.add(node); + else contained = false; + }, + }); + return contained ? { ...shape, cacheSites: sites } : undefined; +}; + +/** Accept `module.exports` or `cache[key].exports` read through a stable binding. */ +const returnsModuleExports = ( + argument: t.Expression | null | undefined, + shape: LoaderShape, + state: JavaScriptSemanticAnalysisState, +): boolean => { + const value = t.isSequenceExpression(argument) + ? argument.expressions.at(-1) + : argument; + const before = t.isSequenceExpression(argument) + ? argument.expressions.slice(0, -1) + : []; + if ( + !before.every((expression) => + sideEffectBeforeReturn(expression, shape, state), + ) + ) + return false; + if ( + !t.isMemberExpression(value) || + !t.isIdentifier(value.object) || + semanticStaticPropertyKey(value.property, value.computed) !== "exports" + ) + return false; + const id = bindingOf(state, value.object); + if (id === shape.moduleId) return true; + if (id === undefined || shape.cacheId === undefined) return false; + const binding = state.bindingsById.get(id); + const initializer = semanticStableInitializer(binding, state); + return ( + binding?.definitions.length === 1 && + initializer?.projection.length === 0 && + keyedSlot(initializer.node, shape.parameterId, state) === shape.cacheId + ); +}; + +/** The dispatch itself, or a write to a module member other than `exports`. */ +const sideEffectBeforeReturn = ( + expression: t.Expression, + shape: LoaderShape, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (expression === shape.dispatch) return true; + if ( + !t.isAssignmentExpression(expression) || + !t.isMemberExpression(expression.left) || + !t.isIdentifier(expression.left.object) || + bindingOf(state, expression.left.object) !== shape.moduleId + ) + return false; + const key = semanticStaticPropertyKey( + expression.left.property, + expression.left.computed, + ); + return key !== null && key !== "exports"; +}; + +/** The binding of `owner` in `owner[key]` when `key` is the loader parameter. */ +const keyedSlot = ( + node: t.Node, + parameterId: string, + state: JavaScriptSemanticAnalysisState, +): string | undefined => + t.isMemberExpression(node) && + node.computed && + t.isIdentifier(node.object) && + t.isIdentifier(node.property) && + bindingOf(state, node.property) === parameterId + ? bindingOf(state, node.object) + : undefined; + +const referencesAreContained = ( + call: t.CallExpression, + { + bootstrap, + shape, + loaderIds, + }: { + readonly bootstrap: Bootstrap; + readonly shape: LoaderShape; + readonly loaderIds: ReadonlySet; + }, + state: JavaScriptSemanticAnalysisState, +): JavaScriptBootstrapIntegrity | undefined => { + const tracked = new Map([ + [bootstrap.tableBindingId, "table"], + ...(shape.cacheId === undefined ? [] : [[shape.cacheId, "cache"] as const]), + ...[...loaderIds].map((id) => [id, "loader"] as const), + ]); + const names = new Set( + [...tracked.keys()].flatMap((id) => state.bindingsById.get(id)?.name ?? []), + ); + const replacedKeys = new Set(); + let contained = true; + traverseJavaScriptAst(call, { + skip: () => !contained, + enter: (node, parent, readAncestors) => { + if (!t.isIdentifier(node) || !names.has(node.name) || parent === null) + return; + const ancestors = readAncestors(); + if (!t.isReferenced(node, parent, ancestors.at(-2))) return; + const id = bindingOf(state, node); + const role = id === undefined ? undefined : tracked.get(id); + if (role === undefined) return; + const use: Use = { node, parent, ancestors, shape, loaderIds, state }; + if (role !== "loader") contained = containedUse(role, use); + else if (exposedTableUse(use) === "escaped") contained = false; + else exposedSlotWrite(use, replacedKeys); + }, + }); + return contained ? { replacedKeys } : undefined; +}; + +interface Use { + readonly node: t.Identifier; + readonly parent: t.Node; + readonly ancestors: readonly t.Node[]; + readonly shape: LoaderShape; + readonly loaderIds: ReadonlySet; + readonly state: JavaScriptSemanticAnalysisState; +} + +/** The table is read only by the dispatch; the cache only by the loader's own slot operations. */ +const containedUse = (role: "table" | "cache", use: Use): boolean => { + if (role === "table" && use.shape.dispatches.has(use.node)) return true; + if (role === "cache" && use.shape.cacheSites.has(use.parent)) return true; + return isRuntimeExposure(use, role === "table" ? "m" : "c"); +}; + +/** `loader.m = table` or `loader.c = cache`, the runtime's own exposure. */ +const isRuntimeExposure = (use: Use, key: "m" | "c"): boolean => { + const { parent, node, loaderIds, state } = use; + return ( + t.isAssignmentExpression(parent, { operator: "=" }) && + parent.right === node && + t.isMemberExpression(parent.left) && + t.isIdentifier(parent.left.object) && + semanticStaticPropertyKey(parent.left.property, parent.left.computed) === + key && + loaderIds.has(bindingOf(state, parent.left.object) ?? "") + ); +}; + +/** + * `loader.m`/`loader.c` may only be assigned (the exposure) or indexed + * directly; any other use, such as `const a = loader.m`, lets later writes + * escape this proof. Writing a member of an indexed entry, such as + * `loader.c[id].exports = value`, can replace what the loader returns. + */ +const exposedTableUse = (use: Use): "other" | "contained" | "escaped" => { + const { parent, ancestors } = use; + if ( + !t.isMemberExpression(parent) || + parent.object !== use.node || + !["m", "c"].includes( + semanticStaticPropertyKey(parent.property, parent.computed) ?? "", + ) + ) + return "other"; + const outer = ancestors.at(-2); + const assigned = + t.isAssignmentExpression(outer, { operator: "=" }) && outer.left === parent; + const indexed = + t.isMemberExpression(outer) && outer.object === parent && outer.computed; + if (assigned) return "contained"; + if (!indexed) return "escaped"; + const member = ancestors.at(-3); + const target = ancestors.at(-4); + const entryWritten = + t.isMemberExpression(member) && + member.object === outer && + ((t.isAssignmentExpression(target) && target.left === member) || + t.isUpdateExpression(target) || + t.isUnaryExpression(target, { operator: "delete" })); + return entryWritten ? "escaped" : "contained"; +}; + +/** + * `loader.m[key] = factory` or `delete loader.c[key]` replaces a literal slot. + * Writes with runtime keys are chunk installation, which adds modules under + * their own ids; they are not treated as replacing a declared slot. + */ +const exposedSlotWrite = (use: Use, replacedKeys: Set): void => { + const { parent, ancestors } = use; + const slot = ancestors.at(-2); + const write = ancestors.at(-3); + if ( + !t.isMemberExpression(parent) || + parent.object !== use.node || + !["m", "c"].includes( + semanticStaticPropertyKey(parent.property, parent.computed) ?? "", + ) || + !t.isMemberExpression(slot) || + slot.object !== parent || + !( + (t.isAssignmentExpression(write) && write.left === slot) || + t.isUnaryExpression(write, { operator: "delete" }) || + t.isUpdateExpression(write) + ) + ) + return; + const key = slot.computed + ? argumentValue(slot.property) + : semanticStaticPropertyKey(slot.property, false); + if (key !== undefined && key !== null) replacedKeys.add(key); +}; + +const bindingOf = ( + state: JavaScriptSemanticAnalysisState, + identifier: t.Identifier, +): string | undefined => + resolveSemanticBindingState(state, identifier, identifier.name)?.bindingId; diff --git a/src/domain/javascript/javascriptBuiltinResourceMethods.ts b/src/domain/javascript/javascriptBuiltinResourceMethods.ts new file mode 100644 index 000000000..8034b902f --- /dev/null +++ b/src/domain/javascript/javascriptBuiltinResourceMethods.ts @@ -0,0 +1,11 @@ +/** Supported native resource receivers, shared by acquisition and validity owners. */ +export const BUILTIN_RESOURCE_METHODS = { + fs: ["open", "openSync", "createReadStream", "createWriteStream"], + "fs/promises": ["open"], + net: ["connect", "createConnection"], + tls: ["connect", "createConnection"], +} as const; + +export const BUILTIN_RESOURCE_SPECIFIERS = Object.keys( + BUILTIN_RESOURCE_METHODS, +); diff --git a/src/domain/javascript/javascriptBundledRequireOrigin.ts b/src/domain/javascript/javascriptBundledRequireOrigin.ts new file mode 100644 index 000000000..4bec81042 --- /dev/null +++ b/src/domain/javascript/javascriptBundledRequireOrigin.ts @@ -0,0 +1,225 @@ +import * as t from "@babel/types"; + +import { javascriptBootstrapAt } from "./javascriptBootstrap.js"; +import { proveJavaScriptBootstrapIntegrity } from "./javascriptBootstrapIntegrity.js"; +import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; +import { + semanticMemberChain, + semanticRequireOrigin, + type JavaScriptRequireOrigin, +} from "./javascriptSemanticRequireOrigin.js"; +import { + resolveSemanticBindingState, + semanticResolutionBlocked, + type JavaScriptSemanticAnalysisState, +} from "./javascriptSemanticState.js"; +import { + argumentValue, + moduleFactory, + modulePropertyName, +} from "./javascriptStaticAnalysisHelpers.js"; + +/** Loader bindings and literal externals proven for one standalone bootstrap. */ +interface BootstrapExternals { + /** The runtime loader and every factory parameter it is passed as. */ + readonly loaderBindingIds: ReadonlySet; + /** Table keys whose factory only re-exports one unshadowed literal require; null when a key repeats. */ + readonly externals: ReadonlyMap; +} + +const proofs = new WeakMap< + JavaScriptSemanticAnalysisState, + WeakMap +>(); + +/** + * Recover a literal require reached through a proven Webpack bootstrap. + * + * Webpack emits externals as `{45840: e => { e.exports = require("electron") }}` + * and consumers call the runtime loader, `n(45840)`, instead of `require`. The + * origin is accepted only when the loader binding is the bootstrap's own loader + * or a factory parameter that the bootstrap proves it passes the loader as, and + * the keyed factory consists solely of that re-export. + */ +export const semanticBundledRequireOrigin = ( + node: t.Node | null | undefined, + state: JavaScriptSemanticAnalysisState, +): JavaScriptRequireOrigin | undefined => { + const chain = semanticMemberChain(node); + const current = chain?.root; + if ( + chain === undefined || + !t.isCallExpression(current) || + !t.isIdentifier(current.callee) || + current.arguments.length !== 1 + ) + return undefined; + const key = argumentValue(current.arguments[0]); + if (key === undefined) return undefined; + const loader = current.callee; + if (semanticResolutionBlocked(state, loader, loader.name)) return undefined; + const binding = resolveSemanticBindingState(state, loader, loader.name); + if ( + binding === undefined || + binding.definitions.length !== 1 || + binding.definitions[0]?.kind === "assignment" + ) + return undefined; + for ( + let ancestor = state.parentsByNode.get(current); + ancestor !== undefined; + ancestor = state.parentsByNode.get(ancestor) + ) { + if (!t.isCallExpression(ancestor)) continue; + const proof = bootstrapExternals(ancestor, state); + if (proof === null || !proof.loaderBindingIds.has(binding.bindingId)) + continue; + const specifier = proof.externals.get(key); + return typeof specifier === "string" + ? { + origin: { + specifier, + importedPath: chain.members, + resolutionMode: "require", + }, + call: current, + } + : undefined; + } + return undefined; +}; + +const bootstrapExternals = ( + call: t.CallExpression, + state: JavaScriptSemanticAnalysisState, +): BootstrapExternals | null => { + let byCall = proofs.get(state); + if (byCall === undefined) { + byCall = new WeakMap(); + proofs.set(state, byCall); + } + const cached = byCall.get(call); + if (cached !== undefined) return cached; + const proof = proveBootstrapExternals(call, state); + byCall.set(call, proof); + return proof; +}; + +const proveBootstrapExternals = ( + call: t.CallExpression, + state: JavaScriptSemanticAnalysisState, +): BootstrapExternals | null => { + const bootstrap = javascriptBootstrapAt(call, state); + // A replaced loader no longer dispatches the declared table. + if ( + bootstrap === undefined || + !singleDefinition(state, bootstrap.loaderBindingId, "function") + ) + return null; + const loaderBindingIds = new Set([bootstrap.loaderBindingId]); + const factories = bootstrap.table.properties.flatMap((property) => { + const factory = moduleFactory(property); + return factory === undefined ? [] : [{ property, factory }]; + }); + for (const { factory } of factories) { + // The bootstrap proves `table[key](module, module.exports, loader)`. + const loaderParameter = factory.params[2]; + const bindingId = t.isIdentifier(loaderParameter) + ? resolveSemanticBindingState( + state, + loaderParameter, + loaderParameter.name, + )?.bindingId + : undefined; + if ( + bindingId !== undefined && + singleDefinition(state, bindingId, "parameter") + ) + loaderBindingIds.add(bindingId); + } + const integrity = proveJavaScriptBootstrapIntegrity( + call, + bootstrap, + loaderBindingIds, + state, + ); + if (integrity === undefined) return null; + const externals = new Map(); + for (const { property, factory } of factories) { + const key = modulePropertyName(property); + const specifier = integrity.replacedKeys.has(key) + ? undefined + : externalSpecifier(factory, state); + if (externals.has(key)) externals.set(key, null); + else externals.set(key, specifier ?? null); + } + return { loaderBindingIds, externals }; +}; + +/** + * Accept only `module.exports = require("specifier")` as the whole body of an + * ordinary factory; generator, async and accessor factories do not run it. + */ +const externalSpecifier = ( + factory: t.FunctionExpression | t.ArrowFunctionExpression | t.ObjectMethod, + state: JavaScriptSemanticAnalysisState, +): string | undefined => { + const moduleParameter = factory.params[0]; + const names = factory.params.map((parameter) => + t.isIdentifier(parameter) ? parameter.name : null, + ); + if ( + !t.isIdentifier(moduleParameter) || + // A duplicated name binds the later argument, not the module object. + new Set(names).size !== names.length || + factory.generator || + factory.async || + (t.isObjectMethod(factory) && factory.kind !== "method") + ) + return undefined; + let expression: t.Node | undefined; + if (t.isBlockStatement(factory.body)) { + const [statement] = factory.body.body; + if (factory.body.body.length !== 1 || !t.isExpressionStatement(statement)) + return undefined; + expression = statement.expression; + } else expression = factory.body; + if ( + !t.isAssignmentExpression(expression, { operator: "=" }) || + !t.isMemberExpression(expression.left) || + !t.isIdentifier(expression.left.object) || + semanticStaticPropertyKey( + expression.left.property, + expression.left.computed, + ) !== "exports" + ) + return undefined; + const moduleBinding = resolveSemanticBindingState( + state, + moduleParameter, + moduleParameter.name, + ); + const target = resolveSemanticBindingState( + state, + expression.left.object, + expression.left.object.name, + ); + if ( + moduleBinding === undefined || + target?.bindingId !== moduleBinding.bindingId + ) + return undefined; + const required = semanticRequireOrigin(expression.right, state)?.origin; + return required !== undefined && required.importedPath.length === 0 + ? required.specifier + : undefined; +}; + +const singleDefinition = ( + state: JavaScriptSemanticAnalysisState, + bindingId: string, + kind: "function" | "parameter", +): boolean => { + const definitions = state.bindingsById.get(bindingId)?.definitions; + return definitions?.length === 1 && definitions[0]?.kind === kind; +}; diff --git a/src/domain/javascript/javascriptElectronMemberWrites.ts b/src/domain/javascript/javascriptElectronMemberWrites.ts index 14741af36..9ca7c2d89 100644 --- a/src/domain/javascript/javascriptElectronMemberWrites.ts +++ b/src/domain/javascript/javascriptElectronMemberWrites.ts @@ -2,6 +2,7 @@ import * as t from "@babel/types"; import { semanticStaticPropertyKey, + semanticWriteSiteTarget, semanticWriteTargets, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; @@ -62,6 +63,7 @@ export function* collectJavaScriptIdentityWritesSteps( origin: ( root: t.Node, kind: "binding" | "member" | "opaque-call", + writeSite: t.Node, ) => readonly (readonly (string | null)[])[], intrinsicInvocation: ( node: t.CallExpression | t.OptionalCallExpression, @@ -88,7 +90,8 @@ export function* collectJavaScriptIdentityWritesSteps( if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) { const invocation = intrinsicInvocation(node); if (invocation?.kind === "opaque") { - for (const scope of origin(node, "opaque-call")) retain(node, scope); + for (const scope of origin(node, "opaque-call", node)) + retain(node, scope); return; } const callee = invocation?.path; @@ -117,11 +120,11 @@ export function* collectJavaScriptIdentityWritesSteps( // A known mutator with an opaque argument list can overwrite any // captured intrinsic. Do not turn an unexamined target into proof. if (target === undefined || t.isSpreadElement(target)) { - for (const scope of origin(node, "opaque-call")) + for (const scope of origin(node, "opaque-call", node)) retain(node, scope); } if (t.isNode(target) && !t.isSpreadElement(target)) { - const identities = origin(target, "member"); + const identities = origin(target, "member", node); if (identities.length > 0) { const key = keyedMutators.has(method ?? "") ? [staticString(arguments_?.[1]) ?? null] @@ -129,26 +132,18 @@ export function* collectJavaScriptIdentityWritesSteps( for (const identity of identities) retain(node, [...identity, ...key]); } else if (!freshObject(target)) { - for (const scope of origin(node, "opaque-call")) + for (const scope of origin(node, "opaque-call", node)) retain(node, scope); } } } } } - const target = t.isAssignmentExpression(node) - ? node.left - : t.isUpdateExpression(node) - ? node.argument - : t.isUnaryExpression(node, { operator: "delete" }) - ? node.argument - : t.isForOfStatement(node) || t.isForInStatement(node) - ? node.left - : undefined; - if (target === undefined) return; + const target = semanticWriteSiteTarget(node); + if (target === null) return; for (const leaf of semanticWriteTargets(target)) { if (t.isIdentifier(leaf)) { - for (const identity of origin(leaf, "binding")) + for (const identity of origin(leaf, "binding", node)) retain(node, identity); continue; } @@ -159,7 +154,7 @@ export function* collectJavaScriptIdentityWritesSteps( ) continue; const key = semanticStaticPropertyKey(member.property, member.computed); - for (const identity of origin(member.object, "member")) + for (const identity of origin(member.object, "member", node)) retain(node, [...identity, key]); } }, diff --git a/src/domain/javascript/javascriptIntrinsicInvocation.test.ts b/src/domain/javascript/javascriptIntrinsicInvocation.test.ts new file mode 100644 index 000000000..66c58bff6 --- /dev/null +++ b/src/domain/javascript/javascriptIntrinsicInvocation.test.ts @@ -0,0 +1,54 @@ +import { expect, it } from "vitest"; +import * as t from "@babel/types"; + +import { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { indexJavaScriptBindingsSteps } from "./javascriptSemanticAnalysis.js"; +import { + completeSemanticSteps, + traverseJavaScriptAst, +} from "./javascriptSemanticTraversal.js"; + +const invocationKinds = (source: string) => { + const parsed = parseJavaScriptSource(source); + if (parsed === null) throw new Error("invocation fixture did not parse"); + const state = completeSemanticSteps( + indexJavaScriptBindingsSteps(parsed.program), + ); + const interop = createSemanticModuleInterop(state); + const invocations: (ReturnType | null)[] = + []; + traverseJavaScriptAst(parsed.program, { + enter: (node) => { + if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) + invocations.push(interop.intrinsicInvocation(node) ?? null); + }, + }); + return invocations; +}; + +it.each([ + 'const change = Object.defineProperty.call; change(null,Object,"create",{value:customCreate});', + 'const {apply:change} = Object.defineProperty; change(null,[Object,"create",{value:customCreate}]);', + 'const change = Reflect.apply.call; change(null,Object.defineProperty,null,[Object,"create",{value:customCreate}]);', +])( + "keeps a bare invocation member alias unknown, not effect-free: %s", + (source) => { + expect(invocationKinds(source)).toEqual([ + expect.objectContaining({ kind: "opaque" }), + ]); + }, +); + +it("keeps a comma-expression intrinsic call unknown when its receiver is erased", () => { + expect( + invocationKinds( + '(0, Object.defineProperty.call)(null, Object, "create", {value: customCreate});', + ), + ).toEqual([ + expect.objectContaining({ + kind: "opaque", + path: ["intrinsic", "Object", "defineProperty", "call"], + }), + ]); +}); diff --git a/src/domain/javascript/javascriptIntrinsicInvocation.ts b/src/domain/javascript/javascriptIntrinsicInvocation.ts index b46f28b42..6aeb5ab78 100644 --- a/src/domain/javascript/javascriptIntrinsicInvocation.ts +++ b/src/domain/javascript/javascriptIntrinsicInvocation.ts @@ -81,6 +81,15 @@ export const resolveJavaScriptIntrinsicInvocation = ( let explicitReceiver = false; for (;;) { callee = unwrapJavaScriptExpression(callee).node; + if (t.isSequenceExpression(callee)) { + const finalExpression = callee.expressions.at(-1); + if (finalExpression === undefined) return undefined; + // The comma operator returns the final value without its member receiver. + // Keep that lost receiver explicit so intrinsic call/apply stays unknown. + explicitReceiver = true; + callee = finalExpression; + continue; + } if (visited.has(callee)) return undefined; visited.add(callee); let path = intrinsicPath(callee); @@ -129,7 +138,8 @@ export const resolveJavaScriptIntrinsicInvocation = ( : t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee) ? callee.object : undefined; - if (!t.isExpression(callable)) return undefined; + if (!t.isExpression(callable)) + return { kind: "opaque", path, arguments: undefined }; callee = callable; receiver = arguments_?.[0]; explicitReceiver = true; diff --git a/src/domain/javascript/javascriptSemanticAmbientWrites.ts b/src/domain/javascript/javascriptSemanticAmbientWrites.ts index dd24f3541..f7c4a8e7a 100644 --- a/src/domain/javascript/javascriptSemanticAmbientWrites.ts +++ b/src/domain/javascript/javascriptSemanticAmbientWrites.ts @@ -16,6 +16,7 @@ import { type JavaScriptSemanticAnalysisState, } from "./javascriptSemanticState.js"; import { isImportMetaUrl } from "./javascriptStaticAnalysisHelpers.js"; +import { semanticMutationInitializers } from "./javascriptSemanticMutationInitializers.js"; const GLOBAL_OBJECT_NAMES = ["globalThis", "global", "window", "self"]; @@ -51,8 +52,8 @@ const createAmbientPathResolver = ( possibleWrites: boolean, ) => { const cached = new WeakMap(); - return (node: t.Node): readonly AmbientPath[] => { - const prior = cached.get(node); + return (node: t.Node, capture?: t.Node): readonly AmbientPath[] => { + const prior = possibleWrites ? undefined : cached.get(node); if (prior !== undefined) return prior; const result = new Map(); const pending: { node: t.Node; suffix: AmbientProjection }[] = [ @@ -126,7 +127,13 @@ const createAmbientPathResolver = ( if (binding === undefined) continue; const stable = semanticStableInitializer(binding, state); const initializers = possibleWrites - ? binding.initializers + ? capture === undefined + ? binding.initializers + : semanticMutationInitializers( + binding, + capture, + state.parentsByNode, + ).initializers : stable === undefined ? [] : [stable]; @@ -221,7 +228,7 @@ const createAmbientPathResolver = ( } } const paths = [...result.values()]; - cached.set(node, paths); + if (!possibleWrites) cached.set(node, paths); return paths; }; }; @@ -248,7 +255,7 @@ export function* indexJavaScriptAmbientWritesSteps( }); yield* collectJavaScriptIdentityWritesSteps( program, - (root, kind) => { + (root, kind, writeSite) => { if (kind === "opaque-call") return []; // Reassigning a local alias changes the binding, not the referenced object. if (kind === "binding") @@ -256,13 +263,14 @@ export function* indexJavaScriptAmbientWritesSteps( isUnshadowedGlobal(root, state, root.name) ? [["global", root.name]] : []; - return ambientWrites(root); + return ambientWrites(root, writeSite); }, interop.intrinsicInvocation, interop.staticString, interop.isFreshObject, - (_node, path) => { + (node, path) => { state.ambientWrites.retain(path); + state.ambientWriteEffects.push({ path, node }); }, ); } diff --git a/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts index 15a4619c3..42c7cce6e 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts @@ -2,6 +2,428 @@ import { expect, it } from "vitest"; import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; +it("shares required process writes with globals while keeping earlier observations", () => { + const ir = analyzeJavaScriptSemantics(`const proc=require('process'); + process.cwd(); const before=proc.env.BEFORE; + proc.env={}; const after=process.env.AFTER;`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["BEFORE"]); +}); + +it("does not propagate replacement-object writes from import-equals direct origins", () => { + const ir = analyzeJavaScriptSemantics(`import fs = require('fs'); + const other=require('fs'); fs={}; fs.readFileSync=custom; other.readFileSync('kept');`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it("retains original import-equals aliases captured before reassignment", () => { + const ir = analyzeJavaScriptSemantics(`import fs = require('fs'); + const alias=fs; const other=require('fs'); fs={}; alias.readFileSync=custom; other.readFileSync('wrong');`); + expect(ir.configurationOperations).toEqual([]); +}); + +it("keeps computed namespace calls as unknown receiver escapes", () => { + const ir = analyzeJavaScriptSemantics( + `const fs=require('fs'); fs.hook=custom; fs[key](); fs.readFileSync('wrong');`, + ); + expect(ir.configurationOperations).toEqual([]); +}); + +it.each(["(0, fs.readFileSync)", "(assigned = fs.readFileSync)"])( + "checks captured function validity through %s", + (value) => { + const ir = analyzeJavaScriptSemantics(`const fs=require('fs'); let assigned; + const saved=${value}; fs.readFileSync=custom; const late=${value}; saved('kept'); late('wrong');`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); + }, +); + +it.each(['process.env={MODE:"local"};', "process.argv=[];", "patch(process);"])( + "shares unbound process object effects with required aliases: %s", + (write) => { + const ir = + analyzeJavaScriptSemantics(`${write} const proc=require('process'); + ${write.includes("argv") ? "const arg=proc.argv[2];" : "const mode=proc.env.MODE;"}`); + expect(ir.configurationOperations).toEqual([]); + }, +); + +it.each([ + "function local(process) {process.env={};}", + "process={}; process.env={};", +])("keeps unrelated process replacements separate: %s", (write) => { + const ir = analyzeJavaScriptSemantics( + `${write} const proc=require('process'); const mode=proc.env.MODE;`, + ); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["MODE"]); +}); + +it("preserves exports captured before a later loader overwrite", () => { + const ir = analyzeJavaScriptSemantics(` + const fs=require('node:fs'); fs.readFileSync('before'); + const read=fs.readFileSync; + require('fs').readFileSync('direct-before'); + require=replacement; + fs.readFileSync('after'); read('saved'); + const late=require('fs'); late.readFileSync('wrong'); + require('fs').readFileSync('also-wrong'); + `); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual([ + "before", + "direct-before", + "after", + "saved", + ]); +}); + +it.each([ + `for (;;) { require('fs').readFileSync('wrong'); require=replacement; }`, + `function run() { require('fs').readFileSync('wrong'); require=replacement; }`, + `require=replacement; const fs=require('fs'); fs.readFileSync('wrong'); require=again;`, +])( + "keeps repeated or already replaced loader captures uncertain: %s", + (source) => { + expect(analyzeJavaScriptSemantics(source).configurationOperations).toEqual( + [], + ); + }, +); + +it.each([ + ["fs.readFileSync = custom;", "other.readFileSync('wrong');"], + ["other.readFileSync = custom;", "fs.readFileSync('wrong');"], + ["patch(fs);", "other.readFileSync('wrong');"], +])( + "shares require-mode import-equals effects in both directions: %s", + (write, call) => { + const ir = analyzeJavaScriptSemantics(`import fs = require('node:fs'); + const other = require('fs'); other.readFileSync('before'); ${write} ${call}`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual([ + "before", + ]); + }, +); + +it.each([ + "import {readFileSync as read} from 'node:fs';", + "const {readFileSync:read}=require('node:fs');", + "const fs=require('fs'); const read=fs.readFileSync;", +])( + "retains captured native function calls after argument escape: %s", + (capture) => { + const ir = analyzeJavaScriptSemantics( + `${capture} observe(read); read('kept');`, + ); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); + }, +); + +it("distinguishes escaping a method value from escaping its namespace", () => { + const ir = analyzeJavaScriptSemantics(`const fs=require('fs'); + observe(fs.readFileSync); fs.readFileSync('kept'); + observe(fs); fs.readFileSync('wrong');`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it("still invalidates escaped process configuration containers", () => { + const ir = analyzeJavaScriptSemantics(`const proc=require('process'); + observe(proc.env); const mode=proc.env.MODE;`); + expect(ir.configurationOperations).toEqual([]); +}); + +it("retains direct require effects without poisoning later cached receivers", () => { + const ir = analyzeJavaScriptSemantics(` + require('node:fs').readFileSync('first'); + const fs = require('fs'); fs.readFileSync('second'); + require('node:fs').createReadStream('stream'); + require('node:child_process').spawn('tool', []); + require('node:timers').setTimeout(callback, 1); + require('node:http').get('http://example.test'); + require('fs').readFileSync = custom; + require('node:fs').readFileSync('wrong'); fs.readFileSync('wrong'); + `); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual([ + "first", + "second", + ]); + expect(ir.resourceOperations).toHaveLength(1); + expect(ir.childProcessSpawns).toHaveLength(1); + expect(ir.timerOperations).toHaveLength(1); + expect(ir.requestOperations).toHaveLength(1); +}); + +it.each([ + "const fs = require('fs'); fs.promises.open = custom;", + "require('node:fs').promises.open = custom;", + "const fsp = require('fs/promises'); fsp.open = custom;", +])("shares fs.promises object writes with fs/promises: %s", (write) => { + const ir = analyzeJavaScriptSemantics(` + const before = require('node:fs/promises'); before.open('before'); + ${write} + const after = require('node:fs/promises'); after.open('wrong'); + `); + expect(ir.resourceOperations).toHaveLength(1); +}); + +it.each([ + "require = () => ({ readFileSync() {} });", + "({ loader: require } = other);", + "for (require of loaders) {}", +])("rejects provenance from an overwritten ambient loader: %s", (write) => { + const ir = analyzeJavaScriptSemantics(`${write} + const fs = require('node:fs'); fs.readFileSync('wrong'); + require('node:fs').readFileSync('also-wrong'); + `); + expect(ir.configurationOperations).toEqual([]); + expect( + ir.bindings.find(({ name }) => name === "fs")?.provenance.status, + ).not.toBe("module"); +}); + +it("keeps local loader reassignment separate from the ambient loader", () => { + const ir = analyzeJavaScriptSemantics(` + function local(require) { require = replacement; } + const fs = require('node:fs'); fs.readFileSync('kept'); + `); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it("keeps wrapper require separate from global-object property writes", () => { + const ir = analyzeJavaScriptSemantics(` + globalThis.require = replacement; + function polyfill(key) { globalThis[key] = replacement; } + require('node:fs').readFileSync('kept'); + `); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it.each(["import fs from 'node:fs';", "const fs = require('node:fs');"])( + "retains reads and saved methods before a later write: %s", + (declaration) => { + const ir = analyzeJavaScriptSemantics(`${declaration} + fs.readFileSync('before'); + const saved = fs.readFileSync; + const { readFileSync: destructured } = fs; + const namespace = fs; + fs.readFileSync = custom; + saved('saved'); destructured('destructured'); + namespace.readFileSync('wrong'); fs.readFileSync('wrong'); + `); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual([ + "before", + "saved", + "destructured", + ]); + }, +); + +it.each([ + "const { readFileSync: read } = require('fs');", + "const read = require('fs').readFileSync;", +])("checks cached module writes at the projected capture: %s", (capture) => { + const ir = analyzeJavaScriptSemantics(` + const fs = require('node:fs'); + const saved = require('fs').readFileSync; + fs.readFileSync = custom; + ${capture} + read('wrong'); saved('saved'); + `); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["saved"]); +}); + +it("retains captured methods across later escapes without freezing a namespace alias", () => { + const ir = analyzeJavaScriptSemantics(` + const fs = require('node:fs'); const saved = require('fs').readFileSync; + const namespace = require('fs'); patch(fs); + const late = require('fs').readFileSync; + saved('saved'); late('wrong'); namespace.readFileSync('wrong'); + `); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["saved"]); +}); + +it("keeps writes in repeated callable and loop activations conservative", () => { + const ir = analyzeJavaScriptSemantics(` + const fs = require('node:fs'); + function run() { fs.readFileSync('wrong'); fs.readFileSync = custom; } + for (;;) { const read = require('fs').readFileSync; read('wrong'); fs.readFileSync = custom; } + `); + expect(ir.configurationOperations).toEqual([]); +}); + +it("does not propagate a reassigned local object's writes into cached exports", () => { + const ir = analyzeJavaScriptSemantics( + `let a = require('node:fs'); const b = require('node:fs'); a = {}; a.readFileSync = custom; b.readFileSync('kept');`, + ); + expect(ir.configurationOperations).toMatchObject([{ key: "kept" }]); +}); + +it("shares repeated require mutations for every built-in effect family", () => { + const ir = analyzeJavaScriptSemantics(` + const fs = require('fs'); const otherFs = require('node:fs'); fs.createReadStream = custom; otherFs.createReadStream('wrong'); + const cp = require('child_process'); const otherCp = require('node:child_process'); cp.spawn = custom; otherCp.spawn('wrong', []); + const timers = require('timers'); const otherTimers = require('node:timers'); timers.setTimeout = custom; otherTimers.setTimeout(callback, 1); + const http = require('http'); const otherHttp = require('node:http'); http.get = custom; otherHttp.get('http://wrong.test'); + const proc = require('process'); const otherProc = require('node:process'); proc.env = replacement; const mode = otherProc.env.MODE; + `); + expect(ir.configurationOperations).toEqual([]); + expect(ir.resourceOperations).toEqual([]); + expect(ir.childProcessSpawns).toEqual([]); + expect(ir.timerOperations).toEqual([]); + expect(ir.requestOperations).toEqual([]); +}); + +it.each(["node:fs", "fs"])( + "shares repeated built-in require identity across %s", + (specifier) => { + const ir = analyzeJavaScriptSemantics( + `const a = require('node:fs'); const b = require('${specifier}'); const alias = b; a.readFileSync = custom; b.createReadStream('kept'); alias.readFileSync('wrong');`, + ); + expect(ir.configurationOperations).toEqual([]); + expect(ir.resourceOperations).toMatchObject([ + { method: "createReadStream" }, + ]); + }, +); + +it("shares unknown escapes across repeated requires without losing earlier effects", () => { + const ir = analyzeJavaScriptSemantics( + `const a = require('node:fs'); const b = require('fs'); b.readFileSync('before'); patch(a); b.readFileSync('after');`, + ); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["before"]); +}); + +it("preserves exact default ESM import mutation paths", () => { + const ir = analyzeJavaScriptSemantics( + `import fs from 'node:fs'; fs.writeFileSync = custom; fs.readFileSync('kept'); fs.writeFileSync('replaced');`, + ); + expect(ir.configurationOperations).toMatchObject([ + { kind: "file", key: "kept" }, + ]); +}); + +it("retains built-in effects before later top-level escapes", () => { + const ir = analyzeJavaScriptSemantics(` + const fs = require('node:fs'); const cp = require('node:child_process'); const timers = require('node:timers'); const http = require('node:http'); + fs.readFileSync('before'); fs.createReadStream('before'); cp.spawn('tool', []); timers.setTimeout(callback, 1); http.get('http://example.test'); + patch(fs); patch(cp); patch(timers); patch(http); + fs.readFileSync('after'); fs.createReadStream('after'); cp.spawn('wrong', []); timers.setTimeout(callback, 1); http.get('http://wrong.test'); + `); + expect(ir.configurationOperations).toMatchObject([{ key: "before" }]); + expect(ir.resourceOperations).toHaveLength(1); + expect(ir.childProcessSpawns).toMatchObject([{ command: "tool" }]); + expect(ir.timerOperations).toHaveLength(1); + expect(ir.requestOperations).toMatchObject([ + { endpoint: "http://example.test" }, + ]); +}); + +it.each(["env", "argv"])( + "does not treat a replaced process %s as configuration", + (family) => { + const ir = analyzeJavaScriptSemantics( + `const proc = require('node:process'); proc.${family} = replacement; const value = proc.${family}[${family === "env" ? "'MODE'" : "2"}];`, + ); + expect(ir.configurationOperations).toEqual([]); + }, +); + +it("does not infer temporal safety across repeated callable or loop activations", () => { + const ir = analyzeJavaScriptSemantics( + `const fs = require('node:fs'); function run() { fs.readFileSync('callable'); patch(fs); } for (;;) { fs.readFileSync('loop'); patch(fs); }`, + ); + expect(ir.configurationOperations).toEqual([]); +}); + +it("does not recover a replaced method through a subsequent alias", () => { + const ir = analyzeJavaScriptSemantics( + `const fs = require('node:fs'); fs.readFileSync = custom; const read = fs.readFileSync; read('config.json');`, + ); + expect(ir.configurationOperations).toEqual([]); +}); + +it.each([ + "fs.readFileSync = custom; fs.createReadStream = custom; cp.spawn = custom; timers.setTimeout = custom; http.get = custom;", + "patch(fs); patch(cp); patch(timers); patch(http);", + "const fsAlias = fs; fsAlias.readFileSync = custom; fsAlias.createReadStream = custom; const cpAlias = cp; cpAlias.spawn = custom; timers[key] = custom; delete http.get;", +])("excludes replaced or escaped CommonJS methods: %s", (mutation) => { + const ir = analyzeJavaScriptSemantics(` + const fs = require('node:fs'); const cp = require('node:child_process'); + const timers = require('node:timers'); const http = require('node:http'); + ${mutation} + fs.readFileSync('config.json'); fs.createReadStream('config.json'); + cp.spawn('tool', []); timers.setTimeout(callback, 1); http.get('http://example.test'); + `); + expect(ir.configurationOperations).toEqual([]); + expect(ir.resourceOperations).toEqual([]); + expect(ir.childProcessSpawns).toEqual([]); + expect(ir.timerOperations).toEqual([]); + expect(ir.requestOperations).toEqual([]); +}); + +it("preserves unaffected CommonJS methods after sibling property mutation", () => { + const ir = analyzeJavaScriptSemantics( + `const fs = require('node:fs'); fs.writeFileSync = custom; fs.readFileSync('config.json');`, + ); + expect(ir.configurationOperations).toMatchObject([ + { kind: "file", key: "config.json" }, + ]); +}); + +it.each([ + `const fs = require("node:fs"); const cp = require("node:child_process"); + const timers = require("node:timers"); const http = require("node:http"); + const read = fs.readFileSync; const launch = cp.spawn; + const schedule = timers.setTimeout; const get = http.get;`, + `const fs = require("fs"); const { readFileSync: read } = fs; + const { spawn: launch } = require("child_process"); + const { setTimeout: schedule } = require("timers"); + const { get } = require("http");`, +])("recovers builtin effects from CommonJS provenance: %s", (declarations) => { + const ir = analyzeJavaScriptSemantics(`${declarations} + const text = read("config.json", "utf8"); + const stream = fs.createReadStream("config.json"); stream.destroy(); + const child = launch("tool", []); child.kill("SIGTERM"); + const timer = schedule(callback, 10); clearTimeout(timer); + get("http://example.test"); + `); + expect(ir.configurationOperations).toMatchObject([ + { kind: "file", key: "config.json" }, + ]); + expect( + ir.resourceOperations.map(({ kind, method }) => ({ kind, method })), + ).toEqual([ + { kind: "acquire", method: "createReadStream" }, + { kind: "release", method: "destroy" }, + ]); + expect(ir.childProcessSpawns).toMatchObject([ + { method: "spawn", command: "tool" }, + ]); + expect(ir.childProcessInteractions).toMatchObject([ + { kind: "signal", signalName: "SIGTERM" }, + ]); + expect( + ir.timerOperations.map(({ kind, method }) => ({ kind, method })), + ).toEqual([ + { kind: "schedule", method: "setTimeout" }, + { kind: "cancel", method: "clearTimeout" }, + ]); + expect(ir.requestOperations).toMatchObject([ + { method: "get", endpoint: "http://example.test" }, + ]); +}); + +it("does not promote ambiguous or non-namespace CommonJS origins to builtin effects", () => { + const ir = analyzeJavaScriptSemantics(` + const fake = require("./fs.js"); fake.readFileSync("external"); + const absentDefault = require("node:fs").default; absentDefault.readFileSync("default"); + const { constants } = require("node:fs"); constants.createReadStream("nested"); + let replaced = require("node:fs"); replaced = other; replaced.readFileSync("replaced"); + const mixed = condition ? require("node:fs") : other; mixed.readFileSync("mixed"); + function shadowed(require) { const fs = require("node:fs"); fs.readFileSync("shadowed"); } + `); + expect(ir.configurationOperations).toEqual([]); + expect(ir.resourceOperations).toEqual([]); +}); + it("does not assign built-in APIs to unrelated or shadowed bindings", () => { const ir = analyzeJavaScriptSemantics(` import fs from "./fs.js"; @@ -13,3 +435,244 @@ it("does not assign built-in APIs to unrelated or shadowed bindings", () => { expect(ir.resourceOperations).toEqual([]); }); + +it.each([ + `const fs = require('fs'); class Reader { text = fs.readFileSync('wrong'); } fs.readFileSync = custom; new Reader();`, + `const fs = require('fs'); class Reader { #text = fs.readFileSync('wrong'); } fs.readFileSync = custom; new Reader();`, + `const fs = require('fs'); fs.__defineGetter__('readFileSync', () => custom); fs.readFileSync('wrong');`, + `import proc from 'node:process'; proc.env = {}; const value = process.env.MODE;`, + `import fs from 'node:fs'; fs.readFileSync = custom; const other = require('fs'); other.readFileSync('wrong');`, +])("keeps delayed or replaced built-in identities unknown: %s", (source) => { + expect(analyzeJavaScriptSemantics(source).configurationOperations).toEqual( + [], + ); +}); + +it.each([ + `const fs = require('fs'); const before = fs.readFileSync('kept'), patch = (fs.readFileSync = custom);`, + `globalThis.process = {env: {}}; process.env = {}; const proc = require('process'); const value = proc.env.kept;`, + `import fs from 'node:fs'; fs.writeFileSync = custom; const other = require('fs'); other.readFileSync('kept');`, +])("keeps earlier and independent built-in observations: %s", (source) => { + expect( + analyzeJavaScriptSemantics(source).configurationOperations.map( + ({ key }) => key, + ), + ).toEqual(["kept"]); +}); + +it("shares equivalent alias subproblems and repeated intact receiver facts", () => { + const aliases = Array.from({ length: 30 }, (_, index) => { + const previous = index === 0 ? "fs" : `a${index - 1}`; + return `const a${index} = flag ? ${previous} : ${previous};`; + }); + const calls = Array.from( + { length: 128 }, + (_, index) => `a29.readFileSync('file-${index}');`, + ); + const ir = analyzeJavaScriptSemantics( + ["const fs = require('fs');", ...aliases, ...calls].join("\n"), + ); + expect(ir.configurationOperations).toHaveLength(128); +}); + +it.each([ + `import tls from 'node:tls'; const socket = tls.connect(443, 'example.test'); socket.end();`, + `import * as tls from 'node:tls'; const socket = tls.connect(443, 'example.test'); socket.end();`, + `const tls = require('tls'); const socket = tls.connect(443, 'example.test'); socket.end();`, +])("retains TLS acquisitions and their release: %s", (source) => { + expect( + analyzeJavaScriptSemantics(source).resourceOperations.map( + ({ kind, method }) => ({ kind, method }), + ), + ).toEqual([ + { kind: "acquire", method: "connect" }, + { kind: "release", method: "end" }, + ]); +}); + +it.each([ + `globalThis.process.env = {}; const proc = require('process'); const value = proc.env.MODE;`, + `const host = globalThis; host.process.env = {}; const proc = require('process'); const value = proc.env.MODE;`, + `import * as procNs from 'node:process'; procNs.default.env = {}; const value = process.env.MODE;`, + `globalThis.process.env = { MODE: 'local' }; const proc = require('process'); const value = proc.env.MODE;`, + `const proc = require('process'); proc.env.MODE = 'local'; const value = proc.env.MODE;`, +])("shares writes through the original process object: %s", (source) => { + expect(analyzeJavaScriptSemantics(source).configurationOperations).toEqual( + [], + ); +}); + +it.each([ + [ + `const fs = require('fs'); const result = (fs.readFileSync('kept'), fs.readFileSync = custom);`, + "kept", + ], + [ + `const fs = require('fs'); const read = fs.readFileSync; const result = (read('kept'), fs.readFileSync = custom);`, + "kept", + ], + [ + `const fs = require('fs'); consume(fs.readFileSync('before'), fs.readFileSync = custom);`, + "before", + ], +])( + "retains reads before a later sequence operand: %s", + (source, expectedKey) => { + expect( + analyzeJavaScriptSemantics(source).configurationOperations.map( + ({ key }) => key, + ), + ).toEqual([expectedKey]); + }, +); + +it("does not use source offsets for destructuring assignment evaluation order", () => { + const ir = + analyzeJavaScriptSemantics(`const fs = require('fs'); const target = {}; + [target[fs.readFileSync('wrong')]] = (fs.readFileSync = custom, []);`); + expect(ir.configurationOperations).toEqual([]); +}); + +it("keeps process reads after an alias is reassigned away from the global", () => { + const ir = + analyzeJavaScriptSemantics(`let host = globalThis; host = {process: {env: {}}}; + host.process.env = {}; const proc = require('process'); const value = proc.env.MODE;`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["MODE"]); +}); + +it("rejects CommonJS require after the wrapper module replaces its delegate", () => { + const ir = + analyzeJavaScriptSemantics(`module.require = () => ({ readFileSync() {} }); + const fs = require('node:fs'); fs.readFileSync('unknown');`); + expect(ir.configurationOperations).toEqual([]); +}); + +it.each([ + `module.require.extra = true; const fs = require('fs'); fs.readFileSync('kept');`, + `const owner = module; owner.require = () => ({}); const fs = require('fs'); fs.readFileSync('wrong');`, + `Object.defineProperty(module, 'require', {value: () => ({})}); const fs = require('fs'); fs.readFileSync('wrong');`, +])("tracks only actual wrapper require replacements: %s", (source) => { + const ir = analyzeJavaScriptSemantics(source); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual( + source.includes("extra") ? ["kept"] : [], + ); +}); + +it("preserves a saved promises object after deleting its parent export", () => { + const ir = + analyzeJavaScriptSemantics(`const fs = require('fs'); const fsp = fs.promises; + delete fs.promises; const handle = fsp.open('kept'); handle.close();`); + expect( + ir.resourceOperations.map(({ kind, method }) => ({ kind, method })), + ).toEqual([ + { kind: "acquire", method: "open" }, + { kind: "release", method: "close" }, + ]); +}); + +it("does not infer network acquisition methods from another built-in", () => { + const ir = analyzeJavaScriptSemantics( + `const { open } = require('node:net'); open();`, + ); + expect(ir.resourceOperations).toEqual([]); +}); + +it("recognizes nested ESM namespace receivers as original built-ins", () => { + const ir = + analyzeJavaScriptSemantics(`import * as ns from 'node:fs'; import fs from 'node:fs'; + ns.default.readFileSync('first'); fs.readFileSync('second');`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual([ + "first", + "second", + ]); +}); + +it("keeps intact fs/promises identity after deleting the fs.promises slot", () => { + const ir = + analyzeJavaScriptSemantics(`const fs=require('fs'); delete fs.promises; + const fsp=require('fs/promises'); const handle=fsp.open('kept'); handle.close();`); + expect( + ir.resourceOperations.map(({ kind, method }) => ({ kind, method })), + ).toEqual([ + { kind: "acquire", method: "open" }, + { kind: "release", method: "close" }, + ]); +}); + +it.each([ + `import fs from 'node:fs'; fs.promises = { open() {} }; fs.promises.open('wrong');`, + `const fs = require('fs'); patch(fs.promises); require('fs/promises').open('wrong');`, + `const fs = require('fs'); patch(fs.promises); require('node:fs/promises').open('wrong');`, +])("keeps replaced or escaped promises receivers unknown: %s", (source) => { + expect(analyzeJavaScriptSemantics(source).resourceOperations).toEqual([]); +}); + +it("follows final sequence expressions for direct built-in method calls", () => { + const ir = analyzeJavaScriptSemantics(`const fs = require('fs'); + (0, fs.readFileSync)('kept'); fs.readFileSync = custom; + (0, fs.readFileSync)('wrong');`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it.each([ + `import { env } from 'node:process'; patch(env); process.env.MODE;`, + `import { argv } from 'node:process'; patch(argv); process.argv[2];`, +])( + "shares named process containers with their namespace aliases: %s", + (source) => { + expect(analyzeJavaScriptSemantics(source).configurationOperations).toEqual( + [], + ); + }, +); + +it("retains uncertainty after a computed write to module's require delegate", () => { + const ir = + analyzeJavaScriptSemantics(`const key = 'require'; module[key] = replacement; + const fs = require('fs'); fs.readFileSync('wrong');`); + expect(ir.configurationOperations).toEqual([]); +}); + +it("retains a module namespace captured before its binding is reassigned", () => { + const ir = + analyzeJavaScriptSemantics(`let fs = require('fs'); const saved = fs; + fs = {}; saved.readFileSync('kept');`); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it("does not let own fs probe calls hide later builtin reads", () => { + const ir = analyzeJavaScriptSemantics( + `const fs=require('fs'); fs.existsSync('probe'); fs.readFileSync('kept');`, + ); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it("resolves nested ESM default namespace methods", () => { + const ir = analyzeJavaScriptSemantics( + `import * as ns from 'node:fs'; ns.default.readFileSync('kept');`, + ); + expect(ir.configurationOperations.map(({ key }) => key)).toEqual(["kept"]); +}); + +it("tracks process objects passed through qualified global references", () => { + const ir = + analyzeJavaScriptSemantics(`patch(globalThis.process); const proc = require('process'); + const mode = proc.env.MODE;`); + expect(ir.configurationOperations).toEqual([]); +}); + +it("checks a saved child object's method when that function is selected", () => { + const ir = + analyzeJavaScriptSemantics(`const fs = require('fs'); const fsp = fs.promises; + fs.promises.open = custom; const open = fsp.open; open('wrong');`); + expect(ir.resourceOperations).toEqual([]); +}); + +it("retains an earlier captured function after its object method is replaced", () => { + const ir = + analyzeJavaScriptSemantics(`const fs = require('fs'); const fsp = fs.promises; + const open = fsp.open; fs.promises.open = custom; open('kept');`); + expect(ir.resourceOperations).toMatchObject([ + { kind: "acquire", method: "open" }, + ]); +}); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts index 15738d5b4..2e32b0fe3 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts @@ -210,6 +210,64 @@ describe("JavaScript semantic analysis: read-modify-write", () => { ]); }); + it.each([ + ["destructuring assignment", "[source.count] = [2];"], + ["destructuring assignment with default", "[source.count = 2] = [];"], + [ + "nested destructuring assignment", + "({ value: { count: source.count } } = {});", + ], + ["rest destructuring assignment", "[...source.items] = [];"], + ["for-of head", "for (source.count of [4]) {}"], + ["for-in head", "for (source.count in { key: true }) {}"], + ["delete", "delete source.count;"], + ])("records a member target in %s as a write", (_name, statement) => { + const ir = analyzeJavaScriptSemantics(` + const source = { count: 1, items: [] }; + ${statement} + `); + expect(ir.objectOperations.map(({ kind }) => kind)).toEqual(["write"]); + }); + + it.each([ + ["non-null update", "source.count!++;"], + ["non-null compound assignment", "source.count! += 1;"], + ["cast compound assignment", "(source.count as number) += 1;"], + ["parenthesized update", "(source.count)++;"], + ])( + "retains the property read in a wrapped read-modify-write: %s", + (_name, statement) => { + const ir = analyzeJavaScriptSemantics(` + const source = { count: 1 }; + ${statement} + `); + expect(ir.objectOperations.map(({ kind }) => kind)).toEqual([ + "read", + "write", + ]); + }, + ); + + it.each([ + ["cast plain assignment", "(source.count as number) = 2;"], + ["non-null delete", "delete source.count!;"], + ])("keeps a wrapped %s write-only", (_name, statement) => { + const ir = analyzeJavaScriptSemantics(` + const source = { count: 1 }; + ${statement} + `); + expect(ir.objectOperations.map(({ kind }) => kind)).toEqual(["write"]); + }); + + it("keeps a member in a destructuring default value a read", () => { + const ir = analyzeJavaScriptSemantics(` + const source = { count: 1 }; + let value; + [value = source.count] = []; + `); + expect(ir.objectOperations.map(({ kind }) => kind)).toEqual(["read"]); + }); + it("does not model a compound binding write as a competing replacement", () => { const ir = analyzeJavaScriptSemantics(` let count = 1; @@ -222,6 +280,37 @@ describe("JavaScript semantic analysis: read-modify-write", () => { }); }); +describe("JavaScript semantic analysis: wrapped binding targets", () => { + it("records a wrapped identifier update as a read before a write", () => { + const ir = analyzeJavaScriptSemantics(` + let count = 1; + count! += 1; + (count as number)++; + const copy = count!; + `); + expect( + ir.references + .filter(({ name }) => name === "count") + .map(({ role }) => role), + ).toEqual(["read", "write", "read", "write", "read"]); + expect(topLevelBinding(ir, "count").value).toMatchObject({ + status: "unknown", + }); + }); + + it("records a wrapped identifier assignment as a write of its value", () => { + const ir = analyzeJavaScriptSemantics(` + let count = 1; + count! = 2; + `); + expect( + ir.references + .filter(({ name }) => name === "count") + .map(({ role }) => role), + ).toEqual(["write"]); + }); +}); + describe("JavaScript semantic analysis: dataflow 2", () => { it("retains partial property coverage and empty returns", () => { const partial = analyzeJavaScriptSemantics(` diff --git a/src/domain/javascript/javascriptSemanticAnalysis.empty_keys.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.empty_keys.test.ts index 1ef0d72af..e97b9e284 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.empty_keys.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.empty_keys.test.ts @@ -115,3 +115,18 @@ it("retains exact empty request field names without dynamic fields", () => { ), ).toEqual(["", "body"]); }); + +it("retains the exact empty CommonJS export name", () => { + const ir = analyzeJavaScriptSemantics(` + exports[''] = 1; + module.exports[''] = 2; + exports['ok'] = 3; + exports[key] = 4; + `); + expect(ir.moduleLinks.map(({ exportedName }) => exportedName)).toEqual([ + "", + "", + "ok", + "*", + ]); +}); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.native_bindings.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.native_bindings.test.ts index 3c137cc0a..784a5a5b6 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.native_bindings.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.native_bindings.test.ts @@ -170,3 +170,39 @@ describe("JavaScript semantic analysis: native bindings 2", () => { ); }); }); + +describe("JavaScript semantic analysis: child-process signals", () => { + it("keeps unresolved kill signals distinct from default and literal signals", () => { + const ir = analyzeJavaScriptSemantics(` + import { spawn } from "node:child_process"; + function run(signal) { + const child = spawn("/bin/tool"); + child.kill(signal); + child.kill(); + child.kill("SIGINT"); + child.kill(undefined); + child.kill(undefined as NodeJS.Signals | undefined); + child.kill(void 0); + child.kill(void signal); + } + function shadowed(undefined) { + const child = spawn("/bin/tool"); + child.kill(undefined); + } + `); + + expect(ir.childProcessSpawns).toHaveLength(2); + expect( + ir.childProcessInteractions.map(({ signalName }) => signalName), + ).toEqual([ + null, + "SIGTERM", + "SIGINT", + "SIGTERM", + "SIGTERM", + "SIGTERM", + "SIGTERM", + null, + ]); + }); +}); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 578789ad5..dcb913afd 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -21,6 +21,7 @@ import { runtimeFunctionParameters, semanticStaticPropertyKey, semanticWriteTargets, + unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; import type { JavaScriptSemanticAnalysisState, @@ -42,6 +43,7 @@ import { traverseJavaScriptAstSteps, } from "./javascriptSemanticTraversal.js"; import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; +import { semanticBundledRequireOrigin } from "./javascriptBundledRequireOrigin.js"; import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js"; import { indexJavaScriptAmbientWritesSteps } from "./javascriptSemanticAmbientWrites.js"; import { hasUncertainTypeOnlyModuleSideEffect } from "./javascriptModuleDeclaration.js"; @@ -157,6 +159,7 @@ export function* analyzeParsedJavaScriptSemanticsSteps( ): Generator { const state = createState(file.program, staticImportResolutionMode); yield* collectDefinitionsSteps(file.program, state); + yield* indexJavaScriptAmbientWritesSteps(file.program, state); yield* collectSemanticMemberMutationsSteps(file.program, state); yield* traverseJavaScriptAstSteps(file.program, { enter: (node) => collectSemanticModuleLink(node, state), @@ -258,7 +261,8 @@ export type JavaScriptOpenReceiverFact = | "local" | "local-indexed-db" | "local-cache-storage" - | "configuration"; + | "configuration" + | "extension-state"; /** * Resolve only the lexical receiver identity needed to distinguish browser @@ -278,6 +282,7 @@ export function* classifyParsedJavaScriptOpenReceiversSteps( const state = createState(file.program); yield* collectDefinitionsSteps(file.program, state); const facts = new Map(); + const extensionStateReads: number[] = []; yield* traverseJavaScriptAstSteps(file.program, { enter: (node) => { if ( @@ -293,6 +298,8 @@ export function* classifyParsedJavaScriptOpenReceiversSteps( if (method === "get") { if (isConfigurationReceiver(state, node.callee.object)) facts.set(node.start ?? -1, "configuration"); + else if (isExtensionStateMember(node.callee.object)) + extensionStateReads.push(node.start ?? -1); return; } if (method !== "open" || !t.isIdentifier(node.callee.object)) return; @@ -358,9 +365,31 @@ export function* classifyParsedJavaScriptOpenReceiversSteps( if (aliasFact !== undefined) facts.set(node.start ?? -1, aliasFact); }, }); + if (extensionStateReads.length > 0 && importsVsCodeModule(state)) + for (const start of extensionStateReads) + facts.set(start, "extension-state"); return facts; } +/** VS Code passes extension state through ExtensionContext, not an import. */ +const isExtensionStateMember = (receiver: t.Node): boolean => + (t.isMemberExpression(receiver) || t.isOptionalMemberExpression(receiver)) && + (() => { + const key = semanticStaticPropertyKey(receiver.property, receiver.computed); + return key === "globalState" || key === "workspaceState"; + })(); + +/** Require module evidence before treating a familiar member as VS Code state. */ +const importsVsCodeModule = (state: JavaScriptSemanticAnalysisState): boolean => + [...state.bindingsById.values()].some( + (binding) => + binding.directOrigins.some(({ specifier }) => specifier === "vscode") || + semanticRequireOrigin( + semanticStableInitializer(binding, state)?.node, + state, + )?.origin.specifier === "vscode", + ); + /** * Whether a call spells VS Code's `workspace.getConfiguration(section)`, whose * result `.get` reads setting keys rather than request paths. The `workspace` @@ -522,7 +551,7 @@ export function* indexJavaScriptElectronIdentitySteps( const wrapped = interop.callOrigin(root); const origin = wrapped === undefined - ? semanticRequireOrigin(root, state)?.origin + ? electronRequireOrigin(root, state)?.origin : collectedWrites && interop.invalidatedCall(root) ? undefined : wrapped.origin; @@ -606,6 +635,13 @@ const electronExport = ( : null; }; +const electronRequireOrigin = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +) => + semanticRequireOrigin(node, state) ?? + semanticBundledRequireOrigin(node, state); + const bindingOrigin = ( binding: JavaScriptSemanticBindingState, state: JavaScriptSemanticAnalysisState, @@ -616,7 +652,7 @@ const bindingOrigin = ( : undefined; const initializer = semanticStableInitializer(binding, state); if (initializer === undefined) return undefined; - const origin = semanticRequireOrigin(initializer.node, state)?.origin; + const origin = electronRequireOrigin(initializer.node, state)?.origin; const projection = initializer.projection; return origin === undefined || !projection.every((key): key is string => typeof key === "string") @@ -637,6 +673,7 @@ const createState = ( bindings: new Map(), }; return { + program, staticImportResolutionMode, parentsByNode: new WeakMap(), scopes: [root], @@ -649,7 +686,10 @@ const createState = ( linkedRequireCalls: new WeakSet(), moduleLinkBindings: new WeakMap(), conditionalInitializers: new WeakSet(), + requiredModuleEffects: new Map(), + ambientBindingWrites: new Map(), ambientWrites: new SemanticPropertyPathCoverage(), + ambientWriteEffects: [], }; }; @@ -940,12 +980,13 @@ const bindAssignment = ( state: JavaScriptSemanticAnalysisState, ): void => { if (t.isAssignmentExpression(node)) { - if (t.isIdentifier(node.left)) + const left = unwrapJavaScriptExpression(node.left).node; + if (t.isIdentifier(left)) addAssignment( - node.left, + left, node.operator === "=" || ((node.operator === "||=" || node.operator === "??=") && - !hasIncomingValue(node.left, scope, state)) + !hasIncomingValue(left, scope, state)) ? node.right : node, scope, @@ -955,9 +996,10 @@ const bindAssignment = ( for (const identifier of semanticWriteTargets(node.left)) if (t.isIdentifier(identifier)) addAssignment(identifier, node, scope, state); - } else if (t.isUpdateExpression(node) && t.isIdentifier(node.argument)) - addAssignment(node.argument, node, scope, state); - else if (t.isForOfStatement(node) || t.isForInStatement(node)) + } else if (t.isUpdateExpression(node)) { + const argument = unwrapJavaScriptExpression(node.argument).node; + if (t.isIdentifier(argument)) addAssignment(argument, node, scope, state); + } else if (t.isForOfStatement(node) || t.isForInStatement(node)) for (const pattern of t.isVariableDeclaration(node.left) ? node.left.declarations.map(({ id }) => id) : [node.left]) { @@ -1339,7 +1381,7 @@ const createBinding = ( name, kind, mutable, - mutatedPaths: [], + memberWrites: [], escapedPaths: [], definitions: [], initializers: [], @@ -1359,7 +1401,14 @@ const addAssignment = ( identifier.name, state, ); - if (binding === undefined) return; + if (binding === undefined) { + // Unbound identifiers can denote host wrapper bindings (such as require), + // which are separate from properties written through the global object. + const writes = state.ambientBindingWrites.get(identifier.name) ?? []; + writes.push(identifier); + state.ambientBindingWrites.set(identifier.name, writes); + return; + } binding.mutable = true; binding.definitions.push({ kind: "assignment", location: range(identifier) }); binding.initializers.push({ node: initializer, projection: [] }); diff --git a/src/domain/javascript/javascriptSemanticAsyncEffects.ts b/src/domain/javascript/javascriptSemanticAsyncEffects.ts index 5b3068101..6ab62bd59 100644 --- a/src/domain/javascript/javascriptSemanticAsyncEffects.ts +++ b/src/domain/javascript/javascriptSemanticAsyncEffects.ts @@ -1,7 +1,5 @@ import * as t from "@babel/types"; -import { semanticBuiltinNamespacePath } from "./javascriptSemanticProvenance.js"; - import type { JavaScriptSemanticCallable, JavaScriptSemanticEventOperation, @@ -18,6 +16,7 @@ import { import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { range } from "./javascriptStaticAnalysisHelpers.js"; import { + builtinDataEffectMethod, dataEffectLiteralString, dataEffectMemberCallee, assignedSemanticResultBindings, @@ -308,22 +307,11 @@ const timerMethod = ( ? null : method; } - return timerMethodFromBinding(binding); } - if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) - return null; - const method = admittedTimerMethod( - semanticStaticPropertyName(callee.property, callee.computed), + return admittedTimerMethod( + builtinDataEffectMethod(callee, state, ["timers", "node:timers"]) ?? "", ); - return method !== null && - t.isIdentifier(callee.object) && - timerNamespaceBinding( - resolveSemanticBindingState(state, callee.object, callee.object.name), - ) - ? method - : null; }; - const admittedTimerMethod = ( value: string, ): JavaScriptSemanticTimerOperation["method"] | null => @@ -331,28 +319,6 @@ const admittedTimerMethod = ( (candidate) => candidate === value, ) ?? null; -const timerMethodFromBinding = ( - binding: JavaScriptSemanticBindingState, -): JavaScriptSemanticTimerOperation["method"] | null => { - for (const { specifier, importedPath } of binding.directOrigins) { - if (!timerModule(specifier)) continue; - const method = admittedTimerMethod(importedPath.at(-1) ?? ""); - if (method !== null) return method; - } - return null; -}; - -const timerNamespaceBinding = ( - binding: JavaScriptSemanticBindingState | undefined, -): boolean => - binding?.directOrigins.some( - ({ specifier, importedPath }) => - timerModule(specifier) && semanticBuiltinNamespacePath(importedPath), - ) ?? false; - -const timerModule = (specifier: string): boolean => - specifier === "timers" || specifier === "node:timers"; - const bindingId = ( node: | t.Expression diff --git a/src/domain/javascript/javascriptSemanticChildProcesses.ts b/src/domain/javascript/javascriptSemanticChildProcesses.ts index 1cffb5164..c9aec78e7 100644 --- a/src/domain/javascript/javascriptSemanticChildProcesses.ts +++ b/src/domain/javascript/javascriptSemanticChildProcesses.ts @@ -1,15 +1,17 @@ import * as t from "@babel/types"; -import { semanticBuiltinNamespacePath } from "./javascriptSemanticProvenance.js"; - import type { JavaScriptSemanticCallable, JavaScriptSemanticChildProcessInteraction, JavaScriptSemanticChildProcessSpawn, } from "./javascriptSemanticIr.js"; import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js"; -import { semanticStaticPropertyName } from "./javascriptAstValues.js"; import { + semanticStaticPropertyName, + unwrapJavaScriptExpression, +} from "./javascriptAstValues.js"; +import { + isUnshadowedGlobal, resolveSemanticBindingState, type JavaScriptSemanticAnalysisState, type JavaScriptSemanticBindingState, @@ -18,6 +20,7 @@ import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { range } from "./javascriptStaticAnalysisHelpers.js"; import { + builtinDataEffectMethod, dataEffectLiteralString, dataEffectMemberCallee, assignedSemanticResultBindings, @@ -222,9 +225,7 @@ const childInteraction = ( .sort(compareUnicodeCodePoints), eventName: kind === "listener" ? eventName : null, signalName: - kind === "signal" - ? (dataEffectLiteralString(node.arguments[0]) ?? "SIGTERM") - : null, + kind === "signal" ? killSignalName(node.arguments, state) : null, listenerLocation: kind === "listener" && t.isNode(listener) ? range(listener) : null, resolution: @@ -237,6 +238,23 @@ const childInteraction = ( }; }; +// Only an omitted argument or the unshadowed global undefined selects the +// SIGTERM default; an unresolved argument leaves the signal unknown. +const killSignalName = ( + args: t.CallExpression["arguments"], + state: JavaScriptSemanticAnalysisState, +): string | null => { + const [signal] = args; + if (signal === undefined) return "SIGTERM"; + const expression = unwrapJavaScriptExpression(signal).node; + if ( + t.isUnaryExpression(expression, { operator: "void" }) || + isUnshadowedGlobal(expression, state, "undefined") + ) + return "SIGTERM"; + return dataEffectLiteralString(signal); +}; + const classifyInteraction = ( method: string, argument: @@ -266,52 +284,15 @@ const childProcessMethod = ( callee: t.Node, state: JavaScriptSemanticAnalysisState, ): JavaScriptSemanticChildProcessSpawn["method"] | null => { - if (t.isIdentifier(callee)) { - const binding = resolveSemanticBindingState(state, callee, callee.name); - return binding === undefined ? null : methodFromBinding(binding); - } - if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) - return null; - const method = admittedChildMethod( - semanticStaticPropertyName(callee.property, callee.computed), + const method = builtinDataEffectMethod(callee, state, [ + "child_process", + "node:child_process", + ]); + return ( + CHILD_PROCESS_METHODS.find((candidate) => candidate === method) ?? null ); - if (method === null || !t.isIdentifier(callee.object)) return null; - const binding = resolveSemanticBindingState( - state, - callee.object, - callee.object.name, - ); - return childProcessNamespace(binding) ? method : null; }; -const methodFromBinding = ( - binding: JavaScriptSemanticBindingState, -): JavaScriptSemanticChildProcessSpawn["method"] | null => { - for (const { specifier, importedPath } of binding.directOrigins) { - if (!childProcessModule(specifier)) continue; - const method = admittedChildMethod(importedPath.at(-1) ?? ""); - if (method !== null) return method; - } - return null; -}; - -const admittedChildMethod = ( - value: string, -): JavaScriptSemanticChildProcessSpawn["method"] | null => - CHILD_PROCESS_METHODS.find((candidate) => candidate === value) ?? null; - -const childProcessNamespace = ( - binding: JavaScriptSemanticBindingState | undefined, -): boolean => - binding?.directOrigins.some( - ({ specifier, importedPath }) => - childProcessModule(specifier) && - semanticBuiltinNamespacePath(importedPath), - ) ?? false; - -const childProcessModule = (specifier: string): boolean => - specifier === "child_process" || specifier === "node:child_process"; - const resolveChildCandidates = ( node: t.Node, binding: JavaScriptSemanticBindingState | undefined, diff --git a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts index 0f7a6d942..99ae17097 100644 --- a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts +++ b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts @@ -1,19 +1,36 @@ import * as t from "@babel/types"; +import { BUILTIN_RESOURCE_METHODS } from "./javascriptBuiltinResourceMethods.js"; import { semanticBuiltinNamespacePath } from "./javascriptSemanticProvenance.js"; +import { + semanticRequireOrigin, + semanticRequireExportsIdentity, + semanticGlobalProcessOrigin, + semanticMutableBuiltinOrigin, +} from "./javascriptSemanticRequireOrigin.js"; +import type { JavaScriptModuleOrigin } from "./javascriptSemanticIr.js"; +import { + evaluateSemanticProvenance, + evaluateSemanticProvenanceAtRead, +} from "./javascriptSemanticValues.js"; +import { + semanticPropertyPathKeyMatches, + type JavaScriptSemanticPropertyPath, +} from "./javascriptSemanticPropertyPaths.js"; import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js"; +import { semanticTopLevelEffectFollowsCapture } from "./javascriptSemanticMutationInitializers.js"; import { resolveSemanticBindingState, - semanticResolutionBlocked, type JavaScriptSemanticAnalysisState, type JavaScriptSemanticBindingState, + type JavaScriptSemanticMemberEffect, + type JavaScriptSemanticRequiredModuleEffect, } from "./javascriptSemanticState.js"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { readExactJavaScriptLiteral, semanticStaticPropertyKey, - semanticStaticPropertyName, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; @@ -137,59 +154,715 @@ export interface DataEffectTraversalContext { readonly ancestors: t.Node[]; } -/** Resolve a direct or namespace-imported built-in method. */ +/** Resolve built-in methods from the shared, unambiguous binding provenance. */ export const builtinDataEffectMethod = ( callee: t.Node, state: JavaScriptSemanticAnalysisState, specifiers: readonly string[], ): string | null => { + const expression = unwrapJavaScriptExpression(callee).node; + if (expression !== callee) + return builtinDataEffectMethod(expression, state, specifiers); + if (t.isSequenceExpression(expression)) { + const final = expression.expressions.at(-1); + return final === undefined + ? null + : builtinDataEffectMethod(final, state, specifiers); + } + const required = semanticRequireOrigin(callee, state); + if (required !== undefined) { + const origin = builtinNamespaceOrigin(required.origin); + return origin.importedPath.length === 1 && + specifiers.some( + (specifier) => specifier.replace(/^node:/u, "") === origin.specifier, + ) && + !requiredModuleChanged( + required.origin.specifier, + required.origin.importedPath, + callee, + state, + true, + ) + ? (origin.importedPath[0] ?? null) + : null; + } if (t.isIdentifier(callee)) { const binding = resolveSemanticBindingState(state, callee, callee.name); - return moduleMethod(binding, specifiers); + const path = builtinBindingPath( + binding, + state, + specifiers, + callee, + [], + true, + ); + return path?.length === 1 ? (path[0] ?? null) : null; } if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) return null; - if (!t.isIdentifier(callee.object)) return null; + if (!t.isIdentifier(callee.object)) { + const method = semanticStaticPropertyKey(callee.property, callee.computed); + const receiver = semanticBoundMemberPath(callee.object, state); + if (method !== null && receiver !== undefined) { + const provenance = evaluateSemanticProvenance(receiver.binding, state); + if (provenance.status === "module") { + const origin = builtinNamespaceOrigin({ + ...provenance.origins[0], + importedPath: [ + ...provenance.origins[0].importedPath, + ...receiver.path, + ], + }); + if ( + specifiers.some( + (specifier) => + specifier.replace(/^node:/u, "") === origin.specifier, + ) + ) + return builtinBindingPath( + receiver.binding, + state, + specifiers, + callee, + [...receiver.path, method], + true, + )?.length === 0 + ? method + : null; + } + } + const source = semanticExpressionModuleOrigin(callee.object, state); + if (method === null || source === undefined) return null; + const origin = builtinNamespaceOrigin(source); + return origin.importedPath.length === 0 && + specifiers.some( + (specifier) => specifier.replace(/^node:/u, "") === origin.specifier, + ) && + !requiredModuleChanged( + origin.specifier, + [...origin.importedPath, method], + callee, + state, + true, + callee.object, + ) + ? method + : null; + } const binding = resolveSemanticBindingState( state, callee.object, callee.object.name, ); - const namespace = binding?.directOrigins.some( - ({ specifier, importedPath }) => - specifiers.includes(specifier) && - semanticBuiltinNamespacePath(importedPath), + const method = semanticStaticPropertyKey(callee.property, callee.computed); + if (method === null) return null; + const resolvedPath = builtinBindingPath( + binding, + state, + specifiers, + callee, + [method], + true, ); - return namespace - ? semanticStaticPropertyName(callee.property, callee.computed) + if (resolvedPath?.length === 0) return method; + const initializer = binding?.initializers[0]; + if ( + initializer === undefined || + binding?.initializers.length !== 1 || + (!t.isMemberExpression(initializer.node) && + !t.isOptionalMemberExpression(initializer.node)) + ) { + return null; + } + const provenance = evaluateSemanticProvenance(binding, state); + if (provenance.status !== "module") return null; + const origin = builtinNamespaceOrigin(provenance.origins[0]); + const selected = [...origin.importedPath, method]; + return specifiers.some( + (specifier) => specifier.replace(/^node:/u, "") === origin.specifier, + ) && + builtinBindingPath( + binding, + state, + specifiers, + initializer.node, + [method], + true, + )?.length === 0 && + !requiredModuleChanged( + provenance.origins[0].specifier, + selected, + callee, + state, + true, + initializer.node, + ) + ? method : null; }; -const moduleMethod = ( - binding: JavaScriptSemanticBindingState | undefined, - specifiers: readonly string[], -): string | null => { - for (const { specifier, importedPath } of binding?.directOrigins ?? []) { - if (!specifiers.includes(specifier)) continue; - return importedPath.at(-1) ?? null; +const semanticBoundMemberPath = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): + | { + readonly binding: JavaScriptSemanticBindingState; + readonly path: string[]; + } + | undefined => { + const path: string[] = []; + let current = unwrapJavaScriptExpression(node).node; + while ( + t.isMemberExpression(current) || + t.isOptionalMemberExpression(current) + ) { + const key = semanticStaticPropertyKey(current.property, current.computed); + if (key === null || !t.isNode(current.object)) return undefined; + path.unshift(key); + current = unwrapJavaScriptExpression(current.object).node; } - return null; + if (!t.isIdentifier(current)) return undefined; + const binding = resolveSemanticBindingState(state, current, current.name); + return binding === undefined ? undefined : { binding, path }; +}; + +type BuiltinBindingMemo = Map>; + +const builtinBindingPath = ( + binding: JavaScriptSemanticBindingState | undefined, + state: JavaScriptSemanticAnalysisState, + specifiers: readonly string[], + capture: t.Node, + selectedPath: readonly string[] = [], + callableValue = false, + visited: ReadonlySet = new Set(), + memo: BuiltinBindingMemo = new Map(), + provenanceCapture?: t.Identifier, + savedObjectCapture?: t.Node, +): readonly string[] | null => { + if (binding === undefined) return null; + const key = JSON.stringify([ + binding.bindingId, + selectedPath, + callableValue, + provenanceCapture?.start, + provenanceCapture?.end, + savedObjectCapture?.start, + savedObjectCapture?.end, + ]); + let captured = memo.get(capture); + if (captured?.has(key)) return captured.get(key) ?? null; + if (visited.has(binding.bindingId)) return null; + if (captured === undefined) { + captured = new Map(); + memo.set(capture, captured); + } + const result = computeBuiltinBindingPath( + binding, + state, + specifiers, + capture, + selectedPath, + callableValue, + visited, + memo, + provenanceCapture, + savedObjectCapture, + ); + captured.set(key, result); + return result; +}; + +const computeBuiltinBindingPath = ( + binding: JavaScriptSemanticBindingState | undefined, + state: JavaScriptSemanticAnalysisState, + specifiers: readonly string[], + capture: t.Node, + selectedPath: readonly string[] = [], + callableValue = false, + visited: ReadonlySet, + memo: BuiltinBindingMemo, + provenanceCapture?: t.Identifier, + savedObjectCapture?: t.Node, +): readonly string[] | null => { + if (binding === undefined || visited.has(binding.bindingId)) return null; + // Provenance identifies the module, independently of whether a selected slot + // still refers to its original built-in after member mutation or escape. + if ( + binding.memberWrites.some( + ({ path, node }) => + affectsBuiltinPath(path, selectedPath) && + !( + semanticTopLevelEffectFollowsCapture( + path.length === 0 && provenanceCapture !== undefined + ? provenanceCapture + : capture, + node, + state.parentsByNode, + ) || + (savedObjectCapture !== undefined && + path.length < selectedPath.length && + semanticTopLevelEffectFollowsCapture( + savedObjectCapture, + node, + state.parentsByNode, + )) + ), + ) || + builtinEscapes(binding, state).some( + ({ path, node }) => + affectsBuiltinPath(path, selectedPath) && + // A function-value escape cannot replace its native [[Call]]. An + // escaped containing object can still replace the selected method. + (!callableValue || path.length < selectedPath.length) && + !semanticTopLevelEffectFollowsCapture( + capture, + node, + state.parentsByNode, + ), + ) + ) + return null; + const provenance = + provenanceCapture === undefined + ? evaluateSemanticProvenance(binding, state) + : evaluateSemanticProvenanceAtRead(binding, provenanceCapture, state); + if (provenance.status !== "module") return null; + const { importedPath } = provenance.origins[0]; + const namespace = builtinNamespaceOrigin(provenance.origins[0]); + if ( + !specifiers.some((item) => { + const name = item.replace(/^node:/u, ""); + return ( + name === namespace.specifier || + (namespace.specifier === "fs" && + selectedPath[0] === "promises" && + name === "fs/promises") + ); + }) + ) + return null; + let resolvedPath: readonly string[] = importedPath; + for (const origin of binding.directOrigins) { + const shared = semanticMutableBuiltinOrigin(origin, selectedPath); + if ( + shared !== undefined && + requiredModuleChanged( + shared.specifier, + shared.path, + capture, + state, + callableValue, + ) + ) + return null; + } + const ancestors = new Set([...visited, binding.bindingId]); + for (const initializer of binding.initializers) { + const pending = [ + { + node: initializer.node, + path: [...initializer.projection.map(String), ...selectedPath], + savedObjectCapture: + selectedPath.length > 0 && + (t.isMemberExpression(initializer.node) || + t.isOptionalMemberExpression(initializer.node)) + ? initializer.node + : savedObjectCapture, + capture: + callableValue && + selectedPath.length === 0 && + initializer.projection.length > 0 + ? initializer.node + : capture, + }, + ]; + while (pending.length > 0) { + const current = pending.pop()!; + const node = unwrapJavaScriptExpression(current.node).node; + if (t.isIdentifier(node)) { + const source = resolveSemanticBindingState(state, node, node.name); + const sourcePath = builtinBindingPath( + source, + state, + specifiers, + current.capture, + current.path, + callableValue, + ancestors, + memo, + node, + current.savedObjectCapture, + ); + if (sourcePath === null) return null; + // Normalize ESM default namespaces at their owner before projecting an alias. + resolvedPath = [...sourcePath, ...current.path].slice( + 0, + sourcePath.length + current.path.length - selectedPath.length, + ); + } else if ( + t.isMemberExpression(node) || + t.isOptionalMemberExpression(node) + ) { + const key = semanticStaticPropertyKey(node.property, node.computed); + if (key === null) return null; + pending.push({ + node: node.object, + path: [key, ...current.path], + savedObjectCapture: current.savedObjectCapture, + // Freeze a selected function, not later member reads through a saved object. + capture: + callableValue && selectedPath.length === 0 ? node : current.capture, + }); + } else if (t.isSequenceExpression(node)) { + const final = node.expressions.at(-1); + if (final === undefined) return null; + pending.push({ ...current, node: final }); + } else if (t.isAssignmentExpression(node) && node.operator === "=") { + pending.push({ ...current, node: node.right }); + } else if (t.isConditionalExpression(node)) { + pending.push( + { ...current, node: node.consequent }, + { ...current, node: node.alternate }, + ); + } else if (t.isLogicalExpression(node)) { + pending.push( + { ...current, node: node.left }, + { ...current, node: node.right }, + ); + } else { + const required = semanticRequireOrigin(node, state); + if ( + required !== undefined && + requiredModuleChanged( + required.origin.specifier, + [...required.origin.importedPath, ...current.path], + current.capture, + state, + callableValue, + current.savedObjectCapture, + ) + ) + return null; + } + } + } + // Only a direct ESM default import denotes the built-in namespace; + // require("node:fs").default is a literal property, not interop syntax. + return binding.directOrigins.length > 0 && + semanticBuiltinNamespacePath(importedPath) + ? [] + : builtinNamespaceOrigin({ + ...provenance.origins[0], + importedPath: resolvedPath, + }).importedPath; +}; + +/** Interpret known Node namespace aliases without changing source provenance. */ +const builtinNamespaceOrigin = ( + origin: JavaScriptModuleOrigin, +): JavaScriptModuleOrigin => { + const specifier = origin.specifier.replace(/^node:/u, ""); + const importedPath = + origin.resolutionMode !== "require" && origin.importedPath[0] === "default" + ? origin.importedPath.slice(1) + : origin.importedPath; + return specifier === "fs" && importedPath[0] === "promises" + ? { + ...origin, + specifier: "fs/promises", + importedPath: importedPath.slice(1), + } + : { ...origin, specifier, importedPath }; +}; + +const affectsBuiltinPath = ( + path: JavaScriptSemanticPropertyPath, + selected: readonly string[], +): boolean => + path.length <= selected.length && + path.every((key, index) => + semanticPropertyPathKeyMatches(key, selected[index]!), + ); + +const BUILTIN_RECEIVER_METHODS: Readonly>> = + { + fs: new Set([ + "readFile", + "readFileSync", + "writeFile", + "writeFileSync", + "close", + "closeSync", + "existsSync", + ...BUILTIN_RESOURCE_METHODS.fs, + ]), + "fs/promises": new Set([ + "readFile", + "writeFile", + ...BUILTIN_RESOURCE_METHODS["fs/promises"], + ]), + net: new Set(BUILTIN_RESOURCE_METHODS.net), + tls: new Set(BUILTIN_RESOURCE_METHODS.tls), + http: new Set(["request", "get"]), + https: new Set(["request", "get"]), + child_process: new Set(["spawn", "exec", "execFile", "fork"]), + process: new Set(["cwd", "uptime", "hrtime", "nextTick"]), + timers: new Set([ + "setTimeout", + "setInterval", + "setImmediate", + "clearTimeout", + "clearInterval", + "clearImmediate", + ]), + }; + +interface BuiltinEffectIndex { + readonly writesByModule: Map< + string, + readonly JavaScriptSemanticRequiredModuleEffect[] + >; + readonly moduleEffects: Map< + string, + readonly JavaScriptSemanticRequiredModuleEffect[] + >; + readonly bindingEscapes: Map< + string, + readonly JavaScriptSemanticMemberEffect[] + >; + readonly receivers: WeakMap; +} + +// Derived passes run after mutation collection. These indexes share that state's +// lifetime and never cache facts while its mutation owner is still producing them. +const builtinEffectIndexes = new WeakMap< + JavaScriptSemanticAnalysisState, + BuiltinEffectIndex +>(); + +const builtinEffectIndex = ( + state: JavaScriptSemanticAnalysisState, +): BuiltinEffectIndex => { + let index = builtinEffectIndexes.get(state); + if (index === undefined) { + index = { + writesByModule: new Map( + [...state.requiredModuleEffects].map(([name, effects]) => [ + name, + effects.filter(({ kind }) => kind === "write"), + ]), + ), + moduleEffects: new Map(), + bindingEscapes: new Map(), + receivers: new WeakMap(), + }; + builtinEffectIndexes.set(state, index); + } + return index; +}; + +const builtinEscapes = ( + binding: JavaScriptSemanticBindingState, + state: JavaScriptSemanticAnalysisState, +): readonly JavaScriptSemanticMemberEffect[] => { + const cache = builtinEffectIndex(state).bindingEscapes; + let effects = cache.get(binding.bindingId); + if (effects === undefined) { + effects = binding.escapedPaths.filter( + ({ node }) => !originalBuiltinReceiver(node, state), + ); + cache.set(binding.bindingId, effects); + } + return effects; +}; + +const builtinModuleEffects = ( + specifier: string, + state: JavaScriptSemanticAnalysisState, +): readonly JavaScriptSemanticRequiredModuleEffect[] => { + const cache = builtinEffectIndex(state).moduleEffects; + let effects = cache.get(specifier); + if (effects === undefined) { + effects = (state.requiredModuleEffects.get(specifier) ?? []).filter( + ({ kind, node }) => + kind === "write" || !originalBuiltinReceiver(node, state), + ); + if (specifier === "fs/promises") { + const sharedFsEscapes = (state.requiredModuleEffects.get("fs") ?? []) + .filter( + ({ kind, path, node }) => + kind === "escape" && + (path.length === 0 || path[0] === "promises") && + !originalBuiltinReceiver(node, state), + ) + .map((effect) => ({ + ...effect, + path: + effect.path[0] === "promises" ? effect.path.slice(1) : effect.path, + })); + effects = [...effects, ...sharedFsEscapes]; + } + cache.set(specifier, effects); + } + return effects; +}; + +/** Check shared exports at the use or method-capture point, not at every later call. */ +const requiredModuleChanged = ( + specifier: string, + selected: readonly string[], + capture: t.Node, + state: JavaScriptSemanticAnalysisState, + callableValue = false, + savedObjectCapture?: t.Node, +): boolean => { + const identity = semanticRequireExportsIdentity(specifier, selected); + return builtinModuleEffects(identity.specifier, state).some( + ({ path, node, kind }) => + !( + savedObjectCapture !== undefined && + path.length < identity.path.length && + semanticTopLevelEffectFollowsCapture( + savedObjectCapture, + node, + state.parentsByNode, + ) + ) && + affectsBuiltinPath(path, identity.path) && + !semanticTopLevelEffectFollowsCapture( + capture, + node, + state.parentsByNode, + ) && + (kind === "write" || + !callableValue || + path.length < identity.path.length), + ); +}; + +/** Calling an unchanged built-in as a member is not an escape to unknown code. */ +const originalBuiltinReceiver = ( + node: t.Node | undefined, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (node === undefined) return false; + const receivers = builtinEffectIndex(state).receivers; + const previous = receivers.get(node); + if (previous !== undefined) return previous; + const result = computeOriginalBuiltinReceiver(node, state); + receivers.set(node, result); + return result; +}; + +const computeOriginalBuiltinReceiver = ( + node: t.Node | undefined, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (!t.isCallExpression(node) && !t.isOptionalCallExpression(node)) + return false; + const callee = node.callee; + if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) + return false; + const receiver = t.isIdentifier(callee.object) + ? resolveSemanticBindingState(state, callee.object, callee.object.name) + : undefined; + const provenance = + receiver === undefined + ? undefined + : evaluateSemanticProvenance(receiver, state); + const sourceOrigin = + provenance?.status === "module" + ? provenance.origins[0] + : (semanticExpressionModuleOrigin(callee.object, state) ?? + semanticRequireOrigin(callee.object, state)?.origin ?? + semanticGlobalProcessOrigin(callee.object, state)); + if (sourceOrigin === undefined) return false; + const origin = builtinNamespaceOrigin(sourceOrigin); + if (origin.importedPath.length !== 0) return false; + const method = semanticStaticPropertyKey(callee.property, callee.computed); + const identity = semanticRequireExportsIdentity(origin.specifier, [ + method ?? "", + ]); + if ( + method === null || + !BUILTIN_RECEIVER_METHODS[origin.specifier]?.has(method) || + receiver?.memberWrites.some( + ({ path, node: write }) => + affectsBuiltinPath(path, [method]) && + !semanticTopLevelEffectFollowsCapture(node, write, state.parentsByNode), + ) || + ( + builtinEffectIndex(state).writesByModule.get(identity.specifier) ?? [] + ).some( + ({ path, node: write }) => + affectsBuiltinPath(path, identity.path) && + !semanticTopLevelEffectFollowsCapture(node, write, state.parentsByNode), + ) + ) + return false; + return !node.arguments.some((argument) => { + if (!t.isIdentifier(argument)) return false; + const passed = resolveSemanticBindingState(state, argument, argument.name); + if (passed === undefined) return false; + const passedOrigin = evaluateSemanticProvenance(passed, state); + return ( + passedOrigin.status === "module" && + builtinNamespaceOrigin(passedOrigin.origins[0]).specifier === + origin.specifier && + builtinNamespaceOrigin(passedOrigin.origins[0]).importedPath.length === 0 + ); + }); +}; + +const semanticExpressionModuleOrigin = ( + expression: t.Node, + state: JavaScriptSemanticAnalysisState, + visited = new Set(), +): JavaScriptModuleOrigin | undefined => { + const node = unwrapJavaScriptExpression(expression).node; + const required = semanticRequireOrigin(node, state); + if (required !== undefined) return required.origin; + if (t.isIdentifier(node)) { + const binding = resolveSemanticBindingState(state, node, node.name); + if (binding === undefined || visited.has(binding.bindingId)) + return undefined; + const provenance = evaluateSemanticProvenance(binding, state); + return provenance.status === "module" ? provenance.origins[0] : undefined; + } + if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { + const base = semanticExpressionModuleOrigin(node.object, state, visited); + const key = semanticStaticPropertyKey(node.property, node.computed); + return base === undefined || key === null + ? undefined + : { ...base, importedPath: [...base.importedPath, key] }; + } + return undefined; }; /** Recognize global or node:process aliases without accepting shadowing. */ export const isSemanticProcessObject = ( node: t.Expression | t.Super, state: JavaScriptSemanticAnalysisState, + family: "env" | "argv", + key?: string | null, ): boolean => { if (!t.isIdentifier(node)) return false; const binding = resolveSemanticBindingState(state, node, node.name); + const selected = key === undefined || key === null ? [family] : [family, key]; if (binding === undefined) return ( - node.name === "process" && - !semanticResolutionBlocked(state, node, node.name) + semanticGlobalProcessOrigin(node, state) !== undefined && + !requiredModuleChanged("process", selected, node, state) ); - return binding.directOrigins.some( - ({ specifier }) => specifier === "process" || specifier === "node:process", + return ( + builtinBindingPath( + binding, + state, + ["process", "node:process"], + node, + selected, + )?.length === 0 ); }; diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index 607843aab..c6d47dfd2 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -1,7 +1,5 @@ import * as t from "@babel/types"; -import { semanticBuiltinNamespacePath } from "./javascriptSemanticProvenance.js"; - import type { JavaScriptSemanticBoundaryOperation, JavaScriptSemanticCallable, @@ -140,12 +138,12 @@ const processConfigurationMember = ( const object = node.object; if (!t.isMemberExpression(object) && !t.isOptionalMemberExpression(object)) return null; - if (!isProcessObject(object.object, state)) return null; const family = semanticStaticPropertyName(object.property, object.computed); if (family !== "env" && family !== "argv") return null; // `process.env[""]` names an exact (empty) key; only a dynamic key is unknown. const key = semanticStaticPropertyKey(node.property, node.computed); if (family === "argv" && key !== null && !isArrayIndexKey(key)) return null; + if (!isProcessObject(object.object, state, family, key)) return null; return { kind: family === "env" ? "environment" : "argv", key, @@ -375,22 +373,13 @@ const requestMethod = ( if (t.isNewExpression(node) && unbound && name === "WebSocket") return "WebSocket"; } - const member = memberCallee(node); - if (member === null || !t.isIdentifier(member.object)) return null; - const method = semanticStaticPropertyName(member.property, member.computed); - if (method !== "request" && method !== "get") return null; - const binding = resolveSemanticBindingState( - state, - member.object, - member.object.name, - ); - return binding?.directOrigins.some( - ({ specifier, importedPath }) => - ["http", "https", "node:http", "node:https"].includes(specifier) && - semanticBuiltinNamespacePath(importedPath), - ) - ? method - : null; + const method = builtinMethod(node.callee, state, [ + "http", + "https", + "node:http", + "node:https", + ]); + return method === "request" || method === "get" ? method : null; }; const requestFields = ( diff --git a/src/domain/javascript/javascriptSemanticGraphSchemas.ts b/src/domain/javascript/javascriptSemanticGraphSchemas.ts index 4048ae698..115d3762e 100644 --- a/src/domain/javascript/javascriptSemanticGraphSchemas.ts +++ b/src/domain/javascript/javascriptSemanticGraphSchemas.ts @@ -184,7 +184,6 @@ const relativePathSchema = z .refine( (path) => !path.startsWith("/") && - !path.includes("\\") && path.split("/").every((part) => !["", ".", ".."].includes(part)), "Expected a normalized relative POSIX path without traversal", ); diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 167e8899e..f3d7063b7 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -1,5 +1,14 @@ import * as t from "@babel/types"; +import { + semanticGlobalProcessOrigin, + semanticGlobalProcessPathOrigin, + semanticOriginalGlobalProcessAt, + semanticMutableBuiltinOrigin, + semanticRequireExportsIdentity, + semanticRequireOrigin, +} from "./javascriptSemanticRequireOrigin.js"; + import { resolveSemanticBindingState, type JavaScriptSemanticBindingState, @@ -35,6 +44,7 @@ import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; import { semanticStaticPropertyKey, + semanticWriteSiteTarget, semanticWriteTargets, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; @@ -50,7 +60,41 @@ export function* collectSemanticMemberMutationsSteps( const parents = state.parentsByNode; const observablePaths = observableMutationPaths(state); const returnedReferences = createSemanticReturnedReferences(state); - const recordedEffects = new Set(); + const recordedRequiredEffects = new Set(); + let recordedEffectCount = 0; + const recordRequiredEffect = ( + specifier: string, + path: PropertyPath, + effect: ValueEffect, + node: t.Node | undefined, + ): void => { + const identity = semanticRequireExportsIdentity(specifier, path); + const key = JSON.stringify([ + "required", + identity.specifier, + effect, + identity.path, + node?.start, + node?.end, + ]); + if (recordedRequiredEffects.has(key)) return; + recordedRequiredEffects.add(key); + recordedEffectCount += 1; + const effects = state.requiredModuleEffects.get(identity.specifier) ?? []; + effects.push({ path: identity.path, node, kind: effect }); + state.requiredModuleEffects.set(identity.specifier, effects); + }; + // Ambient indexing owns aliases of global objects. Consume its canonical + // writes instead of reparsing qualified or aliased process assignments. + for (const { path, node } of state.ambientWriteEffects) + if ( + path[0] === "global" && + path[1] === "process" && + path.length > 2 && + node !== undefined && + semanticOriginalGlobalProcessAt(node, state) + ) + recordRequiredEffect("node:process", path.slice(2), "write", node); const selections = new Map(); const coveringPaths = new Map(); let arrayIterationUnknown = false; @@ -171,9 +215,37 @@ export function* collectSemanticMemberMutationsSteps( const current = pending[cursor]; if (current === undefined) break; const expression = unwrapJavaScriptExpression(current.node).node; + const required = semanticRequireOrigin(expression, state); + if (required !== undefined) { + recordRequiredEffect( + required.origin.specifier, + [...required.origin.importedPath, ...current.path], + effect, + mutation, + ); + continue; + } + const globalProcess = semanticGlobalProcessPathOrigin(expression, state); + if (globalProcess !== undefined) { + recordRequiredEffect( + globalProcess.specifier, + [...globalProcess.importedPath, ...current.path], + effect, + mutation, + ); + continue; + } if (t.isIdentifier(expression)) { const binding = resolveBinding(expression, expression.name); if (binding === undefined) { + const processOrigin = semanticGlobalProcessOrigin(expression, state); + if (processOrigin !== undefined) + recordRequiredEffect( + processOrigin.specifier, + current.path, + effect, + mutation, + ); if ( (expression.name === "Array" && (current.path[0] === "prototype" || current.path[0] === null)) || @@ -270,52 +342,6 @@ export function* collectSemanticMemberMutationsSteps( index === selectorIndex ? { ...selector, excludedKeys } : key, ); } - // Keep an antichain of exact paths. Widening distinct member keys would - // also invalidate unrelated siblings that never reach this effect. - const pathState = JSON.stringify([ - binding.bindingId, - effect, - // Short writes stop at owned literal slots, whereas longer writes - // reach shared children. Only equal-depth writes cover one another. - effect === "write" ? path.length : null, - mutation?.start, - mutation?.end, - current.originAt?.start, - current.originAt?.end, - current.ownerInitializer?.start, - current.ownerInitializer?.end, - ]); - let covering = coveringPaths.get(pathState); - if (covering === undefined) { - covering = new SemanticPropertyPathCoverage(); - coveringPaths.set(pathState, covering); - } - if (!covering.retain(path)) continue; - const identity = JSON.stringify([ - binding.bindingId, - effect, - path, - mutation?.start, - mutation?.end, - current.originAt?.start, - current.originAt?.end, - current.ownerInitializer?.start, - current.ownerInitializer?.end, - ]); - if (recordedEffects.has(identity)) continue; - // A recorded write invalidates the binding to a non-primitive value, so - // only its first write can target a primitive. Re-evaluating it would - // replay every earlier write and make repeated writes superlinear. - const value = - effect === "write" && binding.mutatedPaths.length === 0 - ? evaluateSemanticBinding(binding, state) - : undefined; - const primitiveWrite = - value?.status === "literal" || value?.status === "union"; - recordedEffects.add(identity); - const nested = new Set([...current.bindings, lifetime, captureKey]); - // Returning a literal's receiver reaches that allocation, not the - // binding's older values read while evaluating its initializer. const origins = current.ownerInitializer !== undefined ? { @@ -323,6 +349,7 @@ export function* collectSemanticMemberMutationsSteps( ({ node }) => node === current.ownerInitializer, ), referenceInitializers: [], + directOrigins: [], } : current.originAt === undefined ? binding @@ -331,10 +358,57 @@ export function* collectSemanticMemberMutationsSteps( current.originAt, parents, ); + // Keep an antichain of exact paths. Widening distinct member keys would + // also invalidate unrelated siblings that never reach this effect. + // This map is local to the file scan. Escaping the binding id keeps a + // field separator injective while avoiding a fresh JSON array encode. + const pathState = `${JSON.stringify(binding.bindingId)}\u0001${ + effect + }\u0001${ + // Short writes stop at owned literal slots, whereas longer writes + // reach shared children. Only equal-depth writes cover one another. + effect === "write" ? path.length : -1 + }\u0001${mutation?.start ?? -1}\u0001${mutation?.end ?? -1}\u0001${ + current.originAt?.start ?? -1 + }\u0001${current.originAt?.end ?? -1}\u0001${ + current.ownerInitializer?.start ?? -1 + }\u0001${current.ownerInitializer?.end ?? -1}`; + let covering = coveringPaths.get(pathState); + if (covering === undefined) { + covering = new SemanticPropertyPathCoverage(); + coveringPaths.set(pathState, covering); + } + if (!covering.retain(path)) continue; + // A recorded write invalidates the binding to a non-primitive value, so + // only its first write can target a primitive. Re-evaluating it would + // replay every earlier write and make repeated writes superlinear. + const value = + effect === "write" && binding.memberWrites.length === 0 + ? evaluateSemanticBinding(binding, state) + : undefined; + const primitiveWrite = + value?.status === "literal" || value?.status === "union"; + // The coverage index already owns binding-effect deduplication. + recordedEffectCount += 1; + const nested = new Set(current.bindings); + nested.add(lifetime); + nested.add(captureKey); + // Returning a literal's receiver reaches that allocation, not the + // binding's older values read while evaluating its initializer. if (!primitiveWrite) { + for (const origin of origins.directOrigins) { + const shared = semanticMutableBuiltinOrigin(origin, path); + if (shared !== undefined) + recordRequiredEffect( + shared.specifier, + shared.path, + effect, + mutation, + ); + } if (effect === "escape") binding.escapedPaths.push({ path, node: mutation }); - else binding.mutatedPaths.push(path); + else binding.memberWrites.push({ path, node: mutation }); clearSemanticValueCaches(state); for (const initializer of origins.initializers) pending.push({ @@ -514,12 +588,8 @@ export function* collectSemanticMemberMutationsSteps( originAt: source.mutation, iterationOnly: source.kind === "iteration", }); - if (t.isAssignmentExpression(node)) markTarget(node.left, node); - else if (t.isUpdateExpression(node)) markTarget(node.argument, node); - else if (t.isUnaryExpression(node, { operator: "delete" })) - markTarget(node.argument, node); - else if (t.isForOfStatement(node) || t.isForInStatement(node)) - markTarget(node.left, node); + const writeTarget = semanticWriteSiteTarget(node); + if (writeTarget !== null) markTarget(writeTarget, node); else if ( t.isCallExpression(node) || t.isOptionalCallExpression(node) || @@ -552,7 +622,7 @@ export function* collectSemanticMemberMutationsSteps( // A later effect can make a previously defined destructuring source uncertain. // Reconsider defaults and iterable origins until their effects are stable. while (pendingReferences.length > 0 || iterableReferences.length > 0) { - const effectsBefore = recordedEffects.size; + const effectsBefore = recordedEffectCount; const iterablesBefore = iterableReferences.length; const iterationBefore: boolean = arrayIterationUnknown; for (const [index, reference] of pendingReferences.splice(0).entries()) { @@ -631,7 +701,7 @@ export function* collectSemanticMemberMutationsSteps( ); } if ( - recordedEffects.size === effectsBefore && + recordedEffectCount === effectsBefore && iterableReferences.length === iterablesBefore && arrayIterationUnknown === iterationBefore ) @@ -881,6 +951,15 @@ const observableMutationPaths = ( for (const binding of state.bindingsById.values()) { const observer = bindingNodes.get(binding.bindingId); if (observer === undefined) continue; + // Imported values are external objects with observable member slots even + // when their syntax has no local object-allocation node (for example a + // CommonJS namespace from `require("node:fs")`). Keep those writes at their + // exact member paths instead of widening the first path to a wildcard. + const hasParsedRequireOrigin = binding.initializers.some( + ({ node }) => semanticRequireOrigin(node, state) !== undefined, + ); + if (binding.directOrigins.length > 0 || hasParsedRequireOrigin) + (observer.origins ??= new Set()).add("opaque"); for (const initializer of binding.initializers) { link( project(expressionNode(initializer.node), initializer.projection), diff --git a/src/domain/javascript/javascriptSemanticMutationInitializers.test.ts b/src/domain/javascript/javascriptSemanticMutationInitializers.test.ts index 8b5364578..8e1c92d5f 100644 --- a/src/domain/javascript/javascriptSemanticMutationInitializers.test.ts +++ b/src/domain/javascript/javascriptSemanticMutationInitializers.test.ts @@ -48,7 +48,7 @@ describe("parameter references copied into function body bindings", () => { name: "arg", kind: "variable", mutable: true, - mutatedPaths: [], + memberWrites: [], escapedPaths: [], definitions: [], initializers: [entry, assigned], diff --git a/src/domain/javascript/javascriptSemanticMutationInitializers.ts b/src/domain/javascript/javascriptSemanticMutationInitializers.ts index 414d3e26d..fba388226 100644 --- a/src/domain/javascript/javascriptSemanticMutationInitializers.ts +++ b/src/domain/javascript/javascriptSemanticMutationInitializers.ts @@ -73,6 +73,94 @@ const positionPrecedes = ( left.declaratorIndex >= 0 && left.declaratorIndex < right.declaratorIndex)); +/** Order effects only in a non-repeating top-level activation. */ +export const semanticTopLevelEffectFollowsCapture = ( + capture: t.Node, + effect: t.Node | undefined, + parents: WeakMap, +): boolean => { + const position = (node: t.Node | undefined): StatementPosition | null => { + let selected: StatementPosition | null = null; + let declarator: t.VariableDeclarator | undefined; + let child: t.Node | undefined; + while (node !== undefined) { + if (t.isFunction(node) || t.isLoop(node)) return null; + if ( + (t.isClassProperty(node) || + t.isClassPrivateProperty(node) || + t.isClassAccessorProperty(node)) && + !node.static && + node.value === child + ) + return null; + if (t.isVariableDeclarator(node)) declarator = node; + if (selected === null && t.isStatement(node)) + selected = statementPosition(node, parents, declarator); + if (t.isProgram(node)) return selected; + child = node; + node = parents.get(node); + } + return null; + }; + const before = position(capture); + const after = position(effect); + if (before === null || after === null) return false; + if (positionPrecedes(before, after)) return true; + if ( + before.body !== after.body || + before.index !== after.index || + before.declaratorIndex !== after.declaratorIndex + ) + return false; + return expressionEvaluationPrecedes(capture, effect, parents); +}; + +/** Prove ordering only where JavaScript evaluates sibling expressions left-to-right. */ +const expressionEvaluationPrecedes = ( + earlier: t.Node, + later: t.Node | undefined, + parents: WeakMap, +): boolean => { + if (later === undefined) return false; + const earlierAncestors = new Set(); + let current: t.Node | undefined = earlier; + while (current !== undefined) { + earlierAncestors.add(current); + current = parents.get(current); + } + const laterPath: t.Node[] = []; + current = later; + while (current !== undefined && !earlierAncestors.has(current)) { + laterPath.push(current); + current = parents.get(current); + } + const common = current; + if (common === undefined || common === earlier || common === later) + return false; + let earlierChild: t.Node = earlier; + while (parents.get(earlierChild) !== common) { + const parent = parents.get(earlierChild); + if (parent === undefined) return false; + earlierChild = parent; + } + const laterChild = laterPath.at(-1); + if (laterChild === undefined) return false; + const orderedChildren = ((): readonly t.Node[] | null => { + if (t.isSequenceExpression(common)) return common.expressions; + if (t.isCallExpression(common) || t.isOptionalCallExpression(common)) + return [common.callee, ...common.arguments]; + if (t.isNewExpression(common)) return [common.callee, ...common.arguments]; + if (t.isBinaryExpression(common) || t.isLogicalExpression(common)) + return [common.left, common.right]; + if (t.isConditionalExpression(common)) return [common.test]; + return null; + })(); + if (orderedChildren === null) return false; + const earlierIndex = orderedChildren.indexOf(earlierChild); + const laterIndex = orderedChildren.indexOf(laterChild); + return earlierIndex >= 0 && laterIndex > earlierIndex; +}; + const computeDirectInitializerPosition = ( node: t.Node, parents: WeakMap, @@ -211,12 +299,64 @@ export const semanticMutationInitializers = ( parents: WeakMap, ): Pick< JavaScriptSemanticBindingState, - "initializers" | "referenceInitializers" + "initializers" | "referenceInitializers" | "directOrigins" > => { const mutationPosition = directMutationPosition(mutation, parents) ?? referenceInitializerPosition(mutation, parents); - return selectInitializers(binding, mutationPosition, parents); + if (mutationPosition === null) return binding; + const origins = [ + ...binding.initializers.map((initializer) => ({ + initializer, + position: + initializer.entryBody === undefined + ? directInitializerPosition(initializer.node, parents) + : { + body: initializer.entryBody.body, + index: -1, + declaratorIndex: -1, + }, + })), + ...binding.referenceInitializers.map((initializer) => ({ + initializer, + position: referenceInitializerPosition(initializer.node, parents), + })), + ]; + if ( + origins.some( + ({ position }) => + position === null || !positionPrecedes(position, mutationPosition), + ) + ) + return binding; + const latest = origins.reduce( + (latest, { position }) => + position !== null && + (latest === null || positionPrecedes(latest, position)) + ? position + : latest, + null, + ); + const retained = new Set( + origins + .filter( + ({ position }) => + position?.index === latest?.index && + position?.declaratorIndex === latest?.declaratorIndex, + ) + .map(({ initializer }) => initializer), + ); + return { + // A proven earlier assignment replaces even an import-equals value. A + // capture before or inside an uncertain assignment retains the full binding. + directOrigins: origins.length === 0 ? binding.directOrigins : [], + initializers: binding.initializers.filter((initializer) => + retained.has(initializer), + ), + referenceInitializers: binding.referenceInitializers.filter((initializer) => + retained.has(initializer), + ), + }; }; /** Select a local alias at its read, before any later straight-line reassignment. */ @@ -309,6 +449,16 @@ const selectInitializers = ( ) .map(({ initializer }) => initializer), ); + // Preserve the owner's identity when selection removes no facts. Consumers + // can then reuse the binding's dependency-aware provenance memo rather than + // treating every equivalent read as a new temporal snapshot. + if ( + binding.initializers.every((initializer) => retained.has(initializer)) && + binding.referenceInitializers.every((initializer) => + retained.has(initializer), + ) + ) + return binding; return { initializers: binding.initializers.filter((initializer) => retained.has(initializer), diff --git a/src/domain/javascript/javascriptSemanticObjects.ts b/src/domain/javascript/javascriptSemanticObjects.ts index 08acf25f8..8c0b7af8b 100644 --- a/src/domain/javascript/javascriptSemanticObjects.ts +++ b/src/domain/javascript/javascriptSemanticObjects.ts @@ -4,12 +4,12 @@ import type { JavaScriptSemanticCallable, JavaScriptSemanticObjectOperation, } from "./javascriptSemanticIr.js"; -import { - semanticCallableIdForNode, - semanticReadsBeforeWrite, -} from "./javascriptSemanticProjection.js"; +import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js"; import { semanticStaticPropertyKey, + semanticWriteSiteReadsFirst, + semanticWriteSiteTarget, + semanticWriteTargets, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; import { @@ -28,13 +28,21 @@ export const collectJavaScriptSemanticObjects = ( const output: JavaScriptSemanticObjectOperation[] = []; const callableIds = new Set(callables.map(({ callableId }) => callableId)); const callableStack: string[] = []; - const context: ObjectCollectionContext = { state, output }; + const context: ObjectCollectionContext = { + state, + output, + writeTargets: new Map(), + }; traverseJavaScriptAst(program, { enter: (node, parent) => { const callableId = semanticCallableIdForNode(node); if (callableId !== null && callableIds.has(callableId)) callableStack.push(callableId); const ownerCallableId = callableStack.at(-1) ?? null; + const writeTarget = semanticWriteSiteTarget(node); + if (writeTarget !== null) + for (const leaf of semanticWriteTargets(writeTarget)) + if (!t.isIdentifier(leaf)) context.writeTargets.set(leaf, node); if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) collectMember(node, parent, ownerCallableId, context); else if ( @@ -73,6 +81,7 @@ export const collectJavaScriptSemanticObjects = ( interface ObjectCollectionContext { readonly state: JavaScriptSemanticAnalysisState; readonly output: JavaScriptSemanticObjectOperation[]; + readonly writeTargets: Map; } const collectMember = ( @@ -90,10 +99,11 @@ const collectMember = ( ) return; const identity = expressionObjectIdentity(node, state); - const write = - (t.isAssignmentExpression(parent) && parent.left === node) || - (t.isUpdateExpression(parent) && parent.argument === node); - const readsBeforeWrite = write && semanticReadsBeforeWrite(node, parent); + const writeSite = context.writeTargets.get(node); + context.writeTargets.delete(node); + const write = writeSite !== undefined; + const readsBeforeWrite = + writeSite !== undefined && semanticWriteSiteReadsFirst(writeSite); if (readsBeforeWrite) addObjectOperation( { diff --git a/src/domain/javascript/javascriptSemanticProjection.ts b/src/domain/javascript/javascriptSemanticProjection.ts index b477bb5af..59e92b98d 100644 --- a/src/domain/javascript/javascriptSemanticProjection.ts +++ b/src/domain/javascript/javascriptSemanticProjection.ts @@ -27,6 +27,10 @@ import { range } from "./javascriptStaticAnalysisHelpers.js"; import { propertyName, semanticStaticPropertyKey, + semanticWrappedWriteSite, + semanticWriteSiteReadsFirst, + semanticWriteSiteTarget, + unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; import { isTypeOnlyExportDeclaration, @@ -175,12 +179,17 @@ export function* collectSemanticReferencesSteps( !patterns.some((pattern) => bindsIdentifier(pattern, parent, node))) ) return; - const role = semanticIdentifierRole(node, parent, readAncestors()); + const ancestors = readAncestors(); + const role = semanticIdentifierRole(node, parent, ancestors); if (role === null) return; // A compound assignment or update reads before it writes; emit both so // reference consumers see the read they actually execute. const roles = - role === "write" && semanticReadsBeforeWrite(node, parent) + role === "write" && + semanticReadsBeforeWrite( + node, + semanticWrappedWriteSite(node, parent, ancestors) ?? parent, + ) ? (["read", "write"] as const) : ([role] as const); for (const resolvedRole of roles) { @@ -594,12 +603,15 @@ const declarationCallableId = ( */ export const semanticReadsBeforeWrite = ( node: t.Node, - parent: t.Node, -): boolean => - (t.isAssignmentExpression(parent) && - parent.left === node && - parent.operator !== "=") || - (t.isUpdateExpression(parent) && parent.argument === node); + site: t.Node, +): boolean => { + const target = semanticWriteSiteTarget(site); + return ( + target !== null && + unwrapJavaScriptExpression(target).node === node && + semanticWriteSiteReadsFirst(site) + ); +}; /** * Role of one identifier occurrence, given the node's ancestry. @@ -620,6 +632,8 @@ export const semanticIdentifierRole = ( if (isPatternOwnerDeclaration(patternAncestor)) return null; return "write"; } + const site = semanticWrappedWriteSite(node, parent, ancestors); + if (site !== null && site !== parent && !isDeleteSite(site)) return "write"; return semanticReferenceRole(node, parent); }; @@ -684,6 +698,9 @@ const isPatternNode = (node: t.Node): boolean => t.isAssignmentPattern(node) || t.isRestElement(node); +const isDeleteSite = (site: t.Node): boolean => + t.isUnaryExpression(site, { operator: "delete" }); + const isPatternOwnerDeclaration = (node: t.Node): boolean => (t.isVariableDeclarator(node) && t.isPattern(node.id)) || (t.isFunction(node) && @@ -815,16 +832,15 @@ const commonJsExportName = ( if (!t.isMemberExpression(node) && !t.isOptionalMemberExpression(node)) return undefined; const key = semanticStaticPropertyKey(node.property, node.computed); - // `exports[key]`/`module.exports[key]` assign an export whose name is not - // knowable. Report the wildcard rather than the variable name, and never - // collapse it into `default`, which would claim a real default export. + // A computed nonliteral key is unknown, but the empty string is a real exact + // CommonJS export name and must not collapse into the wildcard/default. if (isUnshadowedGlobal(node.object, state, "exports")) - return key === null ? "*" : key || "*"; + return key === null ? "*" : key; if ( t.isMemberExpression(node.object) && isCommonJsModuleExports(node.object, state) ) - return key === null ? "*" : key || "default"; + return key === null ? "*" : key; if (isCommonJsModuleExports(node, state)) return "default"; return undefined; }; diff --git a/src/domain/javascript/javascriptSemanticRequireOrigin.ts b/src/domain/javascript/javascriptSemanticRequireOrigin.ts index 63fde3a95..a4a153597 100644 --- a/src/domain/javascript/javascriptSemanticRequireOrigin.ts +++ b/src/domain/javascript/javascriptSemanticRequireOrigin.ts @@ -1,5 +1,6 @@ import * as t from "@babel/types"; +import { BUILTIN_RESOURCE_METHODS } from "./javascriptBuiltinResourceMethods.js"; import type { JavaScriptModuleOrigin } from "./javascriptSemanticIr.js"; import { resolveSemanticBindingState, @@ -10,18 +11,243 @@ import { semanticStaticPropertyKey, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; +import { semanticPropertyPathKeyMatches } from "./javascriptSemanticPropertyPaths.js"; import { stringValue } from "./javascriptStaticAnalysisHelpers.js"; +import { semanticTopLevelEffectFollowsCapture } from "./javascriptSemanticMutationInitializers.js"; +import { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; -interface JavaScriptRequireOrigin { +export interface JavaScriptRequireOrigin { readonly origin: JavaScriptModuleOrigin; readonly call: t.CallExpression; } -/** Recover an unshadowed literal require origin and its exact member path. */ -export const semanticRequireOrigin = ( - node: t.Node | null | undefined, +// Ambient indexing finishes before these derived facts are consumed. Keep only +// replacement sites, rather than rescanning unrelated global member writes. +const globalProcessReplacements = new WeakMap< + JavaScriptSemanticAnalysisState, + readonly (t.Node | undefined)[] +>(); +const moduleRequireWrites = new WeakMap< + JavaScriptSemanticAnalysisState, + readonly t.Node[] +>(); + +const writesModuleRequire = ( state: JavaScriptSemanticAnalysisState, -): JavaScriptRequireOrigin | undefined => { +): readonly t.Node[] => { + const cached = moduleRequireWrites.get(state); + if (cached !== undefined) return cached; + // Interop invocation resolution may ask for require provenance while this + // index is being built. Publish a temporary empty result to break that cycle. + moduleRequireWrites.set(state, []); + const writes: t.Node[] = []; + const interop = createSemanticModuleInterop(state); + const isModuleObject = ( + node: t.Node, + visited = new Set(), + ): boolean => { + const value = unwrapJavaScriptExpression(node).node; + if (!t.isIdentifier(value)) return false; + if (value.name === "module") + return ( + resolveSemanticBindingState(state, value, "module") === undefined && + !semanticResolutionBlocked(state, value, "module") + ); + const binding = resolveSemanticBindingState(state, value, value.name); + if (binding === undefined || visited.has(binding.bindingId)) return false; + const next = new Set(visited).add(binding.bindingId); + return ( + binding.initializers.length > 0 && + binding.initializers.every(({ node: init }) => isModuleObject(init, next)) + ); + }; + const pending: t.Node[] = [state.program]; + while (pending.length > 0) { + const node = pending.pop(); + if (node === undefined) break; + const target = t.isAssignmentExpression(node) + ? node.left + : t.isUpdateExpression(node) || + t.isUnaryExpression(node, { operator: "delete" }) + ? node.argument + : undefined; + if (target !== undefined) { + const members: string[] = []; + let root: t.Node = target; + while (t.isMemberExpression(root) || t.isOptionalMemberExpression(root)) { + const key = semanticStaticPropertyKey(root.property, root.computed); + if (key === null) break; + members.unshift(key); + root = root.object; + } + if ( + members.length === 1 && + members[0] === "require" && + isModuleObject(root) + ) + writes.push(node); + else if ( + members.length === 0 && + (t.isMemberExpression(target) || + t.isOptionalMemberExpression(target)) && + target.computed && + semanticStaticPropertyKey(target.property, true) === null && + isModuleObject(target.object) + ) + // An unknown computed key may be "require"; retain uncertainty rather + // than treating this direct delegate write as a proven unrelated slot. + writes.push(node); + } + if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) { + const invocation = interop.intrinsicInvocation(node); + const owner = + invocation?.path?.length === 3 ? invocation.path[1] : undefined; + const method = invocation?.path?.at(-1); + const args = invocation?.arguments; + if ( + (owner === "Object" || owner === "Reflect") && + method === "defineProperty" && + isModuleObject(args?.[0] ?? t.nullLiteral()) + ) { + const key = interop.staticString(args?.[1]); + if (key === "require" || key === undefined) writes.push(node); + } else if ( + owner === "Object" && + method === "defineProperties" && + isModuleObject(args?.[0] ?? t.nullLiteral()) + ) { + const descriptors = args?.[1]; + if ( + !t.isObjectExpression(descriptors) || + descriptors.properties.some( + (property) => + t.isSpreadElement(property) || + (t.isObjectProperty(property) && + semanticStaticPropertyKey(property.key, property.computed) === + "require"), + ) + ) + writes.push(node); + } else if ( + owner === "Object" && + method === "assign" && + args?.some((arg) => isModuleObject(arg)) + ) { + const target = args?.[0]; + if (target !== undefined && isModuleObject(target)) { + const sourceHasRequire = args + ?.slice(1) + .some( + (arg) => + t.isObjectExpression(arg) && + arg.properties.some( + (property) => + t.isObjectProperty(property) && + semanticStaticPropertyKey( + property.key, + property.computed, + ) === "require", + ), + ); + if ( + sourceHasRequire || + args?.slice(1).some((arg) => !t.isObjectExpression(arg)) + ) + writes.push(node); + } + } + } + for (const key of t.VISITOR_KEYS[node.type] ?? []) { + const value = (node as unknown as Record)[key]; + if (Array.isArray(value)) { + for (const child of value) if (t.isNode(child)) pending.push(child); + } else if (t.isNode(value)) pending.push(value); + } + } + moduleRequireWrites.set(state, writes); + return writes; +}; + +/** Whether the original global process object can still own effects at this source site. */ +export const semanticOriginalGlobalProcessAt = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): boolean => { + let replacements = globalProcessReplacements.get(state); + if (replacements === undefined) { + replacements = [ + ...state.ambientWriteEffects + .filter( + ({ path }) => + path.length <= 2 && + path.every((key, index) => + semanticPropertyPathKeyMatches( + key, + ["global", "process"][index]!, + ), + ), + ) + .map(({ node: write }) => write), + ...(state.ambientBindingWrites.get("process") ?? []), + ]; + globalProcessReplacements.set(state, replacements); + } + return !replacements.some( + (write) => + !semanticTopLevelEffectFollowsCapture(node, write, state.parentsByNode), + ); +}; + +/** Node's unbound process reference shares the cached process exports object. */ +export const semanticGlobalProcessOrigin = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): JavaScriptModuleOrigin | undefined => + t.isIdentifier(node, { name: "process" }) && + resolveSemanticBindingState(state, node, "process") === undefined && + !semanticResolutionBlocked(state, node, "process") && + semanticOriginalGlobalProcessAt(node, state) + ? { specifier: "node:process", importedPath: [], resolutionMode: "require" } + : undefined; + +/** Resolve qualified global-object process references, preserving their selected path. */ +export const semanticGlobalProcessPathOrigin = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): JavaScriptModuleOrigin | undefined => { + const members: string[] = []; + let current = unwrapJavaScriptExpression(node).node; + while ( + t.isMemberExpression(current) || + t.isOptionalMemberExpression(current) + ) { + const key = semanticStaticPropertyKey(current.property, current.computed); + if (key === null) return undefined; + members.unshift(key); + current = unwrapJavaScriptExpression(current.object).node; + } + if ( + !t.isIdentifier(current) || + !["globalThis", "global", "window", "self"].includes(current.name) || + resolveSemanticBindingState(state, current, current.name) !== undefined || + semanticResolutionBlocked(state, current, current.name) || + members[0] !== "process" || + !semanticOriginalGlobalProcessAt(node, state) + ) + return undefined; + return { + specifier: "node:process", + importedPath: members.slice(1), + resolutionMode: "require", + }; +}; + +/** Strip a static member chain, returning its root expression and member path. */ +export const semanticMemberChain = ( + node: t.Node | null | undefined, +): + | { readonly root: t.Node | null | undefined; readonly members: string[] } + | undefined => { const members: string[] = []; let current = node; while (current !== null && current !== undefined) { @@ -39,15 +265,45 @@ export const semanticRequireOrigin = ( members.push(member); current = current.object; } + return { root: current, members: members.reverse() }; +}; + +/** Recover an unshadowed literal require origin and its exact member path. */ +export const semanticRequireOrigin = ( + node: t.Node | null | undefined, + state: JavaScriptSemanticAnalysisState, +): JavaScriptRequireOrigin | undefined => { + const chain = semanticMemberChain(node); + const current = chain?.root; if ( + chain === undefined || !t.isCallExpression(current) || !t.isIdentifier(current.callee, { name: "require" }) ) return undefined; + if ( + writesModuleRequire(state).some( + (write) => + !semanticTopLevelEffectFollowsCapture( + current, + write, + state.parentsByNode, + ), + ) + ) + return undefined; if ( resolveSemanticBindingState(state, current.callee, "require") !== undefined || - semanticResolutionBlocked(state, current.callee, "require") + semanticResolutionBlocked(state, current.callee, "require") || + (state.ambientBindingWrites.get("require") ?? []).some( + (write) => + !semanticTopLevelEffectFollowsCapture( + current, + write, + state.parentsByNode, + ), + ) ) return undefined; const specifier = stringValue(current.arguments[0]); @@ -56,9 +312,50 @@ export const semanticRequireOrigin = ( : { origin: { specifier, - importedPath: members.reverse(), + importedPath: chain.members, resolutionMode: "require", }, call: current, }; }; + +/** Node's fs.promises export and fs/promises require share one object. */ +export const semanticRequireExportsIdentity = ( + specifier: string, + path: readonly Key[], +): { readonly specifier: string; readonly path: readonly (Key | string)[] } => { + const name = specifier.replace(/^node:/u, ""); + if (name === "fs/promises") return { specifier: name, path }; + return name === "fs" && path[0] === "promises" && path.length > 1 + ? { specifier: "fs/promises", path: path.slice(1) } + : { specifier: name, path }; +}; + +const MUTABLE_BUILTIN_DEFAULTS = new Set([ + ...Object.keys(BUILTIN_RESOURCE_METHODS), + "http", + "https", + "child_process", + "timers", + "process", +]); + +/** Mutable default/object selections share CommonJS identity; namespace slots do not. */ +export const semanticMutableBuiltinOrigin = ( + origin: JavaScriptModuleOrigin, + selected: readonly Key[] = [], +): + | { readonly specifier: string; readonly path: readonly (Key | string)[] } + | undefined => { + const path = [...origin.importedPath, ...selected]; + if (origin.resolutionMode === "require") + return { specifier: origin.specifier, path }; + const specifier = origin.specifier.replace(/^node:/u, ""); + if (!MUTABLE_BUILTIN_DEFAULTS.has(specifier)) return undefined; + if (path[0] === "default") return { specifier, path: path.slice(1) }; + if (specifier === "process" && (path[0] === "env" || path[0] === "argv")) + return { specifier, path }; + return specifier === "fs" && path[0] === "promises" + ? { specifier, path } + : undefined; +}; diff --git a/src/domain/javascript/javascriptSemanticResources.ts b/src/domain/javascript/javascriptSemanticResources.ts index 0a080fc90..4cfd066e9 100644 --- a/src/domain/javascript/javascriptSemanticResources.ts +++ b/src/domain/javascript/javascriptSemanticResources.ts @@ -1,7 +1,9 @@ import * as t from "@babel/types"; -import { semanticBuiltinNamespacePath } from "./javascriptSemanticProvenance.js"; - +import { + BUILTIN_RESOURCE_SPECIFIERS, + BUILTIN_RESOURCE_METHODS, +} from "./javascriptBuiltinResourceMethods.js"; import type { JavaScriptSemanticCallable, JavaScriptSemanticResourceOperation, @@ -13,6 +15,7 @@ import { type JavaScriptSemanticBindingState, } from "./javascriptSemanticState.js"; import { + builtinDataEffectMethod, dataEffectMemberCallee, outerDataEffectBinding, traverseDataEffects, @@ -20,14 +23,6 @@ import { } from "./javascriptSemanticDataEffectHelpers.js"; import { range } from "./javascriptStaticAnalysisHelpers.js"; -const ACQUIRE_METHODS = [ - "open", - "openSync", - "createReadStream", - "createWriteStream", - "connect", - "createConnection", -] as const; const RELEASE_METHODS = ["close", "destroy", "end"] as const; interface ResourceCandidate { @@ -137,58 +132,19 @@ const acquisitionMethod = ( callee: t.Node, state: JavaScriptSemanticAnalysisState, ): JavaScriptSemanticResourceOperation["method"] | null => { - if (t.isIdentifier(callee)) { - const binding = resolveSemanticBindingState(state, callee, callee.name); - return methodFromBinding(binding); - } - if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) - return null; - const method = ACQUIRE_METHODS.find( - (candidate) => - candidate === - semanticStaticPropertyName(callee.property, callee.computed), - ); - if (method === undefined || !t.isIdentifier(callee.object)) return null; - const binding = resolveSemanticBindingState( - state, - callee.object, - callee.object.name, - ); - return resourceNamespace(binding) ? method : null; -}; - -const methodFromBinding = ( - binding: JavaScriptSemanticBindingState | undefined, -): JavaScriptSemanticResourceOperation["method"] | null => { - for (const { specifier, importedPath } of binding?.directOrigins ?? []) { - if (!resourceModule(specifier)) continue; - const method = ACQUIRE_METHODS.find( - (candidate) => candidate === importedPath.at(-1), - ); - if (method !== undefined) return method; + for (const specifier of BUILTIN_RESOURCE_SPECIFIERS) { + const method = builtinDataEffectMethod(callee, state, [specifier]); + if ( + method !== null && + ( + BUILTIN_RESOURCE_METHODS[ + specifier as keyof typeof BUILTIN_RESOURCE_METHODS + ] as readonly string[] + ).includes(method) + ) + return method as JavaScriptSemanticResourceOperation["method"]; } return null; }; - -const resourceNamespace = ( - binding: JavaScriptSemanticBindingState | undefined, -): boolean => - binding?.directOrigins.some( - ({ specifier, importedPath }) => - resourceModule(specifier) && semanticBuiltinNamespacePath(importedPath), - ) ?? false; - -const resourceModule = (specifier: string): boolean => - [ - "fs", - "fs/promises", - "net", - "node:fs", - "node:fs/promises", - "node:net", - "node:tls", - "tls", - ].includes(specifier); - const resourceCandidateId = (candidate: ResourceCandidate): string => `resource:acquire:${String(candidate.node.start ?? -1)}:${String(candidate.node.end ?? -1)}`; diff --git a/src/domain/javascript/javascriptSemanticReturnedReferences.ts b/src/domain/javascript/javascriptSemanticReturnedReferences.ts index fbf1dbb56..bdaf2af58 100644 --- a/src/domain/javascript/javascriptSemanticReturnedReferences.ts +++ b/src/domain/javascript/javascriptSemanticReturnedReferences.ts @@ -74,10 +74,35 @@ export const createSemanticReturnedReferences = ( const pending: Candidate[] = [ { node: callee, path: [], bindings: new Set() }, ]; + // A shared initializer can be reached through both arms of every alias + // diamond. Expand an expression/path again only when its cycle context + // permits a binding that an earlier traversal blocked. + const expanded = new WeakMap< + t.Node, + Map[]> + >(); for (let cursor = 0; cursor < pending.length; cursor++) { const item = pending[cursor]; if (item === undefined) break; const node = unwrapJavaScriptExpression(item.node).node; + const pathKey = JSON.stringify(item.path); + const paths = + expanded.get(node) ?? new Map[]>(); + const contexts = paths.get(pathKey) ?? []; + if ( + contexts.some((context) => + [...context].every((binding) => item.bindings.has(binding)), + ) + ) + continue; + paths.set(pathKey, [ + ...contexts.filter( + (context) => + ![...item.bindings].every((binding) => context.has(binding)), + ), + item.bindings, + ]); + expanded.set(node, paths); if (t.isIdentifier(node)) { const binding = resolveSemanticBindingState(state, node, node.name); if (binding === undefined || item.bindings.has(binding.bindingId)) diff --git a/src/domain/javascript/javascriptSemanticState.ts b/src/domain/javascript/javascriptSemanticState.ts index d01a4be52..7dac75c97 100644 --- a/src/domain/javascript/javascriptSemanticState.ts +++ b/src/domain/javascript/javascriptSemanticState.ts @@ -32,17 +32,23 @@ interface JavaScriptSemanticReferenceInitializer extends JavaScriptSemanticIniti } /** Mutable binding state used only while constructing the immutable IR. */ +export interface JavaScriptSemanticMemberEffect { + readonly path: JavaScriptSemanticPropertyPath; + readonly node: t.Node | undefined; +} + +export interface JavaScriptSemanticRequiredModuleEffect extends JavaScriptSemanticMemberEffect { + readonly kind: "write" | "escape"; +} + export interface JavaScriptSemanticBindingState { readonly bindingId: string; readonly scopeId: string; readonly name: string; kind: JavaScriptSemanticDefinition["kind"]; mutable: boolean; - readonly mutatedPaths: JavaScriptSemanticPropertyPath[]; - readonly escapedPaths: { - readonly path: JavaScriptSemanticPropertyPath; - readonly node: t.Node | undefined; - }[]; + readonly memberWrites: JavaScriptSemanticMemberEffect[]; + readonly escapedPaths: JavaScriptSemanticMemberEffect[]; readonly definitions: JavaScriptSemanticDefinition[]; readonly initializers: JavaScriptSemanticInitializer[]; readonly referenceInitializers: JavaScriptSemanticReferenceInitializer[]; @@ -63,6 +69,7 @@ export interface JavaScriptSemanticScopeState { /** Shared state for semantic collection and evaluation. */ export interface JavaScriptSemanticAnalysisState { + readonly program: t.Program; /** Effective loader for static imports in this source artifact. */ readonly staticImportResolutionMode: JavaScriptModuleResolutionMode; readonly parentsByNode: WeakMap; @@ -71,6 +78,11 @@ export interface JavaScriptSemanticAnalysisState { /** Scope boundaries and descendant scopes resolved on demand. */ readonly scopeByNode: WeakMap; readonly bindingsById: Map; + /** Effects on a cached require exports object, independent of local projections. */ + readonly requiredModuleEffects: Map< + string, + JavaScriptSemanticRequiredModuleEffect[] + >; readonly callables: JavaScriptSemanticCallable[]; readonly callableNodesById: Map; readonly moduleLinks: JavaScriptSemanticModuleLink[]; @@ -81,8 +93,12 @@ export interface JavaScriptSemanticAnalysisState { JavaScriptSemanticBindingState >; readonly conditionalInitializers: WeakSet; - /** Source-local writes, tagged by global-object or import.meta identity. */ + /** Unbound identifier writes retain their source sites for capture ordering. */ + readonly ambientBindingWrites: Map; + /** Source-local writes, tagged by global object or import.meta identity. */ readonly ambientWrites: SemanticPropertyPathCoverage; + /** Source sites distinguish replacement before an ambient identity capture from later writes. */ + readonly ambientWriteEffects: JavaScriptSemanticMemberEffect[]; } /** diff --git a/src/domain/javascript/javascriptSemanticValues.memo.test.ts b/src/domain/javascript/javascriptSemanticValues.memo.test.ts index 8f32a723c..5a9395cdb 100644 --- a/src/domain/javascript/javascriptSemanticValues.memo.test.ts +++ b/src/domain/javascript/javascriptSemanticValues.memo.test.ts @@ -86,7 +86,7 @@ it("invalidates retained alias values at the producing mutation boundary", () => expect( property(property(evaluateSemanticBinding(root, state), "alias"), "value"), ).toEqual({ status: "literal", value: 1 }); - target.mutatedPaths.push(["value"]); + target.memberWrites.push({ path: ["value"], node: undefined }); clearSemanticValueCaches(state); expect( property(property(evaluateSemanticBinding(root, state), "alias"), "value") diff --git a/src/domain/javascript/javascriptSemanticValues.ts b/src/domain/javascript/javascriptSemanticValues.ts index d15c5edfc..276131f08 100644 --- a/src/domain/javascript/javascriptSemanticValues.ts +++ b/src/domain/javascript/javascriptSemanticValues.ts @@ -9,6 +9,7 @@ import { semanticCapturePosition, semanticEscapeFollowsCapture, semanticBindingPrecedesCapture, + semanticReadInitializers, } from "./javascriptSemanticMutationInitializers.js"; import { semanticSlotAtPath } from "./javascriptSemanticSlots.js"; @@ -228,6 +229,21 @@ export const evaluateSemanticProvenance = ( memo: createEvaluationMemo(state), }); +/** Follow a namespace binding as it was read at a specific source position. */ +export const evaluateSemanticProvenanceAtRead = ( + binding: JavaScriptSemanticBindingState, + read: t.Identifier, + state: JavaScriptSemanticAnalysisState, +): JavaScriptBindingProvenance => + provenanceForBindingAtRead(binding, read, { + state, + bindings: new Map(), + expressionDepth: 0, + primitiveBindingValues: primitiveBindingValuesFor(state), + dependence: { lowest: CALLER_INDEPENDENT }, + memo: createEvaluationMemo(state), + }); + const readMemoizedEvaluation = ( entry: MemoizedEvaluation | undefined, context: EvaluationContext, @@ -383,7 +399,10 @@ const evaluateBinding = ( ? callerDependence : Math.min(callerDependence, context.dependence.lowest); } - let projected = invalidateSemanticMutationPaths(value, binding.mutatedPaths); + let projected = invalidateSemanticMutationPaths( + value, + binding.memberWrites.map(({ path }) => path), + ); for (const escape of binding.escapedPaths) { // Escapes only invalidate object references; other lattice values survive. if (projected.status !== "object" && projected.status !== "array") break; @@ -912,7 +931,7 @@ const provenanceForExpression = ( "unknown", `Unbound identifier ${node.name}.`, ) - : provenanceForBinding(binding, nestedContext(context)); + : provenanceForBindingAtRead(binding, node, nestedContext(context)); } if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { const members: string[] = []; @@ -983,6 +1002,40 @@ const provenanceForExpression = ( ); }; +/** Resolve an alias from the value visible at this read, before later rebinds. */ +const provenanceForBindingAtRead = ( + binding: JavaScriptSemanticBindingState, + read: t.Identifier, + context: EvaluationContext, +): JavaScriptBindingProvenance => { + const selected = semanticReadInitializers( + binding, + read, + context.state.parentsByNode, + ); + if (selected.initializers === binding.initializers) + return provenanceForBinding(binding, context); + if (context.bindings.has(binding.bindingId)) + return semanticUnresolvedProvenance( + "cycle", + `Alias cycle at ${binding.name}.`, + ); + const snapshot: JavaScriptSemanticBindingState = { + ...binding, + ...selected, + // Assignment facts after the read cannot change the captured value. + definitions: binding.definitions.filter( + ({ kind }) => kind !== "assignment", + ), + }; + context.bindings.set(binding.bindingId, context.bindings.size); + try { + return provenanceForBindingValue(snapshot, context); + } finally { + context.bindings.delete(binding.bindingId); + } +}; + const isLocallyCreatedValue = (node: t.Node): boolean => readExactJavaScriptLiteral(node).found || t.isObjectExpression(node) || diff --git a/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts b/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts index 6dbf855fc..36cb537ae 100644 --- a/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts +++ b/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts @@ -8,9 +8,252 @@ import { expect, it } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; const execute = promisify(execFile); +it("preserves earlier and sibling effects without inventing replaced cached-module or process effects", async () => { + const root = await createTestTempDirectory("rea-builtin-identity-"); + await writeFile( + join(root, "app.cjs"), + ` + throw new Error('parse only'); + const a = require('node:fs'); const b = require('fs'); + b.readFileSync('before.json'); const stream = b.createReadStream('before.json'); stream.destroy(); + patch(a); b.readFileSync('wrong-after.json'); + const proc = require('node:process'); proc.env = {}; proc.argv = []; const mode = proc.env.MODE; const arg = proc.argv[2]; + `, + ); + await writeFile( + join(root, "sibling.mjs"), + `import fs from 'node:fs'; fs.writeFileSync = custom; fs.readFileSync('sibling.json'); fs.readFileSync = custom;`, + ); + await writeFile( + join(root, "captures.cjs"), + `const fs = require('node:fs'); + const saved = require('fs').readFileSync; + const { readFileSync: savedDestructured } = fs; + fs.readFileSync('before-write.json'); fs.readFileSync = custom; + const { readFileSync: late } = require('fs'); + const lateMember = require('fs').readFileSync; + saved('saved.json'); savedDestructured('saved-destructured.json'); + late('wrong-late.json'); lateMember('wrong-member.json');`, + ); + await writeFile( + join(root, "direct.cjs"), + `require('node:fs').readFileSync('direct.json'); + const fs = require('fs'); fs.readFileSync('later.json'); + require('fs/promises').open('before'); + fs.promises.open = custom; require('node:fs/promises').open('wrong');`, + ); + await writeFile( + join(root, "loader.cjs"), + `require = () => ({ readFileSync() {} }); + const fs = require('node:fs'); fs.readFileSync('wrong-loader.json'); + require('node:fs').readFileSync('wrong-direct-loader.json');`, + ); + await writeFile( + join(root, "imports.ts"), + `import fs = require('node:fs'); const other = require('fs'); + const read = other.readFileSync; observe(read); read('escaped.json'); + fs.readFileSync = custom; other.readFileSync('wrong-import-equals.json');`, + ); + await writeFile( + join(root, "late-loader.cjs"), + `const fs=require('fs'); fs.readFileSync('loader-before.json'); + require=replacement; fs.readFileSync('loader-after.json'); + require('fs').readFileSync('wrong-late-loader.json');`, + ); + await writeFile( + join(root, "replacement.ts"), + `import fs = require('fs'); const other=require('fs'); fs={}; fs.readFileSync=custom; other.readFileSync('replacement-kept.json');`, + ); + await writeFile( + join(root, "dynamic.cjs"), + `const fs=require('fs'); fs.hook=custom; fs[key](); fs.readFileSync('wrong-dynamic.json');`, + ); + await writeFile( + join(root, "wrapped.cjs"), + `const fs=require('fs'); let assigned; const saved=(0, fs.readFileSync); fs.readFileSync=custom; const late=(assigned=fs.readFileSync); saved('wrapped-kept.json'); late('wrong-wrapped.json');`, + ); + await writeFile( + join(root, "global-process.cjs"), + `process.env={MODE:'local'}; const proc=require('process'); const mode=proc.env.MODE;`, + ); + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", "analyze-javascript-application", root, "--json"], + { cwd: process.cwd(), maxBuffer: 16 * 1024 * 1024 }, + ); + const evidence = parseEvidence(JSON.parse(stdout)); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + expect( + result.semantic_graph.nodes + .filter(({ kind }) => kind === "config-source") + .map(({ properties }) => properties.key) + .sort(), + ).toEqual([ + "before-write.json", + "before.json", + "direct.json", + "escaped.json", + "later.json", + "loader-after.json", + "loader-before.json", + "replacement-kept.json", + "saved-destructured.json", + "saved.json", + "sibling.json", + "wrapped-kept.json", + ]); + expect( + result.semantic_graph.nodes.filter(({ kind }) => kind === "resource"), + ).toHaveLength(2); + expect( + result.semantic_graph.relations.filter( + ({ relation }) => relation === "releases", + ), + ).toHaveLength(1); + const { client, call } = await connectLocalToolsMcp(); + const response = await call("analyze_javascript_application", { + input_path: root, + format: "auto", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(evidence); + await client.ping(); +}, 60000); + +it("excludes monkey-patched CommonJS effects from CLI and MCP evidence", async () => { + const root = await createTestTempDirectory("rea-builtin-mutated-"); + await writeFile( + join(root, "app.cjs"), + ` + throw new Error("parse only"); + const fs = require("node:fs"); fs.readFileSync = custom; fs.createReadStream = custom; + const cp = require("node:child_process"); cp.spawn = custom; + const timers = require("node:timers"); patch(timers); + const http = require("node:http"); delete http.get; + fs.readFileSync("config.json"); const stream = fs.createReadStream("config.json"); stream.destroy(); + const child = cp.spawn("tool", []); child.kill(); + const timer = timers.setTimeout(callback, 1); clearTimeout(timer); + http.get("http://example.test"); + `, + ); + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", "analyze-javascript-application", root, "--json"], + { cwd: process.cwd(), maxBuffer: 16 * 1024 * 1024 }, + ); + const evidence = parseEvidence(JSON.parse(stdout)); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + expect( + result.semantic_graph.nodes.filter(({ kind }) => + [ + "config-source", + "resource", + "child-process", + "timer", + "request", + ].includes(kind), + ), + ).toEqual([]); + expect( + result.semantic_graph.relations.filter(({ relation }) => + ["releases", "forwards-signal", "cancels-timer"].includes(relation), + ), + ).toEqual([]); + const { client, call } = await connectLocalToolsMcp(); + const response = await call("analyze_javascript_application", { + input_path: root, + format: "auto", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(evidence); + await client.ping(); +}, 60000); + +it("retains CommonJS builtin effects and handle links through CLI and MCP", async () => { + const root = await createTestTempDirectory("rea-builtin-commonjs-"); + await writeFile( + join(root, "app.cjs"), + ` + throw new Error("This source must only be parsed, never executed"); + const fs = require("node:fs"); + const {spawn: launch} = require("node:child_process"); + const timers = require("timers"); + const {get} = require("http"); + const proc = require("node:process"); + const text = fs.readFileSync("config.json", "utf8"); + const stream = fs.createReadStream("config.json"); stream.destroy(); + const child = launch("tool", []); child.kill("SIGTERM"); + const timer = timers.setTimeout(callback, 10); timers.clearTimeout(timer); + get("http://example.test"); + const mode = proc.env.MODE; + const absent = require("node:fs").default; absent.readFileSync("wrong-default"); + const external = require("./fs.js"); external.readFileSync("wrong-external"); + function shadowed(require) { const fs = require("node:fs"); fs.readFileSync("wrong"); } + `, + ); + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", "analyze-javascript-application", root, "--json"], + { cwd: process.cwd(), maxBuffer: 16 * 1024 * 1024 }, + ); + const evidence = parseEvidence(JSON.parse(stdout)); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + const nodes = result.semantic_graph.nodes; + expect( + nodes.filter(({ kind }) => + [ + "config-source", + "resource", + "child-process", + "timer", + "request", + ].includes(kind), + ), + ).toHaveLength(6); + expect(nodes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "config-source", + properties: expect.objectContaining({ key: "config.json" }), + }), + expect.objectContaining({ + kind: "config-source", + properties: expect.objectContaining({ key: "MODE" }), + }), + ...["createReadStream", "spawn", "setTimeout", "get"].map((method) => + expect.objectContaining({ + properties: expect.objectContaining({ method }), + }), + ), + ]), + ); + const relations = result.semantic_graph.relations; + for (const relation of ["releases", "forwards-signal", "cancels-timer"]) + expect(relations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ relation, resolution: "resolved" }), + ]), + ); + const { client, call } = await connectLocalToolsMcp(); + const response = await call("analyze_javascript_application", { + input_path: root, + format: "auto", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(evidence); + await client.ping(); +}, 60000); + it.each([ "import fs from 'node:fs';", "import {default as fs} from 'node:fs';", diff --git a/tests/fixtures/javascriptSemanticGraph.ts b/tests/fixtures/javascriptSemanticGraph.ts index 0615b5419..04e7e009e 100644 --- a/tests/fixtures/javascriptSemanticGraph.ts +++ b/tests/fixtures/javascriptSemanticGraph.ts @@ -26,6 +26,7 @@ export const graphForJavaScript = ( packages: [], json_modules: [], html_scripts: [], + html_inline_scripts: [], source_maps: [], visited_ast_nodes: 0, findings: 0, From 41a2e846470bd2af40dd298d975b1a1d2fef5786 Mon Sep 17 00:00:00 2001 From: William Blair <60581672+williamblair333@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:45:50 -0400 Subject: [PATCH 17/26] docs(installation): limit REA to one project or agent profile Document existing Claude Code profile and per-project installation workflows, including scoped maintenance, backups, caller metadata and Windows commands. Clarify authentication storage and the filesystem/network restrictions required for enforced separation. Validated 68 boundary tests, docs checks, formatting, typecheck/lint and two real REA setup dry-runs with zero applied actions or profile writes. Native Claude sessions, PowerShell execution and enforcement policies were not tested. Setup automation remains separate in #2187. Co-authored-by: William Blair <60581672+williamblair333@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- docs/installation.md | 144 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index 38c4d7966..75da703cf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -129,6 +129,150 @@ Setup pins package-runner MCP registrations to the exact installed REA version, installs the matching skill and on-demand references in the same plan, and adds `startup_timeout_sec = 30` for Codex and Grok Build. +## Limit REA to one project or agent profile + +Setup registers REA for the whole client. For Claude Code it writes +`mcpServers.rea` in `~/.claude.json` and installs the bundled skill under +`~/.claude/skills`, so every project that Claude Code opens can call REA. To +offer REA's tools only where they're wanted, use one of the routes below. + +Both routes control which sessions discover REA's MCP server and skill. They +leave shell commands, filesystem access and network access available. An agent +under the same OS account can run `rea` from `PATH` or `npx` and read other +profiles and REA output. For clean-room separation, use an implementation +environment with filesystem and network restrictions that prevent reading or +fetching REA, the target and analysis output, including through shared +installations and package registries. A separate account or container must +enforce those restrictions. + +For Claude Code, check existing default-profile state before choosing either +route. Setup is additive: a previous user-level REA registration and personal +skill remain active in other projects. Back up the default configuration and +REA skill first, then, with `CLAUDE_CONFIG_DIR` unset, remove the REA user +registration with `claude mcp remove --scope user rea` and remove the known +REA-installed `~/.claude/skills/reverse-engineer-anything` directory. Preserve +unrelated registrations and skills. `rea uninstall` applies to other clients +as well, so it is broader than this Claude-only migration. + +**A separate Claude Code profile.** Claude Code keeps settings, skills and session +history under `CLAUDE_CONFIG_DIR`. Setup follows the same +variable: it writes the registration to `$CLAUDE_CONFIG_DIR/.claude.json` and +the skill to `$CLAUDE_CONFIG_DIR/skills`. + +```bash +CLAUDE_CONFIG_DIR=~/.claude-rea npx rea-agents setup --client claude_code +CLAUDE_CONFIG_DIR=~/.claude-rea claude +``` + +A new profile may need authentication. Console sign-ins without an API key are +stored outside `CLAUDE_CONFIG_DIR`; see +[Claude Code authentication](https://code.claude.com/docs/en/authentication#log-in-with-multiple-accounts). +Sessions started without the variable use the default profile's registration, +skill and history. + +Inline assignments apply only to that command. Keep the same prefix for every +profile check, setup rerun, update, or removal; bare commands use the current +environment and can otherwise inspect or recreate default-profile state: + +```bash +CLAUDE_CONFIG_DIR=~/.claude-rea claude mcp list +CLAUDE_CONFIG_DIR=~/.claude-rea npx rea-agents doctor --client claude_code --skill --json +CLAUDE_CONFIG_DIR=~/.claude-rea npx -y rea-agents@latest setup --client claude_code --dry-run +``` + +After reviewing an update plan, rerun that same prefixed setup command without +`--dry-run` and approve its exact changes. Profile removal likewise needs the +prefix on `claude mcp remove --scope user rea` and any `rea uninstall` command; +keep the broader client scope of uninstall in mind. Restart the selected +profile after changing its pinned registration. + +`claude mcp remove` removes only the registration. To remove REA from this +profile without running the broader uninstall flow, first back up the profile's +REA-managed skill directory, then remove +`$CLAUDE_CONFIG_DIR/skills/reverse-engineer-anything` as well. Keep other skills +in the profile intact. + +The commands above use POSIX shell assignment syntax. In native Windows +PowerShell, select the profile for the whole setup/launch/check session and +restore the previous selection afterward: + +```powershell +$previousClaudeConfigDir = $env:CLAUDE_CONFIG_DIR +try { + $env:CLAUDE_CONFIG_DIR = Join-Path $HOME ".claude-rea" + npx rea-agents setup --client claude_code + claude mcp list + npx rea-agents doctor --client claude_code --skill --json + npx -y rea-agents@latest setup --client claude_code --dry-run + # After reviewing the plan, repeat setup without --dry-run and approve it. + claude + # Profile removal, when intended, also belongs inside this selected scope: + # claude mcp remove --scope user rea +} finally { + if ($null -eq $previousClaudeConfigDir) { + Remove-Item Env:CLAUDE_CONFIG_DIR -ErrorAction SilentlyContinue + } else { + $env:CLAUDE_CONFIG_DIR = $previousClaudeConfigDir + } +} +``` + +Use PowerShell for these Windows examples rather than pasting POSIX assignments +into cmd.exe. Keep profile updates, readiness checks and removal inside the +same selected environment; the `finally` block restores the previous profile +or unsets the variable when it was absent. + +**One project.** Install without configuring Claude Code: use the curl +installer's `--no-setup`, or run setup only for other clients. Then register +REA from the project directory at Claude Code's local scope. + +This route uses the default Claude profile. Keep `CLAUDE_CONFIG_DIR` unset for +the entire local lifecycle: registration, backup, listing, version replacement +and removal. In POSIX shells run `unset CLAUDE_CONFIG_DIR`; in PowerShell run +`Remove-Item Env:CLAUDE_CONFIG_DIR -ErrorAction SilentlyContinue` before these +commands. Record an existing selection first if you intend to restore it after +finishing. The configuration path below is the default profile's +`~/.claude.json`; a different active profile uses its own config root instead. + +Register with the variable unset: + + + +```bash +claude mcp add --scope local rea -- npx -y rea-agents@6.4.0 mcp +``` + +On native Windows, use the same project directory and wrap the package-runner +shim so the stdio client can launch it: + +```powershell +claude mcp add --scope local rea -- cmd.exe /d /c npx -y rea-agents@6.4.0 mcp +``` + + + +Local scope stores the entry under that directory's path in `~/.claude.json`, +so other projects don't load it. This route doesn't install the bundled skill. +Avoid `--scope project`, which writes `.mcp.json` into the repository where it +can be committed or copied. `rea doctor` and `rea uninstall` read only the +user-level entry, so they don't report or remove a local-scope registration; +remove it with `claude mcp remove --scope local rea` from the same directory. +`rea update` doesn't refresh it either, so it stays on the version it names. +To move it to a newer release, first back up the project entry in +`~/.claude.json` and record its environment and other metadata. In particular, +preserve caller-selected overrides such as `REA_MCP_INPUT_SCHEMA_PROFILE` and +`REA_MCP_MAX_RESPONSE_BYTES`. Run `claude mcp remove --scope local rea` from the +same directory, then recreate it with the platform-appropriate command above +and the new version. Reapply every recorded environment value with Claude +Code's `mcp add --env NAME=value` options and restore any other retained entry +metadata before restarting the project session. Removal and re-addition do not +preserve these values automatically. Running setup for Claude Code instead +would add a user-level entry for every project. + +Check the separate profile with the prefixed `claude mcp list` above and compare +with the default profile with `CLAUDE_CONFIG_DIR` unset. For local scope, run +`claude mcp list` from the selected project and again from another project. + ## Check readiness for your task Run `rea doctor` when you need diagnosis. It reads host prerequisites and agent From b58adafa4fa16d578132810f677eabc6d9ffdc09 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:46:50 +0800 Subject: [PATCH 18/26] fix(javascript): align module lookup and source evidence Preserve CommonJS literal paths and confined ESM URL semantics, including package main fallback, directory intent, Unicode identity, and present opaque modules. Parse extraction inputs at their AST/source owner, retain skipped HTML and source-map coverage, and distinguish endpoint candidates from proven receiver facts. Version structural fingerprints with computed literal contents and propagate them through worker contracts. Reviewed fixes include #1810, #1840, #2642, #2700, #2702, #2714, #2803, #2960, #2968, #2986, #3035, and #3036. Validation: 158 resolver tests passed; public CLI/MCP module lookup controls, package-main/Unicode workflows, and six builtin workflows passed in the current compiled lane. Worker schema/fingerprint regressions passed. Large public pdf.js completion and real Electron analysis remain unverified. --- docs/javascript-artifact-reconstruction.md | 17 +- .../javascript/ElectronBoundaryAnalysis.ts | 7 + .../javascript/ElectronBoundaryGraphNative.ts | 1 + .../ElectronBoundaryGraphWindows.ts | 57 ++- .../javascript/JavaScriptArtifactAnalysis.ts | 95 ++-- .../JavaScriptArtifactAnalysisTypes.ts | 11 + .../JavaScriptArtifactGraphBuilder.ts | 46 +- .../JavaScriptArtifactGraphContext.ts | 46 +- .../JavaScriptArtifactGraphDocuments.ts | 2 + .../JavaScriptArtifactGraphFindings.ts | 63 ++- .../JavaScriptArtifactGraphStructure.ts | 39 +- .../JavaScriptArtifactPathResolution.test.ts | 225 +++++++- .../JavaScriptArtifactPathResolution.ts | 343 ++++++++++--- .../JavaScriptArtifactReconstruction.ts | 9 +- .../JavaScriptModuleRelationships.ts | 2 + ...avaScriptRuntimeObservationService.test.ts | 46 +- .../JavaScriptRuntimeObservationService.ts | 2 + .../javascript/htmlArtifactReferences.test.ts | 83 ++- .../javascript/htmlArtifactReferences.ts | 126 ++++- .../javascriptApplicationEvidenceSchemas.ts | 2 +- .../javascript/javascriptArtifactFiles.ts | 26 + .../javascriptAstFingerprint.test.ts | 60 +++ .../javascript/javascriptAstFingerprint.ts | 10 +- ...scriptExportShapeComparisonSchemas.test.ts | 21 + .../javascriptExportShapeComparisonSchemas.ts | 11 +- .../javascriptExportShapeVariants.ts | 2 +- .../javascript/javascriptRpcEndpoints.test.ts | 19 + .../javascript/javascriptRpcEndpoints.ts | 33 +- ...javascriptRuntimeReconciliationMatching.ts | 50 +- .../javascriptRuntimeReconciliationSchemas.ts | 4 +- .../javascriptRuntimeStaticCandidates.ts | 164 +++++- .../javascript/javascriptSourceMapPaths.ts | 21 +- .../javascript/javascriptStaticAnalysis.ts | 2 + .../javascriptStaticAnalysisBundler.ts | 3 +- ...scriptStaticAnalysisCalleeBoundary.test.ts | 41 ++ .../javascriptStaticAnalysisCalls.ts | 189 ++++++- .../javascriptStaticAnalysisHelpers.ts | 77 ++- .../javascriptStaticAnalysisState.ts | 2 + .../javascriptStaticAnalysisTypes.ts | 7 +- .../javascriptSymbolicBindings.test.ts | 80 +++ .../javascript/javascriptSymbolicBindings.ts | 64 ++- .../javascriptVitePreloadDependencies.test.ts | 86 ++++ .../sourceToBundleComparison.test.ts | 12 +- .../javascript/sourceToBundleSignals.ts | 55 +- src/domain/webBundleAnalysis.ts | 7 +- src/domain/webBundleAnalyzer.test.ts | 344 ++++++++++++- src/domain/webBundleAnalyzerInspection.ts | 40 +- src/domain/webBundleAnalyzerResult.ts | 2 + .../JavaScriptAnalysisWorkerSchemas.test.ts | 33 ++ .../JavaScriptAnalysisWorkerSchemas.ts | 5 +- .../JavaScriptModuleLookupCli.test.ts | 27 +- .../applications/packageMainLookupCli.test.ts | 284 +++++++++++ .../packageSpecifierSuffixCli.test.ts | 106 ++++ .../applications/packageUrlUnicodeCli.test.ts | 268 ++++++++++ .../electronApplicationAnalysis.test.ts | 28 +- .../javascriptArtifactReconstruction.test.ts | 2 +- .../javascriptRuntimeReconciliation.test.ts | 482 ++++++++++++++++++ .../javascriptExportShapeComparison.test.ts | 80 +++ .../javascriptHtmlEntrypoints.test.ts | 182 +++++++ .../javascriptRequireLiteralPaths.test.ts | 36 ++ .../filesystem/sourceToBundlePaths.test.ts | 108 +++- .../filesystem/webScriptManifestPaths.test.ts | 81 +++ tests/support/cli/javascriptApplication.ts | 16 + 63 files changed, 4066 insertions(+), 326 deletions(-) create mode 100644 src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts create mode 100644 src/domain/javascript/javascriptVitePreloadDependencies.test.ts create mode 100644 src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts create mode 100644 tests/acceptance/applications/packageMainLookupCli.test.ts create mode 100644 tests/acceptance/applications/packageUrlUnicodeCli.test.ts diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 090902ff8..b166bf65e 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -282,9 +282,14 @@ suffix. Utility calls such as `useUtils().invoice.list.fetch()`, `useQueries`, and links whose callee was renamed by a minifier are not recognized. Each recovered bundle module retains the exact factory-source digest. A complete -bounded AST also receives a `babel-ast-v1` structural fingerprint that ignores +bounded AST also receives a `babel-ast-v2` structural fingerprint that ignores ordinary identifier names while retaining syntax, literals, operators, object -keys, and member properties. If fingerprint construction reaches its structural +keys, and member properties. Computed member identifiers retain their access +syntax without treating a local variable's spelling as a fixed property; template +literal contents remain part of the fingerprint. Version comparison keeps the +algorithm in its matching key, so retained `babel-ast-v1` fingerprints cannot match +these new fingerprints. Re-analyze both snapshots to compare their new structure. +If fingerprint construction reaches its structural bound, the fingerprint is unavailable and its status is `truncated`; a bounded prefix is never presented as a complete stable fingerprint. @@ -619,3 +624,11 @@ Unresolved HTML references retain their declaration, source range, and resolutio reason in the renderer observations. HTML script source ranges follow the HTML parser across LF, CRLF, and bare CR line endings, preserving UTF-16 columns. +Inline JavaScript in local HTML, meaning classic or module `', []], @@ -35,17 +38,21 @@ it.each([ ).toEqual(expected); }); -it("retains exact opening-tag offsets rather than the script body", () => { +it("retains exact opening-tag coordinates rather than the script body", () => { const opening = ''; const prefix = "\n\n"; const text = `${prefix}${opening}ignored body`; expect(htmlArtifactReferences(text)).toEqual({ baseHref: null, + inlineScripts: [], + inlineScriptCount: 0, scripts: [ { scriptPath: "actual.js", - startOffset: prefix.length, - endOffset: prefix.length + opening.length, + location: { + start: { line: 3, column: 0 }, + end: { line: 4, column: ' title=">" src=actual.js>'.length }, + }, }, ], }); @@ -68,3 +75,71 @@ it("traverses deep valid document content without recursive stack growth", () => htmlArtifactReferences(text).scripts.map(({ scriptPath }) => scriptPath), ).toEqual(["actual.js"]); }); + +it.each([ + ["", ["classic"]], + ['', ["classic"]], + ['', ["classic"]], + ['', ["classic"]], + ['', ["classic"]], + ['', ["classic"]], + ['', ["module"]], + // Only the JavaScript MIME check strips whitespace; `module` must match. + ['', []], + ['', []], + ["", ["classic"]], + // A src attribute, even an empty one, means the browser ignores the body. + ['', []], + ['', []], + ["", []], + ['', []], + ['', []], + ['', []], + ['', []], + ['', []], + ['', []], + ["", []], + ["", []], + ["", []], +])("classifies inline JavaScript in %s", (text, expected) => { + expect( + htmlArtifactReferences(text).inlineScripts.map( + ({ scriptType }) => scriptType, + ), + ).toEqual(expected); +}); + +it("retains exact inline script body coordinates, including unclosed scripts", () => { + const prefix = "\n

after

`; + expect(htmlArtifactReferences(closed).inlineScripts).toEqual([ + { + scriptType: "classic", + location: { start: { line: 2, column: 8 }, end: { line: 4, column: 0 } }, + }, + ]); + const unclosed = `${prefix}${body}`; + expect(htmlArtifactReferences(unclosed).inlineScripts).toEqual([ + { + scriptType: "classic", + location: { start: { line: 2, column: 8 }, end: { line: 4, column: 0 } }, + }, + ]); +}); + +it("counts every inline body but retains bounded locations", () => { + const count = 150_000; + const text = "".repeat(count); + const { inlineScripts, inlineScriptCount } = htmlArtifactReferences(text); + expect(inlineScriptCount).toBe(count); + expect(inlineScripts).toHaveLength(HTML_INLINE_SCRIPT_LOCATION_LIMIT); + const column = 18 * (HTML_INLINE_SCRIPT_LOCATION_LIMIT - 1) + 8; + expect(inlineScripts.at(-1)).toEqual({ + scriptType: "classic", + location: { + start: { line: 1, column }, + end: { line: 1, column: column + 1 }, + }, + }); +}); diff --git a/src/domain/javascript/htmlArtifactReferences.ts b/src/domain/javascript/htmlArtifactReferences.ts index fa9945e36..10a52f21a 100644 --- a/src/domain/javascript/htmlArtifactReferences.ts +++ b/src/domain/javascript/htmlArtifactReferences.ts @@ -4,23 +4,99 @@ import { type DefaultTreeAdapterTypes, } from "parse5"; +import type { + JavaScriptSourcePoint, + JavaScriptSourceRange, +} from "./javascriptStaticAnalysisTypes.js"; + interface HtmlScriptReference { readonly scriptPath: string; - readonly startOffset: number; - readonly endOffset: number; + /** The opening tag's range in parse5 coordinates. */ + readonly location: JavaScriptSourceRange; } -/** Extract inert HTML script references and the first document base href. */ +/** JavaScript written directly in a script element instead of loaded by URL. */ +export interface HtmlInlineScript { + readonly scriptType: "classic" | "module"; + /** The body's range, between the opening and closing tags. */ + readonly location: JavaScriptSourceRange; +} + +/** + * Inline bodies whose exact locations are retained per HTML file. Every body + * is still counted; locations beyond this bound are reported as omitted. + */ +export const HTML_INLINE_SCRIPT_LOCATION_LIMIT = 100; + +// The JavaScript MIME type essences from the WHATWG MIME Sniffing standard. +const JAVASCRIPT_MIME_TYPES = new Set([ + "application/ecmascript", + "application/javascript", + "application/x-ecmascript", + "application/x-javascript", + "text/ecmascript", + "text/javascript", + "text/javascript1.0", + "text/javascript1.1", + "text/javascript1.2", + "text/javascript1.3", + "text/javascript1.4", + "text/javascript1.5", + "text/jscript", + "text/livescript", + "text/x-ecmascript", + "text/x-javascript", +]); + +/** + * Classify a script element's type as HTML "prepare the script element" does. + * Only a type attribute is whitespace-stripped, and only for the JavaScript + * MIME check; `module` must match the type string exactly, ignoring ASCII + * case. Import maps, speculation rules and data blocks are not JavaScript. + */ +const inlineScriptType = ( + attrs: readonly { readonly name: string; readonly value: string }[], +): HtmlInlineScript["scriptType"] | null => { + const type = attrs.find(({ name }) => name === "type"); + const language = attrs.find(({ name }) => name === "language"); + const typeString = + type !== undefined + ? type.value + : language !== undefined && language.value !== "" + ? `text/${language.value}` + : ""; + if (typeString === "") return "classic"; + const mimeCandidate = + type === undefined + ? typeString + : typeString.replace(/^[\t\n\f\r ]+|[\t\n\f\r ]+$/gu, ""); + if (JAVASCRIPT_MIME_TYPES.has(asciiLowercase(mimeCandidate))) + return "classic"; + return asciiLowercase(typeString) === "module" ? "module" : null; +}; + +const asciiLowercase = (value: string): string => + value.replace(/[A-Z]/gu, (letter) => letter.toLowerCase()); + +/** + * Extract inert HTML script references, inline JavaScript bodies, and the + * first document base href. + */ export const htmlArtifactReferences = ( text: string, ): { readonly baseHref: string | null; readonly scripts: readonly HtmlScriptReference[]; + /** The first inline bodies in document order, up to the location limit. */ + readonly inlineScripts: readonly HtmlInlineScript[]; + readonly inlineScriptCount: number; } => { const pending: DefaultTreeAdapterTypes.Node[] = [ parse(text, { sourceCodeLocationInfo: true }), ]; const scripts: HtmlScriptReference[] = []; + const inlineScripts: HtmlInlineScript[] = []; + let inlineScriptCount = 0; let baseHref: string | null = null; while (pending.length > 0) { const node = pending.pop(); @@ -39,10 +115,20 @@ export const htmlArtifactReferences = ( if (src !== undefined && src.value !== "" && location !== undefined) { scripts.push({ scriptPath: src.value, - startOffset: location.startOffset, - endOffset: location.endOffset, + location: { + start: sourcePoint(location.startLine, location.startCol), + end: sourcePoint(location.endLine, location.endCol), + }, }); } + const inline = src === undefined ? inlineScriptBody(node) : undefined; + const scriptType = + inline === undefined ? null : inlineScriptType(node.attrs); + if (inline !== undefined && scriptType !== null) { + inlineScriptCount += 1; + if (inlineScripts.length < HTML_INLINE_SCRIPT_LOCATION_LIMIT) + inlineScripts.push({ scriptType, location: inline }); + } } } // Template content belongs to a separate inert document fragment. Only @@ -51,5 +137,33 @@ export const htmlArtifactReferences = ( for (const child of node.childNodes.toReversed()) pending.push(child); } } - return { baseHref, scripts }; + return { baseHref, scripts, inlineScripts, inlineScriptCount }; }; + +/** Locate a script element's non-blank body; browsers ignore it when src is set. */ +const inlineScriptBody = ( + node: DefaultTreeAdapterTypes.Element, +): JavaScriptSourceRange | undefined => { + const startTag = node.sourceCodeLocation?.startTag; + if (startTag === undefined) return undefined; + let blank = true; + let end: DefaultTreeAdapterTypes.TextNode["sourceCodeLocation"]; + for (const child of node.childNodes) { + if (!defaultTreeAdapter.isTextNode(child)) continue; + if (blank && /\S/u.test(child.value)) blank = false; + end = child.sourceCodeLocation ?? end; + } + // An unclosed script's element range ends at its opening tag, so the body's + // own text location bounds it. + if (blank || end == null) return undefined; + return { + start: sourcePoint(startTag.endLine, startTag.endCol), + end: sourcePoint(end.endLine, end.endCol), + }; +}; + +/** parse5 lines and UTF-16 columns are one-based; ranges use zero-based columns. */ +const sourcePoint = (line: number, column: number): JavaScriptSourcePoint => ({ + line, + column: column - 1, +}); diff --git a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts index 65bce595b..73f379895 100644 --- a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts +++ b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts @@ -6,7 +6,7 @@ const boundedTextSchema = z.string().min(1); const boundedKeySchema = z.string().min(1); const unsafePathSegments = new Set(["", ".", ".."]); const isSafeRelativePath = (path: string): boolean => { - if (path.startsWith("/") || path.includes("\\")) return false; + if (path.startsWith("/")) return false; return path.split("/").every((part) => !unsafePathSegments.has(part)); }; const relativePathSchema = z diff --git a/src/domain/javascript/javascriptArtifactFiles.ts b/src/domain/javascript/javascriptArtifactFiles.ts index 819c6ceae..e5c7e05fa 100644 --- a/src/domain/javascript/javascriptArtifactFiles.ts +++ b/src/domain/javascript/javascriptArtifactFiles.ts @@ -1,5 +1,9 @@ import type { JavaScriptSourceSelection } from "./javascriptSourceSelection.js"; +/** Whether an inventoried path has a source extension read by the analyzer. */ +export const isJavaScriptSourcePath = (path: string): boolean => + /\.(?:cjs|cts|mjs|mts|js|jsx|ts|tsx)$/iu.test(path); + /** Relevant file categories projected from the complete artifact inventory. */ export type JavaScriptArtifactFileKind = | "package-json" @@ -29,6 +33,26 @@ export interface JavaScriptArtifactFile { }; } +/** + * One inventoried regular file outside the JavaScript-relevant kinds. Its presence + * was inventoried, including unavailable or unhashed members. The source projection + * does not read its bytes; references remain present-but-unanalyzed rather than absent. + */ +export interface JavaScriptUnanalyzedArtifactFile { + readonly path: string; + readonly container_sha256: string; + readonly sha256: string | null; + readonly bytes: number | null; + readonly inventory_artifact_id: string | null; + readonly limitations?: readonly string[]; +} + +/** Borrowed inventory lookup; metadata is materialized only for requested paths. */ +export interface JavaScriptUnanalyzedArtifactLookup { + get(path: string): JavaScriptUnanalyzedArtifactFile | undefined; + hasDirectory(path: string, containerSha256: string | undefined): boolean; +} + /** One filesystem-backed ASAR nested beneath a directory input. */ export interface JavaScriptArtifactContainer { readonly path: string; @@ -40,6 +64,8 @@ export interface JavaScriptArtifactContainer { /** Deterministic relevant-file projection plus explicit omissions. */ export interface JavaScriptArtifactFileSet { readonly files: readonly JavaScriptArtifactFile[]; + /** Borrowed inventory lookup for regular files the source projection does not read. */ + readonly unanalyzed_files?: JavaScriptUnanalyzedArtifactLookup; readonly containers: readonly JavaScriptArtifactContainer[]; readonly text_bytes_read: number; readonly invalid_utf8_files: number; diff --git a/src/domain/javascript/javascriptAstFingerprint.test.ts b/src/domain/javascript/javascriptAstFingerprint.test.ts index ddbb85ade..4a12ac8e3 100644 --- a/src/domain/javascript/javascriptAstFingerprint.test.ts +++ b/src/domain/javascript/javascriptAstFingerprint.test.ts @@ -2,6 +2,7 @@ import { parse } from "@babel/parser"; import { describe, expect, it } from "vitest"; import { fingerprintJavaScriptAst } from "./javascriptAstFingerprint.js"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; describe("JavaScript AST fingerprints", () => { it("includes complete literal values in the fingerprint", () => { @@ -14,3 +15,62 @@ describe("JavaScript AST fingerprints", () => { ); }); }); + +it.each(["obj[key]", "obj?.[key]"])( + "ignores a local computed key binding in %s", + (expression) => { + const left = parse(`function read(obj, key) { return ${expression}; }`); + const renamed = expression + .replace("obj", "value") + .replace("key", "selector"); + const right = parse( + `function read(value, selector) { return ${renamed}; }`, + ); + expect(fingerprintJavaScriptAst(left)).toBe( + fingerprintJavaScriptAst(right), + ); + }, +); + +it("distinguishes computed reads from fixed named members", () => { + expect(fingerprintJavaScriptAst(parse("obj[key]"))).not.toBe( + fingerprintJavaScriptAst(parse("obj.key")), + ); + expect(fingerprintJavaScriptAst(parse("obj.key"))).not.toBe( + fingerprintJavaScriptAst(parse("obj.other")), + ); + expect(fingerprintJavaScriptAst(parse('obj["key"]'))).not.toBe( + fingerprintJavaScriptAst(parse('obj["other"]')), + ); +}); + +it.each([ + ["obj[`left`]", "obj[`right`]"], + ["obj[``]", "obj[`right`]"], + ["obj[true]", "obj[false]"], + ["obj[null]", "obj[false]"], + ['obj["" ]', 'obj["right"]'], + ["obj[`prefix-${key}`]", "obj[`changed-${key}`]"], +])("retains computed literal contents in %s", (left, right) => { + expect(fingerprintJavaScriptAst(parse(left))).not.toBe( + fingerprintJavaScriptAst(parse(right)), + ); +}); + +it("keeps renamed computed-key bundle modules structurally matchable", () => { + const source = (name: string) => `globalThis.webpackChunkApp.push([[1], { + 17(module, exports) { const ${name} = selectKey(); exports.read = value[${name}]; } + }]);`; + const before = analyzeJavaScriptStaticSource(source("key")) + .bundler_registrations[0]?.modules[0]; + const after = analyzeJavaScriptStaticSource(source("selector")) + .bundler_registrations[0]?.modules[0]; + expect(before).toBeDefined(); + expect(after).toBeDefined(); + expect(before?.structural_fingerprint_algorithm).toBe("babel-ast-v2"); + expect(before?.source_sha256).not.toBe(after?.source_sha256); + expect(before?.structural_fingerprint_sha256).toMatch(/^[a-f0-9]{64}$/u); + expect(before?.structural_fingerprint_sha256).toBe( + after?.structural_fingerprint_sha256, + ); +}); diff --git a/src/domain/javascript/javascriptAstFingerprint.ts b/src/domain/javascript/javascriptAstFingerprint.ts index b9e53f975..1dee83fa0 100644 --- a/src/domain/javascript/javascriptAstFingerprint.ts +++ b/src/domain/javascript/javascriptAstFingerprint.ts @@ -84,13 +84,19 @@ const semanticTokens = (node: t.Node): string[] => { tokens.push(`boolean:${String(node.value)}`); else if (t.isRegExpLiteral(node)) tokens.push(`regexp:${node.pattern}/${node.flags}`); + else if (t.isTemplateElement(node)) + tokens.push(`template:${JSON.stringify(node.value)}`); else if (t.isBinaryExpression(node) || t.isLogicalExpression(node)) tokens.push(`operator:${node.operator}`); else if (t.isUnaryExpression(node) || t.isUpdateExpression(node)) tokens.push(`operator:${node.operator}`); if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { - const property = propertyName(node.property); - if (property !== "") tokens.push(`property:${property}`); + // A computed identifier names a local binding, not a fixed member. + tokens.push(`computed:${String(node.computed)}`); + if (!node.computed) { + const property = propertyName(node.property); + if (property !== "") tokens.push(`property:${property}`); + } } if ((t.isObjectProperty(node) || t.isObjectMethod(node)) && !node.computed) { const key = propertyName(node.key); diff --git a/src/domain/javascript/javascriptExportShapeComparisonSchemas.test.ts b/src/domain/javascript/javascriptExportShapeComparisonSchemas.test.ts index 067452b7e..82e826d3b 100644 --- a/src/domain/javascript/javascriptExportShapeComparisonSchemas.test.ts +++ b/src/domain/javascript/javascriptExportShapeComparisonSchemas.test.ts @@ -1,10 +1,31 @@ import { expect, it } from "vitest"; import { + compareJavaScriptExportShapesInputSchema, javaScriptExportShapeComparisonChangeSchema, projectedExportReturnShapesSchema, } from "./javascriptExportShapeComparisonSchemas.js"; +it("accepts exact empty export names in selectors and projections", () => { + expect( + compareJavaScriptExportShapesInputSchema.shape.left_export_name.safeParse( + "", + ).success, + ).toBe(true); + expect( + projectedExportReturnShapesSchema.safeParse({ + ...projectionWithField({ + path: "/kind", + presence: "present", + state: "literal", + value: "ok", + reason: null, + }), + exported_name: "", + }).success, + ).toBe(true); +}); + const projectionWithField = (field: unknown) => ({ semantic_role: "export-return-shapes", module_path: "index.js", diff --git a/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts b/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts index 5f9425015..b7ac7ee4f 100644 --- a/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts +++ b/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts @@ -7,6 +7,7 @@ const evidenceIdSchema = prefixedDigestSchema("ev"); const graphIdSchema = prefixedDigestSchema("jag"); const nodeIdSchema = prefixedDigestSchema("jag_node"); const textSchema = z.string().min(1); +const exportNameSchema = z.string(); const semanticPrimitiveSchema = z.union([ z.string(), z.number().finite(), @@ -85,9 +86,9 @@ export const compareJavaScriptExportShapesInputSchema = z left: evidenceSchema, right: evidenceSchema, left_module_path: textSchema, - left_export_name: textSchema, + left_export_name: exportNameSchema, right_module_path: textSchema, - right_export_name: textSchema, + right_export_name: exportNameSchema, }) .superRefine(validateJavaScriptExportShapeIdentities); @@ -170,7 +171,7 @@ export const projectedExportReturnShapesSchema = z .strictObject({ semantic_role: z.literal("export-return-shapes"), module_path: textSchema, - exported_name: textSchema, + exported_name: exportNameSchema, callable_id: textSchema, callable_kind: z.enum(["function", "class", "method"]), static_return_shapes: z.array(projectedReturnShapeSchema), @@ -198,7 +199,7 @@ export const projectedExportReturnShapesSchema = z const exportCandidateSchema = z.strictObject({ node_id: nodeIdSchema, module_path: textSchema, - export_name: textSchema, + export_name: exportNameSchema, matches_requested_module: z.boolean(), matches_requested_export: z.boolean(), }); @@ -207,7 +208,7 @@ const selectorResultShape = { evidence_id: evidenceIdSchema, graph_id: graphIdSchema, requested_module_path: textSchema, - requested_export_name: textSchema, + requested_export_name: exportNameSchema, candidates: z.array(exportCandidateSchema), omitted_candidates: z.number().int().min(0), }; diff --git a/src/domain/javascript/javascriptExportShapeVariants.ts b/src/domain/javascript/javascriptExportShapeVariants.ts index bedf57cbe..c4f0839ac 100644 --- a/src/domain/javascript/javascriptExportShapeVariants.ts +++ b/src/domain/javascript/javascriptExportShapeVariants.ts @@ -98,7 +98,7 @@ export const buildJavaScriptExportShapeChanges = ( input.rightSelection.selection.status !== "selected" ) return [selectionUnknownChange(input)]; - if (input.leftShapes.length === 0 || input.rightShapes.length === 0) + if (input.leftShapes.length === 0 && input.rightShapes.length === 0) return [emptyShapeUnknownChange(input)]; const paired = input.pairing.pairs.flatMap((pair) => { const left = input.leftShapes[pair.leftIndex]; diff --git a/src/domain/javascript/javascriptRpcEndpoints.test.ts b/src/domain/javascript/javascriptRpcEndpoints.test.ts index b086470f3..c416c8461 100644 --- a/src/domain/javascript/javascriptRpcEndpoints.test.ts +++ b/src/domain/javascript/javascriptRpcEndpoints.test.ts @@ -278,6 +278,25 @@ describe("GraphQL operation recognition", () => { ]); }); + it("recognizes operations after leading fragment interpolations only when prior quasis are ignored", () => { + expect( + rpcEndpoints(` + const one = gql\` \${CORE_FIELDS} + query CommentsForPost($postId: ID!) { comments { id } }\`; + const two = gql\`\${CORE_FIELDS} + # another fragment + ,\${MORE_FIELDS} + mutation UpdatePost($id: ID!) { updatePost(id: $id) { id } }\`; + const prose = gql\`fragment prose before interpolation + \${CORE_FIELDS} + query NotAnOperation { viewer { id } }\`; + `), + ).toEqual([ + { kind: "network", value: "CommentsForPost", mechanism: "graphql:query" }, + { kind: "network", value: "UpdatePost", mechanism: "graphql:mutation" }, + ]); + }); + it("retains operations after long ignored prefixes and long headers", () => { const name = "Operation" + "x".repeat(512); const text = diff --git a/src/domain/javascript/javascriptRpcEndpoints.ts b/src/domain/javascript/javascriptRpcEndpoints.ts index 1ccf6d3e5..d706c9aed 100644 --- a/src/domain/javascript/javascriptRpcEndpoints.ts +++ b/src/domain/javascript/javascriptRpcEndpoints.ts @@ -163,11 +163,8 @@ const GRAPHQL_OPERATION_HEADER = const isGraphqlIgnoredCharacter = (character: string): boolean => character === "," || character === "" || /\s/u.test(character); -/** Recover a named operation from GraphQL document text. */ -export const graphqlOperationFromText = ( - text: string, -): JavaScriptRpcEndpoint | undefined => { - // Skip GraphQL ignored tokens: whitespace, commas, BOM and # comments. +/** Return the first offset after GraphQL ignored tokens. */ +const skipGraphqlIgnored = (text: string): number => { const limit = text.length; let index = 0; while (index < limit) { @@ -177,7 +174,15 @@ export const graphqlOperationFromText = ( while (index < limit && !/[\n\r]/u.test(text.charAt(index))) index += 1; } else break; } - if (index >= limit) return undefined; + return index; +}; + +/** Recover a named operation from GraphQL document text. */ +export const graphqlOperationFromText = ( + text: string, +): JavaScriptRpcEndpoint | undefined => { + const index = skipGraphqlIgnored(text); + if (index >= text.length) return undefined; const header = GRAPHQL_OPERATION_HEADER.exec(text.slice(index)); const type = header?.[1]; const name = header?.[2]; @@ -310,11 +315,17 @@ export const createJavaScriptRpcScan = ( ? undefined : graphqlOperationFromText(node.value); if (t.isTemplateLiteral(node)) { - // A template recovered without a cooked value stays dynamic. - const cooked = node.quasis[0]?.value.cooked; - return cooked === undefined || cooked === null - ? undefined - : graphqlOperationFromText(cooked); + // Interpolations may contain fragment definitions. Continue only + // across quasis that contain ignored tokens; prose or dynamic text + // before an operation prevents claiming a later operation belongs to + // this document prefix. + for (const quasi of node.quasis) { + const cooked = quasi.value.cooked; + if (cooked === undefined || cooked === null) return undefined; + if (skipGraphqlIgnored(cooked) === cooked.length) continue; + return graphqlOperationFromText(cooked); + } + return undefined; } return undefined; }, diff --git a/src/domain/javascript/javascriptRuntimeReconciliationMatching.ts b/src/domain/javascript/javascriptRuntimeReconciliationMatching.ts index 96bbe2e59..0c3d6e80b 100644 --- a/src/domain/javascript/javascriptRuntimeReconciliationMatching.ts +++ b/src/domain/javascript/javascriptRuntimeReconciliationMatching.ts @@ -51,6 +51,13 @@ interface CandidateIndex { string, readonly StaticRuntimeCandidate[] >; + readonly byNativeDeclarationPath: ReadonlyMap< + string, + readonly { + readonly candidate: StaticRuntimeCandidate; + readonly sourcePath: string | null; + }[] + >; readonly byLayerCategoryPath: ReadonlyMap< string, readonly StaticRuntimeCandidate[] @@ -170,9 +177,25 @@ const candidatesWithLocation = ( const located = new Map(); for (const layer of layers) for (const mapping of mappedRuntimePaths(layer, entity)) - for (const candidate of index.byLayerCategoryPath.get( - layerCategoryPathKey(layer.layerId, entity.staticKind, mapping.path), - ) ?? []) { + for (const candidate of [ + ...(index.byLayerCategoryPath.get( + layerCategoryPathKey(layer.layerId, entity.staticKind, mapping.path), + ) ?? []), + ...(mapping.fileStyle === null || mapping.nativeRelativePath === null + ? [] + : ( + index.byNativeDeclarationPath.get( + `${mapping.fileStyle}\0${layerCategoryPathKey(layer.layerId, entity.staticKind, mapping.nativeRelativePath)}`, + ) ?? [] + ) + .filter( + ({ sourcePath }) => + mapping.artifactPrefix === "" || + (sourcePath !== null && + sourcePath.startsWith(`${mapping.artifactPrefix}/`)), + ) + .map(({ candidate }) => candidate)), + ]) { const key = candidateKey(candidate); if (!located.has(key)) located.set(key, { candidate, mapping }); } @@ -189,6 +212,10 @@ const indexCandidates = ( ): CandidateIndex => { const byCategoryDigest = new Map(); const byLayerCategoryPath = new Map(); + const byNativeDeclarationPath = new Map< + string, + { candidate: StaticRuntimeCandidate; sourcePath: string | null }[] + >(); for (const candidate of candidates) { for (const { sha256 } of candidate.digests) appendIndex( @@ -202,14 +229,23 @@ const indexCandidates = ( layerCategoryPathKey(candidate.layer.layerId, candidate.category, path), candidate, ); + for (const style of ["posix", "windows"] as const) + for (const { path, sourcePath } of candidate.nativeDeclarationPaths[ + style + ]) + appendIndex( + byNativeDeclarationPath, + `${style}\0${layerCategoryPathKey(candidate.layer.layerId, candidate.category, path)}`, + { candidate, sourcePath }, + ); } - return { byCategoryDigest, byLayerCategoryPath }; + return { byCategoryDigest, byLayerCategoryPath, byNativeDeclarationPath }; }; -const appendIndex = ( - index: Map, +const appendIndex = ( + index: Map, key: string, - candidate: StaticRuntimeCandidate, + candidate: Value, ): void => { const values = index.get(key); if (values === undefined) index.set(key, [candidate]); diff --git a/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts b/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts index ada1c8048..684a4a4d2 100644 --- a/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts +++ b/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; +import { logicalPathEscapesRoot } from "../artifactIdentity.js"; import { evidenceSchema } from "../evidence.js"; import { javascriptRuntimeUnresolvedLocationSchema } from "./javascriptRuntimeObservation.js"; import { javascriptApplicationGraphSchema } from "./javascriptApplicationGraph.js"; @@ -16,8 +17,7 @@ const safePrefixSchema = z .refine( (value) => value === "" || - (!value.startsWith("/") && - !value.includes("\\") && + (!logicalPathEscapesRoot(value) && value.split("/").every((part) => !["", ".", ".."].includes(part))), "Artifact prefix must be empty or a safe relative POSIX path", ); diff --git a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts index 810ab7f5e..a7e1afa73 100644 --- a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts +++ b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts @@ -16,6 +16,12 @@ export interface StaticRuntimeCandidate { readonly node: ApplicationNode; readonly category: "renderer" | "javascript" | "worker" | "service-worker"; readonly paths: readonly string[]; + readonly nativeDeclarationPaths: Readonly< + Record< + "posix" | "windows", + readonly { readonly path: string; readonly sourcePath: string | null }[] + > + >; readonly digests: readonly { readonly sha256: string; readonly basis: "content-sha256" | "module-source-sha256"; @@ -24,6 +30,9 @@ export interface StaticRuntimeCandidate { export interface MappedRuntimePath { readonly path: string; + readonly fileStyle: "posix" | "windows" | null; + readonly nativeRelativePath: string | null; + readonly artifactPrefix: string; readonly basis: | "artifact-path" | "operator-file-mapping" @@ -54,9 +63,20 @@ export const mappedRuntimePaths = ( entity: RuntimeReconciliationEntity, ): MappedRuntimePath[] => { const paths: MappedRuntimePath[] = []; + const fileStyle = + entity.location.kind === "file" + ? absoluteFilePathStyle(entity.location.value) + : null; if (entity.location.kind === "file" && layer.result.format === "directory") { const path = pathBelowRoot(layer.result.input_path, entity.location.value); - if (path !== null) paths.push({ path, basis: "artifact-path" }); + if (path !== null) + paths.push({ + path, + fileStyle, + nativeRelativePath: path, + artifactPrefix: "", + basis: "artifact-path", + }); } for (const mapping of layer.runtimeMappings) { const suffix = @@ -70,6 +90,9 @@ export const mappedRuntimePaths = ( if (path === null) continue; paths.push({ path, + fileStyle, + nativeRelativePath: mapping.kind === "file-root" ? suffix : null, + artifactPrefix: normalizeArtifactPrefix(mapping.artifact_prefix), basis: mapping.kind === "file-root" ? "operator-file-mapping" @@ -111,6 +134,7 @@ const candidatesForLayer = ( node, category, paths: pathsByNodeId.get(node.node_id) ?? [], + nativeDeclarationPaths: nativeDeclarationPaths(node), digests: candidateDigests(node), }, ]; @@ -157,7 +181,9 @@ const indexStaticPaths = ( node.node_id, uniqueSorted([ ...nodePaths(node), - ...(relatedBySource.get(node.node_id) ?? []).flatMap(nodePaths), + ...(usesApplicationFileDeclaration(node) + ? [] + : (relatedBySource.get(node.node_id) ?? []).flatMap(nodePaths)), ]), ]), ); @@ -168,9 +194,21 @@ const nodePaths = (node: ApplicationNode): string[] => { if (node.identity.strategy === "canonical-path") paths.push(node.identity.path); for (const observation of node.observations) { - for (const property of ["path", "resolved_path", "declared_path"]) - if (typeof observation.properties[property] === "string") - paths.push(observation.properties[property]); + const rawApplicationFile = + node.kind === "electron-renderer" && + isRawApplicationFileObservation(observation); + if (typeof observation.properties.path === "string") + paths.push(observation.properties.path); + if (rawApplicationFile) continue; + if (typeof observation.properties.resolved_path === "string") + paths.push(observation.properties.resolved_path); + const declaration = observation.properties.declared_path; + if ( + !hasOwnedArtifactPath(node) && + typeof declaration === "string" && + !declaration.includes("\\") + ) + paths.push(declaration); if ( observation.evidence.location.available && observation.evidence.location.value.kind === "artifact-path" @@ -184,6 +222,95 @@ const nodePaths = (node: ApplicationNode): string[] => { ); }; +// Raw filesystem expressions have host syntax only when a captured file location +// establishes that format. Genuinely owned artifact facts take precedence. +const hasOwnedArtifactPath = (node: ApplicationNode): boolean => + node.identity.strategy === "canonical-path" || + node.observations.some( + ({ properties }) => typeof properties.path === "string", + ) || + node.observations.some( + (observation) => + !isRawApplicationFileObservation(observation) && + typeof observation.properties.resolved_path === "string", + ); + +// loadFile receives a raw literal here: the syntax producer does not retain +// computed __dirname expressions. Electron resolves this relative to the app +// root, so its source-relative inference is not an owned runtime target. +const isRawApplicationFileObservation = ( + observation: ApplicationNode["observations"][number], +): boolean => + observation.properties.resolution_context === "filesystem-expression" && + typeof observation.properties.mechanism === "string" && + observation.properties.mechanism.startsWith("call:") && + observation.properties.mechanism.endsWith("loadFile"); + +const usesApplicationFileDeclaration = (node: ApplicationNode): boolean => + node.kind === "electron-renderer" && + !hasOwnedArtifactPath(node) && + node.observations.some(isRawApplicationFileObservation); + +const nativeDeclarationPaths = ( + node: ApplicationNode, +): StaticRuntimeCandidate["nativeDeclarationPaths"] => { + if (hasOwnedArtifactPath(node)) return { posix: [], windows: [] }; + const declarations = node.observations.flatMap((observation) => { + const { properties } = observation; + return properties.resolution_context === "filesystem-expression" && + typeof properties.declared_path === "string" && + (isRawApplicationFileObservation(observation) || + properties.declared_path.includes("\\")) + ? [ + { + path: properties.declared_path, + sourcePath: nativeDeclarationSource(observation), + }, + ] + : []; + }); + const normalized = (windows: boolean) => { + const values = declarations.flatMap(({ path, sourcePath }) => { + if (windows && (win32.isAbsolute(path) || /^[a-z]:/iu.test(path))) + return []; + const normalizedPath = relativeLogicalPath( + windows ? path.replaceAll("\\", "/") : path, + ); + return normalizedPath === null + ? [] + : [{ path: normalizedPath, sourcePath }]; + }); + return [ + ...new Map( + values.map((value) => [JSON.stringify(value), value]), + ).values(), + ].sort( + (left, right) => + compareUnicodeCodePoints(left.path, right.path) || + compareUnicodeCodePoints(left.sourcePath ?? "", right.sourcePath ?? ""), + ); + }; + return { posix: normalized(false), windows: normalized(true) }; +}; + +// A source anchor constrains the selected application namespace; it does not +// supply a module-relative base for the raw filesystem declaration. +const nativeDeclarationSource = ( + observation: ApplicationNode["observations"][number], +): string | null => { + const location = observation.evidence.location; + if (!location.available) return null; + const source = + location.value.kind === "source-range" + ? location.value.source + : location.value.kind === "artifact-path" + ? location.value.path + : null; + return source === null || source === "artifact-root" + ? null + : relativeLogicalPath(source); +}; + const candidateDigests = ( node: ApplicationNode, ): StaticRuntimeCandidate["digests"] => { @@ -217,7 +344,8 @@ const pathBelowRoot = (root: string, value: string): string | null => { rootStyle === "windows" ? win32.relative(root, value) : posix.relative(root, value); - const portableRemainder = remainder.replaceAll("\\", "/"); + const portableRemainder = + rootStyle === "windows" ? remainder.replaceAll("\\", "/") : remainder; if ( remainder === "" || (rootStyle === "windows" && win32.isAbsolute(remainder)) || @@ -288,10 +416,13 @@ const activeArtifactPrefixes = ( : false, ) ) - prefixes.add(mapping.artifact_prefix); + prefixes.add(normalizeArtifactPrefix(mapping.artifact_prefix)); return [...prefixes].sort(compareUnicodeCodePoints); }; +const normalizeArtifactPrefix = (prefix: string): string => + prefix === "" ? "" : normalizeJoinedLogicalPath(prefix); + const pathWithinPrefix = (path: string, prefix: string): boolean => prefix === "" || path === prefix || path.startsWith(`${prefix}/`); @@ -299,6 +430,7 @@ const safeArtifactPath = (prefix: string, suffix: string): string | null => relativeLogicalPath(prefix === "" ? suffix : posix.join(prefix, suffix)); const relativeLogicalPath = (value: string): string | null => { + // These are artifact names or already parsed runtime paths, not declarations. const normalized = normalizeJoinedLogicalPath(value); return normalized !== "" && normalized !== "." && @@ -312,10 +444,24 @@ const uniqueMappedPaths = ( ): MappedRuntimePath[] => [ ...new Map( - values.map((value) => [`${value.basis}:${value.path}`, value]), + values.map((value) => [ + JSON.stringify([ + value.basis, + value.path, + value.fileStyle, + value.nativeRelativePath, + value.artifactPrefix, + ]), + value, + ]), ).values(), ].sort( (left, right) => compareUnicodeCodePoints(left.path, right.path) || - compareUnicodeCodePoints(left.basis, right.basis), + compareUnicodeCodePoints(left.basis, right.basis) || + compareUnicodeCodePoints( + left.nativeRelativePath ?? "", + right.nativeRelativePath ?? "", + ) || + compareUnicodeCodePoints(left.artifactPrefix, right.artifactPrefix), ); diff --git a/src/domain/javascript/javascriptSourceMapPaths.ts b/src/domain/javascript/javascriptSourceMapPaths.ts index 06b5f8265..5966f89e6 100644 --- a/src/domain/javascript/javascriptSourceMapPaths.ts +++ b/src/domain/javascript/javascriptSourceMapPaths.ts @@ -93,10 +93,7 @@ const resolveJavaScriptSourceMapPath = ( const windowsPath = /^[a-z]:\//iu.test(portable) || source.startsWith("\\\\"); // Preserve parent traversal rather than letting URL resolution clamp at '/'. const anchor = `/${"source/".repeat(portable.split("/").length + 1)}`; - const directory = posix.join( - anchor, - posix.dirname(mapPath.replaceAll("\\", "/")), - ); + const directory = posix.join(anchor, posix.dirname(mapPath)); const base = pathToFileURL(`${directory}/`, { windows: false }); const absolute = windowsPath ? windowsFileUrl(source) : URL.parse(source); const resolved = windowsPath ? absolute : URL.parse(source, base); @@ -116,8 +113,11 @@ const resolveJavaScriptSourceMapPath = ( kind: "unresolved", reason: "Source reference names a directory rather than a file.", }; - // Encoded separators must not manufacture a different filesystem hierarchy. - if (/%(?:2f|5c)/iu.test(resolved.pathname)) + // Reject separators in declarations, not escaped literal names in the owned base. + const declaredPath = windowsPath + ? (absolute?.pathname ?? "") + : (URL.parse(source, "file:///")?.pathname ?? ""); + if (/%(?:2f|5c)/iu.test(declaredPath)) return { kind: "unresolved", reason: "Source reference contains an encoded path separator.", @@ -170,10 +170,11 @@ const windowsFileUrl = (path: string): URL | null => { const decodedUrlPath = (url: URL): string | null => { try { - if (url.protocol === "file:") - return fileURLToPath(url, { - windows: url.hostname !== "" && url.hostname !== "localhost", - }).replaceAll("\\", "/"); + if (url.protocol === "file:") { + const windows = url.hostname !== "" && url.hostname !== "localhost"; + const path = fileURLToPath(url, { windows }); + return windows ? path.replaceAll("\\", "/") : path; + } return `${url.host}${decodeURIComponent(url.pathname)}`; } catch (cause: unknown) { if (cause instanceof URIError || cause instanceof TypeError) return null; diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index c34462498..165ca7fd3 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -25,6 +25,7 @@ import { inspectRouteProperty, inspectRoleProperty, inspectRpcNode, + indexVitePreloadDependencyTable, } from "./javascriptStaticAnalysisCalls.js"; import { inspectBundlerRegistration, @@ -101,6 +102,7 @@ export function* analyzeParsedJavaScriptStaticSourceSteps( { source, accumulator, + vitePreloadDependencyTable: indexVitePreloadDependencyTable(file.program), ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), electronBindings: electronIdentity.bindings, rpcScan: createJavaScriptRpcScan(source, clientRoots), diff --git a/src/domain/javascript/javascriptStaticAnalysisBundler.ts b/src/domain/javascript/javascriptStaticAnalysisBundler.ts index b49f4c2ae..0d51cacd0 100644 --- a/src/domain/javascript/javascriptStaticAnalysisBundler.ts +++ b/src/domain/javascript/javascriptStaticAnalysisBundler.ts @@ -32,6 +32,7 @@ import type { JavaScriptBundlerModule, JavaScriptBundlerRegistration, } from "./javascriptStaticAnalysisTypes.js"; +import { JAVASCRIPT_AST_FINGERPRINT_ALGORITHM } from "./javascriptStaticAnalysisTypes.js"; /** Inspect a bundler registration call and recover its module table. */ export const inspectBundlerRegistration = ( @@ -269,7 +270,7 @@ const recoverBundlerModule = ( factory_require_name: requireName, source_sha256: sha256Text(sourceSlice(source, factory)), structural_fingerprint_sha256: fingerprint, - structural_fingerprint_algorithm: "babel-ast-v1", + structural_fingerprint_algorithm: JAVASCRIPT_AST_FINGERPRINT_ALGORITHM, exports: exportsValue.values, location: range(factory), }, diff --git a/src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts b/src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts new file mode 100644 index 000000000..477bef1ac --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +describe("static callee boundary matching", () => { + it("matches complete callee members without suffix collisions", () => { + const analysis = analyzeJavaScriptStaticSource(` + win.loadFile("./window.html"); + BrowserWindow.loadURL("https://app.example.test"); + navigator.serviceWorker.register("./worker.js"); + loadFile("./bare.html"); + win?.loadURL("https://optional.example.test"); + + uploadFile("./upload.html"); + preloadFile("./preload.html"); + downloadURL("https://download.example.test"); + reloadURL("https://reload.example.test"); + notloadURL("https://other.example.test"); + myserviceWorker.register("./not-worker.js"); + `); + + expect( + analysis.role_paths.map(({ path, mechanism }) => [path, mechanism]), + ).toEqual([ + ["./window.html", "call:win.loadFile"], + ["https://app.example.test", "call:BrowserWindow.loadURL"], + ["./bare.html", "call:loadFile"], + ["https://optional.example.test", "call:win.loadURL"], + ]); + expect( + analysis.references + .filter(({ kind }) => kind === "service-worker") + .map(({ specifier }) => specifier), + ).toEqual(["./worker.js"]); + expect( + analysis.endpoints + .filter(({ mechanism }) => mechanism.endsWith("loadURL")) + .map(({ value }) => value), + ).toEqual(["https://app.example.test", "https://optional.example.test"]); + }); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisCalls.ts b/src/domain/javascript/javascriptStaticAnalysisCalls.ts index b098b3c26..b031aea9e 100644 --- a/src/domain/javascript/javascriptStaticAnalysisCalls.ts +++ b/src/domain/javascript/javascriptStaticAnalysisCalls.ts @@ -71,7 +71,12 @@ export const inspectCall = ( kind: "dynamic-import", specifier: chunkLoad.specifier, }); - for (const specifier of vitePreloadDependencySpecifiers(source, node, name)) + for (const specifier of vitePreloadDependencySpecifiers( + node, + name, + context.vitePreloadDependencyTable, + bindings, + )) addReference(context, { node, kind: "dynamic-import", specifier }); const worker = t.isNewExpression(node) ? node : undefined; const workerConstructor = @@ -94,13 +99,19 @@ export const inspectCall = ( ? undefined : workerUrlSpecifier(worker, bindings)), }); - if (name.endsWith("serviceWorker.register")) + if ( + name === "serviceWorker.register" || + name.endsWith(".serviceWorker.register") + ) addReference(context, { node, kind: "service-worker", specifier: first }); inspectEndpointCall(node, name, first, context); inspectStorageCall(node, name, first, context); if (first === "audio" || first === "video") noteMediaElementPreloads(node, accumulator); - if (name.endsWith("loadFile") && first !== undefined) + if ( + (name === "loadFile" || name.endsWith(".loadFile")) && + first !== undefined + ) addLocatedFinding(context, { collection: accumulator.roles, key: `role\0renderer\0${first}`, @@ -114,7 +125,7 @@ export const inspectCall = ( location: range(node), }, }); - if (name.endsWith("loadURL") && first !== undefined) + if ((name === "loadURL" || name.endsWith(".loadURL")) && first !== undefined) addLocatedFinding(context, { collection: accumulator.roles, key: `role\0renderer\0${first}`, @@ -241,28 +252,156 @@ const chunkLoadSpecifier = ( }; const vitePreloadDependencySpecifiers = ( - source: string, node: JavaScriptCallLike, name: string, -): readonly string[] => { - if (name !== "__vite__mapDeps" || !t.isArrayExpression(node.arguments[0])) + table: readonly (string | undefined)[] | null, + bindings: JavaScriptSemanticAnalysisState, +): readonly (string | undefined)[] => { + if ( + name !== "__vite__mapDeps" || + table === null || + !t.isArrayExpression(node.arguments[0]) + ) return []; - const indexes = node.arguments[0].elements.flatMap((element) => - t.isNumericLiteral(element) ? [element.value] : [], - ); - if (indexes.length !== node.arguments[0].elements.length) return []; - const table = vitePreloadDependencyTable(source); - return indexes.flatMap((index) => table[index] ?? []); + const indexes = node.arguments[0].elements; + // Explicit cache overrides change which table the helper reads. Retain the + // call slots as unknown unless every override is statically undefined. + if ( + !node.arguments + .slice(1) + .every((argument) => isExplicitDefaultArgument(argument, bindings)) + ) + return indexes.map(() => undefined); + return indexes.map((element) => { + if (element === null || !t.isNumericLiteral(element)) return undefined; + const index = element.value; + return Number.isSafeInteger(index) && index >= 0 ? table[index] : undefined; + }); }; -const vitePreloadDependencyTable = (source: string): readonly string[] => { - const match = - /\.\s*f\s*\|\|\s*\(\s*[^)]*?\.f\s*=\s*\[([\s\S]*?)\]\s*\)/u.exec(source); - const body = match?.[1]; - if (body === undefined) return []; - return [ - ...body.matchAll(/"([^"\\]*(?:\\.[^"\\]*)*)"|'([^'\\]*(?:\\.[^'\\]*)*)'/gu), - ].map((literal) => literal[1] ?? literal[2] ?? ""); +const isExplicitDefaultArgument = ( + argument: t.CallExpression["arguments"][number], + bindings: JavaScriptSemanticAnalysisState, +): boolean => { + if (!t.isNode(argument)) return false; + const value = unwrapJavaScriptExpression(argument).node; + return ( + (t.isUnaryExpression(value) && value.operator === "void") || + (t.isIdentifier(value, { name: "undefined" }) && + isUnmodifiedGlobalReference(value, bindings, "undefined")) + ); +}; + +type ViteMapDepsFunction = + | t.FunctionDeclaration + | t.FunctionExpression + | t.ArrowFunctionExpression; + +/** Index the canonical Vite helper's cache initializer from parsed syntax. */ +export const indexVitePreloadDependencyTable = ( + program: t.Program, +): readonly (string | undefined)[] | null => { + const helpers: ViteMapDepsFunction[] = []; + visitJavaScriptNodes(program, (node) => { + if (t.isFunctionDeclaration(node) && node.id?.name === "__vite__mapDeps") + helpers.push(node); + else if ( + t.isVariableDeclarator(node) && + t.isIdentifier(node.id, { name: "__vite__mapDeps" }) + ) { + const init = node.init; + if (t.isFunctionExpression(init) || t.isArrowFunctionExpression(init)) + helpers.push(init); + } + }); + if (helpers.length !== 1) return null; + const helper = helpers[0]; + if (helper === undefined) return null; + const mapParameter = helper.params[1]; + const mapIdentifier = t.isAssignmentPattern(mapParameter) + ? mapParameter.left + : mapParameter; + if (!t.isIdentifier(mapIdentifier)) return []; + const tables: t.ArrayExpression[] = []; + visitJavaScriptNodes( + helper, + (node) => { + if ( + t.isAssignmentExpression(node) && + node.operator === "=" && + (t.isMemberExpression(node.left) || + t.isOptionalMemberExpression(node.left)) && + t.isIdentifier(node.left.object, { name: mapIdentifier.name }) && + semanticStaticPropertyName(node.left.property, node.left.computed) === + "f" && + t.isArrayExpression(node.right) + ) + tables.push(node.right); + }, + undefined, + true, + ); + if (tables.length !== 1) return []; + const table = tables[0]; + if (table === undefined) return []; + const dependencies = table.elements.map((element) => + element === null || t.isSpreadElement(element) + ? undefined + : stringValue(element), + ); + let escapedOrMutated = false; + visitJavaScriptNodes( + program, + (node, parent) => { + if (!t.isIdentifier(node, { name: "__vite__mapDeps" })) return; + if ( + (parent === helper && + t.isFunctionDeclaration(parent) && + parent.id === node) || + (t.isVariableDeclarator(parent) && + parent.init === helper && + parent.id === node) + ) + return; + if ( + (t.isCallExpression(parent) || t.isOptionalCallExpression(parent)) && + parent.callee === node + ) + return; + escapedOrMutated = true; + }, + helper, + ); + return escapedOrMutated ? dependencies.map(() => undefined) : dependencies; +}; + +const visitJavaScriptNodes = ( + root: t.Node, + visit: (node: t.Node, parent: t.Node | undefined) => void, + skip?: t.Node, + skipNestedFunctions = false, +): void => { + const pending: { readonly node: t.Node; readonly parent?: t.Node }[] = [ + { node: root }, + ]; + while (pending.length > 0) { + const current = pending.pop(); + if (current === undefined) continue; + const { node, parent } = current; + if (node === skip) continue; + visit(node, parent); + if (skipNestedFunctions && node !== root && t.isFunction(node)) continue; + const keys = t.VISITOR_KEYS[node.type] ?? []; + for (const key of keys) { + const child = (node as unknown as Record)[key]; + if (Array.isArray(child)) { + for (let index = child.length - 1; index >= 0; index -= 1) { + const item = child[index]; + if (t.isNode(item)) pending.push({ node: item, parent: node }); + } + } else if (t.isNode(child)) pending.push({ node: child, parent: node }); + } + } }; const inspectEndpointCall = ( @@ -292,8 +431,9 @@ const inspectEndpointCall = ( addEndpoint(context, { node, kind: "network", - value: endpoint, + value: endpoint.value, mechanism: `call:${name}`, + ...(endpoint.kind === "candidate" ? { interpretation: "candidate" } : {}), }); }; @@ -401,12 +541,15 @@ export const addReference = ( const addEndpoint = (context: FindingContext, input: EndpointInput): void => addLocatedFinding(context, { collection: context.accumulator.endpoints, - key: `endpoint\0${input.kind}\0${input.value}`, + key: `endpoint\0${input.kind}\0${input.interpretation ?? "literal"}\0${input.value}`, node: input.node, value: { kind: input.kind, value: sanitizeEndpointCandidate(input.value), mechanism: input.mechanism, + ...(input.interpretation === undefined + ? {} + : { interpretation: input.interpretation }), module_key: null, location: range(input.node), }, diff --git a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts index 14827e1c2..952abdd0e 100644 --- a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts +++ b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts @@ -410,13 +410,27 @@ const XHR_METHODS = new Set([ "TRACE", ]); -/** An absolute HTTP(S)/WebSocket URL, or an absolute or relative path. */ -const isRequestUrlLiteral = (value: string): boolean => +/** Legacy URL/path shapes that bypass the overloaded lookup exclusions. */ +const isRequestUrlOrPathLiteral = (value: string): boolean => /^(?:https?|wss?):\/\//iu.test(value) || value.startsWith("/") || value.startsWith("./") || value.startsWith("../"); +/** Explicit request references, classified after receiver-specific exclusions. */ +const isRequestUrlReference = (value: string): boolean => { + if ( + isRequestUrlOrPathLiteral(value) || + value.startsWith("?") || + value.startsWith("#") + ) + return true; + // WHATWG accepts HTTP/WS scheme forms without authority slashes. Parse + // without a base so bare keys stay ambiguous, and keep the source spelling. + if (!/^(?:https?|wss?):/iu.test(value)) return false; + return URL.parse(value) !== null; +}; + /** * Browsing-context and document `open` take a URL or type first and a target * name second. The name is only a spelling, since `parent` or `self` may be a @@ -538,10 +552,20 @@ const isKeyedCollectionReceiver = (callee: t.Node): boolean => { receiver.property, receiver.computed, ); - return property === "headers" || property === "searchParams"; + return ( + property === "headers" || + property === "headersList" || + property === "searchParams" + ); }; -/** Select the literal URL argument for recognized network callees. */ +/** A literal endpoint shape or an ambiguous argument of an HTTP-method-named call. */ +export interface JavaScriptEndpointArgument { + readonly kind: "endpoint" | "candidate"; + readonly value: string; +} + +/** Classify one literal without treating a generic method name as network identity. */ export const endpointArgument = ( name: string, args: readonly ( @@ -552,7 +576,11 @@ export const endpointArgument = ( )[], callee: t.Node, receiverFact?: JavaScriptOpenReceiverFact, -): string | undefined => { +): JavaScriptEndpointArgument | undefined => { + const literal = ( + value: string | undefined, + ): JavaScriptEndpointArgument | undefined => + value === undefined ? undefined : { kind: "endpoint", value }; if (receiverFact === "window") return undefined; if ( name === "fetch" || @@ -561,9 +589,18 @@ export const endpointArgument = ( name === "EventSource" || name.endsWith(".EventSource") ) - return stringValue(args[0]); - if (name.endsWith(".open")) - return xhrOpenUrl(name, args[0], args[1], receiverFact); + return literal(stringValue(args[0])); + if (name.endsWith(".open")) { + const value = xhrOpenUrl(name, args[0], args[1], receiverFact); + if (value === undefined) return undefined; + return { + kind: + stringValue(args[0]) !== undefined || isRequestUrlReference(value) + ? "endpoint" + : "candidate", + value, + }; + } const method = ["get", "post", "put", "patch", "delete", "request"].find( (candidate) => name === candidate || name.endsWith(`.${candidate}`), ); @@ -572,18 +609,26 @@ export const endpointArgument = ( // Only get and delete also belong to the keyed-collection APIs, whose // keys are header, field, and parameter names rather than URLs or paths. // Configuration `get` reads setting keys. - return value !== undefined && + if (value === undefined) return undefined; + if ( (method === "get" || method === "delete") && - !isRequestUrlLiteral(value) && + !isRequestUrlOrPathLiteral(value) && (isKeyedCollectionReceiver(callee) || - (method === "get" && receiverFact === "configuration")) - ? undefined - : value; + (method === "get" && + (receiverFact === "configuration" || + receiverFact === "extension-state"))) + ) + return undefined; + return { + kind: isRequestUrlReference(value) ? "endpoint" : "candidate", + value, + }; } - // navigator.sendBeacon(url, data) queues a POST to its first argument. + // Beacon URLs are endpoint observations, retaining the upstream API support. if (name === "sendBeacon" || name.endsWith(".sendBeacon")) - return stringValue(args[0]); - if (name.endsWith("loadURL")) return stringValue(args[0]); + return literal(stringValue(args[0])); + if (name === "loadURL" || name.endsWith(".loadURL")) + return literal(stringValue(args[0])); return undefined; }; diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index 5173362fa..c60b76763 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -65,6 +65,7 @@ export interface JavaScriptFindingContext { readonly rpcScan: JavaScriptRpcScan; readonly source: string; readonly accumulator: JavaScriptAnalysisAccumulator; + readonly vitePreloadDependencyTable: readonly (string | undefined)[] | null; readonly openReceiverFacts?: ReadonlyMap; /** Proven Electron export path segments by root offset. */ readonly electronBindings?: ReadonlyMap; @@ -83,6 +84,7 @@ export interface JavaScriptEndpointInput { readonly kind: JavaScriptStaticEndpoint["kind"]; readonly value: string; readonly mechanism: string; + readonly interpretation?: "candidate"; } /** Values required to append one module-attributable static finding. */ diff --git a/src/domain/javascript/javascriptStaticAnalysisTypes.ts b/src/domain/javascript/javascriptStaticAnalysisTypes.ts index 1e72deb1d..887277950 100644 --- a/src/domain/javascript/javascriptStaticAnalysisTypes.ts +++ b/src/domain/javascript/javascriptStaticAnalysisTypes.ts @@ -1,5 +1,8 @@ import type { ElectronStaticFindings } from "./electronStaticAnalysisTypes.js"; +/** Token semantics shared by bundle producers and worker admission. */ +export const JAVASCRIPT_AST_FINGERPRINT_ALGORITHM = "babel-ast-v2"; + /** Static syntax context retained for later artifact path resolution. */ export type JavaScriptStaticPathContext = | "module-specifier" @@ -42,6 +45,8 @@ export interface JavaScriptStaticEndpoint { readonly kind: "route" | "network"; readonly value: string; readonly mechanism: string; + /** An overloaded callee name suggests this value; its network receiver is unproven. */ + readonly interpretation?: "candidate"; readonly module_key: string | null; /** Recovered factory runtime; absent in older or unbundled findings. */ readonly module_runtime?: string; @@ -75,7 +80,7 @@ interface JavaScriptBundlerModuleState { /** One exact module factory recovered from a Webpack/Rspack registration. */ export interface JavaScriptBundlerModule extends JavaScriptBundlerModuleState { readonly structural_fingerprint_sha256: string; - readonly structural_fingerprint_algorithm: "babel-ast-v1"; + readonly structural_fingerprint_algorithm: typeof JAVASCRIPT_AST_FINGERPRINT_ALGORITHM; } /** One statically recognized bundler chunk or module registration. */ diff --git a/src/domain/javascript/javascriptSymbolicBindings.test.ts b/src/domain/javascript/javascriptSymbolicBindings.test.ts index a4248e30b..4e0a2acac 100644 --- a/src/domain/javascript/javascriptSymbolicBindings.test.ts +++ b/src/domain/javascript/javascriptSymbolicBindings.test.ts @@ -170,6 +170,34 @@ it("invalidates a bootstrap module slot mutated through an alias", () => { ]); }); +it.each([ + [ + "destructuring assignment", + `[table[1]] = [function(module) { module.exports = require("other"); }];`, + ], + ["for-of head", `for (table[1] of replacements) {}`], + [ + "wrapped assignment", + `(table as never)[1] = function(module) { module.exports = require("other"); };`, + ], +])( + "invalidates a bootstrap module slot overwritten through %s", + (_name, write) => { + const values = inspectedValues(`(() => { + const modules = { + 1(module) { module.exports = require("electron"); }, + 2(module, exports, load) { inspect(load(1)); } + }; + const table = modules; + function load(id) { const m = {exports:{}}; modules[id](m,m.exports,load); return m.exports; } + ${write} + load(2); + })();`); + expect(values).toHaveLength(1); + expect(values[0]).toEqual(expect.objectContaining({ kind: "unknown" })); + }, +); + it.each([ `const electron=require("electron"); const ipc=electron.ipcMain; ipc.handle=()=>null; inspect(electron.ipcMain.handle);`, `require("electron").ipcMain.handle=()=>null; inspect(require("electron").ipcMain.handle);`, @@ -438,3 +466,55 @@ it("propagates branch guards through object properties and function calls", () = ]); } }); + +it("completes a loaded module past a very large array without stack overflow", () => { + const parsed = parseJavaScriptSource(`(() => { + const modules = { + 4(module) { + module.exports = { table: [0] }; + if (module) { return; } + return; + }, + 8(module, exports, load) { const dep = load(4); inspect(dep); } + }; + function load(id) { const m = {exports:{}}; modules[id](m,m.exports,load); return m.exports; } + load(8); + })();`); + if (parsed === null) throw new Error("symbolic source fixture did not parse"); + const arrays: t.ArrayExpression[] = []; + const bareReturns: t.ReturnStatement[] = []; + let inspectCall: t.CallExpression | undefined; + traverseJavaScriptAst(parsed.program, { + enter: (node) => { + if (t.isArrayExpression(node)) arrays.push(node); + if (t.isReturnStatement(node) && node.argument == null) + bareReturns.push(node); + if ( + t.isCallExpression(node) && + t.isIdentifier(node.callee, { name: "inspect" }) + ) + inspectCall = node; + }, + }); + for (const array of arrays) + array.elements = Array.from({ length: 1_000_000 }, () => + t.numericLiteral(0), + ); + const earliestReturn = bareReturns.toSorted( + (left, right) => (left.start ?? 0) - (right.start ?? 0), + )[0]; + const inspected = inspectCall?.arguments[0]; + if (earliestReturn === undefined || inspected === undefined) + throw new Error("symbolic source fixture is missing its probes"); + + const state = completeSemanticSteps( + indexJavaScriptBindingsSteps(parsed.program), + ); + const resolver = new JavaScriptSymbolicBindings(state, parsed.program); + const namespace = symbolicVariants(resolver.evaluate(inspected)).find( + (variant) => variant.kind === "namespace", + ); + if (namespace?.kind !== "namespace") + throw new Error("loaded module namespace missing"); + expect(namespace.environment.completedAt).toBe(earliestReturn); +}); diff --git a/src/domain/javascript/javascriptSymbolicBindings.ts b/src/domain/javascript/javascriptSymbolicBindings.ts index faddf545f..ec1585a1b 100644 --- a/src/domain/javascript/javascriptSymbolicBindings.ts +++ b/src/domain/javascript/javascriptSymbolicBindings.ts @@ -1,6 +1,8 @@ import * as t from "@babel/types"; import { semanticStaticPropertyKey, + semanticWriteSiteTarget, + semanticWriteTargets, runtimeFunctionParameters, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; @@ -16,7 +18,6 @@ import { modulePropertyName, } from "./javascriptStaticAnalysisHelpers.js"; import { - childNodes, completeSemanticSteps, traverseJavaScriptAst, } from "./javascriptSemanticTraversal.js"; @@ -105,12 +106,8 @@ export class JavaScriptSymbolicBindings { }); traverseJavaScriptAst(program, { enter: (node) => { - if (t.isAssignmentExpression(node)) - this.#indexWrite(node.left, node.right, node); - if (t.isUpdateExpression(node)) - this.#indexWrite(node.argument, node, node); - if (t.isUnaryExpression(node, { operator: "delete" })) - this.#indexWrite(node.argument, node, node); + const target = semanticWriteSiteTarget(node); + if (target !== null) this.#indexWriteSite(target, node); if ( t.isCallExpression(node) && t.isMemberExpression(node.callee) && @@ -636,20 +633,19 @@ export class JavaScriptSymbolicBindings { #moduleCompletionPoint(factory: t.Function): t.Node { let earliestReturn: t.ReturnStatement | undefined; - const pending: t.Node[] = [factory.body]; - while (pending.length > 0) { - const node = pending.pop(); - if (!node) continue; - if (node !== factory.body && (t.isFunction(node) || t.isClass(node))) - continue; - if ( - t.isReturnStatement(node) && - (earliestReturn === undefined || - (node.start ?? Infinity) < (earliestReturn.start ?? Infinity)) - ) - earliestReturn = node; - pending.push(...childNodes(node)); - } + // Use the shared traversal so source-sized child lists are pushed one at + // a time. Spreading array elements or call arguments overflows the stack. + traverseJavaScriptAst(factory.body, { + skip: (node) => t.isFunction(node) || t.isClass(node), + enter: (node) => { + if ( + t.isReturnStatement(node) && + (earliestReturn === undefined || + (node.start ?? Infinity) < (earliestReturn.start ?? Infinity)) + ) + earliestReturn = node; + }, + }); return earliestReturn ?? factory.body; } @@ -943,15 +939,33 @@ export class JavaScriptSymbolicBindings { else for (const key of invalidSlots) modules.delete(key); } - #indexWrite(target: t.Node, value: t.Node, site: t.Node): void { + #indexWriteSite(target: t.Node, site: t.Node): void { + const direct = unwrapJavaScriptExpression(target).node; + if ( + (t.isIdentifier(direct) || t.isMemberExpression(direct)) && + !t.isForOfStatement(site) && + !t.isForInStatement(site) + ) { + this.#indexWrite( + direct, + t.isAssignmentExpression(site) ? site.right : site, + site, + ); + return; + } + for (const leaf of semanticWriteTargets(target)) + this.#indexWrite(leaf, site, site, false); + } + + #indexWrite(target: t.Node, value: t.Node, site: t.Node, exact = true): void { const path: string[] = []; - let root = target; - let dynamic = false; + let root = unwrapJavaScriptExpression(target).node; + let dynamic = !exact; while (t.isMemberExpression(root)) { const key = semanticStaticPropertyKey(root.property, root.computed); if (key === null) dynamic = true; else path.unshift(key); - root = root.object; + root = unwrapJavaScriptExpression(root.object).node; } if (dynamic && t.isIdentifier(root)) { const binding = resolveSemanticBindingState(this.state, root, root.name); diff --git a/src/domain/javascript/javascriptVitePreloadDependencies.test.ts b/src/domain/javascript/javascriptVitePreloadDependencies.test.ts new file mode 100644 index 000000000..4995dd06c --- /dev/null +++ b/src/domain/javascript/javascriptVitePreloadDependencies.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +describe("Vite preload dependency recovery", () => { + it("decodes helper table values and preserves sparse and dynamic slots", () => { + const source = String.raw` + const __vite__mapDeps = (indexes, map = __vite__mapDeps, dependencies = + (map.f || (map.f = ["./caf\u00e9.js", , getDependency(), "./last.js"]))) => + indexes.map((index) => dependencies[index]); + const dynamicIndex = chooseIndex(); + __vite__mapDeps([0, , 3, dynamicIndex]); + `; + const analysis = analyzeJavaScriptStaticSource(source); + const references = analysis.references.filter( + ({ kind }) => kind === "dynamic-import", + ); + + expect(references.map(({ specifier }) => specifier).sort()).toEqual([ + "./café.js", + "./last.js", + null, + ]); + expect(references.every(({ location }) => location.start.line === 6)).toBe( + true, + ); + expect(references.filter(({ specifier }) => specifier === null)).toEqual([ + expect.objectContaining({ + expression: expect.stringContaining( + "__vite__mapDeps([0, , 3, dynamicIndex])", + ), + }), + ]); + }); + + it("does not take dependency strings from comments or unrelated helpers", () => { + const analysis = analyzeJavaScriptStaticSource(` + // cache.f = ["./comment.js"] + const unrelated = (cache) => cache.f || (cache.f = ["./unrelated.js"]); + __vite__mapDeps([0]); + `); + + expect( + analysis.references.filter(({ kind }) => kind === "dynamic-import"), + ).toEqual([]); + }); + + it("keeps cache escapes and explicit cache overrides unknown", () => { + const helper = ` + const __vite__mapDeps = (indexes, map = __vite__mapDeps, dependencies = + (map.f || (map.f = ["./original.js"]))) => + indexes.map((index) => dependencies[index]); + `; + const escaped = analyzeJavaScriptStaticSource(` + ${helper} + const dependencies = __vite__mapDeps.f; + dependencies[0] = "./changed.js"; + __vite__mapDeps([0]); + `); + expect( + escaped.references + .filter(({ kind }) => kind === "dynamic-import") + .map(({ specifier }) => specifier), + ).toEqual([null]); + + const overridden = analyzeJavaScriptStaticSource(` + ${helper} + __vite__mapDeps([0], customMap); + `); + expect( + overridden.references + .filter(({ kind }) => kind === "dynamic-import") + .map(({ specifier }) => specifier), + ).toEqual([null]); + + const defaulted = analyzeJavaScriptStaticSource(` + ${helper} + __vite__mapDeps([0], undefined, void 0); + `); + expect( + defaulted.references + .filter(({ kind }) => kind === "dynamic-import") + .map(({ specifier }) => specifier), + ).toEqual(["./original.js"]); + }); +}); diff --git a/src/domain/javascript/sourceToBundleComparison.test.ts b/src/domain/javascript/sourceToBundleComparison.test.ts index d231cb5ec..0853ce378 100644 --- a/src/domain/javascript/sourceToBundleComparison.test.ts +++ b/src/domain/javascript/sourceToBundleComparison.test.ts @@ -330,11 +330,13 @@ describe("source to bundle path syntax", () => { }); }); - it.each([ - "https://example.test/src/modified.ts?revision=2#section", - "webpack://fixture/./src/modified.ts?revision=2#section", - ])("still strips URL query and fragment from %s", (path) => { - const node = moduleNode("url", { path }); + it.each( + [ + "https://example.test/src/modified.ts?revision=2#section", + "webpack://fixture/./src/modified.ts?revision=2#section", + ].flatMap((path) => ["path", "module_path"].map((key) => ({ key, path }))), + )("still strips URL query and fragment from $key $path", ({ key, path }) => { + const node = moduleNode("url", { [key]: path }); const graph = createJavaScriptApplicationGraph({ schema: "JavaScriptApplicationGraph", root_node_ids: [node.node_id], diff --git a/src/domain/javascript/sourceToBundleSignals.ts b/src/domain/javascript/sourceToBundleSignals.ts index b3546aecc..b9706f714 100644 --- a/src/domain/javascript/sourceToBundleSignals.ts +++ b/src/domain/javascript/sourceToBundleSignals.ts @@ -188,7 +188,7 @@ const projectCurrentNode = (node: ApplicationNode): CurrentProjection => { digests.add(node.identity.source_sha256); } if (node.identity.strategy === "canonical-path") - addPath(paths, "canonical-path", node.identity.path); + addPath(paths, "canonical-path", node.identity.path, "artifact"); for (const observation of node.observations) { if (node.identity.strategy === "source-map-original") { const resolution = observation.source_map_reference?.resolution; @@ -202,8 +202,26 @@ const projectCurrentNode = (node: ApplicationNode): CurrentProjection => { const sourceDigest = observation.properties.source_sha256; if (typeof sourceDigest === "string" && isDigest(sourceDigest)) digests.add(sourceDigest); - for (const key of PATH_PROPERTIES) - addJsonPath(paths, observation.properties[key]); + for (const key of PATH_PROPERTIES) { + // A resolved declaration already has an owning artifact-path fact. + if ( + key === "declared_path" && + typeof observation.properties.resolved_path === "string" + ) + continue; + addJsonPath( + paths, + observation.properties[key], + key === "resolved_path" || + ((key === "path" || key === "module_path") && + !( + typeof observation.properties[key] === "string" && + SCHEME_URL.test(observation.properties[key]) + )) + ? "artifact" + : "reference", + ); + } } } return { @@ -323,29 +341,40 @@ const matchingUnicodePaths = ( .map(({ value }) => value); }; -const addJsonPath = (paths: CurrentPath[], value: unknown): void => { - if (typeof value === "string") addPath(paths, "observation-path", value); +const addJsonPath = ( + paths: CurrentPath[], + value: unknown, + format: "artifact" | "reference", +): void => { + if (typeof value === "string") + addPath(paths, "observation-path", value, format); }; const addPath = ( paths: CurrentPath[], kind: CurrentPathKind, raw: string, + format: "artifact" | "reference", ): void => { - const value = normalizeCurrentPath(raw); + const value = normalizeCurrentPath(raw, format); if (value !== null) paths.push({ kind, value, allowSuffix: true }); }; const SCHEME_URL = /^[a-z][a-z0-9+.-]*:\/\//iu; const SCHEME_URL_PREFIX = /^[a-z][a-z0-9+.-]*:\/\/+/iu; -const normalizeCurrentPath = (raw: string): string | null => { - const withoutQuery = SCHEME_URL.test(raw) - ? (raw.split(/[?#]/u, 1)[0] ?? "") - : raw; - const portable = withoutQuery - .replace(SCHEME_URL_PREFIX, "") - .replaceAll("\\", "/"); +const normalizeCurrentPath = ( + raw: string, + format: "artifact" | "reference", +): string | null => { + const withoutQuery = + format === "reference" && SCHEME_URL.test(raw) + ? (raw.split(/[?#]/u, 1)[0] ?? "") + : raw; + const portable = + format === "artifact" + ? withoutQuery + : withoutQuery.replace(SCHEME_URL_PREFIX, "").replaceAll("\\", "/"); const parts: string[] = []; for (const part of portable.split("/")) { if (part === "" || part === ".") continue; diff --git a/src/domain/webBundleAnalysis.ts b/src/domain/webBundleAnalysis.ts index 27c79d126..b4e4c11f8 100644 --- a/src/domain/webBundleAnalysis.ts +++ b/src/domain/webBundleAnalysis.ts @@ -202,7 +202,12 @@ export const webBundleAnalysisSchema = z.object({ }), inferences: z.array( z.object({ - kind: z.enum(["vendor_fingerprint", "route_framework", "bundle_runtime"]), + kind: z.enum([ + "vendor_fingerprint", + "route_framework", + "bundle_runtime", + "endpoint_candidate", + ]), value: z.string(), confidence: z.enum(["low", "medium", "high"]), basis: z.array(basisSchema).min(1), diff --git a/src/domain/webBundleAnalyzer.test.ts b/src/domain/webBundleAnalyzer.test.ts index 93bb24fdc..0450bc774 100644 --- a/src/domain/webBundleAnalyzer.test.ts +++ b/src/domain/webBundleAnalyzer.test.ts @@ -257,9 +257,15 @@ describe("web bundle static-analysis parity", () => { "/dynamic-method", "/shadowed-parent-xhr", "/xhr-get", - "api/relative", "https://xhr.example.test/submit", ]); + expect(result.inferences).toContainEqual( + expect.objectContaining({ + kind: "endpoint_candidate", + value: "api/relative", + confidence: "low", + }), + ); }); }); @@ -356,11 +362,252 @@ describe("web bundle API classification parity", () => { "/search", "/users", "/v1/headers", - "cache-key", "https://api.example.test/headers", "https://api.example.test/users", + ]); + expect( + result.inferences + .filter(({ kind }) => kind === "endpoint_candidate") + .map(({ value }) => value) + .sort(), + ).toEqual(["cache-key", "users"]); + }); + + it("keeps undici header-store reads out of endpoint observations", () => { + const result = analyzeCapturedWebBundle( + inspection(` + response.headersList.get("location", true); + request.headersList.delete("authorization"); + request.headersList.get("/header-list"); + api.get("/api/users"); + `), + ); + expect(result.observations.endpoints.map(({ value }) => value)).toEqual([ + "/header-list", + "/api/users", + ]); + expect( + result.inferences + .filter(({ kind }) => kind === "endpoint_candidate") + .map(({ value }) => value), + ).toEqual([]); + }); +}); + +describe("web bundle explicit request reference forms", () => { + it("recognizes URL, path, query and fragment forms after excluding lookup receivers", () => { + const references = [ + "http://api.example.test/users", + "https://api.example.test/users?q=1#part", + "ws://api.example.test/events", + "wss://api.example.test/events", + "http:api.example.test/users", + "https:api.example.test/users?q=1&tag=a&tag=b#part", + "https:api.example.test:443/a/../b?x=a+b&x=two#part", + "ws:api.example.test/events", + "wss:api.example.test/events", + "HtTpS:/api.example.test/users", + String.raw`https:\api.example.test\users`, + "https://[invalid", + "//api.example.test/users", + "/api/users", + "./users", + "../users", + "?", + "?page=2&tag=a&tag=b#section", + "#", + "#section", + "#frag?tag=a&tag=b", + ]; + const ambiguous = [ + "", + ".", + "..", + "users", + "%23section", + "%3Fpage=2", + " #section", + " https:api.example.test/users", + "https:", + "http:?page=2", + "ws:#section", + "https:[invalid", + "https:api.example.test:99999", + "ftp:api.example.test/users", + "file:/users", + "data:text/plain,users", + ]; + const source = ` + ${references.map((value) => `api.get(${JSON.stringify(value)});`).join("\n")} + ${ambiguous.map((value) => `api.get(${JSON.stringify(value)});`).join("\n")} + xhr.open(method, "?view=1"); + xhr.open(method, "#view=1"); + xhr.open("GET", "#literal=1"); + xhr.open(method, "https:api.example.test/xhr"); + request("POST", target); + snackBar.open(message, "X"); + new Map().get("?map-key"); + new Map().get("#map-key"); + new Map().get("https:api.example.test/map-key"); + new Headers().delete("?header-key"); + new Headers().delete("#header-key"); + new URLSearchParams().get("?parameter-key"); + new URLSearchParams().get("#parameter-key"); + response.headers.get("?response-key"); + response.headers.get("#response-key"); + const config = vscode.workspace.getConfiguration("git"); + config.get("?setting"); + config.get("#setting"); + config.get("https:api.example.test/setting"); + window.open(url, "?window-target"); + window.open(url, "#window-target"); + indexedDB.open(name, "?database-version"); + indexedDB.open(name, "#database-version"); + caches.open(name, "#cache-name"); + fs.open(path, "r"); + `; + const expected = [ + ...references, + "?view=1", + "#view=1", + "#literal=1", + "https:api.example.test/xhr", + ].sort(); + const expectedCandidates = [...ambiguous, "POST", "X"].sort(); + const web = analyzeCapturedWebBundle(inspection(source)); + expect(web.observations.endpoints.map(({ value }) => value).sort()).toEqual( + expected, + ); + expect( + web.inferences + .filter(({ kind }) => kind === "endpoint_candidate") + .map(({ value }) => value) + .sort(), + ).toEqual(expectedCandidates); + const analyzed = analyzeJavaScriptStaticSource(source); + expect( + analyzed.endpoints + .filter(({ interpretation }) => interpretation === undefined) + .map(({ value }) => value) + .sort(), + ).toEqual(expected); + expect( + analyzed.endpoints + .filter(({ interpretation }) => interpretation === "candidate") + .map(({ value }) => value) + .sort(), + ).toEqual(expectedCandidates); + }); +}); + +describe("web bundle endpoint candidate classification", () => { + it("keeps ambiguous HTTP-method arguments as candidates instead of endpoint observations", () => { + const source = ` + fetch("login"); + api.get("/api/users"); + api.get("users"); + form.get("passwordControl"); + translate.get("ADDRESS_ADDED"); + cookieService.put("token", value); + http.request("POST", target); + snackBar.open(message, "X"); + xhr.open(method, "login"); + xhr.open("GET", "ping"); + `; + const result = analyzeCapturedWebBundle(inspection(source)); + expect( + result.observations.endpoints.map(({ value }) => value).sort(), + ).toEqual(["/api/users", "login", "ping"]); + const candidates = result.inferences.filter( + ({ kind }) => kind === "endpoint_candidate", + ); + expect(candidates.map(({ value }) => value).sort()).toEqual([ + "ADDRESS_ADDED", + "POST", + "X", + "login", + "passwordControl", + "token", "users", ]); + expect( + candidates.every( + ({ confidence, basis }) => + confidence === "low" && + basis.length === 1 && + basis[0]?.artifact_sha256 === + createWebTextArtifact(source, "text/javascript").sha256 && + basis[0]?.line !== null && + basis[0]?.detector.startsWith("call:"), + ), + ).toBe(true); + const staticEndpoints = analyzeJavaScriptStaticSource(source).endpoints; + expect( + staticEndpoints + .filter(({ interpretation }) => interpretation === "candidate") + .map(({ value }) => value) + .sort(), + ).toEqual(candidates.map(({ value }) => value).sort()); + expect( + staticEndpoints + .filter(({ interpretation }) => interpretation === undefined) + .map(({ value }) => value) + .sort(), + ).toEqual(result.observations.endpoints.map(({ value }) => value).sort()); + expect(webBundleAnalysisSchema.parse(result)).toEqual(result); + }); + + it("deduplicates repeated candidate sites while keeping script, value and detector identity", () => { + const source = `${'form.get("name");\n'.repeat(5_000)}translate.get("name");\nform.get("other");`; + const captured = inspection(source); + const first = captured.scripts.items[0]; + if (first === undefined) throw new Error("Missing captured script fixture"); + const secondKey = `scr_${"2".repeat(64)}`; + const result = analyzeCapturedWebBundle({ + ...captured, + scripts: { + total: 2, + items: [ + first, + { + ...first, + script_key: secondKey, + url: `${origin}/assets/second.js`, + }, + ], + }, + }); + const candidates = result.inferences.filter( + ({ kind }) => kind === "endpoint_candidate", + ); + expect(candidates).toHaveLength(6); + for (const scriptKey of [first.script_key, secondKey]) { + const local = candidates.filter( + ({ basis }) => basis[0]?.script_key === scriptKey, + ); + expect( + local.map(({ value, basis }) => [value, basis[0]?.detector]), + ).toEqual([ + ["name", "call:form.get:unverified-network-receiver"], + ["name", "call:translate.get:unverified-network-receiver"], + ["other", "call:form.get:unverified-network-receiver"], + ]); + expect(local[0]?.basis[0]).toMatchObject({ + line: 1, + column: 0, + artifact_sha256: createWebTextArtifact(source, "text/javascript") + .sha256, + }); + } + expect(result.observations.endpoints).toEqual([]); + expect(result.capture.source_artifacts).toHaveLength(2); + expect(result.completeness).toMatchObject({ + status: "complete", + parsed_scripts: 2, + }); + expect(result.limitations).toContain( + 'Endpoint candidates are deduplicated by script, value and detector; basis retains only the first matching source coordinate. They are not a complete call-site inventory or a runtime occurrence count. completeness.status="complete" describes examination of captured source, not a call-site census.', + ); }); }); @@ -408,11 +655,16 @@ describe("web bundle configuration and navigation parity", () => { "shadowed", "users", ]; + const web = analyzeCapturedWebBundle(inspection(source)); + expect(web.observations.endpoints.map(({ value }) => value)).toEqual([ + "/v1/settings", + ]); expect( - analyzeCapturedWebBundle(inspection(source)) - .observations.endpoints.map(({ value }) => value) + web.inferences + .filter(({ kind }) => kind === "endpoint_candidate") + .map(({ value }) => value) .sort(), - ).toEqual(expected); + ).toEqual(expected.slice(1)); expect( analyzeJavaScriptStaticSource(source) .endpoints.map(({ value }) => value) @@ -420,6 +672,68 @@ describe("web bundle configuration and navigation parity", () => { ).toEqual(expected); }); + it("classifies proven VS Code extension state reads as non-endpoints", () => { + const reads = ` + context.globalState.get("lastVersion"); + this.workspaceState.get("closedRepositories", []); + context?.globalState.get("onboarded"); + context.globalState.get("/v1/state"); + context.secrets.get("token"); + api.get("users"); + `; + const analyze = (source: string) => { + const web = analyzeCapturedWebBundle(inspection(source)); + const staticAnalysis = analyzeJavaScriptStaticSource(source); + return { + web: { + endpoints: web.observations.endpoints + .map(({ value }) => value) + .sort(), + candidates: web.inferences + .filter(({ kind }) => kind === "endpoint_candidate") + .map(({ value }) => value) + .sort(), + }, + application: { + endpoints: staticAnalysis.endpoints + .filter(({ interpretation }) => interpretation === undefined) + .map(({ value }) => value) + .sort(), + candidates: staticAnalysis.endpoints + .filter(({ interpretation }) => interpretation === "candidate") + .map(({ value }) => value) + .sort(), + }, + }; + }; + const proven = { + endpoints: ["/v1/state"], + candidates: ["token", "users"], + }; + expect(analyze(`const vscode = require("vscode");${reads}`)).toEqual({ + web: proven, + application: proven, + }); + expect(analyze(`import * as vscode from "vscode";${reads}`)).toEqual({ + web: proven, + application: proven, + }); + const unproven = { + endpoints: ["/v1/state"], + candidates: [ + "closedRepositories", + "lastVersion", + "onboarded", + "token", + "users", + ], + }; + expect(analyze(`const vscode = require("./vscode");${reads}`)).toEqual({ + web: unproven, + application: unproven, + }); + }); + it("recognizes loadURL endpoints like static analysis", () => { const result = analyzeCapturedWebBundle( inspection('window.win.loadURL("https://embedded.test/app");'), @@ -662,6 +976,26 @@ describe("web metadata key semantics", () => { }); }); +it("retains WebMCP declarations through optional member and call chaining", () => { + const result = analyzeCapturedWebBundle( + inspection(` + document.modelContext?.registerTool({ name: "optional member" }); + document.modelContext.registerTool?.({ name: "optional call" }); + `), + ); + + expect(result.observations.webmcp_declarations).toEqual([ + expect.objectContaining({ + name: "optional member", + trust: "page-declared-untrusted", + }), + expect.objectContaining({ + name: "optional call", + trust: "page-declared-untrusted", + }), + ]); +}); + it("reads expression-free template dynamic imports as chunk edges", () => { const result = analyzeCapturedWebBundle( inspection(` diff --git a/src/domain/webBundleAnalyzerInspection.ts b/src/domain/webBundleAnalyzerInspection.ts index f48faf552..2c6b8c7ed 100644 --- a/src/domain/webBundleAnalyzerInspection.ts +++ b/src/domain/webBundleAnalyzerInspection.ts @@ -192,19 +192,20 @@ const inspectCall = ( node.callee, openReceiverFacts.get(node.start ?? -1), ); - if (endpoint !== undefined) + if (endpoint?.kind === "candidate") + addEndpointCandidate(script, node, name, endpoint.value, accumulator); + else if (endpoint !== undefined) addFinding({ collection: accumulator.endpoints, script, - rawValue: endpoint, + rawValue: endpoint.value, mechanism: `call:${name}`, node, accumulator, }); if ( - (t.isCallExpression(node) || t.isNewExpression(node)) && - (name.endsWith("modelContext.registerTool") || - name === "modelContext.registerTool") + name.endsWith("modelContext.registerTool") || + name === "modelContext.registerTool" ) addWebMcpDeclaration(script, node, accumulator); }; @@ -260,6 +261,33 @@ interface AddFindingContext { readonly accumulator: AnalysisAccumulator; } +const addEndpointCandidate = ( + script: IncludedScript, + node: t.Node, + name: string, + rawValue: string, + accumulator: AnalysisAccumulator, +): void => { + const value = sanitizeEndpointCandidate(rawValue); + const detector = `call:${name}:unverified-network-receiver`; + const key = compositeKey([ + "endpoint-candidate", + script.script_key, + value, + detector, + ]); + // This is a per-script candidate inventory, not a callsite list. Keep the + // first representative source position without multiplying inline records. + addUnique(accumulator, key, () => + accumulator.inferences.push({ + kind: "endpoint_candidate", + value, + confidence: "low", + basis: [{ ...basis(script, detector), ...locationFields(node) }], + }), + ); +}; + const addFinding = (context: AddFindingContext): void => { const value = sanitizeEndpointCandidate(context.rawValue); const key = `finding\0${context.mechanism}\0${context.script.script_key}\0${value}`; @@ -274,7 +302,7 @@ const addFinding = (context: AddFindingContext): void => { const addWebMcpDeclaration = ( script: IncludedScript, - node: t.CallExpression | t.NewExpression, + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, accumulator: AnalysisAccumulator, ): void => { const declaration = node.arguments[0]; diff --git a/src/domain/webBundleAnalyzerResult.ts b/src/domain/webBundleAnalyzerResult.ts index 4d2509675..b34867f85 100644 --- a/src/domain/webBundleAnalyzerResult.ts +++ b/src/domain/webBundleAnalyzerResult.ts @@ -201,6 +201,8 @@ const buildCompleteness = ( const bundleLimitations = (): WebBundleAnalysis["limitations"] => [ "Static bundle findings are observations or bounded inferences; REA does not execute captured JavaScript.", + "Bare string arguments of unbound HTTP-method-named calls are low-confidence endpoint candidates; form, translation, storage and other APIs can use the same syntax. URL-shaped findings still do not prove network execution.", + 'Endpoint candidates are deduplicated by script, value and detector; basis retains only the first matching source coordinate. They are not a complete call-site inventory or a runtime occurrence count. completeness.status="complete" describes examination of captured source, not a call-site census.', "String-built routes and endpoints, encrypted configuration, and server-side behavior may remain unknown.", "Page-declared WebMCP metadata is untrusted and is never registered or invoked as an REA tool.", ]; diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts new file mode 100644 index 000000000..bb6f8b585 --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts @@ -0,0 +1,33 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; +import { javaScriptWorkerStaticAnalysisSchema } from "./JavaScriptAnalysisWorkerSchemas.js"; + +it("versions fingerprints through the worker schema and rejects v1", () => { + const facts = + analyzeJavaScriptStaticSource(`globalThis.webpackChunkApp.push([[1], { + 17(module, exports) { const key = selectKey(); exports.read = value[key]; } + }]);`); + expect(facts.bundler_registrations[0]?.modules[0]).toMatchObject({ + structural_fingerprint_algorithm: "babel-ast-v2", + structural_fingerprint_sha256: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + expect( + javaScriptWorkerStaticAnalysisSchema.safeParse( + JSON.parse(JSON.stringify(facts)) as unknown, + ).success, + ).toBe(true); + const old = { + ...facts, + bundler_registrations: facts.bundler_registrations.map((registration) => ({ + ...registration, + modules: registration.modules.map((module) => ({ + ...module, + structural_fingerprint_algorithm: "babel-ast-v1", + })), + })), + }; + expect(javaScriptWorkerStaticAnalysisSchema.safeParse(old).success).toBe( + false, + ); +}); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index 40073f1ae..b8358ec47 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { digestSchema } from "../../domain/digests.js"; import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; +import { JAVASCRIPT_AST_FINGERPRINT_ALGORITHM } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptSemanticProperty, JavaScriptSemanticValue, @@ -363,7 +364,9 @@ export const javaScriptWorkerStaticAnalysisSchema = z exports: strings, location: range, structural_fingerprint_sha256: digestSchema, - structural_fingerprint_algorithm: z.literal("babel-ast-v1"), + structural_fingerprint_algorithm: z.literal( + JAVASCRIPT_AST_FINGERPRINT_ALGORITHM, + ), }), ), location: range, diff --git a/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts b/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts index 3a7fad35f..24dc748b7 100644 --- a/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts +++ b/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts @@ -18,6 +18,9 @@ const createModuleLookupFixture = async () => { type: "commonjs", }), "esm/entry.mjs": [ + 'import "./a%2eb.mjs";', + 'import "./sub/%2e%2e/x.mjs";', + 'import "./sub%2fx.mjs";', 'import "./extensionless";', 'import "./directory";', 'import "./explicit.js";', @@ -43,6 +46,8 @@ const createModuleLookupFixture = async () => { "esm/directory/index.js": "module.exports = 2;", "esm/explicit.js.js": "module.exports = 3;", "esm/exact.js": "module.exports = 4;", + "esm/a.b.mjs": "export default 'encoded dot';", + "esm/x.mjs": "export default 'confined parent';", "esm/actual.cjs": "module.exports = 5;", "esm/.hidden.js": "module.exports = 'local hidden';", "esm/..pkg.js": "module.exports = 'local dot package';", @@ -51,6 +56,8 @@ const createModuleLookupFixture = async () => { "esm/typed.mts": 'import value from "./typed-dep.js"; void value;', "esm/typed-dep.ts": "export default 42;", "cjs/entry.cjs": [ + 'require("./lit%2eb");', + 'require("./x%2Fy");', 'require("./extensionless");', 'require("./directory");', 'require("./feature");', @@ -76,6 +83,8 @@ const createModuleLookupFixture = async () => { "index.js": "module.exports = 'parent directory';", "cjs/.hidden.js": "module.exports = 'local hidden';", "cjs/extensionless.cjs": "module.exports = 1;", + "cjs/lit%2eb.js": "module.exports = 'literal dot bytes';", + "cjs/x%2Fy.js": "module.exports = 'literal slash bytes';", "cjs/feature.cjs": "module.exports = 2;", "cjs/module-only.mjs": "export default 42;", "cjs/valid-directory/index.js": "module.exports = 42;", @@ -98,7 +107,7 @@ const createModuleLookupFixture = async () => { "node_modules/ancestor-blocked/index.js": "module.exports = 9;", "node_modules/ancestor-sub/parser.js": "module.exports = 10;", "esm/runtime-control.mjs": [ - 'const specifiers = ["./extensionless", "./directory", "./explicit.js", "./exact.js", "./actual.cjs?cache=1", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser", "./.hidden.js", "./..pkg.js", "./exact.js/.", "./exact.js/child/..", ".hidden.js", "..pkg.js", ".", ".."];', + 'const specifiers = ["./a%2eb.mjs", "./sub/%2e%2e/x.mjs", "./sub%2fx.mjs", "./extensionless", "./directory", "./explicit.js", "./exact.js", "./actual.cjs?cache=1", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser", "./.hidden.js", "./..pkg.js", "./exact.js/.", "./exact.js/child/..", ".hidden.js", "..pkg.js", ".", ".."];', "const results = [];", "for (const specifier of specifiers) {", " try { const value = (await import(specifier)).default; results.push({ specifier, ok: true, value }); }", @@ -107,7 +116,7 @@ const createModuleLookupFixture = async () => { "console.log(JSON.stringify(results));", ].join("\n"), "cjs/runtime-control.cjs": [ - 'const specifiers = ["./extensionless", "./directory", "./feature", "./feature.cjs", "./module-only", "./valid-directory", "./default-js", "./default-json", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser", ".hidden.js", "./.hidden.js", ".", ".."];', + 'const specifiers = ["./lit%2eb", "./x%2Fy", "./extensionless", "./directory", "./feature", "./feature.cjs", "./module-only", "./valid-directory", "./default-js", "./default-json", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser", ".hidden.js", "./.hidden.js", ".", ".."];', "const results = [];", "for (const specifier of specifiers) {", " try { const value = require(specifier); results.push({ specifier, ok: true, value }); }", @@ -142,6 +151,13 @@ const expectNodeRuntimeControls = ( const esmResults = JSON.parse(esmOutput) as RuntimeControlResult[]; const cjsResults = JSON.parse(cjsOutput) as RuntimeControlResult[]; expect(esmResults).toEqual([ + { specifier: "./a%2eb.mjs", ok: true, value: "encoded dot" }, + { specifier: "./sub/%2e%2e/x.mjs", ok: true, value: "confined parent" }, + { + specifier: "./sub%2fx.mjs", + ok: false, + code: "ERR_INVALID_MODULE_SPECIFIER", + }, { specifier: "./extensionless", ok: false, @@ -199,6 +215,8 @@ const expectNodeRuntimeControls = ( { specifier: "..", ok: false, code: "ERR_UNSUPPORTED_DIR_IMPORT" }, ]); expect(cjsResults).toEqual([ + { specifier: "./lit%2eb", ok: true, value: "literal dot bytes" }, + { specifier: "./x%2Fy", ok: true, value: "literal slash bytes" }, { specifier: "./extensionless", ok: false, @@ -250,6 +268,9 @@ cliTest.for(["CLI", "stdio MCP"] as const)( const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); const expectations = [ + ["import", "./a%2eb.mjs", "esm/a.b.mjs", "resolved"], + ["import", "./sub/%2e%2e/x.mjs", "esm/x.mjs", "resolved"], + ["import", "./sub%2fx.mjs", null, "rejected"], ["import", "./extensionless", null, "not-found"], ["import", "./directory", null, "not-found"], ["import", "./explicit.js", null, "not-found"], @@ -271,6 +292,8 @@ cliTest.for(["CLI", "stdio MCP"] as const)( ["import", "./exact.js/.", null, "not-found"], ["import", "./exact.js/child/..", null, "not-found"], ["require", "./extensionless", null, "not-found"], + ["require", "./lit%2eb", "cjs/lit%2eb.js", "resolved"], + ["require", "./x%2Fy", "cjs/x%2Fy.js", "resolved"], ["require", "./directory", null, "not-found"], ["require", "./feature", null, "not-found"], ["require", "./feature.cjs", "cjs/feature.cjs", "resolved"], diff --git a/tests/acceptance/applications/packageMainLookupCli.test.ts b/tests/acceptance/applications/packageMainLookupCli.test.ts new file mode 100644 index 000000000..efdef8a55 --- /dev/null +++ b/tests/acceptance/applications/packageMainLookupCli.test.ts @@ -0,0 +1,284 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + analyzeJavaScriptDirectory, + runNodeJsonControl, +} from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + { name: "null-main", metadata: { main: null } }, + { name: "false-main", metadata: { main: false } }, + { name: "zero-main", metadata: { main: 0 } }, + { name: "true-main", metadata: { main: true } }, + { name: "object-main", metadata: { main: {} } }, + { name: "empty-main", metadata: { main: "" } }, + { name: "missing-main", metadata: {} }, + { name: "string-main", metadata: { main: "./index.js" } }, +] as const; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "matches Node package main interpretation through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-package-main-"); + const files: Record = { + "package.json": JSON.stringify({ type: "commonjs" }), + "entry.cjs": cases + .map(({ name }) => `require(${JSON.stringify(name)});`) + .join("\n"), + "entry.mjs": cases + .map(({ name }) => `import ${JSON.stringify(name)};`) + .join("\n"), + "control.cjs": [ + `const specifiers = ${JSON.stringify(cases.map(({ name }) => name))};`, + "console.log(JSON.stringify(specifiers.map((specifier) => require(specifier))));", + ].join("\n"), + "control.mjs": [ + `const specifiers = ${JSON.stringify(cases.map(({ name }) => name))};`, + "console.log(JSON.stringify(await Promise.all(specifiers.map(async (specifier) => (await import(specifier)).default))));", + ].join("\n"), + }; + for (const { name, metadata } of cases) { + files[`node_modules/${name}/package.json`] = JSON.stringify(metadata); + files[`node_modules/${name}/index.js`] = + `module.exports = ${JSON.stringify(`node_modules/${name}/index.js`)};`; + } + await writeFixtureFiles(root, files); + + for (const filename of ["control.cjs", "control.mjs"]) { + expect(await runNodeJsonControl(root, filename)).toEqual( + cases.map(({ name }) => `node_modules/${name}/index.js`), + ); + } + + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + for (const kind of ["require", "import"]) + for (const { name } of cases) + expect( + graph.edges + .filter(({ relation }) => relation === "imports") + .map(({ properties }) => properties), + ).toContainEqual( + expect.objectContaining({ + module_link_kind: kind, + specifier: name, + resolved_path: `node_modules/${name}/index.js`, + resolution_status: "resolved", + }), + ); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "preserves loader-specific package main probes and finalization through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-package-main-path-"); + const packages = [ + { + name: "dot-slash-main", + main: "./lib/", + files: ["lib.js", "lib/index.js", "index.js"], + require: "lib.js", + import: null, + }, + { + name: "slash-main", + main: "lib/", + files: ["lib.js", "lib/index.js", "index.js"], + require: "lib.js", + import: null, + }, + { + name: "plain-main", + main: "./lib", + files: ["lib.js", "lib/index.js"], + require: "lib.js", + import: "lib.js", + }, + { + name: "directory-main", + main: "./lib/", + files: ["lib/index.js", "index.js"], + require: "lib/index.js", + import: "lib/index.js", + }, + { + name: "fallback-main", + main: "./missing/", + files: ["index.js"], + require: "index.js", + import: "index.js", + }, + { + name: "exact-main", + main: "./lib.js/", + files: ["lib.js", "index.js"], + require: "lib.js", + import: "lib.js", + }, + { + name: "finalized-directory-main", + main: "./lib/", + files: ["lib.js", "lib/.js/child.js"], + require: "lib.js", + import: null, + importError: "ERR_UNSUPPORTED_DIR_IMPORT", + }, + { + name: "finalized-main", + main: "./lib/", + files: ["lib.js", "lib/.js", "lib/index.js"], + require: "lib.js", + import: "lib/.js", + }, + ] as const; + const files: Record = { + "package.json": JSON.stringify({ type: "commonjs" }), + "entry.cjs": [ + ...packages.map(({ name }) => `require(${JSON.stringify(name)});`), + 'require("./local/");', + ].join("\n"), + "entry.mjs": packages + .map(({ name }) => `import ${JSON.stringify(name)};`) + .join("\n"), + "local.js": 'module.exports = "local.js";', + "local/index.js": 'module.exports = "local/index.js";', + }; + for (const item of packages) { + files[`node_modules/${item.name}/package.json`] = JSON.stringify({ + main: item.main, + }); + for (const path of item.files) { + const target = `node_modules/${item.name}/${path}`; + files[target] = `module.exports = ${JSON.stringify(target)};`; + } + } + for (const kind of ["require", "import"] as const) { + const filename = `control.${kind === "require" ? "cjs" : "mjs"}`; + files[filename] = [ + `const specifiers = ${JSON.stringify(packages.map(({ name }) => name))};`, + "const results = [];", + "for (const specifier of specifiers) {", + ` try { results.push({ path: ${kind === "require" ? "require(specifier)" : "(await import(specifier)).default"} }); }`, + " catch (error) { results.push({ code: error.code }); }", + "}", + "console.log(JSON.stringify(results));", + ].join("\n"); + } + files["local-control.cjs"] = + 'console.log(JSON.stringify(require("./local/")));'; + await writeFixtureFiles(root, files); + for (const kind of ["require", "import"] as const) { + const control = await runNodeJsonControl( + root, + `control.${kind === "require" ? "cjs" : "mjs"}`, + ); + expect(control).toEqual( + packages.map((item) => + item[kind] === null + ? { + code: + "importError" in item + ? item.importError + : "ERR_MODULE_NOT_FOUND", + } + : { path: `node_modules/${item.name}/${item[kind]}` }, + ), + ); + } + expect(await runNodeJsonControl(root, "local-control.cjs")).toBe( + "local/index.js", + ); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const relationships = graph.edges + .filter(({ relation }) => relation === "imports") + .map(({ properties }) => properties); + for (const kind of ["require", "import"] as const) + for (const item of packages) + expect(relationships).toContainEqual( + expect.objectContaining({ + module_link_kind: kind, + specifier: item.name, + resolved_path: + item[kind] === null + ? null + : `node_modules/${item.name}/${item[kind]}`, + resolution_status: + item[kind] === null + ? "importError" in item + ? "rejected" + : "not-found" + : "resolved", + }), + ); + expect(relationships).toContainEqual( + expect.objectContaining({ + module_link_kind: "require", + specifier: "./local/", + resolved_path: "local/index.js", + resolution_status: "resolved", + }), + ); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "resolves package mains that name the artifact root through %s", + async (surface, { cli }) => { + const mains = ["../..", "../../", "../../."]; + const names = mains.map((_, index) => `root-main-${index}`); + const requires = names.map((name) => `require("${name}")`); + for (const rootIndex of ["index.js", "index.json", null]) { + const root = await createTestTempDirectory("rea-package-main-root-"); + const importOptions = + rootIndex === "index.json" ? ', { with: { type: "json" } }' : ""; + const imports = names.map( + (name) => + `import("${name}"${importOptions}).then(module => module.default)`, + ); + const files: Record = { + "package.json": '{"type":"commonjs"}', + "entry.cjs": requires.join(";\n"), + "entry.mjs": names.map((name) => `import "${name}";`).join("\n"), + ".js": 'module.exports = ".js decoy";', + ".json": '".json decoy"', + "control.cjs": `console.log(JSON.stringify([${requires.join(",")}]));`, + "control.mjs": `console.log(JSON.stringify(await Promise.all([${imports.join(",")}])));`, + }; + if (rootIndex !== null) + files[rootIndex] = + rootIndex === "index.js" + ? 'module.exports = "index.js";' + : '"index.json"'; + for (const [index, main] of mains.entries()) { + const directory = `node_modules/${names[index]}`; + files[`${directory}/package.json`] = JSON.stringify({ main }); + files[`${directory}/index.js`] = + `module.exports = "${directory}/index.js";`; + } + await writeFixtureFiles(root, files); + const expected = names.map( + (name) => rootIndex ?? `node_modules/${name}/index.js`, + ); + for (const filename of ["control.cjs", "control.mjs"]) + expect(await runNodeJsonControl(root, filename)).toEqual(expected); + + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const relationships = graph.edges + .filter(({ relation }) => relation === "imports") + .map(({ properties }) => properties); + for (const kind of ["require", "import"]) + for (const [index, name] of names.entries()) + expect(relationships).toContainEqual( + expect.objectContaining({ + module_link_kind: kind, + specifier: name, + resolved_path: expected[index], + resolution_status: "resolved", + }), + ); + } + }, +); diff --git a/tests/acceptance/applications/packageSpecifierSuffixCli.test.ts b/tests/acceptance/applications/packageSpecifierSuffixCli.test.ts index ad175006e..ee319ff7e 100644 --- a/tests/acceptance/applications/packageSpecifierSuffixCli.test.ts +++ b/tests/acceptance/applications/packageSpecifierSuffixCli.test.ts @@ -18,6 +18,23 @@ type PackageSpecifierCase = { readonly error: string | null; }; +const directoryFormSpecifiers = [ + "legacy/feature.js/.", + "legacy/feature.js/child/..", + "legacy/feature.js/", + "legacy/feature.js/.?cache=1#entry", + "legacy/with%20space.js/.", + "legacy/missing/.", + "legacy/feature.js/.\t", + "legacy/feature.js/.\n?cache=1#entry", + "legacy/feature.js/ch\til\rd/..", + "legacy/feature.js/. ", +]; +const encodedDotSpecifiers = ["\t", "\n", "\r"].flatMap((control) => [ + `legacy/child/%2${control}e%2${control}e/feature.js`, + `legacy/child/.%2${control}E/feature.js`, +]); + const requests = [ { specifier: "rea-self-fixture", @@ -131,6 +148,74 @@ const requests = [ value: "legacy spaced feature", error: null, }, + ...directoryFormSpecifiers.map((specifier) => ({ + specifier, + target: null, + status: "rejected" as const, + value: null, + error: "ERR_UNSUPPORTED_DIR_IMPORT", + })), + ...encodedDotSpecifiers.map((specifier) => ({ + specifier, + target: null, + status: "rejected" as const, + value: "legacy feature", + error: null, + })), + { + specifier: "legacy/feature.js/%2e", + target: null, + status: "rejected", + value: null, + error: "ERR_UNSUPPORTED_DIR_IMPORT", + }, + { + specifier: "legacy/feature.js/.%09", + target: null, + status: "not-found", + value: null, + error: "ERR_MODULE_NOT_FOUND", + }, + { + specifier: "legacy/feature.js/%2f", + target: null, + status: "rejected", + value: null, + error: "ERR_INVALID_MODULE_SPECIFIER", + }, + ...[ + "legacy/feature.js?cache=/..", + "legacy/feature.js#entry/.", + "legacy/fea\tture.js", + "legacy/feature.js?cache=%2\te", + "legacy/feature.js#entry=%2\nf", + "legacy/child/../feature.js", + ].map((specifier) => ({ + specifier, + target: "node_modules/legacy/feature.js", + status: "resolved" as const, + value: "legacy feature", + error: null, + })), + ...[ + "fixture/feature.js/.", + "fixture/feature.js/.\t", + "fixture/child/%2\te%2\te/feature", + "legacy/../fixture/literal.cjs", + ].map((specifier) => ({ + specifier, + target: "node_modules/fixture/literal.cjs", + status: "resolved" as const, + value: "literal", + error: null, + })), + { + specifier: "legacy/literal%252e.js", + target: "node_modules/legacy/literal%2e.js", + status: "resolved", + value: "legacy literal percent", + error: null, + }, { specifier: "legacy/feature", target: null, @@ -184,6 +269,9 @@ const createPackageSpecifierFixture = async (): Promise<{ exports: { "./feature": "./feature.cjs", "./literal?cache=1": "./literal.cjs", + "./feature.js/.": "./literal.cjs", + "./feature.js/.\t": "./literal.cjs", + "./child/%2\te%2\te/feature": "./literal.cjs", }, }), ); @@ -206,6 +294,10 @@ const createPackageSpecifierFixture = async (): Promise<{ join(legacyPackagePath, "with space.js"), 'module.exports = "legacy spaced feature";', ), + writeFile( + join(legacyPackagePath, "literal%2e.js"), + 'module.exports = "legacy literal percent";', + ), writeFile( join(legacyPackagePath, "nested", "package.json"), JSON.stringify({ main: "./main.cjs" }), @@ -258,6 +350,20 @@ cliTest.for(["CLI", "stdio MCP"] as const)( resolution_status: status, }), evidence: expect.objectContaining({ + ...(encodedDotSpecifiers.includes(specifier) + ? { + limitations: expect.arrayContaining([ + "Encoded dot or separator bytes are rejected before artifact path resolution.", + ]), + } + : {}), + ...(directoryFormSpecifiers.includes(specifier) + ? { + limitations: expect.arrayContaining([ + `ESM package subpath ${JSON.stringify(specifier)} has URL directory form; directory imports are unsupported and no exact-file, package main or index lookup was attempted.`, + ]), + } + : {}), ...(specifier === "legacy/feature" ? { limitations: expect.arrayContaining([ diff --git a/tests/acceptance/applications/packageUrlUnicodeCli.test.ts b/tests/acceptance/applications/packageUrlUnicodeCli.test.ts new file mode 100644 index 000000000..a50b4c837 --- /dev/null +++ b/tests/acceptance/applications/packageUrlUnicodeCli.test.ts @@ -0,0 +1,268 @@ +import { createPackageWithOptions } from "@electron/asar"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const escapingTarget = "./%2\te%2\te/outside.js"; +const targets = { + feature: { value: "./feature.js", imports: "relative", exports: "relative" }, + encodedSeparator: { + value: "./feature%2\tf.js", + imports: "rejected", + exports: "rejected", + }, + malformedEncoding: { + value: "./feature%ZZ.js", + imports: "rejected", + exports: "rejected", + }, + blocked: { value: null, imports: "external", exports: "external" }, + empty: { value: [], imports: "external", exports: "external" }, + inactive: { + value: { browser: "./feature.js" }, + imports: "external", + exports: "external", + }, + external: { + value: "another-package", + imports: "external", + exports: "rejected", + }, + recursive: { value: "#another", imports: "external", exports: "rejected" }, + invalid: { value: "../feature.js", imports: "rejected", exports: "rejected" }, + scalar: { value: false, imports: "rejected", exports: "rejected" }, + numeric: { value: { 0: null }, imports: "rejected", exports: "rejected" }, + arrayNull: { value: [false, null], imports: "external", exports: "external" }, + arrayInactive: { + value: [{ browser: "./feature.js" }], + imports: "external", + exports: "external", + }, + arrayExternal: { + value: [false, "another-package"], + imports: "external", + exports: "rejected", + }, + arrayRelative: { + value: [false, "./feature.js"], + imports: "relative", + exports: "relative", + }, + relativeFirst: { + value: ["./feature.js", null], + imports: "relative", + exports: "relative", + }, + escapeExternal: { + value: [escapingTarget, "another-package"], + imports: "external", + exports: "rejected", + }, + escapeNull: { + value: [escapingTarget, null], + imports: "external", + exports: "external", + }, + escapeRelative: { + value: [escapingTarget, "./feature.js"], + imports: "relative", + exports: "relative", + }, + escapeCondition: { + value: { node: escapingTarget, default: "./feature.js" }, + imports: "rejected", + exports: "rejected", + }, + escapeNested: { + value: [{ node: escapingTarget }, "another-package"], + imports: "external", + exports: "rejected", + }, + escapeConfig: { + value: [{ 0: escapingTarget }, "another-package"], + imports: "rejected", + exports: "rejected", + }, + wtf8Reentry: { + value: ["./%2\te%2\te/%ED%A0%80/feature.js", "another-package"], + imports: "external", + exports: "rejected", + }, + opaqueReentry: { + value: ["./%2\te%2\te/rea-op\taque-0/feature.js", "another-package"], + imports: "external", + exports: "rejected", + }, + structuralReentry: { + value: "./%2\te%2\te/mapped/feature.js", + imports: "rejected", + exports: "relative", + }, +} as const; +const requests = [ + { + specifier: "legacy/feature.js", + path: "node_modules/legacy/feature.js", + status: "legacy" as const, + }, + { + specifier: "legacy/feature.js/.", + path: "", + status: "rejected" as const, + }, + { + specifier: "legacy/../mapped/feature.js", + path: "node_modules/mapped/feature.js", + status: "legacy" as const, + }, + { + specifier: "legacy/../mapped/feature.js/.", + path: "", + status: "rejected" as const, + }, + { + specifier: "legacy/../../../outside.js", + path: "outside.js", + status: "root-legacy" as const, + }, + { + specifier: "legacy/../../../absent.js", + path: "", + status: "not-found" as const, + }, + ...Object.entries(targets).flatMap(([name, target]) => [ + { specifier: `#${name}`, path: "feature.js", status: target.imports }, + { + specifier: `mapped/${name}`, + path: "node_modules/mapped/feature.js", + status: target.exports, + }, + ]), + { specifier: "#missing", path: "", status: "external" as const }, + { specifier: "mapped/missing", path: "", status: "external" as const }, +]; +const sources = ["import", "require"].map((mode) => ({ + mode, + path: mode === "import" ? "main.mjs" : "main.cjs", + source: requests + .filter(({ specifier }) => mode === "import" || !specifier.endsWith("/.")) + .map(({ specifier }) => + mode === "import" + ? `import ${JSON.stringify(specifier)};` + : `require(${JSON.stringify(specifier)});`, + ) + .join("\n"), +})); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "preserves package target facts under unrepresentable URL paths through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-package-url-unicode-"); + const application = join(root, "application"); + await mkdir(application); + const fixture = join(root, "source"); + await writeFixtureFiles(fixture, { "outside.js": "export default 4;" }); + const prefixes = ["highxx", "lowxxx", "valid\ud83d\ude00"]; + for (const prefix of prefixes) + await writeFixtureFiles(join(fixture, prefix), { + ...Object.fromEntries( + sources.map(({ path, source }) => [path, source]), + ), + "package.json": JSON.stringify({ + imports: Object.fromEntries( + Object.entries(targets).map(([name, { value }]) => [ + `#${name}`, + value, + ]), + ), + }), + "feature.js": "export default 1;", + "node_modules/legacy/package.json": "{}", + "node_modules/legacy/feature.js": "export default 2;", + "node_modules/mapped/package.json": JSON.stringify({ + exports: Object.fromEntries( + Object.entries(targets).map(([name, { value }]) => [ + `./${name}`, + value, + ]), + ), + }), + "node_modules/mapped/feature.js": "export default 3;", + }); + const archive = join(application, "app.asar"); + await createPackageWithOptions(fixture, archive, {}); + const bytes = await readFile(archive); + const header = bytes.subarray(16, 16 + bytes.readUInt32LE(12)); + const escaped = Buffer.from( + header + .toString("utf8") + .replace('"highxx":', '"\\ud800":') + .replace('"lowxxx":', '"\\udc00":'), + ); + expect(escaped.length).toBe(header.length); + expect(escaped.equals(header)).toBe(false); + escaped.copy(bytes, 16); + await writeFile(archive, bytes); + + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + ); + for (const prefix of ["\ud800", "\udc00", "valid\ud83d\ude00"]) + for (const { mode, path, source } of sources) + for (const request of requests) { + if (mode === "require" && request.specifier.endsWith("/.")) continue; + const base = `app.asar/${prefix}`; + const available = + prefix.startsWith("valid") || + request.status === "root-legacy" || + (mode === "require" && request.status === "legacy"); + const fileStatus = available ? "resolved" : "unavailable"; + const status = + request.status === "relative" || + request.status === "legacy" || + request.status === "root-legacy" + ? fileStatus + : request.status; + expect(graph.edges).toContainEqual( + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + module_link_kind: mode, + specifier: request.specifier, + resolution_status: status, + resolved_path: + status === "resolved" + ? `${request.status === "root-legacy" ? "app.asar" : base}/${request.path}` + : null, + }), + evidence: expect.objectContaining({ + limitations: + status === "unavailable" + ? expect.arrayContaining([ + expect.stringContaining("unpaired UTF-16 surrogate"), + ]) + : expect.not.arrayContaining([ + expect.stringContaining("unpaired UTF-16 surrogate"), + ]), + artifact: expect.objectContaining({ + sha256: createHash("sha256").update(source).digest("hex"), + }), + location: expect.objectContaining({ + value: expect.objectContaining({ source: `${base}/${path}` }), + }), + }), + }), + ); + } + expect(await readFile(archive)).toEqual(bytes); + }, +); diff --git a/tests/boundary/filesystem/javascript-applications/electronApplicationAnalysis.test.ts b/tests/boundary/filesystem/javascript-applications/electronApplicationAnalysis.test.ts index c2eb1070e..d04be62ee 100644 --- a/tests/boundary/filesystem/javascript-applications/electronApplicationAnalysis.test.ts +++ b/tests/boundary/filesystem/javascript-applications/electronApplicationAnalysis.test.ts @@ -77,17 +77,8 @@ describe("static Electron application analysis", () => { const result = await reconstructJavaScriptArtifact({ input_path: root }); const graph = parseJavaScriptApplicationGraph(result.graph); - // Computed access keeps no known members; the unknown access is an - // explicit dynamic flag rather than a "*" sentinel inside members. - const accesses = graph.nodes.flatMap((node) => - node.kind === "native-export" - ? node.observations.map(({ properties }) => ({ - members: Array.isArray(properties.requested_members) - ? properties.requested_members - : [], - dynamicMemberAccess: properties.dynamic_member_access ?? false, - })) - : [], + const nativeBindings = graph.nodes.filter( + (node) => node.kind === "native-export", ); expect(result.electron_summary).toMatchObject({ @@ -95,7 +86,20 @@ describe("static Electron application analysis", () => { native_addon_bindings: 1, resolved_native_addon_bindings: 1, }); - expect(accesses).toEqual([{ members: [], dynamicMemberAccess: true }]); + expect(nativeBindings).toMatchObject([ + { + observations: [ + { + properties: { + requested_members: [], + namespace_access: false, + dynamic_member_access: true, + native_export_verification: "not-performed", + }, + }, + ], + }, + ]); }); it("maps Electron boundaries through bundler-renamed bindings", async () => { diff --git a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts index 416337097..c35b8aef1 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -361,7 +361,7 @@ it("reconstructs package, Electron roles, Webpack/Rspack modules, and cross-laye expect.objectContaining({ properties: expect.objectContaining({ module_key: "1", - structural_fingerprint_algorithm: "babel-ast-v1", + structural_fingerprint_algorithm: "babel-ast-v2", }), }), ]), diff --git a/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts index a3b2231a2..ca747c90c 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts @@ -8,6 +8,9 @@ import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js import { createElectronActiveEvidence } from "../../../../src/application/javascript/ElectronActiveEvidence.js"; import { reconcileJavaScriptRuntimeEvidence } from "../../../../src/application/javascript/JavaScriptRuntimeReconciliationService.js"; import { createEvidence } from "../../../../src/domain/evidence.js"; +import { mappedRuntimePaths } from "../../../../src/domain/javascript/javascriptRuntimeStaticCandidates.js"; +import { projectRuntimeCaptures } from "../../../../src/domain/javascript/javascriptRuntimeReconciliationRuntime.js"; +import { electronPageInspectionSchema } from "../../../../src/domain/javascript/electronObservation.js"; import { parseRuntimeReconciliationInput, reconcileJavaScriptRuntime, @@ -419,3 +422,482 @@ it("imports an operator-provided cache layer through an explicit file mapping", sharedFile?.observations.map(({ properties }) => properties.path), ).toEqual(expect.arrayContaining(["app.js", "mapped/chunk.js"])); }); + +const sourceFreeRuntime = (runtimeRoot: string, scriptFile: string) => { + const runtime = electronRuntimeEvidence(runtimeRoot, "", { + scriptFile, + sourceIncluded: false, + includeWorker: false, + }); + const observation = electronPageInspectionSchema.parse( + runtime.normalized_result, + ); + for (const script of observation.scripts.items) + script.source = { included: false, reason: "not-requested" }; + return createEvidence(undefined, runtime.provider, { + operation: runtime.operation, + predicateType: runtime.predicate_type, + parameters: runtime.parameters, + result: observation, + confidence: runtime.confidence, + authority: runtime.authority, + }); +}; + +const reconcileScriptLocation = ( + application: Awaited>, + root: string, + scriptFile: string, + runtimeRoot = root, +) => { + return reconcileInput({ + static_layers: [ + { + role: "application", + analysis: application, + runtime_mappings: + runtimeRoot === root + ? [] + : [{ kind: "file-root", root: runtimeRoot, artifact_prefix: "" }], + }, + ], + runtime_observations: [sourceFreeRuntime(runtimeRoot, scriptFile)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "script"); +}; + +it.skipIf(process.platform === "win32")( + "keeps distinct POSIX backslash and slash script locations separate", + async () => { + const root = await createTestTempDirectory("rea-backslash-runtime-"); + const literal = "literal\\name.js"; + await writeFile(join(root, literal), "export const marker = 'literal';\n"); + const application = await analyzeFixture(root); + expect( + reconcileScriptLocation(application, root, "literal/name.js"), + ).toMatchObject({ status: "unmatched" }); + expect(reconcileScriptLocation(application, root, literal)).toMatchObject({ + status: "matched", + basis: "artifact-path", + }); + }, +); + +it.skipIf(process.platform === "win32")( + "does not make POSIX literal and slash paths ambiguous, including Windows runtime mappings", + async () => { + const root = await createTestTempDirectory("rea-backslash-runtime-pair-"); + const literal = "literal\\name.js"; + await mkdir(join(root, "literal")); + await writeFile(join(root, literal), "export const marker = 'literal';\n"); + await writeFile( + join(root, "literal", "name.js"), + "export const marker = 'slash';\n", + ); + const application = await analyzeFixture(root); + const slash = reconcileScriptLocation(application, root, "literal/name.js"); + const backslash = reconcileScriptLocation(application, root, literal); + expect(slash).toMatchObject({ status: "matched", basis: "artifact-path" }); + expect(backslash).toMatchObject({ + status: "matched", + basis: "artifact-path", + }); + expect(slash?.static_node_id).not.toBe(backslash?.static_node_id); + // These recorded Windows paths exercise matching only, not native Windows IO. + expect( + reconcileScriptLocation( + application, + root, + "literal\\name.js", + "C:\\runtime", + ), + ).toMatchObject({ + status: "matched", + static_node_id: slash?.static_node_id, + }); + }, +); + +it.skipIf(process.platform === "win32")( + "preserves URL-prefix path semantics separately from logical artifact names", + async () => { + const root = await createTestTempDirectory("rea-backslash-runtime-url-"); + await writeFile( + join(root, "literal\\name.js"), + "export const marker = 42;\n", + ); + const analysis = await analyzeFixture(root); + const parsed = parseRuntimeReconciliationInput( + reconcileJavaScriptRuntimeInputSchema.parse({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { + kind: "url-prefix", + prefix: "https://example.test/assets/", + artifact_prefix: "", + }, + ], + }, + ], + runtime_observations: [sourceFreeRuntime(root, "literal\\name.js")], + }), + ); + const layer = parsed.layers[0]; + const entity = projectRuntimeCaptures(parsed.captures).entities.find( + ({ kind }) => kind === "script", + ); + if (layer === undefined || entity === undefined) + throw new Error("Expected parsed static layer and runtime script"); + const mapped = (value: string) => + mappedRuntimePaths(layer, { + ...entity, + location: { kind: "url", value }, + }).map(({ path }) => path); + expect(mapped("https://example.test/assets/literal%5Cname.js")).toEqual([ + "literal\\name.js", + ]); + expect(mapped("https://example.test/assets/literal/name.js")).toEqual([ + "literal/name.js", + ]); + expect(mapped("https://example.test/assets/literal\\name.js")).toEqual([ + "literal/name.js", + ]); + expect(mapped("https://other.test/assets/literal/name.js")).toEqual([]); + expect(mapped("https://example.test/assets/%2e%2e/outside.js")).toEqual([]); + }, +); + +const rendererCapture = (runtimeRoot: string, filePath: string) => { + const runtime = sourceFreeRuntime(runtimeRoot, "main.cjs"); + const observation = electronPageInspectionSchema.parse( + runtime.normalized_result, + ); + observation.target.file_path = filePath; + for (const frame of observation.frames) frame.file_path = filePath; + return createEvidence(undefined, runtime.provider, { + operation: runtime.operation, + predicateType: runtime.predicate_type, + parameters: runtime.parameters, + result: observation, + confidence: runtime.confidence, + authority: runtime.authority, + }); +}; + +it.skipIf(process.platform === "win32")( + "interprets raw loadFile declarations using the captured native path format", + async () => { + const root = await createTestTempDirectory( + "rea-native-loadfile-declaration-", + ); + await mkdir(join(root, "dist")); + await writeFile(join(root, "dist", "index.html"), "slash file"); + await writeFile(join(root, "dist\\index.html"), "literal file"); + await writeFile( + join(root, "main.cjs"), + `const { BrowserWindow } = require('electron'); const win = new BrowserWindow(); win.loadFile(${JSON.stringify("dist\\index.html")});`, + ); + const analysis = await analyzeFixture(root); + const reconcileTarget = (runtimeRoot: string, path: string) => + reconcileInput({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { kind: "file-root", root: runtimeRoot, artifact_prefix: "" }, + ], + }, + ], + runtime_observations: [rendererCapture(runtimeRoot, path)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "target"); + // These Windows paths are recorded data, not native Windows execution. + expect( + reconcileTarget("C:\\runtime", "C:\\runtime\\dist\\index.html"), + ).toMatchObject({ status: "matched" }); + expect(reconcileTarget(root, join(root, "dist\\index.html"))).toMatchObject( + { status: "matched" }, + ); + expect( + reconcileTarget(root, join(root, "dist", "index.html")), + ).toMatchObject({ status: "unmatched" }); + }, +); + +it.skipIf(process.platform === "win32").each(["literal\\dir", "literal%5Cdir"])( + "maps a runtime root to the exact logical artifact prefix %j", + async (prefix) => { + const root = await createTestTempDirectory("rea-literal-artifact-prefix-"); + await mkdir(join(root, prefix)); + await writeFile( + join(root, prefix, "app.js"), + "export const marker = 42;\n", + ); + const analysis = await analyzeFixture(root); + const result = reconcileInput({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { kind: "file-root", root: "/deployed", artifact_prefix: prefix }, + ], + }, + ], + runtime_observations: [sourceFreeRuntime("/deployed", "app.js")], + }); + expect( + result.reconciliations.find( + ({ entity_kind }) => entity_kind === "script", + ), + ).toMatchObject({ status: "matched", basis: "operator-file-mapping" }); + }, +); + +it.skipIf(process.platform === "win32")( + "uses resolved artifact paths instead of a raw declaration alias", + async () => { + const root = await createTestTempDirectory("rea-resolved-loadfile-alias-"); + const directory = "literal\\dir"; + await mkdir(join(root, directory)); + await writeFile(join(root, directory, "index.html"), "owned target"); + await writeFile(join(root, "index.html"), "different target"); + await writeFile( + join(root, directory, "package.json"), + JSON.stringify({ + name: "literal-renderer", + version: "1.0.0", + renderer: "./index.html", + }), + ); + const analysis = await analyzeFixture(root); + const reconcileTarget = (path: string) => + reconcileInput({ + static_layers: [{ analysis, role: "application" }], + runtime_observations: [rendererCapture(root, path)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "target"); + expect(reconcileTarget(join(root, directory, "index.html"))).toMatchObject({ + status: "matched", + }); + expect(reconcileTarget(join(root, "index.html"))).toMatchObject({ + status: "unmatched", + }); + }, +); + +it("rejects unsafe logical artifact prefixes without decoding literal names", async () => { + const root = await createTestTempDirectory("rea-prefix-admission-"); + await writeFile(join(root, "app.js"), "export const marker = 42;\n"); + const analysis = await analyzeFixture(root); + const accepts = (artifact_prefix: string) => + reconcileJavaScriptRuntimeInputSchema.safeParse({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { kind: "file-root", root: "/deployed", artifact_prefix }, + ], + }, + ], + runtime_observations: [sourceFreeRuntime("/deployed", "app.js")], + }).success; + for (const prefix of [ + "\0", + "/absolute", + ".", + "..", + "a/../b", + "a//b", + "a/", + "./a", + ]) + expect(accepts(prefix), prefix).toBe(false); + for (const prefix of ["", "literal\\dir", "literal%5Cdir", "literal%2Fdir"]) + expect(accepts(prefix), prefix).toBe(true); +}); + +it.skipIf(process.platform === "win32")( + "keeps a leading backslash declaration literal only for POSIX captures", + async () => { + const root = await createTestTempDirectory("rea-leading-loadfile-"); + await writeFile(join(root, "\\index.html"), "literal target"); + await writeFile( + join(root, "main.cjs"), + `const { BrowserWindow } = require('electron'); const win = new BrowserWindow(); win.loadFile(${JSON.stringify("\\index.html")});`, + ); + const analysis = await analyzeFixture(root); + const reconcileTarget = (runtimeRoot: string, path: string) => + reconcileInput({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { kind: "file-root", root: runtimeRoot, artifact_prefix: "" }, + ], + }, + ], + runtime_observations: [rendererCapture(runtimeRoot, path)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "target"); + expect(reconcileTarget(root, join(root, "\\index.html"))).toMatchObject({ + status: "matched", + }); + expect( + reconcileTarget("C:\\runtime", "C:\\runtime\\index.html"), + ).toMatchObject({ status: "unmatched" }); + }, +); + +it("keeps a raw nested loadFile target application-relative when the producer also resolves a nested decoy", async () => { + const root = await createTestTempDirectory("rea-raw-loadfile-ownership-"); + await mkdir(join(root, "dist")); + await mkdir(join(root, "sub", "dist"), { recursive: true }); + await writeFile(join(root, "dist", "index.html"), "application target"); + await writeFile(join(root, "sub", "dist", "index.html"), "nested decoy"); + await writeFile( + join(root, "sub", "main.cjs"), + "const { BrowserWindow } = require('electron'); const win = new BrowserWindow(); win.loadFile('dist/index.html');", + ); + const analysis = await analyzeFixture(root); + const reconcileTarget = (path: string) => + reconcileInput({ + static_layers: [{ analysis, role: "application" }], + runtime_observations: [rendererCapture(root, path)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "target"); + expect(reconcileTarget(join(root, "dist", "index.html"))).toMatchObject({ + status: "matched", + }); + expect( + reconcileTarget(join(root, "sub", "dist", "index.html")), + ).toMatchObject({ status: "unmatched" }); +}); + +it + .skipIf(process.platform === "win32") + .each(["app", "literal\\app", "café", "cafe\u0301"])( + "maps raw renderer declarations through the logical application prefix %j", + async (prefix) => { + const root = await createTestTempDirectory("rea-prefixed-native-renderer-"); + await mkdir(join(root, prefix, "dist"), { recursive: true }); + await writeFile( + join(root, prefix, "dist", "index.html"), + "application target", + ); + await writeFile( + join(root, prefix, "main.cjs"), + "const { BrowserWindow } = require('electron'); const win = new BrowserWindow(); win.loadFile('dist/index.html');", + ); + const analysis = await analyzeFixture(root); + const reconcileTarget = (runtimeRoot: string, path: string) => + reconcileInput({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { kind: "file-root", root: runtimeRoot, artifact_prefix: prefix }, + ], + }, + ], + runtime_observations: [rendererCapture(runtimeRoot, path)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "target"); + expect( + reconcileTarget("/deployed", "/deployed/dist/index.html"), + ).toMatchObject({ status: "matched", basis: "operator-file-mapping" }); + expect( + reconcileTarget("C:\\deployed", "C:\\deployed\\dist\\index.html"), + ).toMatchObject({ status: "matched", basis: "operator-file-mapping" }); + expect(reconcileTarget("/deployed", "/deployed/index.html")).toMatchObject({ + status: "unmatched", + }); + }, +); + +it("confines raw renderer declarations to the operator-selected artifact prefix", async () => { + const root = await createTestTempDirectory("rea-native-prefix-isolation-"); + for (const app of ["appA", "appB"]) { + await mkdir(join(root, app, "dist"), { recursive: true }); + await writeFile(join(root, app, "dist", "index.html"), app); + await writeFile( + join(root, app, "main.cjs"), + "const { BrowserWindow } = require('electron'); const win = new BrowserWindow(); win.loadFile('dist/index.html');", + ); + } + const analysis = await analyzeFixture(root); + const result = reconcileInput({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { kind: "file-root", root: "/deployed", artifact_prefix: "appA" }, + ], + }, + ], + runtime_observations: [ + rendererCapture("/deployed", "/deployed/dist/index.html"), + ], + }); + const target = result.reconciliations.find( + ({ entity_kind }) => entity_kind === "target", + ); + expect(target).toMatchObject({ + status: "matched", + candidate_static_count: 1, + }); + const node = result.graph.nodes.find( + ({ node_id }) => node_id === target?.static_node_id, + ); + expect( + node?.observations.flatMap(({ evidence }) => + evidence.location.available && + evidence.location.value.kind === "source-range" + ? [evidence.location.value.source] + : [], + ), + ).toEqual(["appA/main.cjs"]); +}); + +it("retains uncertainty for overlapping file mappings with distinct application interpretations", async () => { + const root = await createTestTempDirectory("rea-overlapping-native-prefix-"); + await mkdir(join(root, "app", "dist"), { recursive: true }); + await writeFile(join(root, "app", "dist", "index.html"), "shared target"); + for (const [path, declaration] of [ + ["app/main.cjs", "dist/index.html"], + ["app/dist/main.cjs", "index.html"], + ]) + await writeFile( + join(root, path!), + `const { BrowserWindow } = require('electron'); const win = new BrowserWindow(); win.loadFile(${JSON.stringify(declaration)});`, + ); + const analysis = await analyzeFixture(root); + const result = reconcileInput({ + static_layers: [ + { + analysis, + role: "application", + runtime_mappings: [ + { kind: "file-root", root: "/deployed", artifact_prefix: "app" }, + { + kind: "file-root", + root: "/deployed/dist", + artifact_prefix: "app/dist", + }, + ], + }, + ], + runtime_observations: [ + rendererCapture("/deployed", "/deployed/dist/index.html"), + ], + }); + expect( + result.reconciliations.find(({ entity_kind }) => entity_kind === "target"), + ).toMatchObject({ + status: "ambiguous", + confidence: "unknown", + candidate_static_count: 2, + }); +}); diff --git a/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts b/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts index 97090c029..705fb7b18 100644 --- a/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts +++ b/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts @@ -306,7 +306,9 @@ describe("JavaScript export return-shape uncertain presence", () => { ).toBe("partial"); } }); +}); +describe("JavaScript export return-shape array presence", () => { it("excludes array holes while preserving observed unknown elements", async () => { const holes = 'export default () => ({ kind: "result", items: [,] });'; const empty = await analyzeSources({ @@ -331,6 +333,84 @@ describe("JavaScript export return-shape uncertain presence", () => { }), ); }); +}); + +describe("JavaScript export unpaired return locations", () => { + it.each([ + { + description: "two right return variants", + left: "export default function f() {}", + right: `export default function f(flag) { + if (flag) return { count: call() }; + return { total: call() }; + }`, + side: "right" as const, + lines: [2, 3], + }, + { + description: "one right return variant", + left: "export default function f() {}", + right: `export default function f() { + return { count: call() }; + }`, + side: "right" as const, + lines: [2], + }, + { + description: "two left return variants", + left: `export default function f(flag) { + if (flag) return { count: call() }; + return { total: call() }; + }`, + right: "export default function f() {}", + side: "left" as const, + lines: [2, 3], + }, + { + description: "one left return variant", + left: `export default function f() { + return { count: call() }; + }`, + right: "export default function f() {}", + side: "left" as const, + lines: [2], + }, + ])( + "retains each location for $description when the opposite export has no return", + async ({ left, right, side, lines }) => { + const graphs = await analyzeSources({ left, right }); + const result = compare(...graphs); + expect(result.coverage).toMatchObject({ + status: "partial", + unpaired_left_variants: side === "left" ? lines.length : 0, + unpaired_right_variants: side === "right" ? lines.length : 0, + }); + expect(result.changes).toHaveLength(lines.length); + expect( + result.changes.every( + ({ status, left_source_range, right_source_range }) => + status === "unknown" && + (side === "left" + ? left_source_range !== null && right_source_range === null + : left_source_range === null && right_source_range !== null), + ), + ).toBe(true); + const locations = result.changes + .map((change) => + side === "left" + ? change.left_source_range?.start.line + : change.right_source_range?.start.line, + ) + .sort((first, second) => (first ?? 0) - (second ?? 0)); + expect(locations).toEqual(lines); + expect(result.summary).toMatchObject({ + added: 0, + removed: 0, + changed: 0, + unknown: lines.length, + }); + }, + ); it("retains unpaired inventory locations when the opposite export has no return", async () => { const graphs = await analyzeSources({ diff --git a/tests/boundary/filesystem/javascriptHtmlEntrypoints.test.ts b/tests/boundary/filesystem/javascriptHtmlEntrypoints.test.ts index 29a9847df..ee3f9ced0 100644 --- a/tests/boundary/filesystem/javascriptHtmlEntrypoints.test.ts +++ b/tests/boundary/filesystem/javascriptHtmlEntrypoints.test.ts @@ -191,3 +191,185 @@ const htmlScriptRanges = (html: string) => { } return ranges; }; + +it("reports unanalyzed inline HTML scripts instead of claiming complete coverage", async () => { + const root = await createTestTempDirectory("rea-html-inline-scripts-"); + const html = [ + "", + '', + '', + "", + "", + '', + "", + ].join("\n"); + await mkdir(join(root, "www")); + await Promise.all([ + writeFile( + join(root, "package.json"), + JSON.stringify({ name: "game", version: "1.0.0" }), + ), + writeFile(join(root, "www", "index.html"), html), + writeFile(join(root, "www", "cordova.js"), "globalThis.cordova = {};"), + ]); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + const gaps = result.graph.nodes.filter( + ({ kind, observations }) => + kind === "unknown" && + observations.some( + ({ properties }) => + properties.operation === "parse-html-inline-scripts", + ), + ); + + expect(result.graph.coverage.status).toBe("partial"); + expect(gaps).toHaveLength(1); + expect(gaps[0]?.observations[0]).toMatchObject({ + properties: { + path: "www/index.html", + inline_script_count: 2, + inline_scripts: [ + { + script_type: "classic", + location: { + start: { line: 5, column: 8 }, + end: { line: 7, column: 0 }, + }, + }, + { + script_type: "module", + location: { + start: { line: 8, column: 22 }, + end: { line: 8, column: 67 }, + }, + }, + ], + }, + evidence: { + state: "unavailable", + artifact: { + sha256: createHash("sha256").update(html).digest("hex"), + }, + coverage: { status: "partial" }, + }, + }); + expect(result.graph.limitations).toContain( + "www/index.html has 2 inline JavaScript bodies starting at lines 5, 8 that were not analyzed; endpoints, storage, imports and other facts in them are absent from the evidence.", + ); + // The external script is still projected as before. + expect( + result.graph.edges.some( + ({ relation, properties }) => + relation === "loads" && properties.resolved_path === "www/cordova.js", + ), + ).toBe(true); +}); + +it("keeps complete coverage when HTML has only external scripts and data blocks", async () => { + const root = await createTestTempDirectory("rea-html-no-inline-scripts-"); + await Promise.all([ + writeFile( + join(root, "index.html"), + '', + ), + writeFile(join(root, "actual.js"), "globalThis.ownedActual = true;"), + ]); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + + expect(result.graph.coverage.status).toBe("complete"); + expect( + result.graph.limitations.some((limitation) => + limitation.includes("inline JavaScript"), + ), + ).toBe(false); +}); + +it.each(coordinateCases)( + "retains parse5 source coordinates for inline script bodies with $name", + async ({ prefix }) => { + const root = await createTestTempDirectory("rea-html-inline-coordinates-"); + const html = `${prefix}`; + await writeFile(join(root, "index.html"), html); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + const gap = result.graph.nodes + .flatMap(({ observations }) => observations) + .find( + ({ properties }) => + properties.operation === "parse-html-inline-scripts", + ); + + expect(gap?.properties.inline_scripts).toEqual([ + { script_type: "classic", location: inlineBodyRange(html) }, + ]); + }, +); + +const inlineBodyRange = (html: string) => { + const pending: DefaultTreeAdapterTypes.Node[] = [ + parse(html, { sourceCodeLocationInfo: true }), + ]; + while (pending.length > 0) { + const node = pending.pop(); + if (node === undefined) continue; + if (defaultTreeAdapter.isElementNode(node) && node.tagName === "script") { + const opening = node.sourceCodeLocation?.startTag; + const body = node.childNodes[0]?.sourceCodeLocation; + if (opening === undefined || body === undefined || body === null) + throw new Error("Missing parse5 inline script range"); + return { + start: { line: opening.endLine, column: opening.endCol - 1 }, + end: { line: body.endLine, column: body.endCol - 1 }, + }; + } + if ("childNodes" in node) { + for (const child of node.childNodes.toReversed()) pending.push(child); + } + } + throw new Error("Missing parse5 inline script"); +}; + +it("bounds retained inline script locations and names the omitted remainder", async () => { + const root = await createTestTempDirectory("rea-html-many-inline-scripts-"); + const count = 5_000; + await writeFile( + join(root, "index.html"), + "\n".repeat(count), + ); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + const gap = result.graph.nodes + .flatMap(({ observations }) => observations) + .find( + ({ properties }) => properties.operation === "parse-html-inline-scripts", + ); + const limit = { + name: "html_inline_script_locations_per_file", + value: 100, + unit: "items", + }; + + expect(gap?.properties.inline_script_count).toBe(count); + expect(gap?.properties.inline_scripts).toHaveLength(100); + // The gap record knows exactly which locations the bound dropped. + expect(gap?.evidence.coverage).toEqual({ + status: "partial", + truncated: true, + omitted_count: count - 100, + limits: [limit], + }); + // The graph is truncated too, but unanalyzed bodies hide an unknown total. + expect(result.graph.coverage).toMatchObject({ + status: "partial", + truncated: true, + omitted_count: null, + limits: expect.arrayContaining([limit]), + }); + expect(result.graph.limitations).toContain( + `index.html has ${String(count)} inline JavaScript bodies starting at lines 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 and ${String(count - 10)} more that were not analyzed; endpoints, storage, imports and other facts in them are absent from the evidence.`, + ); +}); diff --git a/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts b/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts index d5495a7f2..ab47a4181 100644 --- a/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts +++ b/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { mkdir, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; import { dirname, join, relative, sep } from "node:path"; @@ -196,3 +197,38 @@ describe("literal CommonJS percent bytes in application graphs", () => { expect(edge.properties.resolved_path).toBeNull(); }); }); + +it("reports a present non-JavaScript require target as unavailable with inventory identity", async () => { + const present = await createTestTempDirectory("rea-require-opaque-present-"); + const absent = await createTestTempDirectory("rea-require-opaque-absent-"); + const bytes = Buffer.from([0xde, 0xc0, 0x17, 0x0b, 0x00, 0x01, 0x02]); + const main = 'require("./core.jsc");'; + await Promise.all([ + writeFile(join(present, "main.cjs"), main), + writeFile(join(present, "core.jsc"), bytes), + writeFile(join(absent, "main.cjs"), main), + ]); + const edge = async (root: string) => { + const { graph } = await analyze(root); + return graph.edges.find( + ({ properties }) => + properties.specifier === "./core.jsc" && + properties.module_link_kind === "require", + ); + }; + const presentEdge = await edge(present); + expect(presentEdge?.properties).toMatchObject({ + resolution_status: "unavailable", + candidate_path: "core.jsc", + resolved_path: null, + }); + expect(presentEdge?.evidence.limitations).toContainEqual( + expect.stringContaining( + `core.jsc exists in the inventoried artifact container (sha256 ${createHash("sha256").update(bytes).digest("hex")}, ${String(bytes.length)} bytes)`, + ), + ); + expect((await edge(absent))?.properties).toMatchObject({ + resolution_status: "not-found", + resolved_path: null, + }); +}); diff --git a/tests/boundary/filesystem/sourceToBundlePaths.test.ts b/tests/boundary/filesystem/sourceToBundlePaths.test.ts index 8bc9caa27..d546605de 100644 --- a/tests/boundary/filesystem/sourceToBundlePaths.test.ts +++ b/tests/boundary/filesystem/sourceToBundlePaths.test.ts @@ -6,7 +6,10 @@ import { analyzeJavaScriptApplication } from "../../support/javascriptApplicatio import { parseApplicationGraphEvidence } from "../../../src/application/javascript/JavaScriptApplicationEvidenceGraph.js"; import { findApplicationFeatureSeeds } from "../../../src/domain/javascript/javascriptFeatureSeed.js"; import { compareSourceToBundle } from "../../../src/domain/javascript/sourceToBundleComparison.js"; -import { resolveJavaScriptSourceMapUrl } from "../../../src/domain/javascript/javascriptSourceMapPaths.js"; +import { + resolveJavaScriptSourceMapReferences, + resolveJavaScriptSourceMapUrl, +} from "../../../src/domain/javascript/javascriptSourceMapPaths.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { importReferenceSource } from "../../support/referenceSourceResourceScope.js"; @@ -704,3 +707,106 @@ const compareTrees = async (previous: string, current: string) => { }); return { reference: reference.value, comparison, application }; }; + +it.skipIf(process.platform === "win32")( + "does not promote a POSIX literal backslash name to a different historical slash path", + async () => { + const root = await createTestTempDirectory("rea-backslash-source-match-"); + const previous = join(root, "previous"); + const current = join(root, "current"); + await mkdir(join(previous, "literal"), { recursive: true }); + await mkdir(current); + await writeFile( + join(previous, "literal", "name.js"), + "export const value = 1;\n", + ); + await writeFile( + join(current, "literal\\name.js"), + "export const value = 2;\n", + ); + const { comparison } = await compareTrees(previous, current); + const item = comparison.items.find( + ({ source_path }) => source_path === "literal/name.js", + ); + expect(item?.current_node_ids).toEqual([]); + expect( + item?.candidates + .flatMap(({ signals }) => signals) + .some(({ kind }) => kind === "current-path-exact"), + ).toBe(false); + }, +); + +it.skipIf(process.platform === "win32").each([ + { mapPath: "literal\\bundle.js.map", expected: "a.js" }, + { mapPath: "literal\\dir/bundle.js.map", expected: "literal\\dir/a.js" }, +])( + "resolves source-map declarations from literal artifact base $mapPath", + async ({ mapPath, expected }) => { + const root = await createTestTempDirectory("rea-backslash-source-map-"); + const previous = join(root, "previous"); + const current = join(root, "current"); + await mkdir(previous); + await mkdir(current); + await writeFile(join(previous, "a.js"), "export const value = 1;\n"); + await mkdir(join(current, "literal\\dir"), { recursive: true }); + await writeFile(join(current, mapPath.slice(0, -4)), "var value = 2;\n"); + await writeFile( + join(current, mapPath), + JSON.stringify({ + version: 3, + file: mapPath.slice(0, -4), + sources: ["a.js"], + sourcesContent: ["export const value = 2;\n"], + names: [], + mappings: "AAAA", + }), + ); + const { application, comparison } = await compareTrees(previous, current); + const references = application.graph.nodes + .flatMap(({ observations }) => + observations.map(({ source_map_reference }) => source_map_reference), + ) + .filter((reference) => reference?.map_path === mapPath); + expect(references).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + resolution: { kind: "artifact-relative", path: expected }, + }), + ]), + ); + const item = comparison.items.find( + ({ source_path }) => source_path === "a.js", + ); + const signals = item?.candidates.flatMap(({ signals }) => signals) ?? []; + if (expected === "a.js") + expect(signals).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "source-map-original-path", + current_values: ["a.js"], + }), + ]), + ); + else + expect( + signals.some(({ kind }) => kind === "source-map-original-path"), + ).toBe(false); + }, +); + +it.each([ + { source: "https://user%2Fname@host.test/a.js", expected: "host.test/a.js" }, + { + source: "C:\\code\\literal%5Cname.js", + expected: "C:/code/literal%5Cname.js", + }, +])( + "checks encoded source-map separators in the parsed pathname of $source", + ({ source, expected }) => { + expect( + resolveJavaScriptSourceMapReferences([source], null, "ordinary.js.map")[0] + ?.resolution, + ).toEqual({ kind: "suffix", path: expected }); + }, +); diff --git a/tests/boundary/filesystem/webScriptManifestPaths.test.ts b/tests/boundary/filesystem/webScriptManifestPaths.test.ts index 9996fa8a9..762ecd397 100644 --- a/tests/boundary/filesystem/webScriptManifestPaths.test.ts +++ b/tests/boundary/filesystem/webScriptManifestPaths.test.ts @@ -75,3 +75,84 @@ it("still rejects changed source bytes after normalizing the selected manifest p "artifact_integrity_mismatch", ); }); + +const literalManifest = async () => { + const fixture = await materialize(); + const selected = fixture.artifact.manifest.scripts[0]; + if (selected?.content.state !== "exported") + throw new Error("Expected exported source"); + selected.content.relative_path = "..\\sibling.js"; + await writeFile(join(fixture.root, "sibling.js"), fixture.artifact.source); + if (process.platform !== "win32") + await writeFile( + join(fixture.root, "files", selected.content.relative_path), + fixture.artifact.source, + ); + await writeFile( + fixture.manifestPath, + JSON.stringify(fixture.artifact.manifest), + ); + return fixture; +}; + +it.each(["trace_web_module_imports", "trace_web_source_location"] as const)( + "refuses Windows backslash source interpretation for %s", + async (operation) => { + const fixture = await literalManifest(); + const originalPlatform = Object.getOwnPropertyDescriptor( + process, + "platform", + ); + const artifacts = new LocalWebScriptArtifacts(operation); + let response; + try { + // Exercise the Windows admission branch; this is not native Windows IO. + Object.defineProperty(process, "platform", { value: "win32" }); + response = await artifacts.load({ + manifest_path: fixture.manifestPath, + script_index: 0, + }); + } finally { + if (originalPlatform !== undefined) + Object.defineProperty(process, "platform", originalPlatform); + await artifacts.close(); + } + if (response.ok) throw new Error("Expected Windows source path refusal"); + expect(projectAnalysisError(response.error)).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["script_index"], + reason: "invalid_value", + message: expect.stringContaining( + "Windows source path cannot contain backslashes", + ), + }, + ], + }, + }); + }, +); + +it.skipIf(process.platform === "win32")( + "reads the same saved POSIX literal name without selecting its sibling", + async () => { + const fixture = await literalManifest(); + await writeFile(join(fixture.root, "sibling.js"), "wrong sibling source"); + const artifacts = new LocalWebScriptArtifacts("trace_web_module_imports"); + try { + const result = await artifacts.load({ + manifest_path: fixture.manifestPath, + script_index: 0, + }); + if (!result.ok) throw result.error; + expect(result.value.source).toBe(fixture.artifact.source); + expect(result.value.sourceFile.path).toBe( + join(fixture.root, "files", "..\\sibling.js"), + ); + } finally { + await artifacts.close(); + } + }, +); diff --git a/tests/support/cli/javascriptApplication.ts b/tests/support/cli/javascriptApplication.ts index 085792100..fe799e349 100644 --- a/tests/support/cli/javascriptApplication.ts +++ b/tests/support/cli/javascriptApplication.ts @@ -1,3 +1,6 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + import { expect } from "vitest"; import { z } from "zod"; @@ -5,6 +8,19 @@ import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/j import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import type { TestCli } from "./cliFixture.js"; +const execute = promisify(execFile); + +export const runNodeJsonControl = async ( + root: string, + filename: string, +): Promise => { + const { stdout } = await execute(process.execPath, [filename], { + cwd: root, + timeout: 5_000, + }); + return JSON.parse(stdout) as unknown; +}; + /** Analyze a directory through a public adapter and validate its result contract. */ export const analyzeJavaScriptDirectory = async ( surface: "CLI" | "stdio MCP", From a769915f6e49c783596c361150d1d665f6d2375e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:51:32 +0800 Subject: [PATCH 19/26] refactor(javascript): reuse canonical AST traversal Remove the separate Vite walker and raw visitor-key require scan. Keep subtree selection at the shared traversal owner. All 103 focused builtin/Vite tests and the complete static and architecture checks pass. --- .../javascriptSemanticRequireOrigin.ts | 187 +++++++++--------- .../javascriptStaticAnalysisCalls.ts | 73 +++---- 2 files changed, 114 insertions(+), 146 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticRequireOrigin.ts b/src/domain/javascript/javascriptSemanticRequireOrigin.ts index a4a153597..b11d64af2 100644 --- a/src/domain/javascript/javascriptSemanticRequireOrigin.ts +++ b/src/domain/javascript/javascriptSemanticRequireOrigin.ts @@ -1,4 +1,5 @@ import * as t from "@babel/types"; +import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { BUILTIN_RESOURCE_METHODS } from "./javascriptBuiltinResourceMethods.js"; import type { JavaScriptModuleOrigin } from "./javascriptSemanticIr.js"; @@ -61,109 +62,105 @@ const writesModuleRequire = ( binding.initializers.every(({ node: init }) => isModuleObject(init, next)) ); }; - const pending: t.Node[] = [state.program]; - while (pending.length > 0) { - const node = pending.pop(); - if (node === undefined) break; - const target = t.isAssignmentExpression(node) - ? node.left - : t.isUpdateExpression(node) || - t.isUnaryExpression(node, { operator: "delete" }) - ? node.argument - : undefined; - if (target !== undefined) { - const members: string[] = []; - let root: t.Node = target; - while (t.isMemberExpression(root) || t.isOptionalMemberExpression(root)) { - const key = semanticStaticPropertyKey(root.property, root.computed); - if (key === null) break; - members.unshift(key); - root = root.object; - } - if ( - members.length === 1 && - members[0] === "require" && - isModuleObject(root) - ) - writes.push(node); - else if ( - members.length === 0 && - (t.isMemberExpression(target) || - t.isOptionalMemberExpression(target)) && - target.computed && - semanticStaticPropertyKey(target.property, true) === null && - isModuleObject(target.object) - ) - // An unknown computed key may be "require"; retain uncertainty rather - // than treating this direct delegate write as a proven unrelated slot. - writes.push(node); - } - if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) { - const invocation = interop.intrinsicInvocation(node); - const owner = - invocation?.path?.length === 3 ? invocation.path[1] : undefined; - const method = invocation?.path?.at(-1); - const args = invocation?.arguments; - if ( - (owner === "Object" || owner === "Reflect") && - method === "defineProperty" && - isModuleObject(args?.[0] ?? t.nullLiteral()) - ) { - const key = interop.staticString(args?.[1]); - if (key === "require" || key === undefined) writes.push(node); - } else if ( - owner === "Object" && - method === "defineProperties" && - isModuleObject(args?.[0] ?? t.nullLiteral()) - ) { - const descriptors = args?.[1]; + traverseJavaScriptAst(state.program, { + enter: (node) => { + const target = t.isAssignmentExpression(node) + ? node.left + : t.isUpdateExpression(node) || + t.isUnaryExpression(node, { operator: "delete" }) + ? node.argument + : undefined; + if (target !== undefined) { + const members: string[] = []; + let root: t.Node = target; + while ( + t.isMemberExpression(root) || + t.isOptionalMemberExpression(root) + ) { + const key = semanticStaticPropertyKey(root.property, root.computed); + if (key === null) break; + members.unshift(key); + root = root.object; + } if ( - !t.isObjectExpression(descriptors) || - descriptors.properties.some( - (property) => - t.isSpreadElement(property) || - (t.isObjectProperty(property) && - semanticStaticPropertyKey(property.key, property.computed) === - "require"), - ) + members.length === 1 && + members[0] === "require" && + isModuleObject(root) ) writes.push(node); - } else if ( - owner === "Object" && - method === "assign" && - args?.some((arg) => isModuleObject(arg)) - ) { - const target = args?.[0]; - if (target !== undefined && isModuleObject(target)) { - const sourceHasRequire = args - ?.slice(1) - .some( - (arg) => - t.isObjectExpression(arg) && - arg.properties.some( - (property) => - t.isObjectProperty(property) && - semanticStaticPropertyKey( - property.key, - property.computed, - ) === "require", - ), - ); + else if ( + members.length === 0 && + (t.isMemberExpression(target) || + t.isOptionalMemberExpression(target)) && + target.computed && + semanticStaticPropertyKey(target.property, true) === null && + isModuleObject(target.object) + ) + // An unknown computed key may be "require"; retain uncertainty rather + // than treating this direct delegate write as a proven unrelated slot. + writes.push(node); + } + if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) { + const invocation = interop.intrinsicInvocation(node); + const owner = + invocation?.path?.length === 3 ? invocation.path[1] : undefined; + const method = invocation?.path?.at(-1); + const args = invocation?.arguments; + if ( + (owner === "Object" || owner === "Reflect") && + method === "defineProperty" && + isModuleObject(args?.[0] ?? t.nullLiteral()) + ) { + const key = interop.staticString(args?.[1]); + if (key === "require" || key === undefined) writes.push(node); + } else if ( + owner === "Object" && + method === "defineProperties" && + isModuleObject(args?.[0] ?? t.nullLiteral()) + ) { + const descriptors = args?.[1]; if ( - sourceHasRequire || - args?.slice(1).some((arg) => !t.isObjectExpression(arg)) + !t.isObjectExpression(descriptors) || + descriptors.properties.some( + (property) => + t.isSpreadElement(property) || + (t.isObjectProperty(property) && + semanticStaticPropertyKey(property.key, property.computed) === + "require"), + ) ) writes.push(node); + } else if ( + owner === "Object" && + method === "assign" && + args?.some((arg) => isModuleObject(arg)) + ) { + const target = args?.[0]; + if (target !== undefined && isModuleObject(target)) { + const sourceHasRequire = args + ?.slice(1) + .some( + (arg) => + t.isObjectExpression(arg) && + arg.properties.some( + (property) => + t.isObjectProperty(property) && + semanticStaticPropertyKey( + property.key, + property.computed, + ) === "require", + ), + ); + if ( + sourceHasRequire || + args?.slice(1).some((arg) => !t.isObjectExpression(arg)) + ) + writes.push(node); + } } } - } - for (const key of t.VISITOR_KEYS[node.type] ?? []) { - const value = (node as unknown as Record)[key]; - if (Array.isArray(value)) { - for (const child of value) if (t.isNode(child)) pending.push(child); - } else if (t.isNode(value)) pending.push(value); - } - } + }, + }); moduleRequireWrites.set(state, writes); return writes; }; diff --git a/src/domain/javascript/javascriptStaticAnalysisCalls.ts b/src/domain/javascript/javascriptStaticAnalysisCalls.ts index b031aea9e..b9bc3c96e 100644 --- a/src/domain/javascript/javascriptStaticAnalysisCalls.ts +++ b/src/domain/javascript/javascriptStaticAnalysisCalls.ts @@ -1,4 +1,5 @@ import * as t from "@babel/types"; +import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { htmlUrlText, stripQueryAndFragment } from "../artifactPathSyntax.js"; import { sanitizeEndpointCandidate } from "../browserObservation.js"; @@ -302,17 +303,19 @@ export const indexVitePreloadDependencyTable = ( program: t.Program, ): readonly (string | undefined)[] | null => { const helpers: ViteMapDepsFunction[] = []; - visitJavaScriptNodes(program, (node) => { - if (t.isFunctionDeclaration(node) && node.id?.name === "__vite__mapDeps") - helpers.push(node); - else if ( - t.isVariableDeclarator(node) && - t.isIdentifier(node.id, { name: "__vite__mapDeps" }) - ) { - const init = node.init; - if (t.isFunctionExpression(init) || t.isArrowFunctionExpression(init)) - helpers.push(init); - } + traverseJavaScriptAst(program, { + enter: (node) => { + if (t.isFunctionDeclaration(node) && node.id?.name === "__vite__mapDeps") + helpers.push(node); + else if ( + t.isVariableDeclarator(node) && + t.isIdentifier(node.id, { name: "__vite__mapDeps" }) + ) { + const init = node.init; + if (t.isFunctionExpression(init) || t.isArrowFunctionExpression(init)) + helpers.push(init); + } + }, }); if (helpers.length !== 1) return null; const helper = helpers[0]; @@ -323,9 +326,9 @@ export const indexVitePreloadDependencyTable = ( : mapParameter; if (!t.isIdentifier(mapIdentifier)) return []; const tables: t.ArrayExpression[] = []; - visitJavaScriptNodes( - helper, - (node) => { + traverseJavaScriptAst(helper, { + skip: (node) => t.isFunction(node), + enter: (node) => { if ( t.isAssignmentExpression(node) && node.operator === "=" && @@ -338,9 +341,7 @@ export const indexVitePreloadDependencyTable = ( ) tables.push(node.right); }, - undefined, - true, - ); + }); if (tables.length !== 1) return []; const table = tables[0]; if (table === undefined) return []; @@ -350,9 +351,9 @@ export const indexVitePreloadDependencyTable = ( : stringValue(element), ); let escapedOrMutated = false; - visitJavaScriptNodes( - program, - (node, parent) => { + traverseJavaScriptAst(program, { + skip: (node) => node === helper, + enter: (node, parent) => { if (!t.isIdentifier(node, { name: "__vite__mapDeps" })) return; if ( (parent === helper && @@ -370,40 +371,10 @@ export const indexVitePreloadDependencyTable = ( return; escapedOrMutated = true; }, - helper, - ); + }); return escapedOrMutated ? dependencies.map(() => undefined) : dependencies; }; -const visitJavaScriptNodes = ( - root: t.Node, - visit: (node: t.Node, parent: t.Node | undefined) => void, - skip?: t.Node, - skipNestedFunctions = false, -): void => { - const pending: { readonly node: t.Node; readonly parent?: t.Node }[] = [ - { node: root }, - ]; - while (pending.length > 0) { - const current = pending.pop(); - if (current === undefined) continue; - const { node, parent } = current; - if (node === skip) continue; - visit(node, parent); - if (skipNestedFunctions && node !== root && t.isFunction(node)) continue; - const keys = t.VISITOR_KEYS[node.type] ?? []; - for (const key of keys) { - const child = (node as unknown as Record)[key]; - if (Array.isArray(child)) { - for (let index = child.length - 1; index >= 0; index -= 1) { - const item = child[index]; - if (t.isNode(item)) pending.push({ node: item, parent: node }); - } - } else if (t.isNode(child)) pending.push({ node: child, parent: node }); - } - } -}; - const inspectEndpointCall = ( node: JavaScriptCallLike, name: string, From 537467ab9ecad4db8f984c792cd165d01a10054d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:58:34 +0800 Subject: [PATCH 20/26] fix(test-fixtures): keep managed metadata and method bodies disjoint Lay out resources, ReadyToRun data and IL bodies from their actual extents. Preserve RVA zero for bodyless methods and verify adjacent bodies through the production inspector. --- scripts/fixtures/managed/pe.d.mts | 3 + scripts/fixtures/managed/pe.mjs | 91 +++++++++++++++++++---- src/dotnet/ManagedPeFixtureLayout.test.ts | 74 ++++++++++++++++++ 3 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 src/dotnet/ManagedPeFixtureLayout.test.ts diff --git a/scripts/fixtures/managed/pe.d.mts b/scripts/fixtures/managed/pe.d.mts index 57567b917..633707596 100644 --- a/scripts/fixtures/managed/pe.d.mts +++ b/scripts/fixtures/managed/pe.d.mts @@ -3,6 +3,8 @@ export interface ManagedPeFixtureMethod { readonly flags?: number; readonly implFlags?: number; readonly name: string; + /** An explicit metadata RVA; zero represents a method without an IL body. */ + readonly rva?: number; } export interface ManagedPeFixtureOptions { @@ -19,6 +21,7 @@ export interface ManagedPeFixtureOptions { readonly pinvoke?: { readonly importName?: string; readonly mappingFlags?: number; + readonly memberRow?: number; readonly moduleName?: string; }; readonly readyToRun?: boolean; diff --git a/scripts/fixtures/managed/pe.mjs b/scripts/fixtures/managed/pe.mjs index e505fcdd0..d97130c1c 100644 --- a/scripts/fixtures/managed/pe.mjs +++ b/scripts/fixtures/managed/pe.mjs @@ -280,7 +280,7 @@ const buildManagedMetadata = (options, methods) => { methodDefRow({ name: stringsForMethods[index], signature: methodSignature, - rva: method.rva ?? 0x2800 + index * 0x20, + rva: method.rva, flags: method.flags ?? 0x0016, implFlags: method.implFlags ?? 0, }), @@ -325,13 +325,64 @@ const buildManagedMetadata = (options, methods) => { return metadata; }; +const align = (value, boundary) => Math.ceil(value / boundary) * boundary; + +const managedFixtureLayout = (options, methods, metadataLength) => { + const resourceLength = + 4 + (options.resourceData ?? Buffer.from("resource-data")).length; + const metadataOffset = 0x0300; + const resourceOffset = Math.max( + 0x0800, + align(metadataOffset + metadataLength, 0x200), + ); + const readyToRunOffset = Math.max( + 0x0900, + align(resourceOffset + resourceLength, 0x200), + ); + const methodOffsets = []; + let nextMethodOffset = Math.max( + 0x0a00, + align(readyToRunOffset + (options.readyToRun === true ? 4 : 0), 0x200), + ); + for (const method of methods) { + // RVA zero denotes a method without a mapped IL body, such as P/Invoke. + if (method.rva === 0) { + methodOffsets.push(null); + continue; + } + const offset = + method.rva === undefined + ? nextMethodOffset + : 0x0200 + (method.rva - 0x2000); + methodOffsets.push(offset); + nextMethodOffset = + Math.max(offset, nextMethodOffset) + + align(Math.max(0x20, (method.body ?? retOnlyIlBody).length), 0x20); + } + return { metadataOffset, resourceOffset, readyToRunOffset, methodOffsets }; +}; + const buildManagedImage = (options, methods, metadata) => { const resourceData = options.resourceData ?? Buffer.from("resource-data"); const resourceDirectory = Buffer.concat([ u32(resourceData.length), resourceData, ]); - const image = Buffer.alloc(0x1000); + const { metadataOffset, resourceOffset, readyToRunOffset, methodOffsets } = + managedFixtureLayout(options, methods, metadata.length); + const contentEnd = Math.max( + metadataOffset + metadata.length, + resourceOffset + resourceDirectory.length, + readyToRunOffset + (options.readyToRun === true ? 4 : 0), + ...methodOffsets.flatMap((offset, index) => + offset === null + ? [] + : [offset + (methods[index].body ?? retOnlyIlBody).length], + ), + 0x1000, + ); + const imageSize = align(contentEnd, 0x200); + const image = Buffer.alloc(imageSize); image.write("MZ", 0, "ascii"); image.writeUInt32LE(0x80, 0x3c); image.writeUInt32LE(0x0000_4550, 0x80); @@ -351,16 +402,16 @@ const buildManagedImage = (options, methods, metadata) => { else image.writeUInt32LE(0x0040_0000, optional + 28); image.writeUInt32LE(0x1000, optional + 32); image.writeUInt32LE(0x200, optional + 36); - image.writeUInt32LE(0x3000, optional + 56); + image.writeUInt32LE(align(0x2000 + imageSize - 0x200, 0x1000), optional + 56); image.writeUInt32LE(0x200, optional + 60); image.writeUInt32LE(16, optional + directoryOffset - 4); image.writeUInt32LE(0x2000, optional + directoryOffset + 14 * 8); image.writeUInt32LE(72, optional + directoryOffset + 14 * 8 + 4); const section = optional + optionalSize; image.write(".text\0\0\0", section, "ascii"); - image.writeUInt32LE(0x1000, section + 8); + image.writeUInt32LE(imageSize - 0x200, section + 8); image.writeUInt32LE(0x2000, section + 12); - image.writeUInt32LE(0x0e00, section + 16); + image.writeUInt32LE(imageSize - 0x200, section + 16); image.writeUInt32LE(0x0200, section + 20); image.writeUInt32LE(0x6000_0020, section + 36); const cli = 0x0200; @@ -371,27 +422,41 @@ const buildManagedImage = (options, methods, metadata) => { image.writeUInt32LE(metadata.length, cli + 12); image.writeUInt32LE(options.cliFlags ?? 1, cli + 16); image.writeUInt32LE(0x0600_0001, cli + 20); - image.writeUInt32LE(0x2600, cli + 24); + image.writeUInt32LE(0x2000 + (resourceOffset - 0x0200), cli + 24); image.writeUInt32LE(resourceDirectory.length, cli + 28); if (options.readyToRun === true) { - image.writeUInt32LE(0x2700, cli + 64); + image.writeUInt32LE(0x2000 + (readyToRunOffset - 0x0200), cli + 64); image.writeUInt32LE(4, cli + 68); - image.write("RTR\0", 0x0900, "ascii"); + image.write("RTR\0", readyToRunOffset, "ascii"); } methods.forEach((method, index) => { - const offset = 0x0a00 + index * 0x20; + const offset = methodOffsets[index]; + if (offset === null) return; (method.body ?? retOnlyIlBody).copy(image, offset); }); - metadata.copy(image, 0x0300); - resourceDirectory.copy(image, 0x0800); + metadata.copy(image, metadataOffset); + resourceDirectory.copy(image, resourceOffset); return image; }; /** Build a source-owned PE/CLI fixture without committing compiled binaries. */ export const buildManagedPeFixture = (options = {}) => { const methods = managedFixtureMethods(options); - const metadata = buildManagedMetadata(options, methods); - return buildManagedImage(options, methods, metadata); + const provisionalMetadata = buildManagedMetadata( + options, + methods.map((method) => ({ ...method, rva: method.rva ?? 0x2800 })), + ); + const layout = managedFixtureLayout( + options, + methods, + provisionalMetadata.length, + ); + const methodsWithRvas = methods.map((method, index) => ({ + ...method, + rva: method.rva ?? 0x2000 + (layout.methodOffsets[index] - 0x0200), + })); + const metadata = buildManagedMetadata(options, methodsWithRvas); + return buildManagedImage(options, methodsWithRvas, metadata); }; export const buildNativePeFixture = () => { const image = buildManagedPeFixture(); diff --git a/src/dotnet/ManagedPeFixtureLayout.test.ts b/src/dotnet/ManagedPeFixtureLayout.test.ts new file mode 100644 index 000000000..b3c9f1f6e --- /dev/null +++ b/src/dotnet/ManagedPeFixtureLayout.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; + +import { buildManagedPeFixture } from "../../scripts/fixtures/managed/pe.mjs"; +import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js"; +import { managedPeFixtureTarget } from "./ManagedPe.fixture.js"; + +const rawOffset = (rva: number) => 0x200 + (rva - 0x2000); + +describe("managed PE fixture layout", () => { + it("preserves bodyless native methods without allocating an IL extent", () => { + const image = buildManagedPeFixture({ + methods: [ + { name: "NativeMethod", flags: 0x2016, rva: 0 }, + { name: "ManagedMethod", body: Buffer.from([0x06, 0x2a]) }, + ], + }); + const inspected = inspectManagedMembersBytes( + image, + managedPeFixtureTarget(image), + ); + expect(inspected.methods[0]?.rva).toBe(0); + expect(inspected.methods[0]?.body.status).toBe("absent"); + expect(inspected.methods[0]?.body.opcode_counts).toEqual({}); + expect(inspected.methods[1]?.body.opcode_counts).toEqual({ ret: 1 }); + }); + + it("preserves complete adjacent method bodies", () => { + const firstBody = Buffer.concat([ + Buffer.from([0x9e]), + Buffer.alloc(38), + Buffer.from([0x2a]), + ]); + const secondBody = Buffer.from([0x06, 0x2a]); + const image = buildManagedPeFixture({ + methods: [ + { name: "LongBody", body: firstBody }, + { name: "NextBody", body: secondBody }, + ], + }); + + expect( + image.subarray(rawOffset(0x2800), rawOffset(0x2800) + firstBody.length), + ).toEqual(firstBody); + expect( + image.subarray(rawOffset(0x2840), rawOffset(0x2840) + secondBody.length), + ).toEqual(secondBody); + const inspected = inspectManagedMembersBytes( + image, + managedPeFixtureTarget(image), + ); + expect(inspected.methods.map(({ body }) => body.opcode_counts)).toEqual([ + { nop: 38, ret: 1 }, + { ret: 1 }, + ]); + }); + + it("keeps the ReadyToRun signature outside expanded metadata", () => { + const image = buildManagedPeFixture({ + readyToRun: true, + methods: Array.from({ length: 47 }, (_, index) => ({ + name: `Method${index}`, + })), + }); + const nativeHeaderRva = image.readUInt32LE(0x240); + const nativeHeaderSize = image.readUInt32LE(0x244); + + expect(nativeHeaderSize).toBe(4); + expect( + image + .subarray(rawOffset(nativeHeaderRva), rawOffset(nativeHeaderRva) + 4) + .toString("latin1"), + ).toBe("RTR\0"); + }); +}); From a688c797848dc6b1e7500666f32eebced8d2ab2a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:58:46 +0800 Subject: [PATCH 21/26] docs(ida): describe catalog-based lifecycle compatibility --- docs/ida-provider.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/ida-provider.md b/docs/ida-provider.md index fca4d0bb5..4e4460516 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -6,7 +6,9 @@ Choose one lifecycle mode in an upstream MCP JSON registration. Set `REA_IDA_MCP ## Supported upstream versions -Each mode accepts one upstream tool profile, checked against `tools/list` before any analysis runs. Upstream's `main` branch has since moved to 2.0.0, which renamed the GUI tools and dropped `idb_close`, so it serves neither profile. Installing from that branch fails at startup with the missing tool names in the error diagnostics. +Each mode accepts one upstream tool profile, checked against `tools/list` before any analysis runs. The attached profile requires the legacy GUI tool names; upstream 2.0.0 renamed those tools, and REA has not established an equivalent attached-mode lifecycle and ownership contract for that catalog. It therefore fails closed with the missing tool names in the error diagnostics. + +The headless profile is defined by the database-supervisor API, not by an upstream version string. The upstream source snapshot [`c133c385`](https://github.com/mrexodia/ida-pro-mcp/tree/c133c3853faa111a9b00ee615c013b720d0c4acd) includes `idb_close` (introduced by [upstream PR #495](https://github.com/mrexodia/ida-pro-mcp/pull/495)) alongside `idb_open` and `idb_list`. Earlier builds may report the same `2.0.0` version while lacking that tool, so REA checks the live catalog rather than treating a version number or branch name as a compatibility guarantee. | Mode | Upstream profile | Known working | | -------- | ------------------- | ------------------------------------------------------ | From 1b14870ace1bd9b93ece3dc73654e958b054bf33 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:58:53 +0800 Subject: [PATCH 22/26] refactor(evidence): remove superseded ordering wrapper and document exclusions --- docs/cli.md | 7 ++++++- src/domain/browserScenarioDiffEvidence.ts | 15 --------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index c801bf006..de180122b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -263,7 +263,12 @@ tree matches that commit remains unknown. It honors the source tree's `.gitignor and applies default exclusions for common generated and dependency paths such as `node_modules/`, `dist/`, and `*.log`. Pattern-based exclusions record the matched pattern and whether it came from project, default, caller, or sensitive -path policy. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore +path policy. Links into excluded paths retain their internal target metadata; +the exclusion explains why the target has no inventory entry, and the inventory +remains partial. Exclusions record `entry_kind` when the importer observed it; +older saved exclusions without that field have unknown kind. An excluded +ancestor proves an internal descendant only when its kind is `directory`. +Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore patterns when you want to mark selected paths as sensitive; those patterns are reported separately as configured-secret exclusions. diff --git a/src/domain/browserScenarioDiffEvidence.ts b/src/domain/browserScenarioDiffEvidence.ts index a6b5a0b5d..380f90303 100644 --- a/src/domain/browserScenarioDiffEvidence.ts +++ b/src/domain/browserScenarioDiffEvidence.ts @@ -5,7 +5,6 @@ import type { BrowserStepArtifacts, } from "./browserScenarioCaptureValues.js"; import type { BrowserScenarioCapture } from "./browserScenarioCapture.js"; -import { z } from "zod"; import { fingerprintScenarioNetworkTransactions, scenarioNetworkTransactions, @@ -145,20 +144,6 @@ export const prepareScenarioEvents = ( }; }; -const scenarioStepEventsSchema = z.strictObject({ - events: z.array(z.unknown()), - network_transactions: z.array(z.string()), -}); - -/** Network ancestry is already fingerprinted per capture; keep the collection unordered. */ -export const orderNetworkTransactions = (value: unknown): unknown => { - const events = scenarioStepEventsSchema.parse(value); - return { - events: events.events, - network_transactions: events.network_transactions, - }; -}; - /** Different content selection is missing comparability, not an observed website change. */ export const networkContentCoverageReason = ( before: BrowserScenarioCapture, From 4d4631ac94431bb8aae2fe3edcf1e8227048848d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:59:27 +0800 Subject: [PATCH 23/26] test: harden verification fixtures and runtime coverage Use exact discovered test identities, production MCP sessions and byte-exact setup backups. Add CLI signal cleanup coverage and independent Ghidra, Go and firmware verification assertions; prune redundant test configuration and preserve platform limits. --- .github/workflows/ci.yml | 3 + docs/testing.md | 7 +- knip.json | 2 +- package.json | 2 +- scripts/ci/plan.mjs | 3 + scripts/ci/scopes.mjs | 4 + scripts/create-ghidra-windows-fixture.mjs | 5 +- scripts/lib/ghidra-windows-byte-search.mjs | 149 +++++++++++++++ .../lib/real-ghidra-address-boundaries.mjs | 88 ++++++--- scripts/lib/real-hopper-search.d.mts | 7 +- scripts/run-development-tests.mjs | 6 +- scripts/run-exact-tests.mjs | 134 +++++++++++++ scripts/verify-agent-experience.mjs | 12 ++ scripts/verify-go-binary.mjs | 73 +++++-- scripts/verify-module-boundaries.mjs | 66 +++++-- scripts/verify-opencode-client.mjs | 20 +- scripts/verify-package-setup.d.mts | 6 + scripts/verify-package-setup.mjs | 22 ++- scripts/verify-packaged-ghidra-windows.mjs | 54 +++++- scripts/verify-real-ghidra-swift.mjs | 179 +++++++++++++++++- scripts/verify-real-ghidra-windows.mjs | 19 +- scripts/verify/clients/model-fixture.d.mts | 21 ++ scripts/verify/clients/model-fixture.mjs | 5 +- scripts/verify/firmware/analysis.mjs | 35 +++- src/evaluation/AgentEvaluationFixtures.ts | 156 +++++++++++++-- .../applicationWorkflowCli.test.ts | 9 +- tests/acceptance/webRuntimeCli.test.ts | 150 ++++++++++++--- .../boundary/captures/captureProvider.test.ts | 17 ++ tests/boundary/cli/developmentTests.test.ts | 131 +++++++++++++ .../cli/keyedArchiveXmlExtensionCli.test.ts | 38 ++++ .../filesystem/android/application.test.ts | 36 ++++ .../boundary/filesystem/ciChangeScope.test.ts | 4 + .../swift/ReaSwiftDecoderRegressions.java | 152 +++++++++++++++ tests/conformance/swift/decoder-safeguards.S | 58 ++++++ .../swift/unresolved-string-jump.s | 8 + .../model-evals/knownAnswerEvaluation.test.ts | 2 + .../model-evals/openAiModelFixture.test.ts | 52 +++++ tests/evaluation/moduleBoundaries.test.ts | 41 ++++ .../process-global/updateMaintenance.test.ts | 29 ++- tests/support/mcp/productionSession.ts | 33 ++++ 40 files changed, 1697 insertions(+), 141 deletions(-) create mode 100644 scripts/lib/ghidra-windows-byte-search.mjs create mode 100644 scripts/run-exact-tests.mjs create mode 100644 scripts/verify-package-setup.d.mts create mode 100644 scripts/verify/clients/model-fixture.d.mts create mode 100644 tests/boundary/cli/keyedArchiveXmlExtensionCli.test.ts create mode 100644 tests/conformance/swift/ReaSwiftDecoderRegressions.java create mode 100644 tests/conformance/swift/decoder-safeguards.S create mode 100644 tests/conformance/swift/unresolved-string-jump.s create mode 100644 tests/evaluation/model-evals/openAiModelFixture.test.ts create mode 100644 tests/support/mcp/productionSession.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b35b6a6e1..f55667dce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -634,6 +634,7 @@ jobs: tests/boundary/cli/nativeSignatureCli.test.ts tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts src/process/capture/FilesystemSnapshot.test.ts + tests/process-global/filesystemSnapshotFallbackCancellation.test.ts tests/boundary/process/darwinProcessRunTokenReader.test.ts tests/boundary/process/directAnalysisSignals.test.ts tests/boundary/process/processCaptureLifecycle.test.ts @@ -714,6 +715,8 @@ jobs: - name: Run curated Windows capability tests run: >- npx vitest run --coverage.enabled=false + src/process/capture/FilesystemSnapshot.test.ts + tests/process-global/filesystemSnapshotFallbackCancellation.test.ts tests/boundary/filesystem/binaryTarget.test.ts tests/boundary/filesystem/safeOutputTree.test.ts tests/boundary/providers/native/artifactProvider.archiveSafety.test.ts diff --git a/docs/testing.md b/docs/testing.md index aa8d2163f..042248a26 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -19,7 +19,7 @@ behavior, and complete CI evidence before merging. | `npm run test:boundary` | Boundary, CLI boundary, MCP boundary, process boundary, and process-global projects | | `npm run test:mcp` | MCP boundary project | | `npm run test:acceptance` | Complete compiled CLI and MCP acceptance workflows | -| `npm run test:python` | LLDB producer retention and launch contracts with a fake LLDB API; no native debugger required | +| `npm run test:python` | LLDB and mitmproxy producer contracts plus analysis-efficiency checks; no real debugger or proxy required | | `npm run test:watch` | Dirty source tests in watch mode, without build | | `npm run test:watch:all` | Changed tests from every project; builds at startup, so rebuild after production edits before relying on compiled tests | | `npm run check:changed` | Cached typecheck/lint and branch-related source feedback | @@ -449,8 +449,9 @@ adds in-place reparse, breakaway, environment, and token observations. The separate `scripts/verify-windows-native-inspection.mjs` lane uses the `.cache/windows-native-inspection-fixture` artifact built on Linux with `node scripts/build-windows-native.mjs --inspection-fixture`. Its controlled -first deletion failure checks retained NTFS ownership, later-query recovery, -successful-admission caching, and environment-teardown cleanup. It neither +first deletion failure checks retained NTFS ownership across a same-volume +rename, later-query handle-based recovery, successful-admission caching, and +environment-teardown cleanup. It neither establishes the cause of the intermittent #1701 host failure nor replaces verification with the shipped production artifact. `npm run verify:ghidra:windows:package` additionally packs and installs REA into diff --git a/knip.json b/knip.json index 5ed82edf0..97a323b6c 100644 --- a/knip.json +++ b/knip.json @@ -9,8 +9,8 @@ "src/application/RealHopperAssertions.ts", "scripts/verify/fridaBindingsCompatibility.ts", "scripts/verify/termux/browser.mjs", - "scripts/verify-real-cutter.mjs", "scripts/verify-real-ghidra-noreturn.mjs", + "scripts/run-exact-tests.mjs", "tests/processOwnershipRunner.ts" ], "project": ["src/**/*.ts", "tests/**/*.ts", "scripts/**/*.mjs"], diff --git a/package.json b/package.json index 5ccf53c99..45e73afc0 100644 --- a/package.json +++ b/package.json @@ -138,7 +138,7 @@ "test": "npm run test:with-build", "test:with-build": "npm run test:prepare && vitest run", "test:run": "vitest run", - "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", + "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", "test:local": "node scripts/run-development-tests.mjs local", "test:fast": "node scripts/ci/test-group.mjs source", "test:built": "node scripts/ci/test-group.mjs built", diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 6e3f950b8..cb942bfd9 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -202,6 +202,9 @@ if (tier === "reused") { if (GUARDED_WORKFLOWS.has(file)) selected.code = true; if (file !== ".github/workflows/ci.yml") continue; } + // The Notes example is also the source-owned agent evaluation fixture. + if (file.startsWith("website/public/examples/notes-electron/")) + selected.code = true; if (owners.length === 1 && owners[0] === "website") continue; if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true; if (file === "package.json") { diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index b13057b33..01cb8e329 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -79,6 +79,7 @@ export const SCOPES = { "tests/boundary/cli/nativeSignatureCli.test.ts", "tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts", "src/process/capture/FilesystemSnapshot.test.ts", + "tests/process-global/filesystemSnapshotFallbackCancellation.test.ts", "tests/boundary/process/darwinProcessRunTokenReader.test.ts", "tests/boundary/process/directAnalysisSignals.test.ts", "tests/boundary/process/processCaptureLifecycle.test.ts", @@ -89,6 +90,9 @@ export const SCOPES = { package: [ "src/windows/**", "native/windows/**", + "src/process/capture/FilesystemSnapshot.ts", + "src/process/capture/FilesystemSnapshot.test.ts", + "tests/process-global/filesystemSnapshotFallbackCancellation.test.ts", "src/inspector/**", "scripts/verify/inspector/**", "scripts/verify-package.mjs", diff --git a/scripts/create-ghidra-windows-fixture.mjs b/scripts/create-ghidra-windows-fixture.mjs index 214d7dadc..aa22c8165 100644 --- a/scripts/create-ghidra-windows-fixture.mjs +++ b/scripts/create-ghidra-windows-fixture.mjs @@ -1,6 +1,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { writeWindowsGhidraByteSearchFixture } from "./lib/ghidra-windows-byte-search.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); // Source-owned static fixtures: no compiler, target loading or execution. @@ -61,7 +62,7 @@ for (const architecture of ["x86_64", "x86"]) { } bytes.write(".text", sectionTable, "ascii"); - bytes.writeUInt32LE(dll ? 0x180 : 0x50, sectionTable + 8); + bytes.writeUInt32LE(0x200, sectionTable + 8); bytes.writeUInt32LE(0x1000, sectionTable + 12); bytes.writeUInt32LE(0x200, sectionTable + 16); bytes.writeUInt32LE(0x200, sectionTable + 20); @@ -136,6 +137,8 @@ for (const architecture of ["x86_64", "x86"]) { bytes.write("fixture_return42\0", 0x340, "binary"); } + writeWindowsGhidraByteSearchFixture(bytes); + await mkdir(dirname(output), { recursive: true }); await writeFile(output, bytes); if (!x86 && !dll) process.stdout.write(`${output}\n`); diff --git a/scripts/lib/ghidra-windows-byte-search.mjs b/scripts/lib/ghidra-windows-byte-search.mjs new file mode 100644 index 000000000..b432b9fc7 --- /dev/null +++ b/scripts/lib/ghidra-windows-byte-search.mjs @@ -0,0 +1,149 @@ +import assert from "node:assert/strict"; + +export const WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE = Object.freeze({ + pattern: "a0 ?? c0", + mask: "f0 ff f0", + startFileOffset: 0x380, + endFileOffset: 0x392, + markerOffsets: Object.freeze([0x380, 0x390]), +}); + +/** Put two masked, wildcarded signatures in loaded, source-owned PE bytes. */ +export function writeWindowsGhidraByteSearchFixture(bytes) { + bytes.set([0xa1, 0xb2, 0xc3], 0x380); + bytes.set([0xa4, 0xb5, 0xc6], 0x390); +} + +/** Verify real Ghidra search pages against the immutable source fixture. */ +export async function verifyWindowsGhidraByteSearch({ + invoke, + sourceBytes, + architecture, + sourceOwnedFixture = true, + fallbackAddress, + fallbackFileOffset, + runCliPage, +}) { + const imageBase = architecture === "x86" ? 0x400000n : 0x140000000n; + const query = sourceOwnedFixture + ? { + pattern: WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.pattern, + mask: WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.mask, + start: addressAtFileOffset( + imageBase, + WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.startFileOffset, + ), + end: addressAtFileOffset( + imageBase, + WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.endFileOffset, + ), + limit: 1, + } + : { + pattern: sourceBytes + .subarray(fallbackFileOffset, fallbackFileOffset + 1) + .toString("hex"), + start: fallbackAddress, + end: fallbackAddress, + limit: 1, + }; + + if (!sourceOwnedFixture) { + const result = await invoke("search_bytes", query); + assert.equal(result.complete, true); + assert.equal(result.next_cursor, null); + assert.deepEqual( + result.matches.map(({ address, file_offset }) => ({ + address, + file_offset, + })), + [{ address: fallbackAddress, file_offset: fallbackFileOffset }], + ); + assert.ok(result.searched.bytes > 0); + return { bounded_source_match: true }; + } + + const { startFileOffset, endFileOffset } = WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE; + const expected = []; + for ( + let fileOffset = startFileOffset; + fileOffset + 2 <= endFileOffset; + fileOffset += 1 + ) { + const first = sourceBytes[fileOffset]; + const last = sourceBytes[fileOffset + 2]; + if (first === undefined || last === undefined) break; + if ((first & 0xf0) === 0xa0 && (last & 0xf0) === 0xc0) + expected.push({ + address: addressAtFileOffset(imageBase, fileOffset), + file_offset: fileOffset, + }); + } + assert.deepEqual( + expected.map(({ file_offset }) => file_offset), + [...WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.markerOffsets], + "The source-owned byte oracle must retain both distinct signatures", + ); + + const first = await invoke("search_bytes", query); + assert.deepEqual( + first.matches.map(({ address, file_offset }) => ({ address, file_offset })), + [expected[0]], + ); + assert.equal(first.complete, false); + assert.equal(typeof first.next_cursor, "string"); + assert.ok(first.searched.bytes > 0); + if (runCliPage !== undefined) { + const cli = await runCliPage(query); + assert.deepEqual(cli.matches, first.matches); + assert.deepEqual(cli.searched, first.searched); + assert.equal(cli.complete, first.complete); + } + + const second = await invoke("search_bytes", { + ...query, + cursor: first.next_cursor, + }); + assert.deepEqual( + second.matches.map(({ address, file_offset }) => ({ + address, + file_offset, + })), + [expected[1]], + ); + assert.equal(second.complete, true); + assert.equal(second.next_cursor, null); + assert.ok(second.searched.bytes > 0); + + const covered = new Set(); + for (const page of [first, second]) { + assert.equal( + page.searched.bytes, + page.searched.blocks.reduce((sum, block) => sum + block.bytes, 0), + ); + for (const block of page.searched.blocks) { + const start = BigInt(block.start); + const end = BigInt(block.end); + assert.ok(start >= BigInt(query.start) && end <= BigInt(query.end)); + for (let offset = start; offset <= end; offset += 1n) + covered.add(offset.toString()); + } + } + assert.equal( + covered.size, + endFileOffset - startFileOffset + 1, + "Paged byte-search coverage must account for the complete bounded source range", + ); + return { + bounded: true, + wildcardAndMask: true, + matches: expected, + continuation: true, + examinedBytes: covered.size, + cliMcpParity: runCliPage !== undefined, + }; +} + +function addressAtFileOffset(imageBase, fileOffset) { + return `0x${(imageBase + 0x1000n + BigInt(fileOffset - 0x200)).toString(16)}`; +} diff --git a/scripts/lib/real-ghidra-address-boundaries.mjs b/scripts/lib/real-ghidra-address-boundaries.mjs index 0835ecec3..a2211a1ba 100644 --- a/scripts/lib/real-ghidra-address-boundaries.mjs +++ b/scripts/lib/real-ghidra-address-boundaries.mjs @@ -62,36 +62,68 @@ export async function verifyGhidraAddressBoundaries(client, entry, cli) { await query("read_bytes", { address: padded, length: 1 }), await query("read_bytes", { address: entry, length: 1 }), ); - const aliasKeys = [entry, padded]; - assert.deepEqual( - await query("set_addresses_names", { - names: { - [entry]: "rea_same_address_alias", - [padded]: "rea_same_address_alias", - }, - }), - Object.fromEntries(aliasKeys.map((address) => [address, true])), - "Equivalent address spellings with one requested name share one edit", - ); - const nameBeforeAliasedBatch = await query("address_name", { - address: entry, - }); - assert.equal(nameBeforeAliasedBatch, "rea_same_address_alias"); - assert.deepEqual( - await query("set_addresses_names", { - names: { - [entry]: "rea_first_address_alias", - [padded]: "rea_second_address_alias", - }, - }), - Object.fromEntries(aliasKeys.map((address) => [address, false])), - "Conflicting spellings of one Ghidra address must be rejected together", - ); + const originalName = await query("address_name", { address: entry }); assert.equal( - await query("address_name", { address: entry }), - nameBeforeAliasedBatch, - "A rejected alias collision must leave the preexisting target name unchanged", + typeof originalName, + "string", + "The function entry needs an observed name that can be restored", ); + const aliasKeys = [entry, padded]; + let primaryFailure; + try { + assert.deepEqual( + await query("set_addresses_names", { + names: { + [entry]: "rea_same_address_alias", + [padded]: "rea_same_address_alias", + }, + }), + Object.fromEntries(aliasKeys.map((address) => [address, true])), + "Equivalent address spellings with one requested name share one edit", + ); + const nameBeforeAliasedBatch = await query("address_name", { + address: entry, + }); + assert.equal(nameBeforeAliasedBatch, "rea_same_address_alias"); + assert.deepEqual( + await query("set_addresses_names", { + names: { + [entry]: "rea_first_address_alias", + [padded]: "rea_second_address_alias", + }, + }), + Object.fromEntries(aliasKeys.map((address) => [address, false])), + "Conflicting spellings of one Ghidra address must be rejected together", + ); + assert.equal( + await query("address_name", { address: entry }), + nameBeforeAliasedBatch, + "A rejected alias collision must leave the preexisting target name unchanged", + ); + } catch (cause) { + primaryFailure = cause; + throw cause; + } finally { + try { + assert.equal( + await query("set_address_name", { address: entry, name: originalName }), + true, + "The temporary alias edit must be restored", + ); + assert.equal( + await query("address_name", { address: entry }), + originalName, + "The original function-entry name must survive restoration", + ); + } catch (cleanupCause) { + if (primaryFailure !== undefined) + throw new AggregateError( + [primaryFailure, cleanupCause], + "Ghidra address-boundary verification and alias restoration failed", + ); + throw cleanupCause; + } + } for (const [command, flags, operation] of [ ["read-bytes", ["--length", "1"], "read_bytes"], [ diff --git a/scripts/lib/real-hopper-search.d.mts b/scripts/lib/real-hopper-search.d.mts index 0a7260227..d3bb4c0d3 100644 --- a/scripts/lib/real-hopper-search.d.mts +++ b/scripts/lib/real-hopper-search.d.mts @@ -1,3 +1,7 @@ +import type { z } from "zod"; + +import type { analysisStringSchema } from "../../src/domain/hopperValues.js"; + type VerifierCall = ( name: string, arguments_?: Record, @@ -44,6 +48,3 @@ export function verifyHopperStringObjects( }[]; readonly displayEscapesExcludedFromMatching: true; }>; -import type { z } from "zod"; - -import type { analysisStringSchema } from "../../src/domain/hopperValues.js"; diff --git a/scripts/run-development-tests.mjs b/scripts/run-development-tests.mjs index bd3c65b28..d53b59c84 100644 --- a/scripts/run-development-tests.mjs +++ b/scripts/run-development-tests.mjs @@ -68,7 +68,11 @@ try { } if (!process.exitCode && !shutdownSignal) { const tested = await run(process.execPath, [ - resolve("node_modules/vitest/vitest.mjs"), + resolve( + request.paths.length > 0 + ? "scripts/run-exact-tests.mjs" + : "node_modules/vitest/vitest.mjs", + ), ...plan.vitestArguments, ]); process.exitCode = tested.code; diff --git a/scripts/run-exact-tests.mjs b/scripts/run-exact-tests.mjs new file mode 100644 index 000000000..98e723b3b --- /dev/null +++ b/scripts/run-exact-tests.mjs @@ -0,0 +1,134 @@ +import { lstat, realpath, stat } from "node:fs/promises"; +import { parse, relative, resolve, sep } from "node:path"; + +import { createVitest, parseCLI } from "vitest/node"; + +// CLI file filters are substring matches. Select discovered specifications by +// identity instead, retaining each project's include/exclude and runner settings. +const { filter, options } = parseCLI(["vitest", ...process.argv.slice(2)]); +const vitest = await createVitest({ ...options, watch: false }); +const signalExitCodes = { SIGINT: 130, SIGTERM: 143 }; +let shutdownSignal; +let closePromise; +const close = () => (closePromise ??= vitest.close()); +for (const signal of Object.keys(signalExitCodes)) + process.on(signal, () => { + if (shutdownSignal !== undefined) return; + shutdownSignal = signal; + void close().catch((cause) => { + console.error(cause instanceof Error ? cause.message : String(cause)); + process.exitCode = 1; + }); + }); +try { + if (shutdownSignal === undefined) await vitest.standalone(); + if (shutdownSignal === undefined) { + const discovered = new Map(); + for (const specification of await vitest.globTestSpecifications()) { + const path = resolve(specification.moduleId); + const specifications = discovered.get(path) ?? []; + specifications.push(specification); + discovered.set(path, specifications); + } + const selected = new Set(); + const missing = []; + let discoveredByIdentity; + for (const path of filter) { + const absolute = resolve(path); + let matches = discovered.get(absolute) ?? []; + if (matches.length === 0) { + try { + await lstat(absolute); + } catch (cause) { + if (cause?.code === "ENOENT" || cause?.code === "ENOTDIR") { + missing.push(path); + continue; + } + throw cause; + } + const [canonical, selectedIdentity] = await Promise.all([ + realpath(absolute), + stat(absolute, { bigint: true }), + ]); + if (discoveredByIdentity === undefined) { + discoveredByIdentity = new Map(); + for (const [candidate, specifications] of discovered) { + try { + const entry = await stat(candidate, { bigint: true }); + const identity = `${entry.dev}:${entry.ino}`; + const entries = discoveredByIdentity.get(identity) ?? []; + entries.push([candidate, specifications]); + discoveredByIdentity.set(identity, entries); + } catch (cause) { + if (cause?.code === "ENOENT" || cause?.code === "ENOTDIR") + continue; + throw cause; + } + } + } + const identity = `${selectedIdentity.dev}:${selectedIdentity.ino}`; + for (const [candidate, specifications] of discoveredByIdentity.get( + identity, + ) ?? []) { + try { + await lstat(candidate); + } catch (cause) { + if (cause?.code === "ENOENT" || cause?.code === "ENOTDIR") continue; + throw cause; + } + if ( + !(await hasCaseMismatchedSymlinkComponent(absolute, candidate)) && + (await realpath(candidate)) === canonical + ) + matches = [...matches, ...specifications]; + } + } + if (matches.length === 0) missing.push(path); + for (const specification of matches) selected.add(specification); + } + if (missing.length > 0 || selected.size === 0) + throw new Error( + `Selected test files were not discovered: ${missing.join(", ")}`, + ); + if (shutdownSignal === undefined) + await vitest.runTestSpecifications([...selected]); + } +} finally { + await close(); +} +if (shutdownSignal !== undefined) + process.exitCode = signalExitCodes[shutdownSignal]; + +async function hasCaseMismatchedSymlinkComponent(selected, candidate) { + const selectedAbsolute = resolve(selected); + const candidateAbsolute = resolve(candidate); + const root = parse(selectedAbsolute).root; + if (root !== parse(candidateAbsolute).root) return true; + const selectedParts = relative(root, selectedAbsolute).split(sep); + const candidateParts = relative(root, candidateAbsolute).split(sep); + let selectedPath = root; + let candidatePath = root; + for (let index = 0; index < selectedParts.length; index += 1) { + const selectedPart = selectedParts[index]; + const candidatePart = candidateParts[index]; + if (selectedPart === undefined || candidatePart === undefined) return true; + selectedPath = selectedPath + ? resolve(selectedPath, selectedPart) + : selectedPart; + candidatePath = candidatePath + ? resolve(candidatePath, candidatePart) + : candidatePart; + if ( + selectedPart !== candidatePart && + selectedPart.toLowerCase() === candidatePart.toLowerCase() + ) { + const [selectedEntry, candidateEntry] = await Promise.all([ + lstat(selectedPath), + lstat(candidatePath), + ]); + if (selectedEntry.isSymbolicLink() || candidateEntry.isSymbolicLink()) + return true; + } + } + return false; +} diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 71cd6383c..349877075 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -155,6 +155,18 @@ try { requiredAnswerTermGroups: [["profileapi"], ["profile:read"]], prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`, }, + { + id: "notes-export", + expectedFirstTool: "analyze_javascript_application", + requiresEvidence: true, + requiredAnswerTermGroups: [], + requiredToolSubsequence: [ + "analyze_javascript_application", + "trace_application_feature", + "trace_javascript_semantics", + ], + prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`, + }, { id: "javascript-module-view", expectedFirstTool: "analyze_javascript_application", diff --git a/scripts/verify-go-binary.mjs b/scripts/verify-go-binary.mjs index 0724c5f4d..4af896656 100644 --- a/scripts/verify-go-binary.mjs +++ b/scripts/verify-go-binary.mjs @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { isUtf8 } from "node:buffer"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { lstat, mkdir, readFile, writeFile } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; @@ -24,6 +24,8 @@ if (process.env.REA_VERIFY_GO_COMMAND !== undefined && !isAbsolute(compiler)) let compilerVersion = null; const run = createVerifierRun(); const root = await PrivateRuntimeRoot.create({ prefix: "rea-go-verifier-" }); +// Keep Go's directory enumeration below the private root's Windows ownership lease. +const sourceRoot = join(root.path, "source"); const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url)); const environment = Object.fromEntries( Object.entries(process.env).filter(([, value]) => typeof value === "string"), @@ -115,14 +117,54 @@ try { ], { GOOS: os, GOARCH: arch }, ); - await verify(path, target, oracle, null, "stripped"); + const strippedInfo = await verify(path, target, oracle, null, "stripped"); if (os === "linux" && arch === "amd64") { const sectionless = join(root.path, "linux-amd64-sectionless"); const bytes = Buffer.from(await readFile(path)); bytes.fill(0, 40, 48); // ELF64 e_shoff. bytes.fill(0, 58, 64); // e_shentsize, e_shnum and e_shstrndx. await writeFile(sectionless, bytes); - await verify(sectionless, target, oracle, null, "sectionless"); + const sectionlessInfo = await verify( + sectionless, + target, + oracle, + null, + "sectionless", + path, + ); + assert.deepEqual(sectionlessInfo, strippedInfo); + + const pie = join(root.path, "linux-amd64-pie"); + await go( + [ + "build", + "-trimpath", + "-buildvcs=false", + "-ldflags=-s -w", + "-buildmode=pie", + "-o", + pie, + ".", + ], + { GOOS: os, GOARCH: arch }, + ); + const originalInfo = await verify(pie, target, oracle, null, "pie"); + const pieBytes = Buffer.from(await readFile(pie)); + pieBytes.fill(0, 40, 48); + pieBytes.fill(0, 58, 64); + const pieSectionless = `${pie}-sectionless`; + await writeFile(pieSectionless, pieBytes); + // Keep the producer image as the independent oracle. Removing section + // headers must preserve every metadata field and original source span. + const retainedInfo = await verify( + pieSectionless, + target, + oracle, + null, + "sectionless-pie", + pie, + ); + assert.deepEqual(retainedInfo, originalInfo); } } for (const target of targets.filter(([os]) => os === "darwin")) { @@ -143,7 +185,7 @@ try { const expected = JSON.parse( ( await execute(oracle, ["--build-byte-fixture", compiler, path], { - cwd: root.path, + cwd: sourceRoot, env: { ...goEnvironment, GOOS: "linux", GOARCH: "amd64" }, timeout: 120_000, maxBuffer: 1024 * 1024, @@ -169,7 +211,7 @@ try { } const verifier = await completeVerifierRun(run); try { - await assert.rejects(readFile(join(root.path, "go.mod")), { code: "ENOENT" }); + await assert.rejects(lstat(root.path), { code: "ENOENT" }); if (verifier.process_lineage.status === "verified") assert.deepEqual(verifier.process_lineage.descendants, []); } catch (cause) { @@ -214,7 +256,7 @@ console.log( async function go(args, overrides = {}) { return execute(compiler, args, { - cwd: root.path, + cwd: sourceRoot, env: { ...goEnvironment, ...overrides }, timeout: 120_000, maxBuffer: 1024 * 1024, @@ -222,29 +264,30 @@ async function go(args, overrides = {}) { } async function prepareFixture() { + await mkdir(sourceRoot); await mkdir(join(root.path, "tmp")); await mkdir(join(root.path, "telemetry")); await writeFile(join(root.path, "telemetry", "mode"), "off\n"); - await mkdir(join(root.path, "dependency")); - await mkdir(join(root.path, "oracle")); + await mkdir(join(sourceRoot, "dependency")); + await mkdir(join(sourceRoot, "oracle")); await writeFile( - join(root.path, "go.mod"), + join(sourceRoot, "go.mod"), "module example.test/rea-go-fixture\n\ngo 1.20\n\nrequire example.test/dependency v1.2.3\nreplace example.test/dependency => ./dependency\n", ); await writeFile( - join(root.path, "dependency", "go.mod"), + join(sourceRoot, "dependency", "go.mod"), "module example.test/dependency\n\ngo 1.20\n", ); await writeFile( - join(root.path, "dependency", "dependency.go"), + join(sourceRoot, "dependency", "dependency.go"), 'package dependency\nfunc Value() string { return "REA fixture" }\n', ); await writeFile( - join(root.path, "main.go"), + join(sourceRoot, "main.go"), 'package main\nimport ("fmt"; "example.test/dependency")\nfunc main() { fmt.Println(dependency.Value()) }\n', ); await writeFile( - join(root.path, "oracle", "main.go"), + join(sourceRoot, "oracle", "main.go"), String.raw`package main import ( @@ -332,13 +375,14 @@ async function verify( oracle, byteExpected = null, fileKind = "stripped", + oraclePath = path, ) { const bytes = await readFile(path); const expected = byteExpected ?? JSON.parse( ( - await execute(oracle, [path], { + await execute(oracle, [oraclePath], { timeout: 10_000, maxBuffer: 1024 * 1024, }) @@ -408,6 +452,7 @@ async function verify( ? {} : { go_version_bytes_base64: expected.GoVersionBytesBase64 }), }); + return reports[0].build_info; } function verifyByteStrings(info, expected) { diff --git a/scripts/verify-module-boundaries.mjs b/scripts/verify-module-boundaries.mjs index 1064d2ae4..2069c96db 100644 --- a/scripts/verify-module-boundaries.mjs +++ b/scripts/verify-module-boundaries.mjs @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; -import { readFile } from "node:fs/promises"; -import { resolve } from "node:path"; +import { readdir, readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; import { promisify } from "node:util"; import { inspectModuleBoundaries } from "./lib/module-boundaries.mjs"; @@ -11,24 +11,44 @@ const selected = process.argv.slice(2); for (const file of selected) if (!file.endsWith(".ts")) throw new TypeError(`Expected an exact TypeScript source file: ${file}`); -const files = - selected.length > 0 - ? selected - : ( - await execute( - "git", - [ - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-standard", - "--", - "src", - ], - { cwd: root, maxBuffer: 8 * 1024 * 1024 }, - ) - ).stdout.split("\0"); + +const enumerateSourceFiles = async () => { + if (selected.length > 0) return selected; + let isGitRoot = false; + try { + const { stdout } = await execute("git", ["rev-parse", "--show-cdup"], { + cwd: root, + maxBuffer: 8 * 1024 * 1024, + }); + isGitRoot = stdout.trim() === ""; + } catch { + // Archives and source snapshots may not be in a Git worktree. + } + if (isGitRoot) { + try { + const { stdout } = await execute( + "git", + [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + "src", + ], + { cwd: root, maxBuffer: 8 * 1024 * 1024 }, + ); + return stdout.split("\0"); + } catch { + // Fall back to disk when Git's index is unavailable or unreadable. + } + } + const entries = await readdir(resolve(root, "src"), { recursive: true }); + return entries.map((entry) => join("src", entry)); +}; + +const files = await enumerateSourceFiles(); let checked = 0; for (const file of new Set(files.filter((path) => path.endsWith(".ts")))) { @@ -36,7 +56,11 @@ for (const file of new Set(files.filter((path) => path.endsWith(".ts")))) { try { source = await readFile(resolve(root, file), "utf8"); } catch (cause) { - if (selected.length === 0 && cause.code === "ENOENT") continue; + if ( + selected.length === 0 && + (cause.code === "ENOENT" || cause.code === "EISDIR") + ) + continue; throw cause; } checked += 1; diff --git a/scripts/verify-opencode-client.mjs b/scripts/verify-opencode-client.mjs index a29610ab3..b1006f8ba 100644 --- a/scripts/verify-opencode-client.mjs +++ b/scripts/verify-opencode-client.mjs @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { isAbsolute, join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; import { @@ -42,6 +42,21 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); +const isManagedSkillTarget = (target) => { + const value = relative(skillDirectory, target); + return ( + value === "" || + (value !== ".." && !value.startsWith(`..${sep}`) && !isAbsolute(value)) + ); +}; +assert.equal( + isManagedSkillTarget(`${skillDirectory}-sibling/AGENTS.md`), + false, +); +assert.equal( + isManagedSkillTarget(join(skillDirectory, "..", "outside", "AGENTS.md")), + false, +); await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); const environment = { @@ -264,8 +279,7 @@ try { assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); assert( parsed.plannedActions.every( - (a) => - a.target === configPath || a.target.startsWith(skillDirectory), + (a) => a.target === configPath || isManagedSkillTarget(a.target), ), ); } diff --git a/scripts/verify-package-setup.d.mts b/scripts/verify-package-setup.d.mts new file mode 100644 index 000000000..2c7244291 --- /dev/null +++ b/scripts/verify-package-setup.d.mts @@ -0,0 +1,6 @@ +/** Validate a live registration and its byte-identical pre-setup backup. */ +export function assertClientConfig( + cli: string, + configPath: string, + original: Uint8Array, +): Promise; diff --git a/scripts/verify-package-setup.mjs b/scripts/verify-package-setup.mjs index 9a87a1695..44f28c142 100644 --- a/scripts/verify-package-setup.mjs +++ b/scripts/verify-package-setup.mjs @@ -414,7 +414,8 @@ const verifySetupApply = async ({ }; }; -const assertClientConfig = async (cli, configPath) => { +/** Verify the live registration and exact pre-setup backup bytes. */ +export const assertClientConfig = async (cli, configPath, original) => { const config = json(await readFile(configPath, "utf8")); if ( config.existing !== true || @@ -422,9 +423,7 @@ const assertClientConfig = async (cli, configPath) => { JSON.stringify(config.mcpServers?.rea?.args) !== JSON.stringify(["mcp"]) ) throw new Error("packaged client readback failed"); - if ( - !(await readFile(`${configPath}.rea.backup`, "utf8")).includes("existing") - ) + if (!(await readFile(`${configPath}.rea.backup`)).equals(original)) throw new Error("packaged client backup failed"); }; @@ -467,9 +466,7 @@ const assertCodexSymlink = async ({ !codex.includes(`command = "${cli}"`) || !codex.includes("startup_timeout_sec = 30") || !codex.startsWith(CODEX_ORIGINAL) || - (await readFile(`${codexConfig}.rea.backup`, "utf8")) !== CODEX_ORIGINAL || - (await readFile(`${cursorConfig}.rea.backup`, "utf8")) !== - '{"existing":true}\n' + (await readFile(`${codexConfig}.rea.backup`, "utf8")) !== CODEX_ORIGINAL ) throw new Error("packaged setup did not preserve config symlinks"); }; @@ -534,9 +531,11 @@ const verifyConfigReadback = async ({ siblingSkillPath, opencodeConfig, opencodeOriginal, + claudeOriginal, + cursorOriginal, }) => { - await assertClientConfig(cli, claudeConfig); - await assertClientConfig(cli, cursorConfig); + await assertClientConfig(cli, claudeConfig, claudeOriginal); + await assertClientConfig(cli, cursorConfig, cursorOriginal); await assertCodexSymlink({ cli, codexConfig, @@ -729,6 +728,9 @@ export async function verifyPackageSetup({ opencodeConfig, skillPath, ]; + const [claudeOriginal, cursorOriginal] = supportedSetupHost + ? await Promise.all([readFile(claudeConfig), readFile(cursorConfig)]) + : []; await verifyInteractiveSetup({ command: packageRunnerCli, environment, @@ -772,6 +774,8 @@ export async function verifyPackageSetup({ siblingSkillPath, opencodeConfig, opencodeOriginal: OPENCODE_ORIGINAL, + claudeOriginal, + cursorOriginal, }); await verifySetupFailureRecovery({ cli, diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index 2c5f15e62..4d85e6782 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -6,6 +6,7 @@ import { requireMcpOperationResult, } from "./lib/mcp-verifier-results.mjs"; import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs"; +import { verifyWindowsGhidraByteSearch } from "./lib/ghidra-windows-byte-search.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -358,11 +359,12 @@ try { const mapping = await call("address_to_file_offset", { address: procedure.address, }); + const sourceBytes = await readFile(target); assert.equal(memory.complete, true); assert.equal(memory.returned_bytes, 16); assert.equal( memory.bytes_hex, - (await readFile(target)) + sourceBytes .subarray(mapping.file_offset, mapping.file_offset + 16) .toString("hex"), ); @@ -374,6 +376,35 @@ try { await call("resolve_containing_procedure", { address: procedure.address }); await call("search_procedures", { pattern: procedure.value }); await call("search_strings", { pattern: "rea" }); + report.byteSearch = await verifyWindowsGhidraByteSearch({ + invoke: call, + sourceBytes, + architecture: parsedTarget.value.architecture, + sourceOwnedFixture: targetArgument === undefined, + fallbackAddress: procedure.address, + fallbackFileOffset: mapping.file_offset, + runCliPage: async (query) => { + const cliArguments = [entry, "search-bytes", target, query.pattern]; + for (const [option, value] of [ + ["--mask", query.mask], + ["--start", query.start], + ["--end", query.end], + ["--limit", String(query.limit)], + ]) { + if (value !== undefined) cliArguments.push(option, value); + } + cliArguments.push("--provider", "ghidra", "--json"); + const result = await exec(process.execPath, cliArguments, { + env: environment, + cwd: callerDirectory, + timeout: 360_000, + maxBuffer: 8 * 1024 * 1024, + }); + const evidence = JSON.parse(result.stdout); + assert.ok(evidence.error === undefined, "CLI byte search failed"); + return evidence.normalized_result; + }, + }); for (const name of [ "procedure_assembly", "procedure_callees", @@ -429,9 +460,24 @@ try { name !== "shutdown" && !GHIDRA_MUTATING_OPERATIONS.has(name), ); - assert.ok( - expected.every((name) => report.mcpOperations.includes(name)), - "Packaged verification omitted an admitted operation.", + const expectedOperations = new Set([ + ...expected, + "open_binary", + "binary_overview", + "binary_session", + "close_binary", + ]); + const observedOperations = new Set(report.mcpOperations); + const missingOperations = expected.filter( + (name) => !observedOperations.has(name), + ); + const unexpectedOperations = [...observedOperations].filter( + (name) => !expectedOperations.has(name), + ); + assert.deepEqual( + { missing: missingOperations, unexpected: unexpectedOperations }, + { missing: [], unexpected: [] }, + "Packaged verification operation coverage drifted.", ); const session = await call("binary_session", {}); assert.equal( diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 0aa58e2f0..5cdf04427 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -1,7 +1,14 @@ #!/usr/bin/env node import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { copyFile, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + readdir, + rm, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,6 +16,11 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js"; +import { cleanupOwnedProcessGroup } from "../dist/process/ProcessOwnership.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../dist/process/ProviderProcess.js"; import { toolContract } from "../dist/contracts/toolContracts.js"; import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -48,10 +60,21 @@ const source = fileURLToPath( const literalSource = fileURLToPath( new URL("../tests/conformance/swift/string-literals.swift", import.meta.url), ); +const unresolvedJumpSource = fileURLToPath( + new URL( + "../tests/conformance/swift/unresolved-string-jump.s", + import.meta.url, + ), +); const binary = join(root, "swift-inventory"); const literalBinary = join(root, "swift-literals"); +const literalAssembly = fileURLToPath( + new URL("../tests/conformance/swift/decoder-safeguards.S", import.meta.url), +); +const literalObject = join(root, "decoder-safeguards.o"); const reports = []; let literalReport; +let decoderCleaned = true; try { await execute("swiftc", ["-O", "-g", source, "-o", binary], { timeout: 60_000, @@ -59,6 +82,19 @@ try { await mkdir(join(root, "copied")); const copied = join(root, "copied", "swift-inventory"); await copyFile(binary, copied); + await execute( + "xcrun", + [ + "clang", + "-target", + "arm64-apple-macosx13.0", + "-c", + literalAssembly, + "-o", + literalObject, + ], + { timeout: 60_000 }, + ); // Decoding covers AArch64 register pairs, so build that slice on any Mac host. await execute( "swiftc", @@ -67,11 +103,14 @@ try { "-target", "arm64-apple-macosx13.0", literalSource, + literalObject, + unresolvedJumpSource, "-o", literalBinary, ], { timeout: 60_000 }, ); + await verifyDecoderSafeguards(); await client.connect(transport); for (const [variant, path] of [ ["debug", binary], @@ -154,7 +193,7 @@ try { } finally { await client.close(); await transport.close(); - await rm(root, { recursive: true, force: true }); + if (decoderCleaned) await rm(root, { recursive: true, force: true }); } console.log( JSON.stringify( @@ -222,8 +261,144 @@ async function verifyStringLiterals() { } decoded[name] = observed.length; } + const unresolvedJump = procedures.find(({ value }) => + value.includes("rea_unresolved_string_jump"), + ); + assert( + unresolvedJump, + "unresolved jump fixture is missing from the procedure inventory", + ); + const unresolvedDossier = await call("analyze_function", { + procedure: unresolvedJump.address, + }); + assert.deepEqual(unresolvedDossier.swift_string_literals, []); + assert.deepEqual( + unresolvedDossier.swift_string_literal_omissions.map( + ({ reason }) => reason, + ), + ["unresolved_jump"], + ); + decoded.unresolvedStringJump = + unresolvedDossier.swift_string_literal_omissions.length; return decoded; } finally { await call("close_binary", {}); } } + +// Exercise the Java producer inside an ephemeral Ghidra database. The explicit +// budget seam makes exhaustion deterministic without exposing a runtime option. +async function verifyDecoderSafeguards() { + const scripts = join(root, "decoder-scripts"); + await mkdir(scripts); + const bridgeSources = fileURLToPath( + new URL("../bridge/ghidra/", import.meta.url), + ); + for (const name of await readdir(bridgeSources)) { + if (name.endsWith(".java")) + await copyFile(join(bridgeSources, name), join(scripts, name)); + } + await copyFile( + fileURLToPath( + new URL( + "../tests/conformance/swift/ReaSwiftDecoderRegressions.java", + import.meta.url, + ), + ), + join(scripts, "ReaSwiftDecoderRegressions.java"), + ); + const report = join(root, "decoder-safeguards.txt"); + await runDecoderProbe(scripts, report); + assert.equal( + await readFile(report, "utf8"), + "REA_SWIFT_DECODER_SAFEGUARDS_OK\n", + ); +} + +async function runDecoderProbe(scripts, report) { + const controller = new AbortController(); + let interrupt; + const interrupted = new Promise((resolve) => { + interrupt = () => { + controller.abort(new Error("Swift decoder probe interrupted")); + resolve(false); + }; + }); + process.on("SIGINT", interrupt); + process.on("SIGTERM", interrupt); + let supervisor; + let launch; + try { + launch = await spawnOwnedProviderProcess({ + command: installation.analyzeHeadlessPath, + arguments: [ + root, + "decoder-regressions", + "-import", + literalBinary, + "-scriptPath", + scripts, + "-postScript", + "ReaSwiftDecoderRegressions.java", + report, + "-deleteProject", + "-max-cpu", + "2", + ], + runId: run.run_id, + expectedCommand: null, + env: { JAVA_HOME: installation.javaHome }, + signal: controller.signal, + }); + decoderCleaned = false; + supervisor = new ProviderProcessSupervisor( + { + ...launch, + ownsProcessLifetime: true, + cleanup: () => cleanupOwnedProcessGroup(launch.ownership), + }, + { maxDiagnosticBytes: 8 * 1024 * 1024 }, + ); + assert.ok( + await Promise.race([supervisor.waitForExit(120_000), interrupted]), + controller.signal.aborted + ? "Swift decoder probe interrupted" + : "Swift decoder probe timed out", + ); + assert.ok( + await supervisor.waitForOutputClose(1_000), + "Swift decoder probe output did not close", + ); + const snapshot = supervisor.snapshot(); + assert.equal( + snapshot.exitCode, + 0, + `${snapshot.stdout.text}\n${snapshot.stderr.text}`, + ); + assert.ok( + !snapshot.diagnosticTruncated, + "Swift decoder probe exceeded its diagnostic capacity", + ); + } finally { + try { + if (supervisor !== undefined) { + const stopped = await supervisor.stop(); + decoderCleaned = stopped.status === "verified-cleanup"; + assert.ok( + decoderCleaned, + `Owned Ghidra cleanup incomplete; retained ${root}: ${JSON.stringify(stopped)}`, + ); + } else if (launch !== undefined) { + const stopped = await cleanupOwnedProcessGroup(launch.ownership); + decoderCleaned = stopped.cleaned; + assert.ok( + decoderCleaned, + `Owned Ghidra cleanup incomplete; retained ${root}: ${JSON.stringify(stopped)}`, + ); + } + } finally { + process.off("SIGINT", interrupt); + process.off("SIGTERM", interrupt); + } + } +} diff --git a/scripts/verify-real-ghidra-windows.mjs b/scripts/verify-real-ghidra-windows.mjs index bc822211f..2e6e57d22 100644 --- a/scripts/verify-real-ghidra-windows.mjs +++ b/scripts/verify-real-ghidra-windows.mjs @@ -29,6 +29,7 @@ import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs"; import { verifyGhidraFileOffsets } from "./lib/real-ghidra-file-offsets.mjs"; +import { verifyWindowsGhidraByteSearch } from "./lib/ghidra-windows-byte-search.mjs"; const verifierRun = createVerifierRun(); const SESSION_CAPABILITIES = GHIDRA_SESSION_CAPABILITIES.filter( @@ -215,6 +216,11 @@ try { }); await inventory("search_procedures", { pattern: procedure.value }); await inventory("search_strings", { pattern: "rea" }); + const byteSearch = await verifyWindowsGhidraByteSearch({ + invoke: inventory, + sourceBytes, + architecture, + }); for (const operation of [ "procedure_assembly", @@ -242,12 +248,18 @@ try { ...GHIDRA_INVENTORY_OPERATIONS, ...GHIDRA_FUNCTION_OPERATIONS, ].filter((name) => !GHIDRA_MUTATING_OPERATIONS.has(name)); + const expectedOperations = new Set(expected); + const missingOperations = expected.filter((name) => !observed.has(name)); + const unexpectedOperations = [...observed].filter( + (name) => !expectedOperations.has(name), + ); if ( - observed.size !== expected.length || - expected.some((operation) => !observed.has(operation)) + missingOperations.length > 0 || + unexpectedOperations.length > 0 || + observed.size !== expectedOperations.size ) throw new Error( - `Windows Ghidra operation proof was incomplete: ${JSON.stringify([...observed])}`, + `Windows Ghidra operation proof was incomplete: ${JSON.stringify({ missing: missingOperations, unexpected: unexpectedOperations })}`, ); // The reference controls occupy the EXE fixture's additional function. @@ -271,6 +283,7 @@ try { }, transport: "authenticated-loopback-tcp", operations: [...observed].sort((left, right) => left.localeCompare(right)), + byte_search: byteSearch, cleanup: "complete", compact_references: compactReferences, native_controls: { diff --git a/scripts/verify/clients/model-fixture.d.mts b/scripts/verify/clients/model-fixture.d.mts new file mode 100644 index 000000000..b162fe315 --- /dev/null +++ b/scripts/verify/clients/model-fixture.d.mts @@ -0,0 +1,21 @@ +export interface OpenAiModelFixture { + readonly baseUrl: string; + readonly requests: readonly { readonly path: string }[]; + readonly probes: readonly { + readonly path: string | undefined; + readonly status: 404; + }[]; + readonly failure: string | undefined; + close(): Promise; +} + +export function createOpenAiModelFixture(options: { + readonly directory: string; + readonly prefix: string; + readonly onRequest: () => + | { + readonly role: "assistant"; + readonly content: string; + } + | Promise<{ readonly role: "assistant"; readonly content: string }>; +}): Promise; diff --git a/scripts/verify/clients/model-fixture.mjs b/scripts/verify/clients/model-fixture.mjs index cdc99a31c..b218a0985 100644 --- a/scripts/verify/clients/model-fixture.mjs +++ b/scripts/verify/clients/model-fixture.mjs @@ -15,7 +15,8 @@ export const createOpenAiModelFixture = async ({ let failure; const server = createServer(async (request, response) => { try { - if (request.method === "GET" && request.url?.endsWith("/models")) { + const path = new URL(request.url ?? "/", "http://localhost").pathname; + if (request.method === "GET" && path.endsWith("/models")) { response.setHeader("Content-Type", "application/json"); response.end( JSON.stringify({ @@ -47,7 +48,7 @@ export const createOpenAiModelFixture = async ({ join(directory, `${prefix}-incoming-${++incoming}.json`), JSON.stringify({ method: request.method, path: request.url, body }), ); - if (request.url !== "/v1/chat/completions") { + if (path !== "/v1/chat/completions") { // Hermes probes custom endpoints for Ollama/llama.cpp metadata. // Unsupported HTTP endpoints return 404, as on an OpenAI-only server. probes.push({ diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index 804e8fffa..b90dba14b 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -201,6 +201,16 @@ async function stageCli() { ), ); + assert.ok( + files.extraction_gaps.some( + (gap) => + gap.kind === "unknown-chunk-handler" && + gap.range !== null && + gap.range.offset === 0 && + gap.range.length === 64, + ), + "The unknown chunk must carry its explicit recovery row", + ); return { regions, files }; } @@ -247,6 +257,20 @@ async function stageMcp(cli) { ); assert.equal(depthParsed.coverage, "partial"); assert.ok(depthParsed.depth_limited_paths.length > 0); + assert.ok( + depthParsed.extraction_gaps.some( + (gap) => + gap.kind === "depth-limited" && + gap.dependencies.length === 0 && + depthParsed.extraction_continuations.some( + ({ id, instruction }) => + id === gap.continuation_id && + instruction.includes("max_depth") && + instruction.includes("fresh output_directory"), + ), + ), + "Depth-limited paths must carry the explicit depth continuation", + ); console.log( "PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits", @@ -292,8 +316,17 @@ async function stageExt4() { ); assert.equal(degradedParsed.coverage, "partial"); assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs")); + assert.ok( + degradedParsed.extraction_gaps.some( + (gap) => + gap.kind === "missing-extractor-dependency" && + gap.dependencies.includes("debugfs") && + gap.handler === "extfs", + ), + "The missing extractor must be named in its explicit recovery row", + ); console.log( - "PASS real missing-extractor diagnostics retained despite Unblob exit 1", + "PASS real missing-extractor diagnostics and recovery row retained despite Unblob exit 1", ); } diff --git a/src/evaluation/AgentEvaluationFixtures.ts b/src/evaluation/AgentEvaluationFixtures.ts index b0535a96e..83b22db41 100644 --- a/src/evaluation/AgentEvaluationFixtures.ts +++ b/src/evaluation/AgentEvaluationFixtures.ts @@ -8,6 +8,11 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js"; import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js"; import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { + applicationFeatureTraceResultSchema, + featureSeedMatchMode, +} from "../domain/javascript/javascriptFeatureTraceSchemas.js"; +import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js"; import { managedArtifactInspectionSchema, managedMemberInspectionSchema, @@ -28,6 +33,8 @@ export interface AgentEvaluationFixtureTargets { readonly javascriptSha256: string; readonly javascriptShapeLeft: string; readonly javascriptShapeRight: string; + readonly notesElectron: string; + readonly notesElectronSha256: string; readonly managed?: string; readonly managedSha256?: string; } @@ -50,6 +57,8 @@ export const createAgentEvaluationFixtures = async ( const javascript = join(root, "desktop-app.asar"); const javascriptShapeLeft = join(root, "parser-v1"); const javascriptShapeRight = join(root, "parser-v2"); + const notesDirectory = join(root, "notes-electron"); + const notesElectron = join(root, "notes-electron.asar"); await Promise.all([ mkdir(join(desktop, "renderer"), { recursive: true }), mkdir(javascriptShapeLeft, { recursive: true }), @@ -67,8 +76,14 @@ export const createAgentEvaluationFixtures = async ( join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"), join(javascriptShapeRight, "parser.mjs"), ), + cp( + join(repositoryRoot, "website/public/examples/notes-electron"), + notesDirectory, + { recursive: true }, + ), ]); await createPackage(desktop, javascript); + await createPackage(notesDirectory, notesElectron); return { javascript, javascriptSha256: createHash("sha256") @@ -76,6 +91,10 @@ export const createAgentEvaluationFixtures = async ( .digest("hex"), javascriptShapeLeft, javascriptShapeRight, + notesElectron, + notesElectronSha256: createHash("sha256") + .update(await readFile(notesElectron)) + .digest("hex"), }; }; @@ -189,21 +208,116 @@ const runtimeSemantics = (value: JsonValue): JsonValue | undefined => { : undefined; }; -const asarClaims = ( +const featureModules = (value: JsonValue): JsonValue | undefined => { + const result = applicationFeatureTraceResultSchema.safeParse(value); + if (!result.success || result.data.graph === null) return undefined; + if ( + result.data.seed.kind !== "channel" || + result.data.seed.value !== "notes:export" || + featureSeedMatchMode(result.data.seed) !== "exact" + ) + return undefined; + const paths = result.data.graph.nodes + .filter((node) => node.kind === "javascript-module") + .flatMap((node) => node.observations) + .filter(({ properties }) => properties.semantic_role === "source-module") + .map(({ properties }) => properties.path); + if (paths.length === 0 || !paths.every((path) => typeof path === "string")) + return undefined; + return { + paths: [...new Set(paths)].sort(compareUnicodeCodePoints), + coverage: result.data.coverage.status, + }; +}; + +const semanticBoundary = (value: JsonValue): JsonValue | undefined => { + const result = javaScriptSemanticTraceResultSchema.safeParse(value); + if (!result.success) return undefined; + return { + status: result.data.status, + coverage: result.data.coverage.status, + relevant_unknowns: result.data.summary.relevant_unknowns, + }; +}; + +const notesClaims = ( targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => { + const subject = { + path: targets.notesElectron, + sha256: targets.notesElectronSha256, + }; + const derived = { + authority: "analyst-inference", + confidence: "inferred", + linkedSubjects: [ + { operation: "analyze_javascript_application", ...subject }, + ], + } satisfies Pick< + FixtureClaimExpectation["source"], + "authority" | "confidence" | "linkedSubjects" + >; + // Fixture source establishes bridge/module facts; the semantic claim checks + // the bounded static-query contract, not execution or CSV formatter behavior. + return [ + ...bridgeClaims(subject, "notes", ["exportCsv"], "notes:export"), + { + id: "feature_modules", + expectedValue: { + paths: ["csv.js", "main.js", "preload.js", "renderer.js"], + coverage: "complete-within-source", + }, + source: { + ...derived, + operation: "trace_application_feature", + parameters: { + direction: "both", + }, + select: featureModules, + }, + }, + { + id: "semantic_boundary", + expectedValue: { + status: "ambiguous", + coverage: "partial", + relevant_unknowns: 1, + }, + source: { + ...derived, + operation: "trace_javascript_semantics", + parameters: { + query: { + seed: { kind: "property", name: "invoke" }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: false, + expected: null, + }, + }, + select: semanticBoundary, + }, + }, + ]; +}; + +const bridgeClaims = ( + subject: { readonly path: string; readonly sha256: string }, + key: string, + members: readonly string[], + channel: string, ): readonly FixtureClaimExpectation[] => { const source = { operation: "analyze_javascript_application", authority: "shipped-artifact", confidence: "derived", - subject: { path: targets.javascript, sha256: targets.javascriptSha256 }, + subject, } satisfies Omit; return [ { id: "renderer_api", expectedValue: { - key: "profileApi", - members: ["read"], + key, + members: [...members], world: "main", source: "preload.js", }, @@ -212,7 +326,7 @@ const asarClaims = ( { id: "renderer_ipc", expectedValue: { - channel: "profile:read", + channel, operation: "invoke", source: "preload.js", }, @@ -221,7 +335,7 @@ const asarClaims = ( { id: "main_ipc", expectedValue: { - channel: "profile:read", + channel, operation: "handle", source: "main.js", }, @@ -235,6 +349,16 @@ const asarClaims = ( ]; }; +const asarClaims = ( + targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => + bridgeClaims( + { path: targets.javascript, sha256: targets.javascriptSha256 }, + "profileApi", + ["read"], + "profile:read", + ); + const parserClaims = ( targets: AgentEvaluationFixtureTargets, ): readonly FixtureClaimExpectation[] => { @@ -399,25 +523,33 @@ export const agentFixtureClaims = ( if (scenarioId === "asar") return asarClaims(targets); if (scenarioId === "managed") return managedClaims(targets); if (scenarioId === "javascript-export-shape") return parserClaims(targets); + if (scenarioId === "notes-export") return notesClaims(targets); return undefined; }; +const bridgeAnswerDescriptions = [ + 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', + 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', + 'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.', + 'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.', +]; + const answerValueDescriptions: Readonly> = { managed: [ "public_types: string[], listing the exact public top-level type full names in Unicode code-point order from unfiltered member Evidence (omit type_names).", 'entry_point: {"declaring_type": string, "name": string}, resolving the CLI metadata entry-point token against the member inventory from the same artifact SHA-256 and module MVID. Cite member Evidence; both artifact and member Evidence must be delivered.', ], - asar: [ - 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', - 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', - 'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.', - 'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.', - ], + asar: bridgeAnswerDescriptions, "javascript-export-shape": [ 'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.', 'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.', 'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.', ], + "notes-export": [ + ...bridgeAnswerDescriptions, + 'feature_modules: {"paths": string[], "coverage": string}, listing the unique source-module observation paths in the feature trace in Unicode code-point order, and its coverage status.', + 'semantic_boundary: {"status": string, "coverage": string, "relevant_unknowns": number}, copying the semantic trace status, coverage status, and summary.relevant_unknowns. Static traces do not establish runtime execution.', + ], }; /** Describe the strict answer contract without revealing any golden fixture values. */ diff --git a/tests/acceptance/applications/applicationWorkflowCli.test.ts b/tests/acceptance/applications/applicationWorkflowCli.test.ts index 6548d210b..ed1ca8d16 100644 --- a/tests/acceptance/applications/applicationWorkflowCli.test.ts +++ b/tests/acceptance/applications/applicationWorkflowCli.test.ts @@ -873,8 +873,9 @@ describe("rest parameter semantic trace CLI", () => { kind === "expression" && label === `argument ${String(index)}`, ); if (argument === undefined) throw new Error("Missing argument node"); - const traced = await runCli([ - "trace-javascript-semantics", + const requestPath = join(root, `trace-request-${index}.json`); + await writeFile( + requestPath, JSON.stringify({ application, query: { @@ -884,6 +885,10 @@ describe("rest parameter semantic trace CLI", () => { expected: { role: "sink", classes: ["parameter"] }, }, }), + ); + const traced = await runCli([ + "trace-javascript-semantics", + requestPath, "--json", ]); expect(traced).toMatchObject({ diff --git a/tests/acceptance/webRuntimeCli.test.ts b/tests/acceptance/webRuntimeCli.test.ts index 084687dc2..2b37fecdd 100644 --- a/tests/acceptance/webRuntimeCli.test.ts +++ b/tests/acceptance/webRuntimeCli.test.ts @@ -111,17 +111,124 @@ const cancellationSignals = process.platform === "win32" ? (["SIGINT"] as const) : (["SIGINT", "SIGTERM"] as const); -const webRuntimeCancellationCases = ( - ["observe-web-execution", "inspect-web-event-listeners"] as const -).flatMap((command) => - cancellationSignals.map( - (signal) => [command, signal, signal === "SIGINT" ? 130 : 143] as const, - ), + +interface CliCancellationCommand { + readonly command: string; + readonly operation: string; + readonly buildArgs: (endpoint: string) => readonly string[]; +} + +const cancellationCommands: readonly CliCancellationCommand[] = [ + { + command: "observe-web-execution", + operation: "observe_web_execution", + buildArgs: (endpoint) => [ + "observe-web-execution", + endpoint, + "page-fixture", + "--observation-ms", + "30000", + "--json", + ], + }, + { + command: "inspect-web-event-listeners", + operation: "inspect_web_event_listeners", + buildArgs: (endpoint) => [ + "inspect-web-event-listeners", + endpoint, + "page-fixture", + "body", + "--json", + ], + }, + { + command: "list-browser-targets", + operation: "list_browser_targets", + buildArgs: (endpoint) => ["list-browser-targets", endpoint, "--json"], + }, + { + command: "inspect-web-page", + operation: "inspect_web_page", + buildArgs: (endpoint) => [ + "inspect-web-page", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "analyze-web-bundle", + operation: "analyze_web_bundle", + buildArgs: (endpoint) => [ + "analyze-web-bundle", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "observe-web-session", + operation: "observe_web_session", + buildArgs: (endpoint) => [ + "observe-web-session", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "discover-webmcp-tools", + operation: "discover_webmcp_tools", + buildArgs: (endpoint) => [ + "discover-webmcp-tools", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "capture-web-screenshot", + operation: "capture_web_screenshot", + buildArgs: (endpoint) => [ + "capture-web-screenshot", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "list-javascript-runtime-targets", + operation: "list_javascript_runtime_targets", + buildArgs: (endpoint) => [ + "list-javascript-runtime-targets", + endpoint, + "--json", + ], + }, + { + command: "observe-javascript-runtime", + operation: "observe_javascript_runtime", + buildArgs: (endpoint) => [ + "observe-javascript-runtime", + endpoint, + "target-fixture", + "--json", + ], + }, +]; + +const cancellationCases = cancellationCommands.flatMap((command) => + cancellationSignals.map((signal) => ({ + ...command, + signal, + expectedExitCode: signal === "SIGINT" ? 130 : 143, + })), ); -it.each(webRuntimeCancellationCases)( - "%s preserves %s status when cancellation interrupts CDP discovery", - async (command, signal, expectedExitCode) => { +it.each(cancellationCases)( + "$command preserves $signal status and awaited cleanup during CDP discovery", + async ({ command, operation, buildArgs, signal, expectedExitCode }) => { const sockets = new Set(); const server = createServer(() => undefined); server.on("connection", (socket) => { @@ -136,21 +243,14 @@ it.each(webRuntimeCancellationCases)( const endpoint = `http://127.0.0.1:${String(address.port)}`; const contacted = once(server, "request"); - const args = - command === "observe-web-execution" - ? [ - command, - endpoint, - "page-fixture", - "--observation-ms", - "30000", - "--json", - ] - : [command, endpoint, "page-fixture", "body", "--json"]; - const child = spawn(process.execPath, [entrypoint, ...args], { - env: environment(), - stdio: ["ignore", "pipe", "pipe"], - }); + const child = spawn( + process.execPath, + [entrypoint, ...buildArgs(endpoint)], + { + env: environment(), + stdio: ["ignore", "pipe", "pipe"], + }, + ); const closed = once(child, "close"); let stdout = ""; child.stdout.setEncoding("utf8"); @@ -180,7 +280,7 @@ it.each(webRuntimeCancellationCases)( expect(JSON.parse(stdout)).toMatchObject({ code: "cancelled", category: "cancelled", - details: { cleanup: "complete" }, + details: { cleanup: "complete", operation }, }); } finally { clearTimeout(timeout); diff --git a/tests/boundary/captures/captureProvider.test.ts b/tests/boundary/captures/captureProvider.test.ts index 881b43e63..30281bed3 100644 --- a/tests/boundary/captures/captureProvider.test.ts +++ b/tests/boundary/captures/captureProvider.test.ts @@ -53,6 +53,8 @@ else { if(readFileSync('capture', 'utf8') !== 'recording bytes') throw Error('wr }, async (input) => { calls++; + if (mode === "launch-failure" && !input.arguments.includes("--version")) + throw Object.assign(new Error("spawn E2BIG"), { code: "E2BIG" }); if (mode === "file-limit" && !input.arguments.includes("--version")) { const stdout = { text: "", bytes: 0, observedBytes: 0 }; const stderr = { @@ -255,6 +257,21 @@ it.each([ }, ); +it.each(["capture", "fields"] as const)( + "does not attribute the version probe's output to a %s command that fails to launch", + async (operation) => { + const f = await fixture("launch-failure"); + const result = + operation === "capture" + ? await f.service.inspect({ path: f.path, fields: ["ip.addr"] }) + : await f.service.inspectFields({ kind: "fields" }); + if (result.ok) throw new Error("Expected launch failure"); + expect(f.calls()).toBe(2); + expect(result.error.message).toContain("spawn E2BIG"); + expect(result.error.capturedOutput).toBeUndefined(); + }, +); + it("retains a completed observation and cleanup ownership for retry", async () => { const f = await fixture("ok", true); const result = await f.service.inspect({ path: f.path, fields: ["ip.addr"] }); diff --git a/tests/boundary/cli/developmentTests.test.ts b/tests/boundary/cli/developmentTests.test.ts index 7becdd4e7..d9b9fa7f1 100644 --- a/tests/boundary/cli/developmentTests.test.ts +++ b/tests/boundary/cli/developmentTests.test.ts @@ -1,5 +1,6 @@ import { copyFile, + lstat, mkdir, readFile, symlink, @@ -44,6 +45,7 @@ const fixture = async () => { await mkdir(join(root, "src")); for (const path of [ "scripts/run-development-tests.mjs", + "scripts/run-exact-tests.mjs", "scripts/lib/development-tests.mjs", "scripts/lib/test-projects.mjs", ]) @@ -76,6 +78,135 @@ const testCommand = (root: string, ...arguments_: string[]) => ...arguments_, ]); +describe("development test runner exact selection", () => { + it("matches selected path casing to the host filesystem", async () => { + const root = await fixture(); + try { + await lstat(join(root, "src/VALUE.test.ts")); + } catch (cause) { + if ( + !(cause instanceof Error) || + !("code" in cause) || + cause.code !== "ENOENT" + ) + throw cause; + await expect( + testCommand(root, "focused", "src/VALUE.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("ENOENT"), + }); + return; + } + await testCommand(root, "focused", "src/VALUE.test.ts"); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }); + + it("matches Unicode-normalized aliases by filesystem identity", async () => { + const root = await fixture(); + const composed = "caf\u00e9.test.ts"; + const decomposed = composed.normalize("NFD"); + await writeFile( + join(root, "src", composed), + 'import { it } from "vitest"; it("checks the selected Unicode path", () => {});\n', + ); + const selected = join(root, "src", decomposed); + try { + await lstat(selected); + } catch (cause) { + if ( + !(cause instanceof Error) || + !("code" in cause) || + cause.code !== "ENOENT" + ) + throw cause; + await expect( + testCommand(root, "focused", `src/${decomposed}`), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("ENOENT"), + }); + return; + } + const result = await testCommand(root, "focused", `src/${decomposed}`); + expect(result.stdout).toContain("1 passed"); + }); + + it.each(["local", "focused"])( + "runs only the exact selected file in %s mode", + async (mode) => { + const root = await fixture(); + await mkdir(join(root, "src", "nested", "src"), { recursive: true }); + await writeFile( + join(root, "src", "nested", "src", "value.test.ts"), + 'import { it } from "vitest"; it("must not run", () => { throw new Error("unselected nested test ran"); });\n', + ); + const result = await testCommand(root, mode, "src/value.test.ts"); + expect(result.stdout).toContain("1 passed"); + expect(result.stdout).not.toContain("nested/src/value.test.ts"); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }, + ); + + it("fails when any selected existing test file is excluded from discovery", async () => { + const root = await fixture(); + await writeFile( + join(root, "src", "excluded.test.ts"), + 'import { it } from "vitest"; it("excluded", () => {});\n', + ); + await writeFile( + join(root, "vitest.config.mjs"), + 'export default { test: { projects: [{ test: { name: "domain", include: ["src/value.test.ts"] } }] } };\n', + ); + await expect( + testCommand(root, "focused", "src/value.test.ts", "src/excluded.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining( + "Selected test files were not discovered: src/excluded.test.ts", + ), + }); + }); + + it("runs every selected file and preserves selected test failures", async () => { + const root = await fixture(); + await writeFile( + join(root, "src", "failure.test.ts"), + 'import { it } from "vitest"; it("selected failure", () => { throw new Error("selected regression"); });\n', + ); + await expect( + testCommand(root, "focused", "src/value.test.ts", "src/failure.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stdout: expect.stringContaining("1 failed | 1 passed"), + stderr: expect.stringContaining("selected regression"), + }); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }); + + it("preserves all configured projects for an exact file", async () => { + const root = await fixture(); + await writeFile( + join(root, "vitest.config.mjs"), + 'export default { test: { projects: ["domain", "evaluation"].map(name => ({ test: { name, include: ["src/value.test.ts"] } })) } };\n', + ); + const result = await testCommand(root, "focused", "src/value.test.ts"); + expect(result.stdout).toMatch(/Test Files\s+2 passed/u); + }); + + it("fails an exact file with no tests even when another selected file passes", async () => { + const root = await fixture(); + await writeFile(join(root, "src", "empty.test.ts"), "export {};\n"); + await expect( + testCommand(root, "focused", "src/value.test.ts", "src/empty.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("No test suite found"), + }); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }); +}); + describe("development test runner public CLI", () => { it("runs an explicitly selected source test on a clean tree without build artifacts", async () => { const root = await fixture(); diff --git a/tests/boundary/cli/keyedArchiveXmlExtensionCli.test.ts b/tests/boundary/cli/keyedArchiveXmlExtensionCli.test.ts new file mode 100644 index 000000000..6ed029228 --- /dev/null +++ b/tests/boundary/cli/keyedArchiveXmlExtensionCli.test.ts @@ -0,0 +1,38 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { build } from "plist"; +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "compiled CLI inspects an XML keyed archive with a non-plist suffix", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-keyed-xml-extension-cli-"); + const path = join(root, "Model.archive"); + await writeFile( + path, + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "leaf"], + $top: { root: { UID: 1 } }, + }), + ); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", path, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expect(output.exitCode, output.stderr).toBe(0); + expect(parseEvidence(output.json).normalized_result).toMatchObject({ + archive_path: "Model.archive", + archive_format: "xml-plist", + objects: expect.arrayContaining([ + expect.objectContaining({ value: "leaf" }), + ]), + }); + }, +); diff --git a/tests/boundary/filesystem/android/application.test.ts b/tests/boundary/filesystem/android/application.test.ts index 01a5901c8..a29844641 100644 --- a/tests/boundary/filesystem/android/application.test.ts +++ b/tests/boundary/filesystem/android/application.test.ts @@ -205,6 +205,42 @@ describe("APK family without a root manifest", () => { }); }); +it("retains encrypted native library paths as unknown component evidence", async () => { + const root = await createTestTempDirectory("rea-android-encrypted-"); + const path = join(root, "Encrypted.apk"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add( + "lib/arm64-v8a/libentry.so", + new TextReader("encrypted native library"), + { password: "fixture-only" }, + ); + await writeFile(path, await writer.close()); + const inventory = parseEvidence( + await runProviderAnalysis(path, "inventory_artifact", {}), + ); + const projection = androidApplicationProjectionResultSchema.parse( + requireSuccessfulProjection( + projectAndroidApplicationEvidence({ inventory_evidence: [inventory] }), + ).normalized_result, + ); + + expect(projection.unavailable_components).toContainEqual( + expect.objectContaining({ + path: "lib/arm64-v8a/libentry.so", + artifact_id: null, + sha256: null, + encrypted: true, + }), + ); + expect(projection.coverage).toMatchObject({ + status: "partial", + inventory_complete: true, + }); + expect(projection.limitations).toContainEqual( + expect.stringContaining("component absence is unknown"), + ); +}); + describe("case-distinct APK resource names", () => { it("inventories and projects an APK whose resource names differ only in case", async () => { const root = await createTestTempDirectory("rea-android-case-"); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 9265e2e9b..30597523e 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -616,6 +616,10 @@ it.each([ { code: true, runtime: false, browser: false, package: false }, ], ["website/src/index.ts", { code: false, website: true, package: false }], + [ + "website/public/examples/notes-electron/preload.js", + { code: true, website: true, package: false }, + ], [ "website/locales/zh-CN/messages.po", { code: false, website: true, package: false }, diff --git a/tests/conformance/swift/ReaSwiftDecoderRegressions.java b/tests/conformance/swift/ReaSwiftDecoderRegressions.java new file mode 100644 index 000000000..751c0d3ab --- /dev/null +++ b/tests/conformance/swift/ReaSwiftDecoderRegressions.java @@ -0,0 +1,152 @@ +// Producer regressions run inside a real, ephemeral Ghidra analysis of the +// source-owned AArch64 fixture. No target code is executed and no decoded JSON +// fixture substitutes for the Java decoder under test. +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import ghidra.app.util.headless.HeadlessScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionIterator; +import ghidra.program.model.listing.Instruction; +import ghidra.program.model.listing.InstructionIterator; +import ghidra.program.model.lang.Register; +import ghidra.program.model.symbol.Symbol; +import ghidra.program.model.symbol.SymbolIterator; + +public class ReaSwiftDecoderRegressions extends HeadlessScript { + private ReaGhidraBridge decoder; + private Class budgetClass; + + @Override + public void run() throws Exception { + decoder = new ReaGhidraBridge(); + setInherited(decoder, "currentProgram", currentProgram); + setInherited(decoder, "monitor", monitor); + setInherited(decoder, "sessionDefaultAddressSpace", currentProgram.getAddressFactory().getDefaultAddressSpace()); + budgetClass = Class.forName("ReaGhidraBridge$SwiftLiteralBudget", true, decoder.getClass().getClassLoader()); + Function copied = function("rea_swift_decoder_copied"); + JsonArray copiedValues = literals(decode(copied, 65536)); + require(copiedValues.size() > 0, "copied constant pair was not decoded"); + for (var element : copiedValues) { + require(element.getAsJsonObject().get("value").getAsString().equals("ok"), "copied words invented text"); + } + for (String name : new String[] {"select", "selected_copy", "padding", "utf8"}) { + Object result = decode(function("rea_swift_decoder_" + name), 65536); + require(literals(result).isEmpty(), name + " produced an invented Swift literal"); + require(omissions(result).isEmpty(), name + " was confused with capacity exhaustion"); + } + for (String name : new String[] {"select", "selected_copy"}) { + Function selected = function("rea_swift_decoder_" + name); + require((Boolean) invoke("selectedValue", new Class[] {Function.class, Instruction.class, Register.class}, selected, returnInstruction(selected), currentProgram.getRegister("x0")), + name + " did not retain conditional-select provenance"); + } + require(invoke("swiftLiteralLayout", new Class[] {long.class, long.class}, 0x6261L, 0xe100000000000000L) == null, + "nonzero small-string padding was accepted"); + require(invoke("swiftLiteralLayout", new Class[] {long.class, long.class}, 0xc0L, 0xa100000000000000L) == null, + "malformed small-string UTF-8 was accepted"); + rejectLarge("rea_swift_decoder_bad_terminator", 0xd000000000000003L); + rejectLarge("rea_swift_decoder_bad_large_utf8", 0x1000000000000001L); + // Zero covers provenance-state exhaustion; 8192 admits both register + // cache entries and then rejects the retained literal/text charge. + assertCapacityOmission(copied, 0); + assertCapacityOmission(copied, 8192); + require(getScriptArgs().length == 1, "Expected owned result path"); + Files.writeString(Path.of(getScriptArgs()[0]), "REA_SWIFT_DECODER_SAFEGUARDS_OK\n"); + } + + private void assertCapacityOmission(Function function, long remaining) throws Exception { + Object exhausted = decode(function, remaining); + require(literals(exhausted).isEmpty(), "capacity failure published decoded small text at budget " + remaining); + JsonArray omitted = omissions(exhausted); + require(omitted.size() == 1, "capacity failure did not stop subsequent use sites at budget " + remaining); + JsonObject omission = omitted.get(0).getAsJsonObject(); + require(omission.get("reason").getAsString().equals("capacity"), "capacity omission lost its reason"); + require(!omission.has("value"), "omission claimed a retained literal value"); + } + + private Object decode(Function function, long remaining) throws Exception { + Constructor constructor = budgetClass.getDeclaredConstructor(long.class); + constructor.setAccessible(true); + return invoke("swiftStringLiterals", new Class[] {Function.class, budgetClass}, function, constructor.newInstance(remaining)); + } + + private void rejectLarge(String name, long word) throws Exception { + Address address = symbol(name).getAddress(); + long object = 0x8000000000000000L | ((address.getOffset() - 0x20) & 0x00ffffffffffffffL); + Object layout = invoke("swiftLiteralLayout", new Class[] {long.class, long.class}, word, object); + require(layout != null, "large negative fixture did not reach the memory-text decoder"); + require(invoke("swiftLargeLiteralText", new Class[] {layout.getClass()}, layout) == null, + name + " was accepted as literal text"); + } + + private Instruction returnInstruction(Function function) { + InstructionIterator instructions = currentProgram.getListing().getInstructions(function.getBody(), true); + while (instructions.hasNext()) { + Instruction instruction = instructions.next(); + if (instruction.getMnemonicString().equalsIgnoreCase("ret")) return instruction; + } + throw new AssertionError("Fixture has no return instruction"); + } + + private Function function(String name) { + FunctionIterator functions = currentProgram.getFunctionManager().getFunctions(true); + Function found = null; + while (functions.hasNext()) { + Function candidate = functions.next(); + if (!candidate.getName().equals(name) && !candidate.getName().equals("_" + name)) continue; + require(found == null, "ambiguous fixture function " + name); + found = candidate; + } + require(found != null, "missing fixture function " + name); + return found; + } + + private Symbol symbol(String name) { + SymbolIterator symbols = currentProgram.getSymbolTable().getAllSymbols(true); + Symbol found = null; + while (symbols.hasNext()) { + Symbol candidate = symbols.next(); + if (!candidate.getName().equals(name) && !candidate.getName().equals("_" + name)) continue; + require(found == null, "ambiguous fixture symbol " + name); + found = candidate; + } + require(found != null, "missing fixture symbol " + name); + return found; + } + + private Object invoke(String name, Class[] types, Object... arguments) throws Exception { + Method method = ReaGhidraBridge.class.getDeclaredMethod(name, types); + method.setAccessible(true); + return method.invoke(decoder, arguments); + } + + private JsonArray literals(Object result) throws Exception { return array(result, "literals"); } + private JsonArray omissions(Object result) throws Exception { return array(result, "omissions"); } + private JsonArray array(Object result, String name) throws Exception { + Method method = result.getClass().getDeclaredMethod(name); + method.setAccessible(true); + return (JsonArray) method.invoke(result); + } + + private void setInherited(Object target, String name, Object value) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException nextOwner) { } + } + throw new AssertionError("Missing Ghidra state field " + name); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/tests/conformance/swift/decoder-safeguards.S b/tests/conformance/swift/decoder-safeguards.S new file mode 100644 index 000000000..c426cfac7 --- /dev/null +++ b/tests/conformance/swift/decoder-safeguards.S @@ -0,0 +1,58 @@ +/* Static-analysis fixture only. Never executed. Deliberate malformed ABI words + * cannot be produced by a well-formed Swift source literal. */ +.text +.p2align 2 +.globl _rea_swift_decoder_copied +_rea_swift_decoder_copied: + mov x5, #0x6b6f + movz x6, #0xe200, lsl #48 + mov x0, x5 + mov x1, x6 + stp x0, x1, [sp, #-16]! + ldp x0, x1, [sp], #16 + ret + +.p2align 2 +.globl _rea_swift_decoder_select +_rea_swift_decoder_select: + mov x2, #0x61 + mov x3, #0x6262 + movz x6, #0xe100, lsl #48 + movz x7, #0xe200, lsl #48 + cmp w4, #0 + csel x0, x2, x3, eq + csel x1, x6, x7, eq + ret + +.p2align 2 +.globl _rea_swift_decoder_selected_copy +_rea_swift_decoder_selected_copy: + mov x2, #0x61 + mov x3, #0x62 + cmp w4, #0 + csel x5, x2, x3, eq + mov x0, x5 + movz x1, #0xe100, lsl #48 + ret + +.p2align 2 +.globl _rea_swift_decoder_padding +_rea_swift_decoder_padding: + mov x0, #0x6261 + movz x1, #0xe100, lsl #48 + ret + +.p2align 2 +.globl _rea_swift_decoder_utf8 +_rea_swift_decoder_utf8: + mov x0, #0xc0 + movz x1, #0xa100, lsl #48 + ret + +.section __DATA,__const +.globl _rea_swift_decoder_bad_terminator +_rea_swift_decoder_bad_terminator: + .byte 0x61, 0x62, 0x63, 0x78 +.globl _rea_swift_decoder_bad_large_utf8 +_rea_swift_decoder_bad_large_utf8: + .byte 0xc0, 0 diff --git a/tests/conformance/swift/unresolved-string-jump.s b/tests/conformance/swift/unresolved-string-jump.s new file mode 100644 index 000000000..2c3e71b85 --- /dev/null +++ b/tests/conformance/swift/unresolved-string-jump.s @@ -0,0 +1,8 @@ +.text +.globl _rea_unresolved_string_jump +.p2align 2 +_rea_unresolved_string_jump: + movz x0, #0x4220 + movk x0, #0x732f, lsl #16 + movz x1, #0xe400, lsl #48 + br x16 diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts index daed23cac..a9a4cc32b 100644 --- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts +++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts @@ -140,6 +140,8 @@ const managedTranscript = () => { javascriptSha256: "", javascriptShapeLeft: "", javascriptShapeRight: "", + notesElectron: "", + notesElectronSha256: "", managed: target.path, managedSha256: target.sha256, }); diff --git a/tests/evaluation/model-evals/openAiModelFixture.test.ts b/tests/evaluation/model-evals/openAiModelFixture.test.ts new file mode 100644 index 000000000..5fba1a94d --- /dev/null +++ b/tests/evaluation/model-evals/openAiModelFixture.test.ts @@ -0,0 +1,52 @@ +import { expect, it } from "vitest"; + +import { createOpenAiModelFixture } from "../../../scripts/verify/clients/model-fixture.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("routes OpenAI fixture requests by pathname, not query", async () => { + const directory = await createTestTempDirectory("rea-model-fixture-"); + let calls = 0; + const fixture = await createOpenAiModelFixture({ + directory, + prefix: "probe", + onRequest: () => { + calls += 1; + return { role: "assistant", content: "ok" }; + }, + }); + try { + const statusOf = async (path: string, method = "POST") => + ( + await fetch(`${fixture.baseUrl}${path}`, { + method, + ...(method === "POST" + ? { + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "rea-client-fixture", + messages: [{ role: "user", content: "hi" }], + }), + } + : {}), + }) + ).status; + + expect([ + await statusOf("/chat/completions"), + await statusOf("/chat/completions?api-version=1"), + await statusOf("/completions"), + await statusOf("/models?limit=1", "GET"), + ]).toEqual([200, 200, 404, 200]); + expect(calls).toBe(2); + expect(fixture.requests.map(({ path }) => path)).toEqual([ + "/v1/chat/completions", + "/v1/chat/completions?api-version=1", + ]); + expect(fixture.probes).toEqual([ + expect.objectContaining({ path: "/v1/completions", status: 404 }), + ]); + expect(fixture.failure).toBeUndefined(); + } finally { + await fixture.close(); + } +}); diff --git a/tests/evaluation/moduleBoundaries.test.ts b/tests/evaluation/moduleBoundaries.test.ts index 09f906963..786f1032a 100644 --- a/tests/evaluation/moduleBoundaries.test.ts +++ b/tests/evaluation/moduleBoundaries.test.ts @@ -275,6 +275,47 @@ describe("module import syntax and source discovery", () => { expect(result.stderr).toContain("src/domain/probe.ts:1: pure-layer"); expect(result.stderr).not.toContain("old.ts"); }); + + it("discovers forbidden nested sources without a Git worktree", async () => { + const temporary = await createTestTempDirectory("rea-module-no-git-"); + await mkdir(join(temporary, "src", "domain", "nested"), { + recursive: true, + }); + await writeFile( + join(temporary, "src/domain/nested/probe.ts"), + 'import { createServer } from "../../server/createServer.js";\n', + ); + const result = spawnSync(process.execPath, [verifier], { + cwd: temporary, + encoding: "utf8", + }); + if (result.error !== undefined) throw result.error; + expect(result.status).toBe(1); + expect(result.stderr).toContain("src/domain/nested/probe.ts"); + expect(result.stderr).toContain("src/server/createServer.js"); + }); + + it("discovers sources in an ignored archive nested under an ancestor Git worktree", async () => { + const outer = await createTestTempDirectory("rea-module-ancestor-git-"); + const archive = join(outer, "archive"); + await writeFile(join(outer, ".gitignore"), "archive/\n"); + await execute("git", ["init", "--quiet"], { cwd: outer }); + await execute("git", ["add", "--", ".gitignore"], { cwd: outer }); + await mkdir(join(archive, "src", "domain"), { recursive: true }); + await writeFile( + join(archive, "src/domain/probe.ts"), + 'import { createServer } from "../server/createServer.js";\n', + ); + + const result = spawnSync(process.execPath, [verifier], { + cwd: archive, + encoding: "utf8", + }); + if (result.error !== undefined) throw result.error; + expect(result.status).toBe(1); + expect(result.stderr).toContain("src/domain/probe.ts:1: pure-layer"); + expect(result.stderr).toContain("src/server/createServer.js"); + }); }); describe("test lane import boundaries", () => { diff --git a/tests/process-global/updateMaintenance.test.ts b/tests/process-global/updateMaintenance.test.ts index 171761478..141cf23ec 100644 --- a/tests/process-global/updateMaintenance.test.ts +++ b/tests/process-global/updateMaintenance.test.ts @@ -319,6 +319,8 @@ describe("updated executable maintenance planning", () => { it("runs real setup as a noninteractive dry run and creates no new integrations or backups", async () => { const configPath = await writeClient("codex", staleCodex); const untouched = await writeClient("claude_code", '{"mcpServers":{}}'); + const skillContent = `---\nname: ${PRODUCT_IDENTITY.skillName}\nmetadata:\n version: "1"\n---\n# REA\n`; + const skillPath = await writeSkill(skillContent); vi.stubEnv("npm_command", "exec"); const result = await planIntegrationMaintenance( home, @@ -328,18 +330,35 @@ describe("updated executable maintenance planning", () => { ); expect(result).toMatchObject({ status: "planned", - scope: { clients: ["codex"], skill: false }, - plannedActions: [ - { id: "configure_client:codex", kind: "configure_client" }, - ], + scope: { + clients: ["codex"], + skill: true, + skillDestinations: [{ client: "shared", path: dirname(skillPath) }], + }, }); + if (result.status !== "planned") throw new Error(JSON.stringify(result)); + expect(result.plannedActions).toEqual([ + expect.objectContaining({ + id: "configure_client:codex", + kind: "configure_client", + }), + expect.objectContaining({ + id: "install_skill", + kind: "install_skill", + target: dirname(skillPath), + }), + ]); expect(await readFile(configPath, "utf8")).toBe(staleCodex); expect(await readFile(untouched, "utf8")).toBe('{"mcpServers":{}}'); + expect(await readFile(skillPath, "utf8")).toBe(skillContent); await expect(access(`${configPath}.rea.backup`)).rejects.toMatchObject({ code: "ENOENT", }); + await expect(access(`${skillPath}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); await expect( - access(join(home, ".agents", "skills", PRODUCT_IDENTITY.skillName)), + access(join(home, ".claude", "skills", PRODUCT_IDENTITY.skillName)), ).rejects.toMatchObject({ code: "ENOENT" }); }); }); diff --git a/tests/support/mcp/productionSession.ts b/tests/support/mcp/productionSession.ts new file mode 100644 index 000000000..38ce87e09 --- /dev/null +++ b/tests/support/mcp/productionSession.ts @@ -0,0 +1,33 @@ +import type { Client } from "@modelcontextprotocol/client"; +import { expect, type OnTestFinishedHandler } from "vitest"; + +import { createBinarySession } from "../../../src/composition/binary.js"; +import { parseConfig } from "../../../src/config/parseConfig.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createServer } from "../../../src/server/createServer.js"; +import type { TestMcp } from "./mcpFixture.js"; + +/** Open one real target in a production session served over MCP. */ +export const openProductionTarget = async ( + mcp: TestMcp, + onTestFinished: (handler: OnTestFinishedHandler) => void, + path: string, +): Promise => { + const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); + if (!configured.ok) throw configured.error; + const session = createBinarySession( + configured.value, + silentLogger, + process.env, + ); + onTestFinished(async () => { + await session.close(); + }); + const client = await mcp.connect(createServer({ kind: "session", session })); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + return client; +}; From 86afc040e5c9f2e9fbf9604f234bf180d9ebe954 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:05:53 +0800 Subject: [PATCH 24/26] fix(ghidra): discover installations from resolved PATH launchers Reuse installation inspection in doctor and provider readiness. Keep explicit configuration authoritative and validate discovered roots without executing launchers. Clarify the existing per-operation availability contract. --- docs/installation.md | 17 +++++ src/ghidra/GhidraInstallation.test.ts | 98 +++++++++++++++++++++++++++ src/ghidra/GhidraInstallation.ts | 61 ++++++++++++++++- 3 files changed, 173 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 4e95e548e..bb6a2dd5c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -290,6 +290,15 @@ configuration without changing them. Without options, it audits every detected registration, the installed skill and optional analysis engines; its overall `healthy` value can be false while your chosen workflow works. +Doctor's `providerInspections` are host-level engine and registration checks, +not a readiness row for every advertised MCP adapter surface. `tools/list` +remains the complete stable catalog. Call `binary_session` with `{}` before +choosing an operation; it works before a target is opened, and +`result.tool_availability` reports per-operation availability, reasons and +client requirements for the current session state. It does not guarantee that +a particular caller-supplied input will be valid. See +[MCP discovery](mcp-contracts.md#identity-and-discovery). + Select the part you want to check: | Task | Readiness check | @@ -736,6 +745,14 @@ installation remains unavailable on Windows. Setup never installs Ghidra, Java, or Python. It preserves valid detected Ghidra/JDK settings in agent registrations. See [Windows Ghidra P0](windows-ghidra-p0.md) for provider diagnostics. +When `GHIDRA_INSTALL_DIR` is unset, doctor and the Ghidra provider also search +`PATH` for `ghidra` or `ghidraRun`, resolve the selected launcher through its +real path, and accept it only when that location belongs to a Ghidra tree with +the application metadata and headless launcher. An explicitly supplied +`GHIDRA_INSTALL_DIR` always takes precedence, including when it is invalid; +REA reports that error instead of silently selecting another installation. +This discovery is read-only and does not execute the Ghidra launcher. + Doctor validates the platform, architecture, Ghidra 12.1.x application version, `support/analyzeHeadless` or `support/analyzeHeadless.bat`, the installation's Java major range, 64-bit JDK bitness, and the presence of `javac`/`javac.exe`. diff --git a/src/ghidra/GhidraInstallation.test.ts b/src/ghidra/GhidraInstallation.test.ts index 626954c69..a60e7b4b1 100644 --- a/src/ghidra/GhidraInstallation.test.ts +++ b/src/ghidra/GhidraInstallation.test.ts @@ -1,3 +1,5 @@ +import { chmod, mkdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { spawnSync } from "node:child_process"; @@ -6,6 +8,7 @@ import { projectGhidraDoctorInspection } from "./GhidraDoctor.js"; import { ghidraInstallationDiagnostics, ghidraJavaEnvironment, + discoverGhidraInstallationOnPath, inspectGhidraInstallation, type GhidraExecutableHeaderProbe, type GhidraInstallationHost, @@ -17,6 +20,7 @@ import { ghidraApplicationProperties, ghidraJavaRequirement, } from "./GhidraInstallationPolicy.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; // The real Java probe shells out via spawnSync; stub it so one test can run // the real probeJava against captured `-XshowSettings:properties` output. @@ -68,6 +72,100 @@ afterEach(() => { vi.unstubAllEnvs(); }); +it.skipIf(process.platform === "win32")( + "resolves a PATH launcher only when its real target is a complete installation", + async () => { + const root = await createTestTempDirectory("rea-ghidra-path-"); + const installation = join(root, "installation"); + const pathDirectory = join(root, "bin"); + const launcher = join(installation, "ghidraRun"); + await mkdir(join(installation, "Ghidra"), { recursive: true }); + await mkdir(join(installation, "support"), { recursive: true }); + await mkdir(pathDirectory, { recursive: true }); + await writeFile( + join(installation, "Ghidra", "application.properties"), + "application.version=12.1.4\n", + ); + await writeFile(launcher, "#!/bin/sh\nexit 0\n"); + await writeFile( + join(installation, "support", "analyzeHeadless"), + "#!/bin/sh\nexit 0\n", + ); + await chmod(launcher, 0o700); + await chmod(join(installation, "support", "analyzeHeadless"), 0o700); + await symlink(launcher, join(pathDirectory, "ghidra")); + + expect( + discoverGhidraInstallationOnPath( + { PATH: pathDirectory }, + process.platform, + ), + ).toBe(installation); + + const incomplete = join(root, "incomplete"); + const incompletePath = join(root, "incomplete-bin"); + await mkdir(join(incomplete, "Ghidra"), { recursive: true }); + await mkdir(incompletePath, { recursive: true }); + await writeFile( + join(incomplete, "Ghidra", "application.properties"), + "application.version=12.1.4\n", + ); + await writeFile(join(incomplete, "ghidraRun"), "launcher\n"); + await chmod(join(incomplete, "ghidraRun"), 0o700); + await symlink( + join(incomplete, "ghidraRun"), + join(incompletePath, "ghidra"), + ); + expect( + discoverGhidraInstallationOnPath( + { PATH: incompletePath }, + process.platform, + ), + ).toBeUndefined(); + + const nonExecutable = join(root, "non-executable"); + const nonExecutablePath = join(root, "non-executable-bin"); + await mkdir(join(nonExecutable, "Ghidra"), { recursive: true }); + await mkdir(join(nonExecutable, "support"), { recursive: true }); + await mkdir(nonExecutablePath, { recursive: true }); + await writeFile( + join(nonExecutable, "Ghidra", "application.properties"), + "application.version=12.1.4\n", + ); + await writeFile( + join(nonExecutable, "support", "analyzeHeadless"), + "headless\n", + ); + await chmod(join(nonExecutable, "support", "analyzeHeadless"), 0o700); + await writeFile(join(nonExecutablePath, "ghidra"), "not executable\n"); + expect( + discoverGhidraInstallationOnPath( + { PATH: nonExecutablePath }, + process.platform, + ), + ).toBeUndefined(); + }, +); + +it("does not fall back to PATH when an explicit Ghidra installation path is malformed", () => { + const inspection = inspectGhidraInstallation( + { + environment: { PATH: "/valid-ghidra-bin" }, + installDir: "relative/ghidra", + platform: "linux", + architecture: "x64", + }, + host({ discoverInstallDirOnPath: () => INSTALL }), + ); + expect(inspection.status).toBe("unavailable"); + expect(inspection.installDir).toBe("relative/ghidra"); + expect(inspection.checks[0]).toMatchObject({ + name: "configuration", + status: "failed", + detail: "GHIDRA_INSTALL_DIR must be absolute", + }); +}); + it("probes the selected Java environment without reading conflicting ambient settings", () => { vi.stubEnv("REA_GHIDRA_CONTEXT", "ambient"); vi.stubEnv("REA_GHIDRA_AMBIENT", "ambient-only"); diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index f30f6ef7a..6a77ee1af 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -8,9 +8,10 @@ import { openSync, readFileSync, readSync, + realpathSync, statSync, } from "node:fs"; -import { posix, win32 } from "node:path"; +import { delimiter as pathDelimiter, dirname, posix, win32 } from "node:path"; import type { ProviderRejectionCode } from "../contracts/providerSelection.js"; import { @@ -295,6 +296,11 @@ export interface GhidraInstallationHost { command: string, environment: NodeJS.ProcessEnv, ): GhidraJavaObservation | undefined; + /** Resolve a PATH launcher to a validated installation without executing it. */ + discoverInstallDirOnPath?( + environment: Readonly, + platform: NodeJS.Platform, + ): string | undefined; } interface GhidraInstallationCoordinates { @@ -405,7 +411,12 @@ const installationCoordinates = ( host: GhidraInstallationHost, ): GhidraInstallationCoordinates => { const path = platform === "win32" ? win32 : posix; - const installDir = options.installDir ?? null; + // An explicit setting remains authoritative even when malformed or stale; + // automatic discovery is only a fallback when the caller supplied none. + const installDir = + options.installDir ?? + host.discoverInstallDirOnPath?.(options.environment, platform) ?? + null; const applicationPropertiesPath = installDir === null ? null @@ -958,12 +969,56 @@ const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ : "jre", }; }, + discoverInstallDirOnPath: discoverGhidraInstallationOnPath, }); +/** Find a PATH launcher whose real target is inside a complete Ghidra tree. */ +export const discoverGhidraInstallationOnPath = ( + environment: Readonly, + platform: NodeJS.Platform, +): string | undefined => { + if (platform !== process.platform) return undefined; + const path = platform === "win32" ? win32 : posix; + const pathValue = environment.PATH; + if (pathValue === undefined || pathValue.length === 0) return undefined; + const launchers = + platform === "win32" + ? ["ghidra.bat", "ghidraRun.bat", "ghidra.exe", "ghidraRun.exe"] + : ["ghidra", "ghidraRun"]; + for (const directory of pathValue.split(pathDelimiter)) { + if (directory.length === 0) continue; + for (const launcher of launchers) { + let resolved: string; + try { + const candidate = path.resolve(directory, launcher); + if (!executablePath(candidate)) continue; + resolved = realpathSync(candidate); + } catch (cause: unknown) { + void cause; + continue; + } + const root = dirname(resolved); + const properties = path.join(root, "Ghidra", "application.properties"); + const headless = path.join( + root, + "support", + platform === "win32" ? "analyzeHeadless.bat" : "analyzeHeadless", + ); + try { + if (statSync(properties).isFile() && executablePath(headless)) + return root; + } catch (cause: unknown) { + void cause; + } + } + } + return undefined; +}; + const executablePath = (path: string): boolean => { try { accessSync(path, constants.X_OK); - return true; + return statSync(path).isFile(); } catch (cause: unknown) { // best-effort cleanup: optional executable probing; failure means missing. void cause; From fab33a66c6348942bb112f890daaee7614d2bb7b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:09:02 +0800 Subject: [PATCH 25/26] test(package): select the intended Hopper smoke-test provider Keep provider discovery independent from Hopper-specific packaged workflows. Installed Ghidra launchers on PATH must not turn the fixture into an ambiguous auto-selection request; target-free inventory and unknown-provider checks remain explicit. --- scripts/verify-package-capabilities.mjs | 9 ++++++++- scripts/verify-package-platform.mjs | 26 +++++++++++++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/scripts/verify-package-capabilities.mjs b/scripts/verify-package-capabilities.mjs index acf848974..5a9f79ba9 100644 --- a/scripts/verify-package-capabilities.mjs +++ b/scripts/verify-package-capabilities.mjs @@ -31,7 +31,14 @@ export async function verifyPackageCapabilitiesAndSearch({ cli, environment }) { const searchResult = json( await run( cli, - ["search", process.execPath, "fixture", "--json"], + [ + "search", + process.execPath, + "fixture", + "--provider", + "hopper", + "--json", + ], environment, ), ); diff --git a/scripts/verify-package-platform.mjs b/scripts/verify-package-platform.mjs index 253937688..0dd67aa86 100644 --- a/scripts/verify-package-platform.mjs +++ b/scripts/verify-package-platform.mjs @@ -10,7 +10,7 @@ export async function verifyPackagePlatform({ if (platform === "linux") { const unsupportedExecution = await runWithStatus( cli, - ["analyze", process.execPath, "--json"], + ["analyze", process.execPath, "--provider", "hopper", "--json"], environment, ); assertLinuxPackageProviderFailure(unsupportedExecution); @@ -18,7 +18,7 @@ export async function verifyPackagePlatform({ const overview = json( await runCommand( cli, - ["analyze", process.execPath, "--json"], + ["analyze", process.execPath, "--provider", "hopper", "--json"], environment, ), ); @@ -29,7 +29,7 @@ export async function verifyPackagePlatform({ const inspected = json( await runCommand( cli, - ["inspect", process.execPath, "--json"], + ["inspect", process.execPath, "--provider", "hopper", "--json"], environment, ), ); @@ -38,7 +38,14 @@ export async function verifyPackagePlatform({ const functionResult = json( await runCommand( cli, - ["function", process.execPath, "0x1000", "--json"], + [ + "function", + process.execPath, + "0x1000", + "--provider", + "hopper", + "--json", + ], environment, ), ); @@ -53,7 +60,7 @@ export async function verifyPackagePlatform({ const xrefs = json( await runCommand( cli, - ["xrefs", process.execPath, "0x1000", "--json"], + ["xrefs", process.execPath, "0x1000", "--provider", "hopper", "--json"], environment, ), ); @@ -65,7 +72,14 @@ export async function verifyPackagePlatform({ const trace = json( await runCommand( cli, - ["trace", process.execPath, "fixture", "--json"], + [ + "trace", + process.execPath, + "fixture", + "--provider", + "hopper", + "--json", + ], environment, ), ); From bcc141549a071c02ad2c3eeabd25fbf6207eb95b Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Tue, 13 Oct 2026 18:16:33 -0700 Subject: [PATCH 26/26] feat(electron): inspect packaged Electron fuse state without execution (#2399) Add bounded, read-only inspection of packaged Electron fuse state for ELF, PE, and Mach-O applications through CLI and MCP workflows. Preserve artifact provenance and unknown values, resolve app bundles without execution, reject malformed or overlapping containers before scanning, and keep output compact for agent use. Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- .../skills/reverse-engineer-anything/SKILL.md | 2 +- .../references/runtime-observation.md | 8 +- docs/.vitepress/config.ts | 1 + docs/cli.md | 1 + docs/electron-fuses.md | 82 +++ docs/electron-observation.md | 4 + docs/index.md | 2 +- docs/javascript-runtime-observation.md | 6 + scripts/lib/catalog-builders.mjs | 16 +- scripts/lib/catalog-core.mjs | 1 + scripts/lib/test-projects.mjs | 2 +- src/application/CapabilityInventory.ts | 6 +- src/application/electron/ElectronFusePort.ts | 19 + .../electron/ElectronFuseService.ts | 116 +++ src/cli/electronCommands.ts | 32 + src/cliCommandNames.ts | 1 + src/composition/electronFuses.ts | 6 + .../javascript/electronToolContracts.ts | 17 + src/contracts/toolEffects.ts | 1 + src/contracts/toolOutputSchemaGroups.ts | 6 + src/domain/electron/electronFuses.test.ts | 296 ++++++++ src/domain/electron/electronFuses.ts | 477 ++++++++++++ src/electron/ElectronFusePath.ts | 307 ++++++++ src/electron/ElectronFuseProvider.test.ts | 684 ++++++++++++++++++ src/electron/ElectronFuseProvider.ts | 170 +++++ src/electron/ElectronFuseReader.ts | 480 ++++++++++++ src/server/createServer.ts | 4 + src/server/registerElectronTools.ts | 23 + tests/acceptance/electronFusesCli.test.ts | 150 ++++ tests/boundary/mcp/electronFusesMcp.test.ts | 180 +++++ tests/fixtures/electron/fuseImage.ts | 146 ++++ 31 files changed, 3237 insertions(+), 9 deletions(-) create mode 100644 docs/electron-fuses.md create mode 100644 src/application/electron/ElectronFusePort.ts create mode 100644 src/application/electron/ElectronFuseService.ts create mode 100644 src/composition/electronFuses.ts create mode 100644 src/domain/electron/electronFuses.test.ts create mode 100644 src/domain/electron/electronFuses.ts create mode 100644 src/electron/ElectronFusePath.ts create mode 100644 src/electron/ElectronFuseProvider.test.ts create mode 100644 src/electron/ElectronFuseProvider.ts create mode 100644 src/electron/ElectronFuseReader.ts create mode 100644 tests/acceptance/electronFusesCli.test.ts create mode 100644 tests/boundary/mcp/electronFusesMcp.test.ts create mode 100644 tests/fixtures/electron/fuseImage.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index b0b5b8b33..426a05ae5 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -122,7 +122,7 @@ explicit path or endpoint and do not need it. pages; select one trace, stream ID and direction for contribution/range pages. Facts retain all-recorded-events-before-selection scope. Other qlog layouts and PCAP correlation are unsupported. -- User-owned Electron runtime already open: `list_electron_targets`. +- User-owned Electron runtime already open: `list_electron_targets`. Before an electron-main Inspector attempt, check packaged fuses with `inspect_electron_fuses`; disabled `EnableNodeCliInspectArguments` means the app ignores `--inspect`. - Packet-field discovery: `inspect_packet_fields` searches installed protocols and field names/types before choosing fields for `inspect_packet_capture`. Query/page the relevant records; registrations do not establish observed traffic. diff --git a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md index ba197ecb8..df75bcaf3 100644 --- a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md +++ b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md @@ -8,9 +8,11 @@ selected page's current origin. Electron discovery lists eligible local `file://` targets without a configured file-root allowlist. Start passive browser work with `list_browser_targets`; start Electron runtime work with -`list_electron_targets`. Inspect the target selected for the user's task. Observation -with these tools is passive: never claim REA clicked, navigated, evaluated page JavaScript, -invoked IPC, or captured prior activity. REA does not contain the page's network. +`list_electron_targets`. Before an electron-main Inspector attempt, check +packaged fuses with `inspect_electron_fuses`. Inspect the target selected for +the user's task. Observation with these tools is passive: never claim REA +clicked, navigated, evaluated page JavaScript, invoked IPC, or captured prior +activity. REA does not contain the page's network. Passive captures omit raw header collections, credential header values, storage values, and raw WebSocket/JSON bodies. Selected response metadata, including diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index dd562d813..440338f98 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -44,6 +44,7 @@ export default defineConfig({ link: "/web-network-captures", }, { text: "Electron observation", link: "/electron-observation" }, + { text: "Electron fuses", link: "/electron-fuses" }, { text: "Process capture", link: "/process-capture" }, { text: ".NET assemblies", link: "/managed-code-analysis" }, { text: "Android applications", link: "/android-analysis" }, diff --git a/docs/cli.md b/docs/cli.md index de180122b..76dde4f4f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -292,6 +292,7 @@ Choose the guide for the target and intended interaction: - [Browser observation](browser-observation.md): inspect a selected page through CDP. - [Browser scenarios](browser-scenario-contract.md): run declared interactions and capture their results. - [Electron observation](electron-observation.md): inspect an Electron renderer or capture an application scenario. +- [Electron fuses](electron-fuses.md): read packaged fuse bytes before expecting `--inspect` to reach an Electron main process. - [Node/Electron Inspector](javascript-runtime-observation.md): record script locations and execution contexts. - [Process capture](process-capture.md): run an executable and compare terminal, exit and filesystem observations. diff --git a/docs/electron-fuses.md b/docs/electron-fuses.md new file mode 100644 index 000000000..206ea24bb --- /dev/null +++ b/docs/electron-fuses.md @@ -0,0 +1,82 @@ +# Electron fuse state + +`inspect_electron_fuses` / `inspect-electron-fuses` answers whether a packaged +Electron app will honor `--inspect` and `--inspect-brk`, `ELECTRON_RUN_AS_NODE`, +embedded ASAR integrity validation, and loading application code only from +`app.asar`. It reads the fuse wire stored in the Electron binary. It does not +launch the app, modify fuse bytes, or use the network. + +```bash +rea inspect-electron-fuses /Applications/Example.app --json +``` + +The CLI resolves a relative path against the caller's working directory. MCP +requires an absolute filesystem path and preserves that exact selection: + +```json +{ + "name": "inspect_electron_fuses", + "arguments": { "path": "/Applications/Example.app" } +} +``` + +Check this before `observe_javascript_runtime` with `electron-main`. When +`EnableNodeCliInspectArguments` is disabled or removed, Electron ignores +Inspector flags, and a later target listing can simply show no main-process +target. + +## What is read + +On macOS, pass the `.app` bundle or the Electron Framework binary. Bundle and +framework selection first tries the top-level `Electron Framework` binary, +then `Versions/Current/Electron Framework`, then +`Versions/A/Electron Framework`. Symlinks are followed only while they stay +inside that framework. +On Windows and Linux, pass the main Electron executable. The wire lives in +that executable rather than in `app.asar`. + +The inline Evidence result contains: + +- the selected path, the resolved regular file, its SHA-256, size, and container + format (`mach-o`, `mach-o-universal`, `pe`, or `elf`); +- one slice entry for a thin Mach-O, PE, or ELF file, and one entry per slice of + a universal Mach-O, with file-absolute `sentinel_offset` values; +- `wire_version`, `declared_length`, and ordered fuses `{ index, name, state, raw_byte }`; +- `sentinel_count`, including zero and more than one; +- `runtime_implications`, each marked `relationship: "derivation"` and tied to + the fuse index it came from. + +Version 1 names, in index order, are `RunAsNode`, `EnableCookieEncryption`, +`EnableNodeOptionsEnvironmentVariable`, `EnableNodeCliInspectArguments`, +`EnableEmbeddedAsarIntegrityValidation`, `OnlyLoadAppFromAsar`, +`LoadBrowserProcessSpecificV8Snapshot`, `GrantFileProtocolExtraPrivileges`, +and `WasmTrapHandlers`. An index past that table, or any index on another wire +version, keeps `name: null` and the raw byte. + +Wire bytes are `0x30` disabled, `0x31` enabled, and `0x72` removed. Any other +byte is `unknown`. A removed fuse is not enabled: Inspector flags and +`ELECTRON_RUN_AS_NODE` are not honored, and application code is not restricted +to `app.asar`. On macOS and Windows, a removed or disabled +`EnableEmbeddedAsarIntegrityValidation` byte means embedded ASAR integrity is +not enforced, and an enabled byte means it is enforced. Electron compiles that +validation only on macOS and Windows. On a Linux ELF executable the observed +byte is preserved and the enforcement outcome is `unknown`. + +## Unknown and multiple sentinels + +No sentinel means the fuse state is unknown. It does not mean the fuses are +disabled. A wire that continues past the slice is reported as truncated, with +only the fuse bytes that were present. Two or more sentinels in one slice are +all reported. REA does not pick one. + +Fuse bytes do not prove that the application enables Inspector some other way, +for example through `app.commandLine` or a remote-debugging switch in +application code. + +The selected file must be a regular file, or a bundle that resolves to one. +Symlinks are followed only inside `Electron Framework.framework`. The reader +admits at most 1 GiB, searches with 256 KiB windows, and keeps at most 32 +sentinels per slice. A real nine-fuse, one-slice result is about 1,200 CLI +tokens; a two-slice result is about 1,300 tokens. Exact size varies with path +lengths and limitations. This inspection creates no temporary files and does +not modify the selected bytes. diff --git a/docs/electron-observation.md b/docs/electron-observation.md index eb98a41f3..e702978d9 100644 --- a/docs/electron-observation.md +++ b/docs/electron-observation.md @@ -7,6 +7,10 @@ This passive runtime surface is distinct from the target-free static workflow. Static analysis reads a local ASAR or extracted directory without executing application code. Passive observation inspects an existing Electron page. The results keep static inferences and runtime observations distinct. +Packaged fuse state is a separate static read: +[`inspect_electron_fuses`](electron-fuses.md) reports whether this binary will +honor Inspector flags, `ELECTRON_RUN_AS_NODE`, embedded ASAR integrity, and +`app.asar`-only loading. Use the separate [`reconcile_javascript_runtime`](javascript-runtime-reconciliation.md) workflow when both Evidence sets already exist. diff --git a/docs/index.md b/docs/index.md index 9e0912dd1..5acee1b55 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,7 +52,7 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). | Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | | Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [JEB](jeb-analysis.md) | | Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | -| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md) | +| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | | Inspect assemblies, packages, or firmware | [.NET](managed-code-analysis.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | | Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | | Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | diff --git a/docs/javascript-runtime-observation.md b/docs/javascript-runtime-observation.md index 9a0e90b1f..439572eba 100644 --- a/docs/javascript-runtime-observation.md +++ b/docs/javascript-runtime-observation.md @@ -36,6 +36,12 @@ application starts, and REA never resumes it or sends `Runtime.runIfWaitingForDebugger`. Resume a paused target with a separate caller-owned debugger before requesting application runtime observations. +For a packaged Electron app, read +[`inspect_electron_fuses`](electron-fuses.md) before starting an `electron-main` +target. When `EnableNodeCliInspectArguments` is disabled or removed, Electron +ignores `--inspect` and `--inspect-brk`. A missing main-process target after +launch does not, by itself, show that fuse. + The caller supplies the literal-loopback HTTP endpoint directly. Selecting an endpoint exposes all targets it serves and their local script locations or renderer origins. A target must match the exact requested target ID and expose a same-port WebSocket reported by that endpoint. Targets explicitly marked attached are rejected rather than displacing another debugger. Node's `/json/list` omits attachment state, so REA preserves it as `null` (unknown), reports that limitation, and permits observation unless the endpoint explicitly reports `attached: true`. Local paths are canonicalized after symlink resolution. The `attached` field in JavaScript runtime and browser target results is nullable: `true` and `false` are reported only when discovery supplied a boolean; `null` means the endpoint omitted or could not report the state. CDP `TargetInfo` worker records require an `attached` boolean; REA excludes a worker record with a missing or malformed value as an invalid protocol value. diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index 48f90c6e7..64296b3ce 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -167,6 +167,7 @@ export const providerCatalog = (sources) => { reconciliationContracts, observationContracts, activeContracts, + fuseContracts, } = electronContractSlices(sources); return [ { @@ -261,6 +262,10 @@ export const providerCatalog = (sources) => { identity: sources.rustProvider.RUST_BINARY_PROVIDER_IDENTITY, contracts: sources.rustContracts.RUST_TOOL_CONTRACTS, }, + { + identity: sources.electronFuseProvider.ELECTRON_FUSE_PROVIDER_IDENTITY, + contracts: fuseContracts, + }, { identity: sources.sqliteDomain.SQLITE_PROVIDER_IDENTITY, contracts: sources.sqliteContracts.SQLITE_TOOL_CONTRACTS, @@ -425,17 +430,23 @@ const electronContractSlices = (sources) => { ({ name }) => name !== "analyze_javascript_application" && name !== "reconcile_javascript_runtime" && - name !== "capture_electron_scenario", + name !== "capture_electron_scenario" && + name !== "inspect_electron_fuses", ); const activeContracts = sources.electronContracts.ELECTRON_TOOL_CONTRACTS.filter( ({ name }) => name === "capture_electron_scenario", ); + const fuseContracts = + sources.electronContracts.ELECTRON_TOOL_CONTRACTS.filter( + ({ name }) => name === "inspect_electron_fuses", + ); if ( applicationContracts.length !== 1 || reconciliationContracts.length !== 1 || observationContracts.length !== 2 || - activeContracts.length !== 1 + activeContracts.length !== 1 || + fuseContracts.length !== 1 ) throw new Error("Electron provider capability ownership drifted"); return { @@ -443,5 +454,6 @@ const electronContractSlices = (sources) => { reconciliationContracts, observationContracts, activeContracts, + fuseContracts, }; }; diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index 262493b73..2cdfb64d2 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -96,6 +96,7 @@ const SOURCE_PATHS = { webNetworkCaptureProvider: "dist/browser/history/CaptureRelease.js", browserScenarioContracts: "dist/contracts/browserScenarioToolContracts.js", electronContracts: "dist/contracts/javascript/electronToolContracts.js", + electronFuseProvider: "dist/electron/ElectronFuseProvider.js", javascriptRuntimeObservationContracts: "dist/contracts/javascript/javascriptRuntimeObservationToolContracts.js", applicationContracts: "dist/contracts/applicationToolContracts.js", diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs index 68fb90dbd..f284625cc 100644 --- a/scripts/lib/test-projects.mjs +++ b/scripts/lib/test-projects.mjs @@ -17,7 +17,7 @@ export const SOURCE_TEST_PROJECTS = [ group: "source", include: [ "src/*.test.ts", - "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", + "src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", ], }, { diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 029b23357..5f3fba6a6 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -344,7 +344,11 @@ const workflowAvailabilityFor = ({ kind, policy, }: AvailabilityContext): Availability | null => { - if (name === "inspect_go_binary" || name === "inspect_rust_binary") + if ( + name === "inspect_go_binary" || + name === "inspect_rust_binary" || + name === "inspect_electron_fuses" + ) return { reason: "available", remediation: null }; if (name === "inspect_sqlite_database") { const capability = sqliteDatabaseCapability(process.versions.node); diff --git a/src/application/electron/ElectronFusePort.ts b/src/application/electron/ElectronFusePort.ts new file mode 100644 index 000000000..06354f2c5 --- /dev/null +++ b/src/application/electron/ElectronFusePort.ts @@ -0,0 +1,19 @@ +import type { + ExecutionOptions, + ProviderIdentity, +} from "../AnalysisProvider.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { + ElectronFuseReport, + InspectElectronFusesInput, +} from "../../domain/electron/electronFuses.js"; +import type { Result } from "../../domain/result.js"; + +/** Replaceable read-only Electron fuse reader with no target-execution authority. */ +export interface ElectronFusePort { + readonly identity: ProviderIdentity; + inspect( + input: InspectElectronFusesInput, + options?: ExecutionOptions, + ): Promise>; +} diff --git a/src/application/electron/ElectronFuseService.ts b/src/application/electron/ElectronFuseService.ts new file mode 100644 index 000000000..9719a6fe2 --- /dev/null +++ b/src/application/electron/ElectronFuseService.ts @@ -0,0 +1,116 @@ +import { isAbsolute } from "node:path"; + +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import type { ElectronFusePort } from "./ElectronFusePort.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import { BINARY_ARCHITECTURES } from "../../domain/binaryTargetTypes.js"; +import { + electronFuseReportSchema, + inspectElectronFusesInputSchema, +} from "../../domain/electron/electronFuses.js"; +import { createEvidence, type Evidence } from "../../domain/evidence.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; + +const OPERATION = "inspect_electron_fuses"; + +/** Shared exact input validation and artifact-bound Evidence for CLI and MCP. */ +export class ElectronFuseService { + constructor(readonly provider: ElectronFusePort) {} + + /** Read fuse wires for one explicitly selected path without launching it. */ + async inspect( + rawInput: unknown, + options?: ExecutionOptions, + ): Promise> { + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const input = inspectElectronFusesInputSchema.safeParse(rawInput); + if (!input.success) + return err( + analysisInputErrorFromIssues(OPERATION, input.error.issues, rawInput, { + cause: input.error, + }), + ); + if (!isAbsolute(input.data.path)) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["path"], + reason: "invalid_format", + message: "Expected an absolute filesystem path on this host.", + }, + ]), + ); + const inspected = await this.provider.inspect(input.data, options); + if (!inspected.ok) return inspected; + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const report = electronFuseReportSchema.safeParse(inspected.value); + if ( + !report.success || + report.data.artifact.selected_path !== input.data.path + ) + return err( + new AnalysisOutputError( + OPERATION, + "Electron fuse reader returned a malformed report or changed the selected artifact identity.", + ), + ); + const value = report.data; + const architecture = subjectArchitecture(value); + return ok( + createEvidence( + { + path: value.artifact.resolved_path, + sha256: value.artifact.sha256, + format: value.artifact.format, + ...(architecture === undefined ? {} : { architecture }), + }, + this.provider.identity, + { + operation: OPERATION, + parameters: input.data, + result: value, + confidence: "observed", + limitations: value.limitations, + locations: locationsFor(value), + }, + ), + ); + } +} + +const subjectArchitecture = ( + report: ReturnType, +) => { + if (report.artifact.format === "mach-o-universal") return undefined; + if (report.slices.length !== 1) return undefined; + const name = report.slices[0]?.architecture; + return BINARY_ARCHITECTURES.find((supported) => supported === name); +}; + +const locationsFor = ( + report: ReturnType, +) => [ + { kind: "artifact-path" as const, path: report.artifact.selected_path }, + ...(report.artifact.resolved_path === report.artifact.selected_path + ? [] + : [ + { + kind: "artifact-path" as const, + path: report.artifact.resolved_path, + }, + ]), + ...report.slices.flatMap((slice) => + slice.sentinels.map((sentinel) => ({ + kind: "file-offset" as const, + offset: sentinel.sentinel_offset, + })), + ), +]; diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index 3f558930b..37ee3e70b 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -1,3 +1,5 @@ +import { resolve } from "node:path"; + import { Cli, z } from "incur"; import { logCliCommand } from "../cliLogging.js"; @@ -12,6 +14,7 @@ import { projectInputIssues } from "../domain/inputIssueProjection.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { Logger } from "pino"; import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { createElectronFuseService } from "../composition/electronFuses.js"; import { parseCliJsonInput, resolveCliJsonPaths } from "../cliJsonInput.js"; import { electronPageInspectionOptions, @@ -27,11 +30,40 @@ export const registerElectronCommands = ( environment: Readonly>, ): void => { registerElectronObservationCommands(cli, logger); + registerElectronFuseCommand(cli, logger); registerElectronActiveCommand(cli, logger, environment); registerJavaScriptApplicationCommand(cli, logger); registerJavaScriptRuntimeReconciliationCommand(cli, logger); }; +const registerElectronFuseCommand = ( + cli: ReturnType, + logger: Logger, +): void => { + const service = createElectronFuseService(); + cli.command(CLI_COMMANDS.inspectElectronFuses, { + description: + "Inspect packaged Electron fuse bytes without launching the application", + args: z.object({ + path: z + .string() + .describe( + "Local Electron .app, Electron Framework binary, PE executable, or ELF executable", + ), + }), + run: ({ args }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectElectronFuses, async () => { + const result = await service.inspect( + { path: resolve(args.path) }, + { signal }, + ); + return result.ok ? result.value : projectAnalysisError(result.error); + }), + ), + }); +}; + const registerElectronActiveCommand = ( cli: ReturnType, logger: Logger, diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 798451158..8bbca596b 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -138,6 +138,7 @@ export const CLI_COMMANDS = Object.freeze({ captureBrowserScenario: "capture-browser-scenario", listElectronTargets: "list-electron-targets", inspectElectronPage: "inspect-electron-page", + inspectElectronFuses: "inspect-electron-fuses", captureElectronScenario: "capture-electron-scenario", listJavaScriptRuntimeTargets: "list-javascript-runtime-targets", observeJavaScriptRuntime: "observe-javascript-runtime", diff --git a/src/composition/electronFuses.ts b/src/composition/electronFuses.ts new file mode 100644 index 000000000..f74c436c0 --- /dev/null +++ b/src/composition/electronFuses.ts @@ -0,0 +1,6 @@ +import { ElectronFuseService } from "../application/electron/ElectronFuseService.js"; +import { ElectronFuseProvider } from "../electron/ElectronFuseProvider.js"; + +/** Compose read-only fuse inspection without a runtime, network, or fuse edit. */ +export const createElectronFuseService = (): ElectronFuseService => + new ElectronFuseService(new ElectronFuseProvider()); diff --git a/src/contracts/javascript/electronToolContracts.ts b/src/contracts/javascript/electronToolContracts.ts index f7a9ed732..293b20b68 100644 --- a/src/contracts/javascript/electronToolContracts.ts +++ b/src/contracts/javascript/electronToolContracts.ts @@ -1,6 +1,8 @@ import type { ToolContract } from "../toolContractTypes.js"; import { toolContractMetadata } from "../toolEffects.js"; +import { electronFuseOutputSchemas } from "../toolOutputSchemaGroups.js"; import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; +import { inspectElectronFusesInputSchema } from "../../domain/electron/electronFuses.js"; import { electronPageInspectionSchema, electronTargetListSchema, @@ -147,4 +149,19 @@ export const ELECTRON_TOOL_CONTRACTS = [ { title: "Exercise an owned Electron application", input: activeExample }, ], }, + { + name: "inspect_electron_fuses", + ...toolContractMetadata("inspect_electron_fuses"), + kind: "artifact-provider", + description: + "Inspect packaged Electron fuse bytes in one local macOS app bundle, Electron Framework binary, Windows PE executable, or Linux ELF executable. Returns the artifact digest, per-slice wire version, fuse index, version-1 name, state, and raw byte, plus derived implications for Inspector flags, ELECTRON_RUN_AS_NODE, embedded ASAR integrity, and app.asar-only loading. A missing sentinel is unknown fuse state. Reads artifact bytes only and does not modify the selected file or open a runtime target.", + inputSchema: inspectElectronFusesInputSchema, + outputSchema: electronFuseOutputSchemas.inspect_electron_fuses, + examples: [ + { + title: "Read packaged Electron fuse state", + input: { path: "/Applications/Example.app" }, + }, + ], + }, ] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index a010467be..7fe152e9b 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -648,6 +648,7 @@ export const TOOL_EFFECTS: Readonly> = { list_javascript_runtime_targets: browserEvidence, observe_javascript_runtime: browserEvidence, analyze_javascript_application: evidence, + inspect_electron_fuses: evidence, export_web_scripts: effects({ mutatesSession: true, writesFilesystem: true, diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index bbadef367..9263e5ba4 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -88,6 +88,7 @@ import { } from "./toolOutputSchemaPrimitives.js"; import { processCaptureSchema } from "../domain/process/processCapture.js"; +import { electronFuseReportSchema } from "../domain/electron/electronFuses.js"; const contextFacetSchema = z.discriminatedUnion("state", [ z.object({ state: z.literal("available"), value: jsonValueSchema }), z.object({ @@ -584,3 +585,8 @@ export const sessionOutputSchemas = { }), ), } satisfies Readonly>; + +/** Exact Evidence schema for read-only Electron fuse inspection. */ +export const electronFuseOutputSchemas = { + inspect_electron_fuses: evidenceResultOf(electronFuseReportSchema), +}; diff --git a/src/domain/electron/electronFuses.test.ts b/src/domain/electron/electronFuses.test.ts new file mode 100644 index 000000000..c074ae176 --- /dev/null +++ b/src/domain/electron/electronFuses.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, it } from "vitest"; + +import { + describeElectronFuseFindings, + electronFuseName, + electronFuseReportSchema, + electronFuseState, + ELECTRON_FUSE_SENTINEL, + ELECTRON_FUSE_STATE_BYTES, + ELECTRON_FUSE_V1_NAMES, + type ElectronFuseSlice, +} from "./electronFuses.js"; + +const artifact = { + selected_path: "/artifacts/Electron", + resolved_path: "/artifacts/Electron", + sha256: "a".repeat(64), + bytes: 128, + format: "mach-o" as const, +}; + +const sentinel = ( + offset: number, + fuses: readonly number[], + options?: { + readonly version?: number | null; + readonly declared?: number | null; + readonly complete?: boolean; + }, +) => ({ + sentinel_offset: offset, + wire_version: options?.version === undefined ? 1 : options.version, + declared_length: + options?.declared === undefined ? fuses.length : options.declared, + fuses: fuses.map((raw, index) => ({ + index, + name: electronFuseName( + options?.version === undefined ? 1 : (options.version ?? 1), + index, + ), + state: electronFuseState(raw), + raw_byte: raw, + })), + complete: options?.complete ?? true, +}); + +const slice = ( + sentinels: ReturnType[], + index = 0, +): ElectronFuseSlice => ({ + index, + file_offset: 0, + size: 128, + architecture: "arm64", + sentinel_count: sentinels.length, + sentinels, +}); + +it("validates implications across repeated FAT slices and rejects a false claim", () => { + const slices = Array.from({ length: 128 }, (_, index) => ({ + ...slice( + Array.from({ length: 32 }, (_, offset) => + sentinel( + offset * 64, + Array.from({ length: 8 }, () => ELECTRON_FUSE_STATE_BYTES.enabled), + ), + ), + index, + ), + size: 4096, + })); + const report = { + artifact: { ...artifact, bytes: 4096, format: "mach-o-universal" as const }, + slices, + ...describeElectronFuseFindings(slices, "mach-o-universal"), + }; + expect(electronFuseReportSchema.safeParse(report).success).toBe(true); + const last = report.runtime_implications.at(-1); + expect(last).toBeDefined(); + expect( + electronFuseReportSchema.safeParse({ + ...report, + runtime_implications: [{ ...last, outcome: "unknown" }], + }).success, + ).toBe(false); +}); + +describe("Electron fuse wire decoding", () => { + it("keeps the 32-byte sentinel and version-1 names pinned", () => { + expect(ELECTRON_FUSE_SENTINEL).toHaveLength(32); + expect(ELECTRON_FUSE_V1_NAMES[3]).toBe("EnableNodeCliInspectArguments"); + expect(electronFuseName(1, 8)).toBe("WasmTrapHandlers"); + expect(electronFuseName(1, 9)).toBeNull(); + expect(electronFuseName(2, 0)).toBeNull(); + }); + + it("preserves disabled, enabled, removed, and unknown bytes", () => { + expect(electronFuseState(ELECTRON_FUSE_STATE_BYTES.disabled)).toBe( + "disabled", + ); + expect(electronFuseState(ELECTRON_FUSE_STATE_BYTES.enabled)).toBe( + "enabled", + ); + expect(electronFuseState(ELECTRON_FUSE_STATE_BYTES.removed)).toBe( + "removed", + ); + expect(electronFuseState(0x99)).toBe("unknown"); + }); + + it("derives runtime implications from the fuse index and leaves unknowns unknown", () => { + const fuses = [ + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.enabled, + 0x99, + ELECTRON_FUSE_STATE_BYTES.removed, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.enabled, + ]; + const report = electronFuseReportSchema.parse({ + artifact, + slices: [slice([sentinel(8, fuses)])], + ...describeElectronFuseFindings([slice([sentinel(8, fuses)])], "mach-o"), + }); + expect(report.runtime_implications).toEqual([ + expect.objectContaining({ + subject: "node_cli_inspect_arguments", + fuse_index: 3, + outcome: "not_honored", + relationship: "derivation", + }), + expect.objectContaining({ + subject: "electron_run_as_node", + fuse_index: 0, + outcome: "not_honored", + }), + expect.objectContaining({ + subject: "embedded_asar_integrity_validation", + fuse_index: 4, + outcome: "not_enforced", + }), + expect.objectContaining({ + subject: "only_load_app_from_asar", + fuse_index: 5, + outcome: "restricted_to_app_asar", + }), + ]); + expect(report.slices[0] !== undefined && "fuses" in report.slices[0]).toBe( + false, + ); + expect(report.slices[0]?.sentinels[0]?.fuses[2]).toMatchObject({ + index: 2, + name: "EnableNodeOptionsEnvironmentVariable", + state: "unknown", + raw_byte: 0x99, + }); + }); + + it("does not treat a missing sentinel or a short wire as disabled fuses", () => { + const missing = describeElectronFuseFindings([slice([])], "mach-o"); + expect(missing.runtime_implications).toEqual([]); + expect(missing.limitations.join(" ")).toContain( + "unknown, not that every fuse is disabled", + ); + expect(missing.limitations.join(" ")).toContain( + "fuse state for that slice is unknown, not disabled", + ); + const truncated = describeElectronFuseFindings( + [ + slice([ + sentinel(8, [ELECTRON_FUSE_STATE_BYTES.enabled], { + declared: 4, + complete: false, + }), + ]), + ], + "mach-o", + ); + expect(truncated.runtime_implications).toEqual([ + expect.objectContaining({ + subject: "electron_run_as_node", + outcome: "honored", + }), + ]); + expect(truncated.limitations.join(" ")).toContain("truncated"); + expect(truncated.limitations.join(" ")).toContain("fuse index 3"); + }); + + it("reports every sentinel and skips implications for an unpinned wire version", () => { + const findings = describeElectronFuseFindings( + [ + slice([ + sentinel(8, [ELECTRON_FUSE_STATE_BYTES.enabled], { version: 2 }), + sentinel( + 40, + Array.from({ length: 6 }, () => ELECTRON_FUSE_STATE_BYTES.disabled), + ), + ]), + ], + "mach-o", + ); + expect(findings.runtime_implications).toHaveLength(4); + expect(findings.limitations.join(" ")).toContain("none is selected"); + expect(findings.limitations.join(" ")).toContain("wire version 2"); + expect( + findings.runtime_implications.every( + (item) => item.sentinel_offset === 40, + ), + ).toBe(true); + }); +}); + +describe("ELF embedded ASAR integrity applicability", () => { + it("does not claim ASAR integrity enforcement for an enabled fuse on ELF", () => { + const fuses = [ + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.enabled, + ]; + const slices = [slice([sentinel(4, fuses)])]; + const elf = describeElectronFuseFindings(slices, "elf"); + expect(slices[0]?.sentinels[0]?.fuses[4]).toMatchObject({ + index: 4, + name: "EnableEmbeddedAsarIntegrityValidation", + state: "enabled", + raw_byte: ELECTRON_FUSE_STATE_BYTES.enabled, + }); + expect( + elf.runtime_implications.find( + (item) => item.subject === "embedded_asar_integrity_validation", + ), + ).toMatchObject({ + relationship: "derivation", + fuse_index: 4, + outcome: "unknown", + }); + expect(elf.limitations.join(" ")).toContain("only on macOS and Windows"); + expect( + describeElectronFuseFindings(slices, "mach-o").runtime_implications.find( + (item) => item.subject === "embedded_asar_integrity_validation", + ), + ).toMatchObject({ outcome: "enforced" }); + expect( + describeElectronFuseFindings(slices, "pe").runtime_implications.find( + (item) => item.subject === "embedded_asar_integrity_validation", + ), + ).toMatchObject({ outcome: "enforced" }); + expect( + describeElectronFuseFindings( + slices, + "mach-o-universal", + ).runtime_implications.find( + (item) => item.subject === "embedded_asar_integrity_validation", + ), + ).toMatchObject({ outcome: "enforced" }); + }); +}); + +it("rejects claimed wire bytes outside their slice and unsupported derivations", () => { + const slices = [slice([sentinel(4, [ELECTRON_FUSE_STATE_BYTES.disabled])])]; + const findings = describeElectronFuseFindings(slices, "mach-o"); + const report = { artifact, slices, ...findings }; + expect(electronFuseReportSchema.safeParse(report).success).toBe(true); + expect( + electronFuseReportSchema.safeParse({ + ...report, + artifact: { ...artifact, bytes: 36 }, + slices: [{ ...slices[0], size: 36 }], + }).success, + ).toBe(false); + expect( + electronFuseReportSchema.safeParse({ + ...report, + runtime_implications: findings.runtime_implications.map((fact) => ({ + ...fact, + outcome: "honored", + })), + }).success, + ).toBe(false); + expect( + electronFuseReportSchema.safeParse({ + ...report, + runtime_implications: [ + { + relationship: "derivation", + subject: "node_cli_inspect_arguments", + fuse_index: 3, + outcome: "honored", + slice_index: 0, + sentinel_offset: 4, + }, + ], + }).success, + ).toBe(false); +}); diff --git a/src/domain/electron/electronFuses.ts b/src/domain/electron/electronFuses.ts new file mode 100644 index 000000000..ffacd4481 --- /dev/null +++ b/src/domain/electron/electronFuses.ts @@ -0,0 +1,477 @@ +import { z } from "zod"; + +import { digestSchema } from "../digests.js"; +import { localPathStringSchema } from "../localPath.js"; + +/** ASCII sentinel that introduces an Electron fuse wire in an Electron binary. */ +export const ELECTRON_FUSE_SENTINEL = "dL7pKGdnNz796PbbjQWNKmHXBZaB9tsX"; + +/** Wire bytes from the Electron fuse wire: disabled, enabled, and removed. */ +export const ELECTRON_FUSE_STATE_BYTES = { + disabled: 0x30, + enabled: 0x31, + removed: 0x72, +} as const; + +/** + * FuseV1 index order. An index past this table, or any index on another wire + * version, keeps a null name and its raw byte. + */ +export const ELECTRON_FUSE_V1_NAMES = [ + "RunAsNode", + "EnableCookieEncryption", + "EnableNodeOptionsEnvironmentVariable", + "EnableNodeCliInspectArguments", + "EnableEmbeddedAsarIntegrityValidation", + "OnlyLoadAppFromAsar", + "LoadBrowserProcessSpecificV8Snapshot", + "GrantFileProtocolExtraPrivileges", + "WasmTrapHandlers", +] as const; + +const FUSE_IMPLICATIONS = [ + { + subject: "node_cli_inspect_arguments", + fuseIndex: 3, + enabled: "honored", + inactive: "not_honored", + }, + { + subject: "electron_run_as_node", + fuseIndex: 0, + enabled: "honored", + inactive: "not_honored", + }, + { + subject: "embedded_asar_integrity_validation", + fuseIndex: 4, + enabled: "enforced", + inactive: "not_enforced", + }, + { + subject: "only_load_app_from_asar", + fuseIndex: 5, + enabled: "restricted_to_app_asar", + inactive: "not_restricted_to_app_asar", + }, +] as const; + +const OTHER_MEANS_LIMITATION = + "Fuse state does not prove the application enables Inspector through other means, such as app.commandLine or a remote-debugging switch in application code."; +const MISSING_SENTINEL_LIMITATION = + "A missing sentinel means fuse state is unknown, not that every fuse is disabled."; + +const byteSchema = z.number().int().min(0).max(255); +const fuseStateSchema = z.enum(["enabled", "disabled", "removed", "unknown"]); + +/** One fuse byte, with a version-1 name when the pinned table covers its index. */ +export const electronFuseSchema = z.strictObject({ + index: z.number().int().nonnegative(), + name: z.string().min(1).nullable(), + state: fuseStateSchema, + raw_byte: byteSchema, +}); + +const electronFuseSentinelSchema = z.strictObject({ + sentinel_offset: z.number().int().nonnegative(), + wire_version: byteSchema.nullable(), + declared_length: byteSchema.nullable(), + fuses: z.array(electronFuseSchema), + complete: z.boolean(), +}); + +const electronFuseSliceSchema = z.strictObject({ + index: z.number().int().nonnegative(), + file_offset: z.number().int().nonnegative(), + size: z.number().int().nonnegative(), + architecture: z.string().min(1).nullable(), + sentinel_count: z.number().int().nonnegative(), + sentinels: z.array(electronFuseSentinelSchema), +}); + +const implicationLocation = { + relationship: z.literal("derivation"), + slice_index: z.number().int().nonnegative(), + sentinel_offset: z.number().int().nonnegative(), +}; + +/** Derived runtime meaning of one observed fuse byte. The fuse bytes stay observed. */ +export const electronFuseImplicationSchema = z.discriminatedUnion("subject", [ + z.strictObject({ + ...implicationLocation, + subject: z.literal("node_cli_inspect_arguments"), + fuse_index: z.literal(3), + outcome: z.enum(["honored", "not_honored", "unknown"]), + }), + z.strictObject({ + ...implicationLocation, + subject: z.literal("electron_run_as_node"), + fuse_index: z.literal(0), + outcome: z.enum(["honored", "not_honored", "unknown"]), + }), + z.strictObject({ + ...implicationLocation, + subject: z.literal("embedded_asar_integrity_validation"), + fuse_index: z.literal(4), + outcome: z.enum(["enforced", "not_enforced", "unknown"]), + }), + z.strictObject({ + ...implicationLocation, + subject: z.literal("only_load_app_from_asar"), + fuse_index: z.literal(5), + outcome: z.enum([ + "restricted_to_app_asar", + "not_restricted_to_app_asar", + "unknown", + ]), + }), +]); + +/** Inline fuse observation for one selected app bundle or Electron executable. */ +export const electronFuseReportSchema = z + .strictObject({ + artifact: z + .strictObject({ + selected_path: localPathStringSchema, + resolved_path: localPathStringSchema, + sha256: digestSchema, + bytes: z.number().int().nonnegative(), + format: z.enum(["mach-o", "mach-o-universal", "pe", "elf"]), + }) + .describe( + "Caller-selected path, resolved regular file, and the container that was read", + ), + slices: z + .array(electronFuseSliceSchema) + .describe( + "One entry per universal Mach-O slice, or a single entry for a thin Mach-O, PE, or ELF file. Sentinel offsets are file-absolute.", + ), + runtime_implications: z + .array(electronFuseImplicationSchema) + .describe( + "Derivations from version-1 fuse bytes. Each entry names the fuse index it came from.", + ), + limitations: z.array(z.string().min(1)), + }) + .superRefine((report, context) => { + validateReportBounds(report, context); + }); + +export const inspectElectronFusesInputSchema = z.strictObject({ + path: localPathStringSchema.describe( + "Absolute filesystem path to a macOS .app bundle, Electron Framework binary, Windows PE executable, or Linux ELF executable", + ), +}); + +export type ElectronFuseState = z.infer; +export type ElectronFuseSentinel = z.infer; +export type ElectronFuseSlice = z.infer; +export type ElectronFuseImplication = z.infer< + typeof electronFuseImplicationSchema +>; +export type ElectronFuseReport = z.infer; +export type InspectElectronFusesInput = z.infer< + typeof inspectElectronFusesInputSchema +>; + +/** Decode one fuse wire byte. Any other value stays unknown and keeps the raw byte. */ +export const electronFuseState = (byte: number): ElectronFuseState => { + if (byte === ELECTRON_FUSE_STATE_BYTES.disabled) return "disabled"; + if (byte === ELECTRON_FUSE_STATE_BYTES.enabled) return "enabled"; + if (byte === ELECTRON_FUSE_STATE_BYTES.removed) return "removed"; + return "unknown"; +}; + +/** Pinned version-1 name, or null when this wire version or index has no table entry. */ +export const electronFuseName = ( + wireVersion: number, + index: number, +): string | null => + wireVersion === 1 ? (ELECTRON_FUSE_V1_NAMES[index] ?? null) : null; + +/** + * Attach derivations and reading limitations without choosing among multiple + * sentinels. Embedded ASAR integrity is compiled only for Mach-O and PE, so an + * ELF byte stays observed and its enforcement outcome stays unknown. + */ +export const describeElectronFuseFindings = ( + slices: readonly ElectronFuseSlice[], + container: ElectronFuseReport["artifact"]["format"], +): { + readonly runtime_implications: ElectronFuseImplication[]; + readonly limitations: string[]; +} => { + const runtime_implications: ElectronFuseImplication[] = []; + const limitations = [OTHER_MEANS_LIMITATION, MISSING_SENTINEL_LIMITATION]; + for (const slice of slices) { + if (slice.sentinel_count === 0) + limitations.push( + `Slice ${slice.index} at file offset ${slice.file_offset} contains no Electron fuse sentinel; fuse state for that slice is unknown, not disabled.`, + ); + else if (slice.sentinel_count > 1) + limitations.push( + `Slice ${slice.index} at file offset ${slice.file_offset} contains ${slice.sentinel_count} Electron fuse sentinels; every sentinel is reported and none is selected.`, + ); + for (const sentinel of slice.sentinels) + describeSentinel( + slice.index, + sentinel, + container, + runtime_implications, + limitations, + ); + } + return { runtime_implications, limitations }; +}; + +/** Electron compiles this validation for macOS (Mach-O) and Windows (PE) only. */ +const embeddedAsarIntegrityApplies = ( + container: ElectronFuseReport["artifact"]["format"], +): boolean => + container === "mach-o" || + container === "mach-o-universal" || + container === "pe"; + +const describeSentinel = ( + sliceIndex: number, + sentinel: ElectronFuseSentinel, + container: ElectronFuseReport["artifact"]["format"], + implications: ElectronFuseImplication[], + limitations: string[], +): void => { + if (!sentinel.complete) + limitations.push( + `Electron fuse wire at file offset ${sentinel.sentinel_offset} is truncated; fuse bytes past the slice stay unknown.`, + ); + if (sentinel.wire_version === null) return; + if (sentinel.wire_version !== 1) { + limitations.push( + `Electron fuse wire at file offset ${sentinel.sentinel_offset} uses wire version ${sentinel.wire_version}; version 1 is the only pinned name table, so runtime implications for this wire are unknown.`, + ); + return; + } + for (const spec of FUSE_IMPLICATIONS) { + const fuse = sentinel.fuses.find((item) => item.index === spec.fuseIndex); + if (fuse === undefined) { + limitations.push( + `Electron fuse wire at file offset ${sentinel.sentinel_offset} does not include fuse index ${spec.fuseIndex}; that runtime implication is unknown.`, + ); + continue; + } + const applicabilityUnknown = + spec.subject === "embedded_asar_integrity_validation" && + !embeddedAsarIntegrityApplies(container); + if (applicabilityUnknown) + limitations.push( + `Electron compiles embedded ASAR integrity validation only on macOS and Windows; the observed fuse byte at file offset ${sentinel.sentinel_offset} does not establish enforcement.`, + ); + implications.push( + implicationFor(spec, applicabilityUnknown ? "unknown" : fuse.state, { + slice_index: sliceIndex, + sentinel_offset: sentinel.sentinel_offset, + }), + ); + } +}; + +const implicationFor = ( + spec: (typeof FUSE_IMPLICATIONS)[number], + state: ElectronFuseState, + location: { + readonly slice_index: number; + readonly sentinel_offset: number; + }, +): ElectronFuseImplication => { + const outcome = + state === "enabled" + ? spec.enabled + : state === "unknown" + ? "unknown" + : spec.inactive; + return { + relationship: "derivation", + subject: spec.subject, + fuse_index: spec.fuseIndex, + outcome, + ...location, + } as ElectronFuseImplication; +}; + +const validateReportBounds = ( + report: ElectronFuseReport, + context: z.RefinementCtx, +): void => { + const size = report.artifact.bytes; + for (const [sliceIndex, slice] of report.slices.entries()) { + if (slice.index !== sliceIndex) + context.addIssue({ + code: "custom", + path: ["slices", sliceIndex, "index"], + message: "Slice index does not match its position in the report.", + }); + if (slice.sentinel_count !== slice.sentinels.length) + context.addIssue({ + code: "custom", + path: ["slices", sliceIndex, "sentinel_count"], + message: "Sentinel count does not match the reported sentinels.", + }); + if (slice.file_offset > size || slice.size > size - slice.file_offset) + context.addIssue({ + code: "custom", + path: ["slices", sliceIndex], + message: "Slice range extends outside the selected artifact.", + }); + for (const [sentinelIndex, sentinel] of slice.sentinels.entries()) + validateSentinel(slice, sentinel, sliceIndex, sentinelIndex, context); + } + const expected = new Set( + describeElectronFuseFindings( + report.slices, + report.artifact.format, + ).runtime_implications.map(implicationKey), + ); + for (const [index, implication] of report.runtime_implications.entries()) { + if (!expected.has(implicationKey(implication))) + context.addIssue({ + code: "custom", + path: ["runtime_implications", index], + message: + "Runtime implication is not supported by the reported fuse bytes and applicability.", + }); + const slice = report.slices[implication.slice_index]; + if ( + slice?.sentinels.some( + (sentinel) => sentinel.sentinel_offset === implication.sentinel_offset, + ) !== true + ) + context.addIssue({ + code: "custom", + path: ["runtime_implications", index], + message: "Runtime implication does not reference a reported sentinel.", + }); + } +}; + +const implicationKey = (fact: ElectronFuseImplication): string => + JSON.stringify([ + fact.slice_index, + fact.sentinel_offset, + fact.subject, + fact.fuse_index, + fact.outcome, + ]); + +const validateSentinel = ( + slice: ElectronFuseSlice, + sentinel: ElectronFuseSentinel, + sliceIndex: number, + sentinelIndex: number, + context: z.RefinementCtx, +): void => { + const sentinelEnd = sentinel.sentinel_offset + ELECTRON_FUSE_SENTINEL.length; + const wireEnd = + sentinelEnd + + (sentinel.wire_version === null ? 0 : 1) + + (sentinel.declared_length === null ? 0 : 1) + + sentinel.fuses.length; + if (wireEnd > slice.file_offset + slice.size) + context.addIssue({ + code: "custom", + path: ["slices", sliceIndex, "sentinels", sentinelIndex], + message: "Reported wire header or fuse bytes extend outside the slice.", + }); + if ( + sentinel.sentinel_offset < slice.file_offset || + sentinelEnd > slice.file_offset + slice.size + ) + context.addIssue({ + code: "custom", + path: [ + "slices", + sliceIndex, + "sentinels", + sentinelIndex, + "sentinel_offset", + ], + message: "Sentinel is outside its slice.", + }); + if (sentinel.wire_version === null && sentinel.declared_length !== null) + context.addIssue({ + code: "custom", + path: [ + "slices", + sliceIndex, + "sentinels", + sentinelIndex, + "declared_length", + ], + message: "A wire without a version byte cannot declare a fuse count.", + }); + if ( + sentinel.complete && + (sentinel.declared_length === null || + sentinel.fuses.length !== sentinel.declared_length) + ) + context.addIssue({ + code: "custom", + path: ["slices", sliceIndex, "sentinels", sentinelIndex, "complete"], + message: "A complete wire must contain every declared fuse byte.", + }); + if ( + !sentinel.complete && + sentinel.declared_length !== null && + sentinel.fuses.length >= sentinel.declared_length + ) + context.addIssue({ + code: "custom", + path: ["slices", sliceIndex, "sentinels", sentinelIndex, "complete"], + message: "An incomplete wire must omit at least one declared fuse byte.", + }); + for (const [fuseIndex, fuse] of sentinel.fuses.entries()) { + if (fuse.index !== fuseIndex) + context.addIssue({ + code: "custom", + path: [ + "slices", + sliceIndex, + "sentinels", + sentinelIndex, + "fuses", + fuseIndex, + "index", + ], + message: "Fuse index does not match its wire position.", + }); + if (fuse.state !== electronFuseState(fuse.raw_byte)) + context.addIssue({ + code: "custom", + path: [ + "slices", + sliceIndex, + "sentinels", + sentinelIndex, + "fuses", + fuseIndex, + "state", + ], + message: "Fuse state does not match its raw byte.", + }); + if ( + sentinel.wire_version !== null && + fuse.name !== electronFuseName(sentinel.wire_version, fuse.index) + ) + context.addIssue({ + code: "custom", + path: [ + "slices", + sliceIndex, + "sentinels", + sentinelIndex, + "fuses", + fuseIndex, + "name", + ], + message: "Fuse name does not match the pinned version-1 table.", + }); + } +}; diff --git a/src/electron/ElectronFusePath.ts b/src/electron/ElectronFusePath.ts new file mode 100644 index 000000000..399e001f5 --- /dev/null +++ b/src/electron/ElectronFusePath.ts @@ -0,0 +1,307 @@ +import type { BigIntStats } from "node:fs"; +import { lstat, readlink } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, sep } from "node:path"; + +import { isPathWithinRoot } from "../domain/localPath.js"; + +const FRAMEWORK_NAME = "Electron Framework.framework"; +const BINARY_NAME = "Electron Framework"; +const MAX_SYMLINK_HOPS = 32; + +/** Filesystem facts owned by path admission and consumed unchanged by the reader. */ +export interface ElectronFusePathNode { + readonly path: string; + readonly metadata: BigIntStats; +} +export interface ResolvedElectronFuseFile { + readonly path: string; + readonly metadata: BigIntStats; + readonly nodes: readonly ElectronFusePathNode[]; +} +export interface ResolvedElectronFuseTarget { + readonly selectedPath: string; + readonly file: ResolvedElectronFuseFile; +} + +/** A selected path is not an admitted Electron bundle or regular file. */ +export class ElectronFusePathFailure extends Error { + constructor( + readonly reason: "invalid_value" | "invalid_format", + message: string, + ) { + super(message); + this.name = "ElectronFusePathFailure"; + } +} + +type WalkResult = + | ({ readonly kind: "file" } & ResolvedElectronFuseFile) + | { readonly kind: "absent" | "escapes" | "not-file" | "loop" }; + +/** + * Admit a regular executable, a macOS app bundle, or an Electron Framework + * binary reached through `Versions/Current` without leaving the framework. + */ +export const resolveElectronFuseTarget = async ( + selectedPath: string, + signal?: AbortSignal, +): Promise => { + signal?.throwIfAborted(); + const metadata = await lstat(selectedPath, { bigint: true }); + let file: ResolvedElectronFuseFile; + if (metadata.isSymbolicLink()) + file = await resolveFrameworkSymlink(selectedPath, signal); + else if (metadata.isFile()) + file = + frameworkRootOf(selectedPath) === null + ? { path: selectedPath, metadata, nodes: [] } + : await resolveFrameworkSymlink(selectedPath, signal); + else if (metadata.isDirectory()) + file = await resolveBundleBinary(selectedPath, signal); + else + throw new ElectronFusePathFailure( + "invalid_format", + `Expected a regular file or Electron app bundle: ${selectedPath}`, + ); + return { selectedPath, file }; +}; + +const resolveFrameworkSymlink = async ( + selectedPath: string, + signal?: AbortSignal, +): Promise => { + const root = frameworkRootOf(selectedPath); + if (root === null) + throw new ElectronFusePathFailure( + "invalid_format", + `Expected a regular file without a symlink: ${selectedPath}`, + ); + const nodes = await requireFrameworkRoot(root, selectedPath); + const relativePath = relativeInside(root, selectedPath); + if (relativePath === null) + throw new ElectronFusePathFailure( + "invalid_value", + `Framework symlink escapes ${FRAMEWORK_NAME}: ${selectedPath}`, + ); + return requireFile( + await resolveInside(root, relativePath, signal), + selectedPath, + nodes, + ); +}; + +const resolveBundleBinary = async ( + selectedPath: string, + signal?: AbortSignal, +): Promise => { + const name = basename(selectedPath); + if (name === FRAMEWORK_NAME) + return resolveKnownBinary(selectedPath, selectedPath, signal); + if (name.endsWith(".app")) { + return resolveKnownBinary( + join(selectedPath, "Contents", "Frameworks", FRAMEWORK_NAME), + selectedPath, + signal, + ); + } + throw new ElectronFusePathFailure( + "invalid_format", + `Expected an Electron .app bundle, ${FRAMEWORK_NAME} directory, or executable file: ${selectedPath}`, + ); +}; + +/** `root` is the framework directory for both an `.app` and a framework selection. */ +const resolveKnownBinary = async ( + root: string, + bundle: string, + signal: AbortSignal | undefined, +): Promise => { + const nodes = await requireFrameworkRoot(root, bundle); + for (const candidate of [ + BINARY_NAME, + `Versions/Current/${BINARY_NAME}`, + `Versions/A/${BINARY_NAME}`, + ]) { + const selected = await resolveInside(root, candidate, signal); + if (selected.kind === "file") return requireFile(selected, bundle, nodes); + if (selected.kind !== "absent") + throw pathWalkFailure(selected.kind, bundle); + } + throw new ElectronFusePathFailure( + "invalid_value", + `Electron Framework binary was not found inside ${bundle}.`, + ); +}; + +const resolveInside = async ( + root: string, + relativePath: string, + signal?: AbortSignal, +): Promise => { + const pending = relativePath.split("/").filter((segment) => segment !== ""); + const resolved: string[] = []; + const nodes: ElectronFusePathNode[] = []; + let fileMetadata: BigIntStats | undefined; + let hops = 0; + let last: "file" | "directory" | undefined; + while (pending.length > 0) { + signal?.throwIfAborted(); + const segment = pending.shift() ?? ""; + if (segment === ".") continue; + if (segment === "..") { + if (resolved.length === 0) return { kind: "escapes" }; + resolved.pop(); + last = "directory"; + continue; + } + const absolute = join(root, ...resolved, segment); + if (!isPathWithinRoot(root, absolute)) return { kind: "escapes" }; + let metadata; + try { + metadata = await lstat(absolute, { bigint: true }); + } catch (cause: unknown) { + if (isMissing(cause)) return { kind: "absent" }; + throw cause; + } + if (metadata.isSymbolicLink()) { + nodes.push({ path: absolute, metadata }); + hops += 1; + if (hops > MAX_SYMLINK_HOPS) return { kind: "loop" }; + const target = await readlink(absolute); + if (isAbsolute(target)) return { kind: "escapes" }; + pending.unshift(...target.split("/").filter((part) => part !== "")); + continue; + } + if (metadata.isDirectory()) { + nodes.push({ path: absolute, metadata }); + resolved.push(segment); + last = "directory"; + continue; + } + if (!metadata.isFile() || pending.length > 0) return { kind: "not-file" }; + resolved.push(segment); + fileMetadata = metadata; + last = "file"; + } + if (last !== "file" || fileMetadata === undefined) + return { kind: "not-file" }; + const path = join(root, ...resolved); + return isPathWithinRoot(root, path) + ? { kind: "file", path, metadata: fileMetadata, nodes } + : { kind: "escapes" }; +}; + +/** Validate the app-parent chain equally for app, framework, and binary selection. */ +const requireFrameworkRoot = async ( + root: string, + selectedPath: string, +): Promise => { + const frameworks = dirname(root); + const contents = dirname(frameworks); + const app = dirname(contents); + if ( + basename(frameworks) === "Frameworks" && + basename(contents) === "Contents" && + basename(app).endsWith(".app") + ) + return requireRealDirectories( + dirname(app), + [basename(app), "Contents", "Frameworks", FRAMEWORK_NAME], + selectedPath, + ); + else + return requireRealDirectories( + dirname(root), + [basename(root)], + selectedPath, + ); +}; + +/** + * Each named directory below `anchor` must be a real directory, so the + * framework root cannot redirect the walk outside the selected bundle. + */ +const requireRealDirectories = async ( + anchor: string, + segments: readonly string[], + selectedPath: string, +): Promise => { + let current = anchor; + const nodes: ElectronFusePathNode[] = []; + for (const segment of segments) { + current = join(current, segment); + let metadata; + try { + metadata = await lstat(current, { bigint: true }); + } catch (cause: unknown) { + if (isMissing(cause)) + throw new ElectronFusePathFailure( + "invalid_value", + `Electron Framework binary was not found inside ${selectedPath}.`, + ); + throw cause; + } + nodes.push({ path: current, metadata }); + if (metadata.isSymbolicLink()) + throw pathWalkFailure("escapes", selectedPath); + if (!metadata.isDirectory()) + throw pathWalkFailure("not-file", selectedPath); + } + return nodes; +}; + +const requireFile = ( + walk: WalkResult, + selectedPath: string, + nodes: readonly ElectronFusePathNode[], +): ResolvedElectronFuseFile => { + if (walk.kind === "file") + return { + path: walk.path, + metadata: walk.metadata, + nodes: [...nodes, ...walk.nodes], + }; + if (walk.kind === "absent") + throw new ElectronFusePathFailure( + "invalid_value", + `Electron Framework symlink target was not found: ${selectedPath}`, + ); + throw pathWalkFailure(walk.kind, selectedPath); +}; + +const pathWalkFailure = ( + kind: "escapes" | "not-file" | "loop", + path: string, +): ElectronFusePathFailure => + new ElectronFusePathFailure( + "invalid_value", + kind === "escapes" + ? `Framework symlink escapes ${FRAMEWORK_NAME}: ${path}` + : kind === "loop" + ? `Framework symlink hop limit exceeded: ${path}` + : `Electron Framework path is not a regular file: ${path}`, + ); + +const frameworkRootOf = (path: string): string | null => { + let current = dirname(path); + while (basename(current) !== FRAMEWORK_NAME) { + const parent = dirname(current); + if (parent === current) return null; + current = parent; + } + return current; +}; + +const relativeInside = (root: string, path: string): string | null => { + // Keep dot segments until resolveInside has followed preceding symlinks. + // Lexical normalization would select a different file for link/../ paths. + const prefix = root.endsWith(sep) ? root : `${root}${sep}`; + if (!path.startsWith(prefix)) return null; + const value = path.slice(prefix.length); + return value === "" ? null : value.split(sep).join("/"); +}; + +const isMissing = (cause: unknown): boolean => + cause instanceof Error && + "code" in cause && + ["ENOENT", "ENOTDIR"].includes(String(cause.code)); diff --git a/src/electron/ElectronFuseProvider.test.ts b/src/electron/ElectronFuseProvider.test.ts new file mode 100644 index 000000000..fa43f4787 --- /dev/null +++ b/src/electron/ElectronFuseProvider.test.ts @@ -0,0 +1,684 @@ +import { resolveElectronFuseTarget } from "./ElectronFusePath.js"; +import { RegularFileChangedError } from "../filesystem/RegularFile.js"; +import { mkdir, symlink, writeFile, truncate, rename } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { + elfFuseImage, + fuseWire, + FUSE_SENTINEL, + OBSIDIAN_FUSE_BYTES, + peFuseImage, + sentinelFreeMacho, + thinFuseImage, + thinMacho, + truncatedFuseImage, + twoSentinelElf, + universalFuseImage, +} from "../../tests/fixtures/electron/fuseImage.js"; +import { ELECTRON_FUSE_STATE_BYTES } from "../domain/electron/electronFuses.js"; +import { ElectronFuseProvider } from "./ElectronFuseProvider.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { readElectronFuseImage } from "./ElectronFuseReader.js"; + +const provider = new ElectronFuseProvider(); + +const inspect = async (path: string) => { + const result = await provider.inspect({ path }); + if (!result.ok) throw result.error; + return result.value; +}; + +describe("Electron fuse wire reading", () => { + it("reads thin Mach-O, universal Mach-O, PE, and ELF wires", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-"); + const thin = thinFuseImage(OBSIDIAN_FUSE_BYTES); + const universal = universalFuseImage(); + const pe = peFuseImage(OBSIDIAN_FUSE_BYTES); + const elf = elfFuseImage([ + ELECTRON_FUSE_STATE_BYTES.enabled, + ELECTRON_FUSE_STATE_BYTES.removed, + 0x99, + ]); + await writeFile(join(root, "thin"), thin.bytes); + await writeFile(join(root, "universal"), universal.bytes); + await writeFile(join(root, "app.exe"), pe.bytes); + await writeFile(join(root, "app.elf"), elf.bytes); + + const thinReport = await inspect(join(root, "thin")); + expect(thinReport.artifact).toMatchObject({ + format: "mach-o", + bytes: thin.bytes.length, + }); + expect(thinReport.slices).toMatchObject([ + { + architecture: "arm64", + sentinel_count: 1, + sentinels: [ + { + sentinel_offset: thin.sentinelOffset, + wire_version: 1, + declared_length: 8, + complete: true, + }, + ], + }, + ]); + expect( + thinReport.slices[0]?.sentinels[0]?.fuses.map((fuse) => fuse.state), + ).toEqual([ + "disabled", + "enabled", + "disabled", + "disabled", + "disabled", + "disabled", + "disabled", + "enabled", + ]); + + const universalReport = await inspect(join(root, "universal")); + expect(universalReport.artifact.format).toBe("mach-o-universal"); + expect(universalReport.slices).toMatchObject([ + { + file_offset: universal.left.fileOffset, + architecture: "x86_64", + sentinels: [{ sentinel_offset: universal.left.sentinelOffset }], + }, + { + file_offset: universal.right.fileOffset, + architecture: "arm64", + sentinels: [{ sentinel_offset: universal.right.sentinelOffset }], + }, + ]); + expect( + universalReport.runtime_implications.filter( + (item) => item.subject === "node_cli_inspect_arguments", + ), + ).toEqual([ + expect.objectContaining({ + slice_index: 0, + outcome: "not_honored", + sentinel_offset: universal.left.sentinelOffset, + }), + expect.objectContaining({ + slice_index: 1, + outcome: "honored", + sentinel_offset: universal.right.sentinelOffset, + }), + ]); + + const peReport = await inspect(join(root, "app.exe")); + expect(peReport.artifact.format).toBe("pe"); + expect(peReport.slices[0]?.sentinels[0]?.sentinel_offset).toBe( + pe.sentinelOffset, + ); + const elfReport = await inspect(join(root, "app.elf")); + expect(elfReport.slices[0]?.sentinels[0]?.fuses).toMatchObject([ + { name: "RunAsNode", state: "enabled", raw_byte: 0x31 }, + { name: "EnableCookieEncryption", state: "removed", raw_byte: 0x72 }, + { + name: "EnableNodeOptionsEnvironmentVariable", + state: "unknown", + raw_byte: 0x99, + }, + ]); + expect(elfReport.runtime_implications).toEqual([ + expect.objectContaining({ + subject: "electron_run_as_node", + outcome: "honored", + }), + ]); + }); + + it("keeps a truncated wire, duplicate sentinels, and a missing sentinel", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-partial-"); + await writeFile(join(root, "truncated"), truncatedFuseImage()); + const duplicated = twoSentinelElf(); + await writeFile(join(root, "duplicated"), duplicated.bytes); + await writeFile(join(root, "empty"), sentinelFreeMacho()); + + const truncated = await inspect(join(root, "truncated")); + expect(truncated.slices[0]?.sentinels[0]).toMatchObject({ + wire_version: 1, + declared_length: 4, + complete: false, + fuses: [{ index: 0, state: "enabled", raw_byte: 0x31 }], + }); + expect(truncated.limitations.join(" ")).toContain("truncated"); + + const duplicates = await inspect(join(root, "duplicated")); + expect(duplicates.slices[0]?.sentinel_count).toBe(2); + expect( + duplicates.slices[0]?.sentinels.map((item) => item.sentinel_offset), + ).toEqual([...duplicated.offsets]); + expect(duplicates.limitations.join(" ")).toContain("none is selected"); + + const missing = await inspect(join(root, "empty")); + expect(missing.slices[0]?.sentinel_count).toBe(0); + expect(missing.runtime_implications).toEqual([]); + expect(missing.limitations.join(" ")).toContain("unknown, not disabled"); + }); +}); + +describe("ELF embedded ASAR integrity applicability", () => { + it("preserves an enabled ASAR integrity fuse without claiming enforcement on ELF", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-elf-asar-"); + const fuses = [ + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.enabled, + ]; + const image = elfFuseImage(fuses); + const path = join(root, "electron"); + await writeFile(path, image.bytes); + const report = await inspect(path); + expect(report.artifact.format).toBe("elf"); + expect(report.slices[0]?.sentinels[0]?.fuses[4]).toMatchObject({ + index: 4, + name: "EnableEmbeddedAsarIntegrityValidation", + state: "enabled", + raw_byte: ELECTRON_FUSE_STATE_BYTES.enabled, + }); + expect( + report.runtime_implications.find( + (item) => item.subject === "embedded_asar_integrity_validation", + ), + ).toMatchObject({ + relationship: "derivation", + fuse_index: 4, + outcome: "unknown", + sentinel_offset: image.sentinelOffset, + }); + expect(report.limitations.join(" ")).toContain("only on macOS and Windows"); + }); +}); + +describe("Electron fuse reader resource boundaries", () => { + it("rejects invalid scan window sizes before allocating or scanning", async () => { + const root = await createTestTempDirectory( + "rea-electron-fuses-window-limit-", + ); + const path = join(root, "window"); + await writeFile(path, thinFuseImage(OBSIDIAN_FUSE_BYTES).bytes); + for (const chunkBytes of [0, -1, 1.5, 256 * 1024 + 1]) + await expect(readElectronFuseImage(path, { chunkBytes })).rejects.toThrow( + /chunkBytes must be an integer between/, + ); + }); + + it("rejects overlapping universal Mach-O slices before repeated searching", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-overlap-"); + const image = universalFuseImage().bytes; + const firstOffset = image.readUInt32BE(16); + const firstSize = image.readUInt32BE(20); + image.writeUInt32BE(0x01000007, 28); + image.writeUInt32BE(firstOffset, 36); + image.writeUInt32BE(firstSize, 40); + const path = join(root, "overlapping"); + await writeFile(path, image); + + const result = await provider.inspect({ path }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [ + expect.objectContaining({ + path: ["path"], + reason: "invalid_format", + message: expect.stringContaining("slices 0 and 1 overlap"), + }), + ], + }); + }); +}); + +describe("Electron fuse path and container admission", () => { + it("finds a sentinel that crosses a read window", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-window-"); + const image = thinFuseImage(OBSIDIAN_FUSE_BYTES); + const path = join(root, "window"); + await writeFile(path, image.bytes); + const read = await readElectronFuseImage(path, { chunkBytes: 20 }); + expect(read.slices[0]?.sentinels[0]?.sentinel_offset).toBe( + image.sentinelOffset, + ); + expect(read.slices[0]?.sentinels[0]?.fuses).toHaveLength(8); + }); + + it("resolves an app bundle and a framework symlink inside the bundle", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-app-"); + const image = thinFuseImage(OBSIDIAN_FUSE_BYTES); + const app = join(root, "Fixture.app"); + const versions = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + "Versions", + ); + await mkdir(join(versions, "A"), { recursive: true }); + const binary = join(versions, "A", "Electron Framework"); + await writeFile(binary, image.bytes); + await symlink("A", join(versions, "Current")); + await symlink( + "Versions/Current/Electron Framework", + join(versions, "..", "Electron Framework"), + ); + + const fromApp = await inspect(app); + expect(fromApp.artifact.selected_path).toBe(app); + expect(fromApp.artifact.resolved_path).toBe(binary); + expect(fromApp.slices[0]?.sentinel_count).toBe(1); + + const link = join(versions, "..", "Electron Framework"); + const fromLink = await inspect(link); + expect(fromLink.artifact.resolved_path).toBe(binary); + expect(fromLink.artifact.sha256).toBe(fromApp.artifact.sha256); + }); + + it("rejects an app bundle whose Versions/Current symlink leaves the framework", async () => { + const root = await createTestTempDirectory( + "rea-electron-fuses-framework-root-", + ); + const app = join(root, "Fixture.app"); + const framework = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + ); + await mkdir(join(framework, "Versions"), { recursive: true }); + await mkdir(join(app, "Contents", "Other"), { recursive: true }); + await writeFile( + join(app, "Contents", "Other", "Electron Framework"), + elfFuseImage([ELECTRON_FUSE_STATE_BYTES.enabled]).bytes, + ); + await symlink("../../../Other", join(framework, "Versions", "Current")); + + for (const path of [app, framework]) { + const result = await provider.inspect({ path }); + expect(result.ok, path).toBe(false); + if (result.ok) continue; + expect(result.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [ + expect.objectContaining({ + path: ["path"], + message: expect.stringContaining("escapes"), + }), + ], + }); + } + }); + + it("refuses a framework symlink that leaves the bundle", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-escape-"); + const versions = join( + root, + "Fixture.app", + "Contents", + "Frameworks", + "Electron Framework.framework", + "Versions", + ); + await mkdir(versions, { recursive: true }); + await symlink("/etc", join(versions, "Current")); + const result = await provider.inspect({ + path: join(root, "Fixture.app"), + }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [ + expect.objectContaining({ + path: ["path"], + message: expect.stringContaining("escapes"), + }), + ], + }); + }); + + it("rejects an unrecognized container and a big-endian Mach-O image", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-reject-"); + const text = join(root, "notes.txt"); + const bigEndian = join(root, "big"); + const java = join(root, "Main.class"); + await writeFile(text, "This is not an executable container."); + const header = Buffer.alloc(32); + header.writeUInt32BE(0xfeedfacf, 0); + await writeFile(bigEndian, header); + await writeFile( + java, + Buffer.from([0xca, 0xfe, 0xba, 0xbe, 0x00, 0x00, 0x00, 0x37]), + ); + for (const path of [text, bigEndian, java]) { + const result = await provider.inspect({ path }); + expect(result.ok, path).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisUnsupportedTargetError"); + } + }); + + it("does not invent a sentinel from bytes that only straddle two slices", async () => { + const root = await createTestTempDirectory("rea-electron-fuses-span-"); + const left = thinMacho(0x01000007, Buffer.alloc(16)); + const right = thinMacho(0x0100000c, Buffer.alloc(16)); + const overhang = 10; + FUSE_SENTINEL.copy(left, left.length - overhang); + const gap = Buffer.alloc(FUSE_SENTINEL.length - overhang); + FUSE_SENTINEL.copy(gap, 0, overhang); + const tableEnd = 48; + const rightOffset = tableEnd + left.length + gap.length; + const table = Buffer.alloc(tableEnd); + table.writeUInt32BE(0xcafebabe, 0); + table.writeUInt32BE(2, 4); + table.writeUInt32BE(0x01000007, 8); + table.writeUInt32BE(tableEnd, 16); + table.writeUInt32BE(left.length, 20); + table.writeUInt32BE(0x0100000c, 28); + table.writeUInt32BE(rightOffset, 36); + table.writeUInt32BE(right.length, 40); + const path = join(root, "split"); + await writeFile(path, Buffer.concat([table, left, gap, right])); + const report = await inspect(path); + expect(report.slices.map((item) => item.sentinel_count)).toEqual([0, 0]); + }); +}); + +describe("direct Electron Framework binary selection", () => { + const frameworkFixture = async (prefix: string) => { + const root = await createTestTempDirectory(prefix); + const app = join(root, "Fixture.app"); + const framework = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + ); + await mkdir(join(framework, "Versions"), { recursive: true }); + return { app, framework }; + }; + + it("resolves a binary selected through Versions/Current to its target", async () => { + const { framework } = await frameworkFixture("rea-electron-fuses-direct-"); + await mkdir(join(framework, "Versions", "A"), { recursive: true }); + const binary = join(framework, "Versions", "A", "Electron Framework"); + await writeFile(binary, thinFuseImage(OBSIDIAN_FUSE_BYTES).bytes); + await symlink("A", join(framework, "Versions", "Current")); + + const selected = join( + framework, + "Versions", + "Current", + "Electron Framework", + ); + const report = await inspect(selected); + expect(report.artifact.selected_path).toBe(selected); + expect(report.artifact.resolved_path).toBe(binary); + }); + + it("follows symlinks before parent segments in a selected binary path", async () => { + const { framework } = await frameworkFixture("rea-electron-fuses-parent-"); + const versions = join(framework, "Versions"); + await mkdir(join(versions, "nested", "B"), { recursive: true }); + await mkdir(join(versions, "nested", "A"), { recursive: true }); + await mkdir(join(versions, "A"), { recursive: true }); + const binary = join(versions, "nested", "A", "Electron Framework"); + await writeFile(binary, thinFuseImage(OBSIDIAN_FUSE_BYTES).bytes); + await writeFile( + join(versions, "A", "Electron Framework"), + sentinelFreeMacho(), + ); + await symlink("nested/B", join(versions, "Current")); + const selected = `${versions}/Current/../A/Electron Framework`; + const report = await inspect(selected); + expect(report.artifact.selected_path).toBe(selected); + expect(report.artifact.resolved_path).toBe(binary); + expect(report.slices[0]?.sentinel_count).toBe(1); + }); + + it("rejects a binary selected through a Versions/Current link that leaves the framework", async () => { + const { app, framework } = await frameworkFixture( + "rea-electron-fuses-direct-escape-", + ); + await mkdir(join(app, "Contents", "Other"), { recursive: true }); + await writeFile( + join(app, "Contents", "Other", "Electron Framework"), + elfFuseImage([ELECTRON_FUSE_STATE_BYTES.enabled]).bytes, + ); + await symlink("../../../Other", join(framework, "Versions", "Current")); + + const result = await provider.inspect({ + path: join(framework, "Versions", "Current", "Electron Framework"), + }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [ + expect.objectContaining({ + path: ["path"], + message: expect.stringContaining("escapes"), + }), + ], + }); + }); + + it("rejects a framework directory that is a symlink outside the bundle", async () => { + const root = await createTestTempDirectory( + "rea-electron-fuses-framework-link-", + ); + const outside = join(root, "outside"); + await mkdir(join(outside, "Versions", "A"), { recursive: true }); + await writeFile( + join(outside, "Versions", "A", "Electron Framework"), + thinFuseImage(OBSIDIAN_FUSE_BYTES).bytes, + ); + const app = join(root, "Example.app"); + const frameworks = join(app, "Contents", "Frameworks"); + await mkdir(frameworks, { recursive: true }); + const framework = join(frameworks, "Electron Framework.framework"); + await symlink(outside, framework); + + for (const path of [ + app, + join(framework, "Versions", "A", "Electron Framework"), + ]) { + const result = await provider.inspect({ path }); + expect(result.ok, path).toBe(false); + if (result.ok) continue; + expect(result.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [ + expect.objectContaining({ + path: ["path"], + message: expect.stringContaining("escapes"), + }), + ], + }); + } + }); +}); + +it.each(["Contents", "Frameworks"])( + "rejects a redirected %s for every app selection form", + async (segment) => { + const root = await createTestTempDirectory("rea-electron-parent-link-"); + const app = join(root, "Fixture.app"); + const outside = join(root, "outside"); + const actualFramework = join( + outside, + ...(segment === "Contents" ? ["Frameworks"] : []), + "Electron Framework.framework", + ); + await mkdir(join(actualFramework, "Versions", "A"), { recursive: true }); + await writeFile( + join(actualFramework, "Versions", "A", "Electron Framework"), + thinFuseImage(OBSIDIAN_FUSE_BYTES).bytes, + ); + await symlink("A", join(actualFramework, "Versions", "Current")); + await mkdir(segment === "Contents" ? app : join(app, "Contents"), { + recursive: true, + }); + await symlink( + outside, + join( + app, + "Contents", + ...(segment === "Frameworks" ? ["Frameworks"] : []), + ), + ); + const framework = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + ); + for (const path of [ + app, + framework, + join(framework, "Versions", "Current", "Electron Framework"), + ]) { + const result = await provider.inspect({ path }); + expect(result.ok, path).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisInputError"); + } + }, +); + +it("enforces the 32 sentinel boundary with typed and public limit details", async () => { + const root = await createTestTempDirectory("rea-electron-sentinel-limit-"); + const wire = Buffer.concat([FUSE_SENTINEL, fuseWire(1, [0x31])]); + for (const count of [32, 33]) { + const path = join(root, String(count)); + await writeFile( + path, + thinMacho( + 0x0100000c, + Buffer.concat(Array.from({ length: count }, () => wire)), + ), + ); + const result = await provider.inspect({ path }); + if (count === 32) { + expect(result.ok).toBe(true); + if (result.ok) expect(result.value.slices[0]?.sentinel_count).toBe(32); + } else { + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error._tag).toBe("AnalysisResourceConstraintError"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "resource_constraint", + details: { reported_limits: { maximum_sentinels_per_slice: 32 } }, + }); + } + } + } +}); + +it("rejects a sparse artifact over 1 GiB before scanning", async () => { + const root = await createTestTempDirectory("rea-electron-input-limit-"); + const path = join(root, "large"); + await writeFile(path, thinFuseImage(OBSIDIAN_FUSE_BYTES).bytes); + await truncate(path, 1024 * 1024 * 1024 + 1); + const result = await provider.inspect({ path }); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error._tag).toBe("AnalysisResourceConstraintError"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "resource_constraint", + details: { reported_limits: { maximum_input_bytes: 1024 * 1024 * 1024 } }, + }); + } +}); + +it("follows the active framework binary while an inactive A version remains", async () => { + const root = await createTestTempDirectory("rea-electron-current-version-"); + const app = join(root, "Fixture.app"); + const framework = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + ); + for (const [version, state] of [ + ["A", 0x30], + ["B", 0x31], + ] as const) { + await mkdir(join(framework, "Versions", version), { recursive: true }); + await writeFile( + join(framework, "Versions", version, "Electron Framework"), + thinFuseImage([state]).bytes, + ); + } + await symlink("B", join(framework, "Versions", "Current")); + await symlink( + "Versions/Current/Electron Framework", + join(framework, "Electron Framework"), + ); + for (const path of [app, framework, join(framework, "Electron Framework")]) { + const report = await inspect(path); + expect(report.artifact.resolved_path).toBe( + join(framework, "Versions", "B", "Electron Framework"), + ); + expect(report.slices[0]?.sentinels[0]?.fuses[0]?.state).toBe("enabled"); + } +}); + +it("rejects a checked version directory swapped before the reader opens it", async () => { + const root = await createTestTempDirectory("rea-electron-directory-swap-"); + const app = join(root, "Fixture.app"); + const framework = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + ); + const version = join(framework, "Versions", "A"); + const outside = join(root, "outside"); + await mkdir(version, { recursive: true }); + await mkdir(outside); + await writeFile( + join(version, "Electron Framework"), + thinFuseImage([0x30]).bytes, + ); + await writeFile( + join(outside, "Electron Framework"), + thinFuseImage([0x31]).bytes, + ); + const located = await resolveElectronFuseTarget(app); + await rename(version, `${version}.saved`); + await symlink(outside, version); + await expect( + readElectronFuseImage(located.file.path, { selection: located.file }), + ).rejects.toBeInstanceOf(RegularFileChangedError); +}); + +it("reports a checked version directory removed before open as changed", async () => { + const root = await createTestTempDirectory("rea-electron-directory-missing-"); + const app = join(root, "Fixture.app"); + const version = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + "Versions", + "A", + ); + await mkdir(version, { recursive: true }); + await writeFile( + join(version, "Electron Framework"), + thinFuseImage([0x30]).bytes, + ); + const located = await resolveElectronFuseTarget(app); + await rename(version, `${version}.saved`); + await expect( + readElectronFuseImage(located.file.path, { selection: located.file }), + ).rejects.toBeInstanceOf(RegularFileChangedError); +}); diff --git a/src/electron/ElectronFuseProvider.ts b/src/electron/ElectronFuseProvider.ts new file mode 100644 index 000000000..d05baff43 --- /dev/null +++ b/src/electron/ElectronFuseProvider.ts @@ -0,0 +1,170 @@ +import type { ExecutionOptions } from "../application/AnalysisProvider.js"; +import type { ElectronFusePort } from "../application/electron/ElectronFusePort.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, + AnalysisUnsupportedTargetError, +} from "../domain/analysisErrorCore.js"; +import { + describeElectronFuseFindings, + electronFuseReportSchema, + type ElectronFuseReport, + type InspectElectronFusesInput, +} from "../domain/electron/electronFuses.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { ZodError } from "zod"; +import { RegularFileChangedError } from "../filesystem/RegularFile.js"; +import { NonRegularFileReadError } from "../filesystem/RegularFile.js"; +import { + ElectronFusePathFailure, + resolveElectronFuseTarget, +} from "./ElectronFusePath.js"; +import { + electronFuseReadLimitation, + ElectronFuseFormatFailure, + ElectronFuseResourceFailure, + readElectronFuseImage, +} from "./ElectronFuseReader.js"; + +/** Bundled fuse-wire reader. It does not flip fuses or launch the application. */ +export const ELECTRON_FUSE_PROVIDER_IDENTITY = { + id: "electron-fuses", + name: "REA Electron fuse reader", + version: "1", +} as const; + +const OPERATION = "inspect_electron_fuses"; + +/** Read packaged Electron fuse wires from one selected local file or app bundle. */ +export class ElectronFuseProvider implements ElectronFusePort { + readonly identity = ELECTRON_FUSE_PROVIDER_IDENTITY; + + async inspect( + input: InspectElectronFusesInput, + options?: ExecutionOptions, + ): Promise> { + try { + options?.signal?.throwIfAborted(); + const located = await resolveElectronFuseTarget( + input.path, + options?.signal, + ); + options?.signal?.throwIfAborted(); + const image = await readElectronFuseImage(located.file.path, { + selection: located.file, + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }); + const findings = describeElectronFuseFindings(image.slices, image.format); + const parsed = electronFuseReportSchema.safeParse({ + artifact: { + selected_path: located.selectedPath, + resolved_path: located.file.path, + sha256: image.sha256, + bytes: image.bytes, + format: image.format, + }, + slices: image.slices, + runtime_implications: findings.runtime_implications, + limitations: [...findings.limitations, electronFuseReadLimitation()], + }); + if (!parsed.success) + throw new AnalysisOutputError( + OPERATION, + "Electron fuse reader produced a report outside its schema.", + { cause: parsed.error }, + ); + options?.signal?.throwIfAborted(); + return ok(parsed.data); + } catch (cause: unknown) { + return err(failureFor(cause, input.path, options?.signal)); + } + } +} + +const failureFor = ( + cause: unknown, + path: string, + signal?: AbortSignal, +): AnalysisError => { + if (cause instanceof AnalysisError) return cause; + if (signal?.aborted) return new AnalysisCancelledError(OPERATION); + if (cause instanceof ElectronFusePathFailure) + return new AnalysisInputError(OPERATION, { cause }, [ + { path: ["path"], reason: cause.reason, message: cause.message }, + ]); + if (cause instanceof ElectronFuseResourceFailure) + return new AnalysisResourceConstraintError( + OPERATION, + cause.boundary === "file-size" ? "file-size" : "capacity", + cause.message, + cause.boundary === "file-size" + ? { maximum_input_bytes: 1024 * 1024 * 1024 } + : { maximum_sentinels_per_slice: 32 }, + { cause }, + ); + if (cause instanceof ElectronFuseFormatFailure) + return cause.kind === "unsupported" + ? new AnalysisUnsupportedTargetError(OPERATION, path, cause.message, { + cause, + }) + : new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["path"], + reason: "invalid_format", + message: `Selected artifact ${path}: ${cause.message}`, + }, + ]); + if (cause instanceof RegularFileChangedError) + return new AnalysisArtifactChangedError(OPERATION, path, cause.message, { + cause, + }); + if (cause instanceof NonRegularFileReadError) + return new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ]); + if (cause instanceof ZodError) + return new AnalysisOutputError( + OPERATION, + "Electron fuse reader produced a report outside its schema.", + { cause }, + ); + if (cause instanceof Error && "code" in cause) { + if (cause.code === "EACCES" || cause.code === "EPERM") + return new AnalysisAccessDeniedError(OPERATION, path, cause.code, { + cause, + }); + if ( + ["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"].includes( + String(cause.code), + ) + ) + return new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["path"], + reason: "invalid_value", + message: `Selected artifact could not be read (${String(cause.code)}): ${path}.`, + }, + ]); + } + return new ProviderAdapterError( + ELECTRON_FUSE_PROVIDER_IDENTITY.id, + OPERATION, + { + cause, + diagnostics: { + path, + reason: cause instanceof Error ? cause.message : String(cause), + }, + }, + ); +}; diff --git a/src/electron/ElectronFuseReader.ts b/src/electron/ElectronFuseReader.ts new file mode 100644 index 000000000..4db52b9c7 --- /dev/null +++ b/src/electron/ElectronFuseReader.ts @@ -0,0 +1,480 @@ +import type { ResolvedElectronFuseFile } from "./ElectronFusePath.js"; +import { createHash } from "node:crypto"; +import { lstat } from "node:fs/promises"; + +import { + hasMachoMagic, + readMachoImage, +} from "../artifacts/apple/MachoLoadCommandReader.js"; +import type { ReadAt } from "../artifacts/apple/MachoLoadCommandReader.js"; +import { + electronFuseName, + electronFuseState, + ELECTRON_FUSE_SENTINEL, + type ElectronFuseSentinel, + type ElectronFuseSlice, +} from "../domain/electron/electronFuses.js"; +import { + openRegularFile, + RegularFileChangedError, + sameRegularFileState, +} from "../filesystem/RegularFile.js"; + +/** Admission and search bounds for one Electron executable. */ +export const ELECTRON_FUSE_LIMITS = { + maximumFileBytes: 1024 * 1024 * 1024, + chunkBytes: 256 * 1024, + maximumSentinelsPerSlice: 32, + maximumPeHeaderOffset: 1024 * 1024, +} as const; + +const SENTINEL = Buffer.from(ELECTRON_FUSE_SENTINEL, "ascii"); +const FAT_MAGICS = new Set([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca]); + +/** Container identity and per-slice fuse wires. Offsets are file-absolute. */ +export interface ElectronFuseImage { + readonly sha256: string; + readonly bytes: number; + readonly format: "mach-o" | "mach-o-universal" | "pe" | "elf"; + readonly slices: readonly ElectronFuseSlice[]; +} + +type ElectronFuseContainerFormat = ElectronFuseImage["format"]; + +interface SliceRange { + readonly index: number; + readonly fileOffset: number; + readonly size: number; + readonly architecture: string | null; +} + +/** The selected container is malformed or outside the supported formats. */ +export class ElectronFuseFormatFailure extends Error { + constructor( + readonly kind: "malformed" | "unsupported", + message: string, + ) { + super(message); + this.name = "ElectronFuseFormatFailure"; + } +} + +/** A measured bound was hit before a complete fuse report could be returned. */ +export class ElectronFuseResourceFailure extends Error { + constructor( + readonly boundary: "file-size" | "capacity", + message: string, + ) { + super(message); + this.name = "ElectronFuseResourceFailure"; + } +} + +/** + * Hash one admitted regular file and search each slice for the fuse sentinel. + * The search keeps only a sentinel-sized overlap between chunks. + */ +export const readElectronFuseImage = async ( + path: string, + options?: { + readonly signal?: AbortSignal; + readonly chunkBytes?: number; + readonly selection?: ResolvedElectronFuseFile; + }, +): Promise => { + const signal = options?.signal; + signal?.throwIfAborted(); + const selection = options?.selection; + const recheckPath = async (candidate: string) => { + signal?.throwIfAborted(); + try { + const metadata = await lstat(candidate, { bigint: true }); + signal?.throwIfAborted(); + return metadata; + } catch (cause: unknown) { + if (selection !== undefined && isMissingPath(cause)) + throw new RegularFileChangedError(path); + throw cause; + } + }; + const assertPathNodes = async (): Promise => { + if (selection === undefined) return; + if (selection.path !== path) throw new RegularFileChangedError(path); + for (const node of selection.nodes) { + const current = await recheckPath(node.path); + if ( + current.dev !== node.metadata.dev || + current.ino !== node.metadata.ino || + current.isDirectory() !== node.metadata.isDirectory() || + current.isSymbolicLink() !== node.metadata.isSymbolicLink() + ) + throw new RegularFileChangedError(path); + } + }; + await assertPathNodes(); + const before = await recheckPath(path); + if ( + selection !== undefined && + !sameRegularFileState(selection.metadata, before) + ) + throw new RegularFileChangedError(path); + if (!before.isFile() || before.isSymbolicLink()) + throw new ElectronFuseFormatFailure( + "unsupported", + `Expected a regular file without a symlink: ${path}`, + ); + if (before.size > BigInt(ELECTRON_FUSE_LIMITS.maximumFileBytes)) + throw new ElectronFuseResourceFailure( + "file-size", + `Selected artifact exceeds ${ELECTRON_FUSE_LIMITS.maximumFileBytes} bytes: ${path}`, + ); + const size = Number(before.size); + const chunkBytes = options?.chunkBytes ?? ELECTRON_FUSE_LIMITS.chunkBytes; + if ( + !Number.isSafeInteger(chunkBytes) || + chunkBytes < 1 || + chunkBytes > ELECTRON_FUSE_LIMITS.chunkBytes + ) + throw new TypeError( + `chunkBytes must be an integer between 1 and ${ELECTRON_FUSE_LIMITS.chunkBytes}.`, + ); + const handle = await openRegularFile(path, { symlinks: "reject", signal }); + try { + const opened = await handle.stat({ bigint: true }); + if (!sameRegularFileState(before, opened)) + throw new RegularFileChangedError(path); + await assertPathNodes(); + const readAt = readerFor(handle, size, path, signal); + const container = await identifyContainer(readAt, size, path); + const scanned = await scanSlices( + handle, + path, + size, + container.slices, + chunkBytes, + signal, + ); + const current = await recheckPath(path); + if (!sameRegularFileState(before, current)) + throw new RegularFileChangedError(path); + await assertPathNodes(); + return { + sha256: scanned.sha256, + bytes: size, + format: container.format, + slices: scanned.slices, + }; + } finally { + await handle.close(); + } +}; + +/** Sentence describing the file and sentinel bounds enforced by this reader. */ +export const electronFuseReadLimitation = (): string => + `The selected file is admitted as one regular file of at most 1 GiB. Sentinel search uses ${ELECTRON_FUSE_LIMITS.chunkBytes}-byte windows, and each slice keeps at most ${ELECTRON_FUSE_LIMITS.maximumSentinelsPerSlice} sentinels.`; + +const readerFor = + ( + handle: Awaited>, + size: number, + path: string, + signal: AbortSignal | undefined, + ): ReadAt => + async (offset, length) => { + signal?.throwIfAborted(); + if (length <= 0 || offset < 0 || offset >= size) return new Uint8Array(); + const bounded = Math.min(length, size - offset); + const buffer = Buffer.alloc(bounded); + const read = await handle.read(buffer, 0, bounded, offset); + if (read.bytesRead !== bounded) throw new RegularFileChangedError(path); + return buffer.subarray(0, read.bytesRead); + }; + +const identifyContainer = async ( + readAt: ReadAt, + size: number, + path: string, +): Promise<{ + readonly format: ElectronFuseContainerFormat; + readonly slices: readonly SliceRange[]; +}> => { + const head = await readAt(0, Math.min(size, 8)); + if (hasMachoMagic(head)) return identifyMacho(readAt, size, head, path); + if (isElf(head)) + return { + format: "elf", + slices: [{ index: 0, fileOffset: 0, size, architecture: null }], + }; + if (await isPe(readAt, size)) + return { + format: "pe", + slices: [{ index: 0, fileOffset: 0, size, architecture: null }], + }; + throw new ElectronFuseFormatFailure( + "unsupported", + `Selected artifact ${path} does not contain a Mach-O, ELF, or PE container.`, + ); +}; + +const identifyMacho = async ( + readAt: ReadAt, + size: number, + head: Uint8Array, + path: string, +): Promise<{ + readonly format: ElectronFuseContainerFormat; + readonly slices: readonly SliceRange[]; +}> => { + const facts = await readMachoImage(readAt, size); + if (facts.status !== "parsed") { + const reason = + facts.status === "not-mach-o" + ? `Selected artifact ${path} does not contain a Mach-O, ELF, or PE container.` + : facts.reason; + throw new ElectronFuseFormatFailure( + facts.status === "malformed" ? "malformed" : "unsupported", + reason, + ); + } + const format = isFat(head) ? "mach-o-universal" : "mach-o"; + const slices = facts.slices.map((slice, index) => ({ + index, + fileOffset: slice.slice_offset, + size: slice.slice_size, + architecture: slice.architecture, + })); + assertNonOverlappingSlices(slices); + return { + format, + slices, + }; +}; + +/** A FAT architecture owns a distinct byte range; overlap is malformed. */ +const assertNonOverlappingSlices = (ranges: readonly SliceRange[]): void => { + const ordered = [...ranges].sort( + (left, right) => left.fileOffset - right.fileOffset, + ); + for (let index = 1; index < ordered.length; index += 1) { + const previous = ordered[index - 1]; + const current = ordered[index]; + if ( + previous !== undefined && + current !== undefined && + current.fileOffset < previous.fileOffset + previous.size + ) + throw new ElectronFuseFormatFailure( + "malformed", + `Mach-O slices ${previous.index} and ${current.index} overlap.`, + ); + } +}; + +const scanSlices = async ( + handle: Awaited>, + path: string, + size: number, + ranges: readonly SliceRange[], + chunkBytes: number, + signal: AbortSignal | undefined, +): Promise<{ + readonly sha256: string; + readonly slices: readonly ElectronFuseSlice[]; +}> => { + const states = ranges.map((slice) => ({ + slice, + carry: Buffer.alloc(0), + hits: [] as number[], + })); + const hash = createHash("sha256"); + const chunk = Buffer.allocUnsafe(chunkBytes); + let position = 0; + while (position < size) { + signal?.throwIfAborted(); + const length = Math.min(chunkBytes, size - position); + const read = await handle.read(chunk, 0, length, position); + if (read.bytesRead !== length) throw new RegularFileChangedError(path); + const piece = chunk.subarray(0, length); + hash.update(piece); + absorbChunk(states, piece, position); + position += length; + } + if (position !== size) throw new RegularFileChangedError(path); + const slices: ElectronFuseSlice[] = []; + for (const state of states) { + const sentinels: ElectronFuseSentinel[] = []; + for (const offset of state.hits) + sentinels.push( + await parseWire( + handle, + path, + offset, + state.slice.fileOffset + state.slice.size, + signal, + ), + ); + slices.push({ + index: state.slice.index, + file_offset: state.slice.fileOffset, + size: state.slice.size, + architecture: state.slice.architecture, + sentinel_count: sentinels.length, + sentinels, + }); + } + return { sha256: hash.digest("hex"), slices }; +}; + +const absorbChunk = ( + states: readonly { + readonly slice: SliceRange; + carry: Buffer; + readonly hits: number[]; + }[], + chunk: Buffer, + position: number, +): void => { + const chunkEnd = position + chunk.length; + for (const state of states) { + const start = Math.max(position, state.slice.fileOffset); + const end = Math.min(chunkEnd, state.slice.fileOffset + state.slice.size); + if (start >= end) continue; + const piece = chunk.subarray(start - position, end - position); + const carryLength = state.carry.length; + const window = + carryLength === 0 ? piece : Buffer.concat([state.carry, piece]); + const windowFileStart = start - carryLength; + let from = 0; + while (from <= window.length - SENTINEL.length) { + const found = window.indexOf(SENTINEL, from); + if (found < 0) break; + const sentinelOffset = windowFileStart + found; + if (!state.hits.includes(sentinelOffset)) { + if (state.hits.length >= ELECTRON_FUSE_LIMITS.maximumSentinelsPerSlice) + throw new ElectronFuseResourceFailure( + "capacity", + `Slice ${state.slice.index} contains more than ${ELECTRON_FUSE_LIMITS.maximumSentinelsPerSlice} Electron fuse sentinels.`, + ); + state.hits.push(sentinelOffset); + } + from = found + 1; + } + const tail = SENTINEL.length - 1; + state.carry = + window.length <= tail + ? Buffer.from(window) + : Buffer.from(window.subarray(window.length - tail)); + } +}; + +const parseWire = async ( + handle: Awaited>, + path: string, + sentinelOffset: number, + sliceEnd: number, + signal: AbortSignal | undefined, +): Promise => { + const wireStart = sentinelOffset + SENTINEL.length; + const available = sliceEnd - wireStart; + if (available < 1) + return { + sentinel_offset: sentinelOffset, + wire_version: null, + declared_length: null, + fuses: [], + complete: false, + }; + const header = await readExact( + handle, + path, + wireStart, + Math.min(available, 2), + signal, + ); + const wireVersion = header[0]; + if (wireVersion === undefined || header.length < 2) + return { + sentinel_offset: sentinelOffset, + wire_version: wireVersion ?? null, + declared_length: null, + fuses: [], + complete: false, + }; + const declared = header[1] ?? 0; + const fuseLength = Math.min(declared, available - 2); + const fuseBytes = + fuseLength === 0 + ? Buffer.alloc(0) + : await readExact(handle, path, wireStart + 2, fuseLength, signal); + return { + sentinel_offset: sentinelOffset, + wire_version: wireVersion, + declared_length: declared, + fuses: [...fuseBytes].map((raw, index) => ({ + index, + name: electronFuseName(wireVersion, index), + state: electronFuseState(raw), + raw_byte: raw, + })), + complete: fuseBytes.length === declared, + }; +}; + +const readExact = async ( + handle: Awaited>, + path: string, + offset: number, + length: number, + signal: AbortSignal | undefined, +): Promise => { + signal?.throwIfAborted(); + const buffer = Buffer.alloc(length); + const read = await handle.read(buffer, 0, length, offset); + if (read.bytesRead !== length) throw new RegularFileChangedError(path); + return buffer; +}; + +const isElf = (head: Uint8Array): boolean => + head.length >= 4 && + head[0] === 0x7f && + head[1] === 0x45 && + head[2] === 0x4c && + head[3] === 0x46; + +const isFat = (head: Uint8Array): boolean => { + if (head.length < 4) return false; + const magic = Buffer.from(head.buffer, head.byteOffset, 4).readUInt32BE(0); + return FAT_MAGICS.has(magic); +}; + +const isPe = async (readAt: ReadAt, size: number): Promise => { + if (size < 64) return false; + const header = await readAt(0, 64); + if (header.length < 64 || header[0] !== 0x4d || header[1] !== 0x5a) + return false; + const offset = Buffer.from( + header.buffer, + header.byteOffset, + header.byteLength, + ).readUInt32LE(0x3c); + if ( + !Number.isSafeInteger(offset) || + offset < 64 || + offset > ELECTRON_FUSE_LIMITS.maximumPeHeaderOffset || + offset > size - 4 + ) + return false; + const signature = await readAt(offset, 4); + return ( + signature.length === 4 && + signature[0] === 0x50 && + signature[1] === 0x45 && + signature[2] === 0 && + signature[3] === 0 + ); +}; + +const isMissingPath = (cause: unknown): boolean => + cause instanceof Error && + "code" in cause && + ["ENOENT", "ENOTDIR"].includes(String(cause.code)); diff --git a/src/server/createServer.ts b/src/server/createServer.ts index e637f8d0d..ce0873dd3 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -22,6 +22,8 @@ import { registerRecordedCrashTools } from "./registerRecordedCrashTools.js"; import { createRecordedCrashService } from "../composition/binaryDiagnostics.js"; import { createGoBinaryService } from "../composition/go.js"; import type { GoBinaryService } from "../application/go/GoBinaryService.js"; +import { createElectronFuseService } from "../composition/electronFuses.js"; +import type { ElectronFuseService } from "../application/electron/ElectronFuseService.js"; import { registerGoTools } from "./registerGoTools.js"; import { createRustBinaryService } from "../composition/rust.js"; import type { RustBinaryService } from "../application/rust/RustBinaryService.js"; @@ -157,6 +159,7 @@ export interface CreateServerOptions { readonly binaryLayout?: BinaryLayoutService; readonly goBinary?: GoBinaryService; readonly rustBinary?: RustBinaryService; + readonly electronFuses?: ElectronFuseService; readonly recordedCrash?: RecordedCrashService; readonly firmwareAnalysis?: FirmwareAnalysisPort; readonly javascriptRecovery?: JavaScriptRecoveryPort; @@ -884,6 +887,7 @@ const registerObservationTools = ({ evidenceById, electron: options.electronObservation, electronActive: options.electronActiveObservation, + electronFuses: options.electronFuses ?? createElectronFuseService(), observationLoadFailure: options.optionalProviderLoadFailures?.electronObservation, activeLoadFailure: diff --git a/src/server/registerElectronTools.ts b/src/server/registerElectronTools.ts index e8008177d..96a518e0f 100644 --- a/src/server/registerElectronTools.ts +++ b/src/server/registerElectronTools.ts @@ -21,6 +21,7 @@ import { analyzeJavaScriptApplicationValidated } from "../application/javascript import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { reconcileJavaScriptRuntimeEvidenceValidated } from "../application/javascript/JavaScriptRuntimeReconciliationService.js"; import type { ProgressReporter } from "../application/ProgressReporter.js"; +import type { ElectronFuseService } from "../application/electron/ElectronFuseService.js"; import { toolContract } from "../contracts/toolContracts.js"; import type { ToolContract } from "../contracts/toolContractTypes.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; @@ -32,6 +33,7 @@ import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; +import { createEvidenceToolHandler } from "./contractToolHandler.js"; interface ElectronToolRegistration { readonly artifactResources: ArtifactResourceScope; @@ -40,6 +42,7 @@ interface ElectronToolRegistration { readonly activeLoadFailure?: OptionalProviderLoadFailure | undefined; readonly electron: ElectronObservationPort | undefined; readonly electronActive: ElectronActiveObservationPort | undefined; + readonly electronFuses: ElectronFuseService; readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; readonly evidenceById: EvidenceLookup | undefined; readonly withAdmittedAnalysis?: WithAdmittedAnalysis; @@ -57,6 +60,7 @@ export const registerElectronTools = ( options: ElectronToolRegistration, ): void => { const registration = { ...options, delivery: server.delivery, server }; + registerElectronFuseTool(server, options); const listContract = toolContract("list_electron_targets"); const inspectContract = toolContract("inspect_electron_page"); const analyzeContract = toolContract("analyze_javascript_application"); @@ -141,6 +145,25 @@ export const registerElectronTools = ( ); }; +const registerElectronFuseTool = ( + server: EvidenceMcpServer, + options: ElectronToolRegistration, +): void => { + const contract = toolContract("inspect_electron_fuses"); + const handler = createEvidenceToolHandler( + server, + (input, execution) => options.electronFuses.inspect(input, execution), + options.logger, + options.recordEvidence, + options.withAdmittedAnalysis, + ); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + handler(contract), + ); +}; + const runElectronTool = async ( options: ElectronToolRegistration & { readonly delivery: ToolResultDelivery; diff --git a/tests/acceptance/electronFusesCli.test.ts b/tests/acceptance/electronFusesCli.test.ts new file mode 100644 index 000000000..763d2faf5 --- /dev/null +++ b/tests/acceptance/electronFusesCli.test.ts @@ -0,0 +1,150 @@ +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { expect } from "vitest"; + +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { + OBSIDIAN_FUSE_BYTES, + sentinelFreeMacho, + thinFuseImage, + universalFuseImage, +} from "../fixtures/electron/fuseImage.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest( + "reads thin and universal Electron fuse wires through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-electron-fuses-cli-"); + const thin = thinFuseImage(OBSIDIAN_FUSE_BYTES); + const universal = universalFuseImage(); + await writeFile(join(root, "thin"), thin.bytes); + await writeFile(join(root, "universal"), universal.bytes); + + const thinResult = await cli.run({ + arguments: ["inspect-electron-fuses", "thin", "--json"], + cwd: root, + }); + expect(thinResult.exitCode, thinResult.stderr).toBe(0); + const thinEvidence = toolContract( + "inspect_electron_fuses", + ).outputSchema.parse(thinResult.json); + expect(thinEvidence.subject?.local_path).toBe(join(root, "thin")); + expect(thinEvidence.subject?.digest.sha256).toBe( + createHash("sha256").update(thin.bytes).digest("hex"), + ); + expect(thinEvidence.subject?.architecture).toBe("arm64"); + expect( + thinEvidence.normalized_result.slices[0]?.sentinels[0], + ).toMatchObject({ + sentinel_offset: thin.sentinelOffset, + wire_version: 1, + declared_length: 8, + }); + expect( + thinEvidence.normalized_result.runtime_implications.find( + (item) => item.subject === "node_cli_inspect_arguments", + ), + ).toMatchObject({ outcome: "not_honored", fuse_index: 3 }); + + const universalResult = await cli.run({ + arguments: ["inspect-electron-fuses", join(root, "universal"), "--json"], + }); + expect(universalResult.exitCode, universalResult.stderr).toBe(0); + const universalEvidence = toolContract( + "inspect_electron_fuses", + ).outputSchema.parse(universalResult.json); + expect(universalEvidence.normalized_result.artifact.format).toBe( + "mach-o-universal", + ); + expect(universalEvidence.subject?.architecture).toBeNull(); + expect( + universalEvidence.normalized_result.slices.map((slice) => ({ + file_offset: slice.file_offset, + sentinel_offset: slice.sentinels[0]?.sentinel_offset, + })), + ).toEqual([ + { + file_offset: universal.left.fileOffset, + sentinel_offset: universal.left.sentinelOffset, + }, + { + file_offset: universal.right.fileOffset, + sentinel_offset: universal.right.sentinelOffset, + }, + ]); + }, +); + +cliTest( + "reports a missing sentinel as unknown fuse state through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-electron-fuses-cli-empty-"); + const path = join(root, "empty"); + await writeFile(path, sentinelFreeMacho()); + const result = await cli.run({ + arguments: ["inspect-electron-fuses", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_electron_fuses").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.slices[0]?.sentinel_count).toBe(0); + expect(evidence.normalized_result.runtime_implications).toEqual([]); + expect(evidence.limitations.join(" ")).toContain( + "unknown, not that every fuse is disabled", + ); + }, +); + +cliTest( + "resolves an Electron app bundle to its framework binary through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-electron-fuses-cli-app-"); + const image = thinFuseImage(OBSIDIAN_FUSE_BYTES); + const app = join(root, "Fixture.app"); + const binary = join( + app, + "Contents", + "Frameworks", + "Electron Framework.framework", + "Versions", + "A", + "Electron Framework", + ); + await mkdir(join(binary, ".."), { recursive: true }); + await writeFile(binary, image.bytes); + await symlink("A", join(binary, "..", "..", "Current")); + const result = await cli.run({ + arguments: ["inspect-electron-fuses", app, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_electron_fuses").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.artifact).toMatchObject({ + selected_path: app, + resolved_path: binary, + format: "mach-o", + }); + expect(evidence.subject?.local_path).toBe(binary); + }, +); + +cliTest( + "rejects an unrecognized container through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-electron-fuses-cli-text-"); + const path = join(root, "notes.txt"); + await writeFile(path, "This is not an executable container."); + const result = await cli.run({ + arguments: ["inspect-electron-fuses", path, "--json"], + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "unsupported_target", + details: { operation: "inspect_electron_fuses", path }, + }); + }, +); diff --git a/tests/boundary/mcp/electronFusesMcp.test.ts b/tests/boundary/mcp/electronFusesMcp.test.ts new file mode 100644 index 000000000..db398cd0f --- /dev/null +++ b/tests/boundary/mcp/electronFusesMcp.test.ts @@ -0,0 +1,180 @@ +import type { Client } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { ELECTRON_FUSE_STATE_BYTES } from "../../../src/domain/electron/electronFuses.js"; +import { + elfFuseImage, + truncatedFuseImage, + twoSentinelElf, +} from "../../fixtures/electron/fuseImage.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; + +const fuseMcpTest = mcpTest.extend<{ + fuse: { client: Client }; +}>({ + fuse: async ({ mcp }, use) => { + const session = createTestBinarySession(() => { + throw new Error( + "Electron fuse inspection must not start a deep provider", + ); + }); + try { + const client = await mcp.connect( + createServer({ kind: "session", session }), + ); + await use({ client }); + } finally { + await session.close(); + } + }, +}); + +fuseMcpTest( + "advertises fuse inspection and preserves unknown fuse bytes through MCP", + async ({ fuse: { client }, mcp }) => { + const session = createTestBinarySession(() => { + throw new Error( + "Electron fuse inspection must not start a deep provider", + ); + }); + try { + const connected = await mcp.connect( + createServer({ kind: "session", session }), + ); + const status = await connected.callTool({ + name: "binary_session", + arguments: {}, + }); + const availability = expandToolAvailability( + toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ).result.tool_availability, + ); + expect(availability).toContainEqual( + expect.objectContaining({ + name: "inspect_electron_fuses", + available: true, + reason: "available", + }), + ); + const advertised = (await connected.listTools()).tools.find( + (tool) => tool.name === "inspect_electron_fuses", + ); + if (advertised?.outputSchema === undefined) + throw new Error("Electron fuse inspection must publish both schemas"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + expect(advertised.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + + const root = await createTestTempDirectory("rea-electron-fuses-mcp-"); + const elf = elfFuseImage([ELECTRON_FUSE_STATE_BYTES.removed, 0x99]); + const path = join(root, "application"); + await writeFile(path, elf.bytes); + const response = await connected.callTool({ + name: "inspect_electron_fuses", + arguments: { path }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + expect( + ajv.validate(outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const parsed = toolContract("inspect_electron_fuses").outputSchema.parse( + response.structuredContent, + ); + const evidence = parseEvidence(parsed); + expect(evidence.confidence).toBe("observed"); + expect(parsed.normalized_result.artifact.format).toBe("elf"); + expect( + parsed.normalized_result.slices[0]?.sentinels[0]?.fuses, + ).toMatchObject([ + { index: 0, name: "RunAsNode", state: "removed", raw_byte: 0x72 }, + { + index: 1, + name: "EnableCookieEncryption", + state: "unknown", + raw_byte: 0x99, + }, + ]); + expect(parsed.normalized_result.runtime_implications).toEqual([ + expect.objectContaining({ + subject: "electron_run_as_node", + relationship: "derivation", + outcome: "not_honored", + fuse_index: 0, + }), + ]); + expect(session.evidenceById(evidence.evidence_id)).toEqual(evidence); + + const duplicated = twoSentinelElf(); + const duplicatePath = join(root, "duplicated"); + await writeFile(duplicatePath, duplicated.bytes); + const duplicates = await connected.callTool({ + name: "inspect_electron_fuses", + arguments: { path: duplicatePath }, + }); + expect(duplicates.isError).not.toBe(true); + const duplicateEvidence = toolContract( + "inspect_electron_fuses", + ).outputSchema.parse(duplicates.structuredContent); + expect( + duplicateEvidence.normalized_result.slices[0]?.sentinels.map( + (item) => item.sentinel_offset, + ), + ).toEqual([...duplicated.offsets]); + + const truncatedPath = join(root, "truncated"); + await writeFile(truncatedPath, truncatedFuseImage()); + const truncated = await connected.callTool({ + name: "inspect_electron_fuses", + arguments: { path: truncatedPath }, + }); + expect(truncated.isError).not.toBe(true); + const truncatedEvidence = toolContract( + "inspect_electron_fuses", + ).outputSchema.parse(truncated.structuredContent); + expect( + truncatedEvidence.normalized_result.slices[0]?.sentinels[0], + ).toMatchObject({ + complete: false, + declared_length: 4, + wire_version: 1, + }); + + const relative = await client.callTool({ + name: "inspect_electron_fuses", + arguments: { path: "relative.bin" }, + }); + expect(relative.isError).toBe(true); + expect(parseMcpToolError(relative)).toMatchObject({ + error: { + code: "invalid_request", + details: { + operation: "inspect_electron_fuses", + issues: [expect.objectContaining({ path: ["path"] })], + }, + }, + }); + } finally { + await session.close(); + } + }, +); diff --git a/tests/fixtures/electron/fuseImage.ts b/tests/fixtures/electron/fuseImage.ts new file mode 100644 index 000000000..e887f83ec --- /dev/null +++ b/tests/fixtures/electron/fuseImage.ts @@ -0,0 +1,146 @@ +import { + ELECTRON_FUSE_SENTINEL, + ELECTRON_FUSE_STATE_BYTES, +} from "../../../src/domain/electron/electronFuses.js"; + +export const FUSE_SENTINEL = Buffer.from(ELECTRON_FUSE_SENTINEL, "ascii"); + +/** Obsidian's observed version-1 states: disabled, enabled, then six more with the last enabled. */ +export const OBSIDIAN_FUSE_BYTES = [ + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.enabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.disabled, + ELECTRON_FUSE_STATE_BYTES.enabled, +] as const; + +export const fuseWire = (version: number, fuses: readonly number[]): Buffer => + Buffer.from([version, fuses.length, ...fuses]); + +export const thinMacho = (cpuType: number, body: Buffer): Buffer => { + const header = Buffer.alloc(32); + header.writeUInt32LE(0xfeedfacf, 0); + header.writeUInt32LE(cpuType, 4); + header.writeUInt32LE(0, 8); + header.writeUInt32LE(2, 12); + return Buffer.concat([header, body]); +}; + +export const thinFuseImage = ( + fuses: readonly number[], + version = 1, +): { readonly bytes: Buffer; readonly sentinelOffset: number } => ({ + bytes: thinMacho( + 0x0100000c, + Buffer.concat([FUSE_SENTINEL, fuseWire(version, fuses)]), + ), + sentinelOffset: 32, +}); + +export const universalFuseImage = (): { + readonly bytes: Buffer; + readonly left: { + readonly fileOffset: number; + readonly sentinelOffset: number; + }; + readonly right: { + readonly fileOffset: number; + readonly sentinelOffset: number; + }; +} => { + const left = thinMacho( + 0x01000007, + Buffer.concat([FUSE_SENTINEL, fuseWire(1, OBSIDIAN_FUSE_BYTES)]), + ); + const rightFuses = [...OBSIDIAN_FUSE_BYTES]; + rightFuses[3] = ELECTRON_FUSE_STATE_BYTES.enabled; + const right = thinMacho( + 0x0100000c, + Buffer.concat([FUSE_SENTINEL, fuseWire(1, rightFuses)]), + ); + const tableEnd = 48; + const table = Buffer.alloc(tableEnd); + table.writeUInt32BE(0xcafebabe, 0); + table.writeUInt32BE(2, 4); + const writeEntry = ( + index: number, + cpuType: number, + offset: number, + size: number, + ) => { + const base = 8 + index * 20; + table.writeUInt32BE(cpuType, base); + table.writeUInt32BE(0, base + 4); + table.writeUInt32BE(offset, base + 8); + table.writeUInt32BE(size, base + 12); + table.writeUInt32BE(0, base + 16); + }; + writeEntry(0, 0x01000007, tableEnd, left.length); + writeEntry(1, 0x0100000c, tableEnd + left.length, right.length); + return { + bytes: Buffer.concat([table, left, right]), + left: { fileOffset: tableEnd, sentinelOffset: tableEnd + 32 }, + right: { + fileOffset: tableEnd + left.length, + sentinelOffset: tableEnd + left.length + 32, + }, + }; +}; + +export const peFuseImage = ( + fuses: readonly number[], +): { readonly bytes: Buffer; readonly sentinelOffset: number } => { + const header = Buffer.alloc(0x80); + header.write("MZ", 0, "ascii"); + header.writeUInt32LE(0x80, 0x3c); + const body = Buffer.concat([ + Buffer.from("PE\0\0", "binary"), + FUSE_SENTINEL, + fuseWire(1, fuses), + ]); + return { bytes: Buffer.concat([header, body]), sentinelOffset: 0x84 }; +}; + +export const elfFuseImage = ( + fuses: readonly number[], + version = 1, +): { readonly bytes: Buffer; readonly sentinelOffset: number } => ({ + bytes: Buffer.concat([ + Buffer.from([0x7f, 0x45, 0x4c, 0x46]), + FUSE_SENTINEL, + fuseWire(version, fuses), + ]), + sentinelOffset: 4, +}); + +export const truncatedFuseImage = (): Buffer => + thinMacho( + 0x0100000c, + Buffer.concat([FUSE_SENTINEL, Buffer.from([1, 4, 0x31])]), + ); + +export const twoSentinelElf = (): { + readonly bytes: Buffer; + readonly offsets: readonly [number, number]; +} => { + const wire = fuseWire(1, OBSIDIAN_FUSE_BYTES); + const first = 4; + const second = first + FUSE_SENTINEL.length + wire.length + 2; + return { + bytes: Buffer.concat([ + Buffer.from([0x7f, 0x45, 0x4c, 0x46]), + FUSE_SENTINEL, + wire, + Buffer.from([0, 0]), + FUSE_SENTINEL, + wire, + ]), + offsets: [first, second], + }; +}; + +export const sentinelFreeMacho = (): Buffer => + thinMacho(0x0100000c, Buffer.from("no fuse wire in this image"));