diff --git a/src/domain/providerCleanupError.ts b/src/domain/providerCleanupError.ts index 8e582caa8..4503b583e 100644 --- a/src/domain/providerCleanupError.ts +++ b/src/domain/providerCleanupError.ts @@ -1,5 +1,6 @@ import type { JsonValue } from "./jsonValue.js"; import { ProviderAdapterError } from "./providerAdapterError.js"; +import type { AnalysisPartialObservation } from "./analysisErrorBase.js"; /** Provider resources could not be proven closed after bounded cleanup. */ export class ProviderCleanupError extends ProviderAdapterError { @@ -13,10 +14,21 @@ export class ProviderCleanupError extends ProviderAdapterError { resources: readonly string[], diagnostics: Readonly>, /** Identify the operation owning these resources; binary callers retain their default. */ - options?: ErrorOptions & { readonly operation?: string }, + options?: ErrorOptions & { + readonly operation?: string; + readonly partialObservation?: AnalysisPartialObservation; + }, ) { - const { operation = "close_binary", ...errorOptions } = options ?? {}; - super(providerId, operation, { ...errorOptions, diagnostics }); + const { + operation = "close_binary", + partialObservation, + ...errorOptions + } = options ?? {}; + super(providerId, operation, { + ...errorOptions, + ...(partialObservation === undefined ? {} : { partialObservation }), + diagnostics, + }); this.cleanupResources = [...resources]; } } diff --git a/src/native/LldbCallTracer.ts b/src/native/LldbCallTracer.ts index c0ea59c8b..9fa5813b8 100644 --- a/src/native/LldbCallTracer.ts +++ b/src/native/LldbCallTracer.ts @@ -12,10 +12,9 @@ import { AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; -import type { - AnalysisCleanupObservation, - AnalysisError, -} from "../domain/analysisErrorBase.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js"; import type { NativeCallPartialObservation } from "../domain/native/nativeCallPartialObservation.js"; import { nativeCallEventSchema, @@ -23,6 +22,7 @@ import { type NativeCallObservationInput, } from "../domain/native/nativeCallObservation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; import { safeParseJson } from "../domain/safeJson.js"; import { @@ -32,7 +32,6 @@ import { import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js"; import { observeProcessStartIdentity, - prepareProcessOwnershipInspection, signalProcessWithStartIdentity, } from "../process/ProcessOwnershipObservation.js"; import type { ProcessIdentityObservation } from "../process/ProcessOwnership.js"; @@ -158,6 +157,8 @@ export interface NativeCallTracer { }, signal?: AbortSignal, ): Promise>; + /** Retry cleanup retained after a trace could not verify process exit. */ + close(): Promise>; } /** Remediation for a target LLDB may not debug. */ @@ -167,6 +168,8 @@ const ATTACH_REMEDIATION = /** Production tracer: `lldb --batch` running the REA LLDB bridge. */ export class LldbCallTracer implements NativeCallTracer { private readonly environment: NodeJS.ProcessEnv; + #tail: Promise = Promise.resolve(); + #pendingCleanup: RetainedLldbResources | undefined; constructor( environment: Readonly, private readonly launch: typeof runLldb = runLldb, @@ -183,6 +186,15 @@ export class LldbCallTracer implements NativeCallTracer { request: Parameters[0], signal?: AbortSignal, ): Promise> { + return this.#serialize(() => this.#trace(request, signal)); + } + + async #trace( + request: Parameters[0], + signal?: AbortSignal, + ): Promise> { + const pendingCleanup = await this.#retryPendingCleanup(); + if (pendingCleanup !== undefined) return err(pendingCleanup); const lldb = await this.resolveTool("lldb", signal); if (!lldb.ok) return err( @@ -196,30 +208,155 @@ export class LldbCallTracer implements NativeCallTracer { ), ); const root = await PrivateRuntimeRoot.create({ prefix: "rea-lldb-" }); + const paths = lldbRuntimePaths(root.path); + let result: Result; try { - return await this.#run(lldb.value, root.path, request, signal); - } finally { - await root.close(); + result = await this.#run(lldb.value, root, request, signal); + } catch (cause: unknown) { + const partialObservation = await partialNativeObservation({ + request, + pidPath: paths.pid, + stdoutPath: paths.stdoutCapture, + stderrPath: paths.stderrCapture, + observationPath: paths.observations, + version: null, + reason: "tracer-failure", + }); + const retained = this.#pendingCleanup; + const resources = + retained === undefined ? [] : await lldbCleanupResources(retained); + result = err( + new ProviderAdapterError(PROVIDER, OPERATION, { + cause, + diagnostics: { + reason: cause instanceof Error ? cause.message : String(cause), + }, + partialObservation, + ...(retained === undefined + ? {} + : { + cleanup: { + reason: "LLDB trace failed while owned resources remain", + resources, + }, + }), + }), + ); } + if (this.#pendingCleanup?.root === root) return result; + try { + await root.close(); + } catch (cause: unknown) { + this.#pendingCleanup = { + root, + supervisor: undefined, + pidPath: paths.pid, + targetIdentity: undefined, + processStopped: true, + targetStopped: true, + }; + const partialObservation = result.ok + ? partialObservationFromTrace(request, result.value) + : result.error.partialObservation; + return err( + new ProviderCleanupError( + PROVIDER, + [root.path], + { + reason: cause instanceof Error ? cause.message : String(cause), + ...(result.ok + ? {} + : { + execution_failure: projectAnalysisError(result.error).code, + primary_failure: result.error.message, + }), + }, + { + operation: OPERATION, + cause: result.ok + ? cause + : new AggregateError( + [result.error, cause], + "LLDB trace and runtime cleanup both failed", + ), + ...(partialObservation === undefined ? {} : { partialObservation }), + }, + ), + ); + } + return result; + } + + /** Retry retained LLDB cleanup without abandoning its process or runtime owner. */ + async close(): Promise> { + return this.#serialize(() => this.#close()); + } + + async #close(): Promise> { + const failure = await this.#retryPendingCleanup(); + return failure === undefined ? ok(null) : err(failure); + } + + #serialize(operation: () => Promise): Promise { + const call = this.#tail.then(operation, operation); + this.#tail = call.then( + () => undefined, + () => undefined, + ); + return call; + } + + async #retryPendingCleanup(): Promise { + const owner = this.#pendingCleanup; + if (owner === undefined) return undefined; + let firstFailure: string | undefined; + if (!owner.processStopped) { + const stopped = await owner.supervisor?.stop(); + if (stopped?.status === "incomplete") firstFailure = stopped.reason; + else { + owner.supervisor?.dispose(); + owner.processStopped = true; + } + } + if (firstFailure === undefined && !owner.targetStopped) { + const targetTermination = await ensureTerminated( + owner.pidPath, + owner.targetIdentity, + true, + ); + if (targetTermination !== "terminated") + firstFailure = targetTerminationReason(targetTermination); + else owner.targetStopped = true; + } + if (firstFailure === undefined) { + try { + await owner.root.close(); + this.#pendingCleanup = undefined; + return undefined; + } catch (cause: unknown) { + firstFailure = cause instanceof Error ? cause.message : String(cause); + } + } + const resources = await lldbCleanupResources(owner); + return new ProviderCleanupError( + PROVIDER, + resources, + { + reason: firstFailure ?? "LLDB cleanup remains incomplete", + resources, + }, + { operation: OPERATION }, + ); } async #run( lldb: ResolvedTool, - directory: string, + root: PrivateRuntimeRoot, request: Parameters[0], signal?: AbortSignal, ): Promise> { - const paths = { - config: join(directory, "config.json"), - result: join(directory, "result.json"), - pid: join(directory, "pid"), - identityAck: join(directory, "identity.ack"), - stdout: join(directory, "stdout"), - stderr: join(directory, "stderr"), - stdoutCapture: join(directory, "stdout.capture"), - stderrCapture: join(directory, "stderr.capture"), - observations: join(directory, "observations.jsonl"), - }; + const directory = root.path; + const paths = lldbRuntimePaths(directory); const { input } = request; await writeFile( paths.config, @@ -251,7 +388,6 @@ export class LldbCallTracer implements NativeCallTracer { }), { mode: 0o600 }, ); - await prepareProcessOwnershipInspection(signal); const exited = await this.launch( lldb.path, [ @@ -271,15 +407,22 @@ export class LldbCallTracer implements NativeCallTracer { ...(signal === undefined ? {} : { signal }), }, ); + const retained: RetainedLldbResources = { + root, + supervisor: exited.cleanupOwner, + pidPath: paths.pid, + targetIdentity: exited.targetIdentity, + processStopped: exited.cleanupOwner === undefined, + targetStopped: !exited.targetLaunched, + }; + this.#pendingCleanup = retained; const output = exited.kind === "exited" ? await readTracerOutput(paths.result) : undefined; - let targetTermination = await ensureTerminated( - paths.pid, - exited.targetIdentity, - true, - ); + let targetTermination: TargetTermination = exited.targetLaunched + ? await ensureTerminated(paths.pid, exited.targetIdentity, true) + : "terminated"; // These two bridge outcomes are emitted before a target process exists. // Other missing-PID cases, including a missing result, remain unknown. if ( @@ -288,6 +431,9 @@ export class LldbCallTracer implements NativeCallTracer { (await readTargetPid(paths.pid)) === undefined ) targetTermination = "terminated"; + retained.targetStopped = targetTermination === "terminated"; + if (retained.processStopped && retained.targetStopped) + this.#pendingCleanup = undefined; const terminated = targetTermination === "terminated"; const cleanupDetails = [ exited.cleanupFailure, @@ -298,16 +444,7 @@ export class LldbCallTracer implements NativeCallTracer { ? undefined : { reason: cleanupDetails.join("; "), - resources: [ - ...(exited.cleanupFailure === undefined - ? [] - : ["lldb-process-group"]), - ...(terminated - ? [] - : [ - `native-target:${String((await readTargetPid(paths.pid)) ?? "unknown")}`, - ]), - ], + resources: await lldbCleanupResources(retained), }; const partialReason = exited.kind === "cancelled" @@ -357,20 +494,38 @@ export class LldbCallTracer implements NativeCallTracer { ); if (exited.cleanupFailure !== undefined) return err( - new ProviderAdapterError(PROVIDER, OPERATION, { - cleanup: lifecycleCleanup ?? { - reason: exited.cleanupFailure, - resources: ["lldb-process-group"], - }, - diagnostics: { + new ProviderCleanupError( + PROVIDER, + await lldbCleanupResources(retained), + { reason: "LLDB process-group cleanup could not be verified", cleanup_failure: exited.cleanupFailure, ...(lifecycleCleanup === undefined ? {} : { target_cleanup: lifecycleCleanup }), }, - partialObservation: await getPartialObservation(), - }), + { + operation: OPERATION, + partialObservation: await getPartialObservation(), + }, + ), + ); + if (!terminated) + return err( + new ProviderCleanupError( + PROVIDER, + await lldbCleanupResources(retained), + { + reason: "LLDB target process termination could not be verified", + ...(lifecycleCleanup === undefined + ? {} + : { target_cleanup: lifecycleCleanup }), + }, + { + operation: OPERATION, + partialObservation: await getPartialObservation(), + }, + ), ); if (output === undefined) return err( @@ -532,6 +687,43 @@ const partialNativeObservation = async (options: { }); }; +const partialObservationFromTrace = ( + request: Parameters[0], + trace: NativeCallTrace, +): NativeCallPartialObservation => + projectLldbPartialObservation({ + target: { + path: request.executable, + sha256: request.expectedSha256, + architecture: request.architecture, + arguments: request.input.arguments, + environment: request.input.environment, + working_directory: request.input.working_directory ?? null, + }, + pid: trace.run.pid, + stdout: trace.stdout, + stderr: trace.stderr, + version: trace.run.version, + journal: { + events: trace.run.events, + otherStops: trace.run.other_stops, + limitations: [], + }, + reason: "cleanup-failure", + }); + +const lldbRuntimePaths = (directory: string) => ({ + config: join(directory, "config.json"), + result: join(directory, "result.json"), + pid: join(directory, "pid"), + identityAck: join(directory, "identity.ack"), + stdout: join(directory, "stdout"), + stderr: join(directory, "stderr"), + stdoutCapture: join(directory, "stdout.capture"), + stderrCapture: join(directory, "stderr.capture"), + observations: join(directory, "observations.jsonl"), +}); + const readTracerOutput = async ( path: string, ): Promise | undefined> => { @@ -588,24 +780,43 @@ const capturedOutput = async ( } }; -type LldbExit = +interface RetainedLldbResources { + readonly root: PrivateRuntimeRoot; + readonly supervisor: ProviderProcessSupervisor | undefined; + readonly pidPath: string; + readonly targetIdentity: string | undefined; + processStopped: boolean; + targetStopped: boolean; +} + +type LldbOutcome = | { readonly kind: "exited"; readonly exitCode: number | null; readonly output: string; - readonly targetIdentity: string | undefined; - readonly cleanupFailure: string | undefined; } - | { - readonly kind: "cancelled"; - readonly targetIdentity: string | undefined; - readonly cleanupFailure: string | undefined; - } - | { - readonly kind: "timeout"; - readonly targetIdentity: string | undefined; - readonly cleanupFailure: string | undefined; - }; + | { readonly kind: "cancelled" } + | { readonly kind: "timeout" }; + +type LldbExit = LldbOutcome & { + /** False proves LLDB never started and could not have created a target. */ + readonly targetLaunched: boolean; + readonly targetIdentity: string | undefined; + readonly cleanupFailure: string | undefined; + readonly cleanupOwner?: ProviderProcessSupervisor; +}; + +const lldbCleanupResources = async ( + owner: RetainedLldbResources, +): Promise => [ + ...(owner.processStopped ? [] : ["lldb-process-group"]), + ...(owner.targetStopped + ? [] + : [ + `native-target:${String((await readTargetPid(owner.pidPath)) ?? "unknown")}`, + ]), + owner.root.path, +]; /** Supervise LLDB as an owned process group while observing its target PID identity. */ const runLldb = async ( @@ -622,6 +833,7 @@ const runLldb = async ( if (options.signal?.aborted === true) return { kind: "cancelled", + targetLaunched: false, targetIdentity: undefined, cleanupFailure: undefined, }; @@ -631,6 +843,23 @@ const runLldb = async ( ); let targetIdentity: string | undefined; let supervisor: ProviderProcessSupervisor | undefined; + let retained = false; + const finish = async (outcome: LldbOutcome): Promise => { + const stopped = await supervisor?.stop(); + retained = stopped?.status === "incomplete"; + return { + ...(deadline.interruption === undefined + ? outcome + : { kind: deadline.interruption }), + targetLaunched: supervisor !== undefined, + targetIdentity, + cleanupFailure: + stopped?.status === "incomplete" ? stopped.reason : undefined, + ...(retained && supervisor !== undefined + ? { cleanupOwner: supervisor } + : {}), + }; + }; try { const launched = await spawnOwnedProviderProcess({ command: executable, @@ -668,52 +897,36 @@ const runLldb = async ( identityAckWritten = true; } const interruption = deadline.interruption; - if (interruption !== undefined) { - const stopped = await supervisor.stop(); - const cleanupFailure = - stopped.status === "incomplete" ? stopped.reason : undefined; - return interruption === "cancelled" - ? { kind: "cancelled", targetIdentity, cleanupFailure } - : { kind: "timeout", targetIdentity, cleanupFailure }; - } + if (interruption !== undefined) + return await finish({ kind: interruption }); await waitForAbortableDelay(20, deadline.signal); } const snapshot = supervisor.snapshot(); - const stopped = await supervisor.stop(); - const cleanupFailure = - stopped.status === "incomplete" ? stopped.reason : undefined; const interruption = deadline.interruption; - if (interruption === "cancelled") - return { kind: "cancelled", targetIdentity, cleanupFailure }; - if (interruption === "timeout") - return { kind: "timeout", targetIdentity, cleanupFailure }; - return { - kind: "exited", - exitCode: snapshot.exitCode ?? null, - output: `${snapshot.stdout.text}${snapshot.stderr.text}`.slice( - -DIAGNOSTIC_BYTES, - ), - targetIdentity, - cleanupFailure, - }; + return await finish( + interruption === undefined + ? { + kind: "exited", + exitCode: snapshot.exitCode ?? null, + output: `${snapshot.stdout.text}${snapshot.stderr.text}`.slice( + -DIAGNOSTIC_BYTES, + ), + } + : { kind: interruption }, + ); } catch (cause: unknown) { const interruption = deadline.interruption; - const stopped = await supervisor?.stop(); - const cleanupFailure = - stopped?.status === "incomplete" ? stopped.reason : undefined; - return interruption === "timeout" - ? { kind: "timeout", targetIdentity, cleanupFailure } - : interruption === "cancelled" - ? { kind: "cancelled", targetIdentity, cleanupFailure } - : { + return await finish( + interruption === undefined + ? { kind: "exited", exitCode: null, output: cause instanceof Error ? cause.message : String(cause), - targetIdentity, - cleanupFailure, - }; + } + : { kind: interruption }, + ); } finally { - supervisor?.dispose(); + if (!retained) supervisor?.dispose(); deadline.dispose(); } }; @@ -724,8 +937,10 @@ const runLldb = async ( */ const readTargetPid = async (pidPath: string): Promise => { try { - const pid = Number.parseInt(await readFile(pidPath, "utf8"), 10); - return Number.isSafeInteger(pid) && pid > 0 ? pid : undefined; + const contents = (await readFile(pidPath, "utf8")).trim(); + if (!/^[1-9]\d*$/u.test(contents)) return undefined; + const pid = Number(contents); + return Number.isSafeInteger(pid) ? pid : undefined; } catch { return undefined; } diff --git a/src/native/NativeCallObservation.test.ts b/src/native/NativeCallObservation.test.ts index fcff10565..15b9b2604 100644 --- a/src/native/NativeCallObservation.test.ts +++ b/src/native/NativeCallObservation.test.ts @@ -7,7 +7,7 @@ import { describe, expect, it } from "vitest"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; -import { err, type Result } from "../domain/result.js"; +import { ok, type Result } from "../domain/result.js"; import type { NativeCallTrace, NativeCallTracer } from "./LldbCallTracer.js"; import { observeNativeCalls } from "./NativeCallObservation.js"; import { nativeMachoTarget } from "../../tests/fixtures/nativeCommands.js"; @@ -20,6 +20,10 @@ class NeverStartedTracer implements NativeCallTracer { this.calls += 1; throw new Error("tracer must not run after hash cancellation"); } + + close(): Promise> { + return Promise.resolve(ok(null)); + } } describe("native call target digest cancellation", () => { diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 0d25f969c..12c94814f 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -82,6 +82,7 @@ import { export class NativeMacOSProvider implements AnalysisProvider { readonly #capabilities: readonly CapabilityDescriptor[]; readonly #environment: NodeJS.ProcessEnv; + readonly #tracerFactory: () => NativeCallTracer; constructor( environment: Readonly, @@ -89,9 +90,11 @@ export class NativeMacOSProvider implements AnalysisProvider { environment, ), platform: NodeJS.Platform = process.platform, - private readonly tracer: NativeCallTracer = new LldbCallTracer(environment), + tracerFactory?: () => NativeCallTracer, ) { this.#environment = snapshotEnvironment(environment, platform); + this.#tracerFactory = + tracerFactory ?? (() => new LldbCallTracer(this.#environment)); this.#capabilities = nativeMacOSCapabilities(platform); } @@ -107,7 +110,7 @@ export class NativeMacOSProvider implements AnalysisProvider { return new NativeMacOSClient( target, this.runner, - this.tracer, + this.#tracerFactory(), this.#environment, ); } @@ -246,7 +249,7 @@ class NativeMacOSClient implements AnalysisClient { } close(): Promise> { - return Promise.resolve(ok(null)); + return this.tracer.close(); } #dispatch( diff --git a/src/native/NativeMacOSProviderInput.test.ts b/src/native/NativeMacOSProviderInput.test.ts index df4131e69..a795bf76c 100644 --- a/src/native/NativeMacOSProviderInput.test.ts +++ b/src/native/NativeMacOSProviderInput.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { NativeMacOSProvider } from "./NativeMacOSProvider.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import { ok } from "../domain/result.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { NativeFixtureRunner, @@ -57,4 +58,33 @@ describe("native dispatch metadata input boundary", () => { }); } }); + + it("closes only the native call tracer owned by that client", async () => { + const closes = [0, 0]; + let next = 0; + const provider = new NativeMacOSProvider( + {}, + new NativeFixtureRunner(), + "darwin", + () => { + const index = next++; + return { + trace: async () => { + throw new Error("trace is unused in this lifecycle test"); + }, + close: async () => { + closes[index] = (closes[index] ?? 0) + 1; + return ok(null); + }, + }; + }, + ); + const first = provider.createClient(nativeMachoTarget("/first")); + const second = provider.createClient(nativeMachoTarget("/second")); + + expect((await first.close()).ok).toBe(true); + expect(closes).toEqual([1, 0]); + expect((await second.close()).ok).toBe(true); + expect(closes).toEqual([1, 1]); + }); }); diff --git a/tests/boundary/mcp/nativeCallObservationMcp.test.ts b/tests/boundary/mcp/nativeCallObservationMcp.test.ts index cecd3b2f4..52c26c282 100644 --- a/tests/boundary/mcp/nativeCallObservationMcp.test.ts +++ b/tests/boundary/mcp/nativeCallObservationMcp.test.ts @@ -81,6 +81,7 @@ const tracer: NativeCallTracer & { calls: number } = { }), ); }, + close: () => Promise.resolve(ok(null)), }; it("routes observe_native_calls through MCP with schema-checked input and output", async () => { @@ -88,7 +89,12 @@ it("routes observe_native_calls through MCP with schema-checked input and output const path = join(directory, "Tool"); await writeFile(path, machoHeader()); const session = createTestBinarySession( - new NativeMacOSProvider({}, new NativeFixtureRunner(), "darwin", tracer), + new NativeMacOSProvider( + {}, + new NativeFixtureRunner(), + "darwin", + () => tracer, + ), ); const server = createServer({ kind: "session", session }); const client = new Client({ name: "native-calls-mcp-test", version: "1" }); @@ -138,6 +144,62 @@ it("routes observe_native_calls through MCP with schema-checked input and output } }); +const partialObservationFor = (options: { + readonly path: string; + readonly sha256: string; + readonly arguments: string[]; + readonly environment: Record; + readonly workingDirectory: string; + readonly reason: "cancelled" | "cleanup-failure"; +}) => + nativeCallPartialObservationSchema.parse({ + kind: "native-call-observation", + target: { + path: options.path, + sha256: options.sha256, + architecture: "arm64", + arguments: options.arguments, + environment: options.environment, + working_directory: options.workingDirectory, + }, + process: { + pid: 4242, + stdout: { + text: "partial target output", + bytes: 21, + truncated: true, + complete: false, + }, + stderr: { + text: "debugger diagnostic", + bytes: 19, + truncated: false, + complete: false, + }, + other_stops: ["signal SIGSTOP"], + }, + debugger: { version: "lldb-fixture" }, + events: [ + { + sequence: 0, + elapsed_ms: 4, + thread_id: 3, + breakpoint_index: 0, + load_address: "0x1000", + file_address: "0x1000", + module: "Tool", + module_path: options.path, + symbol: "open", + receiver_class: null, + selector: null, + registers: [{ name: "x0", value: "0x2" }], + backtrace: [], + }, + ], + coverage: { status: "partial", reason: options.reason }, + limitations: ["Observation ended before the requested window completed."], + }); + it.each([ ["cancelled", "cancelled", undefined], [ @@ -161,57 +223,16 @@ it.each([ const launchArguments = ["--fixture", "selected argument"]; const launchEnvironment = { REA_NATIVE_FIXTURE: "selected value" }; const workingDirectory = directory; - const partialObservation = nativeCallPartialObservationSchema.parse({ - kind: "native-call-observation", - target: { - path, - sha256, - architecture: "arm64", - arguments: launchArguments, - environment: launchEnvironment, - working_directory: workingDirectory, - }, - process: { - pid: 4242, - stdout: { - text: "partial target output", - bytes: 21, - truncated: true, - complete: false, - }, - stderr: { - text: "debugger diagnostic", - bytes: 19, - truncated: false, - complete: false, - }, - other_stops: ["signal SIGSTOP"], - }, - debugger: { version: "lldb-fixture" }, - events: [ - { - sequence: 0, - elapsed_ms: 4, - thread_id: 3, - breakpoint_index: 0, - load_address: "0x1000", - file_address: "0x1000", - module: "Tool", - module_path: path, - symbol: "open", - receiver_class: null, - selector: null, - registers: [{ name: "x0", value: "0x2" }], - backtrace: [], - }, - ], - coverage: { - status: "partial", - reason: cleanup === undefined ? "cancelled" : "cleanup-failure", - }, - limitations: ["Observation ended before the requested window completed."], + const partialObservation = partialObservationFor({ + path, + sha256, + arguments: launchArguments, + environment: launchEnvironment, + workingDirectory, + reason: cleanup === undefined ? "cancelled" : "cleanup-failure", }); const tracer: NativeCallTracer = { + close: () => Promise.resolve(ok(null)), trace: () => Promise.resolve( err( @@ -231,7 +252,12 @@ it.each([ ), }; const session = createTestBinarySession( - new NativeMacOSProvider({}, new NativeFixtureRunner(), "darwin", tracer), + new NativeMacOSProvider( + {}, + new NativeFixtureRunner(), + "darwin", + () => tracer, + ), ); const server = createServer({ kind: "session", session }); const client = new Client({ diff --git a/tests/boundary/providers/native/LldbCallTracer.cleanup.test.ts b/tests/boundary/providers/native/LldbCallTracer.cleanup.test.ts new file mode 100644 index 000000000..b3e365d5c --- /dev/null +++ b/tests/boundary/providers/native/LldbCallTracer.cleanup.test.ts @@ -0,0 +1,250 @@ +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { access, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js"; +import { ProviderCleanupError } from "../../../../src/domain/providerCleanupError.js"; +import { ok } from "../../../../src/domain/result.js"; +import { cleanupOwnedProcessGroup } from "../../../../src/process/ProcessOwnership.js"; +import { observeProcessStartIdentity } from "../../../../src/process/ProcessOwnershipObservation.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../../../../src/process/ProviderProcess.js"; +import type { NativeCallTracer } from "../../../../src/native/LldbCallTracer.js"; +import { LldbCallTracer } from "../../../../src/native/LldbCallTracer.js"; +import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; +import { + NativeFixtureRunner, + nativeMachoTarget, +} from "../../../fixtures/nativeCommands.js"; + +const request: Parameters[0] = { + executable: "/fixture/target", + architecture: "arm64", + expectedSha256: "0".repeat(64), + input: { + breakpoints: [{ kind: "function", name: "main" }], + arguments: [], + environment: {}, + duration_ms: 100, + max_events: 1, + argument_registers: 0, + backtrace_frames: 0, + }, +}; +const tool = { path: process.execPath, sha256: "0".repeat(64) }; + +const createTracer = ( + launch?: ConstructorParameters[1], +): LldbCallTracer => + new LldbCallTracer(process.env, launch, async () => ok(tool)); + +const createClient = (tracer: NativeCallTracer) => + new NativeMacOSProvider( + {}, + new NativeFixtureRunner(), + "darwin", + () => tracer, + ).createClient(nativeMachoTarget(request.executable)); + +const runtimeRoot = (error: ProviderCleanupError): string => { + const root = error.cleanupResources.find((resource) => + resource.startsWith(`${tmpdir()}/rea-lldb-`), + ); + if (root === undefined) throw new Error("Expected retained runtime root"); + return root; +}; + +const liveChild = async () => { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + stdio: "ignore", + }); + await once(child, "spawn"); + if (child.pid === undefined) throw new Error("Fixture process had no PID"); + return { child, pid: child.pid }; +}; + +const stopChild = async (child: ReturnType) => { + if (child.exitCode === null && child.signalCode === null) { + const exited = once(child, "exit"); + child.kill("SIGKILL"); + await exited; + } +}; + +it.skipIf(process.platform === "win32")( + "retains a real LLDB process owner and runtime until a later cleanup retry succeeds", + async () => { + let cleanupFailures = 2; + let ownedPid: number | undefined; + const tracer = createTracer(async (_exe, _args, options) => { + const target = spawn(process.execPath, ["-e", "process.exit(0)"], { + stdio: "ignore", + }); + await once(target, "exit"); + if (target.pid === undefined) + throw new Error("Fixture target had no PID"); + await writeFile(options.pidPath, String(target.pid)); + const launched = await spawnOwnedProviderProcess({ + command: process.execPath, + arguments: ["-e", "setInterval(() => {}, 1000)"], + runId: randomUUID(), + hostEnvironment: options.environment, + }); + ownedPid = launched.process.pid; + const supervisor = new ProviderProcessSupervisor({ + ...launched, + ownsProcessLifetime: true, + cleanup: async () => { + if (cleanupFailures-- > 0) + return { + cleaned: false, + reason: "fixture transient cleanup failure", + }; + return cleanupOwnedProcessGroup(launched.ownership); + }, + }); + const stopped = await supervisor.stop(); + return { + kind: "exited" as const, + targetLaunched: true, + exitCode: null, + output: "", + targetIdentity: undefined, + cleanupFailure: + stopped.status === "incomplete" ? stopped.reason : undefined, + ...(stopped.status === "incomplete" + ? { cleanupOwner: supervisor } + : {}), + }; + }); + const client = createClient(tracer); + + try { + const trace = await tracer.trace(request); + expect(trace.ok).toBe(false); + if (trace.ok) throw new Error("Expected incomplete cleanup"); + expect(trace.error).toBeInstanceOf(ProviderCleanupError); + if (!(trace.error instanceof ProviderCleanupError)) return; + expect(trace.error.cleanupResources).toContain("lldb-process-group"); + expect(trace.error.partialObservation).toBeDefined(); + const root = runtimeRoot(trace.error); + await expect(access(root)).resolves.toBeUndefined(); + if (ownedPid === undefined) throw new Error("Expected owned process PID"); + const pid = ownedPid; + expect(() => process.kill(pid, 0)).not.toThrow(); + expect((await client.close()).ok).toBe(false); + expect(() => process.kill(pid, 0)).not.toThrow(); + await expect(access(root)).resolves.toBeUndefined(); + expect((await client.close()).ok).toBe(true); + expect(() => process.kill(pid, 0)).toThrow(); + await expect(access(root)).rejects.toThrow(); + } finally { + await client.close(); + } + }, +); + +it.skipIf(process.platform === "win32")( + "does not retain a target when cancellation proves LLDB never launched", + async () => { + const controller = new AbortController(); + const cancelledTracer = new LldbCallTracer( + process.env, + undefined, + async () => { + controller.abort(); + return ok(tool); + }, + ); + const result = await cancelledTracer.trace(request, controller.signal); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancelled trace"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect((await cancelledTracer.close()).ok).toBe(true); + }, +); + +it.skipIf(process.platform === "win32")( + "retains an unidentified debuggee and runtime until exit can be confirmed", + async () => { + const { child, pid } = await liveChild(); + const tracer = createTracer(async (_exe, _args, options) => { + await writeFile(options.pidPath, String(pid)); + return { + kind: "exited", + targetLaunched: true, + exitCode: 0, + output: "", + targetIdentity: undefined, + cleanupFailure: undefined, + }; + }); + const client = createClient(tracer); + + try { + const trace = await tracer.trace(request); + expect(trace.ok).toBe(false); + if (trace.ok) throw new Error("Expected unverified target cleanup"); + expect(trace.error).toBeInstanceOf(ProviderCleanupError); + if (!(trace.error instanceof ProviderCleanupError)) return; + expect(trace.error.cleanupResources).toContain(`native-target:${pid}`); + const root = runtimeRoot(trace.error); + await expect(access(root)).resolves.toBeUndefined(); + expect(() => process.kill(pid, 0)).not.toThrow(); + expect((await client.close()).ok).toBe(false); + expect(() => process.kill(pid, 0)).not.toThrow(); + await expect(access(root)).resolves.toBeUndefined(); + await stopChild(child); + expect((await client.close()).ok).toBe(true); + await expect(access(root)).rejects.toThrow(); + } finally { + await stopChild(child); + await client.close(); + } + }, +); + +it.skipIf(process.platform === "win32")( + "rejects malformed PID authority and retries after the target exits", + async () => { + const { child, pid } = await liveChild(); + const identity = await observeProcessStartIdentity(pid); + if (identity?.state !== "readable") + throw new Error("Host could not read fixture process identity"); + const tracer = createTracer(async (_exe, _args, options) => { + await writeFile(options.pidPath, `${pid}junk`); + return { + kind: "exited", + targetLaunched: true, + exitCode: 0, + output: "", + targetIdentity: identity.identity, + cleanupFailure: undefined, + }; + }); + const client = createClient(tracer); + + try { + const trace = await tracer.trace(request); + expect(trace.ok).toBe(false); + if (trace.ok) throw new Error("Expected malformed PID cleanup failure"); + expect(trace.error).toBeInstanceOf(ProviderCleanupError); + if (!(trace.error instanceof ProviderCleanupError)) return; + expect(trace.error.cleanupResources).toContain("native-target:unknown"); + expect(() => process.kill(pid, 0)).not.toThrow(); + const root = runtimeRoot(trace.error); + await stopChild(child); + await writeFile(join(root, "pid"), String(pid)); + expect((await client.close()).ok).toBe(true); + } finally { + await stopChild(child); + await client.close(); + } + }, +); diff --git a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts index 6f10e6b20..a8d65dedb 100644 --- a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts +++ b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts @@ -12,6 +12,12 @@ import type { NativeCallTracer } from "../../../../src/native/LldbCallTracer.js" import { LldbCallTracer } from "../../../../src/native/LldbCallTracer.js"; import { err, ok } from "../../../../src/domain/result.js"; import { NativeCommandFailure } from "../../../../src/native/CommandRunner.js"; +import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; +import { createDeferred } from "../../../fixtures/binarySession.js"; +import { + NativeFixtureRunner, + nativeMachoTarget, +} from "../../../fixtures/nativeCommands.js"; const EVENT = { sequence: 0, @@ -83,11 +89,13 @@ const fixtureTracer = ( return outcome === "cancelled" ? { kind: "cancelled", + targetLaunched: false, targetIdentity: undefined, cleanupFailure: undefined, } : { kind: "exited", + targetLaunched: false, exitCode: 1, output: "bridge reported target integrity failure", targetIdentity: undefined, @@ -108,6 +116,56 @@ const fixtureTracer = ( }; describe("LLDB failure retention through the production tracer", () => { + it("waits for an active trace before client close removes its runtime root", async () => { + const launchGate = createDeferred(); + const launchStarted = createDeferred(); + const tracer = new LldbCallTracer( + {}, + async (_executable, arguments_) => { + const { config } = await configFromArguments(arguments_); + await writeFile( + config.observation_path, + `${JSON.stringify({ kind: "event", event: EVENT })}\n`, + ); + await writeFile(config.stdout_capture_path, "captured prefix"); + launchStarted.resolve(); + await launchGate.promise; + return { + kind: "exited", + targetLaunched: false, + exitCode: 0, + output: "", + targetIdentity: undefined, + cleanupFailure: undefined, + }; + }, + async () => ok(fakeTool), + ); + const client = new NativeMacOSProvider( + {}, + new NativeFixtureRunner(), + "darwin", + () => tracer, + ).createClient(nativeMachoTarget(request.executable)); + const tracePromise = tracer.trace(request); + await launchStarted.promise; + let closeFinished = false; + const closePromise = client.close().then((result) => { + closeFinished = true; + return result; + }); + await Promise.resolve(); + expect(closeFinished).toBe(false); + launchGate.resolve(); + const [trace, close] = await Promise.all([tracePromise, closePromise]); + expect(close.ok).toBe(true); + if (trace.ok) throw new Error("expected missing LLDB result"); + expect(trace.error.partialObservation).toMatchObject({ + events: [EVENT], + process: { stdout: { text: "captured prefix", complete: false } }, + }); + }); + it("names the missing LLDB requirement instead of asking for another target", async () => { const tracer = new LldbCallTracer( {}, diff --git a/tests/boundary/providers/native/nativeCallObservation.test.ts b/tests/boundary/providers/native/nativeCallObservation.test.ts index 0ae599748..c915da0d9 100644 --- a/tests/boundary/providers/native/nativeCallObservation.test.ts +++ b/tests/boundary/providers/native/nativeCallObservation.test.ts @@ -108,6 +108,10 @@ class FixtureTracer implements NativeCallTracer { this.requests.push(request); return Promise.resolve(this.outcome); } + + close() { + return Promise.resolve(ok(null)); + } } const BREAKPOINTS = [ @@ -131,7 +135,7 @@ const observe = async ( target?: Awaited>, signal?: AbortSignal, ) => - new NativeMacOSProvider({}, new NativeFixtureRunner(), "darwin", tracer) + new NativeMacOSProvider({}, new NativeFixtureRunner(), "darwin", () => tracer) .createClient(target ?? (await fixtureTarget())) .execute( "observe_native_calls", @@ -421,7 +425,7 @@ describe("observe_native_calls failures", () => { {}, new NativeFixtureRunner(), "darwin", - tracer, + () => tracer, ) .createClient({ path: target.path,