diff --git a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md index d239687b9..97e873a04 100644 --- a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md +++ b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md @@ -38,3 +38,11 @@ or absence: Process captures are opt-in behavioral evidence, not a security sandbox. V3 captures cannot be upgraded to V4; rerun the original scenario. Distinguish root exit from descendant settlement and require freshness when the task needs it. +A timed-out target that must write a final report needs `finalization_ms`: REA +sends SIGTERM to the captured root, keeps observing, and sends SIGKILL once +only if the target has not exited by the end of the interval. +`exit.finalization.signals` lists each attempt with its delivery result; the +observed exit signal stays in `exit.signal`, `exit.code` is null for every +deadline exit, and the code of a finalizer that exited on its own is in +`exit.finalization.exit_code` (null on a signal exit or when no exit was +observed). diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 736a86198..69ce9dce0 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -167,6 +167,9 @@ Native call tracing and process capture retain available observations in `details.partial_observation` on failure, including when cleanup succeeds. The observation reports its partial coverage; cleanup details describe host state separately from the execution failure. +A process capture that fails during a finalization interval keeps every signal +attempt and its delivery result in `details.partial_observation`, with +`elapsed_ms` and `exit_code` null when no exit was observed. Derived comparisons and reconstruction verification yield before computation and before publication, so cancellation cannot race with successful Evidence. diff --git a/docs/process-capture.md b/docs/process-capture.md index 772b39602..198ca264d 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -181,6 +181,100 @@ run the same owned-process cleanup path. Settlement reports whether the sampled process group quiesced or whether cleanup was needed or unverifiable; sampling cannot prove that every short-lived or detached descendant was seen. +### Finalization interval + +By default a deadline sends `SIGKILL` at once. Set `finalization_ms` to give the +target time to finish: when `timeout_ms` or `idle_timeout_ms` fires, REA sends +`SIGTERM`, keeps capturing output and selected files, and sends `SIGKILL` once +if the interval ends first. Cancellation is not delayed: it sends `SIGKILL` +immediately, also during finalization, and ends the run as cancelled. + +```json +{ + "executable": "node", + "arguments": ["./job.mjs"], + "filesystem_observation_paths": ["./reports"], + "timeout_ms": 30000, + "finalization_ms": 1500 +} +``` + +REA binds the captured root's first start identity to its ownership token, +checking the identity before and after reading that token. A missing or +mismatched token, or an identity change during validation, leaves the identity +unverified rather than adopting a replacement process. Both finalization +signals recheck that retained identity immediately before the call; the check +and signal remain separate operating-system operations. REA does not fall back +to a bare PID kill for either finalization signal; failed identity or ownership +validation is recorded as `unverified` with its reason. Cancellation before any +deadline keeps the existing terminal kill. Descendants are not signalled during +finalization; owned-process cleanup handles them afterwards. + +`exit.reason` keeps the initiating deadline (`timeout` or `idle_timeout`). +`exit.code` is `null` for every deadline exit; the code of an exit observed +during finalization is `exit.finalization.exit_code`, and the observed signal +stays in `exit.signal`. `exit.finalization` records what was observed and is +absent when no finalization was attempted: + +```json +{ + "code": null, + "signal": 0, + "reason": "timeout", + "finalization": { + "requested_ms": 1500, + "elapsed_ms": 12, + "exit_code": 0, + "signals": [ + { "signal": "SIGTERM", "sent_at_ms": 0, "delivery": "signaled" } + ] + } +} +``` + +- `signals` lists each attempt in order: `SIGTERM` first, at most one `SIGKILL`. + `sent_at_ms` is the attempt time on a monotonic clock, relative to the start of + finalization, also for an attempt that could not be delivered. +- `delivery` is `signaled` (the identity matched and the signal call returned), + `gone` (no live process matched), `identity-changed` (the PID now belongs to a + different process, which was not signalled) or `unverified` (the identity could + not be established or the call failed). None of them proves the process + exited. + An `unverified` attempt may carry a `reason` string naming why: an unavailable + identity and its reason, an inspection or signal error, or delivery that did + not settle within the bound (1000 ms). An attempt recorded `unverified` after + that bound can still complete later; it stays identity-checked, and its late + result is not added to the record. +- `elapsed_ms` is when REA observed the exit, relative to the start of + finalization, or `null` when no exit was observed. The PTY layer can deliver the + exit slightly after the operating system ended the process. +- `exit_code` is the code the PTY reported when the exit was observed during + finalization and was not caused by a signal (`exit.signal` is `0`); a + finalizer that exits with `3` records `3`, one that exits cleanly records `0`. + It is `null` when no exit was observed or the target ended on a signal, and it + is never inferred. A target that exited on its own near the deadline is + recorded the same way. +- The observed signal stays in `exit.signal`; `exit.code` is `null` for every + deadline exit. The record does not attribute an exit to a particular signal, + and a `SIGKILL` from a scenario event next to the escalation cannot be told + apart from it. +- If the `SIGKILL` cannot be delivered, REA stops waiting for an exit, keeps every + observation collected so far with the delivery results, and releases the run + through owned-process cleanup. +- The `SIGKILL` is not sequenced after the `SIGTERM` delivery result: the two + deliveries are independent, because cancellation must not wait for a slow + identity check. + +`timeout_ms + finalization_ms + settle_ms` must stay at or below +`Number.MAX_SAFE_INTEGER` (9007199254740991), the largest total exactly +representable as a JavaScript number; the scenario is rejected otherwise. This is +a representability limit, not a timer limit. The nominal lifecycle budget is that +sum; polling, snapshotting and cleanup lie outside it. Scenario `events` keep +being dispatched during finalization and are recorded as usual. `finalization_ms` +of `0`, the default, keeps today's behaviour and the committed identity of every +existing scenario. `elapsed_ms` and `sent_at_ms` are timing data and take part in +exit comparison like `settlement.elapsed_ms`. + Every capture requires `truncation_details`, with separate accounting for: - `raw_terminal`: original UTF-8 PTY chunk bytes and observed/retained chunk diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index 65c29785f..ef99a809f 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -94,7 +94,7 @@ export const SESSION_TOOL_CONTRACTS = [ ), session( "capture_process_scenario", - "Run one caller-selected command under a PTY and return process capture Evidence with residual unknowns. The renderer admits at most 1,000,000 combined terminal cells, computed as columns × (rows + scrollback); every resize uses the selected scrollback and must fit the same budget. Supply a smaller terminal shape when that product is too large; dimensions are never clamped. A command name resolves through the inherited PATH; the working directory defaults to the caller's current directory; host environment variables are inherited with scenario overrides. Environment keys cannot contain '=' or NUL, and REA_PROCESS_RUN_ID is reserved for process ownership. Filesystem snapshots are opt-in through filesystem_observation_paths. The target runs with the current user's permissions; this is not a security sandbox.", + "Run one caller-selected command under a PTY and return process capture Evidence with residual unknowns. The renderer admits at most 1,000,000 combined terminal cells, computed as columns × (rows + scrollback); every resize uses the selected scrollback and must fit the same budget. Supply a smaller terminal shape when that product is too large; dimensions are never clamped. A command name resolves through the inherited PATH; the working directory defaults to the caller's current directory; host environment variables are inherited with scenario overrides. Environment keys cannot contain '=' or NUL, and REA_PROCESS_RUN_ID is reserved for process ownership. Filesystem snapshots are opt-in through filesystem_observation_paths. A positive finalization_ms makes a timeout or idle deadline send SIGTERM to the captured root through its start identity, keep observing, and send SIGKILL once after the interval; exit.finalization records each attempt with its delivery result and the observed exit, and cancellation still ends the run at once. The target runs with the current user's permissions; this is not a security sandbox.", processScenarioSchema, ), session( diff --git a/src/contracts/toolContractExamples.ts b/src/contracts/toolContractExamples.ts index 0d7e53113..42a6f4940 100644 --- a/src/contracts/toolContractExamples.ts +++ b/src/contracts/toolContractExamples.ts @@ -58,6 +58,8 @@ export const TOOL_EXAMPLE_OVERRIDES: Readonly< capture_process_scenario: { executable: "/usr/bin/true", working_directory: "/tmp", + timeout_ms: 30_000, + finalization_ms: 1_500, }, compare_process_captures: { left: PROCESS_CAPTURE_REFERENCE, diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.ts index 8cbfbd799..4c28f1407 100644 --- a/src/domain/process/processCapture.ts +++ b/src/domain/process/processCapture.ts @@ -2,7 +2,12 @@ import { z } from "zod"; import { jsonValueSchema } from "../jsonValue.js"; import { normalizationSchema } from "./processScenario.js"; -import { collectProcessCaptureIssues } from "./processCaptureValidation.js"; +import { + collectProcessCaptureIssues, + finalizationAgreementIssue, + finalizationConsistencyIssue, + finalizationManifestIssue, +} from "./processCaptureValidation.js"; import { filesystemCoverageSchema, processCaptureTruncationDetailsSchema, @@ -42,6 +47,21 @@ export interface InteractionEvent { readonly outcome: "dispatched" | "target_exited" | "failed"; } +/** Observed finalization attempts after a deadline fired. */ +export interface ProcessCaptureFinalization { + readonly requested_ms: number; + readonly elapsed_ms: number | null; + /** Code the PTY reported for an exit not caused by a signal; otherwise null. */ + readonly exit_code: number | null; + readonly signals: readonly { + readonly signal: "SIGTERM" | "SIGKILL"; + readonly sent_at_ms: number; + readonly delivery: "signaled" | "gone" | "identity-changed" | "unverified"; + /** Why delivery could not be verified; present only for unverified attempts. */ + readonly reason?: string | undefined; + }[]; +} + /** One filesystem state used for before/after comparison. */ interface FileStateIdentity { readonly path: string; @@ -209,6 +229,7 @@ export interface UnverifiedProcessCapture { readonly code: number | null; readonly signal: number | null; readonly reason: "exited" | "timeout" | "idle_timeout"; + readonly finalization?: ProcessCaptureFinalization | undefined; }; readonly settlement: VerifiedProcessSettlement; readonly process_samples: readonly ProcessSample[]; @@ -290,7 +311,11 @@ export interface IncompleteProcessCaptureObservations { readonly code: number | null; readonly signal: number | null; readonly reason: "exited" | "timeout" | "idle_timeout" | "cancelled"; + readonly finalization?: ProcessCaptureFinalization | undefined; }>; + readonly finalization?: + | PartialProcessObservationField + | undefined; readonly settlement: PartialProcessObservationField<{ readonly state: "quiesced" | "alive_at_deadline" | "unverifiable"; readonly elapsed_ms: number; @@ -385,6 +410,44 @@ const unverifiedCleanupProcessesSchema = z.array( z.strictObject({ pid: z.number().int().positive(), reason: z.string() }), ); +const processCaptureFinalizationSchema = z + .strictObject({ + requested_ms: z.number().int().safe().positive(), + elapsed_ms: z.number().int().nonnegative().nullable(), + exit_code: z.number().int().nonnegative().nullable(), + signals: z + .array( + z.strictObject({ + signal: z.enum(["SIGTERM", "SIGKILL"]), + sent_at_ms: z.number().int().nonnegative(), + delivery: z.enum([ + "signaled", + "gone", + "identity-changed", + "unverified", + ]), + reason: z.string().min(1).optional(), + }), + ) + .min(1), + }) + .superRefine((finalization, context) => { + if (finalization.exit_code !== null && finalization.elapsed_ms === null) + context.addIssue({ + code: "custom", + path: ["exit_code"], + message: "finalization exit_code requires an observed exit", + }); + for (const [index, attempt] of finalization.signals.entries()) + if (attempt.reason !== undefined && attempt.delivery !== "unverified") + context.addIssue({ + code: "custom", + path: ["signals", index, "reason"], + message: + "finalization reason is only allowed for an unverified delivery", + }); + }); + const processCaptureShapeSchema = z.strictObject({ manifest: z.strictObject({ rea_version: z.string().min(1), @@ -448,6 +511,7 @@ const processCaptureShapeSchema = z.strictObject({ code: z.number().int().nullable(), signal: z.number().int().nullable(), reason: z.enum(["exited", "timeout", "idle_timeout"]), + finalization: processCaptureFinalizationSchema.optional(), }), settlement: z.discriminatedUnion("state", [ z.object({ @@ -543,8 +607,12 @@ const incompleteProcessCaptureObservationsSchema = z.strictObject({ code: z.number().int().nullable(), signal: z.number().int().nullable(), reason: z.enum(["exited", "timeout", "idle_timeout", "cancelled"]), + finalization: processCaptureFinalizationSchema.optional(), }), ), + finalization: partialObservationFieldSchema( + processCaptureFinalizationSchema, + ).optional(), settlement: partialObservationFieldSchema( z.strictObject({ state: z.enum(["quiesced", "alive_at_deadline", "unverifiable"]), @@ -606,6 +674,84 @@ export const partialProcessCaptureObservationSchema = z "partial observations require incomplete cleanup or an execution failure", }); + const observedExit = + "capture" in partial + ? partial.capture.exit + : partial.observations.exit.state === "available" + ? partial.observations.exit.value + : undefined; + const observedFinalization = + "observations" in partial && + partial.observations.finalization?.state === "available" + ? partial.observations.finalization.value + : undefined; + const committedManifest = + "capture" in partial + ? partial.capture.manifest + : partial.observations.manifest.state === "available" + ? partial.observations.manifest.value + : undefined; + const manifestFinalizationIssue = finalizationManifestIssue( + committedManifest?.scenario, + committedManifest?.comparison_contract, + ); + const exitFinalizationIssue = + manifestFinalizationIssue ?? + (observedExit === undefined + ? undefined + : finalizationConsistencyIssue( + observedExit, + committedManifest?.scenario, + committedManifest?.comparison_contract, + )); + const incompleteFinalizationIssue = + manifestFinalizationIssue ?? + (observedFinalization === undefined + ? undefined + : finalizationConsistencyIssue( + observedExit === undefined + ? undefined + : { reason: observedExit.reason, signal: observedExit.signal }, + committedManifest?.scenario, + committedManifest?.comparison_contract, + { finalization: observedFinalization, allowNullElapsedMs: true }, + )); + if (exitFinalizationIssue !== undefined) + context.addIssue({ + code: "custom", + path: ["exit"], + message: exitFinalizationIssue, + }); + if (incompleteFinalizationIssue !== undefined) + context.addIssue({ + code: "custom", + path: ["finalization"], + message: incompleteFinalizationIssue, + }); + + const agreementIssue = finalizationAgreementIssue( + observedExit?.finalization, + observedFinalization, + ); + if (agreementIssue !== undefined) + context.addIssue({ + code: "custom", + path: ["finalization"], + message: agreementIssue, + }); + if ( + "observations" in partial && + partial.observations.exit.state === "unavailable" && + observedFinalization !== undefined && + observedFinalization?.elapsed_ms !== null + ) + context.addIssue({ + code: "custom", + path: ["finalization"], + message: + "finalization elapsed_ms must be null when the exit observation is unavailable", + }); + if ("observations" in partial) { const { observations } = partial; if ( diff --git a/src/domain/process/processCapture.validation.test.ts b/src/domain/process/processCapture.validation.test.ts index 0573c0052..df0344b7c 100644 --- a/src/domain/process/processCapture.validation.test.ts +++ b/src/domain/process/processCapture.validation.test.ts @@ -2,6 +2,9 @@ import { expect, it } from "vitest"; import { digestProcessCommitment } from "./processScenario.js"; import { parseProcessCapture } from "./processCaptureParsing.js"; +import { partialProcessCaptureObservationSchema } from "./processCapture.js"; +import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "./processCaptureExample.js"; +import { finalizationConsistencyIssue } from "./processCaptureValidation.js"; import { compareUnverifiedProcessCaptures as compareProcessCaptures, emptyUnverifiedProcessCapture as emptyCapture, @@ -83,6 +86,645 @@ it("rejects settlement and cleanup combinations that cannot occur", () => { ).toThrow("cleanup_outcome"); }); +it("rejects finalization evidence that no deadline could have produced", () => { + const capture = emptyCapture(); + const finalization = { + requested_ms: 500, + exit_code: null, + signals: [ + { + signal: "SIGTERM" as const, + sent_at_ms: 0, + delivery: "signaled" as const, + }, + ], + elapsed_ms: 10, + }; + expect( + () => + parseProcessCapture({ + ...capture, + exit: { code: 0, signal: null, reason: "exited", finalization }, + }), + "an exit without a deadline cannot carry a finalization record", + ).toThrow("finalization"); +}); + +it("rejects a finalization reason on a verified delivery", () => { + const finalization = { + requested_ms: 500, + exit_code: null, + elapsed_ms: 10, + signals: [ + { + signal: "SIGTERM" as const, + sent_at_ms: 0, + delivery: "signaled" as const, + reason: "should not be retained", + }, + ], + }; + + expect( + () => + parseProcessCapture({ + ...withCommittedFinalization(500, { + code: null, + signal: null, + reason: "timeout", + finalization, + }), + }), + "a finalization reason is reserved for unverified delivery", + ).toThrow("finalization"); +}); + +const withCommittedFinalization = ( + committed: number | undefined, + exit: Record, + comparisonCommitted: number | "same" | "absent" = "same", +) => { + const capture = emptyCapture(); + const scenario = { + ...capture.manifest.scenario, + ...(committed === undefined ? {} : { finalization_ms: committed }), + }; + const comparisonContract = { + ...capture.manifest.comparison_contract, + ...(comparisonCommitted === "absent" + ? {} + : comparisonCommitted === "same" + ? committed === undefined + ? {} + : { finalization_ms: committed } + : { finalization_ms: comparisonCommitted }), + }; + return { + ...capture, + manifest: { + ...capture.manifest, + scenario, + comparison_contract: comparisonContract, + full_scenario_sha256: digestProcessCommitment(scenario), + comparison_contract_sha256: digestProcessCommitment(comparisonContract), + }, + exit, + }; +}; + +it("binds finalization evidence to the committed finalization interval", () => { + const finalization = { + requested_ms: 500, + exit_code: null, + signals: [ + { + signal: "SIGTERM" as const, + sent_at_ms: 0, + delivery: "signaled" as const, + }, + ], + elapsed_ms: 10, + }; + const exit = { code: null, signal: null, reason: "timeout", finalization }; + + expect( + parseProcessCapture(withCommittedFinalization(500, exit)).exit, + "a record matching the committed interval is accepted", + ).toMatchObject({ finalization }); + expect( + () => parseProcessCapture(withCommittedFinalization(700, exit)), + "a different committed interval contradicts the record", + ).toThrow("finalization"); + expect( + () => parseProcessCapture(withCommittedFinalization(undefined, exit)), + "a scenario that committed no interval cannot have finalized", + ).toThrow("finalization"); + expect( + () => + parseProcessCapture( + withCommittedFinalization(500, { + ...exit, + finalization: { ...finalization, elapsed_ms: null }, + }), + ), + "a complete capture requires an observed finalization exit time", + ).toThrow("finalization"); +}); + +it("validates finalization signal observations", () => { + const finalization = { + requested_ms: 500, + exit_code: null, + signals: [ + { + signal: "SIGTERM" as const, + sent_at_ms: 0, + delivery: "signaled" as const, + }, + ], + elapsed_ms: 10, + }; + expect( + finalizationConsistencyIssue({ + reason: "exited", + finalization, + }), + "an exited reason cannot carry finalization", + ).toContain("finalization"); + expect( + finalizationConsistencyIssue({ + reason: "timeout", + finalization: { + ...finalization, + signals: [{ signal: "SIGKILL", sent_at_ms: 500 }], + }, + }), + "the first attempt is SIGTERM", + ).toContain("SIGTERM"); + expect( + finalizationConsistencyIssue({ + reason: "timeout", + finalization: { + ...finalization, + signals: [ + { signal: "SIGTERM", sent_at_ms: 0 }, + { signal: "SIGKILL", sent_at_ms: 500 }, + { signal: "SIGKILL", sent_at_ms: 600 }, + ], + }, + }), + "only one SIGKILL is allowed", + ).toContain("one final SIGKILL"); + expect( + finalizationConsistencyIssue({ + reason: "timeout", + finalization: { + ...finalization, + signals: [ + { signal: "SIGTERM", sent_at_ms: 10 }, + { signal: "SIGKILL", sent_at_ms: 5 }, + ], + }, + }), + "attempt times stay ordered", + ).toContain("must not decrease"); + expect( + finalizationConsistencyIssue({ + reason: "timeout", + finalization: { + ...finalization, + signals: [ + { signal: "SIGTERM", sent_at_ms: 0 }, + { signal: "SIGKILL", sent_at_ms: 499 }, + ], + }, + }), + "a non-cancelled early SIGKILL is rejected", + ).toContain("requested interval"); + expect( + finalizationConsistencyIssue({ + reason: "cancelled", + finalization: { + ...finalization, + signals: [ + { signal: "SIGTERM", sent_at_ms: 0 }, + { signal: "SIGKILL", sent_at_ms: 1 }, + ], + }, + }), + "a cancelled early SIGKILL is accepted", + ).toBeUndefined(); + expect( + finalizationConsistencyIssue({ + reason: "timeout", + finalization: { + ...finalization, + signals: [{ signal: "SIGTERM", sent_at_ms: 1 }], + elapsed_ms: 0, + }, + }), + "the observed exit cannot precede the first attempt", + ).toContain("elapsed_ms"); + expect( + finalizationConsistencyIssue({ + reason: "timeout", + finalization: { ...finalization, elapsed_ms: null }, + }), + "an observed exit requires elapsed time", + ).toContain("elapsed_ms"); + expect( + finalizationConsistencyIssue( + { reason: "timeout" }, + { finalization_ms: 500 }, + ), + "a deadline exit with a committed interval must carry the record", + ).toContain("finalization evidence"); +}); + +it("validates finalization evidence in partial captures and rejects malformed records", () => { + const { cleanup: _cleanup, ...completed } = emptyCapture(); + const partial = (exit: Record) => + partialProcessCaptureObservationSchema.safeParse({ + capture: { ...completed, exit }, + cleanup: { + owned_process_group: { state: "cleaned", reason: null }, + terminal_renderer: { state: "cleaned", reason: null }, + temporary_root: { state: "cleaned", reason: null }, + }, + execution_failure: "capture ended after a fixture error", + }); + const partialWithCommittedFinalization = (exit: Record) => { + const { cleanup: _cleanup, ...completed } = withCommittedFinalization(500, { + code: null, + signal: null, + reason: "timeout", + }); + return partialProcessCaptureObservationSchema.safeParse({ + capture: { ...completed, exit }, + cleanup: { + owned_process_group: { state: "cleaned", reason: null }, + terminal_renderer: { state: "cleaned", reason: null }, + temporary_root: { state: "cleaned", reason: null }, + }, + execution_failure: "capture ended after a fixture error", + }); + }; + const finalization = { + requested_ms: 500, + exit_code: null, + signals: [ + { + signal: "SIGTERM" as const, + sent_at_ms: 0, + delivery: "signaled" as const, + }, + ], + elapsed_ms: 10, + }; + + expect( + partial({ code: null, signal: null, reason: "timeout" }).success, + "a partial exit without finalization stays valid", + ).toBe(true); + expect( + partial({ code: 0, signal: null, reason: "exited", finalization }).success, + "a partial exit cannot finalize without a deadline", + ).toBe(false); + expect( + partial({ + code: null, + signal: null, + reason: "timeout", + finalization: { ...finalization, requested_ms: 700 }, + }).success, + "a partial record must match the committed interval", + ).toBe(false); + expect( + partialWithCommittedFinalization({ + code: null, + signal: null, + reason: "timeout", + finalization: { ...finalization, elapsed_ms: null }, + }) + .error?.issues.map(({ message }) => message) + .join("; "), + "a capture variant requires an observed finalization exit time", + ).toContain("elapsed_ms"); + expect( + () => + parseProcessCapture( + withCommittedFinalization(500, { + code: null, + signal: null, + reason: "timeout", + finalization: { + ...finalization, + signals: [ + { signal: "SIGINT", sent_at_ms: 0, delivery: "signaled" }, + ], + }, + }), + ), + "only SIGTERM can start finalization", + ).toThrow(); + expect( + () => + parseProcessCapture( + withCommittedFinalization(500, { + code: null, + signal: null, + reason: "timeout", + finalization: { ...finalization, elapsed_ms: -1 }, + }), + ), + "elapsed time cannot be negative", + ).toThrow(); +}); + +it("binds the comparison contract to the committed finalization interval", () => { + const exit = { code: 0, signal: null, reason: "exited" }; + + expect( + () => parseProcessCapture(withCommittedFinalization(500, exit, "absent")), + "a scenario interval missing from the comparison contract is contradictory", + ).toThrow("comparison_contract"); + expect( + () => parseProcessCapture(withCommittedFinalization(500, exit, 900)), + "a different comparison interval is contradictory", + ).toThrow("comparison_contract"); + expect( + () => parseProcessCapture(withCommittedFinalization(undefined, exit, 500)), + "a comparison interval the scenario never committed is contradictory", + ).toThrow("comparison_contract"); +}); + +// This single scenario shares one deliberately complete incomplete-capture fixture. +// oxlint-disable-next-line max-lines-per-function +it("validates finalization evidence in incomplete observations", () => { + const finalization = { + requested_ms: 500, + exit_code: null, + signals: [ + { + signal: "SIGTERM" as const, + sent_at_ms: 0, + delivery: "signaled" as const, + }, + ], + elapsed_ms: 10, + }; + const unavailable = { state: "unavailable", reason: "not observed" }; + const incomplete = ( + exit: Record | undefined, + manifest: ReturnType["manifest"], + incompleteFinalization: Record | undefined = undefined, + ) => { + const result = partialProcessCaptureObservationSchema.safeParse({ + observations: { + target_pid: unavailable, + frames: { state: "available", value: [] }, + rendered_frames: unavailable, + interaction_events: unavailable, + exit: + exit === undefined + ? unavailable + : { state: "available", value: exit }, + finalization: + incompleteFinalization === undefined + ? unavailable + : { state: "available", value: incompleteFinalization }, + settlement: unavailable, + process_samples: unavailable, + filesystem_snapshots: { before: unavailable, after: unavailable }, + event_journal: unavailable, + manifest: { state: "available", value: manifest }, + }, + cleanup: { + owned_process_group: { state: "cleaned", reason: null }, + terminal_renderer: { state: "cleaned", reason: null }, + temporary_root: { state: "cleaned", reason: null }, + }, + execution_failure: "capture ended after a fixture error", + }); + return { + success: result.success, + messages: result.success + ? "" + : result.error.issues.map(({ message }) => message).join("; "), + }; + }; + const timeout = { code: null, signal: null, reason: "timeout" }; + const normal = { code: 0, signal: null, reason: "exited" }; + const committed = withCommittedFinalization(500, timeout).manifest; + + expect( + incomplete({ ...timeout, finalization }, committed).success, + "a consistent incomplete record is accepted", + ).toBe(true); + expect( + incomplete({ ...timeout, finalization }, committed, { ...finalization }) + .success, + "matching exit and sibling finalization observations are accepted", + ).toBe(true); + const disagreement = incomplete({ ...timeout, finalization }, committed, { + ...finalization, + elapsed_ms: 11, + }); + expect( + disagreement.success, + "conflicting finalization observations are rejected", + ).toBe(false); + expect(disagreement.messages, "the conflict names finalization").toContain( + "finalization observation must agree", + ); + expect( + incomplete(undefined, committed, { ...finalization, elapsed_ms: null }) + .success, + "a finalization observed without an exit permits null elapsed time", + ).toBe(true); + const unavailableExitElapsed = incomplete(undefined, committed, finalization); + expect( + unavailableExitElapsed.success, + "a finalization without an exit rejects a claimed elapsed time", + ).toBe(false); + expect( + unavailableExitElapsed.messages, + "the unavailable-exit rule names elapsed_ms", + ).toContain("elapsed_ms"); + const sigkillFirst = incomplete(undefined, committed, { + ...finalization, + elapsed_ms: null, + signals: [{ signal: "SIGKILL", sent_at_ms: 0, delivery: "signaled" }], + }); + expect( + sigkillFirst.success, + "an incomplete finalization still starts with SIGTERM", + ).toBe(false); + expect( + sigkillFirst.messages, + "only the SIGTERM-first rule fires, not the null-elapsed rule", + ).toContain("start with SIGTERM"); + expect( + incomplete(normal, withCommittedFinalization(undefined, normal).manifest) + .success, + "an incomplete capture that never committed an interval is accepted", + ).toBe(true); + expect( + incomplete( + { code: 0, signal: null, reason: "exited", finalization }, + committed, + ).success, + "an incomplete exit cannot finalize without a deadline", + ).toBe(false); + expect( + incomplete( + { ...timeout, finalization: { ...finalization, requested_ms: 700 } }, + committed, + ).success, + "an incomplete record must match the committed interval", + ).toBe(false); + const scenarioOnly = withCommittedFinalization( + 500, + normal, + "absent", + ).manifest; + const comparisonOnly = withCommittedFinalization( + undefined, + normal, + 500, + ).manifest; + const differing = withCommittedFinalization(500, normal, 900).manifest; + for (const [label, manifest] of [ + ["a scenario interval missing from the comparison contract", scenarioOnly], + ["a comparison interval the scenario never committed", comparisonOnly], + ["differing intervals", differing], + ] as const) { + const verdict = incomplete(normal, manifest); + expect(verdict.success, `${label} is rejected`).toBe(false); + expect( + verdict.messages, + `${label} is rejected for the comparison contract`, + ).toContain("comparison_contract"); + } + const earlyKill = { + requested_ms: 500, + exit_code: null, + elapsed_ms: 600, + signals: [ + { signal: "SIGTERM", sent_at_ms: 0, delivery: "signaled" }, + { signal: "SIGKILL", sent_at_ms: 40, delivery: "unverified" }, + ], + }; + expect( + incomplete( + { + code: null, + signal: null, + reason: "cancelled", + finalization: earlyKill, + }, + committed, + earlyKill, + ).success, + "a cancellation may keep an early SIGKILL attempt", + ).toBe(true); + const earlyKillDeadline = incomplete( + { ...timeout, finalization: earlyKill }, + committed, + earlyKill, + ); + expect( + earlyKillDeadline.success, + "a deadline exit cannot carry a SIGKILL before the requested interval", + ).toBe(false); + expect( + earlyKillDeadline.messages, + "the early SIGKILL negative isolates the requested interval rule", + ).toContain("requested interval"); + expect( + incomplete(undefined, differing).success, + "an unavailable exit still cannot hide a contradictory manifest", + ).toBe(false); +}); + +it("rejects chronologically impossible or repeated finalization attempts", () => { + const attempt = (signal: "SIGTERM" | "SIGKILL", sent_at_ms: number) => ({ + signal, + sent_at_ms, + delivery: "signaled" as const, + }); + const exit = ( + elapsed_ms: number | null, + signals: ReturnType[], + ) => ({ + reason: "timeout", + finalization: { requested_ms: 500, elapsed_ms, exit_code: null, signals }, + }); + + expect( + finalizationConsistencyIssue( + exit(10, [attempt("SIGTERM", 0), attempt("SIGKILL", 500)]), + ), + "an exit cannot be observed before the last recorded attempt", + ).toContain("elapsed_ms"); + expect( + finalizationConsistencyIssue( + exit(600, [attempt("SIGTERM", 0), attempt("SIGTERM", 10)]), + ), + "SIGTERM is attempted once", + ).toContain("finalization"); + expect( + finalizationConsistencyIssue( + exit(600, [attempt("SIGTERM", 0), attempt("SIGKILL", 500)]), + ), + "a consistent record is accepted", + ).toBeUndefined(); +}); + +it("keeps the finalization exit code tied to an observed exit not caused by a signal", () => { + const attempt = { + signal: "SIGTERM" as const, + sent_at_ms: 0, + delivery: "signaled" as const, + }; + const record = ( + elapsed_ms: number | null, + exit_code: number | null, + signal?: number | null, + ) => ({ + reason: "timeout", + ...(signal === undefined ? {} : { signal }), + finalization: { + requested_ms: 500, + elapsed_ms, + exit_code, + signals: [attempt], + }, + }); + + expect( + finalizationConsistencyIssue(record(10, 3, 0)), + "a code reported for an exit without a signal is accepted", + ).toBeUndefined(); + expect( + finalizationConsistencyIssue(record(10, 3, 9)), + "a code cannot accompany an exit ended by a signal", + ).toContain("exit_code"); + expect( + finalizationConsistencyIssue(record(10, null, 9)), + "a signal exit records a null code", + ).toBeUndefined(); + + const schemaFinalization = { + requested_ms: 500, + elapsed_ms: null, + exit_code: 3, + signals: [attempt], + }; + expect( + () => + parseProcessCapture( + withCommittedFinalization(500, { + code: null, + signal: null, + reason: "timeout", + finalization: schemaFinalization, + }), + ), + "an exit code needs an observed exit", + ).toThrow("exit_code"); +}); + +it("accepts the canonical finalized capture example", () => { + expect( + parseProcessCapture(FINALIZED_PROCESS_CAPTURE_EXAMPLE).exit, + "the advertised finalized example is a valid capture", + ).toMatchObject({ + reason: "timeout", + finalization: { requested_ms: 1_500, signals: [{ signal: "SIGTERM" }] }, + }); +}); + it("requires an explicit journal and validates complete journals", () => { const capture = emptyCapture(); const { event_journal: _eventJournal, ...oldCapture } = capture; diff --git a/src/domain/process/processCaptureExample.ts b/src/domain/process/processCaptureExample.ts index 6c177a535..c8dbff897 100644 --- a/src/domain/process/processCaptureExample.ts +++ b/src/domain/process/processCaptureExample.ts @@ -109,3 +109,31 @@ export const EMPTY_PROCESS_CAPTURE_EXAMPLE = { temporary_root: "removed", }, } satisfies JsonValue; + +const finalizedScenario = { ...scenario, finalization_ms: 1_500 }; +const finalizedComparisonContract = { finalization_ms: 1_500 }; + +/** Valid capture of a timeout that finalized: SIGTERM delivered, exit observed. */ +export const FINALIZED_PROCESS_CAPTURE_EXAMPLE = { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE, + manifest: { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest, + scenario: finalizedScenario, + comparison_contract: finalizedComparisonContract, + full_scenario_sha256: digestProcessCommitment(finalizedScenario), + comparison_contract_sha256: digestProcessCommitment( + finalizedComparisonContract, + ), + }, + exit: { + code: null, + signal: 0, + reason: "timeout", + finalization: { + requested_ms: 1_500, + elapsed_ms: 12, + exit_code: 0, + signals: [{ signal: "SIGTERM", sent_at_ms: 0, delivery: "signaled" }], + }, + }, +} satisfies JsonValue; diff --git a/src/domain/process/processCaptureValidation.ts b/src/domain/process/processCaptureValidation.ts index a14cd129b..10d6b831b 100644 --- a/src/domain/process/processCaptureValidation.ts +++ b/src/domain/process/processCaptureValidation.ts @@ -1,7 +1,10 @@ import { digestProcessCommitment } from "./processScenario.js"; import { hasCaptureTruncation } from "./processCaptureCoverage.js"; -import type { UnverifiedProcessCapture } from "./processCapture.js"; +import type { + ProcessCaptureFinalization, + UnverifiedProcessCapture, +} from "./processCapture.js"; /** One pure semantic validation failure in a shaped process capture. */ export interface ProcessCaptureValidationIssue { readonly path: string; @@ -106,6 +109,136 @@ const validateEventJournal = ( expectedSize, "event_journal", "nonempty journal must reference every captured observation"); }; +/** + * The committed comparison contract carries `finalization_ms` exactly when the + * committed scenario does, with the same value. Independent of the exit, so a + * partial capture with an unavailable exit is still checked. + */ +export const finalizationManifestIssue = ( + committedScenario?: Readonly>, + committedComparison?: Readonly>, +): string | undefined => + committedScenario !== undefined && + committedComparison !== undefined && + committedScenario["finalization_ms"] !== + committedComparison["finalization_ms"] + ? "comparison_contract finalization_ms must match the committed scenario" + : undefined; + +type FinalizationAttempts = Pick< + ProcessCaptureFinalization, + "requested_ms" | "elapsed_ms" | "exit_code" +> & { + readonly signals: readonly Pick< + ProcessCaptureFinalization["signals"][number], + "signal" | "sent_at_ms" + >[]; +}; + +/** Ensure incomplete exit and sibling finalization observations preserve one fact. */ +export const finalizationAgreementIssue = ( + exitFinalization: ProcessCaptureFinalization | undefined, + observedFinalization: ProcessCaptureFinalization | undefined, +): string | undefined => + exitFinalization !== undefined && + observedFinalization !== undefined && + JSON.stringify(exitFinalization) !== JSON.stringify(observedFinalization) + ? "exit finalization and finalization observation must agree" + : undefined; + +/** + * Cross-field rule for `exit.finalization`, shared by complete and partial + * captures. Returns the violated rule, or undefined when the record is + * consistent. + */ +export const finalizationConsistencyIssue = ( + exit: + | { + readonly reason: string; + readonly signal?: number | null | undefined; + readonly finalization?: FinalizationAttempts | undefined; + } + | undefined, + committedScenario?: Readonly>, + committedComparison?: Readonly>, + options?: { + readonly finalization?: FinalizationAttempts; + readonly allowNullElapsedMs?: boolean; + }, +): string | undefined => { + const manifestIssue = finalizationManifestIssue( + committedScenario, + committedComparison, + ); + if (manifestIssue !== undefined) return manifestIssue; + const finalization = exit?.finalization ?? options?.finalization; + if (finalization === undefined) { + // A deadline that fired under a committed interval must have attempted + // finalization; cancellation before the deadline legitimately has none. + const committed = committedScenario?.["finalization_ms"]; + return (exit?.reason === "timeout" || exit?.reason === "idle_timeout") && + typeof committed === "number" && + committed > 0 + ? "a deadline exit under a committed finalization_ms requires finalization evidence" + : undefined; + } + if (exit?.reason === "exited") + return "finalization requires a deadline exit reason"; + const first = finalization.signals[0]; + if (first?.signal !== "SIGTERM") + return "finalization signals must start with SIGTERM"; + const killIndexes = finalization.signals.flatMap(({ signal }, index) => + signal === "SIGKILL" ? [index] : [], + ); + if ( + killIndexes.length > 1 || + (killIndexes[0] !== undefined && + killIndexes[0] !== finalization.signals.length - 1) + ) + return "finalization may contain at most one final SIGKILL"; + if ( + !finalization.signals.every(({ sent_at_ms }, index) => { + const previous = finalization.signals[index - 1]; + return previous === undefined || sent_at_ms >= previous.sent_at_ms; + }) + ) + return "finalization signal attempt times must not decrease"; + if ( + finalization.signals.filter(({ signal }) => signal === "SIGTERM").length !== + 1 + ) + return "finalization must attempt SIGTERM exactly once"; + const kill = finalization.signals.find(({ signal }) => signal === "SIGKILL"); + if ( + kill !== undefined && + kill.sent_at_ms < finalization.requested_ms && + exit !== undefined && + exit.reason !== "cancelled" + ) + return "finalization SIGKILL cannot precede the requested interval"; + if (finalization.elapsed_ms === null && options?.allowNullElapsedMs !== true) + return "finalization requires an observed elapsed_ms"; + if ( + finalization.exit_code !== null && + exit?.signal !== undefined && + exit.signal !== null && + exit.signal !== 0 + ) + return "finalization exit_code requires an exit not caused by a signal"; + if ( + finalization.elapsed_ms !== null && + finalization.elapsed_ms < + (finalization.signals.at(-1)?.sent_at_ms ?? first.sent_at_ms) + ) + return "finalization elapsed_ms cannot precede the last signal attempt"; + if ( + committedScenario !== undefined && + committedScenario["finalization_ms"] !== finalization.requested_ms + ) + return "finalization must match the committed finalization_ms"; + return undefined; +}; + const validateLifecycle = ( capture: UnverifiedProcessCapture, require: RequireInvariant, @@ -120,6 +253,12 @@ const validateLifecycle = ( require(capture.exit.reason === "exited" || capture.exit.code === null, "exit", "deadline termination cannot declare a normal exit code"); + const finalizationIssue = finalizationConsistencyIssue( + capture.exit, + capture.manifest.scenario, + capture.manifest.comparison_contract, + ); + require(finalizationIssue === undefined, "exit", finalizationIssue ?? ""); }; const validateCoverage = ( diff --git a/src/domain/process/processScenario.test.ts b/src/domain/process/processScenario.test.ts index be7075502..f6a79f4b5 100644 --- a/src/domain/process/processScenario.test.ts +++ b/src/domain/process/processScenario.test.ts @@ -1,11 +1,137 @@ import { expect, it } from "vitest"; import { + digestProcessCommitment, parseProcessScenario, PROCESS_TERMINAL_CELL_BUDGET, + processComparisonContract, + processScenarioCommitment, processScenarioSchema, } from "./processScenario.js"; +const baseScenario = { executable: "/usr/bin/true" }; + +it("defaults the finalization interval to zero without changing committed identity", () => { + const scenario = parseProcessScenario(baseScenario); + + expect(scenario, "parsed scenario carries the default").toMatchObject({ + finalization_ms: 0, + }); + expect( + Object.hasOwn(processScenarioCommitment(scenario), "finalization_ms"), + "full scenario commitment omits a zero interval", + ).toBe(false); + expect( + Object.hasOwn(processComparisonContract(scenario), "finalization_ms"), + "comparison contract omits a zero interval", + ).toBe(false); +}); + +it("accepts an explicit zero finalization interval", () => { + expect( + parseProcessScenario({ ...baseScenario, finalization_ms: 0 }), + "zero is the immediate-kill default, not an invalid budget", + ).toMatchObject({ finalization_ms: 0 }); +}); + +it("commits a positive finalization interval in both identity projections", () => { + const scenario = parseProcessScenario({ + ...baseScenario, + finalization_ms: 500, + }); + + expect( + processScenarioCommitment(scenario), + "full scenario commitment records the interval", + ).toMatchObject({ finalization_ms: 500 }); + expect( + processComparisonContract(scenario), + "comparison contract records the interval", + ).toMatchObject({ finalization_ms: 500 }); +}); + +it.each([-1, 1.5, Number.MAX_SAFE_INTEGER + 1])( + "rejects the finalization interval %j", + (finalization_ms) => { + expect(() => + parseProcessScenario({ ...baseScenario, finalization_ms }), + ).toThrow(); + }, +); + +it("accepts a combined lifecycle budget up to the safe-integer limit", () => { + const limit = Number.MAX_SAFE_INTEGER; + const accepted = parseProcessScenario({ + ...baseScenario, + timeout_ms: limit - 100, + idle_timeout_ms: 1, + settle_ms: 90, + finalization_ms: 10, + }); + + expect( + accepted.timeout_ms + accepted.finalization_ms + accepted.settle_ms, + "a total exactly at the representable limit is accepted", + ).toBe(limit); + expect( + parseProcessScenario(baseScenario), + "the default scenario stays far below the limit", + ).toMatchObject({ timeout_ms: 30_000, settle_ms: 100, finalization_ms: 0 }); +}); + +it.each([ + ["timeout alone over the sum", { timeout_ms: Number.MAX_SAFE_INTEGER }], + [ + "components that only overflow together", + { + timeout_ms: Number.MAX_SAFE_INTEGER - 10, + settle_ms: 6, + finalization_ms: 5, + }, + ], + [ + "a large settle and finalization", + { + timeout_ms: 5, + settle_ms: Number.MAX_SAFE_INTEGER, + finalization_ms: 1, + }, + ], +])("rejects %s with a representability message", (_label, budget) => { + const parse = () => + parseProcessScenario({ ...baseScenario, idle_timeout_ms: 1, ...budget }); + + expect(parse, "the combined budget is rejected").toThrow( + "timeout_ms + finalization_ms + settle_ms", + ); + expect(parse, "the message names the representability limit").toThrow( + "exactly representable", + ); +}); + +it("keeps the committed identity of a scenario without finalization byte for byte", () => { + const scenario = parseProcessScenario({ + executable: "/usr/bin/true", + arguments: ["-x"], + working_directory: "/tmp", + environment: { A: "b" }, + filesystem_observation_paths: ["/tmp"], + timeout_ms: 5_000, + idle_timeout_ms: 2_000, + settle_ms: 50, + }); + + // Digests computed from the compiled base commit before finalization_ms existed. + expect( + digestProcessCommitment(processScenarioCommitment(scenario)), + "the full scenario commitment is unchanged", + ).toBe("b50005d26269bd1db4a49c79e05bca67c3a18d837b45cd1c8ff591c01eb7623a"); + expect( + digestProcessCommitment(processComparisonContract(scenario)), + "the comparison contract commitment is unchanged", + ).toBe("675b759347a737a4f1cbfbf2fc759258d9c877216513f2f4dad3ed1e9e6e7631"); +}); + it("admits the default and the exact combined terminal-cell budget", () => { expect(parseProcessScenario({ executable: "/bin/true" }).terminal).toEqual({ columns: 80, diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts index c489db92e..a87ffd86e 100644 --- a/src/domain/process/processScenario.ts +++ b/src/domain/process/processScenario.ts @@ -95,11 +95,17 @@ const committedScenarioEvents = ( export const processScenarioCommitment = ( scenario: ProcessScenario, executableSha256?: string, -): Readonly> => ({ - ...scenario, - events: committedScenarioEvents(scenario.events, true), - executable_sha256: executableSha256 ?? null, -}); +): Readonly> => { + // A zero interval is today's immediate kill; leaving it out keeps the + // committed identity of every scenario that predates the field unchanged. + const { finalization_ms: finalizationMs, ...committed } = scenario; + return { + ...committed, + ...(finalizationMs > 0 ? { finalization_ms: finalizationMs } : {}), + events: committedScenarioEvents(scenario.events, true), + executable_sha256: executableSha256 ?? null, + }; +}; /** Project observation settings shared by capture and reconstruction. */ export const processComparisonContract = ( @@ -117,6 +123,9 @@ export const processComparisonContract = ( timeout_ms: scenario.timeout_ms, idle_timeout_ms: scenario.idle_timeout_ms, settle_ms: scenario.settle_ms, + ...(scenario.finalization_ms > 0 + ? { finalization_ms: scenario.finalization_ms } + : {}), limits: scenario.limits, normalization: scenario.normalization, ...(scenario.normalization.ports @@ -177,6 +186,15 @@ export const processScenarioSchema = z timeout_ms: positiveBudget.default(30_000), idle_timeout_ms: positiveBudget.default(30_000), settle_ms: z.number().int().safe().nonnegative().default(100), + finalization_ms: z + .number() + .int() + .safe() + .nonnegative() + .default(0) + .describe( + "Milliseconds a target may keep running after SIGTERM when timeout_ms or idle_timeout_ms fires, before SIGKILL; 0 sends SIGKILL immediately. Both signals reach the captured root only while its launch-time start identity still matches; cancellation ends the run at once.", + ), limits: z .strictObject({ output_bytes: positiveBudget.default(1_000_000), @@ -221,6 +239,18 @@ export const processScenarioSchema = z }), }) .superRefine((scenario, context) => { + // Each budget is a safe integer, but their sum is not checked anywhere + // else. BigInt keeps the comparison exact beyond the representable range. + const lifecycleBudget = + BigInt(scenario.timeout_ms) + + BigInt(scenario.finalization_ms) + + BigInt(scenario.settle_ms); + if (lifecycleBudget > BigInt(Number.MAX_SAFE_INTEGER)) + context.addIssue({ + code: "custom", + message: `timeout_ms + finalization_ms + settle_ms must stay at or below ${String(Number.MAX_SAFE_INTEGER)} ms, the largest total exactly representable as a JavaScript number`, + path: ["timeout_ms"], + }); if ( !fitsProcessTerminalCellBudget( scenario.terminal.columns, diff --git a/src/process/ProcessOwnershipObservation.ts b/src/process/ProcessOwnershipObservation.ts index db5c7cb3a..667dd06a1 100644 --- a/src/process/ProcessOwnershipObservation.ts +++ b/src/process/ProcessOwnershipObservation.ts @@ -431,10 +431,15 @@ export const createSystemProcessOwnershipHost = ( export const systemProcessOwnershipHost = createSystemProcessOwnershipHost(); -/** Observe the current OS start identity for one live PID, when supported. */ +/** + * Observe a live PID's start identity, optionally binding it to an owned run. + * Token validation is bracketed by identity reads so PID reuse cannot supply + * the first identity of a different process for an owned launch. + */ export const observeProcessStartIdentity = async ( pid: number, host: ProcessOwnershipHost = systemProcessOwnershipHost, + expectedRunId?: string, ): Promise => { const process = (await host.listProcesses()).find( (entry) => entry.pid === pid, @@ -443,12 +448,42 @@ export const observeProcessStartIdentity = async ( return undefined; if (host.processIdentities === undefined) return { state: "unavailable", reason: "process identity is unsupported" }; - return ( - (await host.processIdentities([process])).get(pid) ?? { + const identity: ProcessIdentityObservation = ( + await host.processIdentities([process]) + ).get(pid) ?? { + state: "unavailable", + reason: "process identity was not returned", + }; + if (expectedRunId === undefined || identity.state !== "readable") + return identity; + let owned = false; + for await (const { observation } of readProcessRunTokens(host, [process])) { + if (observation.state === "unavailable") + return { + state: "unavailable", + reason: `process ownership token could not be read for PID ${String(pid)}: ${observation.reason}`, + }; + if (observation.runId !== expectedRunId) + return { + state: "unavailable", + reason: `process ownership token did not match the captured run for PID ${String(pid)}`, + }; + owned = true; + } + if (!owned) + return { state: "unavailable", - reason: "process identity was not returned", - } - ); + reason: `process ownership token was not returned for PID ${String(pid)}`, + }; + const confirmed = await observeProcessStartIdentity(pid, host); + if (confirmed === undefined || confirmed.state !== "readable") + return confirmed; + return confirmed.identity === identity.identity + ? identity + : { + state: "unavailable", + reason: `process identity changed while validating the captured run token for PID ${String(pid)}`, + }; }; /** Revalidate a launch-time start identity immediately before signaling a PID. */ diff --git a/src/process/ProcessStartIdentity.test.ts b/src/process/ProcessStartIdentity.test.ts index be354b24f..20b2862ee 100644 --- a/src/process/ProcessStartIdentity.test.ts +++ b/src/process/ProcessStartIdentity.test.ts @@ -37,6 +37,54 @@ describe("process start identity lease", () => { }); }); + it("does not adopt a replacement PID's first identity for an owned launch", async () => { + const replacement: ProcessOwnershipHost = { + ...host("replacement-start"), + environment: async () => ({ REA_PROCESS_RUN_ID: "another-run" }), + }; + await expect( + observeProcessStartIdentity(42, replacement, "captured-run"), + ).resolves.toMatchObject({ + state: "unavailable", + reason: expect.stringContaining("ownership"), + }); + }); + + it("does not establish an owned launch when its token is unreadable", async () => { + const unreadable: ProcessOwnershipHost = { + ...host("start-1"), + runTokens: async () => + new Map([ + [42, { state: "unavailable", reason: "environment_errno_EACCES" }], + ]), + }; + await expect( + observeProcessStartIdentity(42, unreadable, "captured-run"), + ).resolves.toMatchObject({ + state: "unavailable", + reason: expect.stringContaining("environment_errno_EACCES"), + }); + }); + + it("rejects an identity that changes while the captured run token is read", async () => { + let identity = "start-1"; + const changing: ProcessOwnershipHost = { + ...host("start-1"), + processIdentities: async () => + new Map([[42, { state: "readable", identity }]]), + environment: async () => { + identity = "replacement-start"; + return { REA_PROCESS_RUN_ID: "captured-run" }; + }, + }; + await expect( + observeProcessStartIdentity(42, changing, "captured-run"), + ).resolves.toMatchObject({ + state: "unavailable", + reason: expect.stringContaining("identity changed"), + }); + }); + it("signals the PID when its launch identity still matches", async () => { const sendSignal = vi.fn(); await expect( diff --git a/src/process/capture/ProcessCaptureLifecycle.test.ts b/src/process/capture/ProcessCaptureLifecycle.test.ts new file mode 100644 index 000000000..765635ccc --- /dev/null +++ b/src/process/capture/ProcessCaptureLifecycle.test.ts @@ -0,0 +1,548 @@ +import { expect, it } from "vitest"; +import type { IPty } from "@lydell/node-pty"; + +import { parseProcessScenario } from "../../domain/process/processScenario.js"; +import type { ProcessCaptureFinalization } from "../../domain/process/processCapture.js"; +import { awaitTerminalExit } from "./ProcessCaptureLifecycle.js"; +import type { ProcessTimer } from "./ProcessTimer.js"; + +type Delivery = ProcessCaptureFinalization["signals"][number]["delivery"]; + +interface FakeExit { + readonly exitCode: number; + readonly signal?: number; +} + +/** + * Terminal whose exit notification the test delivers when it chooses, plus + * the identity-checked signaller that finalization uses instead of a bare + * terminal kill. Both record into `signals`. + */ +const fakeTerminal = ( + deliveries: Partial> = {}, + hold?: Promise, + signalFailure?: Error, +) => { + let listener: ((exit: FakeExit) => void) | undefined; + const signals: string[] = []; + const terminal: Pick = { + onExit: (next) => { + listener = next; + return { dispose: () => undefined }; + }, + kill: (signal) => { + signals.push(`terminal:${signal ?? "SIGHUP"}`); + }, + }; + const signalTarget = async ( + signal: "SIGTERM" | "SIGKILL", + ): Promise<{ readonly delivery: Delivery }> => { + signals.push(signal); + await hold; + if (signalFailure !== undefined) throw signalFailure; + return { delivery: deliveries[signal] ?? "signaled" }; + }; + return { + signals, + terminal, + signalTarget, + deliverExit: (exit: FakeExit) => listener?.(exit), + }; +}; + +const waitFor = async (condition: () => boolean): Promise => { + const deadline = Date.now() + 3_000; + while (!condition()) { + if (Date.now() > deadline) throw new Error("condition not reached"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +}; + +const snapshots: ProcessCaptureFinalization[] = []; + +const observed = ( + result: Result, +): Exclude => { + if (result.unobserved === true) throw new Error("exit was not observed"); + return result as Exclude; +}; + +const start = ( + fake: ReturnType, + scenario: Readonly>, + signal?: AbortSignal, +) => + awaitTerminalExit({ + terminal: fake.terminal, + scenario: parseProcessScenario({ + executable: "unused", + idle_timeout_ms: 10_000, + ...scenario, + }), + started: Date.now(), + lastOutput: () => Date.now(), + signal, + signalTarget: fake.signalTarget, + recordFinalization: (snapshot) => { + snapshots.push(snapshot); + }, + timers: new Set(), + interactions: [], + dispatchedEventIndexes: new Set(), + recordEvent: () => undefined, + }); + +it("records attempts when a self-exit notification arrives after escalation", async () => { + const fake = fakeTerminal(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + await waitFor(() => fake.signals.includes("SIGKILL")); + await new Promise((resolve) => setTimeout(resolve, 150)); + fake.deliverExit({ exitCode: 0, signal: 0 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.signals.map(({ signal }) => signal), + "both attempts are observed in order", + ).toEqual(["SIGTERM", "SIGKILL"]); + expect(exit.signal, "the observed exit remains independent of attempts").toBe( + 0, + ); + expect( + fake.signals.filter((signal) => signal === "SIGKILL"), + "the escalation is sent once, not on every poll", + ).toHaveLength(1); + expect( + fake.signals.filter((signal) => signal === "SIGTERM"), + "the finalization signal is sent once, not on every poll", + ).toHaveLength(1); +}); + +it("kills at once when cancelled during the finalization interval", async () => { + const fake = fakeTerminal(); + const controller = new AbortController(); + const pending = start( + fake, + { timeout_ms: 100, finalization_ms: 60_000 }, + controller.signal, + ); + await waitFor(() => fake.signals.includes("SIGTERM")); + controller.abort(); + await waitFor(() => fake.signals.includes("SIGKILL")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + + const exit = observed(await pending); + + expect(exit.reason, "cancellation replaces the initiating deadline").toBe( + "cancelled", + ); + expect( + fake.signals.filter((signal) => signal === "SIGKILL"), + "the abort branch itself sent one SIGKILL, long before the interval ended", + ).toHaveLength(1); + expect( + exit.finalization?.signals.map(({ signal }) => signal), + "cancellation records both attempts", + ).toEqual(["SIGTERM", "SIGKILL"]); + expect( + fake.signals.filter((signal) => signal.startsWith("terminal:")), + "cancellation during finalization never uses a bare terminal kill", + ).toEqual([]); +}); + +it("records escalation attempts alongside an observed SIGKILL exit", async () => { + const fake = fakeTerminal(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + await waitFor(() => fake.signals.includes("SIGKILL")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.signals, + "both finalization attempts are recorded", + ).toMatchObject([ + { signal: "SIGTERM", delivery: "signaled" }, + { signal: "SIGKILL", delivery: "signaled" }, + ]); + expect(exit.signal, "the observed exit signal is retained separately").toBe( + 9, + ); + expect( + exit.finalization?.elapsed_ms, + "the escalation waited for the whole interval", + ).toBeGreaterThanOrEqual(100); +}); + +it("records a scripted SIGKILL exit without inferring its cause", async () => { + const fake = fakeTerminal(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => fake.signals.includes("SIGTERM")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + + const exit = observed(await pending); + + expect( + fake.signals, + "the harness itself never escalated inside the interval", + ).not.toContain("SIGKILL"); + expect( + exit.finalization?.signals, + "only the SIGTERM attempt was made", + ).toMatchObject([{ signal: "SIGTERM", delivery: "signaled" }]); + expect(exit.signal, "the observed SIGKILL remains an exit fact").toBe(9); +}); + +it.each([ + [3, 3], + [0, 0], + [7, 7], +] as const)( + "records exit code %i of a finalizer that exits on its own as %i", + async (exitCode, expected) => { + const fake = fakeTerminal(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => fake.signals.includes("SIGTERM")); + fake.deliverExit({ exitCode, signal: 0 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.exit_code, + "the PTY-reported code of an exit not caused by a signal is recorded", + ).toBe(expected); + expect(exit.reason, "the initiating deadline stays the exit reason").toBe( + "timeout", + ); + }, +); + +it("records no exit code when the finalizing target was ended by a signal", async () => { + const fake = fakeTerminal(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + await waitFor(() => fake.signals.includes("SIGKILL")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.exit_code, + "a code reported next to a terminating signal is never recorded", + ).toBeNull(); + expect(exit.signal, "the signal remains the observed exit fact").toBe(9); +}); + +it("records no exit code when the PTY reports a negative code without a signal", async () => { + const fake = fakeTerminal(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => fake.signals.includes("SIGTERM")); + fake.deliverExit({ exitCode: -1, signal: 0 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.exit_code, + "a negative PTY code is not a process exit code and is never recorded", + ).toBeNull(); + expect(exit.reason, "the initiating deadline stays the exit reason").toBe( + "timeout", + ); +}); + +it("records no exit code when no exit was observed", async () => { + const fake = fakeTerminal({}, new Promise(() => undefined)); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + + const result = await pending; + + expect(result.unobserved, "no exit is invented").toBe(true); + expect( + result.finalization?.exit_code, + "an unobserved exit has no code to record", + ).toBeNull(); +}, 10_000); + +it("records every delivery result and publishes attempts incrementally", async () => { + snapshots.length = 0; + const fake = fakeTerminal({ SIGTERM: "gone", SIGKILL: "gone" }); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + await waitFor(() => fake.signals.includes("SIGKILL")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.signals.map(({ delivery }) => delivery), + "both delivery results are kept as reported", + ).toEqual(["gone", "gone"]); + expect( + snapshots.map(({ signals }) => signals.length), + "the first attempt was published before the escalation existed", + ).toContain(1); + expect( + snapshots.at(-1)?.signals.length, + "the last published record holds both attempts", + ).toBe(2); +}); + +it.each(["identity-changed", "unverified"] as const)( + "stops waiting for an exit when the escalation reports %s", + async (delivery) => { + snapshots.length = 0; + const fake = fakeTerminal({ SIGTERM: "signaled", SIGKILL: delivery }); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + + const result = await pending; + + expect(result.unobserved, "no exit is invented").toBe(true); + expect(result.reason, "the initiating deadline is retained").toBe( + "timeout", + ); + expect( + result.finalization, + "attempts and the undelivered result are kept", + ).toMatchObject({ + elapsed_ms: null, + signals: [ + { signal: "SIGTERM", delivery: "signaled" }, + { signal: "SIGKILL", delivery }, + ], + }); + expect( + fake.signals.filter((signal) => signal.startsWith("terminal:")), + "a failed identity check never falls back to a bare terminal kill", + ).toEqual([]); + }, +); + +it("returns at once when a cancellation kill cannot be delivered", async () => { + const fake = fakeTerminal({ SIGKILL: "unverified" }); + const controller = new AbortController(); + const pending = start( + fake, + { timeout_ms: 100, finalization_ms: 60_000 }, + controller.signal, + ); + await waitFor(() => fake.signals.includes("SIGTERM")); + controller.abort(); + + const result = await pending; + + expect(result.reason, "cancellation replaces the deadline").toBe("cancelled"); + expect(result.unobserved, "the capture does not wait for an exit").toBe(true); +}); + +it("waits for a pending delivery before reporting an observed exit", async () => { + let release: () => void = () => undefined; + const hold = new Promise((resolve) => { + release = resolve; + }); + const fake = fakeTerminal({ SIGTERM: "signaled" }, hold); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => fake.signals.includes("SIGTERM")); + fake.deliverExit({ exitCode: 0, signal: 0 }); + let settled = false; + void pending.then(() => { + settled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(settled, "the result waits for the outstanding delivery").toBe(false); + + release(); + const exit = observed(await pending); + + expect( + exit.finalization?.signals, + "the delivery result is complete in the published record", + ).toMatchObject([{ signal: "SIGTERM", delivery: "signaled" }]); +}); + +it("keeps the zero-interval deadline as a bare SIGKILL", async () => { + const fake = fakeTerminal(); + const pending = start(fake, { timeout_ms: 60, finalization_ms: 0 }); + await waitFor(() => fake.signals.includes("terminal:SIGKILL")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + + const exit = observed(await pending); + + expect( + fake.signals.filter((signal) => !signal.startsWith("terminal:")), + "the identity-checked signaller is never used without an interval", + ).toEqual([]); + expect(exit.finalization, "no finalization record exists").toBeUndefined(); +}); + +it("stops waiting for a hung delivery once the exit is observed", async () => { + const fake = fakeTerminal({}, new Promise(() => undefined)); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => fake.signals.includes("SIGTERM")); + fake.deliverExit({ exitCode: 0, signal: 0 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.signals, + "an undelivered attempt stays recorded as unverified", + ).toMatchObject([ + { + signal: "SIGTERM", + delivery: "unverified", + reason: "delivery did not settle within 1000 ms", + }, + ]); + expect(exit.signal, "the observed exit is kept").toBe(0); +}, 10_000); + +it("retains a thrown signal error as an unverified delivery reason", async () => { + const fake = fakeTerminal( + {}, + undefined, + new Error("signal transport failed"), + ); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => fake.signals.includes("SIGTERM")); + fake.deliverExit({ exitCode: 0, signal: 0 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.signals, + "a rejected signal retains its error message", + ).toMatchObject([ + { + signal: "SIGTERM", + delivery: "unverified", + reason: "signal transport failed", + }, + ]); +}); + +it("keeps an observed exit when a late kill delivery reports unverified", async () => { + let release: () => void = () => undefined; + const hold = new Promise((resolve) => { + release = resolve; + }); + const fake = fakeTerminal({ SIGKILL: "unverified" }, hold); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + await waitFor(() => fake.signals.includes("SIGKILL")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + release(); + + const result = await pending; + + expect( + result.unobserved, + "an exit that was already observed is never turned into an unobserved one", + ).not.toBe(true); + expect(observed(result).signal, "the observed exit signal is retained").toBe( + 9, + ); +}); + +it("stops waiting when the escalation delivery never settles and no exit arrives", async () => { + const fake = fakeTerminal({}, new Promise(() => undefined)); + const waitStarted = Date.now(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); + + const result = await pending; + + expect( + Date.now() - waitStarted, + "the wait is bounded by the one-second cleanup grace after the interval", + ).toBeLessThan(2_500); + expect(result.unobserved, "no exit is invented").toBe(true); + expect( + result.finalization?.signals.at(-1), + "the undelivered escalation stays recorded as unverified", + ).toMatchObject({ + signal: "SIGKILL", + delivery: "unverified", + reason: "delivery did not settle within 1000 ms", + }); + expect(result.finalization?.elapsed_ms, "no exit was observed").toBeNull(); +}, 10_000); + +const activeTimers = (): number => + process.getActiveResourcesInfo().filter((name) => name === "Timeout").length; + +it("leaves no referenced timer behind after a zero-interval exit", async () => { + const fake = fakeTerminal(); + const before = activeTimers(); + const pending = start(fake, { timeout_ms: 60, finalization_ms: 0 }); + await waitFor(() => fake.signals.includes("terminal:SIGKILL")); + fake.deliverExit({ exitCode: 0, signal: 9 }); + + await pending; + + expect( + activeTimers(), + "the zero path resolves without a bounded-wait timer", + ).toBeLessThanOrEqual(before); +}); + +it("cancels the bounded-wait timer once deliveries settle after an exit", async () => { + const fake = fakeTerminal(); + const before = activeTimers(); + const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => fake.signals.includes("SIGTERM")); + fake.deliverExit({ exitCode: 0, signal: 0 }); + + await pending; + await new Promise((resolve) => setImmediate(resolve)); + + expect( + activeTimers(), + "no one-second grace timer outlives the finalized exit", + ).toBeLessThanOrEqual(before); +}); + +it("keeps a non-empty reason when a signal delivery throws without a message", async () => { + const fake = fakeTerminal(); + const throwing = { + ...fake, + signalTarget: async (): Promise => { + throw new Error(); + }, + }; + const pending = start(throwing, { timeout_ms: 100, finalization_ms: 5_000 }); + await waitFor(() => throwing.signals.length >= 0); + await new Promise((resolve) => setTimeout(resolve, 150)); + throwing.deliverExit({ exitCode: 0, signal: 0 }); + + const exit = observed(await pending); + + expect( + exit.finalization?.signals[0]?.reason?.length, + "a thrown error without a message still leaves a named reason", + ).toBeGreaterThan(0); +}); + +it("never publishes an unverified attempt without a reason, also while it is in flight", async () => { + const fake = fakeTerminal(); + let calls = 0; + const custom = { + ...fake, + signalTarget: (signal: "SIGTERM" | "SIGKILL") => { + calls += 1; + fake.signals.push(signal); + return calls === 1 + ? new Promise(() => undefined) + : Promise.resolve({ delivery: "identity-changed" as const }); + }, + }; + const pending = start(custom, { timeout_ms: 50, finalization_ms: 20 }); + + const result = await pending; + + expect(result.unobserved, "the undeliverable escalation ends the wait").toBe( + true, + ); + expect( + result.finalization?.signals[0], + "the still-undelivered SIGTERM carries a reason, as the capture schema requires", + ).toMatchObject({ signal: "SIGTERM", delivery: "unverified" }); + expect( + result.finalization?.signals[0]?.reason?.length, + "the in-flight attempt is never published without a reason", + ).toBeGreaterThan(0); +}); diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index e4e59d171..01adafbcb 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -52,6 +52,7 @@ import { scheduleProcessInterval, type ProcessTimer } from "./ProcessTimer.js"; import type { FilesystemCheckpoint, InteractionEvent, + ProcessCaptureFinalization, UnverifiedProcessCapture, ProcessSample, ProcessSettlement, @@ -66,11 +67,20 @@ import type { } from "../../domain/process/processCapture.js"; import { partialProcessCaptureObservationSchema } from "../../domain/process/processCapture.js"; interface TerminalExitOptions { - readonly terminal: IPty; + readonly terminal: Pick; readonly scenario: ProcessScenario; readonly started: number; readonly lastOutput: () => number; readonly signal: AbortSignal | undefined; + /** Identity-checked signal to the captured root, used only during finalization. */ + readonly signalTarget: (signal: FinalizationSignal) => Promise<{ + readonly delivery: FinalizationDelivery; + readonly reason?: string | undefined; + }>; + /** Receives the attempts so far, so a later failure keeps them. */ + readonly recordFinalization: ( + finalization: ProcessCaptureFinalization, + ) => void; readonly timers: Set; readonly interactions: InteractionEvent[]; readonly dispatchedEventIndexes: ReadonlySet; @@ -78,12 +88,33 @@ interface TerminalExitOptions { readonly onLiveProgress?: () => void; } +type FinalizationSignal = + ProcessCaptureFinalization["signals"][number]["signal"]; +type FinalizationDelivery = + ProcessCaptureFinalization["signals"][number]["delivery"]; + +/** The capture's exit, or the facts known when no exit could be observed. */ +export type TerminalExitResult = + | { + readonly unobserved?: false; + readonly exitCode: number; + readonly signal?: number; + readonly reason: "exited" | "timeout" | "idle_timeout" | "cancelled"; + readonly finalization?: ProcessCaptureFinalization; + } + | { + readonly unobserved: true; + readonly reason: "timeout" | "idle_timeout" | "cancelled"; + readonly finalization: ProcessCaptureFinalization; + }; + interface CaptureResultOptions { readonly frames: readonly TerminalFrame[]; readonly exit: { readonly exitCode: number; readonly signal?: number; readonly reason: "exited" | "timeout" | "idle_timeout"; + readonly finalization?: ProcessCaptureFinalization; }; readonly samples: readonly ProcessSample[]; readonly before: ProcessFilesystemSnapshot; @@ -130,6 +161,7 @@ export interface ProcessCaptureObservationBuffer { UnverifiedProcessCapture["interaction_events"] >; exit: IncompleteProcessCaptureObservations["exit"]; + finalization?: IncompleteProcessCaptureObservations["finalization"]; settlement: IncompleteProcessCaptureObservations["settlement"]; process_samples: PartialProcessObservationField; filesystem_snapshots: { @@ -149,6 +181,8 @@ export const createProcessCaptureObservationBuffer = (options: { readonly samples: readonly ProcessSample[]; readonly eventJournal: readonly ProcessCaptureEventJournalEntry[]; readonly before: ProcessFilesystemSnapshot; + /** Seed a finalization observation only for a scenario with an interval. */ + readonly finalizationEnabled?: boolean; }): ProcessCaptureObservationBuffer => ({ target_pid: { state: "unavailable", @@ -165,6 +199,15 @@ export const createProcessCaptureObservationBuffer = (options: { state: "unavailable", reason: "Terminal exit was not observed before the run failed.", }, + ...(options.finalizationEnabled === true + ? { + finalization: { + state: "unavailable" as const, + reason: + "Finalization was not observed before capture completion finished.", + }, + } + : {}), settlement: { state: "unavailable", reason: @@ -271,6 +314,9 @@ export const buildCaptureResult = ( : null, signal: options.exit.signal ?? null, reason: options.exit.reason, + ...(options.exit.finalization === undefined + ? {} + : { finalization: options.exit.finalization }), }, settlement: options.settlement, process_samples: normalizeProcessSamples( @@ -644,29 +690,172 @@ export const observeSettlement = async ( return { state: "alive_at_deadline", elapsed_ms: Date.now() - started }; }; +/** Name why an attempt is unverified; a blank message never reaches the capture schema. */ +export const describeUnverifiedCause = ( + cause: unknown, + fallback: string, +): string => { + const message = cause instanceof Error ? cause.message : String(cause); + return message.trim() === "" ? fallback : message; +}; + +/** + * Wait for the captured terminal to exit and report how it ended. + * + * A timeout or idle deadline kills the root at once when `finalization_ms` is + * 0. With a positive interval it sends SIGTERM through `signalTarget`, keeps + * observing, and sends SIGKILL once if the interval ends first; cancellation + * during the interval sends that SIGKILL at once. Every attempt is recorded + * with its delivery result before the delivery is awaited, and each wait is + * bounded by the cleanup verification grace. If the SIGKILL cannot be + * delivered the wait ends with `unobserved: true` and no invented exit, so the + * caller can keep its observations and release the run through cleanup. + */ export const awaitTerminalExit = async ({ terminal, scenario, started, lastOutput, signal, + signalTarget, + recordFinalization, timers, interactions, dispatchedEventIndexes, recordEvent, onLiveProgress, -}: TerminalExitOptions): Promise<{ - exitCode: number; - signal?: number; - reason: "exited" | "timeout" | "idle_timeout" | "cancelled"; -}> => +}: TerminalExitOptions): Promise => new Promise((resolveExit) => { // The kill caused by a deadline is observed later as an ordinary PTY exit. // Keep the initiating lifecycle reason so comparisons distinguish a target // exit from harness-owned timeout, idle-timeout, and cancellation cleanup. let reason: "exited" | "timeout" | "idle_timeout" | "cancelled" = "exited"; onLiveProgress?.(); + // A deadline with a finalization interval sends SIGTERM first and keeps + // observing; SIGKILL follows only when the interval elapses or the request + // is cancelled. Both go through the identity-checked signaller. + // The interval is measured on the monotonic clock so a wall-clock change + // cannot shorten it or produce a negative elapsed time. + let finalizationStartedAt: number | undefined; + let escalationSent = false; + let settled = false; + let exitObserved = false; + const finalizationSignals: { + signal: FinalizationSignal; + sent_at_ms: number; + delivery: FinalizationDelivery; + reason?: string | undefined; + }[] = []; + const pendingDeliveries: Promise[] = []; + const snapshot = ( + elapsed: number | null, + exitCode: number | null = null, + ): ProcessCaptureFinalization => ({ + requested_ms: scenario.finalization_ms, + elapsed_ms: elapsed, + exit_code: exitCode, + signals: finalizationSignals.map((attempt) => ({ ...attempt })), + }); + const unobservedExit = (): void => { + if (settled || exitObserved || finalizationStartedAt === undefined) + return; + settled = true; + for (const timer of timers) timer.cancel(); + timers.clear(); + resolveExit({ + unobserved: true, + reason: reason === "exited" ? "timeout" : reason, + finalization: snapshot(null), + }); + }; + // An attempt is recorded before its delivery is awaited, so a hung or + // failed delivery still leaves the attempt in the observations. + const attempt = (finalizationSignal: FinalizationSignal): void => { + if (finalizationStartedAt === undefined) return; + const record: (typeof finalizationSignals)[number] = { + signal: finalizationSignal, + sent_at_ms: Math.round(performance.now() - finalizationStartedAt), + delivery: "unverified" as FinalizationDelivery, + reason: "delivery had not settled when the record was published", + }; + finalizationSignals.push(record); + recordFinalization(snapshot(null)); + const deliveryBound = new AbortController(); + // The wait is bounded by the cleanup verification grace: a delivery that + // never settles is recorded as unverified instead of holding the capture. + pendingDeliveries.push( + Promise.race([ + signalTarget(finalizationSignal), + delay( + PROCESS_CLEANUP_VERIFICATION_GRACE_MS, + { + delivery: "unverified" as const, + reason: `delivery did not settle within ${String(PROCESS_CLEANUP_VERIFICATION_GRACE_MS)} ms`, + }, + { + ref: false, + signal: deliveryBound.signal, + }, + ).catch(() => ({ + delivery: "unverified" as const, + reason: "delivery wait was interrupted", + })), + ]) + .finally(() => { + deliveryBound.abort(); + }) + .then( + ({ delivery, reason: deliveryReason }) => { + record.delivery = delivery; + if (deliveryReason !== undefined) record.reason = deliveryReason; + else delete record.reason; + }, + (cause: unknown) => { + record.delivery = "unverified"; + record.reason = describeUnverifiedCause( + cause, + "signal delivery failed without a message", + ); + }, + ) + .then(() => { + recordFinalization(snapshot(null)); + // A kill that cannot reach the root leaves no exit to wait for. + if ( + finalizationSignal === "SIGKILL" && + record.delivery !== "signaled" && + record.delivery !== "gone" + ) + unobservedExit(); + }), + ); + }; + const forceKill = (): void => { + if (finalizationStartedAt !== undefined) { + if (escalationSent) return; + escalationSent = true; + attempt("SIGKILL"); + return; + } + terminal.kill("SIGKILL"); + }; + const terminateAtDeadline = (): void => { + if (scenario.finalization_ms === 0) { + terminal.kill("SIGKILL"); + return; + } + finalizationStartedAt = performance.now(); + attempt("SIGTERM"); + }; terminal.onExit((exit) => { + if (settled) return; + exitObserved = true; + // Measures when the exit was observed, not when the operating system + // ended the process; the PTY layer can deliver the notification late. + const finalizationElapsedMs = + finalizationStartedAt === undefined + ? undefined + : Math.round(performance.now() - finalizationStartedAt); recordEvent("lifecycle", 0); for (const [eventIndex, event] of scenario.events.entries()) { if (dispatchedEventIndexes.has(eventIndex)) continue; @@ -690,19 +879,60 @@ export const awaitTerminalExit = async ({ timer.cancel(); } timers.clear(); - resolveExit({ ...exit, reason }); + const finish = (): void => { + if (settled) return; + settled = true; + resolveExit({ + ...exit, + reason, + ...(finalizationElapsedMs === undefined + ? {} + : { + finalization: snapshot( + finalizationElapsedMs, + exit.exitCode >= 0 && (exit.signal ?? 0) === 0 + ? exit.exitCode + : null, + ), + }), + }); + }; + if (pendingDeliveries.length === 0) { + finish(); + return; + } + // A delivery that never settles must not hold back an observed exit; + // its attempt stays recorded as unverified. + const grace = new AbortController(); + void Promise.race([ + Promise.allSettled(pendingDeliveries), + delay(PROCESS_CLEANUP_VERIFICATION_GRACE_MS, undefined, { + ref: false, + signal: grace.signal, + }).catch(() => undefined), + ]) + .finally(() => { + grace.abort(); + }) + .then(finish); }); const timeout = scheduleProcessInterval(() => { onLiveProgress?.(); if (signal?.aborted === true) { reason = "cancelled"; - terminal.kill("SIGKILL"); + forceKill(); + } else if (finalizationStartedAt !== undefined) { + if ( + performance.now() - finalizationStartedAt >= + scenario.finalization_ms + ) + forceKill(); } else if (Date.now() - started >= scenario.timeout_ms) { reason = "timeout"; - terminal.kill("SIGKILL"); + terminateAtDeadline(); } else if (Date.now() - lastOutput() >= scenario.idle_timeout_ms) { reason = "idle_timeout"; - terminal.kill("SIGKILL"); + terminateAtDeadline(); } }, 20); timers.add(timeout); @@ -1268,6 +1498,7 @@ export const prepareProcessCapture = async ( samples: [], eventJournal: [], before, + finalizationEnabled: scenario.finalization_ms > 0, }), { scenario }, ); diff --git a/src/process/capture/ProcessHarness.test.ts b/src/process/capture/ProcessHarness.test.ts index 1a898ae94..6cc2c77a9 100644 --- a/src/process/capture/ProcessHarness.test.ts +++ b/src/process/capture/ProcessHarness.test.ts @@ -1,7 +1,10 @@ import { expect, it, vi } from "vitest"; import { processScenarioSchema } from "../../domain/process/processScenario.js"; import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; -import { captureProcessScenario } from "./ProcessHarness.js"; +import { + captureProcessScenario, + retainRootSignaller, +} from "./ProcessHarness.js"; import { settleProcessCaptureJournal } from "./ProcessCaptureLifecycle.js"; import { parseProcessCapture } from "../../domain/process/processCaptureParsing.js"; import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../domain/process/processCaptureExample.js"; @@ -116,6 +119,13 @@ it("keeps prelaunch snapshot evidence and the ownership failure when root cleanu } catch (cause: unknown) { if (!(cause instanceof ProcessCaptureError)) throw cause; expect(cause.cause).toBe(ownershipFailure); + const incomplete = cause.partialObservation; + if (incomplete === undefined || !("observations" in incomplete)) + throw new Error("expected incomplete preparation observations"); + expect( + Object.keys(incomplete.observations), + "a default scenario keeps the base observation keys through the preparation path", + ).not.toContain("finalization"); expect( analysisErrorProjectionSchema.parse(projectAnalysisError(cause)), ).toMatchObject({ @@ -795,3 +805,173 @@ it("fails closed on Windows before resolving or launching scenario paths", async }, }); }); + +it("retains the root start identity once at launch and signals only with it", async () => { + const observe = vi.fn(async () => ({ + state: "readable" as const, + identity: "id-launch", + })); + const signal = vi.fn(async () => "signaled" as const); + const signalRoot = retainRootSignaller(4242, 500, "captured-run", { + observe, + signal, + }); + + expect( + observe, + "the identity is read at launch, before any signal", + ).toHaveBeenCalledTimes(1); + expect( + await signalRoot("SIGTERM"), + "a matching identity is signalled", + ).toEqual({ + delivery: "signaled", + }); + await signalRoot("SIGKILL"); + + expect( + observe, + "later signals never re-read the identity", + ).toHaveBeenCalledTimes(1); + expect( + signal.mock.calls, + "both signals carry the pid and the identity retained at launch", + ).toEqual([ + [4242, "id-launch", "SIGTERM"], + [4242, "id-launch", "SIGKILL"], + ]); +}); + +it.each([ + ["a vanished root", async () => undefined, { delivery: "gone" }], + [ + "an unreadable identity", + async () => ({ state: "unavailable" as const, reason: "unsupported" }), + { delivery: "unverified", reason: "unsupported" }, + ], + [ + "a failed identity read", + async () => { + throw new Error("ps failed"); + }, + { delivery: "unverified", reason: "ps failed" }, + ], +])( + "never signals without a retained identity: %s", + async (_label, read, delivery) => { + const signal = vi.fn(async () => "signaled" as const); + const signalRoot = retainRootSignaller(4242, 500, "captured-run", { + observe: read, + signal, + }); + + expect( + await signalRoot("SIGTERM"), + "the delivery result is recorded", + ).toEqual(delivery); + expect(signal, "no identity means no signal at all").not.toHaveBeenCalled(); + }, +); + +it("does not read an identity when there is no finalization interval", async () => { + const observe = vi.fn(async () => undefined); + const signalRoot = retainRootSignaller(4242, 0, "captured-run", { + observe, + signal: vi.fn(async () => "signaled" as const), + }); + + expect( + observe, + "the zero-interval path never inspects the root", + ).not.toHaveBeenCalled(); + expect( + await signalRoot("SIGKILL"), + "an unretained identity stays unverified", + ).toEqual({ + delivery: "unverified", + reason: "root identity was not retained", + }); +}); + +it.each(["signaled", "gone", "identity-changed", "unverified"] as const)( + "reports the identity-checked delivery result %s unchanged", + async (delivery) => { + const signalRoot = retainRootSignaller(4242, 500, "captured-run", { + observe: async () => ({ state: "readable", identity: "id-launch" }), + signal: async () => delivery, + }); + + expect( + await signalRoot("SIGKILL"), + "the signaller result is not coerced", + ).toEqual( + delivery === "unverified" + ? { + delivery, + reason: + "start identity could not be verified or the signal call failed", + } + : { delivery }, + ); + }, +); + +it("seeds a finalization observation only for a scenario with an interval", () => { + const seed = { + frames: [], + interactions: [], + samples: [], + eventJournal: [], + before: { + files: [], + truncated: false, + completeRoots: [], + coverage: emptyFilesystemCoverage, + }, + }; + + expect( + Object.keys(createProcessCaptureObservationBuffer(seed)), + "a default scenario keeps the base observation keys", + ).not.toContain("finalization"); + expect( + Object.keys( + createProcessCaptureObservationBuffer({ + ...seed, + finalizationEnabled: true, + }), + ), + "a scenario with an interval carries the finalization observation", + ).toContain("finalization"); +}); + +it("keeps a non-empty reason when the identity inspection fails without a message", async () => { + const signalRoot = retainRootSignaller(4242, 500, "captured-run", { + observe: async () => { + throw new Error(); + }, + signal: vi.fn(async () => "signaled" as const), + }); + + const result = await signalRoot("SIGTERM"); + + expect(result.delivery, "the attempt is unverified").toBe("unverified"); + expect( + result.reason?.length, + "the reason is never empty, which the capture schema would reject", + ).toBeGreaterThan(0); +}); + +it("keeps a non-empty reason when the identity diagnostic is empty", async () => { + const signalRoot = retainRootSignaller(4242, 500, "captured-run", { + observe: async () => ({ state: "unavailable" as const, reason: "" }), + signal: vi.fn(async () => "signaled" as const), + }); + + const result = await signalRoot("SIGTERM"); + + expect( + result.reason?.length, + "an empty diagnostic is replaced by a named fallback", + ).toBeGreaterThan(0); +}); diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.ts index b5baf242d..10275a7e3 100644 --- a/src/process/capture/ProcessHarness.ts +++ b/src/process/capture/ProcessHarness.ts @@ -43,6 +43,7 @@ import { type ProcessCaptureCleanupHost, type ProcessCaptureObservationBuffer, type PendingProcessCapture, + describeUnverifiedCause, type ProcessCaptureResourceCleanupOptions, ProcessCaptureResourceScope, } from "./ProcessCaptureLifecycle.js"; @@ -65,6 +66,10 @@ import { type ProcessCaptureProgressTracker, } from "./ProcessCaptureProgress.js"; import type { ProcessOwnershipBaseline } from "../ProcessOwnership.js"; +import { + observeProcessStartIdentity, + signalProcessWithStartIdentity, +} from "../ProcessOwnershipObservation.js"; interface StartedCaptureRuntime { readonly renderer: TerminalRenderer; @@ -75,8 +80,79 @@ interface StartedCaptureRuntime { readonly lastOutput: () => number; readonly rawTerminalRetention: () => TerminalRetention; readonly stopSampler: ReturnType; + /** Signals the captured root only while its launch-time start identity still matches. */ + readonly signalRoot: (signal: "SIGTERM" | "SIGKILL") => Promise<{ + readonly delivery: "signaled" | "gone" | "identity-changed" | "unverified"; + readonly reason?: string | undefined; + }>; } +/** Identity observation and signalling used for finalization signals; injectable for tests. */ +export interface FinalizationHost { + readonly observe: typeof observeProcessStartIdentity; + readonly signal: typeof signalProcessWithStartIdentity; +} + +/** Bind the captured root's initial identity to its run token before retaining it. */ +export const retainRootSignaller = ( + pid: number, + finalizationMs: number, + runId: string, + host: FinalizationHost = { + observe: (rootPid) => + observeProcessStartIdentity(rootPid, undefined, runId), + signal: signalProcessWithStartIdentity, + }, +): StartedCaptureRuntime["signalRoot"] => { + const retained = + finalizationMs > 0 + ? host.observe(pid).then( + (observation) => + observation === undefined + ? ({ state: "gone" } as const) + : observation.state === "readable" + ? ({ + state: "identity", + identity: observation.identity, + } as const) + : ({ + state: "unverified", + reason: + observation.reason.trim() === "" + ? "start identity is unavailable without a diagnostic" + : observation.reason, + } as const), + (cause: unknown) => + ({ + state: "unverified", + reason: describeUnverifiedCause( + cause, + "start identity inspection failed without a message", + ), + }) as const, + ) + : undefined; + return async (signal) => { + const identity = await retained; + if (identity === undefined) + return { + delivery: "unverified", + reason: "root identity was not retained", + }; + if (identity.state === "unverified") + return { delivery: "unverified", reason: identity.reason }; + if (identity.state === "gone") return { delivery: "gone" }; + const delivery = await host.signal(pid, identity.identity, signal); + return delivery === "unverified" + ? { + delivery, + reason: + "start identity could not be verified or the signal call failed", + } + : { delivery }; + }; +}; + const cleanupFailedStartup = async (options: { readonly cause: unknown; readonly timers: Set; @@ -176,6 +252,7 @@ interface StartCaptureRuntimeOptions { readonly dispatchedEventIndexes: Set; readonly recordEvent: RecordProcessCaptureEvent; readonly signal?: AbortSignal; + readonly finalizationHost?: FinalizationHost; } const startCaptureRuntime = async ( @@ -245,6 +322,12 @@ const startCaptureRuntime = async ( lastOutput: () => lastOutput, rawTerminalRetention, stopSampler, + signalRoot: retainRootSignaller( + terminal.pid, + scenario.finalization_ms, + options.runId, + options.finalizationHost, + ), }; } catch (cause: unknown) { return cleanupFailedStartup({ @@ -371,7 +454,10 @@ const completeCapture = async (options: { readonly frames: readonly TerminalFrame[]; readonly samples: readonly ProcessSample[]; readonly interactions: readonly InteractionEvent[]; - readonly exit: Awaited>; + readonly exit: Extract< + Awaited>, + { readonly unobserved?: false } + >; readonly signal?: AbortSignal; readonly captureSnapshot: typeof snapshotRoots; readonly eventJournal: readonly ProcessCaptureEventJournalEntry[]; @@ -532,6 +618,7 @@ const runProcessScenario = async ( cleanupHost?: ProcessCaptureCleanupHost, progress?: ProcessCaptureProgress, resourceScope?: ProcessCaptureResourceScope, + finalizationHost?: FinalizationHost, ): Promise => { const progressTracker = createProcessCaptureProgressTracker(progress); const frames: TerminalFrame[] = []; @@ -584,6 +671,7 @@ const runProcessScenario = async ( samples, eventJournal, before, + finalizationEnabled: scenario.finalization_ms > 0, }); let runtime: StartedCaptureRuntime | undefined; let actualRootPid: number | undefined; @@ -613,6 +701,7 @@ const runProcessScenario = async ( dispatchedEventIndexes, recordEvent, ...(signal === undefined ? {} : { signal }), + ...(finalizationHost === undefined ? {} : { finalizationHost }), }); stopSampler = runtime.stopSampler; const exit = await awaitTerminalExit({ @@ -621,12 +710,35 @@ const runProcessScenario = async ( started: runtime.started, lastOutput: runtime.lastOutput, signal, + signalTarget: runtime.signalRoot, + recordFinalization: (finalization) => { + observations.finalization = { state: "available", value: finalization }; + }, timers, interactions, dispatchedEventIndexes, recordEvent, ...(progress === undefined ? {} : { onLiveProgress: reportRunning }), }); + if (exit.unobserved === true) { + observations.finalization = { + state: "available", + value: exit.finalization, + }; + observations.exit = { + state: "unavailable", + reason: `The captured process exit was not observed because the finalization SIGKILL could not be delivered (initiating reason: ${exit.reason}).`, + }; + if (exit.reason === "cancelled") throw processCaptureCancelled(); + throw new Error( + "The captured process exit was not observed because the finalization SIGKILL could not be delivered.", + ); + } + if (exit.finalization !== undefined) + observations.finalization = { + state: "available", + value: exit.finalization, + }; observations.exit = { state: "available", value: { @@ -634,6 +746,9 @@ const runProcessScenario = async ( exit.reason === "exited" && exit.exitCode >= 0 ? exit.exitCode : null, signal: exit.signal ?? null, reason: exit.reason, + ...(exit.finalization === undefined + ? {} + : { finalization: exit.finalization }), }, }; capture = await completeCapture({ @@ -710,6 +825,7 @@ export const captureProcessScenario = async ( cleanupHost?: ProcessCaptureCleanupHost, progress?: ProcessCaptureProgress, resourceScope?: ProcessCaptureResourceScope, + finalizationHost?: FinalizationHost, ): Promise> => { const ownershipReason = processCaptureOwnershipUnavailableReason(platform); if (ownershipReason !== undefined) @@ -740,6 +856,7 @@ export const captureProcessScenario = async ( cleanupHost, progress, runResourceScope, + finalizationHost, ); return ok(capture); } catch (cause: unknown) { diff --git a/tests/acceptance/process/processFinalization.test.ts b/tests/acceptance/process/processFinalization.test.ts new file mode 100644 index 000000000..65690bf5c --- /dev/null +++ b/tests/acceptance/process/processFinalization.test.ts @@ -0,0 +1,560 @@ +import { spawn } from "node:child_process"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { expect, onTestFinished } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { partialProcessCaptureObservationSchema } from "../../../src/domain/process/processCapture.js"; +import { observeProcessStartIdentity } from "../../../src/process/ProcessOwnershipObservation.js"; +import { + parseProcessCapture, + type ProcessCapture, +} from "../../../src/domain/process/processCaptureParsing.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { runCliJson } from "../../fixtures/cliJsonProcess.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { itWithCaptureCapability as captureTest } from "../../boundary/process/processCaptureCapability.js"; +import { expectSerializedUnverifiedHostCleanup } from "../../support/hostCleanup.js"; + +const finalizationFixture = fileURLToPath( + new URL("../../fixtures/processFinalization.mjs", import.meta.url), +); + +type CaptureScenario = Record; +type CaptureOutcome = + | { readonly kind: "capture"; readonly capture: ProcessCapture } + | { readonly kind: "partial"; readonly partial: unknown }; + +const captureScenario = ( + mode: "cooperative" | "ignoring" | "descendant", + root: string, + additions: CaptureScenario = {}, + finalExitCode?: number, +): CaptureScenario => ({ + executable: process.execPath, + arguments: [ + finalizationFixture, + mode, + root, + ...(finalExitCode === undefined ? [] : [String(finalExitCode)]), + ], + working_directory: root, + filesystem_observation_paths: [root], + idle_timeout_ms: 10_000, + ...additions, +}); + +const captureViaCli = async ( + root: string, + scenario: CaptureScenario, +): Promise => { + const scenarioPath = join(root, "scenario.json"); + await writeFile(scenarioPath, JSON.stringify(scenario)); + const document = await runCliJson([ + "capture-process", + scenarioPath, + "--format", + "json", + ]); + if ( + typeof document === "object" && + document !== null && + "code" in document && + document.code === "cleanup_incomplete" + ) + return { + kind: "partial", + partial: expectSerializedUnverifiedHostCleanup({ error: document }), + }; + if (typeof document === "object" && document !== null && "code" in document) + throw new Error( + `CLI capture returned an unexpected error: ${JSON.stringify(document).slice(0, 1_000)}`, + ); + return { + kind: "capture", + capture: parseProcessCapture(parseEvidence(document).normalized_result), + }; +}; + +const captureViaMcp = async ( + scenario: CaptureScenario, +): Promise => { + const { call } = await connectLocalToolsMcp(); + const response = await call("capture_process_scenario", scenario); + if (response.isError === true) { + const parsed = parseMcpToolError(response); + if (parsed.error.code === "cleanup_incomplete") + return { + kind: "partial", + partial: expectSerializedUnverifiedHostCleanup(parsed), + }; + throw new Error( + `MCP capture returned an unexpected error: ${JSON.stringify(parsed).slice(0, 1_000)}`, + ); + } + return { + kind: "capture", + capture: parseProcessCapture( + parseEvidence( + toolContract("capture_process_scenario").outputSchema.parse( + response.structuredContent, + ), + ).normalized_result, + ), + }; +}; + +const waitFor = async ( + description: string, + predicate: () => Promise, + deadlineMs = 7_500, +): Promise => { + const deadline = Date.now() + deadlineMs; + while (!(await predicate())) { + if (Date.now() >= deadline) + throw new Error(`timed out waiting for ${description}`); + await delay(25); + } +}; + +const fixturePid = async (root: string): Promise => { + let pid = Number.NaN; + await waitFor("the finalization fixture PID", async () => { + try { + pid = Number.parseInt(await readFile(join(root, "pid.txt"), "utf8"), 10); + return Number.isSafeInteger(pid) && pid > 0; + } catch { + return false; + } + }); + return pid; +}; + +const waitForFinalization = async (root: string): Promise => { + // The fixture writes the PID before it can receive SIGTERM, so once the + // marker exists the PID file is complete. + await waitFor("the finalization fixture to receive SIGTERM", async () => { + try { + await access(join(root, "sigterm-received")); + return true; + } catch { + return false; + } + }); + return fixturePid(root); +}; + +const waitForProcessExit = async (pid: number): Promise => + waitFor(`finalization fixture ${String(pid)} to exit`, async () => { + try { + process.kill(pid, 0); + return false; + } catch (error: unknown) { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ESRCH" + ); + } + }); + +const finalizationFromPartial = (partial: unknown) => { + const parsed = partialProcessCaptureObservationSchema.parse(partial); + if ("capture" in parsed) return parsed.capture.exit.finalization; + return parsed.observations.exit.state === "available" + ? parsed.observations.exit.value.finalization + : undefined; +}; + +const completedCaptureFromOutcome = (outcome: CaptureOutcome) => { + if (outcome.kind === "capture") return outcome.capture; + const partial = partialProcessCaptureObservationSchema.parse(outcome.partial); + expect(partial.execution_failure).toBeNull(); + if (!("capture" in partial)) + throw new Error("host cleanup uncertainty omitted the completed capture"); + return partial.capture; +}; + +const expectCooperativeFinalization = ( + outcome: CaptureOutcome, + expectedExitCode = 0, +): void => { + const capture = completedCaptureFromOutcome(outcome); + expect(capture.exit.reason, "the deadline remains the exit reason").toBe( + "timeout", + ); + expect(capture.exit.code, "a deadline exit has no normal code").toBeNull(); + expect( + capture.exit.finalization, + "the cooperative target receives only SIGTERM", + ).toMatchObject({ + requested_ms: 1_500, + signals: [{ signal: "SIGTERM", delivery: "signaled" }], + }); + expect( + capture.exit.finalization?.signals, + "cooperative finalization does not send SIGKILL", + ).toHaveLength(1); + expect( + capture.exit.finalization?.elapsed_ms, + "the cooperative target exits within its interval", + ).toBeLessThan(1_500); + expect( + capture.exit.finalization?.exit_code, + "the code the finalizer exited with is recorded", + ).toBe(expectedExitCode); + expect( + capture.frames.map(({ data }) => data).join(""), + "finalization output is captured", + ).toContain("finalized"); + expect( + capture.files_after.find(({ path }) => path.endsWith("final.json"))?.sha256, + "the final filesystem snapshot hashes the final report", + ).toMatch(/^[0-9a-f]{64}$/u); + expect( + capture.manifest.scenario.finalization_ms, + "the scenario commitment retains finalization_ms", + ).toBe(1_500); + expect( + capture.manifest.comparison_contract.finalization_ms, + "the comparison contract retains finalization_ms", + ).toBe(1_500); +}; + +const expectIgnoringFinalization = (outcome: CaptureOutcome): void => { + const capture = completedCaptureFromOutcome(outcome); + expect(capture.exit.reason, "the deadline remains the exit reason").toBe( + "timeout", + ); + expect(capture.exit.signal, "the ignoring child ends on SIGKILL").toBe(9); + expect( + capture.exit.finalization?.signals, + "SIGTERM is followed by exactly one SIGKILL", + ).toMatchObject([ + { signal: "SIGTERM", delivery: "signaled" }, + { signal: "SIGKILL", delivery: "signaled" }, + ]); + expect( + capture.exit.finalization?.signals, + "the ignoring child has exactly two finalization attempts", + ).toHaveLength(2); + expect( + capture.exit.finalization?.signals[1]?.sent_at_ms, + "SIGKILL waits for the requested interval", + ).toBeGreaterThanOrEqual(1_500); + expect( + capture.exit.finalization?.elapsed_ms, + "the elapsed finalization duration is observed", + ).toEqual(expect.any(Number)); + expect( + capture.exit.finalization?.exit_code, + "an exit ended by SIGKILL records no code", + ).toBeNull(); + expect( + capture.files_after.find(({ path }) => path.endsWith("final.json")), + "an ignoring child does not write a final report", + ).toBeUndefined(); +}; + +captureTest.each(["CLI", "MCP"] as const)( + "captures a cooperative finalizer through the compiled %s workflow", + async (adapter) => { + const root = await createTestTempDirectory("rea-finalization-accept-"); + const scenario = captureScenario("cooperative", root, { + timeout_ms: 1_500, + finalization_ms: 1_500, + }); + const outcome = + adapter === "CLI" + ? await captureViaCli(root, scenario) + : await captureViaMcp(scenario); + expectCooperativeFinalization(outcome); + }, + 20_000, +); + +captureTest.each(["CLI", "MCP"] as const)( + "records the non-zero exit code of a finalizer through the compiled %s workflow", + async (adapter) => { + const root = await createTestTempDirectory("rea-finalization-accept-"); + const scenario = captureScenario( + "cooperative", + root, + { timeout_ms: 1_500, finalization_ms: 1_500 }, + 3, + ); + const outcome = + adapter === "CLI" + ? await captureViaCli(root, scenario) + : await captureViaMcp(scenario); + expectCooperativeFinalization(outcome, 3); + }, + 20_000, +); + +captureTest.each(["CLI", "MCP"] as const)( + "forces SIGKILL after the finalization interval through the compiled %s workflow", + async (adapter) => { + const root = await createTestTempDirectory("rea-finalization-accept-"); + const scenario = captureScenario("ignoring", root, { + timeout_ms: 1_500, + finalization_ms: 1_500, + }); + const outcome = + adapter === "CLI" + ? await captureViaCli(root, scenario) + : await captureViaMcp(scenario); + expectIgnoringFinalization(outcome); + }, + 20_000, +); + +captureTest( + "retains finalization facts with omitted digests and cleans the owned descendant", + async () => { + const root = await createTestTempDirectory("rea-finalization-descendant-"); + const childDirectory = join(root, "descendant"); + const outcome = await captureViaMcp( + captureScenario("descendant", root, { + timeout_ms: 1_500, + finalization_ms: 1_500, + settle_ms: 100, + limits: { file_bytes: 1 }, + }), + ); + const capture = completedCaptureFromOutcome(outcome); + expect(capture.exit).toMatchObject({ + reason: "timeout", + code: null, + finalization: { + exit_code: 0, + signals: [{ signal: "SIGTERM", delivery: "signaled" }], + }, + }); + expect(capture.exit.finalization?.signals).toHaveLength(1); + expect(capture.settlement).toMatchObject({ + state: "alive_at_deadline", + cleanup_outcome: "cleaned", + }); + expect(capture.frames.map(({ data }) => data).join("")).toContain( + "finalized", + ); + const final = capture.files_after.find(({ path }) => + path.endsWith(":final.json"), + ); + expect(final).toMatchObject({ type: "file", sha256: null }); + expect(capture.truncation_details.filesystem_after).toMatchObject({ + enumeration_truncated: false, + hash_budget_bytes: 1, + hash_omissions: expect.arrayContaining([ + expect.objectContaining({ + path: final?.path, + reason: "file_bytes_budget", + }), + ]), + }); + expect( + JSON.parse(await readFile(join(root, "final.json"), "utf8")), + ).toEqual({ + phase: "final", + }); + await expect( + access(join(childDirectory, "sigterm-received")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + const childPid = await fixturePid(childDirectory); + await expect( + observeProcessStartIdentity(childPid), + ).resolves.toBeUndefined(); + }, + 20_000, +); + +captureTest.each(["CLI", "MCP"] as const)( + "keeps the default identity through the compiled %s workflow without finalization_ms", + async (adapter) => { + const root = await createTestTempDirectory("rea-finalization-default-"); + const scenario = captureScenario("ignoring", root, { timeout_ms: 500 }); + const outcome = + adapter === "CLI" + ? await captureViaCli(root, scenario) + : await captureViaMcp(scenario); + if (outcome.kind === "partial") { + expect( + finalizationFromPartial(outcome.partial), + "the cleanup-flake partial observation has no default finalization", + ).toBeUndefined(); + return; + } + expect( + outcome.capture.exit, + "default captures have no finalization evidence", + ).not.toHaveProperty("finalization"); + expect( + outcome.capture.manifest.scenario, + "the committed scenario keeps its legacy identity", + ).not.toHaveProperty("finalization_ms"); + expect( + outcome.capture.manifest.comparison_contract, + "the comparison contract keeps its legacy identity", + ).not.toHaveProperty("finalization_ms"); + }, + 20_000, +); + +captureTest.each(["CLI", "MCP"] as const)( + "rejects an unrepresentable combined capture budget through the compiled %s workflow", + async (adapter) => { + const root = await createTestTempDirectory("rea-finalization-budget-"); + const scenario = captureScenario("cooperative", root, { + timeout_ms: Number.MAX_SAFE_INTEGER, + finalization_ms: 1, + settle_ms: 0, + }); + if (adapter === "CLI") { + const scenarioPath = join(root, "scenario.json"); + await writeFile(scenarioPath, JSON.stringify(scenario)); + const response = await runCliJson([ + "capture-process", + scenarioPath, + "--format", + "json", + ]); + expect( + JSON.stringify(response), + "CLI rejects the combined budget", + ).toContain("timeout_ms + finalization_ms + settle_ms"); + } else { + const { call } = await connectLocalToolsMcp(); + const response = await call("capture_process_scenario", scenario); + expect(response.isError, "MCP rejects the combined budget").toBe(true); + expect( + JSON.stringify(response), + "MCP explains the combined budget", + ).toContain("timeout_ms + finalization_ms + settle_ms"); + } + await expect( + access(join(root, "periodic.json")), + "invalid budgets are rejected before launching the fixture", + ).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +captureTest( + "cancels the compiled CLI and MCP workflows during finalization without waiting for grace", + async () => { + const cliRoot = await createTestTempDirectory( + "rea-finalization-cli-cancel-", + ); + const cliScenarioPath = join(cliRoot, "scenario.json"); + await writeFile( + cliScenarioPath, + JSON.stringify( + captureScenario("ignoring", cliRoot, { + timeout_ms: 300, + finalization_ms: 20_000, + }), + ), + ); + const cli = spawn( + process.execPath, + [ + "scripts/rea.mjs", + "capture-process", + cliScenarioPath, + "--format", + "json", + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + let stdout = ""; + cli.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + cli.stderr.resume(); + const cliClosed = new Promise<{ readonly code: number | null }>( + (resolve, reject) => { + cli.once("error", reject); + cli.once("close", (code) => resolve({ code })); + }, + ); + onTestFinished(async () => { + if (cli.exitCode === null && cli.signalCode === null) cli.kill("SIGKILL"); + await cliClosed; + }); + const cliFixturePid = await waitForFinalization(cliRoot); + const cliStarted = Date.now(); + cli.kill("SIGINT"); + const cliExit = await cliClosed; + await waitForProcessExit(cliFixturePid); + expect(cliExit.code, "SIGINT produces the CLI cancellation status").toBe( + 130, + ); + expect( + Date.now() - cliStarted, + "CLI cancellation does not wait for the grace interval", + ).toBeLessThan(8_000); + const cliDocument: unknown = JSON.parse(stdout); + if ( + typeof cliDocument === "object" && + cliDocument !== null && + "code" in cliDocument && + cliDocument.code === "cleanup_incomplete" + ) { + const partial = expectSerializedUnverifiedHostCleanup({ + error: cliDocument, + }); + expect( + finalizationFromPartial(partial), + "CLI cleanup-flake partial observation records SIGKILL", + ).toMatchObject({ + signals: [{ signal: "SIGTERM" }, { signal: "SIGKILL" }], + }); + } else { + expect(cliDocument, "CLI returns a typed cancelled result").toMatchObject( + { + code: "cancelled", + }, + ); + } + + const mcpRoot = await createTestTempDirectory( + "rea-finalization-mcp-cancel-", + ); + const { client } = await connectLocalToolsMcp(); + const controller = new AbortController(); + const call = client.callTool( + { + name: "capture_process_scenario", + arguments: captureScenario("ignoring", mcpRoot, { + timeout_ms: 300, + finalization_ms: 20_000, + }), + }, + { signal: controller.signal }, + ); + const mcpFixturePid = await waitForFinalization(mcpRoot); + const mcpStarted = Date.now(); + controller.abort(new Error("cancel finalization acceptance fixture")); + await expect( + call, + "MCP cancellation rejects the pending SDK request", + ).rejects.toThrow("cancel finalization acceptance fixture"); + await waitForProcessExit(mcpFixturePid); + await expect( + client.listTools(), + "the server accepts a follow-up request after cancelling the capture", + ).resolves.toMatchObject({ tools: expect.any(Array) }); + expect( + Date.now() - mcpStarted, + "MCP cancellation does not wait for the grace interval", + ).toBeLessThan(8_000); + }, + 20_000, +); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index a767af442..24563c883 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -8,10 +8,14 @@ import { expect } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { itWithCaptureCapability } from "./processCaptureCapability.js"; -import { captureProcessScenario } from "../../../src/process/capture/ProcessHarness.js"; +import { + captureProcessScenario, + type FinalizationHost, +} from "../../../src/process/capture/ProcessHarness.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { snapshotRoots } from "../../../src/process/capture/FilesystemSnapshot.js"; import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js"; +import { expectUnverifiedHostCleanup } from "../../support/hostCleanup.js"; import { type ProcessCaptureCleanupHost, observeLaunchedExecutable, @@ -36,35 +40,6 @@ type PartialCapture = Extract< >["capture"]; const emptyFilesystemCoverage = emptyProcessCapture().truncation_details.filesystem_before; -const expectUnverifiedHostCleanup = (error: ProcessCaptureError): void => { - const report = error.cleanupReport; - expect(error.reason, error.message).toBe("cleanup_incomplete"); - expect(report?.owned_process_group.state).toBe("unverified"); - const [summary, diagnostics] = ( - report?.owned_process_group.reason ?? "" - ).split(": "); - expect(summary).toMatch( - /^process ownership token could not be read for [1-9][0-9]* live process\(es\)$/u, - ); - const [breakdown, liveCandidates] = - diagnostics?.split("; live candidates ") ?? []; - const categories = breakdown?.split(", ") ?? []; - expect(categories.length).toBeGreaterThan(0); - const expectedCategory = - process.platform === "linux" - ? /^environment_errno_(?:EACCES|EPERM)=[1-9][0-9]*$/u - : /^environment_unavailable=[1-9][0-9]*$/u; - for (const category of categories) expect(category).toMatch(expectedCategory); - expect(liveCandidates).toMatch( - /^(?:[1-9][0-9]*=(?:environment_unavailable|environment_errno_(?:EACCES|EPERM)))(?:, [1-9][0-9]*=(?:environment_unavailable|environment_errno_(?:EACCES|EPERM)))*$/u, - ); - expect(report?.terminal_renderer.state).toBe("cleaned"); - expect(report?.temporary_root).toEqual({ - state: "unverified", - reason: `temporary root retained because owned process cleanup is unverified: ${report?.owned_process_group.reason}`, - }); -}; - const captureObservations = ( result: CaptureRun, ): { @@ -289,6 +264,10 @@ itWithCaptureCapability( if (partial === undefined || !("observations" in partial)) throw new Error("expected incomplete process observations"); const observations = partial.observations; + expect( + Object.keys(observations), + "a default scenario keeps the base observation keys through the run path", + ).not.toContain("finalization"); expect(observations.frames.state).toBe("available"); if (observations.frames.state !== "available") throw new Error("expected terminal output observations"); @@ -484,6 +463,367 @@ itWithCaptureCapability( }, ); +const finalizationFixture = fileURLToPath( + new URL("../../fixtures/processFinalization.mjs", import.meta.url), +); + +const captureFinalizationFixture = async ( + mode: "cooperative" | "ignoring" | "exits", + scenario: Readonly>, + signal?: AbortSignal, +): Promise<{ readonly result: CaptureRun; readonly root: string }> => { + const root = await createTestTempDirectory("rea-finalization-"); + const result = await captureProcessScenario( + parseProcessScenario({ + executable: process.execPath, + arguments: [finalizationFixture, mode, root], + working_directory: root, + filesystem_observation_paths: [root], + idle_timeout_ms: 10_000, + ...scenario, + }), + signal, + ); + return { result, root }; +}; + +const observedFile = ( + capture: ProcessCapture | PartialCapture, + name: string, +): { readonly sha256: string | null } | undefined => + capture.files_after.find((file) => file.path.endsWith(name)); + +itWithCaptureCapability( + "lets a cooperative target write its final report within the finalization interval", + async () => { + // The deadline leaves Node enough time to register its SIGTERM handler. + const { result } = await captureFinalizationFixture("cooperative", { + timeout_ms: 1_500, + finalization_ms: 1_500, + }); + const { capture } = captureObservations(result); + + expect(capture.exit.reason, "initiating deadline stays the reason").toBe( + "timeout", + ); + expect( + capture.exit.code, + "a deadline reason never declares a normal exit code", + ).toBeNull(); + expect(capture.exit.finalization, "finalization is recorded").toMatchObject( + { + requested_ms: 1_500, + exit_code: 0, + signals: [{ signal: "SIGTERM", delivery: "signaled" }], + }, + ); + expect( + capture.exit.finalization?.elapsed_ms, + "a cooperative exit ends before the interval does", + ).toBeLessThan(1_500); + expect( + observedFile(capture, "final.json")?.sha256, + "final report is retained with a digest", + ).toMatch(/^[0-9a-f]{64}$/u); + expect( + capture.frames.map(({ data }) => data).join(""), + "output written during finalization is captured", + ).toContain("finalized"); + }, + 15_000, +); + +itWithCaptureCapability( + "records no finalization when the target exits before any deadline", + async () => { + const { result } = await captureFinalizationFixture("exits", { + finalization_ms: 1_500, + }); + const { capture } = captureObservations(result); + + expect(capture.exit.reason, "the target exited on its own").toBe("exited"); + expect(capture.exit.code, "its exit code is kept").toBe(0); + expect( + capture.exit, + "no deadline fired, so nothing was finalized", + ).not.toHaveProperty("finalization"); + }, + 15_000, +); + +itWithCaptureCapability( + "starts finalization when the idle deadline fires", + async () => { + const { result } = await captureFinalizationFixture("cooperative", { + timeout_ms: 10_000, + idle_timeout_ms: 1_500, + finalization_ms: 1_200, + }); + const { capture } = captureObservations(result); + + expect(capture.exit.reason, "idle deadline is the initiating reason").toBe( + "idle_timeout", + ); + expect( + capture.exit.finalization, + "idle finalization is recorded", + ).toMatchObject({ + requested_ms: 1_200, + signals: [{ signal: "SIGTERM", delivery: "signaled" }], + }); + expect( + observedFile(capture, "final.json")?.sha256, + "final report is retained with a digest", + ).toMatch(/^[0-9a-f]{64}$/u); + }, + 15_000, +); + +itWithCaptureCapability( + "forces the kill when the target ignores the finalization signal", + async () => { + const { result } = await captureFinalizationFixture("ignoring", { + timeout_ms: 500, + finalization_ms: 1_500, + }); + const { capture } = captureObservations(result); + + expect(capture.exit.reason, "initiating deadline stays the reason").toBe( + "timeout", + ); + expect( + capture.exit.finalization, + "finalization attempts are recorded", + ).toMatchObject({ + requested_ms: 1_500, + signals: [ + { signal: "SIGTERM", delivery: "signaled" }, + { signal: "SIGKILL", delivery: "signaled" }, + ], + }); + expect( + capture.exit.signal, + "the target that ignores SIGTERM and SIGINT ends on SIGKILL", + ).toBe(9); + expect( + capture.exit.finalization?.elapsed_ms, + "the whole interval elapsed before the kill", + ).toBeGreaterThanOrEqual(1_500); + expect( + capture.exit.finalization?.elapsed_ms, + "the kill followed the interval promptly", + ).toBeLessThan(4_000); + expect( + observedFile(capture, "final.json"), + "an ignored signal writes no final report", + ).toBeUndefined(); + }, + 15_000, +); + +itWithCaptureCapability( + "keeps the immediate kill when no finalization interval is configured", + async () => { + const { result } = await captureFinalizationFixture("cooperative", { + timeout_ms: 500, + }); + const { capture } = captureObservations(result); + + expect(capture.exit.reason, "deadline reason is unchanged").toBe("timeout"); + expect( + capture.exit.signal, + "the default deadline still ends on SIGKILL", + ).toBe(9); + expect( + capture.exit, + "default captures carry no finalization record", + ).not.toHaveProperty("finalization"); + expect( + observedFile(capture, "final.json"), + "SIGKILL leaves no final report", + ).toBeUndefined(); + }, + 15_000, +); + +itWithCaptureCapability( + "cancels before any deadline without starting finalization", + async () => { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 1_500); + const started = Date.now(); + const { result } = await captureFinalizationFixture( + "ignoring", + { timeout_ms: 20_000, finalization_ms: 20_000 }, + controller.signal, + ); + + if (result.ok) throw new Error("expected cancellation"); + if (!(result.error instanceof ProcessCaptureError)) throw result.error; + const partial = result.error.partialObservation; + if (partial === undefined || !("observations" in partial)) + throw new Error("expected cancelled process observations"); + expect( + partial.observations.exit, + "cancellation before any deadline is not a finalization", + ).toMatchObject({ state: "available", value: { reason: "cancelled" } }); + expect( + partial.observations.exit, + "no SIGTERM interval ran, so no finalization record exists", + ).not.toHaveProperty("value.finalization"); + expect( + Date.now() - started, + "cancellation does not wait for either deadline", + ).toBeLessThan(10_000); + }, + 15_000, +); + +itWithCaptureCapability( + "cancels immediately while a finalization interval is running", + async () => { + const controller = new AbortController(); + // Abort well after the 300 ms deadline so finalization is already running. + setTimeout(() => controller.abort(), 2_000); + const started = Date.now(); + const { result } = await captureFinalizationFixture( + "ignoring", + { timeout_ms: 300, finalization_ms: 20_000 }, + controller.signal, + ); + + if (result.ok) throw new Error("expected cancellation"); + if (!(result.error instanceof ProcessCaptureError)) throw result.error; + const partial = result.error.partialObservation; + if (partial === undefined || !("observations" in partial)) + throw new Error("expected cancelled process observations"); + expect( + partial.observations.exit, + "cancellation is the exit reason and signal attempts are recorded", + ).toMatchObject({ + state: "available", + value: { + reason: "cancelled", + finalization: { + requested_ms: 20_000, + signals: [ + { signal: "SIGTERM", delivery: "signaled" }, + { signal: "SIGKILL", delivery: "signaled" }, + ], + }, + }, + }); + expect( + partial.observations.finalization, + "the attempts are published beside the exit, also for a failed run", + ).toMatchObject({ + state: "available", + value: { + requested_ms: 20_000, + signals: [ + { signal: "SIGTERM", delivery: "signaled" }, + { signal: "SIGKILL" }, + ], + }, + }); + if (result.error.cleanupIncomplete) { + expectUnverifiedHostCleanup(result.error); + expect( + result.error.executionFailure, + "the host cleanup error wraps the cancellation", + ).toBe("process capture was cancelled"); + } + const cancellation = result.error.cleanupIncomplete + ? result.error.cause + : result.error; + expect(cancellation, "cancellation wins over finalization").toMatchObject({ + reason: "cancelled", + userCategory: "cancelled", + }); + expect( + Date.now() - started, + "cancellation does not wait for the finalization interval", + ).toBeLessThan(8_000); + }, + 15_000, +); + +itWithCaptureCapability( + "keeps the attempts when the escalation cannot be delivered through the retained identity", + async () => { + const calls: string[] = []; + let observedAt: number | undefined; + let firstSignalAt: number | undefined; + const finalizationHost: FinalizationHost = { + observe: async () => { + observedAt ??= performance.now(); + return { state: "readable" as const, identity: "launch" }; + }, + signal: async (_pid, identity, signal) => { + firstSignalAt ??= performance.now(); + calls.push(`${identity}:${signal}`); + return signal === "SIGTERM" ? "signaled" : "unverified"; + }, + }; + const root = await createTestTempDirectory("rea-finalization-host-"); + const result = await captureProcessScenario( + parseProcessScenario({ + executable: process.execPath, + arguments: [finalizationFixture, "ignoring", root], + working_directory: root, + filesystem_observation_paths: [root], + idle_timeout_ms: 10_000, + timeout_ms: 500, + finalization_ms: 500, + }), + undefined, + process.platform, + process.env, + undefined, + undefined, + undefined, + undefined, + finalizationHost, + ); + + if (result.ok) throw new Error("expected an unobserved exit"); + if (!(result.error instanceof ProcessCaptureError)) throw result.error; + if (result.error.cleanupIncomplete) + expectUnverifiedHostCleanup(result.error); + expect( + calls, + "both finalization signals went through the identity-checked host", + ).toEqual(["launch:SIGTERM", "launch:SIGKILL"]); + expect( + (firstSignalAt ?? 0) - (observedAt ?? Number.POSITIVE_INFINITY), + "the start identity was read at launch, well before the 500 ms deadline signalled", + ).toBeGreaterThanOrEqual(300); + const partial = result.error.partialObservation; + if (partial === undefined || !("observations" in partial)) + throw new Error("expected incomplete process observations"); + expect( + partial.observations.finalization, + "the attempts and their delivery results are published for a failed capture", + ).toMatchObject({ + state: "available", + value: { + requested_ms: 500, + elapsed_ms: null, + exit_code: null, + signals: [ + { signal: "SIGTERM", delivery: "signaled" }, + { signal: "SIGKILL", delivery: "unverified" }, + ], + }, + }); + expect( + partial.observations.exit.state, + "no exit is invented when the escalation was not delivered", + ).toBe("unavailable"); + }, + 15_000, +); + itWithCaptureCapability( "classifies cancellation raised by the initial filesystem snapshot", async () => { diff --git a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts index 68c2fceac..1513d6459 100644 --- a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts +++ b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts @@ -9,6 +9,7 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { compareProcessCaptures } from "../../../src/domain/process/processComparison.js"; import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; import { parseProcessScenario } from "../../../src/domain/process/processScenario.js"; +import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -60,6 +61,27 @@ const captureViaMcp = async (scenario: Record) => { return evidence; }; +it("documents a finalization scenario and exit record that match the contract", async () => { + const document = await readFile("docs/process-capture.md", "utf8"); + const blocks = [ + ...(document.split("### Finalization interval")[1] ?? "").matchAll( + /```json\n([\s\S]*?)\n```/gu, + ), + ].map((match) => match[1]); + const [scenario, exit] = blocks; + if (scenario === undefined || exit === undefined) + throw new Error("Missing finalization examples"); + + expect( + parseProcessScenario(JSON.parse(scenario)).finalization_ms, + "the documented scenario is accepted and carries the interval", + ).toBe(1_500); + expect( + JSON.parse(exit), + "the documented exit record is the canonical finalized example", + ).toEqual(FINALIZED_PROCESS_CAPTURE_EXAMPLE.exit); +}); + itWithCaptureCapability.each(["cli", "mcp"] as const)( "hashes the documented complete report despite omitted terminal output through %s", async (adapter) => { diff --git a/tests/fixtures/localToolsMcp.ts b/tests/fixtures/localToolsMcp.ts index 2b08642ac..562deb232 100644 --- a/tests/fixtures/localToolsMcp.ts +++ b/tests/fixtures/localToolsMcp.ts @@ -1,6 +1,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; +import type { ValidateFunction } from "ajv"; import { fileURLToPath } from "node:url"; import { expect, onTestFinished } from "vitest"; import { z } from "zod"; @@ -27,31 +28,55 @@ export async function connectLocalToolsMcp() { }); await client.connect(transport); const tools = (await client.listTools()).tools; - const validators = new Map( - tools.flatMap((tool) => { - new Ajv2020({ strict: false, validateFormats: false }).compile( - tool.inputSchema, + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + for (const tool of tools) + for (const [kind, schema] of [ + ["inputSchema", tool.inputSchema], + ["outputSchema", tool.outputSchema], + ] as const) { + if (schema === undefined) continue; + const valid = schemaValidator.validateSchema( + z.record(z.string(), z.unknown()).parse(schema), ); - return tool.outputSchema === undefined - ? [] - : [ - [ - tool.name, - new Ajv2020({ strict: false, validateFormats: false }).compile( - z.record(z.string(), z.unknown()).parse(tool.outputSchema), - ), - ] as const, - ]; - }), - ); + expect( + valid, + `${tool.name}.${kind}: ${schemaValidator.errorsText(schemaValidator.errors)}`, + ).toBe(true); + } + const definitions = new Map(tools.map((tool) => [tool.name, tool])); + const compiledInputs = new Set(); + const validators = new Map>(); const call = async (name: string, arguments_: Record) => { + const definition = definitions.get(name); + if (definition !== undefined && !compiledInputs.has(name)) { + new Ajv2020({ strict: false, validateFormats: false }).compile( + definition.inputSchema, + ); + compiledInputs.add(name); + } const response = await client.callTool({ name, arguments: arguments_ }); - const validate = validators.get(name); - if (response.structuredContent !== undefined && validate !== undefined) + if ( + response.structuredContent !== undefined && + definition?.outputSchema !== undefined + ) { + let validate = validators.get(name); + if (validate === undefined) { + validate = new Ajv2020({ + strict: false, + validateFormats: false, + }).compile( + z.record(z.string(), z.unknown()).parse(definition.outputSchema), + ); + validators.set(name, validate); + } expect( validate(response.structuredContent), JSON.stringify(validate.errors), ).toBe(true); + } return response; }; return { client, call }; diff --git a/tests/fixtures/processFinalization.mjs b/tests/fixtures/processFinalization.mjs new file mode 100644 index 000000000..f70aa0d8b --- /dev/null +++ b/tests/fixtures/processFinalization.mjs @@ -0,0 +1,51 @@ +import { spawn } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Writes a periodic report, then behaves by mode: +// - "cooperative": stays alive; SIGTERM prints a line, writes a final report +// and exits cleanly. +// - "ignoring": stays alive and swallows SIGTERM and SIGINT, so only SIGKILL +// ends the process. +// - "exits": exits on its own at once. +// - "descendant": cooperates while leaving a SIGTERM-ignoring child for owned cleanup. +// An optional third argument is the exit code a cooperative finalizer uses. +const [mode, directory, finalExitCode = "0"] = process.argv.slice(2); + +writeFileSync(join(directory, "pid.txt"), String(process.pid)); +writeFileSync( + join(directory, "periodic.json"), + JSON.stringify({ phase: "periodic" }), +); + +if (mode === "exits") process.exit(0); + +if (mode === "descendant") { + const childDirectory = join(directory, "descendant"); + mkdirSync(childDirectory); + spawn( + process.execPath, + [fileURLToPath(import.meta.url), "ignoring", childDirectory], + { detached: true, stdio: "ignore" }, + ); +} + +if (mode === "ignoring") { + process.on("SIGTERM", () => { + writeFileSync(join(directory, "sigterm-received"), "received"); + }); + process.on("SIGINT", () => undefined); +} else { + process.on("SIGTERM", () => { + writeFileSync(join(directory, "sigterm-received"), "received"); + process.stdout.write("finalized\n"); + writeFileSync( + join(directory, "final.json"), + JSON.stringify({ phase: "final" }), + ); + process.exit(Number(finalExitCode)); + }); +} + +setInterval(() => undefined, 1_000); diff --git a/tests/support/hostCleanup.ts b/tests/support/hostCleanup.ts new file mode 100644 index 000000000..443ba8581 --- /dev/null +++ b/tests/support/hostCleanup.ts @@ -0,0 +1,120 @@ +import { expect } from "vitest"; + +import type { ProcessCaptureError } from "../../src/process/capture/ProcessCaptureError.js"; + +type SerializedCleanupReport = { + readonly owned_process_group: { + readonly state: unknown; + readonly reason: unknown; + }; + readonly terminal_renderer: { readonly state: unknown }; + readonly temporary_root: { + readonly state: unknown; + readonly reason: unknown; + }; +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + +const isSerializedCleanupReport = ( + value: unknown, +): value is SerializedCleanupReport => + isRecord(value) && + isRecord(value.owned_process_group) && + "state" in value.owned_process_group && + "reason" in value.owned_process_group && + isRecord(value.terminal_renderer) && + "state" in value.terminal_renderer && + isRecord(value.temporary_root) && + "state" in value.temporary_root && + "reason" in value.temporary_root; + +const expectHostCleanupReport = (report: { + readonly owned_process_group: { + readonly state: unknown; + readonly reason: unknown; + }; + readonly terminal_renderer: { readonly state: unknown }; + readonly temporary_root: { + readonly state: unknown; + readonly reason: unknown; + }; +}): void => { + expect(report.owned_process_group.state).toBe("unverified"); + const reason = report.owned_process_group.reason; + expect(typeof reason).toBe("string"); + if (typeof reason !== "string") + throw new Error("cleanup report omitted reason"); + const category = + process.platform === "linux" + ? "environment_errno_(?:EACCES|EPERM)" + : "environment_unavailable"; + const candidate = + "[1-9][0-9]*=(?:environment_unavailable|environment_errno_(?:EACCES|EPERM))"; + expect(reason).toMatch( + new RegExp( + `^process ownership token could not be read for [1-9][0-9]* live process\\(es\\): ` + + `${category}=[1-9][0-9]*(?:, ${category}=[1-9][0-9]*)*; ` + + `live candidates ${candidate}(?:, ${candidate})*$`, + "u", + ), + ); + expect(report.terminal_renderer.state).toBe("cleaned"); + // The temporary root is retained while owned process cleanup is unverified. + expect(report.temporary_root).toEqual({ + state: "unverified", + reason: `temporary root retained because owned process cleanup is unverified: ${reason}`, + }); +}; + +/** Permit only the host ownership-token cleanup flake seen by real captures. */ +export const expectUnverifiedHostCleanup = ( + error: ProcessCaptureError, +): void => { + expect(error.reason, error.message).toBe("cleanup_incomplete"); + const report = error.cleanupReport; + expect(report).toBeDefined(); + if (report === undefined) + throw new Error("cleanup-incomplete error omitted report"); + expectHostCleanupReport(report); +}; + +/** + * Assert the same narrowly-defined cleanup flake after CLI/MCP serialization + * and return the observations that callers must continue to verify. + */ +export const expectSerializedUnverifiedHostCleanup = ( + value: unknown, +): unknown => { + expect(typeof value).toBe("object"); + expect(value).not.toBeNull(); + if (typeof value !== "object" || value === null || !("error" in value)) + throw new Error("cleanup error omitted its serialized error envelope"); + const error = value.error; + expect(typeof error).toBe("object"); + expect(error).not.toBeNull(); + if ( + typeof error !== "object" || + error === null || + !("code" in error) || + !("details" in error) + ) + throw new Error("cleanup error omitted its serialized details"); + expect(error.code).toBe("cleanup_incomplete"); + const details = error.details; + expect(typeof details).toBe("object"); + expect(details).not.toBeNull(); + if ( + typeof details !== "object" || + details === null || + !("cleanup_report" in details) || + !("partial_observation" in details) + ) + throw new Error("cleanup error omitted report or partial observation"); + const cleanupReport = details.cleanup_report; + if (!isSerializedCleanupReport(cleanupReport)) + throw new Error("cleanup error report is incomplete"); + expectHostCleanupReport(cleanupReport); + return details.partial_observation; +};