diff --git a/docs/process-capture.md b/docs/process-capture.md index 0c5841470..d93a968f3 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -119,7 +119,7 @@ all endpoint numbers in comparison text. Captures commit the port-normalization version so older broad numeric normalization cannot silently compare as the same contract. -Output, file count, file size, process sampling, filesystem depth, total +Output retention, file count, aggregate file-hashing bytes, process sampling, filesystem depth, total runtime, idle time, and post-exit settlement are bounded by the scenario's limits. The result marks truncated observations and residual unknowns rather than treating missing data as proof of equivalence. Cancellation and timeout @@ -127,6 +127,35 @@ run the same owned-process cleanup path. Settlement reports whether the sampled process group quiesced or whether cleanup was needed or unverifiable; sampling cannot prove that every short-lived or detached descendant was seen. +New captures include `truncation_details`, with separate accounting for: + +- `raw_terminal`: original UTF-8 PTY chunk bytes and observed/retained chunk + counts. A chunk that does not fit `limits.output_bytes` is omitted whole; + a later smaller chunk can still fit. +- `rendered_terminal`: cumulative UTF-8 serialized-state and visible-line + bytes, plus observed/retained frame counts. This is independent of the raw + chunk budget and does not count JSON encoding. Repeated screen snapshots can + exhaust it even when all raw output fits. Rendered states use retained PTY + input, so raw omissions also limit rendered coverage. +- `filesystem_before` and `filesystem_after`: file-count/depth limits, + enumeration failures, whole-file hash budget and bytes successfully hashed. + Each retained regular file without a digest has an aliased path, size, + remaining budget and reason: `file_bytes_budget` or + `file_changed_or_short_read`. A file too large for the remaining budget is + skipped; a later smaller file can still be hashed. Hash omissions do not + imply incomplete path enumeration. +- `process`: sampling limit and whether sampling ended partially. Coverage + remains `sampled`, including when that limit was not exhausted. + +The legacy `truncated` flag summarizes these observations. Comparisons retain +results for unaffected dimensions and mark affected dimensions unknown; an +incomplete capture cannot locate the first divergence across all dimensions. +Trace assertions use coverage for the sources they select, so an assertion +about complete raw terminal output need not fail because rendered snapshots +were omitted. Older truncated captures without these details retain the +conservative comparison behavior. These diagnostics use the existing scenario +budgets; they do not introduce a separate file-hashing budget. + When the host withholds an unrelated process’s ownership token, REA leaves that process untouched and records its PID and reason in `cleanup.unverified_processes` and process residual unknowns. Successful cleanup verifies the owned group; diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.ts index bbfbdfa0c..414c97d4d 100644 --- a/src/domain/process/processCapture.ts +++ b/src/domain/process/processCapture.ts @@ -3,6 +3,10 @@ import { jsonValueSchema } from "../jsonValue.js"; import { normalizationSchema } from "./processScenario.js"; import { collectProcessCaptureIssues } from "./processCaptureValidation.js"; +import { + processCaptureTruncationDetailsSchema, + type ProcessCaptureTruncationDetails, +} from "./processCaptureCoverage.js"; export * from "./processScenario.js"; @@ -206,6 +210,8 @@ export interface UnverifiedProcessCapture { readonly files_after: readonly FileState[]; readonly filesystem_effects: readonly FileEffect[]; readonly truncated: boolean; + /** Optional on older captures; identifies each producer's actual coverage. */ + readonly truncation_details?: ProcessCaptureTruncationDetails | undefined; readonly limitations: readonly string[]; readonly residual_unknowns: readonly { readonly scope: @@ -477,6 +483,7 @@ const processCaptureShapeSchema = z.strictObject({ files_after: z.array(fileStateSchema), filesystem_effects: z.array(fileEffectSchema), truncated: z.boolean(), + truncation_details: processCaptureTruncationDetailsSchema.optional(), limitations: z.array(z.string()), residual_unknowns: z.array( z.object({ diff --git a/src/domain/process/processCaptureCoverage.ts b/src/domain/process/processCaptureCoverage.ts new file mode 100644 index 000000000..5365961ef --- /dev/null +++ b/src/domain/process/processCaptureCoverage.ts @@ -0,0 +1,101 @@ +import { z } from "zod"; +import type { UnverifiedProcessCapture } from "./processCapture.js"; +import type { ProcessObservationSource } from "./processObservation.js"; + +const byteRetentionSchema = z.strictObject({ + budget_bytes: z.number().int().nonnegative(), + observed_bytes: z.number().int().nonnegative(), + retained_bytes: z.number().int().nonnegative(), + observed_frames: z.number().int().nonnegative(), + retained_frames: z.number().int().nonnegative(), +}); + +/** Filesystem enumeration and whole-file digest coverage for one checkpoint. */ +export const filesystemCoverageSchema = z.strictObject({ + files_limit: z.number().int().nonnegative(), + depth_limit: z.number().int().nonnegative(), + enumeration_truncated: z.boolean(), + enumeration_reasons: z.array( + z.enum(["files_limit", "depth_limit", "identity_changed"]), + ), + hash_budget_bytes: z.number().int().nonnegative(), + hashed_bytes: z.number().int().nonnegative(), + hash_omissions: z.array( + z.strictObject({ + path: z.string(), + size_bytes: z.number().int().nonnegative(), + remaining_budget_bytes: z.number().int().nonnegative(), + reason: z.enum(["file_bytes_budget", "file_changed_or_short_read"]), + }), + ), +}); +export type FilesystemCoverage = z.infer; + +/** Actual budget accounting; rendered bytes count state and visible lines, not JSON. */ +export const processCaptureTruncationDetailsSchema = z.strictObject({ + raw_terminal: byteRetentionSchema.describe( + "UTF-8 bytes of original PTY chunks; whole chunks are retained or omitted.", + ), + rendered_terminal: byteRetentionSchema.describe( + "Cumulative UTF-8 bytes of serialized state plus visible lines, evaluated from retained PTY input; whole frames are retained or omitted.", + ), + filesystem_before: filesystemCoverageSchema, + filesystem_after: filesystemCoverageSchema, + process: z.strictObject({ + sample_limit: z.number().int().nonnegative(), + sampling_partial: z.boolean(), + retained_samples: z.number().int().nonnegative(), + sample_limit_reached: z.boolean(), + sampling_failures: z.number().int().nonnegative(), + first_sampling_failure: z.string().nullable(), + coverage: z.literal("sampled"), + }), +}); +export type ProcessCaptureTruncationDetails = z.infer< + typeof processCaptureTruncationDetailsSchema +>; +export type TerminalRetention = ProcessCaptureTruncationDetails["raw_terminal"]; +export type ProcessSamplingCoverage = + ProcessCaptureTruncationDetails["process"]; + +const omitted = (retention: TerminalRetention): boolean => + retention.observed_frames > retention.retained_frames; +const filesystemTruncated = (coverage: FilesystemCoverage): boolean => + coverage.enumeration_truncated || coverage.hash_omissions.length > 0; + +/** Derive the legacy aggregate flag from independent producer observations. */ +export const hasCaptureTruncation = ( + details: ProcessCaptureTruncationDetails, +): boolean => + omitted(details.raw_terminal) || + omitted(details.rendered_terminal) || + filesystemTruncated(details.filesystem_before) || + filesystemTruncated(details.filesystem_after) || + details.process.sampling_partial; + +/** Older truncated captures lack enough information to localize missing observations. */ +export const processSourceTruncated = ( + capture: UnverifiedProcessCapture, + source: ProcessObservationSource, +): boolean => { + const details = capture.truncation_details; + if (details === undefined) return capture.truncated; + switch (source) { + case "terminal_raw": + return omitted(details.raw_terminal); + case "terminal_rendered": + return ( + omitted(details.raw_terminal) || omitted(details.rendered_terminal) + ); + case "filesystem": + return ( + filesystemTruncated(details.filesystem_before) || + filesystemTruncated(details.filesystem_after) + ); + case "process": + return details.process.sampling_partial; + case "interaction": + case "lifecycle": + return false; + } +}; diff --git a/src/domain/process/processCaptureValidation.ts b/src/domain/process/processCaptureValidation.ts index 9ab0f749c..02d0b8157 100644 --- a/src/domain/process/processCaptureValidation.ts +++ b/src/domain/process/processCaptureValidation.ts @@ -1,5 +1,6 @@ import type { UnverifiedProcessCapture } from "./processCapture.js"; import { digestProcessCommitment } from "./processScenario.js"; +import { hasCaptureTruncation } from "./processCaptureCoverage.js"; /** One pure semantic validation failure in a shaped process capture. */ export interface ProcessCaptureValidationIssue { @@ -121,6 +122,81 @@ const validateLifecycle = ( null, "exit", "deadline termination cannot declare a normal exit code"); }; +const validateCoverage = ( + capture: UnverifiedProcessCapture, + require: RequireInvariant, +): void => { + const details = capture.truncation_details; + if (details === undefined) return; + require(details.process.retained_samples === capture.process_samples.length && + details.process.retained_samples <= details.process.sample_limit && + details.process.sampling_partial === + (details.process.sample_limit_reached || + details.process.sampling_failures > 0) && + (details.process.sampling_failures === 0) === + (details.process.first_sampling_failure === + null), "truncation_details.process", "sampling coverage must agree with retained samples and observed failures"); + require(capture.truncated === + hasCaptureTruncation( + details, + ), "truncation_details", "aggregate truncation must match producer coverage"); + for (const [name, retention, count] of [ + ["raw_terminal", details.raw_terminal, capture.frames.length], + [ + "rendered_terminal", + details.rendered_terminal, + capture.rendered_frames.length, + ], + ] as const) { + require(retention.retained_frames === count && + retention.observed_frames >= count && + retention.observed_bytes >= retention.retained_bytes && + retention.retained_bytes <= + retention.budget_bytes, `truncation_details.${name}`, "retained observations and budget accounting do not agree"); + } + for (const [name, coverage, files, checkpoint] of [ + [ + "filesystem_before", + details.filesystem_before, + capture.files_before, + capture.filesystem_checkpoints[0], + ], + [ + "filesystem_after", + details.filesystem_after, + capture.files_after, + capture.filesystem_checkpoints[1], + ], + ] as const) { + const unhashedFiles = new Map( + files + .filter((file) => file.type === "file" && file.sha256 === null) + .map((file) => [file.path, file]), + ); + require(coverage.hash_omissions.length === unhashedFiles.size && + new Set(coverage.hash_omissions.map(({ path }) => path)).size === + unhashedFiles.size && + coverage.hash_omissions.every( + (omission) => + unhashedFiles.get(omission.path)?.size === omission.size_bytes && + omission.remaining_budget_bytes <= coverage.hash_budget_bytes, + ), `truncation_details.${name}.hash_omissions`, "every retained regular file without a digest must have one matching omission reason"); + require(coverage.hashed_bytes === + files + .filter((file) => file.type === "file" && file.sha256 !== null) + .reduce((sum, file) => sum + file.size, 0) && + coverage.hashed_bytes <= coverage.hash_budget_bytes && + files.length <= + coverage.files_limit, `truncation_details.${name}`, "filesystem budget accounting does not match retained observations"); + require(coverage.enumeration_truncated === + coverage.enumeration_reasons.length > 0 && + checkpoint?.truncated === + (coverage.enumeration_truncated || + coverage.hash_omissions.length > + 0), `truncation_details.${name}`, "filesystem checkpoint truncation must match enumeration and digest coverage"); + } +}; + /** Recompute commitments and cross-field invariants without side effects. */ export const collectProcessCaptureIssues = ( capture: UnverifiedProcessCapture, @@ -133,5 +209,6 @@ export const collectProcessCaptureIssues = ( validateOrdering(capture, require); validateEventJournal(capture, require); validateLifecycle(capture, require); + validateCoverage(capture, require); return issues; }; diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts index 04542631a..f0edec0ff 100644 --- a/src/domain/process/processComparison.ts +++ b/src/domain/process/processComparison.ts @@ -1,5 +1,7 @@ import canonicalize from "canonicalize"; import { z } from "zod"; +import { processSourceTruncated } from "./processCaptureCoverage.js"; +import type { ProcessObservationSource } from "./processObservation.js"; import type { ProcessCapture } from "./processCapture.js"; import { comparableTerminalFrame } from "./processObservation.js"; @@ -363,8 +365,13 @@ const compareDimensions = ( scope: ProcessCapture["residual_unknowns"][number]["scope"], leftValues: readonly unknown[], rightValues: readonly unknown[], + source: ProcessObservationSource, ): ComparisonStatus => - !sameNormalization || hasUnknown(left, scope) || hasUnknown(right, scope) + !sameNormalization || + hasUnknown(left, scope) || + hasUnknown(right, scope) || + processSourceTruncated(left, source) || + processSourceTruncated(right, source) ? "unknown" : classifyCollection(leftValues, rightValues); return { @@ -372,11 +379,13 @@ const compareDimensions = ( "terminal", terminalObservations(left), terminalObservations(right), + "terminal_rendered", ), interaction: classify( "interaction", left.interaction_events, right.interaction_events, + "interaction", ), exit: hasUnknown(left, "exit") || hasUnknown(right, "exit") @@ -391,8 +400,14 @@ const compareDimensions = ( "filesystem", filesystemObservations(left), filesystemObservations(right), + "filesystem", + ), + process: classify( + "process", + left.process_samples, + right.process_samples, + "process", ), - process: classify("process", left.process_samples, right.process_samples), }; }; @@ -412,7 +427,10 @@ export const compareProcessCaptures = ( } = {}, ): ProcessCaptureComparison => { assertComparable(left, right, options); - if (left.truncated || right.truncated) { + if ( + (left.truncated && left.truncation_details === undefined) || + (right.truncated && right.truncation_details === undefined) + ) { const truncated = truncatedComparison(); return options.traceSpecification === undefined ? truncated @@ -438,11 +456,13 @@ export const compareProcessCaptures = ( right, ]); const divergenceCandidates = PROCESS_COMPARISON_DIMENSIONS.map((dimension) => - firstCollectionDivergence( - dimension, - processDimensionObservations(left, dimension), - processDimensionObservations(right, dimension), - ), + dimensions[dimension] === "unknown" + ? null + : firstCollectionDivergence( + dimension, + processDimensionObservations(left, dimension), + processDimensionObservations(right, dimension), + ), ); const observedFirstDivergence = chooseFirstDivergence( divergenceCandidates.map((candidate) => @@ -454,22 +474,30 @@ export const compareProcessCaptures = ( ), ); const firstDivergence = - trace?.verdict === "equivalent" && - deriveProcessComparisonStatus( - PROCESS_COMPARISON_DIMENSIONS.map((dimension) => dimensions[dimension]), - ) === "unchanged" - ? ({ status: "none" } as const) - : observedFirstDivergence.status === "found" - ? observedFirstDivergence - : !sameNormalization || - left.residual_unknowns.length > 0 || - right.residual_unknowns.length > 0 - ? ({ - status: "unknown", - reason: - "Residual unknowns prevent proving that no divergence occurred.", - } as const) - : observedFirstDivergence; + left.truncated || right.truncated + ? ({ + status: "unknown", + reason: + "Incomplete observations prevent locating the first divergence; unaffected dimensions retain their comparisons.", + } as const) + : trace?.verdict === "equivalent" && + deriveProcessComparisonStatus( + PROCESS_COMPARISON_DIMENSIONS.map( + (dimension) => dimensions[dimension], + ), + ) === "unchanged" + ? ({ status: "none" } as const) + : observedFirstDivergence.status === "found" + ? observedFirstDivergence + : !sameNormalization || + left.residual_unknowns.length > 0 || + right.residual_unknowns.length > 0 + ? ({ + status: "unknown", + reason: + "Residual unknowns prevent proving that no divergence occurred.", + } as const) + : observedFirstDivergence; const observedStatus = deriveProcessComparisonStatus( PROCESS_COMPARISON_DIMENSIONS.map((dimension) => dimensions[dimension]), ); diff --git a/src/domain/process/processTraceEvaluation.ts b/src/domain/process/processTraceEvaluation.ts index e4c5e7cb8..3a65b3284 100644 --- a/src/domain/process/processTraceEvaluation.ts +++ b/src/domain/process/processTraceEvaluation.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import type { JsonValue } from "../jsonValue.js"; import type { ProcessCapture } from "./processCapture.js"; +import { processSourceTruncated } from "./processCaptureCoverage.js"; import { processObservationLocationSchema, projectProcessObservation, @@ -414,7 +415,7 @@ export const evaluateProcessTraceSide = ( const sources = new Set(specification.events.map(({ source }) => source)); const relevantScopes = scopesFor(sources); const incomplete = - capture.truncated || + [...sources].some((source) => processSourceTruncated(capture, source)) || capture.residual_unknowns.some(({ scope }) => relevantScopes.has(scope)); if (capture.event_journal.length === 0) return { diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.ts index b8880a630..d8fc9bf5e 100644 --- a/src/process/capture/FilesystemSnapshot.ts +++ b/src/process/capture/FilesystemSnapshot.ts @@ -7,12 +7,14 @@ import type { ProcessScenario, } from "../../domain/process/processCapture.js"; import type { Stats } from "node:fs"; +import type { FilesystemCoverage } from "../../domain/process/processCaptureCoverage.js"; export interface SnapshotResult { readonly files: readonly FileState[]; readonly truncated: boolean; /** Root aliases whose path enumeration was exhausted, regardless of hash coverage. */ readonly completeRoots: readonly string[]; + readonly coverage?: FilesystemCoverage; } const hasSameIdentity = ( @@ -97,6 +99,10 @@ export const snapshotRoots = async ( const completeRoots: string[] = []; let remainingBytes = scenario.limits.file_bytes; let truncated = false; + const enumerationReasons = new Set< + FilesystemCoverage["enumeration_reasons"][number] + >(); + const hashOmissions: FilesystemCoverage["hash_omissions"] = []; const visit = async ( root: string, rootAlias: string, @@ -109,6 +115,10 @@ export const snapshotRoots = async ( depth > scenario.limits.filesystem_depth ) { truncated = true; + if (entries.length >= scenario.limits.files) + enumerationReasons.add("files_limit"); + if (depth > scenario.limits.filesystem_depth) + enumerationReasons.add("depth_limit"); return false; } const stats = await lstatIfPresent(path); @@ -119,6 +129,7 @@ export const snapshotRoots = async ( const afterRead = await lstat(path); if (!hasSameIdentity(stats, afterRead, "symlink")) { truncated = true; + enumerationReasons.add("identity_changed"); return false; } entries.push({ @@ -137,7 +148,18 @@ export const snapshotRoots = async ( ? await hashFile(path, stats, remainingBytes, signal) : null; remainingBytes -= sha256 === null ? 0 : stats.size; - if (sha256 === null) truncated = true; + if (sha256 === null) { + truncated = true; + hashOmissions.push({ + path: `${rootAlias}:${relativePath}`, + size_bytes: stats.size, + remaining_budget_bytes: remainingBytes, + reason: + remainingBytes < stats.size + ? "file_bytes_budget" + : "file_changed_or_short_read", + }); + } entries.push({ path: `${rootAlias}:${relativePath}`, type: "file", @@ -162,6 +184,7 @@ export const snapshotRoots = async ( const afterRead = await lstat(path); if (!hasSameIdentity(stats, afterRead, "directory")) { truncated = true; + enumerationReasons.add("identity_changed"); return false; } let complete = true; @@ -179,5 +202,19 @@ export const snapshotRoots = async ( } if (await visit(root, alias, root, 0)) completeRoots.push(alias); } - return { files: entries, truncated, completeRoots }; + return { + files: entries, + truncated, + completeRoots, + coverage: { + files_limit: scenario.limits.files, + depth_limit: scenario.limits.filesystem_depth, + enumeration_truncated: + completeRoots.length !== scenario.filesystem_observation_paths.length, + enumeration_reasons: [...enumerationReasons], + hash_budget_bytes: scenario.limits.file_bytes, + hashed_bytes: scenario.limits.file_bytes - remainingBytes, + hash_omissions: hashOmissions, + }, + }; }; diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index 86cd88d0c..ca183d3c1 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -59,6 +59,11 @@ import { } from "./ProcessNormalization.js"; import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js"; import { TerminalRenderer } from "./TerminalRenderer.js"; +import { + hasCaptureTruncation, + type TerminalRetention, + type ProcessCaptureTruncationDetails, +} from "../../domain/process/processCaptureCoverage.js"; import { scheduleProcessInterval, type ProcessTimer } from "./ProcessTimer.js"; interface TerminalExitOptions { @@ -84,6 +89,7 @@ interface CaptureResultOptions { readonly before: SnapshotResult; readonly after: SnapshotResult; readonly truncated: boolean; + readonly truncationDetails?: ProcessCaptureTruncationDetails; readonly scenario: ProcessScenario; readonly rootPid: number; readonly samplingPartial: boolean; @@ -279,7 +285,13 @@ export const buildCaptureResult = ( files_before: options.before.files, files_after: options.after.files, filesystem_effects: filesystemEffects, - truncated: options.truncated, + truncated: + options.truncationDetails === undefined + ? options.truncated + : hasCaptureTruncation(options.truncationDetails), + ...(options.truncationDetails === undefined + ? {} + : { truncation_details: options.truncationDetails }), limitations: [ "The executable digest is a prelaunch file sample; matching path metadata immediately after spawn does not prove an atomic operating-system image binding.", "Process trees are sampled and may omit short-lived descendants.", @@ -290,6 +302,11 @@ export const buildCaptureResult = ( "Inherited host environment variables are not recorded and may affect results.", ...(!hasFilesystemObservations ? [filesystemObservationUnknown] : []), ...(hasUnknownFilesystemEffects ? [incompleteFilesystemUnknown] : []), + ...(options.truncationDetails === undefined + ? [] + : captureCoverageUnknowns(options.truncationDetails).map( + ({ reason }) => reason, + )), ...(hasSensitiveScriptedInput ? [sensitiveInputUnknown] : []), ], residual_unknowns: [ @@ -330,10 +347,61 @@ export const buildCaptureResult = ( ...(hasSensitiveScriptedInput ? [{ scope: "interaction" as const, reason: sensitiveInputUnknown }] : []), + ...(options.truncationDetails === undefined + ? [] + : captureCoverageUnknowns(options.truncationDetails).filter( + ({ scope }) => scope !== "terminal", + )), ], }; }; +const captureCoverageUnknowns = ( + details: ProcessCaptureTruncationDetails, +): UnverifiedProcessCapture["residual_unknowns"] => { + const unknowns: Array = + []; + if ( + details.raw_terminal.observed_frames > details.raw_terminal.retained_frames + ) + unknowns.push({ + scope: "terminal", + reason: + "Raw PTY chunks exceeded output_bytes and were omitted; rendered states also lack those inputs. See truncation_details.raw_terminal.", + }); + if ( + details.rendered_terminal.observed_frames > + details.rendered_terminal.retained_frames + ) + unknowns.push({ + scope: "terminal", + reason: + "Cumulative rendered state and visible-line bytes exceeded output_bytes; whole rendered frames were omitted. Raw PTY coverage is reported separately. See truncation_details.rendered_terminal.", + }); + for (const [name, coverage] of [ + ["before", details.filesystem_before], + ["after_settlement", details.filesystem_after], + ] as const) { + if (coverage.enumeration_truncated) + unknowns.push({ + scope: "filesystem", + reason: `Filesystem ${name} path enumeration was incomplete: ${coverage.enumeration_reasons.join(", ")}. See truncation_details.`, + }); + if (coverage.hash_omissions.length > 0) + unknowns.push({ + scope: "filesystem", + reason: `Filesystem ${name} omitted ${String(coverage.hash_omissions.length)} whole-file digests; per-path reasons and remaining file_bytes budget are in truncation_details.`, + }); + } + if (details.process.sampling_partial) + unknowns.push({ + scope: "process", + reason: + "Process sampling stopped with incomplete observations; sample_limit is reported in truncation_details.process.", + }); + return unknowns; +}; + interface ExecutableFileIdentity { readonly dev: bigint; readonly ino: bigint; @@ -840,14 +908,16 @@ export const captureTerminalFrames = (options: { readonly onOutput: () => void; readonly renderer: TerminalRenderer; readonly recordEvent: RecordProcessCaptureEvent; -}): (() => boolean) => { +}): (() => TerminalRetention) => { let outputBytes = 0; - let truncated = false; + let observedBytes = 0; + let observedFrames = 0; options.terminal.onData((data) => { options.onOutput(); const bytes = Buffer.byteLength(data); + observedBytes += bytes; + observedFrames += 1; if (outputBytes + bytes > options.scenario.limits.output_bytes) { - truncated = true; return; } outputBytes += bytes; @@ -871,7 +941,13 @@ export const captureTerminalFrames = (options: { options.renderer.write(data, atMs); options.recordEvent("frames", sequence); }); - return () => truncated; + return () => ({ + budget_bytes: options.scenario.limits.output_bytes, + observed_bytes: observedBytes, + retained_bytes: outputBytes, + observed_frames: observedFrames, + retained_frames: options.frames.length, + }); }; export const resolveProcessResult = ( diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.ts index 672e5209b..f573b32c7 100644 --- a/src/process/capture/ProcessHarness.ts +++ b/src/process/capture/ProcessHarness.ts @@ -1,4 +1,5 @@ import type { IPty } from "@lydell/node-pty"; +import type { TerminalRetention } from "../../domain/process/processCaptureCoverage.js"; import type { InteractionEvent, ProcessCapture, @@ -65,8 +66,8 @@ interface StartedCaptureRuntime { readonly startedAt: Date; readonly executableIdentity: ReturnType; readonly lastOutput: () => number; - readonly framesTruncated: () => boolean; - readonly stopSampler: () => Promise<{ readonly partial: boolean }>; + readonly rawTerminalRetention: () => TerminalRetention; + readonly stopSampler: ReturnType; } const cleanupFailedStartup = async (options: { @@ -200,7 +201,7 @@ const startCaptureRuntime = async ( scenario.executable, selectedExecutable, ); - const framesTruncated = captureTerminalFrames({ + const rawTerminalRetention = captureTerminalFrames({ ...options, terminal, started, @@ -228,7 +229,7 @@ const startCaptureRuntime = async ( startedAt, executableIdentity, lastOutput: () => lastOutput, - framesTruncated, + rawTerminalRetention, stopSampler, }; } catch (cause: unknown) { @@ -365,7 +366,8 @@ const completeCapture = async (options: { state: "available", value: settlement, }; - const samplingPartial = (await runtime.stopSampler()).partial; + const sampling = await runtime.stopSampler(); + const samplingPartial = sampling.partial; await settleProcessCaptureJournal(options.eventJournal); assertNotCancelled(options.signal); let after: Awaited>; @@ -439,7 +441,8 @@ const completeCapture = async (options: { const truncated = options.initiallyTruncated || after.truncated || - runtime.framesTruncated() || + runtime.rawTerminalRetention().observed_frames > + runtime.rawTerminalRetention().retained_frames || checkpoints.some(({ truncated: partial }) => partial) || samplingPartial || runtime.renderer.truncated(); @@ -450,6 +453,17 @@ const completeCapture = async (options: { before: options.before, after, truncated, + ...(options.before.coverage === undefined || after.coverage === undefined + ? {} + : { + truncationDetails: { + raw_terminal: runtime.rawTerminalRetention(), + rendered_terminal: runtime.renderer.retention(), + filesystem_before: options.before.coverage, + filesystem_after: after.coverage, + process: sampling.coverage, + }, + }), scenario, rootPid: runtime.terminal.pid, samplingPartial, diff --git a/src/process/capture/ProcessSampling.ts b/src/process/capture/ProcessSampling.ts index e8ee7cefb..518d91932 100644 --- a/src/process/capture/ProcessSampling.ts +++ b/src/process/capture/ProcessSampling.ts @@ -1,4 +1,5 @@ import { readFile, readdir } from "node:fs/promises"; +import type { ProcessSamplingCoverage } from "../../domain/process/processCaptureCoverage.js"; import type { ProcessSample, RecordProcessCaptureEvent, @@ -455,7 +456,10 @@ export const startProcessSampler = (options: { readonly limit: number; readonly samples: ProcessSample[]; readonly recordEvent?: RecordProcessCaptureEvent; -}): (() => Promise<{ readonly partial: boolean }>) => { +}): (() => Promise<{ + readonly partial: boolean; + readonly coverage: ProcessSamplingCoverage; +}>) => { const { rootPid, runId, started, limit, samples } = options; const identities = new Map(); const lastObservations = new Map(); @@ -463,6 +467,9 @@ export const startProcessSampler = (options: { let stopped = false; let abortCurrent: (() => void) | undefined; let partial = false; + let limitReached = false; + let samplingFailures = 0; + let firstSamplingFailure: string | null = null; let rootInitialized = false; const sample = (): void => { @@ -501,6 +508,7 @@ export const startProcessSampler = (options: { if (lastObservations.get(value.pid) === observation) continue; if (samples.length >= limit) { partial = true; + limitReached = true; continue; } lastObservations.set(value.pid, observation); @@ -510,8 +518,16 @@ export const startProcessSampler = (options: { } }) .catch((cause: unknown) => { - if (!(cause instanceof Error && cause.name === "AbortError")) + if (!(cause instanceof Error && cause.name === "AbortError")) { partial = true; + samplingFailures += 1; + firstSamplingFailure ??= + cause instanceof Error + ? `${cause.name}: ${cause.message}` + : typeof cause === "string" + ? cause + : `Unexpected sampling rejection (${typeof cause})`; + } }) .then(() => { abortCurrent = undefined; @@ -527,6 +543,17 @@ export const startProcessSampler = (options: { clearInterval(timer); abortCurrent?.(); await pending; - return { partial }; + return { + partial, + coverage: { + sample_limit: limit, + sampling_partial: partial, + retained_samples: samples.length, + sample_limit_reached: limitReached, + sampling_failures: samplingFailures, + first_sampling_failure: firstSamplingFailure, + coverage: "sampled", + }, + }; }; }; diff --git a/src/process/capture/TerminalRenderer.ts b/src/process/capture/TerminalRenderer.ts index 0a0492008..e7f1a0fc8 100644 --- a/src/process/capture/TerminalRenderer.ts +++ b/src/process/capture/TerminalRenderer.ts @@ -1,4 +1,5 @@ import { createRequire } from "node:module"; +import type { TerminalRetention } from "../../domain/process/processCaptureCoverage.js"; import type { RecordProcessCaptureEvent, @@ -36,6 +37,8 @@ export class TerminalRenderer { readonly #frames: RenderedTerminalFrame[] = []; #pending: Promise = Promise.resolve(); #capturedBytes = 0; + #observedBytes = 0; + #observedFrames = 0; #truncated = false; constructor(private readonly options: TerminalRendererOptions) { @@ -80,6 +83,17 @@ export class TerminalRenderer { return this.#truncated; } + /** Budget accounting after awaiting queued writes with frames(). */ + retention(): TerminalRetention { + return { + budget_bytes: this.options.maxBytes, + observed_bytes: this.#observedBytes, + retained_bytes: this.#capturedBytes, + observed_frames: this.#observedFrames, + retained_frames: this.#frames.length, + }; + } + /** Release addon and terminal resources after all writes settle. */ async dispose(): Promise { await this.#pending; @@ -107,6 +121,8 @@ export class TerminalRenderer { const bytes = Buffer.byteLength(serializedState) + lines.reduce((total, line) => total + Buffer.byteLength(line), 0); + this.#observedBytes += bytes; + this.#observedFrames += 1; if (this.#capturedBytes + bytes > this.options.maxBytes) { this.#truncated = true; return; diff --git a/tests/acceptance/tools/localToolContracts.test.ts b/tests/acceptance/tools/localToolContracts.test.ts index 7bfc9dfc0..c00678633 100644 --- a/tests/acceptance/tools/localToolContracts.test.ts +++ b/tests/acceptance/tools/localToolContracts.test.ts @@ -207,16 +207,16 @@ captureTest( 60_000, ); -captureTest.each(["unknown", "truncated"] as const)( +captureTest.each(["untruncated", "truncated"] as const)( "captures real process choices and composes a %s self-comparison", - async (status) => { + async (captureState) => { const root = await createTestTempDirectory("rea-local-capture-"); const { call } = await connectLocalToolsMcp(); const captured = await call("capture_process_scenario", { executable: process.execPath, arguments: [ "-e", - `require('node:fs').writeFileSync('state','selected');console.log(process.env['app.setting'],process.env.MY_PASSWORD,process.env['REA_PROCESS_RUN_ID'+String.fromCharCode(10)]);${status === "truncated" ? "console.log('x'.repeat(5000))" : ""}`, + `require('node:fs').writeFileSync('state','selected');console.log(process.env['app.setting'],process.env.MY_PASSWORD,process.env['REA_PROCESS_RUN_ID'+String.fromCharCode(10)]);${captureState === "truncated" ? "console.log('x'.repeat(5000))" : ""}`, ], working_directory: root, environment: { @@ -226,14 +226,14 @@ captureTest.each(["unknown", "truncated"] as const)( "REA_PROCESS_RUN_ID\n": "caller-value", }, filesystem_observation_paths: [root], - limits: status === "truncated" ? { output_bytes: 128 } : {}, + limits: captureState === "truncated" ? { output_bytes: 128 } : {}, }); expect(captured.isError, JSON.stringify(captured)).not.toBe(true); const capture = toolContract("capture_process_scenario").outputSchema.parse( captured.structuredContent, ); const source = capture; - if (status === "unknown") + if (captureState === "untruncated") expect( capture.normalized_result.frames.map((frame) => frame.data).join(""), ).toContain("value ordinary-evidence caller-value"); @@ -266,7 +266,13 @@ captureTest.each(["unknown", "truncated"] as const)( const comparison = toolContract( "compare_process_captures", ).outputSchema.parse(compared.structuredContent); - expect(comparison.normalized_result.status).toBe(status); + expect(comparison.normalized_result).toMatchObject({ + status: "unknown", + terminal: captureState === "truncated" ? "unknown" : "unchanged", + interaction: "unchanged", + exit: "unchanged", + process: "unknown", + }); const verification = await call("verify_reconstruction", { specification: { name: "Self comparison retains uncertainty", @@ -304,7 +310,7 @@ captureTest.each(["unknown", "truncated"] as const)( }); expect(behavior.isError, JSON.stringify(behavior)).not.toBe(true); expect(behavior.structuredContent).toMatchObject({ - normalized_result: { behavior_status: status }, + normalized_result: { behavior_status: "unknown" }, }); const unknowns = await call("list_unknowns", { domain: "process-comparison", diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 468339c86..9006cd41e 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -129,7 +129,7 @@ itWithLinuxCaptureCapability.each([false, true])( state: "quiesced", cleanup_outcome: "not_required", }); - expect(capture.cleanup).toEqual({ + expect(capture.cleanup).toMatchObject({ owned_process_group: "verified", temporary_root: "removed", }); diff --git a/tests/boundary/process/processTruncationAdapters.test.ts b/tests/boundary/process/processTruncationAdapters.test.ts new file mode 100644 index 000000000..7a46e5aab --- /dev/null +++ b/tests/boundary/process/processTruncationAdapters.test.ts @@ -0,0 +1,270 @@ +import { execFile } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, onTestFinished } from "vitest"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + compareProcessCaptures, + parseProcessCapture, + processCaptureSchema, +} from "../../../src/domain/process/processCapture.js"; +import { processSourceTruncated } from "../../../src/domain/process/processCaptureCoverage.js"; +import { compareProcessTraces } from "../../../src/domain/process/processTraceComparison.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { itWithCaptureCapability } from "./processCaptureCapability.js"; + +const exec = promisify(execFile); +const capture = async ( + adapter: "cli" | "mcp", + scenario: Record, +) => { + if (adapter === "cli") { + const path = join( + await createTestTempDirectory("rea-truncation-scenario-"), + "scenario.json", + ); + await writeFile(path, JSON.stringify(scenario)); + const { stdout } = await exec(process.execPath, [ + "scripts/rea.mjs", + "capture-process", + path, + "--format", + "json", + ]); + return parseProcessCapture( + parseEvidence(JSON.parse(stdout)).normalized_result, + ); + } + const session = createTestBinarySession(() => { + throw new Error("Capture must not launch a binary provider"); + }); + const server = createServer(session, session); + const client = new Client({ name: "truncation-coverage", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const result = await client.callTool({ + name: "capture_process_scenario", + arguments: scenario, + }); + expect(result.isError, JSON.stringify(result)).not.toBe(true); + await client.ping(); + return parseProcessCapture( + parseEvidence(result.structuredContent).normalized_result, + ); +}; + +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "localizes sparse rendered-state exhaustion through %s", + async (adapter) => { + const root = await createTestTempDirectory("rea-rendered-budget-"); + const result = await capture(adapter, { + executable: process.execPath, + working_directory: root, + arguments: [ + "-e", + "let n=0;const timer=setInterval(()=>{console.log('board '+String(n).padStart(2,'0')+' completed; score '+String(n*10));if(++n===65)clearInterval(timer)},20)", + ], + normalization: { + ports: false, + pids: false, + paths: false, + time_bucket_ms: 1, + }, + limits: { output_bytes: 30_000 }, + }); + const details = result.truncation_details; + if (details === undefined) throw new Error("Expected producer coverage"); + expect(result.frames.map(({ data }) => data).join("")).toContain( + "board 64 completed; score 640", + ); + expect(details.raw_terminal.observed_bytes).toBe(2004); + expect(details.raw_terminal.retained_bytes).toBe( + details.raw_terminal.observed_bytes, + ); + expect(details.raw_terminal.observed_frames).toBe(result.frames.length); + expect(details.rendered_terminal.observed_frames).toBeGreaterThan( + result.rendered_frames.length, + ); + expect(details.rendered_terminal.retained_bytes).toBeLessThanOrEqual( + 30_000, + ); + expect(details.process.coverage).toBe("sampled"); + expect(processSourceTruncated(result, "terminal_raw")).toBe(false); + expect(processSourceTruncated(result, "terminal_rendered")).toBe(true); + const events = result.frames.map(({ sequence, at_ms, data }) => ({ + id: `chunk_${String(sequence)}`, + source: "terminal_raw" as const, + exact: { sequence, at_ms, data }, + cardinality: { kind: "required" as const }, + })); + const specification = { + events, + language: { + kind: "finite_traces" as const, + variants: [{ id: "observed", trace: events.map(({ id }) => id) }], + }, + }; + expect(compareProcessTraces(result, result, specification).verdict).toBe( + "equivalent", + ); + const legacy = parseProcessCapture({ + ...result, + truncation_details: undefined, + }); + expect(compareProcessTraces(legacy, legacy, specification).verdict).toBe( + "unknown", + ); + expect(compareProcessCaptures(result, result)).toMatchObject({ + terminal: "unknown", + interaction: "unchanged", + exit: "unchanged", + }); + expect( + processCaptureSchema.safeParse({ ...result, truncated: false }).success, + ).toBe(false); + }, + 20_000, +); + +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "distinguishes raw chunk omission and filesystem exhaustion through %s", + async (adapter) => { + const root = await createTestTempDirectory("rea-raw-files-budget-"); + await writeFile(join(root, "a.txt"), "abc"); + await writeFile(join(root, "b.txt"), "12345678"); + const result = await capture(adapter, { + executable: process.execPath, + working_directory: root, + arguments: [ + "-e", + "process.stdout.write('x'.repeat(1000));require('node:fs').writeFileSync('c.txt','new');setTimeout(()=>{},100)", + ], + filesystem_observation_paths: [root], + limits: { output_bytes: 50, files: 3, file_bytes: 4 }, + }); + const details = result.truncation_details; + if (details === undefined) throw new Error("Expected producer coverage"); + expect(details.raw_terminal.observed_bytes).toBe(1000); + expect(details.raw_terminal.retained_bytes).toBeLessThan( + details.raw_terminal.observed_bytes, + ); + expect(details.filesystem_before).toMatchObject({ + enumeration_truncated: false, + hashed_bytes: 3, + hash_omissions: [ + { + path: "root_0:b.txt", + size_bytes: 8, + remaining_budget_bytes: 1, + reason: "file_bytes_budget", + }, + ], + }); + expect(details.filesystem_after.enumeration_reasons).toContain( + "files_limit", + ); + expect( + result.files_after.find(({ path }) => path === "root_0:b.txt")?.sha256, + ).toBeNull(); + expect(processSourceTruncated(result, "terminal_rendered")).toBe(true); + expect(compareProcessCaptures(result, result)).toMatchObject({ + terminal: "unknown", + filesystem: "unknown", + interaction: "unchanged", + exit: "unchanged", + }); + }, + 20_000, +); + +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "keeps terminal facts comparable when only file digests are omitted through %s", + async (adapter) => { + const root = await createTestTempDirectory("rea-hash-budget-"); + await writeFile(join(root, "a.txt"), "abc"); + await writeFile(join(root, "b.txt"), "12345678"); + await writeFile(join(root, "c.txt"), "z"); + const result = await capture(adapter, { + executable: process.execPath, + working_directory: root, + arguments: ["-e", "console.log('done');setTimeout(()=>{},100)"], + filesystem_observation_paths: [root], + limits: { file_bytes: 4 }, + }); + expect(result.truncation_details?.filesystem_after).toMatchObject({ + enumeration_truncated: false, + hashed_bytes: 4, + hash_omissions: [ + { + path: "root_0:b.txt", + reason: "file_bytes_budget", + remaining_budget_bytes: 1, + }, + ], + }); + expect( + result.files_after.find(({ path }) => path === "root_0:c.txt")?.sha256, + ).toMatch(/^[a-f0-9]{64}$/u); + expect(compareProcessCaptures(result, result)).toMatchObject({ + terminal: "unchanged", + filesystem: "unknown", + interaction: "unchanged", + exit: "unchanged", + }); + const differentExit = parseProcessCapture({ + ...result, + exit: { ...result.exit, code: 1 }, + }); + expect(compareProcessCaptures(result, differentExit)).toMatchObject({ + status: "changed", + terminal: "unchanged", + filesystem: "unknown", + exit: "changed", + first_divergence: { status: "unknown" }, + }); + }, + 20_000, +); + +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "reports process sample-limit exhaustion without erasing other dimensions through %s", + async (adapter) => { + const root = await createTestTempDirectory("rea-process-budget-"); + const result = await capture(adapter, { + executable: process.execPath, + working_directory: root, + arguments: [ + "-e", + "setTimeout(()=>require('node:child_process').spawn(process.execPath,['-e','setTimeout(()=>{},800)'],{stdio:'ignore'}),100);console.log('ready')", + ], + limits: { processes: 1 }, + }); + expect(result.truncation_details?.process).toMatchObject({ + sample_limit: 1, + sampling_partial: true, + retained_samples: 1, + sample_limit_reached: true, + sampling_failures: 0, + first_sampling_failure: null, + coverage: "sampled", + }); + expect(compareProcessCaptures(result, result)).toMatchObject({ + terminal: "unchanged", + interaction: "unchanged", + exit: "unchanged", + process: "unknown", + }); + }, + 20_000, +);