diff --git a/docs/process-capture.md b/docs/process-capture.md index b000994c0..316d36c99 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -143,8 +143,10 @@ Every capture requires `truncation_details`, with separate accounting for: - `filesystem_before` and `filesystem_after`: file-count/depth limits, enumeration failures, whole-file hash budget and bytes successfully hashed. Each retained regular file without a digest has an aliased path, size, - remaining budget and reason: `file_bytes_budget` or - `file_changed_or_short_read`. A file too large for the remaining budget is + remaining budget and reason: `file_bytes_budget`, + `file_changed_or_short_read`, or `file_unavailable`. Its `system_code` is + null unless an OS file operation failed, in which case it preserves the + reported error code. A file too large for the remaining budget is skipped; a later smaller file can still be hashed. Hash omissions do not imply incomplete path enumeration. - `process`: sampling limit and whether sampling ended partially. Coverage diff --git a/src/domain/process/processCaptureCoverage.ts b/src/domain/process/processCaptureCoverage.ts index a93972912..72bbb9fb1 100644 --- a/src/domain/process/processCaptureCoverage.ts +++ b/src/domain/process/processCaptureCoverage.ts @@ -25,7 +25,12 @@ export const filesystemCoverageSchema = z.strictObject({ path: z.string(), size_bytes: z.number().int().nonnegative(), remaining_budget_bytes: z.number().int().nonnegative(), - reason: z.enum(["file_bytes_budget", "file_changed_or_short_read"]), + reason: z.enum([ + "file_bytes_budget", + "file_changed_or_short_read", + "file_unavailable", + ]), + system_code: z.string().nullable(), }), ), }); diff --git a/src/domain/process/processCaptureValidation.ts b/src/domain/process/processCaptureValidation.ts index fe5479689..a14cd129b 100644 --- a/src/domain/process/processCaptureValidation.ts +++ b/src/domain/process/processCaptureValidation.ts @@ -200,7 +200,10 @@ const validateCoverage = ( coverage.hash_omissions.every( (omission) => unhashedFiles.get(omission.path)?.size === omission.size_bytes && - omission.remaining_budget_bytes <= coverage.hash_budget_bytes, + omission.remaining_budget_bytes <= coverage.hash_budget_bytes && + (omission.reason === "file_unavailable" + ? omission.system_code !== null + : omission.system_code === null), ), `truncation_details.${name}.hash_omissions`, "every retained regular file without a digest must have one matching omission reason"); require(coverage.hashed_bytes === files diff --git a/src/process/capture/FilesystemCoverage.test.ts b/src/process/capture/FilesystemCoverage.test.ts index ee8de9fac..36ebfc6de 100644 --- a/src/process/capture/FilesystemCoverage.test.ts +++ b/src/process/capture/FilesystemCoverage.test.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; import { parseProcessScenario } from "../../domain/process/processScenario.js"; @@ -81,6 +81,51 @@ it("distinguishes unavailable content hashes from completed path enumeration", a ).toMatchObject({ status: "deleted", before: { sha256: null }, after: null }); }); +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "retains unreadable-file and sibling observations in a complete snapshot", + async () => { + const root = await createTestTempDirectory("rea-fs-unreadable-file-"); + const unreadable = join(root, "unreadable.txt"); + const readable = join(root, "readable.txt"); + await writeFile(unreadable, "permission denied"); + await writeFile(readable, "still observed"); + await chmod(unreadable, 0); + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + }); + + try { + const snapshot = await snapshotRoots(scenario); + expect(snapshot.files.map(({ path }) => path)).toEqual([ + "root_0:.", + "root_0:readable.txt", + "root_0:unreadable.txt", + ]); + expect( + snapshot.files.find(({ path }) => path === "root_0:readable.txt") + ?.sha256, + ).not.toBeNull(); + expect( + snapshot.files.find(({ path }) => path === "root_0:unreadable.txt"), + ).toMatchObject({ type: "file", size: 17, sha256: null }); + expect(snapshot.coverage.hash_omissions).toContainEqual({ + path: "root_0:unreadable.txt", + size_bytes: 17, + remaining_budget_bytes: + scenario.limits.file_bytes - Buffer.byteLength("still observed"), + reason: "file_unavailable", + system_code: "EACCES", + }); + expect(snapshot.completeRoots).toEqual(["root_0"]); + } finally { + await chmod(unreadable, 0o600); + await rm(root, { recursive: true, force: true }); + } + }, +); + it("preserves creation when the selected root was observed missing before capture", async () => { const root = await createTestTempDirectory("rea-fs-missing-root-"); const file = join(root, "created.txt"); diff --git a/src/process/capture/FilesystemSnapshot.test.ts b/src/process/capture/FilesystemSnapshot.test.ts index b7e141018..16799b973 100644 --- a/src/process/capture/FilesystemSnapshot.test.ts +++ b/src/process/capture/FilesystemSnapshot.test.ts @@ -44,7 +44,11 @@ it("does not hash a replacement file using the earlier path metadata", async () await rm(path); await writeFile(path, "replacement with a different identity and size\n"); - await expect(hashFile(path, expected, 1_000)).resolves.toBeNull(); + await expect(hashFile(path, expected, 1_000)).resolves.toEqual({ + state: "omitted", + reason: "file_changed_or_short_read", + system_code: null, + }); } finally { await rm(directory, { recursive: true, force: true }); } @@ -76,7 +80,29 @@ it("rejects a same-size rewrite even when its modification time is restored", as }); expect(rewritten.ctimeMs).not.toBe(expected.ctimeMs); - await expect(hashFile(path, expected, 1_000)).resolves.toBeNull(); + await expect(hashFile(path, expected, 1_000)).resolves.toMatchObject({ + state: "omitted", + reason: "file_changed_or_short_read", + system_code: null, + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +it("reports a file removed between metadata observation and open", async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-snapshot-open-race-")); + const path = join(directory, "observed-file"); + try { + await writeFile(path, "captured file\n"); + const expected = await lstat(path); + await rm(path); + + await expect(hashFile(path, expected, 1_000)).resolves.toEqual({ + state: "omitted", + reason: "file_unavailable", + system_code: "ENOENT", + }); } finally { await rm(directory, { recursive: true, force: true }); } @@ -116,9 +142,12 @@ it.skipIf(process.platform === "win32")( // hashFile reaches `open` synchronously before its first suspension, so // this abort is observed after the descriptor is acquired and is handled // by the `finally` close path. - controller.abort(); + const reason = Object.assign(new Error("cancelled while opening"), { + code: "EACCES", + }); + controller.abort(reason); - await expect(hashing).rejects.toMatchObject({ name: "AbortError" }); + await expect(hashing).rejects.toBe(reason); expect(await countOpenDescriptorsFor(path)).toBe(0); } finally { await rm(directory, { recursive: true, force: true }); diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.ts index 281151afc..17dd91174 100644 --- a/src/process/capture/FilesystemSnapshot.ts +++ b/src/process/capture/FilesystemSnapshot.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { constants as fsConstants } from "node:fs"; import { lstat, open, readdir, readlink } from "node:fs/promises"; import { dirname, join, relative, resolve } from "node:path"; +import type { FileHandle } from "node:fs/promises"; import type { ProcessScenario } from "../../domain/process/processScenario.js"; import type { FilesystemCoverage } from "../../domain/process/processCaptureCoverage.js"; import type { Stats } from "node:fs"; @@ -29,6 +30,23 @@ const hasSameFileState = (before: Stats, after: Stats): boolean => before.mtimeMs === after.mtimeMs && before.ctimeMs === after.ctimeMs; +const fileUnavailableSystemCode = (cause: unknown): string | undefined => { + if (!(cause instanceof Error) || !("code" in cause)) return undefined; + const code = cause.code; + return typeof code === "string" && + [ + "EACCES", + "EISDIR", + "ELOOP", + "ENOTDIR", + "ENOENT", + "EPERM", + "ESTALE", + ].includes(code) + ? code + : undefined; +}; + const lstatIfPresent = async (path: string): Promise => { try { return await lstat(path); @@ -39,26 +57,51 @@ const lstatIfPresent = async (path: string): Promise => { } }; +/** Reports a complete digest or an explicit per-file reason hashing was omitted. */ +export type FileHashOutcome = + | { readonly state: "hashed"; readonly sha256: string } + | { + readonly state: "omitted"; + readonly reason: "file_changed_or_short_read" | "file_unavailable"; + readonly system_code: string | null; + }; + /** Hash a file only when its opened descriptor still matches the captured path state. */ export const hashFile = async ( path: string, expected: Stats, maxBytes: number, signal?: AbortSignal, -): Promise => { +): Promise => { signal?.throwIfAborted(); - const handle = await open( - path, - fsConstants.O_RDONLY | - (fsConstants.O_NOFOLLOW ?? 0) | - (fsConstants.O_NONBLOCK ?? 0), - ); + let handle: FileHandle; + try { + handle = await open( + path, + fsConstants.O_RDONLY | + (fsConstants.O_NOFOLLOW ?? 0) | + (fsConstants.O_NONBLOCK ?? 0), + ); + } catch (cause: unknown) { + signal?.throwIfAborted(); + const systemCode = fileUnavailableSystemCode(cause); + if (systemCode === undefined) throw cause; + return { + state: "omitted", + reason: "file_unavailable", + system_code: systemCode, + }; + } try { signal?.throwIfAborted(); const stats = await handle.stat(); signal?.throwIfAborted(); if (!hasSameFileState(expected, stats) || stats.size > maxBytes) - return null; + return { + state: "omitted", + reason: "file_changed_or_short_read", + system_code: null, + }; const hash = createHash("sha256"); const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes)); let position = 0; @@ -70,14 +113,33 @@ export const hashFile = async ( Math.min(buffer.length, stats.size - position), position, ); - if (bytesRead === 0) return null; + if (bytesRead === 0) + return { + state: "omitted", + reason: "file_changed_or_short_read", + system_code: null, + }; hash.update(buffer.subarray(0, bytesRead)); position += bytesRead; } const after = await handle.stat(); signal?.throwIfAborted(); - if (!hasSameFileState(stats, after)) return null; - return hash.digest("hex"); + if (!hasSameFileState(stats, after)) + return { + state: "omitted", + reason: "file_changed_or_short_read", + system_code: null, + }; + return { state: "hashed", sha256: hash.digest("hex") }; + } catch (cause: unknown) { + signal?.throwIfAborted(); + const systemCode = fileUnavailableSystemCode(cause); + if (systemCode === undefined) throw cause; + return { + state: "omitted", + reason: "file_unavailable", + system_code: systemCode, + }; } finally { await handle.close(); } @@ -136,22 +198,34 @@ export const snapshotRoots = async ( return true; } if (stats.isFile()) { - const sha256 = - remainingBytes >= stats.size - ? await hashFile(path, stats, remainingBytes, signal) - : null; - remainingBytes -= sha256 === null ? 0 : stats.size; - if (sha256 === null) { - truncated = true; - hashOmissions.push({ + let sha256: string | null = null; + let omission: FilesystemCoverage["hash_omissions"][number] | undefined; + if (remainingBytes < stats.size) { + omission = { path: `${rootAlias}:${relativePath}`, size_bytes: stats.size, remaining_budget_bytes: remainingBytes, - reason: - remainingBytes < stats.size - ? "file_bytes_budget" - : "file_changed_or_short_read", - }); + reason: "file_bytes_budget", + system_code: null, + }; + } else { + const outcome = await hashFile(path, stats, remainingBytes, signal); + if (outcome.state === "hashed") { + sha256 = outcome.sha256; + remainingBytes -= stats.size; + } else { + omission = { + path: `${rootAlias}:${relativePath}`, + size_bytes: stats.size, + remaining_budget_bytes: remainingBytes, + reason: outcome.reason, + system_code: outcome.system_code, + }; + } + } + if (omission !== undefined) { + truncated = true; + hashOmissions.push(omission); } entries.push({ path: `${rootAlias}:${relativePath}`, diff --git a/tests/boundary/process/processTruncationAdapters.test.ts b/tests/boundary/process/processTruncationAdapters.test.ts index 34c1ccb3a..20d41e639 100644 --- a/tests/boundary/process/processTruncationAdapters.test.ts +++ b/tests/boundary/process/processTruncationAdapters.test.ts @@ -187,6 +187,7 @@ itWithCaptureCapability.each(["cli", "mcp"] as const)( size_bytes: 8, remaining_budget_bytes: 1, reason: "file_bytes_budget", + system_code: null, }, ], }); @@ -229,6 +230,7 @@ itWithCaptureCapability.each(["cli", "mcp"] as const)( path: "root_0:b.txt", reason: "file_bytes_budget", remaining_budget_bytes: 1, + system_code: null, }, ], }); diff --git a/tests/fixtures/filesystemSnapshotFifoProbe.mjs b/tests/fixtures/filesystemSnapshotFifoProbe.mjs index de3e466b3..984d6e9f7 100644 --- a/tests/fixtures/filesystemSnapshotFifoProbe.mjs +++ b/tests/fixtures/filesystemSnapshotFifoProbe.mjs @@ -18,6 +18,13 @@ execFileSync("mkfifo", [path]); // This invokes the production lstat-to-open hashing seam with a known captured // regular-file identity. The owner test bounds this child so an O_RDONLY FIFO // regression cannot hang Vitest. -const digest = await hashFile(path, expected, 1_000); -if (digest !== null) throw new Error("Expected the FIFO to be rejected"); +const outcome = await hashFile(path, expected, 1_000); +if ( + outcome.state !== "omitted" || + outcome.reason !== "file_changed_or_short_read" || + outcome.system_code !== null +) + throw new Error( + `Expected the FIFO to be omitted as a changed file: ${JSON.stringify(outcome)}`, + ); process.stdout.write("rejected\n");