diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 881b244d6..3a996e5f9 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -113,11 +113,21 @@ smaller view, for example `--format json --filter-output evidence_id,normalized_result.statistics`. Streaming output does not bound the memory needed to construct the analysis graph itself. -CLI workflows still parse each JSON input as one value. A valid input exceeding -the Node runtime's maximum string length returns `resource_constraint` with -`input_reason: "too-large"` and the runtime limit. Produce smaller independent -analysis inputs before retrying; streaming output does not imply streaming -input parsing. +CLI workflows parse larger JSON input files incrementally from one verified +regular-file handle, without constructing a document-sized string. Smaller +files retain native JSON parsing. Both paths require strict JSON and valid +UTF-8, and return one complete value; the assembled object still needs memory. +An individual JSON string exceeding the native length limit or the available +heap headroom for assembly returns +`resource_constraint` with `input_reason: "too-large"`. Supply a smaller valid +value; for Evidence workflows, re-analyze a smaller selection of the original +target and use its complete Evidence. Splitting JSON text or trimming Evidence +fields does not produce a valid workflow input. + +Evidence bundle and analysis snapshot file readers still decode whole +documents. Their runtime string-limit failures report a resource constraint +with the selected path, observed bytes and UTF-16 limit, rather than malformed +JSON. MCP prepares the complete repeated response incrementally against the pinned SDK's 10 MiB stdio receive-buffer budget. Oversized results return an actionable diff --git a/package-lock.json b/package-lock.json index 9e674e202..212d6b8eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "plist": "5.0.0", "semver": "7.8.5", "smol-toml": "1.9.0", + "stream-json": "3.7.0", "write-file-atomic": "7.0.0", "ws": "8.21.0", "zod": "4.4.3" @@ -7070,6 +7071,27 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-chain": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-4.2.6.tgz", + "integrity": "sha512-1zeJ8CrtJfmiba26ui8jXkq/xLRFvhzkdH02D5QLO9Cnovgeb28IJxg88DdraWnjnkbOol/++uIAybJbJhk7ig==", + "license": "BSD-3-Clause", + "funding": { + "url": "https://github.com/sponsors/uhop" + } + }, + "node_modules/stream-json": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-3.7.0.tgz", + "integrity": "sha512-rCSBdcBP/bPk6T8QFcxAj1MSzAuc5i49cYW6IE7sYObNEPccBJIUiL6fU9c9BWt40aJK0siVynLX7lWaW8alXw==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^4.2.5" + }, + "funding": { + "url": "https://github.com/sponsors/uhop" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/package.json b/package.json index 0ec211497..a3d3bb0e1 100644 --- a/package.json +++ b/package.json @@ -263,6 +263,7 @@ "plist": "5.0.0", "semver": "7.8.5", "smol-toml": "1.9.0", + "stream-json": "3.7.0", "write-file-atomic": "7.0.0", "ws": "8.21.0", "zod": "4.4.3", diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts index 8132255da..0ccca760a 100644 --- a/src/application/JsonFiles.ts +++ b/src/application/JsonFiles.ts @@ -3,7 +3,6 @@ import { lstat, mkdtemp, open, - readFile, realpath, rename, rm, @@ -18,6 +17,8 @@ import { } from "../domain/analysisErrorCore.js"; import { err, ok, type Result } from "../domain/result.js"; import { parseUtf8Json } from "./Utf8JsonInput.js"; +import { readRegularFile } from "./RegularFileRead.js"; +import { NonRegularFileReadError } from "../filesystem/RegularFile.js"; /** Request control and its owning operation for an interruptible atomic write. */ export interface TextWriteCancellation { @@ -36,13 +37,7 @@ export const readJsonFile = async ( > => { const requestedPath = resolve(path); try { - const canonicalPath = await realpath(requestedPath); - const stats = await lstat(canonicalPath); - if (!stats.isFile()) - return err( - new EvidenceFileError("read", "not-file", { path: requestedPath }), - ); - const encoded = await readFile(canonicalPath); + const encoded = await readRegularFile(requestedPath); const decoded = parseUtf8Json(encoded, "read_evidence_file", requestedPath); if (!decoded.ok) { return err( @@ -56,10 +51,16 @@ export const readJsonFile = async ( } catch (cause: unknown) { if (cause instanceof AnalysisResourceConstraintError) return err(cause); return err( - new EvidenceFileError("read", missingOrIo(cause), { - cause, - path: requestedPath, - }), + new EvidenceFileError( + "read", + cause instanceof NonRegularFileReadError + ? "not-file" + : missingOrIo(cause), + { + cause, + path: requestedPath, + }, + ), ); } }; diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index 486e8c54d..df87b7c5a 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -1,16 +1,29 @@ +import type { Stats } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; + import { openRegularFile } from "../filesystem/RegularFile.js"; -/** Admit a regular file without waiting for a pipe, then read its verified handle. */ -export const readRegularFile = async ( +/** Admit one regular-file handle, retain it through the consumer, and always close it. */ +export const withRegularFile = async ( path: string, + read: (handle: FileHandle, stats: Stats) => Promise, signal?: AbortSignal, -): Promise => { +): Promise => { const handle = await openRegularFile(path, { symlinks: "follow", signal }); try { - const bytes = await handle.readFile({ signal }); + const stats = await handle.stat(); signal?.throwIfAborted(); - return bytes; + const value = await read(handle, stats); + signal?.throwIfAborted(); + return value; } finally { await handle.close(); } }; + +/** Admit a regular file without waiting for a pipe, then read its verified handle. */ +export const readRegularFile = ( + path: string, + signal?: AbortSignal, +): Promise => + withRegularFile(path, (handle) => handle.readFile({ signal }), signal); diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.ts index 4b9ecf10b..86204e3c9 100644 --- a/src/application/Utf8JsonInput.ts +++ b/src/application/Utf8JsonInput.ts @@ -3,6 +3,10 @@ import { constants } from "node:buffer"; import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js"; +/** Recovery that preserves complete, authenticated Evidence workflow inputs. */ +export const JSON_INPUT_RESOURCE_REMEDIATION = + "Provide a smaller valid JSON value. For Evidence workflows, re-analyze a smaller selection of the original target and use its complete Evidence; splitting JSON text or trimming Evidence fields does not produce valid workflow input."; + /** Decode JSON bytes without confusing a runtime string limit with malformed input. */ export const parseUtf8Json = ( bytes: Uint8Array, @@ -29,8 +33,7 @@ export const parseUtf8Json = ( }, { ...(cause === undefined ? {} : { cause }), - remediationAction: - "Provide a smaller JSON document. For Evidence-based workflows, re-analyze a smaller selection of the original target and use its Evidence; splitting JSON text alone does not produce a valid workflow input.", + remediationAction: JSON_INPUT_RESOURCE_REMEDIATION, }, ); let text: string; diff --git a/src/cliJsonFile.ts b/src/cliJsonFile.ts new file mode 100644 index 000000000..ea75f7c29 --- /dev/null +++ b/src/cliJsonFile.ts @@ -0,0 +1,261 @@ +import { constants } from "node:buffer"; +import type { FileHandle } from "node:fs/promises"; +import { pipeline } from "node:stream/promises"; +import { getHeapStatistics } from "node:v8"; + +import parser from "stream-json/parser.js"; +import streamValues from "stream-json/streamers/stream-values.js"; + +import { withRegularFile } from "./application/RegularFileRead.js"; +import { + JSON_INPUT_RESOURCE_REMEDIATION, + parseUtf8Json, +} from "./application/Utf8JsonInput.js"; +import { + AnalysisInputError, + AnalysisResourceConstraintError, +} from "./domain/analysisErrorCore.js"; +import { err, ok, type Result } from "./domain/result.js"; + +type JsonFileFailure = AnalysisInputError | AnalysisResourceConstraintError; +const READ_CHUNK_BYTES = 64 * 1024; +const PACKED_STRING_PART_CODE_UNITS = 64 * 1024; +// Retain native-parser speed for modest inputs while bounding its extra copies. +// This selects an implementation; larger files remain accepted through streaming. +const NATIVE_PARSE_READ_BUDGET = 8 * 1024 * 1024; + +/** Parse strict UTF-8 JSON without requiring a whole-document string. */ +export const readCliJsonFile = ( + path: string, + operation: string, + signal?: AbortSignal, +): Promise> => + withRegularFile( + path, + async (handle, stats) => { + let found = false; + let value: unknown; + try { + const prefix = + stats.size <= NATIVE_PARSE_READ_BUDGET + ? await readPrefix(handle, stats.size + 1, signal) + : undefined; + if (prefix?.complete === true) { + const parsed = parseUtf8Json( + prefix.bytes, + operation, + path, + "cli-json-input", + ); + return parsed.ok + ? ok(parsed.value) + : err(invalidJson(operation, parsed.error, parsed.cause)); + } + await pipeline( + decodedChunks(handle, signal, prefix?.bytes), + parser.asStream({ + jsonStreaming: false, + streamValues: false, + packKeys: false, + packStrings: false, + }), + packJsonStrings(operation, stats.size), + streamValues.asStream(), + async (rows: AsyncIterable) => { + for await (const row of rows) { + if ( + found || + typeof row !== "object" || + row === null || + !("key" in row) || + row.key !== 0 || + !("value" in row) + ) + throw new Error("Unexpected JSON parser result"); + found = true; + value = row.value; + } + }, + signal === undefined ? {} : { signal }, + ); + return found + ? ok(value) + : err(invalidJson(operation, "JSON document is empty")); + } catch (cause: unknown) { + if (cause instanceof AnalysisResourceConstraintError) return err(cause); + if ( + cause instanceof TypeError && + "code" in cause && + cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA" + ) + return err( + invalidJson(operation, "JSON input is not valid UTF-8", cause), + ); + if ( + cause instanceof SyntaxError || + (cause instanceof Error && cause.message.startsWith("Parser ")) + ) + return err(invalidJson(operation, cause.message, cause)); + if ( + cause instanceof Error && + ((cause instanceof RangeError && + cause.message === "Invalid string length") || + ("code" in cause && cause.code === "ERR_STRING_TOO_LONG")) + ) + return err( + new AnalysisResourceConstraintError( + operation, + "memory", + "An individual JSON string exceeds the runtime string limit", + { + input_file_bytes: stats.size, + max_string_code_units: constants.MAX_STRING_LENGTH, + }, + { + cause, + remediationAction: JSON_INPUT_RESOURCE_REMEDIATION, + }, + ), + ); + throw cause; + } + }, + signal, + ); + +const packJsonStrings = (operation: string, inputFileBytes: number) => + async function* (tokens: AsyncIterable): AsyncGenerator { + let kind: "keyValue" | "stringValue" | undefined; + let length = 0; + let value = ""; + let parts: string[] = []; + let partLength = 0; + for await (const token of tokens) { + if (typeof token !== "object" || token === null || !("name" in token)) + throw new Error("Unexpected JSON parser token"); + switch (token.name) { + case "startKey": + case "startString": + kind = token.name === "startKey" ? "keyValue" : "stringValue"; + break; + case "stringChunk": + if ( + kind === undefined || + !("value" in token) || + typeof token.value !== "string" + ) + throw new Error("Unexpected JSON string fragment"); + length += token.value.length; + if (length > constants.MAX_STRING_LENGTH) + throw new RangeError("Invalid string length"); + parts.push(token.value); + partLength += token.value.length; + if (partLength >= PACKED_STRING_PART_CODE_UNITS) { + // The tokenizer emits short fragments. Coalesce them before retaining + // a string so millions of substring/rope nodes cannot exhaust the heap + // before the native string-length constraint can be reported. + requireStringAssemblyHeadroom(operation, inputFileBytes, length); + value += parts.join(""); + parts = []; + partLength = 0; + } + break; + case "endKey": + case "endString": + if (kind === undefined) throw new Error("Unexpected JSON string end"); + requireStringAssemblyHeadroom(operation, inputFileBytes, length); + yield { name: kind, value: value + parts.join("") }; + kind = undefined; + length = 0; + value = ""; + parts = []; + partLength = 0; + break; + default: + yield token; + } + } + }; + +const requireStringAssemblyHeadroom = ( + operation: string, + inputFileBytes: number, + stringCodeUnits: number, +): void => { + if (stringCodeUnits < PACKED_STRING_PART_CODE_UNITS) return; + const heap = getHeapStatistics(); + // Concatenation or key interning may flatten the retained rope. Reserve its + // UTF-16 representation and the next coalesced part before that allocation. + const requiredBytes = 2 * (stringCodeUnits + PACKED_STRING_PART_CODE_UNITS); + if (requiredBytes > heap.total_available_size) + throw new AnalysisResourceConstraintError( + operation, + "memory", + "Insufficient heap headroom to assemble a complete JSON string", + { + input_file_bytes: inputFileBytes, + string_code_units: stringCodeUnits, + string_assembly_headroom_bytes: requiredBytes, + available_heap_bytes: heap.total_available_size, + heap_size_limit_bytes: heap.heap_size_limit, + }, + { + remediationAction: JSON_INPUT_RESOURCE_REMEDIATION, + }, + ); +}; + +async function* decodedChunks( + handle: FileHandle, + signal: AbortSignal | undefined, + prefix?: Uint8Array, +): AsyncGenerator { + const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + if (prefix !== undefined) + for (let offset = 0; offset < prefix.length; offset += READ_CHUNK_BYTES) { + signal?.throwIfAborted(); + yield decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), { + stream: true, + }); + } + const bytes = Buffer.allocUnsafe(READ_CHUNK_BYTES); + while (true) { + signal?.throwIfAborted(); + const read = await handle.read(bytes, 0, bytes.length, null); + signal?.throwIfAborted(); + if (read.bytesRead === 0) break; + yield decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }); + } + yield decoder.decode(); +} + +const readPrefix = async ( + handle: FileHandle, + size: number, + signal: AbortSignal | undefined, +): Promise<{ readonly bytes: Buffer; readonly complete: boolean }> => { + const bytes = Buffer.allocUnsafe(size); + let offset = 0; + while (offset < bytes.length) { + signal?.throwIfAborted(); + const read = await handle.read(bytes, offset, bytes.length - offset, null); + signal?.throwIfAborted(); + if (read.bytesRead === 0) + return { bytes: bytes.subarray(0, offset), complete: true }; + offset += read.bytesRead; + } + // The file grew after admission. Continue from this same handle, preserving + // the prefix already read rather than reopening or dropping its bytes. + return { bytes, complete: false }; +}; + +const invalidJson = ( + operation: string, + message: string, + cause?: unknown, +): AnalysisInputError => + new AnalysisInputError( + operation, + cause === undefined ? undefined : { cause }, + [{ path: [], reason: "invalid_format", expected: "JSON", message }], + ); diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.ts index ebab099fe..47b1431e4 100644 --- a/src/cliJsonInput.ts +++ b/src/cliJsonInput.ts @@ -1,13 +1,10 @@ -import { constants as bufferConstants } from "node:buffer"; import { resolve } from "node:path"; -import { readRegularFile } from "./application/RegularFileRead.js"; +import { readCliJsonFile } from "./cliJsonFile.js"; import { NonRegularFileReadError } from "./filesystem/RegularFile.js"; -import { parseUtf8Json } from "./application/Utf8JsonInput.js"; import { AnalysisAccessDeniedError, AnalysisInputError, - AnalysisResourceConstraintError, } from "./domain/analysisErrorCore.js"; import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; import type { JsonValue } from "./domain/jsonValue.js"; @@ -23,31 +20,23 @@ export const parseCliJsonInput = async ( > => { const inline = safeParseJson(value); if (inline.ok) return { ok: true, value: inline.value }; - if (isStringLengthLimit(inline.cause)) - return jsonTooLargeError(undefined, operation); try { - // Read raw bytes so invalid UTF-8 is rejected instead of - // being silently replaced by lossy "utf8" decoding. - const parsed = parseUtf8Json( - await readRegularFile(value), - operation, - value, - "cli-json-input", - ); + const parsed = await readCliJsonFile(value, operation); return parsed.ok ? { ok: true, value: parsed.value } - : jsonFileError(value, operation, "invalid-json"); + : { + ok: false, + error: { + error: "Application workflow failed", + ...projectAnalysisError(parsed.error), + input_path: value, + input_reason: + parsed.error._tag === "AnalysisInputError" + ? "invalid-json" + : "too-large", + }, + }; } catch (cause: unknown) { - if (cause instanceof AnalysisResourceConstraintError) - return { - ok: false, - error: { - error: "Application workflow failed", - ...projectAnalysisError(cause), - input_path: value, - input_reason: "too-large", - }, - }; const systemCode = accessDeniedSystemCode(cause); if (systemCode !== undefined) return jsonAccessDeniedError(value, operation, systemCode, cause); @@ -57,6 +46,12 @@ export const parseCliJsonInput = async ( !hasExplicitJsonFileExtension(value) ) return { ok: false, error: inputError(operation) }; + if ( + !(cause instanceof Error) || + !("code" in cause) || + typeof cause.code !== "string" + ) + throw cause; return jsonFileError(value, operation, "read-failed", cause); } }; @@ -120,13 +115,6 @@ const cannotBeAnExistingFile = (cause: unknown): boolean => const hasExplicitJsonFileExtension = (value: string): boolean => value.toLowerCase().endsWith(".json"); -const isStringLengthLimit = (cause: unknown): boolean => - cause instanceof Error && - (("code" in cause && cause.code === "ERR_STRING_TOO_LONG") || - /cannot create a string longer than|invalid string length/i.test( - cause.message, - )); - const inputError = (operation: string): JsonValue => ({ error: "Application workflow failed", ...projectAnalysisError( @@ -194,30 +182,3 @@ const readFailureIssue = (path: string | undefined, cause: unknown) => { : `The JSON input file could not be read${code}: ${path ?? ""}`, }; }; - -const jsonTooLargeError = ( - path: string | undefined, - operation: string, -): { readonly ok: false; readonly error: JsonValue } => ({ - ok: false, - error: { - error: "Application workflow failed", - ...projectAnalysisError( - new AnalysisResourceConstraintError( - operation, - "memory", - `The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`, - { - boundary: "cli-json-input", - max_string_code_units: bufferConstants.MAX_STRING_LENGTH, - }, - { - remediationAction: - "Provide a smaller JSON document or rerun its producer on a smaller selection. Splitting JSON text alone does not produce a valid workflow input.", - }, - ), - ), - ...(path === undefined ? {} : { input_path: path }), - input_reason: "too-large", - }, -}); diff --git a/tests/boundary/cli/jsonInputSize.test.ts b/tests/boundary/cli/jsonInputSize.test.ts index fce9412f2..13e99eafa 100644 --- a/tests/boundary/cli/jsonInputSize.test.ts +++ b/tests/boundary/cli/jsonInputSize.test.ts @@ -1,5 +1,5 @@ import { constants } from "node:buffer"; -import { open } from "node:fs/promises"; +import { mkdir, open, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; @@ -8,14 +8,14 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; cliTest( - "preserves the runtime string limit through CLI and Evidence file inputs", + "rejects malformed oversized CLI input and preserves Evidence decoder resource failures", async ({ cli, processes }) => { const root = await createTestTempDirectory("rea-json-string-limit-"); const input = join(root, "oversized.json"); const file = await open(input, "wx"); try { - // Sparse zero bytes are valid UTF-8. Decoded length exceeds the runtime - // string limit before JSON syntax can be inspected; no fixture is retained. + // Sparse zero bytes are valid UTF-8. The Evidence reader reaches the + // decoded-length limit; the streamed CLI rejects their invalid JSON syntax. await file.truncate(constants.MAX_STRING_LENGTH + 1); } finally { await file.close(); @@ -31,17 +31,9 @@ cliTest( arguments: ["trace-application-feature", input, "--json"], }); expect(result.json).toMatchObject({ - code: "resource_constraint", - category: "resource_constraint", + code: "invalid_request", input_path: input, - input_reason: "too-large", - retryable: false, - remediation: { action: expect.stringContaining("re-analyze") }, - details: { - operation: "trace-application-feature", - resource: "memory", - reported_limits: limits, - }, + input_reason: "invalid-json", }); expect(result.exitCode).toBe(1); expect(result.stderr).toBe(""); @@ -67,3 +59,101 @@ cliTest( expect(evidence.exitCode).toBe(0); }, ); + +cliTest( + "traces valid Evidence beyond the whole-document string limit", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-json-streamed-evidence-"); + const source = join(root, "application"); + await mkdir(source); + await writeFile( + join(source, "package.json"), + JSON.stringify({ + name: "streamed-evidence", + type: "module", + main: "entry.mjs", + }), + ); + await writeFile( + join(source, "entry.mjs"), + 'export const observed = "source-owned";', + ); + const application = await cli.run({ + arguments: [ + "analyze-javascript-application", + source, + "--artifact-format", + "directory", + "--json", + ], + }); + expect(application.exitCode).toBe(0); + const request = JSON.stringify({ + application: application.json, + seed: { kind: "module", value: "entry.mjs", match: "contains" }, + direction: "incoming", + }); + const input = join(root, "trace.json"); + await writeFile(input, request); + const ordinary = await cli.run({ + arguments: ["trace-application-feature", input, "--json"], + }); + expect(ordinary.exitCode).toBe(0); + const file = await open(input, "a"); + try { + const whitespace = Buffer.alloc(1024 * 1024, 0x20); + let remaining = + constants.MAX_STRING_LENGTH + 1 - Buffer.byteLength(request); + while (remaining > 0) { + const length = Math.min(remaining, whitespace.length); + await file.writeFile(whitespace.subarray(0, length)); + remaining -= length; + } + } finally { + await file.close(); + } + const streamed = await cli.run({ + arguments: ["trace-application-feature", input, "--json"], + }); + expect(streamed.exitCode).toBe(0); + expect(streamed.json).toEqual(ordinary.json); + expect(streamed.stderr).toBe(""); + }, +); + +cliTest( + "returns a typed string assembly constraint under a bounded heap", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-json-string-headroom-"); + const input = join(root, "long-string.json"); + const file = await open(input, "wx"); + try { + await file.writeFile('"'); + const chunk = Buffer.alloc(1024 * 1024, 0x78); + let remaining = constants.MAX_STRING_LENGTH + 1; + while (remaining > 0) { + const length = Math.min(remaining, chunk.length); + await file.writeFile(chunk.subarray(0, length)); + remaining -= length; + } + await file.writeFile('"'); + } finally { + await file.close(); + } + const result = await cli.run({ + arguments: ["trace-application-feature", input, "--json"], + environment: { NODE_OPTIONS: "--max-old-space-size=768" }, + }); + expect(result.exitCode, result.stderr).toBe(1); + expect(result.stderr).toBe(""); + expect(result.json).toMatchObject({ + code: "resource_constraint", + input_path: input, + input_reason: "too-large", + details: { resource: "memory" }, + remediation: { + action: expect.stringContaining("re-analyze a smaller selection"), + }, + }); + }, +); diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.ts index 1116bf892..099c95b25 100644 --- a/tests/boundary/filesystem/cliJsonInput.test.ts +++ b/tests/boundary/filesystem/cliJsonInput.test.ts @@ -1,21 +1,13 @@ import { execFile } from "node:child_process"; -import { constants as bufferConstants } from "node:buffer"; -import { - appendFile, - chmod, - open, - symlink, - type FileHandle, - writeFile, -} from "node:fs/promises"; +import { chmod, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; import { describe, expect, it, onTestFinished } from "vitest"; import { createCli } from "../../../src/cli.js"; -import { isCliOperationFailure } from "../../../src/cliLogging.js"; import { parseCliJsonInput } from "../../../src/cliJsonInput.js"; +import { readCliJsonFile } from "../../../src/cliJsonFile.js"; import { analysisCliErrorEnvelopeSchema } from "../../../src/contracts/errorSchemas.js"; import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -107,58 +99,106 @@ describe("CLI JSON file selection", () => { ); }); -describe("CLI JSON input", () => { - it("reports a valid file beyond the runtime string limit as too large", async () => { - const root = await createTestTempDirectory("rea-json-input-too-large-"); +describe("CLI JSON streamed input", () => { + it("preserves native JSON meanings through chunked file parsing", async () => { + const root = await createTestTempDirectory("rea-json-input-chunks-"); const path = join(root, "input.json"); - const size = bufferConstants.MAX_STRING_LENGTH + 1; - await writeValidOversizedObject(path, size); + const documents = [ + "null", + "false", + "-0", + "1e400", + '{"__proto__":{"observed":true},"constructor":2,"duplicate":1,"duplicate":3}', + JSON.stringify({ + text: `${"x".repeat(65_524)}😀漢字`, + escaped: "\ud800", + }), + JSON.stringify({ [`${"x".repeat(70_000)}😀`]: "long key" }), + JSON.stringify( + Array.from({ length: 4_000 }, (_, id) => ({ id, text: "å" })), + ), + ]; + for (const text of documents) { + await writeFile(path, `${" ".repeat(9 * 1024 * 1024)}${text}`); + const parsed = await parseCliJsonInput(path, "test-input"); + if (!parsed.ok) throw new Error(JSON.stringify(parsed.error)); + expect(parsed.value).toEqual(JSON.parse(text)); + if (text.includes("__proto__")) { + if (typeof parsed.value !== "object" || parsed.value === null) + throw new Error("Expected the parsed object"); + expect(Object.getPrototypeOf(parsed.value)).toBe(Object.prototype); + expect(Object.hasOwn(parsed.value, "__proto__")).toBe(true); + } + } + }); - const result = await parseCliJsonInput(path, "test-input"); - if (result.ok) throw new Error("Expected oversized JSON to be rejected"); - expect(analysisCliErrorEnvelopeSchema.parse(result.error)).toEqual( - result.error, - ); - expect(result).toMatchObject({ - ok: false, - error: { - code: "resource_constraint", - input_path: path, - input_reason: "too-large", - details: { - resource: "memory", - reported_limits: { - boundary: "cli-json-input", - max_string_code_units: bufferConstants.MAX_STRING_LENGTH, - }, + it.each([ + "", + "[1,]", + "{} {}", + '{"value":', + '"unterminated', + "\uFEFF{}", + "\u00a0{}", + "{\u000b}", + "01", + "NaN", + '"\u0001"', + ])("rejects strict JSON syntax violations in files (%j)", async (text) => { + const root = await createTestTempDirectory("rea-json-input-invalid-"); + const path = join(root, "input.json"); + for (const prefix of ["", " ".repeat(9 * 1024 * 1024)]) { + await writeFile(path, `${prefix}${text}`); + expect(await parseCliJsonInput(path, "test-input")).toMatchObject({ + ok: false, + error: { + code: "invalid_request", + input_path: path, + input_reason: "invalid-json", }, - remediation: { - action: expect.stringContaining("re-analyze a smaller selection"), + }); + } + }); + + it.each([Buffer.from([0xf0, 0x28, 0x8c, 0xbc]), Buffer.from([0xe2, 0x82])])( + "rejects malformed and incomplete UTF-8 at a read boundary (%j)", + async (invalid) => { + const root = await createTestTempDirectory("rea-json-input-utf8-"); + const path = join(root, "input.json"); + await writeFile( + path, + Buffer.concat([ + Buffer.from(`${" ".repeat(9 * 1024 * 1024)}"${"x".repeat(65_534)}`), + invalid, + ]), + ); + expect(await parseCliJsonInput(path, "test-input")).toMatchObject({ + ok: false, + error: { + input_reason: "invalid-json", + details: { issues: [{ message: "JSON input is not valid UTF-8" }] }, }, - }, + }); + }, + ); + + it("cancels a file parse and allows a following read", async () => { + const root = await createTestTempDirectory("rea-json-input-cancel-"); + const path = join(root, "input.json"); + await writeFile(path, `${" ".repeat(9 * 1024 * 1024)}{"after":true}`); + const controller = new AbortController(); + const cancelled = readCliJsonFile(path, "test-input", controller.signal); + setImmediate(() => controller.abort(new Error("cancel JSON file parse"))); + await expect(cancelled).rejects.toThrow("cancel JSON file parse"); + await writeFile(path, '{"after":true}'); + expect(await readCliJsonFile(path, "test-input")).toEqual({ + ok: true, + value: { after: true }, }); + }); +}); - expect(isCliOperationFailure(result.error)).toBe(true); - - const characterCount = - Math.floor(bufferConstants.MAX_STRING_LENGTH / 2) + 2; - await writeValidMultibyteString(path, characterCount); - - const multibyteResult = await parseCliJsonInput(path, "test-input"); - if (!multibyteResult.ok) - throw new Error("Expected large multibyte JSON to fit the string limit"); - if (typeof multibyteResult.value !== "string") - throw new Error("Expected parsed JSON string"); - expect(multibyteResult.value.length).toBe(characterCount); - - // Large-input streaming must flush and reject an incomplete UTF-8 suffix. - await appendFile(path, Buffer.from([0xc3])); - expect(await parseCliJsonInput(path, "test-input")).toMatchObject({ - ok: false, - error: { input_path: path, input_reason: "invalid-json" }, - }); - }, 20_000); - +describe("CLI JSON input", () => { it("distinguishes malformed inline text from files and preserves bracket-prefixed paths", async () => { expect(await parseCliJsonInput("[", "test-input")).toMatchObject({ ok: false, @@ -251,47 +291,3 @@ describe("CLI JSON input", () => { }, ); }); - -const writeRepeatedBytes = async ( - handle: FileHandle, - bytes: Buffer, - length: number, -): Promise => { - let remaining = length; - while (remaining > 0) { - const chunkLength = Math.min(bytes.length, remaining); - await handle.writeFile(bytes.subarray(0, chunkLength)); - remaining -= chunkLength; - } -}; - -const writeValidOversizedObject = async ( - path: string, - size: number, -): Promise => { - const handle = await open(path, "w"); - const whitespace = Buffer.alloc(1024 * 1024, 0x20); - try { - await handle.writeFile("{"); - await writeRepeatedBytes(handle, whitespace, size - 2); - await handle.writeFile("}"); - } finally { - await handle.close(); - } -}; - -const writeValidMultibyteString = async ( - path: string, - characterCount: number, -): Promise => { - const encodedCharacters = Buffer.from("é".repeat(512 * 1024)); - const byteLength = characterCount * 2 + 2; - const handle = await open(path, "w"); - try { - await handle.writeFile('"'); - await writeRepeatedBytes(handle, encodedCharacters, byteLength - 2); - await handle.writeFile('"'); - } finally { - await handle.close(); - } -};