shifathosn
/
rea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(cli): stream JSON inputs beyond the whole-document string limit (#1360) 

Stream large CLI JSON files through a strict UTF-8 decoder and standard JSON parser using the same admitted regular-file handle. Preserve native parsing for small inputs, typed malformed-input/resource/access-denied errors, cancellation, and complete Evidence.

Verified with focused public CLI regressions, a 768 MB whitespace-padded valid request, and the Windows native decoder boundary. Individual strings and the assembled object graph remain subject to native runtime limits. All 27 final-head checks passed.

Refs #1331
This commit is contained in:
Ysopwnable 2026-10-10 04:53:09 +08:00 committed by GitHub
parent 3728c8267b
commit 6ce3288f3b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 555 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
docs/javascript-artifact-reconstruction.md
View File

@ -113,11 +113,21 @@ smaller view, for example `--format json --filter-output
evidence_id,normalized_result.statistics`. Streaming output does not bound the
memory needed to construct the analysis graph itself.
CLI workflows still parse each JSON input as one value. A valid input exceeding
the Node runtime's maximum string length returns `resource_constraint` with
`input_reason: "too-large"` and the runtime limit. Produce smaller independent
analysis inputs before retrying; streaming output does not imply streaming
input parsing.
CLI workflows parse larger JSON input files incrementally from one verified
regular-file handle, without constructing a document-sized string. Smaller
files retain native JSON parsing. Both paths require strict JSON and valid
UTF-8, and return one complete value; the assembled object still needs memory.
An individual JSON string exceeding the native length limit or the available
heap headroom for assembly returns
`resource_constraint` with `input_reason: "too-large"`. Supply a smaller valid
value; for Evidence workflows, re-analyze a smaller selection of the original
target and use its complete Evidence. Splitting JSON text or trimming Evidence
fields does not produce a valid workflow input.
Evidence bundle and analysis snapshot file readers still decode whole
documents. Their runtime string-limit failures report a resource constraint
with the selected path, observed bytes and UTF-16 limit, rather than malformed
JSON.
MCP prepares the complete repeated response incrementally against the pinned
SDK's 10 MiB stdio receive-buffer budget. Oversized results return an actionable

22
package-lock.json generated
View File

@ -38,6 +38,7 @@
"plist": "5.0.0",
"semver": "7.8.5",
"smol-toml": "1.9.0",
"stream-json": "3.7.0",
"write-file-atomic": "7.0.0",
"ws": "8.21.0",
"zod": "4.4.3"
@ -7070,6 +7071,27 @@
"dev": true,
"license": "MIT"
},
"node_modules/stream-chain": {
"version": "4.2.6",
"resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-4.2.6.tgz",
"integrity": "sha512-1zeJ8CrtJfmiba26ui8jXkq/xLRFvhzkdH02D5QLO9Cnovgeb28IJxg88DdraWnjnkbOol/++uIAybJbJhk7ig==",
"license": "BSD-3-Clause",
"funding": {
"url": "https://github.com/sponsors/uhop"
}
},
"node_modules/stream-json": {
"version": "3.7.0",
"resolved": "https://registry.npmjs.org/stream-json/-/stream-json-3.7.0.tgz",
"integrity": "sha512-rCSBdcBP/bPk6T8QFcxAj1MSzAuc5i49cYW6IE7sYObNEPccBJIUiL6fU9c9BWt40aJK0siVynLX7lWaW8alXw==",
"license": "BSD-3-Clause",
"dependencies": {
"stream-chain": "^4.2.5"
},
"funding": {
"url": "https://github.com/sponsors/uhop"
}
},
"node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",

1
package.json
View File

@ -263,6 +263,7 @@
"plist": "5.0.0",
"semver": "7.8.5",
"smol-toml": "1.9.0",
"stream-json": "3.7.0",
"write-file-atomic": "7.0.0",
"ws": "8.21.0",
"zod": "4.4.3",

25
src/application/JsonFiles.ts
View File

@ -3,7 +3,6 @@ import {
lstat,
mkdtemp,
open,
readFile,
realpath,
rename,
rm,
@ -18,6 +17,8 @@ import {
} from "../domain/analysisErrorCore.js";
import { err, ok, type Result } from "../domain/result.js";
import { parseUtf8Json } from "./Utf8JsonInput.js";
import { readRegularFile } from "./RegularFileRead.js";
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
/** Request control and its owning operation for an interruptible atomic write. */
export interface TextWriteCancellation {
@ -36,13 +37,7 @@ export const readJsonFile = async (
> => {
const requestedPath = resolve(path);
try {
const canonicalPath = await realpath(requestedPath);
const stats = await lstat(canonicalPath);
if (!stats.isFile())
return err(
new EvidenceFileError("read", "not-file", { path: requestedPath }),
);
const encoded = await readFile(canonicalPath);
const encoded = await readRegularFile(requestedPath);
const decoded = parseUtf8Json(encoded, "read_evidence_file", requestedPath);
if (!decoded.ok) {
return err(
@ -56,10 +51,16 @@ export const readJsonFile = async (
} catch (cause: unknown) {
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
return err(
new EvidenceFileError("read", missingOrIo(cause), {
cause,
path: requestedPath,
}),
new EvidenceFileError(
"read",
cause instanceof NonRegularFileReadError
? "not-file"
: missingOrIo(cause),
{
cause,
path: requestedPath,
},
),
);
}
};

23
src/application/RegularFileRead.ts
View File

@ -1,16 +1,29 @@
import type { Stats } from "node:fs";
import type { FileHandle } from "node:fs/promises";
import { openRegularFile } from "../filesystem/RegularFile.js";
/** Admit a regular file without waiting for a pipe, then read its verified handle. */
export const readRegularFile = async (
/** Admit one regular-file handle, retain it through the consumer, and always close it. */
export const withRegularFile = async <Value>(
path: string,
read: (handle: FileHandle, stats: Stats) => Promise<Value>,
signal?: AbortSignal,
): Promise<Buffer> => {
): Promise<Value> => {
const handle = await openRegularFile(path, { symlinks: "follow", signal });
try {
const bytes = await handle.readFile({ signal });
const stats = await handle.stat();
signal?.throwIfAborted();
return bytes;
const value = await read(handle, stats);
signal?.throwIfAborted();
return value;
} finally {
await handle.close();
}
};
/** Admit a regular file without waiting for a pipe, then read its verified handle. */
export const readRegularFile = (
path: string,
signal?: AbortSignal,
): Promise<Buffer> =>
withRegularFile(path, (handle) => handle.readFile({ signal }), signal);

7
src/application/Utf8JsonInput.ts
View File

@ -3,6 +3,10 @@ import { constants } from "node:buffer";
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js";
/** Recovery that preserves complete, authenticated Evidence workflow inputs. */
export const JSON_INPUT_RESOURCE_REMEDIATION =
"Provide a smaller valid JSON value. For Evidence workflows, re-analyze a smaller selection of the original target and use its complete Evidence; splitting JSON text or trimming Evidence fields does not produce valid workflow input.";
/** Decode JSON bytes without confusing a runtime string limit with malformed input. */
export const parseUtf8Json = (
bytes: Uint8Array,
@ -29,8 +33,7 @@ export const parseUtf8Json = (
},
{
...(cause === undefined ? {} : { cause }),
remediationAction:
"Provide a smaller JSON document. For Evidence-based workflows, re-analyze a smaller selection of the original target and use its Evidence; splitting JSON text alone does not produce a valid workflow input.",
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
},
);
let text: string;

261
src/cliJsonFile.ts Normal file
View File

@ -0,0 +1,261 @@
import { constants } from "node:buffer";
import type { FileHandle } from "node:fs/promises";
import { pipeline } from "node:stream/promises";
import { getHeapStatistics } from "node:v8";
import parser from "stream-json/parser.js";
import streamValues from "stream-json/streamers/stream-values.js";
import { withRegularFile } from "./application/RegularFileRead.js";
import {
JSON_INPUT_RESOURCE_REMEDIATION,
parseUtf8Json,
} from "./application/Utf8JsonInput.js";
import {
AnalysisInputError,
AnalysisResourceConstraintError,
} from "./domain/analysisErrorCore.js";
import { err, ok, type Result } from "./domain/result.js";
type JsonFileFailure = AnalysisInputError | AnalysisResourceConstraintError;
const READ_CHUNK_BYTES = 64 * 1024;
const PACKED_STRING_PART_CODE_UNITS = 64 * 1024;
// Retain native-parser speed for modest inputs while bounding its extra copies.
// This selects an implementation; larger files remain accepted through streaming.
const NATIVE_PARSE_READ_BUDGET = 8 * 1024 * 1024;
/** Parse strict UTF-8 JSON without requiring a whole-document string. */
export const readCliJsonFile = (
path: string,
operation: string,
signal?: AbortSignal,
): Promise<Result<unknown, JsonFileFailure>> =>
withRegularFile(
path,
async (handle, stats) => {
let found = false;
let value: unknown;
try {
const prefix =
stats.size <= NATIVE_PARSE_READ_BUDGET
? await readPrefix(handle, stats.size + 1, signal)
: undefined;
if (prefix?.complete === true) {
const parsed = parseUtf8Json(
prefix.bytes,
operation,
path,
"cli-json-input",
);
return parsed.ok
? ok(parsed.value)
: err(invalidJson(operation, parsed.error, parsed.cause));
}
await pipeline(
decodedChunks(handle, signal, prefix?.bytes),
parser.asStream({
jsonStreaming: false,
streamValues: false,
packKeys: false,
packStrings: false,
}),
packJsonStrings(operation, stats.size),
streamValues.asStream(),
async (rows: AsyncIterable<unknown>) => {
for await (const row of rows) {
if (
found ||
typeof row !== "object" ||
row === null ||
!("key" in row) ||
row.key !== 0 ||
!("value" in row)
)
throw new Error("Unexpected JSON parser result");
found = true;
value = row.value;
}
},
signal === undefined ? {} : { signal },
);
return found
? ok(value)
: err(invalidJson(operation, "JSON document is empty"));
} catch (cause: unknown) {
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
if (
cause instanceof TypeError &&
"code" in cause &&
cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA"
)
return err(
invalidJson(operation, "JSON input is not valid UTF-8", cause),
);
if (
cause instanceof SyntaxError ||
(cause instanceof Error && cause.message.startsWith("Parser "))
)
return err(invalidJson(operation, cause.message, cause));
if (
cause instanceof Error &&
((cause instanceof RangeError &&
cause.message === "Invalid string length") ||
("code" in cause && cause.code === "ERR_STRING_TOO_LONG"))
)
return err(
new AnalysisResourceConstraintError(
operation,
"memory",
"An individual JSON string exceeds the runtime string limit",
{
input_file_bytes: stats.size,
max_string_code_units: constants.MAX_STRING_LENGTH,
},
{
cause,
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
},
),
);
throw cause;
}
},
signal,
);
const packJsonStrings = (operation: string, inputFileBytes: number) =>
async function* (tokens: AsyncIterable<unknown>): AsyncGenerator<unknown> {
let kind: "keyValue" | "stringValue" | undefined;
let length = 0;
let value = "";
let parts: string[] = [];
let partLength = 0;
for await (const token of tokens) {
if (typeof token !== "object" || token === null || !("name" in token))
throw new Error("Unexpected JSON parser token");
switch (token.name) {
case "startKey":
case "startString":
kind = token.name === "startKey" ? "keyValue" : "stringValue";
break;
case "stringChunk":
if (
kind === undefined ||
!("value" in token) ||
typeof token.value !== "string"
)
throw new Error("Unexpected JSON string fragment");
length += token.value.length;
if (length > constants.MAX_STRING_LENGTH)
throw new RangeError("Invalid string length");
parts.push(token.value);
partLength += token.value.length;
if (partLength >= PACKED_STRING_PART_CODE_UNITS) {
// The tokenizer emits short fragments. Coalesce them before retaining
// a string so millions of substring/rope nodes cannot exhaust the heap
// before the native string-length constraint can be reported.
requireStringAssemblyHeadroom(operation, inputFileBytes, length);
value += parts.join("");
parts = [];
partLength = 0;
}
break;
case "endKey":
case "endString":
if (kind === undefined) throw new Error("Unexpected JSON string end");
requireStringAssemblyHeadroom(operation, inputFileBytes, length);
yield { name: kind, value: value + parts.join("") };
kind = undefined;
length = 0;
value = "";
parts = [];
partLength = 0;
break;
default:
yield token;
}
}
};
const requireStringAssemblyHeadroom = (
operation: string,
inputFileBytes: number,
stringCodeUnits: number,
): void => {
if (stringCodeUnits < PACKED_STRING_PART_CODE_UNITS) return;
const heap = getHeapStatistics();
// Concatenation or key interning may flatten the retained rope. Reserve its
// UTF-16 representation and the next coalesced part before that allocation.
const requiredBytes = 2 * (stringCodeUnits + PACKED_STRING_PART_CODE_UNITS);
if (requiredBytes > heap.total_available_size)
throw new AnalysisResourceConstraintError(
operation,
"memory",
"Insufficient heap headroom to assemble a complete JSON string",
{
input_file_bytes: inputFileBytes,
string_code_units: stringCodeUnits,
string_assembly_headroom_bytes: requiredBytes,
available_heap_bytes: heap.total_available_size,
heap_size_limit_bytes: heap.heap_size_limit,
},
{
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
},
);
};
async function* decodedChunks(
handle: FileHandle,
signal: AbortSignal | undefined,
prefix?: Uint8Array,
): AsyncGenerator<string> {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
if (prefix !== undefined)
for (let offset = 0; offset < prefix.length; offset += READ_CHUNK_BYTES) {
signal?.throwIfAborted();
yield decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), {
stream: true,
});
}
const bytes = Buffer.allocUnsafe(READ_CHUNK_BYTES);
while (true) {
signal?.throwIfAborted();
const read = await handle.read(bytes, 0, bytes.length, null);
signal?.throwIfAborted();
if (read.bytesRead === 0) break;
yield decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true });
}
yield decoder.decode();
}
const readPrefix = async (
handle: FileHandle,
size: number,
signal: AbortSignal | undefined,
): Promise<{ readonly bytes: Buffer; readonly complete: boolean }> => {
const bytes = Buffer.allocUnsafe(size);
let offset = 0;
while (offset < bytes.length) {
signal?.throwIfAborted();
const read = await handle.read(bytes, offset, bytes.length - offset, null);
signal?.throwIfAborted();
if (read.bytesRead === 0)
return { bytes: bytes.subarray(0, offset), complete: true };
offset += read.bytesRead;
}
// The file grew after admission. Continue from this same handle, preserving
// the prefix already read rather than reopening or dropping its bytes.
return { bytes, complete: false };
};
const invalidJson = (
operation: string,
message: string,
cause?: unknown,
): AnalysisInputError =>
new AnalysisInputError(
operation,
cause === undefined ? undefined : { cause },
[{ path: [], reason: "invalid_format", expected: "JSON", message }],
);

79
src/cliJsonInput.ts
View File

@ -1,13 +1,10 @@
import { constants as bufferConstants } from "node:buffer";
import { resolve } from "node:path";
import { readRegularFile } from "./application/RegularFileRead.js";
import { readCliJsonFile } from "./cliJsonFile.js";
import { NonRegularFileReadError } from "./filesystem/RegularFile.js";
import { parseUtf8Json } from "./application/Utf8JsonInput.js";
import {
AnalysisAccessDeniedError,
AnalysisInputError,
AnalysisResourceConstraintError,
} from "./domain/analysisErrorCore.js";
import { projectAnalysisError } from "./domain/analysisErrorProjection.js";
import type { JsonValue } from "./domain/jsonValue.js";
@ -23,31 +20,23 @@ export const parseCliJsonInput = async (
> => {
const inline = safeParseJson(value);
if (inline.ok) return { ok: true, value: inline.value };
if (isStringLengthLimit(inline.cause))
return jsonTooLargeError(undefined, operation);
try {
// Read raw bytes so invalid UTF-8 is rejected instead of
// being silently replaced by lossy "utf8" decoding.
const parsed = parseUtf8Json(
await readRegularFile(value),
operation,
value,
"cli-json-input",
);
const parsed = await readCliJsonFile(value, operation);
return parsed.ok
? { ok: true, value: parsed.value }
: jsonFileError(value, operation, "invalid-json");
: {
ok: false,
error: {
error: "Application workflow failed",
...projectAnalysisError(parsed.error),
input_path: value,
input_reason:
parsed.error._tag === "AnalysisInputError"
? "invalid-json"
: "too-large",
},
};
} catch (cause: unknown) {
if (cause instanceof AnalysisResourceConstraintError)
return {
ok: false,
error: {
error: "Application workflow failed",
...projectAnalysisError(cause),
input_path: value,
input_reason: "too-large",
},
};
const systemCode = accessDeniedSystemCode(cause);
if (systemCode !== undefined)
return jsonAccessDeniedError(value, operation, systemCode, cause);
@ -57,6 +46,12 @@ export const parseCliJsonInput = async (
!hasExplicitJsonFileExtension(value)
)
return { ok: false, error: inputError(operation) };
if (
!(cause instanceof Error) ||
!("code" in cause) ||
typeof cause.code !== "string"
)
throw cause;
return jsonFileError(value, operation, "read-failed", cause);
}
};
@ -120,13 +115,6 @@ const cannotBeAnExistingFile = (cause: unknown): boolean =>
const hasExplicitJsonFileExtension = (value: string): boolean =>
value.toLowerCase().endsWith(".json");
const isStringLengthLimit = (cause: unknown): boolean =>
cause instanceof Error &&
(("code" in cause && cause.code === "ERR_STRING_TOO_LONG") ||
/cannot create a string longer than|invalid string length/i.test(
cause.message,
));
const inputError = (operation: string): JsonValue => ({
error: "Application workflow failed",
...projectAnalysisError(
@ -194,30 +182,3 @@ const readFailureIssue = (path: string | undefined, cause: unknown) => {
: `The JSON input file could not be read${code}: ${path ?? ""}`,
};
};
const jsonTooLargeError = (
path: string | undefined,
operation: string,
): { readonly ok: false; readonly error: JsonValue } => ({
ok: false,
error: {
error: "Application workflow failed",
...projectAnalysisError(
new AnalysisResourceConstraintError(
operation,
"memory",
`The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`,
{
boundary: "cli-json-input",
max_string_code_units: bufferConstants.MAX_STRING_LENGTH,
},
{
remediationAction:
"Provide a smaller JSON document or rerun its producer on a smaller selection. Splitting JSON text alone does not produce a valid workflow input.",
},
),
),
...(path === undefined ? {} : { input_path: path }),
input_reason: "too-large",
},
});

118
tests/boundary/cli/jsonInputSize.test.ts
View File

@ -1,5 +1,5 @@
import { constants } from "node:buffer";
import { open } from "node:fs/promises";
import { mkdir, open, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
@ -8,14 +8,14 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { cliTest } from "../../support/cli/cliFixture.js";
cliTest(
"preserves the runtime string limit through CLI and Evidence file inputs",
"rejects malformed oversized CLI input and preserves Evidence decoder resource failures",
async ({ cli, processes }) => {
const root = await createTestTempDirectory("rea-json-string-limit-");
const input = join(root, "oversized.json");
const file = await open(input, "wx");
try {
// Sparse zero bytes are valid UTF-8. Decoded length exceeds the runtime
// string limit before JSON syntax can be inspected; no fixture is retained.
// Sparse zero bytes are valid UTF-8. The Evidence reader reaches the
// decoded-length limit; the streamed CLI rejects their invalid JSON syntax.
await file.truncate(constants.MAX_STRING_LENGTH + 1);
} finally {
await file.close();
@ -31,17 +31,9 @@ cliTest(
arguments: ["trace-application-feature", input, "--json"],
});
expect(result.json).toMatchObject({
code: "resource_constraint",
category: "resource_constraint",
code: "invalid_request",
input_path: input,
input_reason: "too-large",
retryable: false,
remediation: { action: expect.stringContaining("re-analyze") },
details: {
operation: "trace-application-feature",
resource: "memory",
reported_limits: limits,
},
input_reason: "invalid-json",
});
expect(result.exitCode).toBe(1);
expect(result.stderr).toBe("");
@ -67,3 +59,101 @@ cliTest(
expect(evidence.exitCode).toBe(0);
},
);
cliTest(
"traces valid Evidence beyond the whole-document string limit",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-json-streamed-evidence-");
const source = join(root, "application");
await mkdir(source);
await writeFile(
join(source, "package.json"),
JSON.stringify({
name: "streamed-evidence",
type: "module",
main: "entry.mjs",
}),
);
await writeFile(
join(source, "entry.mjs"),
'export const observed = "source-owned";',
);
const application = await cli.run({
arguments: [
"analyze-javascript-application",
source,
"--artifact-format",
"directory",
"--json",
],
});
expect(application.exitCode).toBe(0);
const request = JSON.stringify({
application: application.json,
seed: { kind: "module", value: "entry.mjs", match: "contains" },
direction: "incoming",
});
const input = join(root, "trace.json");
await writeFile(input, request);
const ordinary = await cli.run({
arguments: ["trace-application-feature", input, "--json"],
});
expect(ordinary.exitCode).toBe(0);
const file = await open(input, "a");
try {
const whitespace = Buffer.alloc(1024 * 1024, 0x20);
let remaining =
constants.MAX_STRING_LENGTH + 1 - Buffer.byteLength(request);
while (remaining > 0) {
const length = Math.min(remaining, whitespace.length);
await file.writeFile(whitespace.subarray(0, length));
remaining -= length;
}
} finally {
await file.close();
}
const streamed = await cli.run({
arguments: ["trace-application-feature", input, "--json"],
});
expect(streamed.exitCode).toBe(0);
expect(streamed.json).toEqual(ordinary.json);
expect(streamed.stderr).toBe("");
},
);
cliTest(
"returns a typed string assembly constraint under a bounded heap",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-json-string-headroom-");
const input = join(root, "long-string.json");
const file = await open(input, "wx");
try {
await file.writeFile('"');
const chunk = Buffer.alloc(1024 * 1024, 0x78);
let remaining = constants.MAX_STRING_LENGTH + 1;
while (remaining > 0) {
const length = Math.min(remaining, chunk.length);
await file.writeFile(chunk.subarray(0, length));
remaining -= length;
}
await file.writeFile('"');
} finally {
await file.close();
}
const result = await cli.run({
arguments: ["trace-application-feature", input, "--json"],
environment: { NODE_OPTIONS: "--max-old-space-size=768" },
});
expect(result.exitCode, result.stderr).toBe(1);
expect(result.stderr).toBe("");
expect(result.json).toMatchObject({
code: "resource_constraint",
input_path: input,
input_reason: "too-large",
details: { resource: "memory" },
remediation: {
action: expect.stringContaining("re-analyze a smaller selection"),
},
});
},
);

196
tests/boundary/filesystem/cliJsonInput.test.ts
View File

@ -1,21 +1,13 @@
import { execFile } from "node:child_process";
import { constants as bufferConstants } from "node:buffer";
import {
appendFile,
chmod,
open,
symlink,
type FileHandle,
writeFile,
} from "node:fs/promises";
import { chmod, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { promisify } from "node:util";
import { describe, expect, it, onTestFinished } from "vitest";
import { createCli } from "../../../src/cli.js";
import { isCliOperationFailure } from "../../../src/cliLogging.js";
import { parseCliJsonInput } from "../../../src/cliJsonInput.js";
import { readCliJsonFile } from "../../../src/cliJsonFile.js";
import { analysisCliErrorEnvelopeSchema } from "../../../src/contracts/errorSchemas.js";
import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
@ -107,58 +99,106 @@ describe("CLI JSON file selection", () => {
);
});
describe("CLI JSON input", () => {
it("reports a valid file beyond the runtime string limit as too large", async () => {
const root = await createTestTempDirectory("rea-json-input-too-large-");
describe("CLI JSON streamed input", () => {
it("preserves native JSON meanings through chunked file parsing", async () => {
const root = await createTestTempDirectory("rea-json-input-chunks-");
const path = join(root, "input.json");
const size = bufferConstants.MAX_STRING_LENGTH + 1;
await writeValidOversizedObject(path, size);
const documents = [
"null",
"false",
"-0",
"1e400",
'{"__proto__":{"observed":true},"constructor":2,"duplicate":1,"duplicate":3}',
JSON.stringify({
text: `${"x".repeat(65_524)}😀漢字`,
escaped: "\ud800",
}),
JSON.stringify({ [`${"x".repeat(70_000)}😀`]: "long key" }),
JSON.stringify(
Array.from({ length: 4_000 }, (_, id) => ({ id, text: "å" })),
),
];
for (const text of documents) {
await writeFile(path, `${" ".repeat(9 * 1024 * 1024)}${text}`);
const parsed = await parseCliJsonInput(path, "test-input");
if (!parsed.ok) throw new Error(JSON.stringify(parsed.error));
expect(parsed.value).toEqual(JSON.parse(text));
if (text.includes("__proto__")) {
if (typeof parsed.value !== "object" || parsed.value === null)
throw new Error("Expected the parsed object");
expect(Object.getPrototypeOf(parsed.value)).toBe(Object.prototype);
expect(Object.hasOwn(parsed.value, "__proto__")).toBe(true);
}
}
});
const result = await parseCliJsonInput(path, "test-input");
if (result.ok) throw new Error("Expected oversized JSON to be rejected");
expect(analysisCliErrorEnvelopeSchema.parse(result.error)).toEqual(
result.error,
);
expect(result).toMatchObject({
ok: false,
error: {
code: "resource_constraint",
input_path: path,
input_reason: "too-large",
details: {
resource: "memory",
reported_limits: {
boundary: "cli-json-input",
max_string_code_units: bufferConstants.MAX_STRING_LENGTH,
},
it.each([
"",
"[1,]",
"{} {}",
'{"value":',
'"unterminated',
"\uFEFF{}",
"\u00a0{}",
"{\u000b}",
"01",
"NaN",
'"\u0001"',
])("rejects strict JSON syntax violations in files (%j)", async (text) => {
const root = await createTestTempDirectory("rea-json-input-invalid-");
const path = join(root, "input.json");
for (const prefix of ["", " ".repeat(9 * 1024 * 1024)]) {
await writeFile(path, `${prefix}${text}`);
expect(await parseCliJsonInput(path, "test-input")).toMatchObject({
ok: false,
error: {
code: "invalid_request",
input_path: path,
input_reason: "invalid-json",
},
remediation: {
action: expect.stringContaining("re-analyze a smaller selection"),
});
}
});
it.each([Buffer.from([0xf0, 0x28, 0x8c, 0xbc]), Buffer.from([0xe2, 0x82])])(
"rejects malformed and incomplete UTF-8 at a read boundary (%j)",
async (invalid) => {
const root = await createTestTempDirectory("rea-json-input-utf8-");
const path = join(root, "input.json");
await writeFile(
path,
Buffer.concat([
Buffer.from(`${" ".repeat(9 * 1024 * 1024)}"${"x".repeat(65_534)}`),
invalid,
]),
);
expect(await parseCliJsonInput(path, "test-input")).toMatchObject({
ok: false,
error: {
input_reason: "invalid-json",
details: { issues: [{ message: "JSON input is not valid UTF-8" }] },
},
},
});
},
);
it("cancels a file parse and allows a following read", async () => {
const root = await createTestTempDirectory("rea-json-input-cancel-");
const path = join(root, "input.json");
await writeFile(path, `${" ".repeat(9 * 1024 * 1024)}{"after":true}`);
const controller = new AbortController();
const cancelled = readCliJsonFile(path, "test-input", controller.signal);
setImmediate(() => controller.abort(new Error("cancel JSON file parse")));
await expect(cancelled).rejects.toThrow("cancel JSON file parse");
await writeFile(path, '{"after":true}');
expect(await readCliJsonFile(path, "test-input")).toEqual({
ok: true,
value: { after: true },
});
});
});
expect(isCliOperationFailure(result.error)).toBe(true);
const characterCount =
Math.floor(bufferConstants.MAX_STRING_LENGTH / 2) + 2;
await writeValidMultibyteString(path, characterCount);
const multibyteResult = await parseCliJsonInput(path, "test-input");
if (!multibyteResult.ok)
throw new Error("Expected large multibyte JSON to fit the string limit");
if (typeof multibyteResult.value !== "string")
throw new Error("Expected parsed JSON string");
expect(multibyteResult.value.length).toBe(characterCount);
// Large-input streaming must flush and reject an incomplete UTF-8 suffix.
await appendFile(path, Buffer.from([0xc3]));
expect(await parseCliJsonInput(path, "test-input")).toMatchObject({
ok: false,
error: { input_path: path, input_reason: "invalid-json" },
});
}, 20_000);
describe("CLI JSON input", () => {
it("distinguishes malformed inline text from files and preserves bracket-prefixed paths", async () => {
expect(await parseCliJsonInput("[", "test-input")).toMatchObject({
ok: false,
@ -251,47 +291,3 @@ describe("CLI JSON input", () => {
},
);
});
const writeRepeatedBytes = async (
handle: FileHandle,
bytes: Buffer,
length: number,
): Promise<void> => {
let remaining = length;
while (remaining > 0) {
const chunkLength = Math.min(bytes.length, remaining);
await handle.writeFile(bytes.subarray(0, chunkLength));
remaining -= chunkLength;
}
};
const writeValidOversizedObject = async (
path: string,
size: number,
): Promise<void> => {
const handle = await open(path, "w");
const whitespace = Buffer.alloc(1024 * 1024, 0x20);
try {
await handle.writeFile("{");
await writeRepeatedBytes(handle, whitespace, size - 2);
await handle.writeFile("}");
} finally {
await handle.close();
}
};
const writeValidMultibyteString = async (
path: string,
characterCount: number,
): Promise<void> => {
const encodedCharacters = Buffer.from("é".repeat(512 * 1024));
const byteLength = characterCount * 2 + 2;
const handle = await open(path, "w");
try {
await handle.writeFile('"');
await writeRepeatedBytes(handle, encodedCharacters, byteLength - 2);
await handle.writeFile('"');
} finally {
await handle.close();
}
};