diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index f3eda9698..90dbb92ef 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -100,17 +100,22 @@ a temporary output file larger than the running engine's string limit, verifies its bytes and digest, and removes it. Use `-- jsonl` for the compact JSONL check. Each check needs space for one output file plus a 1 GiB free-space reserve. -Other CLI formats, `--token-count`, and MCP transport serialization still -assemble whole strings. Field selection remains useful when the caller needs a +Other CLI formats and `--token-count` still assemble whole strings. +Field selection remains useful when the caller needs a smaller view, for example `--format json --filter-output evidence_id,normalized_result.statistics`. Streaming output does not bound the memory needed to construct the analysis graph itself. -The separate large-response transport work is tracked in -[#1053](https://github.com/morluto/rea/issues/1053) and +MCP prepares the complete repeated response incrementally against the pinned +SDK's 10 MiB stdio receive-buffer budget. Oversized results return an actionable +transport constraint and the exact same-session Evidence reference. Use +`trace_application_feature` to inspect a selected module's relationships, or +`export_evidence_bundle` to write the complete canonical bundle without a +document-sized allocation. Same-session analysis reads reuse authenticated +immutable snapshots; foreign inline Evidence is still parsed and authenticated. +See [MCP tool results](mcp-contracts.md#tool-results) for larger client buffers +and `REA_MCP_MAX_RESPONSE_BYTES`. Follow-up results remain complete and can also +exceed the transport budget. Compact result views are tracked separately in [#1050](https://github.com/morluto/rea/issues/1050). -Client framing limits also apply: the pinned Node MCP SDK's stdio transport -defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate -the complete response, including text and structured Evidence projections. ## What is reconstructed diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 2ce8f5c3d..7445df1e6 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -164,6 +164,26 @@ directly to `compare_functions`, and `inspect_artifact` Evidence to `compare_artifacts`. Use `get_evidence_bundle` when the task needs broader retained session history or an explicit bundle for transfer. +REA prepares complete MCP results within the pinned stdio client's 10 MiB +receive-buffer budget, including both text and structured representations and +room for the JSON-RPC envelope. If a result cannot fit, REA returns +`resource_constraint` with `details.resource: "transport"` before constructing +a document-sized string. Analysis Evidence remains complete in the current +session. Its exact reference is reported in +`details.reported_limits.evidence_reference`; use it with a focused application +workflow, or call `export_evidence_bundle` with a destination path. Complete +bundle exports stream canonical JSON into an atomically published file. A broad +follow-up or `get_evidence_bundle` can also exceed the response budget; exporting +preserves the complete session without sending it through a single MCP frame. + +Clients that explicitly configure a larger receive buffer can set the REA +server's `REA_MCP_MAX_RESPONSE_BYTES` environment variable to the same byte +count. This setting must be a safe decimal integer at least 10485760; REA +reserves 1024 bytes for the envelope. Raising it restores complete inline +delivery for responses that fit that buffer and Node's single-string limit. +It does not change the client's buffer, analysis coverage, or retained content. +Ordinary responses keep their existing complete result contract. + ## Retained application Evidence inputs `trace_application_feature`, `trace_javascript_semantics`, diff --git a/scripts/lib/browser-screenshot-e2e.mjs b/scripts/lib/browser-screenshot-e2e.mjs index e7da037ab..9eed016c4 100644 --- a/scripts/lib/browser-screenshot-e2e.mjs +++ b/scripts/lib/browser-screenshot-e2e.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; import { chromium } from "playwright-core"; @@ -62,13 +65,21 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) { ); assert.equal(bytes.readUInt32BE(16), 2048); assert.equal(bytes.readUInt32BE(20), 1536); + await verifyDefaultScreenshotDelivery({ + entrypoint, + environment: env, + endpoint, + targetId: target.id, + screenshot, + }); + const maxBufferSize = 64 * 1024 * 1024; transport = new StdioClientTransport({ command: process.execPath, args: [entrypoint, "mcp"], - env, + env: { ...env, REA_MCP_MAX_RESPONSE_BYTES: String(maxBufferSize) }, stderr: "pipe", // The SDK defaults to a 10 MiB receive buffer; inline PNG output is larger. - maxBufferSize: 64 * 1024 * 1024, + maxBufferSize, }); client = new Client({ name: "browser-screenshot-real-e2e", version: "1" }); await client.connect(transport); @@ -102,6 +113,9 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) { mocked: false, cli: true, stdio_mcp: true, + default_buffer_recovery: true, + complete_export: true, + matched_large_buffer: true, png_bytes: bytes.byteLength, }; } finally { @@ -117,3 +131,73 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) { } } } + +const verifyDefaultScreenshotDelivery = async ({ + entrypoint, + environment, + endpoint, + targetId, + screenshot, +}) => { + const workspace = await mkdtemp(join(tmpdir(), "rea-screenshot-delivery-")); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env: environment, + stderr: "pipe", + }); + const client = new Client({ + name: "browser-screenshot-default-e2e", + version: "1", + }); + try { + await client.connect(transport); + const constrained = await client.callTool( + { + name: "capture_web_screenshot", + arguments: { cdp_endpoint: endpoint, target_id: targetId }, + }, + { timeout: 60000 }, + ); + assert.equal(constrained.isError, true); + assert.equal( + constrained.structuredContent?.error?.details?.resource, + "transport", + ); + const reference = + constrained.structuredContent?.error?.details?.reported_limits + ?.evidence_reference; + assert.equal(reference?.kind, "retained-evidence"); + await client.ping(); + const exported = await client.callTool({ + name: "export_evidence_bundle", + arguments: { path: join(workspace, "screenshot-evidence.json") }, + }); + assert.notEqual(exported.isError, true, JSON.stringify(exported)); + const bundle = JSON.parse( + await readFile(join(workspace, "screenshot-evidence.json"), "utf8"), + ); + const record = bundle.records.find( + (record) => record.evidence_id === reference.evidence_id, + ); + assert.ok( + record, + "Full screenshot Evidence must remain exportable after the delivery constraint", + ); + const retained = webScreenshotSchema.parse( + parseEvidence(record).normalized_result, + ); + assert.deepEqual(retained.artifact, screenshot.artifact); + await client.ping(); + } finally { + try { + await client.close(); + } finally { + try { + await transport.close(); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + } + } +}; diff --git a/scripts/lib/large-result-mcp.mjs b/scripts/lib/large-result-mcp.mjs index d7a4ab30f..2d304110e 100644 --- a/scripts/lib/large-result-mcp.mjs +++ b/scripts/lib/large-result-mcp.mjs @@ -10,7 +10,10 @@ export const withLargeResultMcp = async ( const transport = new StdioClientTransport({ command: process.execPath, args: [entrypoint, "mcp"], - env: environment, + env: { + ...environment, + REA_MCP_MAX_RESPONSE_BYTES: String(maxBufferSize), + }, stderr: "inherit", maxBufferSize, }); diff --git a/skill-src/reverse-engineer-anything/SKILL.md b/skill-src/reverse-engineer-anything/SKILL.md index 629a72731..d1b369060 100644 --- a/skill-src/reverse-engineer-anything/SKILL.md +++ b/skill-src/reverse-engineer-anything/SKILL.md @@ -144,6 +144,13 @@ Start with the default result and use its inline Evidence and graph context. Do not repeat an identical tool call. Make a focused follow-up only when the returned result leaves a specific question unanswered. +If MCP reports `resource_constraint` with `details.resource: "transport"`, use +its reported same-session Evidence reference with a focused application tool, +or call `export_evidence_bundle` to write the complete session to a selected +path. The analysis remains complete in the ledger; avoid repeating it merely +to request the same oversized response. Complete CLI JSON output also streams +without a single MCP frame. + Every conclusion must distinguish observations, inferences, and unknowns. Cite Evidence IDs, preserve limitations and incomplete coverage, and never imply that static analysis observed execution. Runtime requests execute the declared diff --git a/skill-src/reverse-engineer-anything/references/javascript-applications.md b/skill-src/reverse-engineer-anything/references/javascript-applications.md index e19c3cdc8..03868c10c 100644 --- a/skill-src/reverse-engineer-anything/references/javascript-applications.md +++ b/skill-src/reverse-engineer-anything/references/javascript-applications.md @@ -2,10 +2,17 @@ Use `analyze_javascript_application` directly on the operator-supplied ASAR or extracted tree. The complete result, graph, and Evidence context are returned -inline. Source-map contents are part of the static analysis. Start from its +inline when they fit the MCP frame. Source-map contents are part of the static analysis. Start from its findings, coverage, unknowns, and graph context, then make a focused follow-up only when a specific question remains unanswered. +An oversized response reports `resource_constraint` and +`details.resource: "transport"`. Reuse +`details.reported_limits.evidence_reference` with `trace_application_feature` +for the module or feature under investigation. `export_evidence_bundle` writes +the complete canonical session to a caller-selected file. Both workflows keep +the original Evidence and coverage; a broad follow-up can also exceed framing. + BrowserWindow preferences, preload and contextBridge surfaces, IPC registrations, utility processes, and native binding requests are static syntax observations. Only a unique exact literal IPC channel match supports an inferred diff --git a/src/application/EvidenceBundleFiles.ts b/src/application/EvidenceBundleFiles.ts index 6e4c7cda0..57eeb1397 100644 --- a/src/application/EvidenceBundleFiles.ts +++ b/src/application/EvidenceBundleFiles.ts @@ -2,7 +2,6 @@ import { describeEvidenceBundleFailure, describeValidationFailure, parseEvidenceBundle, - serializeEvidenceBundle, type EvidenceBundle, } from "../domain/evidenceBundle.js"; import { @@ -11,7 +10,11 @@ import { } from "../domain/evidenceErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import { parseProcessCapture } from "../domain/process/processCapture.js"; -import { readJsonFile, writeTextFile } from "./JsonFiles.js"; +import { + bufferedJsonParts, + canonicalJsonParts, +} from "../domain/jsonSerialization.js"; +import { readJsonFile, writeTextParts } from "./JsonFiles.js"; type EvidenceReadFailure = EvidenceFileError | EvidenceIntegrityError; type EvidenceWriteFailure = EvidenceFileError | EvidenceIntegrityError; @@ -60,9 +63,9 @@ export const writeEvidenceBundle = async ( EvidenceWriteFailure > > => { - let encoded: string; + let checked: EvidenceBundle; try { - encoded = serializeEvidenceBundle(bundle); + checked = parseEvidenceBundle(bundle); } catch (cause: unknown) { return err( new EvidenceIntegrityError("Evidence bundle validation failed", { @@ -70,5 +73,9 @@ export const writeEvidenceBundle = async ( }), ); } - return writeTextFile(encoded, path, overwrite); + return writeTextParts( + bufferedJsonParts(canonicalJsonParts(checked)), + path, + overwrite, + ); }; diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts index 4fea078a8..079d2c9d1 100644 --- a/src/application/JsonFiles.ts +++ b/src/application/JsonFiles.ts @@ -1,8 +1,15 @@ -import { link, lstat, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { + link, + lstat, + mkdtemp, + open, + readFile, + realpath, + rename, + rm, +} from "node:fs/promises"; import { basename, dirname, resolve } from "node:path"; -import writeFileAtomic from "write-file-atomic"; - import { EvidenceFileError } from "../domain/evidenceErrors.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -52,8 +59,16 @@ export const writeTextFile = async ( overwrite: boolean, ): Promise< Result<{ readonly path: string; readonly bytes: number }, EvidenceFileError> +> => writeTextParts([encoded], path, overwrite); + +/** Publish complete streamed text atomically, keeping only one encoded part in memory. */ +export const writeTextParts = async ( + parts: Iterable, + path: string, + overwrite: boolean, +): Promise< + Result<{ readonly path: string; readonly bytes: number }, EvidenceFileError> > => { - const bytes = Buffer.byteLength(encoded, "utf8"); const requestedPath = resolve(path); try { const canonicalParent = await realpath(dirname(requestedPath)); @@ -72,13 +87,7 @@ export const writeTextFile = async ( new EvidenceFileError("write", "not-file", { path: requestedPath }), ); } - if (overwrite) - await writeFileAtomic(destination, encoded, { - encoding: "utf8", - mode: 0o600, - fsync: true, - }); - else await publishNewFile(destination, encoded); + const bytes = await publishFile(destination, parts, overwrite); return ok({ path: requestedPath, bytes }); } catch (cause: unknown) { if (!overwrite && fileErrorCode(cause) === "EEXIST") @@ -103,22 +112,31 @@ const missingOrIo = (cause: unknown): "missing" | "io" => { return code === "ENOENT" || code === "ENOTDIR" ? "missing" : "io"; }; -const publishNewFile = async ( +const publishFile = async ( destination: string, - encoded: string, -): Promise => { + parts: Iterable, + overwrite: boolean, +): Promise => { const stagingDirectory = await mkdtemp( resolve(dirname(destination), ".rea-write-"), ); try { const staged = resolve(stagingDirectory, "content"); - await writeFileAtomic(staged, encoded, { - encoding: "utf8", - mode: 0o600, - fsync: true, - }); + const file = await open(staged, "wx", 0o600); + let bytes = 0; + try { + for (const part of parts) { + await file.writeFile(part, { encoding: "utf8" }); + bytes += Buffer.byteLength(part, "utf8"); + } + await file.sync(); + } finally { + await file.close(); + } + if (overwrite) await rename(staged, destination); // A hard link publishes complete bytes atomically and cannot replace an existing name. - await link(staged, destination); + else await link(staged, destination); + return bytes; } finally { await rm(stagingDirectory, { recursive: true, force: true }); } diff --git a/src/application/binary/BinarySessionRecords.ts b/src/application/binary/BinarySessionRecords.ts index 581f45b84..ecde344f6 100644 --- a/src/application/binary/BinarySessionRecords.ts +++ b/src/application/binary/BinarySessionRecords.ts @@ -69,10 +69,20 @@ export abstract class BinarySessionRecords { return this.#records.evidenceById(evidenceId); } + /** Borrow immutable investigation Evidence without copying a complete graph. */ + evidenceForAnalysis(evidenceId: string): Evidence | undefined { + return this.#records.evidenceForAnalysis(evidenceId); + } + exportEvidenceBundle(): EvidenceBundle { return this.#records.exportEvidenceBundle(); } + /** Borrow a sealed bundle for complete serialization without cloning retained graphs. */ + evidenceBundleForSerialization(): EvidenceBundle { + return this.#records.evidenceBundleForSerialization(); + } + importEvidenceBundle( bundle: unknown, ): Result { diff --git a/src/application/investigation/EvidenceLedger.test.ts b/src/application/investigation/EvidenceLedger.test.ts index 2eeb94616..c65c5e181 100644 --- a/src/application/investigation/EvidenceLedger.test.ts +++ b/src/application/investigation/EvidenceLedger.test.ts @@ -116,6 +116,36 @@ describe("evidence ledger recording", () => { ledger.clear(); expect(ledger.export().records).toEqual([]); }); + + it("protects borrowed snapshots while preserving detached mutable public reads", () => { + const evidence = createEvidence(TARGET, PROVIDER, { + operation: "health", + parameters: {}, + result: { nested: { value: "original" } }, + }); + const ledger = new EvidenceLedger(); + expect(ledger.record(evidence).ok).toBe(true); + const borrowed = ledger.forAnalysis(evidence.evidence_id); + if (borrowed === undefined) throw new Error("Missing recorded Evidence"); + expect(Object.isFrozen(borrowed)).toBe(true); + expect(Object.isFrozen(borrowed.normalized_result)).toBe(true); + expect(Reflect.set(borrowed, "operation", "forged")).toBe(false); + expect(ledger.forAnalysis(evidence.evidence_id)).toBe(borrowed); + expect(ledger.record(borrowed)).toEqual({ ok: true, value: "duplicate" }); + const detached = ledger.get(evidence.evidence_id); + if (detached === undefined) throw new Error("Missing detached Evidence"); + expect(Reflect.set(detached, "operation", "forged")).toBe(true); + expect(ledger.record(detached).ok).toBe(false); + expect(ledger.forAnalysis(evidence.evidence_id)).toBe(borrowed); + const snapshot = ledger.forSerialization(); + expect(snapshot).toEqual(ledger.export()); + expect(snapshot.records[0]).toBe(borrowed); + expect(Object.isFrozen(snapshot.records)).toBe(true); + ledger.clear(); + expect(ledger.forAnalysis(evidence.evidence_id)).toBeUndefined(); + expect(ledger.forSerialization().records).toEqual([]); + expect(snapshot.records).toEqual([evidence]); + }); }); describe("evidence ledger metadata imports", () => { diff --git a/src/application/investigation/EvidenceLedger.ts b/src/application/investigation/EvidenceLedger.ts index 60fcc0ebd..6a2900c6f 100644 --- a/src/application/investigation/EvidenceLedger.ts +++ b/src/application/investigation/EvidenceLedger.ts @@ -2,6 +2,7 @@ import canonicalize from "canonicalize"; import { createEvidenceBundle, + createImmutableEvidenceBundle, describeEvidenceBundleFailure, parseEvidenceBundle, validateResidualUnknownAddition, @@ -9,7 +10,11 @@ import { } from "../../domain/evidenceBundle.js"; import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import { UnknownRegistryError } from "../../domain/unknownRegistryError.js"; -import { parseEvidence, type Evidence } from "../../domain/evidence.js"; +import { + immutableEvidence, + parseEvidence, + type Evidence, +} from "../../domain/evidence.js"; import { createResidualUnknown, updateResidualUnknown, @@ -42,7 +47,7 @@ export class EvidenceLedger { record(input: Evidence): RecordResult { let evidence: Evidence; try { - evidence = parseEvidence(input); + evidence = immutableEvidence(input); } catch (cause: unknown) { return err( new EvidenceIntegrityError("Evidence record validation failed", { @@ -52,6 +57,10 @@ export class EvidenceLedger { } const existing = this.#records.get(evidence.evidence_id); if (existing !== undefined) { + // A borrowed record is already authenticated and deeply immutable. Its + // identity cannot change between reads, so comparing document-sized + // canonical strings again would add no integrity check. + if (existing === evidence) return ok("duplicate"); return recordsAgree(existing, evidence) ? ok("duplicate") : err(new EvidenceIntegrityError("Conflicting evidence record")); @@ -97,8 +106,9 @@ export class EvidenceLedger { additions.set(evidence.evidence_id, evidence); } for (const [id, evidence] of replacements) - this.#records.set(id, evidence); - for (const [id, evidence] of additions) this.#records.set(id, evidence); + this.#records.set(id, immutableEvidence(evidence)); + for (const [id, evidence] of additions) + this.#records.set(id, immutableEvidence(evidence)); return ok({ recordsAdded: additions.size, unknownsAdded: 0, @@ -165,6 +175,14 @@ export class EvidenceLedger { ); } + /** Borrow a sealed canonical bundle for serialization without copying its records. */ + forSerialization(): EvidenceBundle { + return createImmutableEvidenceBundle( + [...this.#records.values()], + [...this.#unknownRevisions.values()], + ); + } + /** Check whether one immutable Evidence ID is present in this ledger. */ has(evidenceId: string): boolean { return this.#records.has(evidenceId); @@ -176,6 +194,11 @@ export class EvidenceLedger { return evidence === undefined ? undefined : structuredClone(evidence); } + /** Borrow authenticated immutable Evidence for read-only analysis without a full clone. */ + forAnalysis(evidenceId: string): Evidence | undefined { + return this.#records.get(evidenceId); + } + /** Create one approved unknown and atomically append its mutation evidence. */ recordUnknown( input: RecordUnknownInput, @@ -380,7 +403,8 @@ export class EvidenceLedger { } catch (cause: unknown) { return err(new UnknownRegistryError("integrity", { cause })); } - for (const [id, evidence] of additions) this.#records.set(id, evidence); + for (const [id, evidence] of additions) + this.#records.set(id, immutableEvidence(evidence)); if (unknown !== undefined && unknownKey !== undefined) { this.#unknownRevisions.set(unknownKey, unknown); this.#unknownHeads.set(unknown.unknown_id, unknown); @@ -407,7 +431,8 @@ export class EvidenceLedger { unknowns: ReadonlyMap, ): void { this.#records.clear(); - for (const [id, evidence] of records) this.#records.set(id, evidence); + for (const [id, evidence] of records) + this.#records.set(id, immutableEvidence(evidence)); this.#unknownRevisions.clear(); this.#unknownHeads.clear(); for (const [key, unknown] of unknowns) { diff --git a/src/application/investigation/InvestigationRecordPort.ts b/src/application/investigation/InvestigationRecordPort.ts index a99542b5c..aacf8fa26 100644 --- a/src/application/investigation/InvestigationRecordPort.ts +++ b/src/application/investigation/InvestigationRecordPort.ts @@ -15,6 +15,10 @@ import type { Result } from "../../domain/result.js"; export interface EvidenceReader { hasEvidence(evidenceId: string): boolean; evidenceById(evidenceId: string): Evidence | undefined; + /** Optional immutable analysis read; older adapters can keep detached reads. */ + evidenceForAnalysis?(evidenceId: string): Evidence | undefined; + /** Optional immutable bundle snapshot for complete serialization without graph copies. */ + evidenceBundleForSerialization?(): EvidenceBundle; exportEvidenceBundle(): EvidenceBundle; } diff --git a/src/application/investigation/InvestigationRecords.ts b/src/application/investigation/InvestigationRecords.ts index 9b612f11c..b8a1dea99 100644 --- a/src/application/investigation/InvestigationRecords.ts +++ b/src/application/investigation/InvestigationRecords.ts @@ -51,11 +51,21 @@ export class InvestigationRecords return this.#evidence.get(evidenceId); } + /** Borrow an immutable authenticated record for a read-only application workflow. */ + evidenceForAnalysis(evidenceId: string): Evidence | undefined { + return this.#evidence.forAnalysis(evidenceId); + } + /** Export a detached deterministic bundle of Evidence and Unknown revisions. */ exportEvidenceBundle(): EvidenceBundle { return this.#evidence.export(); } + /** Borrow a sealed bundle snapshot for complete JSON serialization. */ + evidenceBundleForSerialization(): EvidenceBundle { + return this.#evidence.forSerialization(); + } + /** Atomically merge a bundle and report changes for the owner's post-commit observers. */ mergeEvidenceBundle( bundle: unknown, diff --git a/src/application/javascript/JavaScriptApplicationEvidence.ts b/src/application/javascript/JavaScriptApplicationEvidence.ts index c291f7673..53d7087e0 100644 --- a/src/application/javascript/JavaScriptApplicationEvidence.ts +++ b/src/application/javascript/JavaScriptApplicationEvidence.ts @@ -8,6 +8,18 @@ import type { JavaScriptApplicationAnalysisResult, } from "../../domain/javascript/javascriptApplicationAnalysis.js"; import { JAVASCRIPT_APPLICATION_PROVIDER } from "../InvestigationProviders.js"; +import { freezeJsonSnapshot } from "../../domain/immutableJson.js"; +import { rememberOwnedApplicationGraphEvidence } from "./JavaScriptApplicationEvidenceGraph.js"; + +/** Transfer an already validated, application-owned result into immutable analysis Evidence. */ +export const createOwnedJavaScriptApplicationEvidence = ( + input: AnalyzeJavaScriptApplicationInput, + result: JavaScriptApplicationAnalysisResult, +): Evidence => + rememberOwnedApplicationGraphEvidence( + createJavaScriptApplicationEvidence(input, freezeJsonSnapshot(result)), + result, + ); /** Create Evidence for one deterministic local JavaScript application graph. */ export const createJavaScriptApplicationEvidence = ( diff --git a/src/application/javascript/JavaScriptApplicationEvidenceGraph.test.ts b/src/application/javascript/JavaScriptApplicationEvidenceGraph.test.ts new file mode 100644 index 000000000..1db99dcbd --- /dev/null +++ b/src/application/javascript/JavaScriptApplicationEvidenceGraph.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; + +import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; +import { parseEvidence } from "../../domain/evidence.js"; +import { + analyzeJavaScriptApplicationInputSchema, + javascriptApplicationAnalysisResultSchema, +} from "../../domain/javascript/javascriptApplicationAnalysis.js"; +import { + createJavaScriptApplicationEvidence, + createOwnedJavaScriptApplicationEvidence, +} from "./JavaScriptApplicationEvidence.js"; +import { + parseApplicationGraphEvidence, + rememberOwnedApplicationGraphEvidence, +} from "./JavaScriptApplicationEvidenceGraph.js"; + +describe("owned application graph Evidence", () => { + it("reuses the exact authenticated graph and preserves ordinary wire identity", () => { + const result = javascriptApplicationAnalysisResultSchema.parse( + JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result, + ); + const input = analyzeJavaScriptApplicationInputSchema.parse({ + input_path: result.input_path, + ...JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.parameters, + }); + const ordinary = createJavaScriptApplicationEvidence(input, result); + const owned = createOwnedJavaScriptApplicationEvidence(input, result); + expect(owned.evidence_id).toBe(ordinary.evidence_id); + expect(JSON.stringify(owned)).toBe(JSON.stringify(ordinary)); + const parsed = parseApplicationGraphEvidence(owned); + expect(parsed.evidence).toBe(owned); + expect(parsed.graph).toBe(result.graph); + expect(parsed.semanticGraph).toBe(result.semantic_graph); + expect(Object.isFrozen(parsed.graph.nodes)).toBe(true); + expect(parseApplicationGraphEvidence(owned)).toBe(parsed); + const wire: unknown = JSON.parse(JSON.stringify(owned)); + expect(parseApplicationGraphEvidence(wire)).toEqual(parsed); + expect(parseApplicationGraphEvidence(wire).graph).not.toBe(parsed.graph); + expect(() => + rememberOwnedApplicationGraphEvidence(ordinary, result), + ).toThrow("exact validated immutable result"); + const tampered = { ...owned, normalized_result: {} }; + expect(() => parseEvidence(Object.freeze(tampered))).toThrow(); + expect(() => parseApplicationGraphEvidence(tampered)).toThrow(); + }); +}); diff --git a/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts b/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts index 8d9106359..459c226ac 100644 --- a/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts +++ b/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts @@ -3,7 +3,11 @@ import { JAVASCRIPT_RUNTIME_RECONCILIATION_PROVIDER, MANAGED_WORKFLOW_PROVIDER, } from "../InvestigationProviders.js"; -import { parseEvidence, type Evidence } from "../../domain/evidence.js"; +import { + isImmutableEvidence, + parseEvidence, + type Evidence, +} from "../../domain/evidence.js"; import { analyzeJavaScriptApplicationInputSchema, javascriptApplicationAnalysisResultSchema, @@ -26,11 +30,39 @@ export interface ApplicationGraphEvidenceSource { readonly semanticGraph: JavaScriptSemanticGraph | null; } +const ownedApplicationSources = new WeakMap< + Evidence, + ApplicationGraphEvidenceSource +>(); + +/** Retain the producer-validated meaning of its exact immutable result snapshot. */ +export const rememberOwnedApplicationGraphEvidence = ( + evidence: Evidence, + result: JavaScriptApplicationAnalysisResult, +): Evidence => { + if ( + !isImmutableEvidence(evidence) || + evidence.normalized_result !== result || + evidence.operation !== "analyze_javascript_application" || + !providerMatches(evidence, JAVASCRIPT_APPLICATION_PROVIDER) + ) + throw new TypeError( + "Owned application Evidence must bind its exact validated immutable result", + ); + ownedApplicationSources.set( + evidence, + Object.freeze(staticApplicationSourceForResult(evidence, result)), + ); + return evidence; +}; + /** Parse and authenticate one REA-produced JavaScript Application Graph Evidence. */ export const parseApplicationGraphEvidence = ( input: unknown, ): ApplicationGraphEvidenceSource => { const evidence = parseEvidence(input); + const owned = ownedApplicationSources.get(evidence); + if (owned !== undefined) return owned; if ( evidence.operation === "analyze_javascript_application" && evidence.predicate_type === "rea.javascript-application-analysis" && @@ -56,6 +88,16 @@ export const parseApplicationGraphEvidence = ( const staticApplicationSource = ( evidence: Evidence, +): ApplicationGraphEvidenceSource => { + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + return staticApplicationSourceForResult(evidence, result); +}; + +const staticApplicationSourceForResult = ( + evidence: Evidence, + result: JavaScriptApplicationAnalysisResult, ): ApplicationGraphEvidenceSource => { if ( evidence.authority !== "shipped-artifact" || @@ -64,9 +106,6 @@ const staticApplicationSource = ( throw new TypeError( "JavaScript application Evidence authority or confidence is invalid", ); - const result = javascriptApplicationAnalysisResultSchema.parse( - evidence.normalized_result, - ); if (evidence.predicate_type !== "rea.javascript-application-analysis") throw new TypeError( "JavaScript application Evidence predicate does not match its result shape", diff --git a/src/application/javascript/JavaScriptApplicationService.ts b/src/application/javascript/JavaScriptApplicationService.ts index b1b180383..76583e476 100644 --- a/src/application/javascript/JavaScriptApplicationService.ts +++ b/src/application/javascript/JavaScriptApplicationService.ts @@ -16,7 +16,7 @@ import { projectInputIssues } from "../../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../../domain/result.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import type { ExecutionOptions } from "../AnalysisProvider.js"; -import { createJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js"; +import { createOwnedJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js"; import { reconstructJavaScriptArtifact } from "./JavaScriptArtifactReconstruction.js"; import { JAVASCRIPT_APPLICATION_PROVIDER } from "../InvestigationProviders.js"; @@ -77,7 +77,7 @@ export const analyzeJavaScriptApplicationValidated = async ( total: 1, message: "Creating and hashing application analysis Evidence", }); - const evidence = createJavaScriptApplicationEvidence(input, result); + const evidence = createOwnedJavaScriptApplicationEvidence(input, result); await options.progress?.report({ phase: "analyze_javascript_application", completed: 1, diff --git a/src/cli/jsonOutput.ts b/src/cli/jsonOutput.ts index cc401ee32..93df9abd3 100644 --- a/src/cli/jsonOutput.ts +++ b/src/cli/jsonOutput.ts @@ -1,5 +1,7 @@ import type { Writable } from "node:stream"; +import { bufferedJsonParts, jsonParts } from "../domain/jsonSerialization.js"; + type JsonOutputFormat = "json" | "jsonl"; /** Write ordinary JSON without constructing a document-sized string. */ @@ -8,7 +10,7 @@ export const writeJsonOutput = async ( destination: Writable, format: JsonOutputFormat = "json", ): Promise => { - for (const part of bufferedParts(documentParts(value, format))) + for (const part of bufferedJsonParts(documentParts(value, format))) await writePart(destination, part); }; @@ -39,115 +41,11 @@ function* documentParts( ): Generator { if (format === "jsonl" && Array.isArray(value)) { for (const item of value) { - yield* encodeValue(item, false, 0, new Set()); + yield* jsonParts(item); yield "\n"; } return; } - yield* encodeValue(value, format === "json", 0, new Set()); + yield* jsonParts(value, format === "json"); yield "\n"; } - -function* bufferedParts(parts: Iterable): Generator { - let buffer = ""; - for (const part of parts) { - if (buffer.length + part.length > 64 * 1024 && buffer !== "") { - yield buffer; - buffer = ""; - } - buffer += part; - } - if (buffer !== "") yield buffer; -} - -function* encodeString(value: string): Generator { - if (value.length <= 8192) { - yield JSON.stringify(value); - return; - } - yield '"'; - for (let start = 0; start < value.length;) { - let end = Math.min(start + 8192, value.length); - const last = value.charCodeAt(end - 1); - const next = value.charCodeAt(end); - // Keep a surrogate pair in the same JSON.stringify and UTF-8 write. - if ( - end < value.length && - last >= 0xd800 && - last <= 0xdbff && - next >= 0xdc00 && - next <= 0xdfff - ) - end -= 1; - yield JSON.stringify(value.slice(start, end)).slice(1, -1); - start = end; - } - yield '"'; -} - -function* encodeValue( - value: unknown, - pretty: boolean, - depth: number, - ancestors: Set, -): Generator { - if (typeof value === "string") { - yield* encodeString(value); - return; - } - if (value === null || typeof value === "boolean") { - yield value === null ? "null" : String(value); - return; - } - if (typeof value === "number" && Number.isFinite(value)) { - yield String(value); - return; - } - if (typeof value !== "object" || value === null) - throw new TypeError("CLI JSON output requires ordinary JSON values"); - if (ancestors.has(value)) - throw new TypeError("CLI JSON output contains a circular reference"); - if ( - !Array.isArray(value) && - Object.prototype.toString.call(value) !== "[object Object]" - ) - throw new TypeError("CLI JSON output requires ordinary JSON objects"); - ancestors.add(value); - try { - const array = Array.isArray(value); - const keys = array - ? undefined - : Object.keys(value).filter((key) => { - const item: unknown = Reflect.get(value, key); - // Incur's filter can select inherited methods. Match native JSON's - // omission of fields that have no JSON representation. - return ( - item !== undefined && - typeof item !== "function" && - typeof item !== "symbol" - ); - }); - const count = array ? value.length : (keys?.length ?? 0); - yield array ? "[" : "{"; - for (let index = 0; index < count; index += 1) { - if (index > 0) yield ","; - if (pretty) yield `\n${" ".repeat(depth + 1)}`; - let item: unknown; - if (array) { - item = value[index]; - } else { - const key = keys?.[index]; - if (key === undefined) - throw new TypeError("CLI JSON object changed during serialization"); - yield* encodeString(key); - yield pretty ? ": " : ":"; - item = Reflect.get(value, key); - } - yield* encodeValue(item, pretty, depth + 1, ancestors); - } - if (pretty && count > 0) yield `\n${" ".repeat(depth)}`; - yield array ? "]" : "}"; - } finally { - ancestors.delete(value); - } -} diff --git a/src/config/environment.ts b/src/config/environment.ts index e292f7f3f..aebcd5be5 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -4,9 +4,11 @@ import { isAbsolute } from "node:path"; import { ConfigurationError } from "../domain/configurationErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisProviderSelectorSchema } from "../contracts/providerSelection.js"; +import { mcpResponseBudgetSchema } from "./mcpResponseBudget.js"; const environmentSchema = z.object({ REA_ANALYSIS_PROVIDER: analysisProviderSelectorSchema.default("auto"), + REA_MCP_MAX_RESPONSE_BYTES: mcpResponseBudgetSchema.optional(), REA_IDA_MCP_CONFIG: z .string() .min(1) diff --git a/src/config/mcpResponseBudget.ts b/src/config/mcpResponseBudget.ts new file mode 100644 index 000000000..01d69db97 --- /dev/null +++ b/src/config/mcpResponseBudget.ts @@ -0,0 +1,31 @@ +import { z } from "zod"; + +import { ConfigurationError } from "../domain/configurationErrors.js"; +import { err, ok, type Result } from "../domain/result.js"; + +// This override supports clients with larger frames. Keep the configuration +// parser independent of the SDK's eager server imports; conformance verifies +// this floor against the pinned SDK's actual default receive-buffer size. +const MINIMUM_RESPONSE_BYTES = 10 * 1024 * 1024; + +/** Optional stdio response budget, including room for the protocol envelope. */ +export const mcpResponseBudgetSchema = z + .string() + .regex(/^[1-9]\d*$/u) + .transform(Number) + .pipe(z.number().int().safe().min(MINIMUM_RESPONSE_BYTES)); + +/** Parse only the MCP-specific budget without loading unrelated provider settings. */ +export const parseMcpResponseBudget = ( + value: string | undefined, +): Result => { + if (value === undefined) return ok(undefined); + const parsed = mcpResponseBudgetSchema.safeParse(value); + return parsed.success + ? ok(parsed.data) + : err( + new ConfigurationError( + `REA_MCP_MAX_RESPONSE_BYTES must be a decimal safe integer at least ${String(MINIMUM_RESPONSE_BYTES)} bytes, the default stdio receive-buffer size.`, + ), + ); +}; diff --git a/src/domain/analysisErrorCore.ts b/src/domain/analysisErrorCore.ts index 03835a2b3..47f0e0867 100644 --- a/src/domain/analysisErrorCore.ts +++ b/src/domain/analysisErrorCore.ts @@ -17,13 +17,13 @@ export class AnalysisUnsupportedTargetError extends AnalysisError { } } -/** Provider reported a resource failure, distinct from malformed input or unsupported coverage. */ +/** An analysis or transport resource failure, distinct from malformed input or unsupported coverage. */ export class AnalysisResourceConstraintError extends AnalysisError { readonly _tag = "AnalysisResourceConstraintError"; constructor( readonly operation: string, - readonly resource: "memory" | "cpu" | "file-size", + readonly resource: "memory" | "cpu" | "file-size" | "transport", readonly reason: string, readonly reportedLimits: Readonly> | null, options?: AnalysisErrorOptions, diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 76cd82b8f..f8684605a 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -37,11 +37,13 @@ export const analysisErrorRemediationAction = ( if (error instanceof AnalysisUnsupportedTargetError) return "Select a target supported by this operation or choose an operation supporting the reported target format."; if (error instanceof AnalysisResourceConstraintError) - return error.resource === "cpu" - ? "Review the reported worker CPU limits and observed signal. Retry with sufficient CPU time or a smaller artifact; REA retains tighter inherited limits." - : error.resource === "file-size" - ? "Review the reported worker file-size limits and write failure. Retry with a sufficient file-size allowance for the evidence reply; REA retains tighter inherited limits." - : "Review the reported worker memory limits and available host memory. Retry with sufficient memory or a smaller artifact; REA retains tighter inherited limits."; + return error.resource === "transport" + ? "Use the reported Evidence reference with a focused analysis tool when retained in this session, or export the session through export_evidence_bundle to a caller-selected path. Complete CLI JSON output is also available. The connection remains usable." + : error.resource === "cpu" + ? "Review the reported worker CPU limits and observed signal. Retry with sufficient CPU time or a smaller artifact; REA retains tighter inherited limits." + : error.resource === "file-size" + ? "Review the reported worker file-size limits and write failure. Retry with a sufficient file-size allowance for the evidence reply; REA retains tighter inherited limits." + : "Review the reported worker memory limits and available host memory. Retry with sufficient memory or a smaller artifact; REA retains tighter inherited limits."; if (error instanceof HopperTimeoutError) return error.providerState === "busy" ? "Check binary_session.analysis_activity, wait for the active Hopper request to finish, then retry." diff --git a/src/domain/analysisErrorProjection.test.ts b/src/domain/analysisErrorProjection.test.ts index 392b97966..3bf294386 100644 --- a/src/domain/analysisErrorProjection.test.ts +++ b/src/domain/analysisErrorProjection.test.ts @@ -62,6 +62,7 @@ it.each([ ["cpu", { cpu_seconds: 1 }], ["file-size", null], ["file-size", { file_size_bytes: 1024 }], + ["transport", { result_budget_bytes: 10485760 }], ] as const)( "projects reported %s constraints with observed or unknown limits: %j", (resource, limits) => { @@ -89,9 +90,11 @@ it.each([ expect(projected.remediation.action).toContain( resource === "cpu" ? "CPU" - : resource === "file-size" - ? "file-size" - : "memory", + : resource === "transport" + ? "Evidence reference" + : resource === "file-size" + ? "file-size" + : "memory", ); expect(projected.remediation.action).not.toContain("doctor"); expect(analysisErrorProjectionSchema.safeParse(projected).success).toBe( diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts index 95dabcc1a..85bdb33aa 100644 --- a/src/domain/evidence.test.ts +++ b/src/domain/evidence.test.ts @@ -3,9 +3,15 @@ import { describe, expect } from "vitest"; import { createAnalysisProfile } from "./analysisProfile.js"; import type { BinaryTarget } from "./binaryTarget.js"; -import { createEvidence, evidenceSchema, parseEvidence } from "./evidence.js"; -import { MAX_JSON_DEPTH } from "./jsonValue.js"; +import { + createEvidence, + evidenceRecordSchema, + evidenceSchema, + parseEvidence, +} from "./evidence.js"; +import { MAX_JSON_DEPTH, type JsonValue } from "./jsonValue.js"; import { createEvidenceBundle } from "./evidenceBundle.js"; +import { freezeJsonSnapshot } from "./immutableJson.js"; const TARGET: BinaryTarget = { path: "/tmp/fixture", @@ -18,6 +24,90 @@ const TARGET: BinaryTarget = { const PROVIDER = { id: "fixture", name: "Fixture provider", version: "1" }; const PROFILE = createAnalysisProfile(PROVIDER, { loader: "default" }); +it("preserves identity and bytes when reusing an owned immutable result", () => { + const value = { nested: { values: ["observed", 1] } }; + const mutable = createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters: {}, + result: value, + }); + const frozen = createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters: {}, + result: freezeJsonSnapshot(value), + }); + expect(JSON.stringify(frozen)).toBe(JSON.stringify(mutable)); + expect(frozen.normalized_result).toBe(value); + expect(Object.isFrozen(frozen)).toBe(true); + expect(parseEvidence(frozen)).toEqual(mutable); + const wire: unknown = JSON.parse(JSON.stringify(frozen)); + expect(parseEvidence(wire)).toEqual(mutable); + expect(() => + parseEvidence({ ...frozen, normalized_result: { changed: true } }), + ).toThrow("semantic identifier"); +}); + +it("applies the JSON depth boundary to owned immutable results", () => { + let result: JsonValue = 1; + for (let index = 0; index <= MAX_JSON_DEPTH; index += 1) + result = { nested: result }; + const observe = () => + createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters: {}, + result, + }); + expect(observe).toThrow("maximum nesting depth"); + freezeJsonSnapshot(result); + expect(observe).toThrow("maximum nesting depth"); +}); + +it("preserves prototype-named own members when reusing an owned result", () => { + const result = { ["__proto__"]: { preserved: 7 }, constructor: "ordinary" }; + const observation = { operation: "inspect", parameters: {}, result }; + const ordinary = createEvidence(TARGET, PROVIDER, observation); + const owned = createEvidence(TARGET, PROVIDER, { + ...observation, + result: freezeJsonSnapshot(result), + }); + expect(owned.normalized_result).toBe(result); + expect(owned).toEqual(ordinary); + expect(parseEvidence(JSON.parse(JSON.stringify(owned)))).toEqual(owned); + expect(Object.getPrototypeOf(owned.normalized_result)).toBe(Object.prototype); + expect(Reflect.get(Object.prototype, "preserved")).toBeUndefined(); +}); + +it("preserves producer envelope admission and profile validation", () => { + const legacy = createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters: {}, + result: { observed: [1, 2, 3] }, + }); + const profiled = createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters: {}, + result: { observed: [1, 2, 3] }, + analysisProfile: PROFILE, + }); + const candidates: unknown[] = [ + legacy, + profiled, + { ...legacy, analysis_profile: undefined }, + { ...legacy, analysis_profile: null }, + { + ...profiled, + analysis_profile: { ...PROFILE, provider: { ...PROVIDER, id: "other" } }, + }, + { ...legacy, unexpected: true }, + ]; + for (const candidate of candidates) { + const canonical = evidenceRecordSchema.safeParse(candidate); + if (canonical.success) + expect(parseEvidence(candidate)).toEqual(canonical.data); + else expect(() => parseEvidence(candidate)).toThrow(); + } +}); + it("snapshots caller-owned payloads before deriving their identity", () => { const result = { nested: { values: ["observed"] } }; const rawResult = { entries: [{ value: "raw" }] }; diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index a634d0c3b..8c6b0fb01 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -10,8 +10,13 @@ import type { BinaryTarget } from "./binaryTarget.js"; import { jsonObjectSchema, jsonValueSchema, + jsonValueValidationIssue, type JsonValue, } from "./jsonValue.js"; +import { + freezeJsonSnapshot, + isImmutableJsonSnapshot, +} from "./immutableJson.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -144,6 +149,14 @@ export const evidenceRecordSchema = z.union([ ]); export type Evidence = z.infer; +const immutableEvidenceSnapshots = new WeakMap(); +const immutableResultSchema = z + .custom(isImmutableJsonSnapshot) + .superRefine((value, context) => { + const issue = jsonValueValidationIssue(value); + if (issue !== undefined) + context.addIssue({ code: "custom", message: issue }); + }); export type EvidenceLocation = z.infer; /** Minimal immutable local artifact identity accepted by Evidence. */ @@ -216,7 +229,19 @@ const computeEvidenceId = (evidence: EvidenceWithoutId): string => /** Parse evidence and reject a syntactically valid but tampered semantic ID. */ export const parseEvidence = (input: unknown): Evidence => { - const evidence = evidenceRecordSchema.parse(input); + const immutable = + typeof input === "object" && input !== null + ? immutableEvidenceSnapshots.get(input) + : undefined; + if (immutable !== undefined) return immutable; + // Select the producer's envelope before traversing its payload. A legacy + // record cannot satisfy the required-profile branch, whose failed parse + // would otherwise clone the entire normalized result before trying legacy. + const schema = + typeof input === "object" && input !== null && "analysis_profile" in input + ? profiledEvidenceSchema + : evidenceBaseSchema; + const evidence = schema.parse(input); const { evidence_id: evidenceId, ...withoutId } = evidence; if (computeEvidenceId(withoutId) !== evidenceId) throw new TypeError( @@ -225,6 +250,20 @@ export const parseEvidence = (input: unknown): Evidence => { return evidence; }; +/** Authenticate and seal a ledger-owned snapshot; external mutable values are copied first. */ +export const immutableEvidence = (input: unknown): Evidence => + rememberImmutableEvidence(parseEvidence(input)); + +/** Recognize authenticated Evidence whose complete reachable JSON data is immutable. */ +export const isImmutableEvidence = (evidence: Evidence): boolean => + immutableEvidenceSnapshots.has(evidence); + +const rememberImmutableEvidence = (evidence: Evidence): Evidence => { + freezeJsonSnapshot(evidence); + immutableEvidenceSnapshots.set(evidence, evidence); + return evidence; +}; + /** Build deterministic Evidence from an immutable artifact subject. */ export const createEvidence = ( target: EvidenceSubjectTarget | BinaryTarget | undefined, @@ -285,7 +324,14 @@ export const createEvidence = ( observation.analysisProfile === undefined ? evidenceBaseSchema : profiledEvidenceSchema; - const normalized = schema.parse({ + const sharedResult = + typeof observation.result === "object" && + observation.result !== null && + isImmutableJsonSnapshot(observation.result); + const selectedSchema = sharedResult + ? schema.safeExtend({ normalized_result: immutableResultSchema }) + : schema; + const normalized = selectedSchema.parse({ ...semantic, evidence_id: `ev_${"0".repeat(64)}`, subject, @@ -293,8 +339,9 @@ export const createEvidence = ( // The envelope has already been parsed into an independent snapshot. Only // its derived identifier changes here; parsing again clones the full payload // and recomputes the same digest while the previous snapshot is still live. - return { + const evidence = { ...normalized, evidence_id: computeEvidenceId(normalized), }; + return sharedResult ? rememberImmutableEvidence(evidence) : evidence; }; diff --git a/src/domain/evidenceBundle.test.ts b/src/domain/evidenceBundle.test.ts index 00e5f7b4f..ff6fc38e2 100644 --- a/src/domain/evidenceBundle.test.ts +++ b/src/domain/evidenceBundle.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { createEvidence } from "./evidence.js"; import { createEvidenceBundle, + createImmutableEvidenceBundle, evidenceBundleForTarget, parseEvidenceBundle, } from "./evidenceBundle.js"; @@ -118,3 +119,26 @@ describe("evidenceBundleForTarget", () => { ); }); }); + +describe("immutable serialization bundles", () => { + it("authenticates sealed records while revalidating detached and malformed input", () => { + const evidence = createEvidence(undefined, provider, { + operation: "health", + parameters: {}, + result: { nested: [true] }, + }); + const bundle = createImmutableEvidenceBundle([evidence]); + expect(parseEvidenceBundle(bundle)).toBe(bundle); + expect(Object.isFrozen(bundle.records[0]?.normalized_result)).toBe(true); + const detached: unknown = JSON.parse(JSON.stringify(bundle)); + expect(parseEvidenceBundle(detached)).toEqual(bundle); + expect(parseEvidenceBundle(detached)).not.toBe(bundle); + const tampered = { + ...bundle, + records: [{ ...evidence, normalized_result: false }], + }; + expect(() => parseEvidenceBundle(Object.freeze(tampered))).toThrow(); + expect(() => createImmutableEvidenceBundle(tampered.records)).toThrow(); + expect(() => createImmutableEvidenceBundle([evidence, evidence])).toThrow(); + }); +}); diff --git a/src/domain/evidenceBundle.ts b/src/domain/evidenceBundle.ts index 807781b66..e6f3a6806 100644 --- a/src/domain/evidenceBundle.ts +++ b/src/domain/evidenceBundle.ts @@ -4,6 +4,7 @@ import canonicalize from "canonicalize"; import { evidenceEnvelopeSchema, evidenceRecordSchema, + immutableEvidence, parseEvidence, providerSchema, type Evidence, @@ -12,6 +13,7 @@ import { residualUnknownSchema, type ResidualUnknown, } from "./residualUnknown.js"; +import { freezeJsonSnapshot } from "./immutableJson.js"; const artifactManifestSchema = evidenceEnvelopeSchema.shape.subject .unwrap() @@ -44,6 +46,29 @@ export const evidenceBundleSchema = z.object({ }); export type EvidenceBundle = z.infer; +const immutableBundles = new WeakMap(); + +/** Build and authenticate a sealed bundle while reusing immutable Evidence records. */ +export const createImmutableEvidenceBundle = ( + records: readonly Evidence[], + unknowns: readonly ResidualUnknown[] = [], +): EvidenceBundle => { + const checkedRecords = records.map(immutableEvidence); + if ( + new Set(checkedRecords.map(({ evidence_id }) => evidence_id)).size !== + checkedRecords.length + ) + throw new TypeError("Evidence bundle contains duplicate record IDs"); + const checkedUnknowns = unknowns.map((unknown) => + residualUnknownSchema.parse(unknown), + ); + validateUnknownGraph(checkedUnknowns, checkedRecords); + const bundle = freezeJsonSnapshot( + createEvidenceBundle(checkedRecords, checkedUnknowns), + ); + immutableBundles.set(bundle, bundle); + return bundle; +}; /** Project records into a deterministic bundle whose order has no semantics. */ export const createEvidenceBundle = ( @@ -212,6 +237,10 @@ export const describeValidationFailure = (cause: unknown): string => { /** Parse records, verify semantic IDs, and reject inconsistent manifests. */ export const parseEvidenceBundle = (input: unknown): EvidenceBundle => { + if (typeof input === "object" && input !== null) { + const owned = immutableBundles.get(input); + if (owned !== undefined) return owned; + } const parsed = evidenceBundleSchema.parse(input); const recordIds = parsed.records.map(({ evidence_id: id }) => id); if (new Set(recordIds).size !== recordIds.length) diff --git a/src/domain/immutableJson.test.ts b/src/domain/immutableJson.test.ts new file mode 100644 index 000000000..16e524a96 --- /dev/null +++ b/src/domain/immutableJson.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import type { JsonValue } from "./jsonValue.js"; +import { + freezeJsonSnapshot, + isImmutableJsonSnapshot, +} from "./immutableJson.js"; + +describe("owned immutable JSON snapshots", () => { + it("seals reachable data and preserves repeated references", () => { + const shared = { values: [1, "observed"] }; + const value = { left: shared, right: shared }; + const encoded = JSON.stringify(value); + expect(freezeJsonSnapshot(value)).toBe(value); + expect(value.left).toBe(value.right); + expect(Object.isFrozen(value)).toBe(true); + expect(Object.isFrozen(shared.values)).toBe(true); + expect(Reflect.set(shared.values, 0, 9)).toBe(false); + expect(JSON.stringify(value)).toBe(encoded); + expect(isImmutableJsonSnapshot(value)).toBe(true); + }); + + it("does not mistake a shallow freeze for validated immutable data", () => { + const value = Object.freeze({ nested: { value: 1 } }); + expect(isImmutableJsonSnapshot(value)).toBe(false); + freezeJsonSnapshot(value); + expect(Object.isFrozen(value.nested)).toBe(true); + expect(isImmutableJsonSnapshot(value)).toBe(true); + }); + + it("composes sealed roots without changing the nested data or authenticating a shallow freeze", () => { + const nested = freezeJsonSnapshot({ values: [1, "observed"] }); + const outer = freezeJsonSnapshot({ nested }); + expect(outer.nested).toBe(nested); + expect(Object.isFrozen(outer)).toBe(true); + expect(Object.isFrozen(nested.values)).toBe(true); + expect(isImmutableJsonSnapshot(outer)).toBe(true); + expect(isImmutableJsonSnapshot(nested)).toBe(true); + const unowned = Object.freeze({ nested: { value: 1 } }); + expect(isImmutableJsonSnapshot(unowned)).toBe(false); + }); + + it("rejects cycles without registering or partially freezing the snapshot", () => { + const value: Record = {}; + value.self = value; + expect(() => freezeJsonSnapshot(value)).toThrow("circular"); + expect(isImmutableJsonSnapshot(value)).toBe(false); + expect(Object.isFrozen(value)).toBe(false); + }); + + it("rejects mutable accessor behavior and sparse arrays", () => { + const accessor = { + get value() { + return 1; + }, + }; + expect(() => freezeJsonSnapshot(accessor)).toThrow("accessors"); + expect(isImmutableJsonSnapshot(accessor)).toBe(false); + expect(() => freezeJsonSnapshot(new Array(2))).toThrow("sparse"); + expect(() => freezeJsonSnapshot({ number: Number.NaN })).toThrow( + "JSON values", + ); + }); +}); diff --git a/src/domain/immutableJson.ts b/src/domain/immutableJson.ts new file mode 100644 index 000000000..4c0579e71 --- /dev/null +++ b/src/domain/immutableJson.ts @@ -0,0 +1,74 @@ +import type { JsonValue } from "./jsonValue.js"; + +const immutableSnapshots = new WeakSet(); + +/** Recognize an owned JSON snapshot sealed by this module, or an immutable scalar. */ +export const isImmutableJsonSnapshot = (value: unknown): value is JsonValue => + value === null || + typeof value === "string" || + typeof value === "boolean" || + (typeof value === "number" && Number.isFinite(value)) || + (typeof value === "object" && immutableSnapshots.has(value)); + +/** Seal an already parsed, owned JSON snapshot for immutable zero-copy reuse. */ +export const freezeJsonSnapshot = (value: T): T => { + const pending: { value: unknown; exiting: boolean }[] = [ + { value, exiting: false }, + ]; + // These sets exist only during traversal, while the owned root already + // retains every child. Strong sets avoid per-node ephemeron GC work. + const visited = new Set(); + const ancestors = new Set(); + const objects: object[] = []; + while (pending.length > 0) { + const entry = pending.pop(); + if (entry === undefined) break; + const item = entry.value; + if (entry.exiting && typeof item === "object" && item !== null) { + ancestors.delete(item); + visited.add(item); + objects.push(item); + continue; + } + if (isImmutableJsonSnapshot(item)) continue; + if (typeof item !== "object" || item === null) + throw new TypeError("Immutable JSON snapshots require JSON values"); + const prototype: unknown = Object.getPrototypeOf(item); + if ( + !Array.isArray(item) && + prototype !== Object.prototype && + prototype !== null + ) + throw new TypeError("Immutable JSON snapshots require ordinary objects"); + if (ancestors.has(item)) + throw new TypeError( + "Immutable JSON snapshots cannot contain circular references", + ); + if (visited.has(item)) continue; + ancestors.add(item); + pending.push({ value: item, exiting: true }); + if (Array.isArray(item)) + for (let index = 0; index < item.length; index += 1) + if (!Object.hasOwn(item, index)) + throw new TypeError( + "Immutable JSON snapshots cannot contain sparse arrays", + ); + for (const key of Object.getOwnPropertyNames(item)) { + const descriptor = Object.getOwnPropertyDescriptor(item, key); + if (descriptor === undefined || !Object.hasOwn(descriptor, "value")) + throw new TypeError( + "Immutable JSON snapshots cannot contain accessors", + ); + const child: unknown = descriptor.value; + pending.push({ value: child, exiting: false }); + } + } + // Register only after every value is checked and every object is frozen. + // An arbitrary Object.freeze call cannot forge this snapshot identity. + for (const item of objects) Object.freeze(item); + // Register the requested root, rather than retaining weak entries for every + // nested object. Sealing an Evidence or bundle skips previously sealed roots. + if (typeof value === "object" && value !== null) + immutableSnapshots.add(value); + return value; +}; diff --git a/src/domain/jsonSerialization.test.ts b/src/domain/jsonSerialization.test.ts new file mode 100644 index 000000000..bca145a95 --- /dev/null +++ b/src/domain/jsonSerialization.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import canonicalize from "canonicalize"; + +import { + bufferedJsonParts, + canonicalJsonParts, + jsonParts, +} from "./jsonSerialization.js"; + +describe("streamed JSON representation", () => { + it.each([ + null, + true, + -0, + [1, false, null, 'quotes: " / slash: \\ / newline: \n'], + { nested: { value: "δΈ­ζ–‡ πŸ˜€ \ud800" }, items: [1, 2] }, + JSON.parse('{"__proto__":{"value":1},"constructor":2,"10":3,"2":4}'), + ])("matches native JSON for ordinary values: %j", (value: unknown) => { + expect([...bufferedJsonParts(jsonParts(value))].join("")).toBe( + JSON.stringify(value), + ); + expect([...jsonParts(value, true)].join("")).toBe( + JSON.stringify(value, null, 2), + ); + }); + + it("preserves surrogate pairs across string and buffer boundaries", () => { + const value = { text: `${"x".repeat(8190)}πŸ˜€${'δΈ­ζ–‡\n"'.repeat(20000)}` }; + const parts = [...bufferedJsonParts(jsonParts(value))]; + expect( + Buffer.concat(parts.map((part) => Buffer.from(part))).toString(), + ).toBe(JSON.stringify(value)); + expect(JSON.parse(parts.join(""))).toEqual(value); + }); + + it("rejects a cycle while allowing repeated references", () => { + const shared = { value: 1 }; + expect([...jsonParts({ left: shared, right: shared })].join("")).toBe( + JSON.stringify({ left: shared, right: shared }), + ); + const cycle: { self?: unknown } = {}; + cycle.self = cycle; + expect(() => [...jsonParts(cycle)]).toThrow("circular reference"); + }); + + it("matches existing canonical bytes for numeric keys, Unicode, and shared values", () => { + const shared = { z: -0, a: [1e-7, 1e30, "ι›ͺπŸ˜€\ud800"] }; + const ownKeys: unknown = JSON.parse( + '{"__proto__":1,"constructor":2,"10":3,"2":4}', + ); + const value = { + z: shared, + a: shared, + keys: ownKeys, + long: 'ι›ͺπŸ˜€"'.repeat(20000), + }; + expect([...bufferedJsonParts(canonicalJsonParts(value))].join("")).toBe( + canonicalize(value), + ); + }); +}); diff --git a/src/domain/jsonSerialization.ts b/src/domain/jsonSerialization.ts new file mode 100644 index 000000000..57462f254 --- /dev/null +++ b/src/domain/jsonSerialization.ts @@ -0,0 +1,118 @@ +/** Encode ordinary JSON as bounded string parts without a document-sized allocation. */ +export function* jsonParts(value: unknown, pretty = false): Generator { + yield* encodeValue(value, pretty, 0, new Set(), false); +} + +/** Encode validated JSON with canonical key ordering without allocating the whole document. */ +export function* canonicalJsonParts(value: unknown): Generator { + yield* encodeValue(value, false, 0, new Set(), true); +} + +/** Buffer JSON parts while preserving complete UTF-16 surrogate pairs. */ +export function* bufferedJsonParts(parts: Iterable): Generator { + let buffer = ""; + for (const part of parts) { + if (buffer.length + part.length > 64 * 1024 && buffer !== "") { + yield buffer; + buffer = ""; + } + buffer += part; + } + if (buffer !== "") yield buffer; +} + +function* encodeString(value: string): Generator { + if (value.length <= 8192) { + yield JSON.stringify(value); + return; + } + yield '"'; + for (let start = 0; start < value.length;) { + let end = Math.min(start + 8192, value.length); + const last = value.charCodeAt(end - 1); + const next = value.charCodeAt(end); + // Keep a surrogate pair in the same JSON.stringify and UTF-8 write. + if ( + end < value.length && + last >= 0xd800 && + last <= 0xdbff && + next >= 0xdc00 && + next <= 0xdfff + ) + end -= 1; + yield JSON.stringify(value.slice(start, end)).slice(1, -1); + start = end; + } + yield '"'; +} + +function* encodeValue( + value: unknown, + pretty: boolean, + depth: number, + ancestors: Set, + canonical: boolean, +): Generator { + if (typeof value === "string") { + yield* encodeString(value); + return; + } + if (value === null || typeof value === "boolean") { + yield value === null ? "null" : String(value); + return; + } + if (typeof value === "number" && Number.isFinite(value)) { + yield String(value); + return; + } + if (typeof value !== "object" || value === null) + throw new TypeError("JSON output requires ordinary JSON values"); + if (ancestors.has(value)) + throw new TypeError("JSON output contains a circular reference"); + if ( + !Array.isArray(value) && + Object.prototype.toString.call(value) !== "[object Object]" + ) + throw new TypeError("JSON output requires ordinary JSON objects"); + ancestors.add(value); + try { + const array = Array.isArray(value); + const keys = array ? undefined : Object.keys(value); + if (canonical) keys?.sort(); + const count = array ? value.length : (keys?.length ?? 0); + let written = 0; + yield array ? "[" : "{"; + for (let index = 0; index < count; index += 1) { + let item: unknown; + let key: string | undefined; + if (array) { + item = value[index]; + } else { + key = keys?.[index]; + if (key === undefined) + throw new TypeError("JSON object changed during serialization"); + item = Reflect.get(value, key); + // Incur's filter can select inherited methods. Match native JSON's + // omission without eagerly reading the rest of a bounded result. + if ( + item === undefined || + typeof item === "function" || + typeof item === "symbol" + ) + continue; + } + if (written > 0) yield ","; + if (pretty) yield `\n${" ".repeat(depth + 1)}`; + if (key !== undefined) { + yield* encodeString(key); + yield pretty ? ": " : ":"; + } + yield* encodeValue(item, pretty, depth + 1, ancestors, canonical); + written += 1; + } + if (pretty && written > 0) yield `\n${" ".repeat(depth)}`; + yield array ? "]" : "}"; + } finally { + ancestors.delete(value); + } +} diff --git a/src/domain/jsonValue.ts b/src/domain/jsonValue.ts index 7b6f6f649..721c22c53 100644 --- a/src/domain/jsonValue.ts +++ b/src/domain/jsonValue.ts @@ -16,7 +16,8 @@ const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); const recordShape = z.record(z.string(), z.unknown()); -const jsonValidationIssue = (root: unknown): string | undefined => { +/** Validate JSON meaning and nesting without cloning a trusted immutable value. */ +export const jsonValueValidationIssue = (root: unknown): string | undefined => { const pending: { value: unknown; depth: number }[] = [ { value: root, depth: 0 }, ]; @@ -59,9 +60,10 @@ const preserveJsonValue = (value: JsonValue): JsonValue => { }; const validatedJsonValue = z.custom( - (value) => jsonValidationIssue(value) === undefined, + (value) => jsonValueValidationIssue(value) === undefined, { - error: (issue) => jsonValidationIssue(issue.input) ?? "Invalid JSON value", + error: (issue) => + jsonValueValidationIssue(issue.input) ?? "Invalid JSON value", }, ); // Keep both stages nonrecursive in public and Standard Schema projections. diff --git a/src/server/createServer.ts b/src/server/createServer.ts index fd71d210c..73d724ffd 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -294,7 +294,9 @@ const createSessionRecorders = ( evidenceById: session === undefined ? undefined - : (evidenceId: string) => session.evidenceById(evidenceId), + : (evidenceId: string) => + session.evidenceForAnalysis?.(evidenceId) ?? + session.evidenceById(evidenceId), activeTarget: session === undefined ? undefined : () => session.activeTarget(), recordEvidence: diff --git a/src/server/registerApplicationTools/helpers.ts b/src/server/registerApplicationTools/helpers.ts index d65f5b966..0a68d97f0 100644 --- a/src/server/registerApplicationTools/helpers.ts +++ b/src/server/registerApplicationTools/helpers.ts @@ -21,7 +21,13 @@ export const recordResult = ( ); if (recorded !== undefined && !recorded.ok) return toCallToolResult(recorded, contract); - return toCallToolResult({ ok: true, value: evidence }, contract); + return toCallToolResult( + { ok: true, value: evidence }, + contract, + recorded === undefined + ? undefined + : { retainedEvidenceId: evidence.evidence_id }, + ); }; const unknownRegistration = ( diff --git a/src/server/registerBrowserTools.ts b/src/server/registerBrowserTools.ts index 498dd098b..55a065a03 100644 --- a/src/server/registerBrowserTools.ts +++ b/src/server/registerBrowserTools.ts @@ -208,5 +208,11 @@ const evidenceResult = ( const recorded = options.recordEvidence?.(evidence); return recorded !== undefined && !recorded.ok ? toCallToolResult(recorded, contract) - : toCallToolResult({ ok: true, value: evidence }, contract); + : toCallToolResult( + { ok: true, value: evidence }, + contract, + recorded === undefined + ? undefined + : { retainedEvidenceId: evidence.evidence_id }, + ); }; diff --git a/src/server/registerElectronTools.ts b/src/server/registerElectronTools.ts index c7766feb7..e338eff4a 100644 --- a/src/server/registerElectronTools.ts +++ b/src/server/registerElectronTools.ts @@ -168,5 +168,11 @@ const evidenceResult = ( const recorded = options.recordEvidence?.(evidence); return recorded !== undefined && !recorded.ok ? toCallToolResult(recorded, contract) - : toCallToolResult({ ok: true, value: evidence }, contract); + : toCallToolResult( + { ok: true, value: evidence }, + contract, + recorded === undefined + ? undefined + : { retainedEvidenceId: evidence.evidence_id }, + ); }; diff --git a/src/server/registerSessionRecordTools.ts b/src/server/registerSessionRecordTools.ts index b7508cd25..8e777a64f 100644 --- a/src/server/registerSessionRecordTools.ts +++ b/src/server/registerSessionRecordTools.ts @@ -43,7 +43,7 @@ const registerExportEvidenceTool = ({ exportContract.name, toolRegistrationOptions(exportContract), async (input) => { - const bundle = session.exportEvidenceBundle(); + const bundle = bundleForSerialization(session); const written = await writeEvidenceBundle( bundle, input.path, @@ -73,7 +73,7 @@ const registerSnapshotEvidenceTool = ({ snapshotContract.name, toolRegistrationOptions(snapshotContract), () => - toCallToolResult(ok(session.exportEvidenceBundle()), snapshotContract), + toCallToolResult(ok(bundleForSerialization(session)), snapshotContract), ); }; @@ -90,9 +90,9 @@ const registerImportEvidenceTool = ({ const loaded = await readEvidenceBundle(path); if (!loaded.ok) return toCallToolResult(loaded, importContract); const retainedUnknownRevisions = new Set( - session - .exportEvidenceBundle() - .unknowns.map((unknown) => unknownRevisionKey(unknown)), + bundleForSerialization(session).unknowns.map((unknown) => + unknownRevisionKey(unknown), + ), ); const imported = session.importEvidenceBundle(loaded.value); return imported.ok @@ -103,7 +103,7 @@ const registerImportEvidenceTool = ({ (unknown) => !retainedUnknownRevisions.has(unknownRevisionKey(unknown)), ).length, - total: session.exportEvidenceBundle().records.length, + total: bundleForSerialization(session).records.length, }), importContract, ) @@ -112,6 +112,9 @@ const registerImportEvidenceTool = ({ ); }; +const bundleForSerialization = (session: BinarySessionPort): EvidenceBundle => + session.evidenceBundleForSerialization?.() ?? session.exportEvidenceBundle(); + const unknownRevisionKey = ( unknown: EvidenceBundle["unknowns"][number], ): string => `${unknown.unknown_id}:${String(unknown.revision)}`; diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index ca2319bdd..6beb1e54d 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -1,10 +1,18 @@ import type { CallToolResult } from "@modelcontextprotocol/server"; +import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import type { ToolContract } from "../contracts/toolContracts.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { type AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { Result } from "../domain/result.js"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { parseMcpResponseBudget } from "../config/mcpResponseBudget.js"; +import { + encodeToolResult, + MCP_RESULT_BUDGET_BYTES, + MCP_RESULT_STRING_LIMIT, +} from "./toolResultEncoding.js"; /** * Serialize an application result as MCP text and structured content. @@ -14,8 +22,11 @@ import type { Result } from "../domain/result.js"; export const toCallToolResult = ( result: Result, contract: ToolContract, + context?: { readonly retainedEvidenceId: string }, ): CallToolResult => - result.ok ? successResult(result.value, contract) : errorResult(result.error); + result.ok + ? successResult(result.value, contract, context) + : errorResult(result.error); const errorResult = (error: AnalysisError): CallToolResult => { const projected = projectAnalysisError(error); @@ -35,21 +46,73 @@ const errorResult = (error: AnalysisError): CallToolResult => { const successResult = ( value: JsonValue, contract: ToolContract, + context: { readonly retainedEvidenceId: string } | undefined, ): CallToolResult => { const candidate = projectEvidence(value) ?? (contract.kind === "session" ? { result: value } : value); + const configured = parseMcpResponseBudget( + process.env.REA_MCP_MAX_RESPONSE_BYTES, + ); + if (!configured.ok) return errorResult(configured.error); + const budget = + configured.value === undefined + ? MCP_RESULT_BUDGET_BYTES + : configured.value - 1024; + const encoded = encodeToolResult(candidate, budget); + if (!encoded.ok) + return errorResult( + new AnalysisResourceConstraintError( + contract.name, + "transport", + encoded.constraint === "string-length" + ? "The complete MCP response exceeds Node's single-string representation limit. Use a focused retained-Evidence query or export to consume the complete analysis." + : "The complete MCP response exceeds the stdio response budget. The analysis result was not truncated; use a focused retained-Evidence query or export to consume it.", + { + boundary: "mcp-response", + default_receive_buffer_bytes: STDIO_DEFAULT_MAX_BUFFER_SIZE, + result_budget_bytes: budget, + max_string_code_units: MCP_RESULT_STRING_LIMIT, + response_bytes_at_least: encoded.bytesAtLeast, + response_code_units_at_least: encoded.codeUnitsAtLeast, + constraint: encoded.constraint, + ...evidenceRecovery(value, context?.retainedEvidenceId), + }, + ), + ); return { content: [ { type: "text", - text: JSON.stringify(candidate), + text: encoded.text, }, ], structuredContent: candidate, }; }; +const evidenceRecovery = ( + value: JsonValue, + retainedEvidenceId: string | undefined, +): Record => { + const projected = projectEvidence(value); + if ( + projected === undefined || + typeof projected !== "object" || + projected === null || + Array.isArray(projected) + ) + return {}; + if (projected.evidence_id !== retainedEvidenceId) + return { evidence_id: projected.evidence_id ?? null }; + return { + evidence_reference: { + kind: "retained-evidence", + evidence_id: projected.evidence_id ?? null, + }, + }; +}; + const projectEvidence = (value: JsonValue): JsonValue | undefined => { if ( typeof value !== "object" || diff --git a/src/server/toolResultEncoding.test.ts b/src/server/toolResultEncoding.test.ts new file mode 100644 index 000000000..367d1bfe9 --- /dev/null +++ b/src/server/toolResultEncoding.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; + +import type { JsonValue } from "../domain/jsonValue.js"; +import { encodeToolResult } from "./toolResultEncoding.js"; + +const projectedResult = (candidate: JsonValue) => ({ + content: [{ type: "text", text: JSON.stringify(candidate) }], + structuredContent: candidate, +}); + +describe("MCP result encoding budget", () => { + it.each([ + { result: { value: 1 } }, + { result: ["δΈ­ζ–‡", "πŸ˜€", "\ud800", '"\\\n'] }, + { result: "x".repeat(8191) + "πŸ˜€" + 'δΈ­ζ–‡\n"'.repeat(20000) }, + ])("counts actual UTF-8 structured and escaped text bytes", (candidate) => { + const encoded = encodeToolResult(candidate); + expect(encoded.ok).toBe(true); + if (!encoded.ok) throw new Error("Expected complete encoding"); + expect(encoded.text).toBe(JSON.stringify(candidate)); + expect(encoded.bytes).toBe( + Buffer.byteLength(JSON.stringify(projectedResult(candidate))), + ); + }); + + it("accounts for repeated Evidence representations", () => { + const normalized = { value: 'δΈ­ζ–‡"'.repeat(30000) }; + const candidate = { + result: normalized, + evidence: { normalized_result: normalized }, + }; + const completeBytes = Buffer.byteLength( + JSON.stringify(projectedResult(candidate)), + ); + expect(encodeToolResult(candidate, completeBytes).ok).toBe(true); + expect(encodeToolResult(candidate, completeBytes - 1)).toMatchObject({ + ok: false, + bytesAtLeast: completeBytes, + }); + }); + + it("stops before reading the remainder of an oversized result", () => { + const candidate: JsonValue = { text: "x".repeat(1000000) }; + Object.defineProperty(candidate, "remainder", { + enumerable: true, + get() { + throw new Error("The remainder must not be read"); + }, + }); + // Only an already-known first property is visited before exhausting the + // budget. The serializer's property discovery must not eagerly read values. + expect(() => encodeToolResult(candidate, 1000)).not.toThrow(); + expect(encodeToolResult(candidate, 1000).ok).toBe(false); + }); +}); diff --git a/src/server/toolResultEncoding.ts b/src/server/toolResultEncoding.ts new file mode 100644 index 000000000..496adb3ef --- /dev/null +++ b/src/server/toolResultEncoding.ts @@ -0,0 +1,55 @@ +import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; +import { constants as bufferConstants } from "node:buffer"; + +import type { JsonValue } from "../domain/jsonValue.js"; +import { bufferedJsonParts, jsonParts } from "../domain/jsonSerialization.js"; + +// Leave room for the SDK's JSON-RPC envelope and protocol metadata. This budget +// follows the pinned client's actual default framing limit, not a result-row cap. +export const MCP_RESULT_BUDGET_BYTES = STDIO_DEFAULT_MAX_BUFFER_SIZE - 1024; +export const MCP_RESULT_STRING_LIMIT = bufferConstants.MAX_STRING_LENGTH - 1024; + +const RESULT_ENVELOPE_BYTES = + Buffer.byteLength( + JSON.stringify({ + content: [{ type: "text", text: "" }], + structuredContent: null, + }), + ) - 4; + +/** Result of encoding the complete repeated MCP representations within a wire budget. */ +export type ToolResultEncoding = + | { readonly ok: true; readonly text: string; readonly bytes: number } + | { + readonly ok: false; + readonly bytesAtLeast: number; + readonly codeUnitsAtLeast: number; + readonly constraint: "receive-buffer" | "string-length"; + }; + +/** Account for structured JSON and escaped text before allocating the complete text. */ +export const encodeToolResult = ( + value: JsonValue, + budgetBytes = MCP_RESULT_BUDGET_BYTES, +): ToolResultEncoding => { + const parts: string[] = []; + let bytes = RESULT_ENVELOPE_BYTES; + let codeUnits = RESULT_ENVELOPE_BYTES; + for (const part of bufferedJsonParts(jsonParts(value))) { + const escapedPart = JSON.stringify(part); + bytes += Buffer.byteLength(part) + Buffer.byteLength(escapedPart) - 2; + codeUnits += part.length + escapedPart.length - 2; + if (bytes > budgetBytes || codeUnits > MCP_RESULT_STRING_LIMIT) + return { + ok: false, + bytesAtLeast: bytes, + codeUnitsAtLeast: codeUnits, + constraint: + codeUnits > MCP_RESULT_STRING_LIMIT + ? "string-length" + : "receive-buffer", + }; + parts.push(part); + } + return { ok: true, text: parts.join(""), bytes }; +}; diff --git a/tests/boundary/filesystem/evidenceBundleFiles.test.ts b/tests/boundary/filesystem/evidenceBundleFiles.test.ts index efc1cd08f..716c65880 100644 --- a/tests/boundary/filesystem/evidenceBundleFiles.test.ts +++ b/tests/boundary/filesystem/evidenceBundleFiles.test.ts @@ -1,4 +1,11 @@ -import { mkdir, readdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { + mkdir, + readdir, + readFile, + stat, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -12,6 +19,7 @@ import { import { compareEvidenceBundlesCommand } from "../../../src/application/EvidenceBundleCommands.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { createEvidence } from "../../../src/domain/evidence.js"; +import { writeTextParts } from "../../../src/application/JsonFiles.js"; import { createEvidenceBundle, serializeEvidenceBundle, @@ -27,6 +35,39 @@ const bundle = (result = true) => ]); describe("evidence bundle publication", () => { + it.each([false, true])( + "never publishes a partial streamed file (overwrite: %s)", + async (overwrite) => { + const root = await createTestTempDirectory("rea-stream-publication-"); + const path = join(root, "bundle.json"); + if (overwrite) await writeFile(path, "original"); + function* brokenParts() { + yield "partial"; + throw new Error("Source stream failed"); + } + expect( + await writeTextParts(brokenParts(), path, overwrite), + ).toMatchObject({ + ok: false, + error: { _tag: "EvidenceFileError", reason: "io" }, + }); + expect(await readdir(root)).toEqual(overwrite ? ["bundle.json"] : []); + if (overwrite) expect(await readFile(path, "utf8")).toBe("original"); + }, + ); + + it("counts complete Unicode bytes and publishes a private file", async () => { + const root = await createTestTempDirectory("rea-stream-bytes-"); + const path = join(root, "bundle.json"); + const parts = ["ι›ͺ", "πŸ˜€", "\n"]; + expect(await writeTextParts(parts, path, false)).toEqual({ + ok: true, + value: { path, bytes: Buffer.byteLength(parts.join("")) }, + }); + expect(await readFile(path, "utf8")).toBe(parts.join("")); + if (process.platform !== "win32") + expect((await stat(path)).mode & 0o777).toBe(0o600); + }); it("allows only one simultaneous export without overwrite approval", async () => { const root = await createTestTempDirectory("rea-evidence-exclusive-"); const path = join(root, "bundle.json"); diff --git a/tests/boundary/mcp/largeToolResult.test.ts b/tests/boundary/mcp/largeToolResult.test.ts new file mode 100644 index 000000000..0bf82e4c0 --- /dev/null +++ b/tests/boundary/mcp/largeToolResult.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { ok } from "../../../src/domain/result.js"; +import { toCallToolResult } from "../../../src/server/toolResult.js"; +import { EvidenceLedger } from "../../../src/application/investigation/EvidenceLedger.js"; + +describe("large complete Evidence MCP delivery", () => { + it("preserves retained content and returns an actionable framing constraint", () => { + const normalized = { + payload: "x".repeat(Math.ceil(STDIO_DEFAULT_MAX_BUFFER_SIZE / 3)), + }; + const evidence = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "analyze_javascript_application", + parameters: {}, + result: normalized, + }, + ); + const ledger = new EvidenceLedger(); + expect(ledger.record(evidence).ok).toBe(true); + const result = toCallToolResult( + ok(evidence), + toolContract("analyze_javascript_application"), + { retainedEvidenceId: evidence.evidence_id }, + ); + expect(result.isError).toBe(true); + expect(result.structuredContent).toMatchObject({ + error: { + code: "resource_constraint", + category: "resource_constraint", + details: { + resource: "transport", + reported_limits: { + boundary: "mcp-response", + constraint: "receive-buffer", + evidence_reference: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + }, + }, + }, + }); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + STDIO_DEFAULT_MAX_BUFFER_SIZE, + ); + expect(result.content).toEqual([ + { type: "text", text: JSON.stringify(result.structuredContent) }, + ]); + const retained = ledger.get(evidence.evidence_id); + expect(parseEvidence(retained)).toEqual(evidence); + expect(retained?.normalized_result).toEqual(normalized); + const detached = toCallToolResult( + ok(evidence), + toolContract("analyze_javascript_application"), + ); + expect(detached.structuredContent).toMatchObject({ + error: { + details: { reported_limits: { evidence_id: evidence.evidence_id } }, + }, + }); + expect(JSON.stringify(detached)).not.toContain("retained-evidence"); + }); +}); diff --git a/tests/conformance/contracts/mcpResponseBudget.test.ts b/tests/conformance/contracts/mcpResponseBudget.test.ts new file mode 100644 index 000000000..206f8b930 --- /dev/null +++ b/tests/conformance/contracts/mcpResponseBudget.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; + +import { parseMcpResponseBudget } from "../../../src/config/mcpResponseBudget.js"; + +describe("MCP response budget configuration", () => { + it("leaves the default policy to the MCP adapter", () => { + expect(parseMcpResponseBudget(undefined)).toEqual({ + ok: true, + value: undefined, + }); + }); + + it("requires the override to accommodate the pinned transport's default frames", () => { + expect( + parseMcpResponseBudget(String(STDIO_DEFAULT_MAX_BUFFER_SIZE)).ok, + ).toBe(true); + expect( + parseMcpResponseBudget(String(STDIO_DEFAULT_MAX_BUFFER_SIZE - 1)).ok, + ).toBe(false); + }); + + it.each(["10485760", "268435456", "1073741824"])( + "accepts an explicit complete-response budget: %s", + (value) => { + expect(parseMcpResponseBudget(value)).toEqual({ + ok: true, + value: Number(value), + }); + }, + ); + + it.each([ + "", + "0", + "-1", + "1024", + "1.5", + "1e8", + " 10485760", + "Infinity", + "9007199254740992", + ])("rejects malformed or unusable budgets: %j", (value) => { + const parsed = parseMcpResponseBudget(value); + expect(parsed.ok).toBe(false); + if (parsed.ok) throw new Error("Expected invalid budget"); + expect(parsed.error.message).toContain("REA_MCP_MAX_RESPONSE_BYTES"); + }); +});