diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.py index cc24331fe..fa0535c02 100644 --- a/bridge/mitmproxy/capture.py +++ b/bridge/mitmproxy/capture.py @@ -133,8 +133,6 @@ def project_record(state, secrets): for key, item in value.items(): if not isinstance(key, str): raise CaptureFailure("format", "Native record has a non-string dictionary key.", path) - if key == "__proto__": - raise CaptureFailure("unsupported", "Native object contains a __proto__ member that the current JSON schema boundary cannot preserve.", path) if any(secret in key for secret in secrets): # Validate the omitted subtree too, then discard all of its # coordinates rather than inventing a replacement identity. diff --git a/docs/product-catalog.json b/docs/product-catalog.json index d7934eda1..50b8db1a1 100644 --- a/docs/product-catalog.json +++ b/docs/product-catalog.json @@ -713,9 +713,9 @@ "mcp_prompts": 6 }, "digests": { - "tools_sha256": "bbf6d1716e6c40e6214ec639ee55904625e37cb07248a49dd166bc0ed954cad9", + "tools_sha256": "18a0131bdea6a7af17c052bc97d343e1f833e1a35218e0098eba9695e9f821a2", "prompts_sha256": "ef2de8814390b340623c32001610aa2709d90bed51537b6e562ab3abf144ab4d", - "combined_sha256": "adbd48a1533f6d7d1cdfddbe7d7f350205e106e636384537bf078b6daf870f18", + "combined_sha256": "b40971d45fb56386599e5f8e904c21ee491b20ef23b089ceb28750641eb58ae7", "providers_sha256": "954b499c76514890062d2b6c936d73cb3a9c45412d522a081125a325e81cdec5" } } diff --git a/docs/verification/managed-conformance-manifest.json b/docs/verification/managed-conformance-manifest.json index 2a2197a7a..2636b5aaf 100644 --- a/docs/verification/managed-conformance-manifest.json +++ b/docs/verification/managed-conformance-manifest.json @@ -1,5 +1,5 @@ { - "manifest_id": "ecm_bfae29da7d32a1f58ae7f87d6bba618590ba476a9a70fbbd0059fe33a65f3cc0", + "manifest_id": "ecm_c39e03fbab9c465463c5c8f729512a712af644971fa1d9cdfd245d583a2aa2e6", "verifier": { "id": "rea-managed-conformance", "version": "1" @@ -13,7 +13,7 @@ "skill_digests": [ { "skill_id": "reverse-engineer-anything", - "sha256": "d62fe4acdd5797ded5f13ef77987d5444fee6517b00e6eda5378069fb501844d" + "sha256": "7542b5f56f36a31aa3a7e75043933747ecf63b6b51ad10ca375fb03b77aca795" } ], "claims": [ diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 5216c2031..01c2041c6 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -32,9 +32,9 @@ HAR uses unchanged `har-schema@2.0.0` draft-06 schemas, `ajv@8.20.0`, `lossless-json@4.3.1` numeric representations, pinned in the npm lockfile. Only the HAR 1.2 profile is supported. Unknown extensions remain reported evidence. Duplicate object keys, including equal values and escaped spellings, -are rejected before materialization. A `__proto__` member returns an explicit -unsupported-boundary error in both formats: the current shared JSON schema -cannot preserve this member. Original capture files remain unchanged. +are rejected before materialization. Prototype-named members such as +`__proto__` are preserved as ordinary own JSON members without prototype +mutation. Original capture files remain unchanged. The observed mitmproxy 12.2.3 `SaveHar` profile can emit `postData.text: null` for a missing request body. REA preserves that reported null, omitting the optional field only from its validation copy for this exact producer profile; diff --git a/scripts/verify-web-network-captures.mjs b/scripts/verify-web-network-captures.mjs index d5d1e2609..316d811ce 100644 --- a/scripts/verify-web-network-captures.mjs +++ b/scripts/verify-web-network-captures.mjs @@ -260,14 +260,16 @@ try { ); const original = await readFile(path); for (const mode of ["cli", "mcp"]) { - const error = await inspect( - mode, - { capture_path: path, format }, - "unsupported_provider", - ); - assert.ok( - JSON.stringify(error).includes("JSON schema boundary cannot preserve"), - ); + const value = await inspect(mode, { capture_path: path, format }); + const reported = + format === "har" + ? value.container.reported._extension + : value.records[0].reported.metadata; + assert.equal(Object.hasOwn(reported, "__proto__"), true); + assert.deepEqual(reported["__proto__"], { preserved: 7 }); + assert.equal(Object.getPrototypeOf(reported), Object.prototype); + assert.equal(Reflect.get(Object.prototype, "preserved"), undefined); + if (format === "har") assert.equal(reported.constructor, "ordinary"); assert.deepEqual(await readFile(path), original); cases++; } diff --git a/skills/reverse-engineer-anything/SKILL.md b/skills/reverse-engineer-anything/SKILL.md index da7a2c98d..60613a101 100644 --- a/skills/reverse-engineer-anything/SKILL.md +++ b/skills/reverse-engineer-anything/SKILL.md @@ -4,7 +4,7 @@ description: Reverse engineer native, managed, Electron/JavaScript, packaged, fi metadata: version: "33" tool_count: 139 - catalog_digest: "adbd48a1533f6d7d1cdfddbe7d7f350205e106e636384537bf078b6daf870f18" + catalog_digest: "b40971d45fb56386599e5f8e904c21ee491b20ef23b089ceb28750641eb58ae7" --- # REA diff --git a/src/browser/history/HarCapture.test.ts b/src/browser/history/HarCapture.test.ts index 7e525ad5a..4c29504a8 100644 --- a/src/browser/history/HarCapture.test.ts +++ b/src/browser/history/HarCapture.test.ts @@ -161,14 +161,28 @@ it("rejects identical duplicate core fields before schema validation", () => { expect(() => decodeHarCapture(text, [])).toThrow("duplicate"); }); -it("reports an unsupported JSON schema boundary instead of dropping a producer property", () => { +it("preserves prototype-named producer members through the capture boundary", () => { const text = JSON.stringify(historicalHar()).replace( '"cache":{}', '"cache":{},"_extension":{"__proto__":{"preserved":7},"constructor":"ordinary"}', ); - expect(() => decodeHarCapture(text, [])).toThrow( - "JSON schema boundary cannot preserve", - ); + const result = decodeHarCapture(text, []); + const reported = result.records[0]?.reported; + expect(reported).toBeDefined(); + if ( + reported === undefined || + reported === null || + typeof reported !== "object" || + Array.isArray(reported) + ) + throw new Error("No HAR object record was retained"); + const extension = reported["_extension"]; + expect(extension).toEqual({ + ["__proto__"]: { preserved: 7 }, + constructor: "ordinary", + }); + expect(Object.getPrototypeOf(extension)).toBe(Object.prototype); + expect(Reflect.get(Object.prototype, "preserved")).toBeUndefined(); }); it("preserves Unicode, offsets, sizes, duplicate URLs and opaque extensions without inventing bytes", () => { diff --git a/src/browser/history/HarJson.ts b/src/browser/history/HarJson.ts index 93f25e7cd..257630932 100644 --- a/src/browser/history/HarJson.ts +++ b/src/browser/history/HarJson.ts @@ -58,11 +58,9 @@ export const parseHarJson = (text: string): unknown => { }); }, onObjectProperty: (key) => { - if (key === "__proto__") - throw new CaptureFormatError( - "unsupported", - "HAR object contains a __proto__ member that the current JSON schema boundary cannot preserve.", - ); + // Prototype-named members are kept as ordinary own properties: the + // members map keys them without prototype setters, and + // Object.fromEntries materializes them as own data properties. const frame = frames.at(-1); if (frame?.kind !== "object") throw new TypeError( diff --git a/src/contracts/webNetworkCaptureToolContracts.ts b/src/contracts/webNetworkCaptureToolContracts.ts index a9e45d606..6acfa2241 100644 --- a/src/contracts/webNetworkCaptureToolContracts.ts +++ b/src/contracts/webNetworkCaptureToolContracts.ts @@ -12,7 +12,7 @@ export const WEB_NETWORK_CAPTURE_TOOL_CONTRACTS = [ name: "inspect_web_network_capture", ...toolContractMetadata("inspect_web_network_capture"), description: - "Inspect retained local HAR 1.2 or native mitmproxy capture records without contacting recorded URLs. Returns original ordinals, pointers/byte ranges, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate HAR members are invalid; prototype-named __proto__ members are unsupported by the current shared JSON boundary. Uses pinned upstream parsers, owned temporary files and bounded offline processes. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", + "Inspect retained local HAR 1.2 or native mitmproxy capture records without contacting recorded URLs. Returns original ordinals, pointers/byte ranges, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate HAR members are invalid; prototype-named members such as __proto__ are preserved as ordinary own JSON members without prototype mutation. Uses pinned upstream parsers, owned temporary files and bounded offline processes. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", kind: "browser-provider", inputSchema: inspectWebNetworkCaptureInputSchema, outputSchema: evidenceResultOf(webNetworkCaptureSchema), diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts index c5c20a521..95dabcc1a 100644 --- a/src/domain/evidence.test.ts +++ b/src/domain/evidence.test.ts @@ -44,13 +44,27 @@ it("snapshots caller-owned payloads before deriving their identity", () => { }); describe("analysis evidence identity", () => { - it("normalizes prototype-named parameter keys", () => { + it("preserves prototype-named parameter keys and their semantic identity", () => { const evidence = createEvidence(TARGET, PROVIDER, { operation: "health", parameters: Object.fromEntries([["__proto__", false]]), result: true, }); + expect(evidence.parameters).toEqual({ ["__proto__"]: false }); + expect(Object.hasOwn(evidence.parameters, "__proto__")).toBe(true); expect(parseEvidence(evidence)).toEqual(evidence); + const stripped = createEvidence(TARGET, PROVIDER, { + operation: "health", + parameters: {}, + result: true, + }); + expect(stripped.evidence_id).not.toBe(evidence.evidence_id); + expect(() => + parseEvidence({ + ...evidence, + parameters: { ["__proto__"]: true }, + }), + ).toThrow(/semantic identifier/u); }); it.prop([ @@ -280,3 +294,27 @@ describe("evidence parameter depth bound", () => { ).toBe(true); }); }); + +it("preserves prototype-named raw and normalized results with aligned identity", () => { + const result = { ["__proto__"]: { preserved: 7 }, constructor: "ordinary" }; + const evidence = createEvidence(TARGET, PROVIDER, { + operation: "health", + parameters: {}, + result, + rawResult: result, + }); + const parsed = parseEvidence(evidence); + expect(parsed.normalized_result).toEqual(result); + expect(parsed.raw_result).toEqual(result); + expect(Object.getPrototypeOf(parsed.normalized_result)).toBe( + Object.prototype, + ); + expect(Reflect.get(Object.prototype, "preserved")).toBeUndefined(); + const stripped = createEvidence(TARGET, PROVIDER, { + operation: "health", + parameters: {}, + result: { constructor: "ordinary" }, + rawResult: { constructor: "ordinary" }, + }); + expect(stripped.evidence_id).not.toBe(evidence.evidence_id); +}); diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 844c9e488..a634d0c3b 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -7,7 +7,11 @@ import { type AnalysisProfileCommitment, } from "./analysisProfile.js"; import type { BinaryTarget } from "./binaryTarget.js"; -import { jsonValueSchema, type JsonValue } from "./jsonValue.js"; +import { + jsonObjectSchema, + jsonValueSchema, + type JsonValue, +} from "./jsonValue.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -89,7 +93,7 @@ const evidenceBaseSchema = z provider: providerSchema, predicate_type: z.string().min(1), operation: z.string().min(1), - parameters: z.record(z.string(), jsonValueSchema), + parameters: jsonObjectSchema, raw_result: jsonValueSchema.nullable(), normalized_result: jsonValueSchema, confidence: z.enum(["observed", "derived", "inferred"]), diff --git a/src/domain/javascript/javascriptSemanticQuerySchemas.ts b/src/domain/javascript/javascriptSemanticQuerySchemas.ts index 0e9faf4f6..0cd94e280 100644 --- a/src/domain/javascript/javascriptSemanticQuerySchemas.ts +++ b/src/domain/javascript/javascriptSemanticQuerySchemas.ts @@ -12,16 +12,21 @@ import { digestSchema } from "../digests.js"; import { prefixedDigestSchema } from "../digests.js"; const semanticNodeIdSchema = prefixedDigestSchema("jsrg_node"); +const literalValueSchema = jsonValueSchema.refine( + (value) => + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean", + "Literal seeds accept only JSON primitive values", +); +// Zod derivations drop the shared schema's JSON Schema projection override; +// re-apply it so MCP catalog generation keeps projecting `{}`. +literalValueSchema._zod.toJSONSchema = () => ({}); + const literalSeedSchema = z.strictObject({ kind: z.literal("literal"), - value: jsonValueSchema.refine( - (value) => - value === null || - typeof value === "string" || - typeof value === "number" || - typeof value === "boolean", - "Literal seeds accept only JSON primitive values", - ), + value: literalValueSchema, }); /** Authenticated starting points accepted by semantic tracing. */ diff --git a/src/domain/jsonValue.test.ts b/src/domain/jsonValue.test.ts index 48f23a15d..9b5931233 100644 --- a/src/domain/jsonValue.test.ts +++ b/src/domain/jsonValue.test.ts @@ -99,3 +99,34 @@ describe("jsonValueSchema depth bound", () => { ).toBe(true); }); }); + +it("preserves prototype-named members without prototype mutation", () => { + const input: unknown = JSON.parse( + '{"__proto__":{"preserved":7},"constructor":"ordinary","prototype":[1],"nested":{"\\u005f\\u005fproto\\u005f\\u005f":"escaped"},"list":[{"__proto__":null}]}', + ); + const parsed = jsonObjectSchema.parse(input); + expect(parsed).toEqual(input); + expect(Object.getPrototypeOf(parsed)).toBe(Object.prototype); + expect(Object.hasOwn(parsed, "__proto__")).toBe(true); + expect(parsed["__proto__"]).toEqual({ preserved: 7 }); + expect(parsed["constructor"]).toBe("ordinary"); + expect(parsed["prototype"]).toEqual([1]); + expect(parsed["nested"]).toEqual({ ["__proto__"]: "escaped" }); + expect(parsed["list"]).toEqual([{ ["__proto__"]: null }]); + expect(Object.getPrototypeOf(parsed["nested"])).toBe(Object.prototype); + expect(Reflect.get(Object.prototype, "preserved")).toBeUndefined(); + expect(JSON.parse(JSON.stringify(parsed))).toEqual(input); +}); + +it.each([undefined, Number.NaN, Infinity, 1n, () => 0, new Date(0)])( + "rejects non-JSON values in prototype-named members (%s)", + (value) => { + for (const input of [ + { ["__proto__"]: value }, + { nested: [{ ["__proto__"]: value }] }, + ]) { + expect(jsonValueSchema.safeParse(input).success).toBe(false); + expect(jsonObjectSchema.safeParse(input).success).toBe(false); + } + }, +); diff --git a/src/domain/jsonValue.ts b/src/domain/jsonValue.ts index ccfb3238f..7b6f6f649 100644 --- a/src/domain/jsonValue.ts +++ b/src/domain/jsonValue.ts @@ -1,67 +1,87 @@ import { z } from "zod"; -/** - * Maximum nesting depth accepted for JSON values. - * - * Sits well below the point where the recursive `z.json()` core overflows its - * stack (measured between 2000 and 3000 on Node 24) while leaving room for - * legitimately deep target-derived trees such as compiled NIB view hierarchies. - */ +/** Maximum JSON nesting accepted before recursive result cloning. */ export const MAX_JSON_DEPTH = 1000; +/** JSON-safe value shared by domain, application, and adapter boundaries. */ +export type JsonValue = + | string + | number + | boolean + | null + | JsonValue[] + | { [key: string]: JsonValue }; + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); +const recordShape = z.record(z.string(), z.unknown()); -const exceedsMaxDepth = (root: unknown): boolean => { - // Depth counts edges from the root. +const jsonValidationIssue = (root: unknown): string | undefined => { const pending: { value: unknown; depth: number }[] = [ { value: root, depth: 0 }, ]; while (pending.length > 0) { const item = pending.pop(); if (item === undefined) break; - if (item.depth > MAX_JSON_DEPTH) return true; + if (item.depth > MAX_JSON_DEPTH) + return `JSON value exceeds maximum nesting depth of ${MAX_JSON_DEPTH}`; const value = item.value; - if (Array.isArray(value)) { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) + continue; + if (typeof value === "number") { + if (!Number.isFinite(value)) return "JSON numbers must be finite"; + } else if (Array.isArray(value)) { for (const child of value) pending.push({ value: child, depth: item.depth + 1 }); - } else if (isRecord(value)) { - for (const key of Object.keys(value)) - pending.push({ value: value[key], depth: item.depth + 1 }); - } + } else if (isRecord(value) && recordShape.safeParse(value).success) { + // Inspect every own member, including keys Zod's record parser skips. + for (const child of Object.values(value)) + pending.push({ value: child, depth: item.depth + 1 }); + } else return "Invalid JSON value"; } - return false; + return undefined; }; -/** - * JSON-safe value shared by domain, application, and adapter boundaries. - * - * An iterative depth guard runs before the recursive `z.json()` core so - * hostile nested payloads surface as validation issues instead of RangeErrors. - * The guard only refuses over-deep values; ordinary JSON parsing semantics - * (including which members `z.json()` keeps) are unchanged. - */ -const jsonCore = z.json(); -// Every JSON value satisfies `{}`. Advertise that instead of Zod's -// self-referential `$defs` projection, which MCP clients and model APIs that -// reject recursive JSON Schemas refuse. Runtime parsing still admits only JSON. -// The override sits on the core as well as on the pipe because the standard -// JSON Schema projection walks into pipe stages without consulting the outer -// node, while the public toJSONSchema projection stops at the pipe. -jsonCore._zod.toJSONSchema = () => ({}); +const preserveJsonValue = (value: JsonValue): JsonValue => { + if (Array.isArray(value)) return value.map(preserveJsonValue); + if (value !== null && typeof value === "object") + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [ + key, + preserveJsonValue(child), + ]), + ); + return value; +}; + +const validatedJsonValue = z.custom( + (value) => jsonValidationIssue(value) === undefined, + { + error: (issue) => jsonValidationIssue(issue.input) ?? "Invalid JSON value", + }, +); +// Keep both stages nonrecursive in public and Standard Schema projections. +validatedJsonValue._zod.toJSONSchema = () => ({}); + +/** Parse bounded JSON and retain all own data members without prototype setters. */ export const jsonValueSchema = z .unknown() - .superRefine((value, context) => { - if (exceedsMaxDepth(value)) - context.addIssue({ - code: "custom", - message: `JSON value exceeds maximum nesting depth of ${MAX_JSON_DEPTH}`, - }); - }) - .pipe(jsonCore); + .nonoptional() + .pipe(validatedJsonValue) + .transform(preserveJsonValue); jsonValueSchema._zod.toJSONSchema = () => ({}); -/** JSON object boundary for caller-visible parameter maps. */ -export const jsonObjectSchema = z.record(z.string(), jsonValueSchema); +const objectValidation = z.custom>( + isRecord, + "Expected JSON object", +); +const objectProjection = () => ({ type: "object", additionalProperties: {} }); +objectValidation._zod.toJSONSchema = objectProjection; -export type JsonValue = z.infer; +/** JSON object boundary for caller-visible parameter maps. */ +export const jsonObjectSchema = jsonValueSchema.pipe(objectValidation); +jsonObjectSchema._zod.toJSONSchema = objectProjection; diff --git a/tests/boundary/mcp/webNetworkCaptureMcp.test.ts b/tests/boundary/mcp/webNetworkCaptureMcp.test.ts index 602479ea7..fdbe0753e 100644 --- a/tests/boundary/mcp/webNetworkCaptureMcp.test.ts +++ b/tests/boundary/mcp/webNetworkCaptureMcp.test.ts @@ -5,6 +5,7 @@ import { WebNetworkCaptureService } from "../../../src/application/WebNetworkCap import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { ok } from "../../../src/domain/result.js"; +import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -17,7 +18,7 @@ it("publishes valid SDK schemas and retains historical inline Evidence with the format: "har", decoder: { id: "test-capture-port", name: "test port", version: "1" }, container: { - reported: { creator: "fixture" }, + reported: { creator: "fixture", ["__proto__"]: { preserved: 7 } }, numeric_literals: [], redactions: [], records_pointer: "/log/entries", @@ -72,6 +73,15 @@ it("publishes valid SDK schemas and retains historical inline Evidence with the ); const evidence = parseEvidence(parsed.evidence); expect(parsed.result).toEqual(evidence.normalized_result); + const capture = webNetworkCaptureSchema.parse(parsed.result); + expect(capture.container.reported).toEqual({ + creator: "fixture", + ["__proto__"]: { preserved: 7 }, + }); + expect(Object.getPrototypeOf(capture.container.reported)).toBe( + Object.prototype, + ); + expect(Reflect.get(Object.prototype, "preserved")).toBeUndefined(); expect(session.evidenceById(evidence.evidence_id)).toEqual(evidence); const invalid = await client.callTool({ name: "inspect_web_network_capture",