diff --git a/src/domain/jsonSerialization.test.ts b/src/domain/jsonSerialization.test.ts index bca145a95..45caf7898 100644 --- a/src/domain/jsonSerialization.test.ts +++ b/src/domain/jsonSerialization.test.ts @@ -7,6 +7,16 @@ import { jsonParts, } from "./jsonSerialization.js"; +/** Compact and pretty streamed JSON must equal native JSON. */ +const expectNativeJson = (value: unknown): void => { + expect([...bufferedJsonParts(jsonParts(value))].join("")).toBe( + JSON.stringify(value), + ); + expect([...jsonParts(value, true)].join("")).toBe( + JSON.stringify(value, null, 2), + ); +}; + describe("streamed JSON representation", () => { it.each([ null, @@ -15,14 +25,7 @@ describe("streamed JSON representation", () => { [1, false, null, 'quotes: " / slash: \\ / newline: \n'], { nested: { value: "δΈ­ζ–‡ πŸ˜€ \ud800" }, items: [1, 2] }, JSON.parse('{"__proto__":{"value":1},"constructor":2,"10":3,"2":4}'), - ])("matches native JSON for ordinary values: %j", (value: unknown) => { - expect([...bufferedJsonParts(jsonParts(value))].join("")).toBe( - JSON.stringify(value), - ); - expect([...jsonParts(value, true)].join("")).toBe( - JSON.stringify(value, null, 2), - ); - }); + ])("matches native JSON for ordinary values: %j", expectNativeJson); it("preserves surrogate pairs across string and buffer boundaries", () => { const value = { text: `${"x".repeat(8190)}πŸ˜€${'δΈ­ζ–‡\n"'.repeat(20000)}` }; @@ -59,3 +62,55 @@ describe("streamed JSON representation", () => { ); }); }); + +describe("iterative JSON encoding", () => { + it("matches native JSON for long keys, omitted members, and deep nesting", () => { + let deep: unknown = { leaf: [1, "two", null] }; + for (let level = 0; level < 500; level += 1) + deep = level % 2 === 0 ? [deep, level] : { [`k${String(level)}`]: deep }; + const value = { + [`${"k".repeat(9000)}πŸ˜€`]: { nested: ["x".repeat(20000), {}, []] }, + omitted: undefined, + method: () => 1, + symbol: Symbol("ignored"), + deep, + }; + expectNativeJson(value); + }); + + it("rejects values native JSON would not encode as ordinary JSON", () => { + expect(() => [...jsonParts([1, undefined])]).toThrow( + "ordinary JSON values", + ); + expect(() => [...jsonParts({ value: Number.NaN })]).toThrow( + "ordinary JSON values", + ); + expect(() => [...jsonParts({ value: new Date(0) })]).toThrow( + "ordinary JSON objects", + ); + }); + + it("yields bounded parts before reading later members", () => { + const value: Record = { first: "x".repeat(100_000) }; + Object.defineProperty(value, "later", { + enumerable: true, + get() { + throw new Error("later members must not be read yet"); + }, + }); + const parts = jsonParts(value); + const first = parts.next(); + expect(first.done).toBe(false); + expect(first.value?.length).toBeLessThanOrEqual(64 * 1024); + // Many small members are grouped into parts instead of one per token. + const small = Object.fromEntries( + Array.from({ length: 10_000 }, (_, index) => [ + `k${String(index)}`, + index, + ]), + ); + const smallParts = [...jsonParts(small)]; + expect(smallParts.join("")).toBe(JSON.stringify(small)); + expect(smallParts.length).toBeLessThan(20); + }); +}); diff --git a/src/domain/jsonSerialization.ts b/src/domain/jsonSerialization.ts index 57462f254..65330950e 100644 --- a/src/domain/jsonSerialization.ts +++ b/src/domain/jsonSerialization.ts @@ -1,11 +1,11 @@ /** Encode ordinary JSON as bounded string parts without a document-sized allocation. */ export function* jsonParts(value: unknown, pretty = false): Generator { - yield* encodeValue(value, pretty, 0, new Set(), false); + yield* encodeValue(value, pretty, false); } /** Encode validated JSON with canonical key ordering without allocating the whole document. */ export function* canonicalJsonParts(value: unknown): Generator { - yield* encodeValue(value, false, 0, new Set(), true); + yield* encodeValue(value, false, true); } /** Buffer JSON parts while preserving complete UTF-16 surrogate pairs. */ @@ -46,73 +46,143 @@ function* encodeString(value: string): Generator { yield '"'; } -function* encodeValue( +/** Encoded text accumulated before it is yielded as one part. */ +const PART_CHARACTERS = 16 * 1024; + +const indentation: string[] = []; +const indent = (depth: number): string => + (indentation[depth] ??= `\n${" ".repeat(depth)}`); + +interface ContainerFrame { + readonly value: object; + readonly array: boolean; + readonly keys: readonly string[] | undefined; + readonly count: number; + readonly depth: number; + index: number; + written: number; +} + +/** Encoded text of a scalar, or undefined for a container or a long string. */ +const scalarText = (value: unknown): string | undefined => { + if (typeof value === "string") + return value.length <= 8192 ? JSON.stringify(value) : undefined; + if (value === null) return "null"; + if (typeof value === "boolean") return String(value); + if (typeof value === "number" && Number.isFinite(value)) return String(value); + return undefined; +}; + +const openContainer = ( value: unknown, - pretty: boolean, depth: number, ancestors: Set, canonical: boolean, -): Generator { - if (typeof value === "string") { - yield* encodeString(value); - return; - } - if (value === null || typeof value === "boolean") { - yield value === null ? "null" : String(value); - return; - } - if (typeof value === "number" && Number.isFinite(value)) { - yield String(value); - return; - } +): ContainerFrame => { if (typeof value !== "object" || value === null) throw new TypeError("JSON output requires ordinary JSON values"); if (ancestors.has(value)) throw new TypeError("JSON output contains a circular reference"); - if ( - !Array.isArray(value) && - Object.prototype.toString.call(value) !== "[object Object]" - ) + const array = Array.isArray(value); + if (!array && Object.prototype.toString.call(value) !== "[object Object]") throw new TypeError("JSON output requires ordinary JSON objects"); ancestors.add(value); - try { - const array = Array.isArray(value); - const keys = array ? undefined : Object.keys(value); - if (canonical) keys?.sort(); - const count = array ? value.length : (keys?.length ?? 0); - let written = 0; - yield array ? "[" : "{"; - for (let index = 0; index < count; index += 1) { - let item: unknown; - let key: string | undefined; - if (array) { - item = value[index]; - } else { - key = keys?.[index]; - if (key === undefined) - throw new TypeError("JSON object changed during serialization"); - item = Reflect.get(value, key); - // Incur's filter can select inherited methods. Match native JSON's - // omission without eagerly reading the rest of a bounded result. - if ( - item === undefined || - typeof item === "function" || - typeof item === "symbol" - ) - continue; - } - if (written > 0) yield ","; - if (pretty) yield `\n${" ".repeat(depth + 1)}`; - if (key !== undefined) { - yield* encodeString(key); - yield pretty ? ": " : ":"; - } - yield* encodeValue(item, pretty, depth + 1, ancestors, canonical); - written += 1; - } - if (pretty && written > 0) yield `\n${" ".repeat(depth)}`; - yield array ? "]" : "}"; - } finally { - ancestors.delete(value); + const keys = array ? undefined : Object.keys(value); + if (canonical) keys?.sort(); + const count = array ? value.length : (keys?.length ?? 0); + return { value, array, keys, count, depth, index: 0, written: 0 }; +}; + +/** The next written member of a container, skipping values JSON omits. */ +const nextMember = ( + frame: ContainerFrame, +): { readonly key: string | undefined; readonly item: unknown } | undefined => { + while (frame.index < frame.count) { + const index = frame.index; + frame.index += 1; + if (frame.array) + return { key: undefined, item: Reflect.get(frame.value, index) }; + const key = frame.keys?.[index]; + if (key === undefined) + throw new TypeError("JSON object changed during serialization"); + const item: unknown = Reflect.get(frame.value, key); + // Incur's filter can select inherited methods. Match native JSON's + // omission without eagerly reading the rest of a bounded result. + if ( + item !== undefined && + typeof item !== "function" && + typeof item !== "symbol" + ) + return { key, item }; } + return undefined; +}; + +/** + * Encode iteratively into bounded parts. Recursive `yield*` delegation passed + * every token through one generator frame per nesting level, which dominated + * the cost of writing large nested results. + */ +function* encodeValue( + root: unknown, + pretty: boolean, + canonical: boolean, +): Generator { + const ancestors = new Set(); + const stack: ContainerFrame[] = []; + let buffer = ""; + let pending: unknown = root; + let hasPending = true; + while (true) { + if (hasPending) { + hasPending = false; + const text = scalarText(pending); + if (text !== undefined) buffer += text; + else if (typeof pending === "string") { + // A long string is split into surrogate-safe parts of its own. + if (buffer !== "") yield buffer; + buffer = ""; + yield* encodeString(pending); + } else { + const frame = openContainer( + pending, + stack.length, + ancestors, + canonical, + ); + stack.push(frame); + buffer += frame.array ? "[" : "{"; + } + } + if (buffer.length >= PART_CHARACTERS) { + yield buffer; + buffer = ""; + } + const frame = stack.at(-1); + if (frame === undefined) break; + const member = nextMember(frame); + if (member === undefined) { + if (pretty && frame.written > 0) buffer += indent(frame.depth); + buffer += frame.array ? "]" : "}"; + ancestors.delete(frame.value); + stack.pop(); + continue; + } + if (frame.written > 0) buffer += ","; + if (pretty) buffer += indent(frame.depth + 1); + if (member.key !== undefined) { + const key = scalarText(member.key); + if (key !== undefined) buffer += key; + else { + if (buffer !== "") yield buffer; + buffer = ""; + yield* encodeString(member.key); + } + buffer += pretty ? ": " : ":"; + } + frame.written += 1; + pending = member.item; + hasPending = true; + } + if (buffer !== "") yield buffer; }