diff --git a/docs/web-source-location.md b/docs/web-source-location.md index 6e677b08f..f1736ec0f 100644 --- a/docs/web-source-location.md +++ b/docs/web-source-location.md @@ -96,7 +96,18 @@ preflights indexed sections, sources, names, generated rows and mapping segments against a 262144-record decode budget before upstream flattening. Indexed maps are limited to 64 nested levels. These structural limits bound decoded object growth while preserving large embedded `sourcesContent` strings within the -response-byte budget; exceeding a decoder limit returns a `fetch_failed` item. +response-byte budget. It also bounds the expanded inline evidence to 32 MiB +across the inspection, checking resolved source identities and mapping rows +before publishing them, and bounds raw resolved-source identities to 64 MiB +before upstream materializes their arrays. Exceeding either decoder budget +returns a `fetch_failed` item while retaining other maps. Decoded coordinates +and source/name indexes are validated per map; a malformed map is reported as +`invalid` without discarding earlier or later maps. +Collection reserves each requested map's failure context before fetching it. +When the aggregate inline-result budget cannot reserve another item, remaining +requests are left unprocessed and reported as unknown through the top-level +`limitation`; `requested` retains the full count and `processed` counts returned +items. Malformed UTF-8 map responses are reported as invalid rather than repaired with replacement characters. diff --git a/scripts/fixtures/browser-verifier-site.mjs b/scripts/fixtures/browser-verifier-site.mjs index 30a7b36e7..841e7f900 100644 --- a/scripts/fixtures/browser-verifier-site.mjs +++ b/scripts/fixtures/browser-verifier-site.mjs @@ -27,6 +27,50 @@ export async function startBrowserVerifierSite() { response.end(sourceMap()); return; } + if (request.url === "/bad.js.map") { + response.setHeader("content-type", "application/source-map+json"); + response.end( + JSON.stringify({ + version: 3, + names: [], + sources: ["../src/bad.ts"], + mappings: "AAAAD", + }), + ); + return; + } + if (request.url === "/expanded.js.map") { + response.setHeader("content-type", "application/source-map+json"); + response.end( + JSON.stringify({ + version: 3, + names: [], + sources: ["../src/" + "x".repeat(300000) + ".ts"], + mappings: "AAAA" + ",CAAA".repeat(127), + }), + ); + return; + } + if (request.url === "/overflow.js.map") { + response.setHeader("content-type", "application/source-map+json"); + response.end( + JSON.stringify({ + version: 3, + sections: [ + { + offset: { line: 0, column: Number.MAX_SAFE_INTEGER }, + map: { + version: 3, + names: [], + sources: ["overflow.ts"], + mappings: "CAAA", + }, + }, + ], + }), + ); + return; + } if (request.url === "/session-generation") { response.setHeader("content-type", "application/json"); response.end(JSON.stringify({ generation: sessionGeneration })); @@ -37,6 +81,27 @@ export async function startBrowserVerifierSite() { response.end(browserScript(port)); return; } + if (request.url?.startsWith("/bad.js") === true) { + response.setHeader("content-type", "text/javascript"); + response.end( + "export const badMapProbe = true;\n//# sourceMappingURL=/bad.js.map", + ); + return; + } + if (request.url?.startsWith("/expanded.js") === true) { + response.setHeader("content-type", "text/javascript"); + response.end( + "export const expandedMapProbe = true;\n//# sourceMappingURL=/expanded.js.map", + ); + return; + } + if (request.url?.startsWith("/overflow.js") === true) { + response.setHeader("content-type", "text/javascript"); + response.end( + "export const overflowMapProbe = true;\n//# sourceMappingURL=/overflow.js.map", + ); + return; + } if (request.url?.startsWith("/api") === true) { response.setHeader("content-type", "application/json"); response.setHeader( @@ -113,6 +178,9 @@ const browserScript = (port) => ` socket.addEventListener("open", () => socket.send(JSON.stringify({ token: "websocket-secret-value" }))); socket.addEventListener("message", () => socket.close()); }; + void import("/bad.js"); + void import("/expanded.js"); + void import("/overflow.js"); observe(); setInterval(observe, 150); let observedGeneration = 0; diff --git a/scripts/lib/browser-verifier-assertions.mjs b/scripts/lib/browser-verifier-assertions.mjs index 7c26e1d98..514fbf267 100644 --- a/scripts/lib/browser-verifier-assertions.mjs +++ b/scripts/lib/browser-verifier-assertions.mjs @@ -150,7 +150,7 @@ export function assertBundleAnalysis(result) { ) throw new Error("Real Chrome static bundle findings were missing"); if ( - result.observations.source_maps.status !== "included" || + result.observations.source_maps.status !== "partial" || !result.observations.source_maps.items.some( ({ status, original_sources, original_module_edges, mappings }) => status === "included" && @@ -166,6 +166,41 @@ export function assertBundleAnalysis(result) { throw new Error( `Real Chrome approved source-map reconstruction was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, ); + if ( + !result.observations.source_maps.items.some( + ({ declared_url, status, artifact, mappings }) => + declared_url.endsWith("/bad.js.map") && + status === "invalid" && + artifact === null && + mappings.length === 0, + ) + ) + throw new Error( + `Real Chrome malformed-map isolation was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, + ); + if ( + !result.observations.source_maps.items.some( + ({ declared_url, status, limitation, mappings }) => + declared_url.endsWith("/expanded.js.map") && + status === "fetch_failed" && + limitation.includes("Expanded source-map evidence") && + mappings.length === 0, + ) + ) + throw new Error( + `Real Chrome expanded-map omission was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, + ); + if ( + !result.observations.source_maps.items.some( + ({ declared_url, status, mappings }) => + declared_url.endsWith("/overflow.js.map") && + status === "invalid" && + mappings.length === 0, + ) + ) + throw new Error( + `Real Chrome overflowing-map isolation was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, + ); } /** Assert explicitly selected console text and JSON/WebSocket shape capture. */ diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index b1cf21006..d35756de5 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -1,9 +1,12 @@ #!/usr/bin/env node import { spawn } from "node:child_process"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { access, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { CdpBrowserProvider } from "../dist/browser/CdpBrowserProvider.js"; import { waitForBrowserDevtoolsPort } from "../dist/browser/BrowserProcessStartup.js"; @@ -12,7 +15,6 @@ import { inspectWebPageInputSchema, listBrowserTargetsInputSchema, } from "../dist/domain/browserObservation.js"; -import { analyzeWebBundleInputSchema } from "../dist/domain/webBundleAnalysis.js"; import { observeWebSessionInputSchema } from "../dist/domain/browserSession.js"; import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiff.js"; import { @@ -39,6 +41,7 @@ import { verifyBrowserNetworkEvidence } from "./lib/browser-network-e2e.mjs"; import { verifyBrowserScriptExport } from "./lib/browser-script-export-e2e.mjs"; import { verifyBrowserModules } from "./lib/browser-module-e2e.mjs"; import { verifyBrowserDomDestinations } from "./lib/browser-dom-destinations-e2e.mjs"; +import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs"; const REAL_BROWSER_STARTUP_TIMEOUT_MS = 60_000; const SCENARIO_SECRET_VALUE = "rea-browser-verifier-secret"; @@ -160,17 +163,7 @@ try { ); assertSensitiveShapes(withSource.value); - const bundle = await provider.analyzeBundle( - analyzeWebBundleInputSchema.parse({ - cdp_endpoint: endpoint, - allowed_origins: [site.origin], - target_id: target, - observation_ms: 200, - fetch_source_maps: true, - }), - ); - if (!bundle.ok) throw bundle.error; - assertBundleAnalysis(bundle.value); + const bundle = await verifyPublicBundle(endpoint, target, site.origin); const captureDiff = await provider.compareCaptures( compareWebCapturesInputSchema.parse({ @@ -349,8 +342,9 @@ try { networkRequests: observed.value.network.requests.length, consoleEvents: observed.value.console.events.length, websocketEvents: observed.value.network.websocket_events.length, - bundleScripts: bundle.value.capture.scripts_analyzed, - sourceMaps: bundle.value.observations.source_maps.processed, + bundleScripts: bundle.capture.scripts_analyzed, + sourceMaps: bundle.observations.source_maps.processed, + bundle_cli_and_stdio_mcp: true, sessionEvents: session.value.timeline.length, pageScopedTransport: true, screenshotBytes: screenshot.value.artifact.bytes, @@ -383,6 +377,43 @@ process.stdout.write( })}\n`, ); +async function verifyPublicBundle(endpoint, target, origin) { + const evidence = await artifactCliEvidence("analyze-web-bundle", endpoint, [ + target, + "--allowed-origins", + origin, + "--fetch-source-maps", + "--observation-ms", + "200", + ]); + assertBundleAnalysis(evidence.normalized_result); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [fileURLToPath(new URL("./rea.mjs", import.meta.url)), "mcp"], + env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + stderr: "pipe", + }); + const client = new Client({ name: "browser-bundle-e2e", version: "1" }); + try { + await client.connect(transport); + const result = await artifactMcpResult(client, "analyze_web_bundle", { + cdp_endpoint: endpoint, + target_id: target, + allowed_origins: [origin], + fetch_source_maps: true, + observation_ms: 200, + }); + assertBundleAnalysis(result); + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } + return evidence.normalized_result; +} + async function verifyPageScopedTransport(provider, proxy, origin) { const target = await pageTarget(provider, proxy.endpoint); const input = inspectWebPageInputSchema.parse({ diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 3cbed28f0..5464fcc0f 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -1,11 +1,9 @@ import * as t from "@babel/types"; import { - AnyMap, - eachMapping, - type EncodedSourceMap, - type Section, - type SectionedSourceMap, -} from "@jridgewell/trace-mapping"; + decodeValidatedSourceMapLeaves, + isBeforeSourceMapLeafStop, + resolveSourceMapSource, +} from "../javascript/sourceMaps/DecodedSourceMap.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; import { isUrlLikeModuleSpecifier } from "../domain/webBundleAnalyzerAst.js"; @@ -20,10 +18,13 @@ import type { import { webSourceMapsSchema } from "../domain/webBundleAnalysis.js"; import { createWebTextArtifact } from "../domain/webContentArtifact.js"; import { safeParseJson } from "../domain/safeJson.js"; +import { jsonParts } from "../domain/jsonSerialization.js"; import { + inspectSourceMapValue, SourceMapFormatFailure, - validateSourceMapStructure, + type SourceMapLeaf, } from "../javascript/sourceMaps/SourceMapFormat.js"; +import { WEB_SOURCE_MAP_LIMITS } from "../domain/webSourceLocation.js"; export interface WebSourceMapRequest { readonly scriptKey: string; @@ -45,6 +46,9 @@ interface SourceMapFetchHost { // hitting either produces an explicit fetch_failed item, never truncated data. const SOURCE_MAP_FETCH_TIMEOUT_MS = 30_000; const SOURCE_MAP_RESPONSE_BYTES = 64 * 1024 * 1024; +const SOURCE_MAP_OUTPUT_BYTES = WEB_SOURCE_MAP_LIMITS.outputBytes; +const SOURCE_MAP_FAILURE_LIMITATION_RESERVE = 512; +const SOURCE_MAP_COLLECTION_ENVELOPE_RESERVE = 1_024; type SourceMaps = WebSourceMaps; type SourceMapItem = WebSourceMapItem; @@ -55,7 +59,11 @@ type ParsedSourceMapItem = Extract< interface SourceMapDecodeContext { readonly signal: AbortSignal | undefined; readonly deadlineAt: number; - readonly budget: { records: number }; + readonly budget: { + records: number; + expandedBytes: number; + resolvedSourceBytes: number; + }; } /** Fetch and validate approved source maps without browser credentials. */ @@ -79,11 +87,27 @@ export const fetchWebSourceMaps = async ( const budget = { retainedBytes: 0, deadlineAt, - decodedRecords: { records: 0 }, + decodedRecords: { + records: 0, + expandedBytes: SOURCE_MAP_COLLECTION_ENVELOPE_RESERVE, + resolvedSourceBytes: 0, + }, }; + let limitation: string | undefined; try { for (const request of requests) { if (signal?.aborted === true) throw signal.reason; + const reservation = sourceMapFailureReservation(request) + 1; + if ( + reservation > + SOURCE_MAP_OUTPUT_BYTES - budget.decodedRecords.expandedBytes + ) { + const unprocessed = requests.length - items.length; + limitation = `Source-map output budget was exhausted before processing ${String(unprocessed)} requested map${unprocessed === 1 ? "" : "s"}. Their results are unknown.`; + break; + } + const expandedBytesBeforeItem = budget.decodedRecords.expandedBytes; + budget.decodedRecords.expandedBytes += reservation; if (operationSignal.aborted || Date.now() >= deadlineAt) { items.push( emptySourceMapItem( @@ -94,7 +118,17 @@ export const fetchWebSourceMaps = async ( ); continue; } - items.push(await fetchOne(request, input, operationSignal, host, budget)); + const item = await fetchOne( + request, + input, + operationSignal, + host, + budget, + ); + if (item.artifact === null) + budget.decodedRecords.expandedBytes = + expandedBytesBeforeItem + reservation; + items.push(item); } } finally { clearTimeout(timeout); @@ -102,20 +136,23 @@ export const fetchWebSourceMaps = async ( } if (signal?.aborted === true) throw signal.reason; return webSourceMapsSchema.parse({ - status: sourceMapsStatus(items), + status: sourceMapsStatus(items, limitation !== undefined), requested: requests.length, processed: items.length, items, + ...(limitation === undefined ? {} : { limitation }), }); }; const sourceMapsStatus = ( items: readonly SourceMapItem[], + omitted: boolean, ): SourceMaps["status"] => { const retained = items.filter( ({ status }) => status === "included" || status === "partial", ).length; if (items.length === 0 || retained === 0) return "unavailable"; + if (omitted) return "partial"; return retained === items.length && !items.some(({ status }) => status === "partial") ? "included" @@ -151,7 +188,11 @@ const fetchOne = async ( budget: { retainedBytes: number; deadlineAt: number; - decodedRecords: { records: number }; + decodedRecords: { + records: number; + expandedBytes: number; + resolvedSourceBytes: number; + }; }, ): Promise => { if (!approvedUrl(request.fetchUrl, input.allowed_origins)) @@ -378,49 +419,6 @@ const promiseWithAbort = ( }); }; -type EncodedBrowserSourceMap = EncodedSourceMap | SectionedSourceMap; - -/** Compose indexed offsets before upstream flattening, retaining parent clipping. */ -const sourceMapWithAbsoluteSections = ( - value: unknown, - check: () => void, -): EncodedBrowserSourceMap => { - const root = value as EncodedBrowserSourceMap; - if (!("sections" in root)) return root; - const sections: Section[] = []; - const pending: Section[] = [{ map: root, offset: { line: 0, column: 0 } }]; - while (pending.length > 0) { - check(); - const node = pending.pop(); - if (node === undefined) break; - if (!("sections" in node.map)) { - sections.push(node); - continue; - } - // A parent's boundary clips the preceding leaf even when its first child - // starts later or it has no children. Empty maps retain that boundary - // without adding source/name identities or point mappings. - sections.push({ - offset: node.offset, - map: { version: 3, sources: [], names: [], mappings: "" }, - }); - for (let index = node.map.sections.length - 1; index >= 0; index -= 1) { - const child = node.map.sections[index]; - if (child === undefined) continue; - pending.push({ - map: child.map, - offset: { - line: node.offset.line + child.offset.line, - column: - child.offset.column + - (child.offset.line === 0 ? node.offset.column : 0), - }, - }); - } - } - return { ...root, sections }; -}; - const normalizeSourceMap = ( request: WebSourceMapRequest, text: string, @@ -434,56 +432,153 @@ const normalizeSourceMap = ( "invalid", "Source-map JSON is not a version 3 map.", ); + const retainedBytesBeforeMap = context.budget.expandedBytes; try { const check = (): void => checkOperation(undefined, context.signal, context.deadlineAt); - validateSourceMapStructure(parsedJson.value, { + const inspected = inspectSourceMapValue(parsedJson.value, { profile: "browser-collection", budget: context.budget, check, }); - check(); - const map = new AnyMap( - sourceMapWithAbsoluteSections(parsedJson.value, check), - fetchedUrl, + context.budget.expandedBytes = + retainedBytesBeforeMap - + sourceMapFailureReservation(request) + + sourceMapSuccessReservation(request); + consumeExpandedBytes( + context.budget, + jsonBytes(text) + + artifactMetadataBytes("application/source-map+json", text) - + 4, + ); + const sourcesByRaw = new Map(); + const resolvedByRoot = new Map>(); + preflightResolvedSourceBytes( + inspected.leaves, + fetchedUrl, + context.budget, + resolvedByRoot, + sourcesByRaw, + check, ); - const resolvedBySource = new Map(); - const originalSources = map.sources.map((source, index) => { - if ((index & 255) === 0) check(); - const content = map.sourcesContent?.[index]; - const resolved = - map.resolvedSources[index] ?? source ?? "[unknown-source]"; - if (source !== null) resolvedBySource.set(source, resolved); - return { - source: sanitizeSource(resolved), - artifact: - typeof content === "string" - ? createWebTextArtifact(content, sourceMediaType(source)) - : null, - }; - }); - const mappings: ParsedSourceMapItem["mappings"] = []; - eachMapping(map, (mapping) => { - if ((mappings.length & 0x3fff) === 0) check(); - if ( - mapping.source === null || - mapping.originalLine === null || - mapping.originalColumn === null - ) - return; - mappings.push({ - generated_line: mapping.generatedLine, - generated_column: mapping.generatedColumn, - source: sanitizeSource( - resolvedBySource.get(mapping.source) ?? mapping.source, - ), - original_line: mapping.originalLine, - original_column: mapping.originalColumn, - name: mapping.name ?? null, - }); - }); - const modules = originalModuleEdges(originalSources, check); check(); + const decoded = decodeValidatedSourceMapLeaves( + inspected.leaves, + fetchedUrl, + check, + ); + const sourceJsonBytes = new Map(); + const nameJsonBytes = new Map(); + const originalSources: ParsedSourceMapItem["original_sources"] = []; + let sourceBytes = 0; + for (const { leaf, resolvedSources } of decoded) { + for (const [index, rawSource] of leaf.map.sources.entries()) { + if ((index & 255) === 0) check(); + const resolved = + resolvedSources[index] ?? rawSource ?? "[unknown-source]"; + const sanitized = cachedSanitized(sourcesByRaw, resolved); + sourceBytes += + cachedJsonBytes(sourceJsonBytes, sanitized) + + sourceEntryOverheadBytes(); + const content = leaf.map.sourcesContent?.[index] ?? null; + if (content !== null) + sourceBytes += + jsonBytes(content) + + artifactMetadataBytes(sourceMediaType(rawSource), content) - + 4; + } + } + consumeExpandedBytes( + context.budget, + sourceBytes + originalSourcesCount(decoded), + ); + for (const { leaf, resolvedSources } of decoded) { + for (const [index, rawSource] of leaf.map.sources.entries()) { + if ((index & 255) === 0) check(); + const resolved = + resolvedSources[index] ?? rawSource ?? "[unknown-source]"; + const sanitized = cachedSanitized(sourcesByRaw, resolved); + const content = leaf.map.sourcesContent?.[index] ?? null; + originalSources.push({ + source: sanitized, + artifact: + content === null + ? null + : createWebTextArtifact(content, sourceMediaType(rawSource)), + }); + } + } + let mappingBytes = 0; + let mappingCount = 0; + for (const { leaf, rows, resolvedSources } of decoded) + for (const [line, row] of rows.entries()) { + if ((line & 255) === 0) check(); + for (const segment of row) { + if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) continue; + if (segment.length === 1) continue; + const rawSource = leaf.map.sources[segment[1]]; + const source = cachedSanitized( + sourcesByRaw, + resolvedSources[segment[1]] ?? rawSource ?? "[unknown-source]", + ); + const name = + segment.length === 5 ? (leaf.map.names[segment[4]] ?? null) : null; + mappingBytes += + cachedJsonBytes(sourceJsonBytes, source) + + (name === null ? 4 : cachedJsonBytes(nameJsonBytes, name)) + + mappingRowBytes( + leaf.offset.line + line + 1, + segment[0] + (line === 0 ? leaf.offset.column : 0), + segment[2] + 1, + segment[3], + ); + mappingCount += 1; + if ( + mappingBytes > + WEB_SOURCE_MAP_LIMITS.outputBytes - context.budget.expandedBytes + ) + throw expandedOutputFailure(); + } + } + consumeExpandedBytes(context.budget, mappingBytes + mappingCount); + const mappings: ParsedSourceMapItem["mappings"] = []; + for (const { leaf, rows, resolvedSources } of decoded) + for (const [line, row] of rows.entries()) { + if ((line & 255) === 0) check(); + for (const segment of row) { + if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) continue; + if (segment.length === 1) continue; + const rawSource = leaf.map.sources[segment[1]]; + const resolved = + resolvedSources[segment[1]] ?? rawSource ?? "[unknown-source]"; + const source = cachedSanitized(sourcesByRaw, resolved); + const name = + segment.length === 5 ? (leaf.map.names[segment[4]] ?? null) : null; + mappings.push({ + generated_line: leaf.offset.line + line + 1, + generated_column: + segment[0] + (line === 0 ? leaf.offset.column : 0), + source, + original_line: segment[2] + 1, + original_column: segment[3], + name, + }); + } + } + const modules = originalModuleEdges(originalSources, check, (edge) => + consumeExpandedBytes( + context.budget, + Buffer.byteLength(JSON.stringify(edge)) + 1, + ), + ); + check(); + if (modules.incomplete.length > 0) + consumeExpandedBytes( + context.budget, + jsonBytes( + `Module edges are incomplete: ${modules.incomplete.length} of ${originalSources.filter(({ artifact }) => artifact !== null).length} original sources could not be parsed in full (${modules.incomplete.join(", ")}).`, + ) - 4, + ); const parsed = { ...sourceMapContext(request), artifact: createWebTextArtifact(text, "application/source-map+json"), @@ -500,6 +595,9 @@ const normalizeSourceMap = ( limitation: `Module edges are incomplete: ${modules.incomplete.length} of ${originalSources.filter(({ artifact }) => artifact !== null).length} original sources could not be parsed in full (${modules.incomplete.join(", ")}).`, }; } catch (cause: unknown) { + // Failed maps publish no artifact or mapping rows. Release their output + // reservation while keeping cumulative fetch and decoder-work accounting. + context.budget.expandedBytes = retainedBytesBeforeMap; if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") return emptySourceMapItem(request, "fetch_failed", cause.message); if ( @@ -524,6 +622,9 @@ interface OriginalModuleEdges { const originalModuleEdges = ( sources: ParsedSourceMapItem["original_sources"], check: () => void, + beforeEdge: ( + edge: ParsedSourceMapItem["original_module_edges"][number], + ) => void, ): OriginalModuleEdges => { const edges: ParsedSourceMapItem["original_module_edges"] = []; const seen = new Set(); @@ -571,12 +672,14 @@ const originalModuleEdges = ( const key = `${source.source}\0${kind}\0${specifier}`; if (seen.has(key)) return; seen.add(key); - edges.push({ + const edge = { from_source: source.source, kind, specifier, resolved_source: resolveOriginalSource(specifier, source.source), - }); + }; + beforeEdge(edge); + edges.push(edge); }, }); } @@ -641,6 +744,151 @@ const sanitizeSource = (value: string): string => { } }; +const preflightResolvedSourceBytes = ( + leaves: readonly SourceMapLeaf[], + mapUrl: string, + budget: { readonly expandedBytes: number; resolvedSourceBytes: number }, + resolvedByRoot: Map>, + sanitizedByResolved: Map, + check: () => void, +): void => { + let estimate = budget.expandedBytes; + let resolvedBytes = budget.resolvedSourceBytes; + for (const { map } of leaves) { + let sourcesForRoot = resolvedByRoot.get(map.sourceRoot ?? null); + if (sourcesForRoot === undefined) { + sourcesForRoot = new Map(); + resolvedByRoot.set(map.sourceRoot ?? null, sourcesForRoot); + } + for (const source of map.sources) { + check(); + const sourceKey = source ?? ""; + let resolved = sourcesForRoot.get(sourceKey); + if (resolved === undefined) + resolved = resolveSourceMapSource(source, map.sourceRoot, mapUrl); + const bytes = Buffer.byteLength(resolved, "utf8"); + if (bytes > SOURCE_MAP_RESPONSE_BYTES - resolvedBytes) + throw resolvedSourceLimitFailure(); + resolvedBytes += bytes; + sourcesForRoot.set(sourceKey, resolved); + const sanitized = cachedSanitized(sanitizedByResolved, resolved); + estimate += jsonBytes(sanitized); + if (estimate > WEB_SOURCE_MAP_LIMITS.outputBytes) + throw expandedOutputFailure(); + } + } + budget.resolvedSourceBytes = resolvedBytes; +}; + +const consumeExpandedBytes = ( + budget: { expandedBytes: number }, + bytes: number, +): void => { + if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes - budget.expandedBytes) + throw expandedOutputFailure(); + budget.expandedBytes += bytes; +}; + +const expandedOutputFailure = (): SourceMapFormatFailure => + new SourceMapFormatFailure( + "limit", + `Expanded source-map evidence exceeds the ${WEB_SOURCE_MAP_LIMITS.outputBytes / (1024 * 1024)} MiB public representation budget.`, + ); + +const resolvedSourceLimitFailure = (): SourceMapFormatFailure => + new SourceMapFormatFailure( + "limit", + `Resolved source-map identities exceed the ${SOURCE_MAP_RESPONSE_BYTES / (1024 * 1024)} MiB decoder representation budget.`, + ); + +const jsonBytes = (value: string): number => { + let bytes = 0; + for (const part of jsonParts(value)) bytes += Buffer.byteLength(part, "utf8"); + return bytes; +}; + +const artifactMetadataBytes = (mediaType: string, text: string): number => + Buffer.byteLength( + JSON.stringify({ + sha256: "0".repeat(64), + bytes: Buffer.byteLength(text), + media_type: mediaType, + charset: "utf-8", + text: "", + }), + ) - 2; + +const sourceMapFailureReservation = (request: WebSourceMapRequest): number => { + const envelope = emptySourceMapItem( + { ...request, declaredUrl: "", scriptKey: "" }, + "fetch_failed", + "x".repeat(SOURCE_MAP_FAILURE_LIMITATION_RESERVE), + ); + return ( + Buffer.byteLength(JSON.stringify(envelope)) - + 4 + + jsonBytes(request.declaredUrl) + + jsonBytes(request.scriptKey) + ); +}; + +const sourceMapSuccessReservation = (request: WebSourceMapRequest): number => { + const context = sourceMapContext({ + ...request, + declaredUrl: "", + scriptKey: "", + }); + const envelope = { + ...context, + status: "included", + artifact: null, + original_sources: [], + original_module_edges: [], + mappings: [], + limitation: null, + }; + return ( + Buffer.byteLength(JSON.stringify(envelope)) - + 4 + + jsonBytes(request.declaredUrl) + + jsonBytes(request.scriptKey) + ); +}; + +const sourceEntryOverheadBytes = (): number => + Buffer.byteLength('{"source":,"artifact":null}'); + +const originalSourcesCount = ( + decoded: ReturnType, +): number => + decoded.reduce((count, { leaf }) => count + leaf.map.sources.length, 0); + +const cachedJsonBytes = (cache: Map, value: string): number => { + const cached = cache.get(value); + if (cached !== undefined) return cached; + const bytes = jsonBytes(value); + cache.set(value, bytes); + return bytes; +}; + +const cachedSanitized = (cache: Map, value: string): string => { + const cached = cache.get(value); + if (cached !== undefined) return cached; + const sanitized = sanitizeSource(value); + cache.set(value, sanitized); + return sanitized; +}; + +const mappingRowBytes = ( + generatedLine: number, + generatedColumn: number, + originalLine: number, + originalColumn: number, +): number => + Buffer.byteLength( + `{"generated_line":${String(generatedLine)},"generated_column":${String(generatedColumn)},"source":,"original_line":${String(originalLine)},"original_column":${String(originalColumn)},"name":}`, + ); + const resolveOriginalSource = ( specifier: string, base: string, diff --git a/src/domain/webBundleAnalysis.ts b/src/domain/webBundleAnalysis.ts index f04f18094..c3bbcf352 100644 --- a/src/domain/webBundleAnalysis.ts +++ b/src/domain/webBundleAnalysis.ts @@ -99,29 +99,44 @@ export const webSourceMapsSchema = z requested: z.number().int().min(0), processed: z.number().int().min(0), items: z.array(sourceMapSchema), + limitation: z.string().min(1).optional(), }) .superRefine((sourceMaps, context) => { if ( - sourceMaps.requested !== sourceMaps.processed || + sourceMaps.processed > sourceMaps.requested || sourceMaps.processed !== sourceMaps.items.length ) context.addIssue({ code: "custom", message: "Source-map coverage counts are inconsistent", }); + if ( + sourceMaps.processed < sourceMaps.requested && + sourceMaps.limitation === undefined + ) + context.addIssue({ + code: "custom", + message: + "Unprocessed source maps require an explicit coverage limitation", + path: ["limitation"], + }); const statuses = sourceMaps.items.map(({ status }) => status); const retained = statuses.filter( (status) => status === "included" || status === "partial", ).length; const allowedStatuses = - sourceMaps.items.length === 0 - ? ["not_requested", "unavailable"] - : retained === 0 + sourceMaps.processed < sourceMaps.requested + ? retained === 0 ? ["unavailable"] - : retained === sourceMaps.items.length && - !statuses.includes("partial") - ? ["included"] - : ["partial"]; + : ["partial"] + : sourceMaps.items.length === 0 + ? ["not_requested", "unavailable"] + : retained === 0 + ? ["unavailable"] + : retained === sourceMaps.items.length && + !statuses.includes("partial") + ? ["included"] + : ["partial"]; if (!allowedStatuses.includes(sourceMaps.status)) context.addIssue({ code: "custom", diff --git a/src/domain/webBundleAnalyzerResult.ts b/src/domain/webBundleAnalyzerResult.ts index f11a3a1b9..b4ac67a29 100644 --- a/src/domain/webBundleAnalyzerResult.ts +++ b/src/domain/webBundleAnalyzerResult.ts @@ -113,6 +113,7 @@ const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [ { dimension: "source_maps" as const, reason: + input.sourceMaps.limitation ?? "One or more requested source maps were unavailable or incomplete", affected_script_keys: input.sourceMaps.items .filter(({ status }) => status !== "included") diff --git a/src/javascript/sourceMaps/DecodedSourceMap.ts b/src/javascript/sourceMaps/DecodedSourceMap.ts new file mode 100644 index 000000000..b329cd68e --- /dev/null +++ b/src/javascript/sourceMaps/DecodedSourceMap.ts @@ -0,0 +1,82 @@ +import { TraceMap, decodedMappings } from "@jridgewell/trace-mapping"; +import { + SourceMapFormatFailure, + type SourceMapLeaf, +} from "./SourceMapFormat.js"; + +/** Determine whether a leaf-local mapping falls before its enclosing boundary. */ +export const isBeforeSourceMapLeafStop = ( + leaf: SourceMapLeaf, + line: number, + column: number, +): boolean => { + if (leaf.stop === null) return true; + const generatedLine = leaf.offset.line + line; + const generatedColumn = column + (line === 0 ? leaf.offset.column : 0); + return ( + generatedLine < leaf.stop.line || + (generatedLine === leaf.stop.line && generatedColumn < leaf.stop.column) + ); +}; + +/** Resolve one declaration with the same pinned upstream implementation used by full maps. */ +export const resolveSourceMapSource = ( + source: string | null, + sourceRoot: string | null | undefined, + mapUrl: string, +): string => { + const map = new TraceMap( + { + version: 3, + names: [], + sources: [source ?? ""], + ...(sourceRoot == null ? {} : { sourceRoot }), + mappings: "", + }, + mapUrl, + ); + const resolved = map.resolvedSources[0]; + if (resolved === undefined) + throw new SourceMapFormatFailure( + "format", + "Source-map resolver did not return the requested source identity.", + ); + return resolved; +}; + +/** Decode leaf mappings and validate producer indices before any consumer publishes them. */ +export const decodeValidatedSourceMapLeaves = ( + leaves: readonly SourceMapLeaf[], + mapUrl: string, + check?: () => void, +): readonly { + readonly leaf: SourceMapLeaf; + readonly rows: ReturnType; + readonly resolvedSources: readonly string[]; +}[] => + leaves.map((leaf) => { + check?.(); + const map = new TraceMap(JSON.stringify(leaf.map), mapUrl); + const rows = decodedMappings(map); + for (const [line, row] of rows.entries()) { + if ((line & 255) === 0) check?.(); + for (const segment of row) { + const generatedLine = leaf.offset.line + line; + const generatedColumn = + segment[0] + (line === 0 ? leaf.offset.column : 0); + if ( + !Number.isSafeInteger(generatedLine) || + !Number.isSafeInteger(generatedColumn) || + ![1, 4, 5].includes(segment.length) || + segment.some((value) => !Number.isSafeInteger(value) || value < 0) || + (segment.length !== 1 && segment[1] >= leaf.map.sources.length) || + (segment.length === 5 && segment[4] >= leaf.map.names.length) + ) + throw new SourceMapFormatFailure( + "format", + "Decoded local or absolute position, source index or name index is invalid.", + ); + } + } + return { leaf, rows, resolvedSources: map.resolvedSources }; + }); diff --git a/src/javascript/sourceMaps/SourceMapFormat.ts b/src/javascript/sourceMaps/SourceMapFormat.ts index f49078628..c1816d51e 100644 --- a/src/javascript/sourceMaps/SourceMapFormat.ts +++ b/src/javascript/sourceMaps/SourceMapFormat.ts @@ -311,18 +311,10 @@ export const inspectSourceMap = ( }; }; -/** Validate shared source-map structure before a consumer decodes mappings. */ -export const validateSourceMapStructure = ( +/** Return validated leaves when a consumer needs decoded rows after preflight. */ +export const inspectSourceMapValue = ( value: unknown, options: SourceMapStructureOptions = {}, -): void => { - inspectSourceMapValue(value, options, false); -}; - -const inspectSourceMapValue = ( - value: unknown, - options: SourceMapStructureOptions = {}, - collect = true, ): { readonly format: "regular" | "indexed"; readonly reportedFile: string | null; @@ -351,10 +343,9 @@ const inspectSourceMapValue = ( continue; } const leaf = inspectLeaf(node, budget, options); - if (collect) - leaves.push({ map: leaf, offset: node.offset, stop: node.stop }); - for (const [sourceIndex, source] of leaf.sources.entries()) - if (collect) + leaves.push({ map: leaf, offset: node.offset, stop: node.stop }); + if (options.profile !== "browser-collection") + for (const [sourceIndex, source] of leaf.sources.entries()) declarations.push({ sectionPath: node.path, sourceIndex, diff --git a/src/javascript/sourceMaps/TraceSourceMap.ts b/src/javascript/sourceMaps/TraceSourceMap.ts index 2fb5ef559..14755d2ec 100644 --- a/src/javascript/sourceMaps/TraceSourceMap.ts +++ b/src/javascript/sourceMaps/TraceSourceMap.ts @@ -20,6 +20,10 @@ import { SourceMapFormatFailure, type SourceMapSourceDeclaration, } from "./SourceMapFormat.js"; +import { + decodeValidatedSourceMapLeaves, + isBeforeSourceMapLeafStop, +} from "./DecodedSourceMap.js"; /** Decode with the pinned upstream codec; preserve every equal-position candidate. */ export const traceSourceMap = ( @@ -28,7 +32,21 @@ export const traceSourceMap = ( position: WebSourcePosition, ): WebSourceMapReport => { const inspected = inspectSourceMap(text); - const { map, rows } = decodeValidatedMap(inspected, url); + const map = new AnyMap(inspected.jsonText, url); + const decodedLeaves = decodeValidatedSourceMapLeaves(inspected.leaves, url); + for (const { leaf, rows } of decodedLeaves) + for (const [line, row] of rows.entries()) + for (const segment of row) + if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) + throw new SourceMapFormatFailure( + "format", + "Embedded source-map mappings overlap the following section boundary.", + ); + const rows = + inspected.format === "regular" + ? (decodedLeaves[0]?.rows ?? []) + : decodedMappings(map); + validateFlattenedRows(inspected, rows, map); const matched = greatestPosition(rows, position); const segments = matched === null @@ -77,49 +95,14 @@ export const traceSourceMap = ( }); }; -const decodeValidatedMap = ( +const validateFlattenedRows = ( inspected: ReturnType, - url: string, -) => { - let regularRows: ReturnType | undefined; - for (const leaf of inspected.leaves) { - const decoded = decodedMappings( - new TraceMap(JSON.stringify(leaf.map), url), - ); - if (inspected.format === "regular") regularRows = decoded; - for (const [line, row] of decoded.entries()) - for (const segment of row) { - if ( - segment.some((value) => !Number.isSafeInteger(value) || value < 0) || - (segment.length !== 1 && segment[1] >= leaf.map.sources.length) || - (segment.length === 5 && segment[4] >= leaf.map.names.length) - ) - throw new SourceMapFormatFailure( - "format", - "Decoded section-local position, source index or name index is invalid.", - ); - const generatedLine = leaf.offset.line + line; - const generatedColumn = - segment[0] + (line === 0 ? leaf.offset.column : 0); - if ( - leaf.stop !== null && - (generatedLine > leaf.stop.line || - (generatedLine === leaf.stop.line && - generatedColumn >= leaf.stop.column)) - ) - throw new SourceMapFormatFailure( - "format", - "Embedded source-map mappings overlap the following section boundary.", - ); - } - } - const map = new AnyMap(inspected.jsonText, url); - const rows = regularRows ?? decodedMappings(map); + rows: ReturnType, + map: ReturnType, +): void => { for (const row of rows) for (const segment of row) { if ( - ![1, 4, 5].includes(segment.length) || - segment.some((v) => !Number.isSafeInteger(v) || v < 0) || (segment.length !== 1 && segment[1] >= inspected.declarations.length) || (segment.length === 5 && segment[4] >= map.names.length) ) @@ -128,8 +111,6 @@ const decodeValidatedMap = ( "Decoded mapping has invalid component count, position, source index or name index.", ); } - - return { map, rows }; }; const greatestPosition = ( diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 2a5d1e963..7d251b41a 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -374,6 +374,57 @@ describe("web source-map fetching and validation: enforcing fetch limits and val }); }); +describe("browser map decoded validation", () => { + it("keeps decoded-index and coordinate failures local to their map", async () => { + const maps = [ + { version: 3, names: [], sources: ["ok.js"], mappings: "AAAA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "ACAA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "AADA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "DAAA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "AAAAD" }, + { + version: 3, + sections: [ + { + offset: { line: 0, column: Number.MAX_SAFE_INTEGER }, + map: { + version: 3, + names: [], + sources: ["bad.js"], + mappings: "CAAA", + }, + }, + ], + }, + { version: 3, names: [], sources: ["ok.js"], mappings: "AAAA" }, + ]; + let index = 0; + const requests = maps.map((_, requestIndex) => ({ + ...request, + scriptKey: `scr_${String(requestIndex + 1).padStart(64, "0")}`, + })); + const result = await fetchWebSourceMaps(requests, input(), undefined, { + fetch: () => + Promise.resolve( + new Response(JSON.stringify(maps[index++]), { status: 200 }), + ), + }); + + expect(result.status).toBe("partial"); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "invalid", + "invalid", + "invalid", + "invalid", + "invalid", + "included", + ]); + expect(result.items[0]?.mappings).toHaveLength(1); + expect(result.items[6]?.mappings).toHaveLength(1); + }); +}); + describe("source-map response encoding", () => { it("rejects malformed UTF-8 source-map response bytes", async () => { const result = await fetchWebSourceMaps([request], input(), undefined, { @@ -809,6 +860,232 @@ describe("web source-map collection", () => { }); }); +describe("source-map output admission", () => { + it("admits a complete maximum-size map set when its real output fits", async () => { + const localOrigin = "http://127.0.0.1:9222"; + const mapUrl = `${localOrigin}/m`; + const requests = Array.from({ length: 50_000 }, (_, index) => ({ + ...request, + scriptKey: `scr_${index.toString(16).padStart(64, "0")}`, + declaredUrl: mapUrl, + fetchUrl: mapUrl, + })); + const map = JSON.stringify({ + version: 3, + names: [], + sources: ["x"], + mappings: "AAAA", + }); + const result = await fetchWebSourceMaps( + requests, + input({ allowed_origins: [localOrigin] }), + undefined, + { + timeoutMs: 120_000, + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }, + ); + + expect(result.status).toBe("included"); + expect(result.limitation).toBeUndefined(); + expect(result.items).toHaveLength(requests.length); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThanOrEqual( + 32 * 1_024 * 1_024, + ); + }, 120_000); + + it("reserves large failure contexts and reports unprocessed requests as unknown", async () => { + const largeDeclaredUrl = `${origin}/${"u".repeat(8 * 1_024 * 1_024)}`; + const requests = [ + request, + ...Array.from({ length: 5 }, (_, index) => ({ + ...request, + scriptKey: `scr_${String(index + 2).padStart(64, "0")}`, + declaredUrl: largeDeclaredUrl, + fetchUrl: `${origin}/invalid-${String(index)}.map`, + })), + ]; + let calls = 0; + const result = await fetchWebSourceMaps(requests, input(), undefined, { + fetch: () => { + calls += 1; + return Promise.resolve( + calls === 1 + ? validMapResponse() + : new Response("not-json", { status: 200 }), + ); + }, + }); + + expect(calls).toBe(4); + expect(result).toMatchObject({ + status: "partial", + requested: 6, + processed: 4, + limitation: expect.stringContaining("2 requested maps"), + }); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "invalid", + "invalid", + "invalid", + ]); + expect( + result.items.slice(1).map(({ declared_url }) => declared_url), + ).toEqual([largeDeclaredUrl, largeDeclaredUrl, largeDeclaredUrl]); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThanOrEqual( + 32 * 1_024 * 1_024, + ); + }); +}); + +describe("expanded source-map evidence budget", () => { + it("retains a large declared source whose exact public representation fits the budget", async () => { + const source = "x".repeat(12 * 1024 * 1024); + const map = JSON.stringify({ + version: 3, + names: [], + sources: [source], + mappings: "", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + expect(result.items[0]?.status).toBe("included"); + expect(result.items[0]?.original_sources[0]?.source).toBe( + `${origin}/assets/${source}`, + ); + }); + + it("releases discarded malformed maps before reserving a later valid map", async () => { + const malformed = JSON.stringify({ + version: 3, + names: [], + sourceRoot: "a/../".repeat(3_300_000), + sources: ["main.ts"], + mappings: "ACAA", + }); + const valid = JSON.stringify({ + version: 3, + names: [], + sources: ["main.js"], + sourcesContent: [`export default "${"x".repeat(2 * 1024 * 1024)}";`], + mappings: "AAAA", + }); + const responses = [malformed, malformed, valid]; + const result = await fetchWebSourceMaps( + responses.map((_, index) => ({ + ...request, + scriptKey: `scr_${String(index + 1).padStart(64, "0")}`, + })), + input(), + undefined, + { + fetch: () => + Promise.resolve(new Response(responses.shift(), { status: 200 })), + }, + ); + expect(result.items.map(({ status }) => status)).toEqual([ + "invalid", + "invalid", + "included", + ]); + expect(result.items[2]?.artifact?.text).toBe(valid); + }, 15_000); + + it("bounds expanded evidence across individually valid maps", async () => { + const expanded = JSON.stringify({ + version: 3, + names: [], + sources: ["x".repeat(60_000)], + mappings: Array.from({ length: 280 }, () => "AAAA").join(","), + }); + const responses = [expanded, expanded]; + const requests = Array.from({ length: 2 }, (_, index) => ({ + ...request, + scriptKey: `scr_${String(index + 1).padStart(64, "0")}`, + })); + const result = await fetchWebSourceMaps(requests, input(), undefined, { + fetch: () => + Promise.resolve(new Response(responses.shift() ?? "", { status: 200 })), + }); + + expect(result.status).toBe("partial"); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "fetch_failed", + ]); + expect(result.items[0]?.mappings).toHaveLength(280); + expect(result.items[1]?.limitation).toContain( + "Expanded source-map evidence", + ); + expect(result.items[1]?.mappings).toEqual([]); + }); + + it("accepts a large raw source root that the pinned resolver normalizes away", async () => { + const map = JSON.stringify({ + version: 3, + names: [], + sourceRoot: "a/../".repeat(2_300_000), + sources: ["main.ts"], + mappings: "", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + + expect(result.items[0]?.status).toBe("included"); + expect(result.items[0]?.original_sources[0]?.source).toBe( + `${origin}/assets/main.ts`, + ); + }); + + it("rejects a large resolved source product before allocating mapping rows", async () => { + const map = JSON.stringify({ + version: 3, + names: [], + sourceRoot: "x".repeat(8 * 1_024 * 1_024), + sources: ["main.ts"], + mappings: "AAAA,AAAA,AAAA", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + + expect(result.items[0]).toMatchObject({ + status: "fetch_failed", + artifact: null, + mappings: [], + }); + expect(result.items[0]?.limitation).toContain( + "Expanded source-map evidence", + ); + }); + + it("bounds raw resolved identities even when URL sanitization removes credentials", async () => { + const userinfo = "u".repeat(1 * 1_024 * 1_024); + const map = JSON.stringify({ + version: 3, + names: [], + sourceRoot: `https://${userinfo}@sources.example.test/`, + sources: Array.from({ length: 65 }, (_, index) => `source-${index}.ts`), + mappings: "", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + + expect(result.items[0]).toMatchObject({ + status: "fetch_failed", + artifact: null, + original_sources: [], + }); + expect(result.items[0]?.limitation).toContain( + "Resolved source-map identities exceed the 64 MiB decoder representation budget", + ); + }); +}); + describe("raw source-map structure preflight", () => { it("budgets source inventory before schema parsing its entries", async () => { const response = new Response(