From bb154d1067b0fc250452b7d468d0b0523031a16f Mon Sep 17 00:00:00 2001 From: Luis <86675342+LuisTaffelli@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:24:05 -0300 Subject: [PATCH] feat(browser): inspect retained aioquic qlog JSON captures Inspect bounded offline aioquic1.5.0 JSON qlog0.3 and page retained event/transport facts with explicit trace and stream selectors. Preserve producer coordinates, exact numeric lexemes, global pre-selection fact scope and endpoint declaration uncertainty. Verified four real HTTP/3 recordings with an independent interval oracle, public CLI/MCP/schema parity, auth exclusions, source-deletion views and actual decoder heap exhaustion. Unverified producer platforms and qlog-specific lifecycle induction remain documented. Co-authored-by: LuisTaffelli <86675342+LuisTaffelli@users.noreply.github.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- .../skills/reverse-engineer-anything/SKILL.md | 9 +- docs/web-network-captures.md | 114 +++- package.json | 3 +- scripts/verify-qlog-captures.mjs | 215 ++++++++ src/application/WebNetworkCaptureService.ts | 10 + src/browser/history/CaptureFailures.ts | 2 +- .../history/{HarJson.ts => CaptureJson.ts} | 27 +- src/browser/history/CaptureRelease.ts | 8 + src/browser/history/HarCapture.test.ts | 20 +- src/browser/history/HarCapture.ts | 18 +- .../history/HistoricalCaptureDecoder.ts | 30 +- .../history/NodeCaptureAdapter.test.ts | 13 +- src/browser/history/NodeCaptureAdapter.ts | 11 +- src/browser/history/NodeCaptureProcess.ts | 15 +- src/browser/history/QlogCapture.test.ts | 259 +++++++++ src/browser/history/QlogCapture.ts | 503 ++++++++++++++++++ src/cli/webNetworkCaptureCommands.ts | 2 +- src/composition/webNetworkCaptures.ts | 1 + src/contracts/analysisViewToolContracts.ts | 2 +- .../webNetworkCaptureToolContracts.ts | 10 +- src/domain/analysisView/analysisView.ts | 15 + src/domain/analysisView/qlogCaptureView.ts | 248 +++++++++ .../analysisView/webNetworkCaptureView.ts | 90 +++- src/domain/qlogCapture.ts | 76 +++ src/domain/webNetworkCapture.ts | 140 +++-- src/domain/webNetworkCaptureCoordinates.ts | 22 + src/server/registerAnalysisViewTool.ts | 11 +- src/server/registerWebNetworkCaptureTool.ts | 14 +- src/server/toolRegistrationOptions.ts | 12 + tests/acceptance/tools/qlogCapture.test.ts | 457 ++++++++++++++++ 30 files changed, 2206 insertions(+), 151 deletions(-) create mode 100644 scripts/verify-qlog-captures.mjs rename src/browser/history/{HarJson.ts => CaptureJson.ts} (83%) create mode 100644 src/browser/history/QlogCapture.test.ts create mode 100644 src/browser/history/QlogCapture.ts create mode 100644 src/domain/analysisView/qlogCaptureView.ts create mode 100644 src/domain/qlogCapture.ts create mode 100644 tests/acceptance/tools/qlogCapture.test.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 35b99d8dc..3b508af1c 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -65,11 +65,16 @@ explicit path or endpoint and do not need it. [Fiddler workflow](references/runtime-observation.md#fiddler-everywhere-traffic-debugging). Inspect saved SAZ with `inspect_web_network_capture` and `format: "saz"`; connect Everywhere's separate MCP server for live traffic. -- Retained HAR/SAZ/native mitmproxy capture: `inspect_web_network_capture`. Select +- Retained HAR/SAZ/native mitmproxy/aioquic qlog capture: `inspect_web_network_capture`. Select original record ordinals when useful; inspect producer fields and byte/number sidecars without fetching recorded URLs or claiming live attribution. See [historical captures](https://github.com/morluto/rea/blob/main/docs/web-network-captures.md) for the exact upstream - profile and credential exclusions. + profile and credential exclusions. Qlog support starts with aioquic 1.5.0 + JSON 0.3; stream coverage and endpoint loss declarations do not recover bodies + or establish network loss. Prefer detail summary, then kind qlog trace/stream + pages; select one trace, stream ID and direction for contribution/range pages. + Facts retain all-recorded-events-before-selection scope. Other qlog layouts + and PCAP correlation are unsupported. - User-owned Electron runtime already open: `list_electron_targets`. - Recorded packet captures: `inspect_packet_capture` with selected TShark fields, display filters and decode-as rules. Requires Linux, util-linux prlimit and `REA_TSHARK_COMMAND`; see diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index e65887a71..c447c4783 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -255,7 +255,7 @@ response fields, including HAR's `status: 0` and an empty `mimeType`, remain unknown, not invented statuses or MIME types. The view makes no handler-identity, schema or causality claim. It performs no -network activity. Native mitmproxy and SAZ captures return a typed unsupported error. +network activity. Native mitmproxy, SAZ and qlog captures return a typed unsupported error. Parents inspected with `sensitive_values`, or whose Evidence does not record `sensitive_value_count`, return a typed unsupported error whose reason is the canonical token `sensitive_values_unretained`. The view would write path @@ -291,14 +291,15 @@ Complete evidence has a 32 MiB input, 96 MiB decoder reply and 64-level nesting budget. Exceeding a budget returns no partial success. Oversized selected captures return an input error identifying `capture_path` and the byte budget; owned reply/output failures remain output or provider errors. -HAR and SAZ run in an owned +HAR, SAZ and qlog run in an owned 192 MiB Node old-generation heap with one V8 worker. Mitmproxy decoding applies Linux limits of 768 MiB address space, 30 seconds CPU and 96 MiB file output; both commands have a 30-second wall deadline and independently supervised process cleanup. Owned decoder diagnostics retain at most 1 MiB across stdout and stderr; the collector drops overflow as it arrives, reports the limit failure and verifies cleanup. Heap limits alone do not establish aggregate RSS limits. -When REA runs on Bun, the HAR/SAZ decoder runs on Bun, which ignores these V8 heap +SAZ and qlog refuse decoding when neither a Node heap limit nor a Windows Job Object bound can be enforced. Existing HAR availability is preserved. +When REA runs on Bun, the HAR/SAZ/qlog decoder runs on Bun, which ignores these V8 heap flags; its limitations and resource failures then omit the heap sizes. On Windows, the decoder's owned Job Object instead limits its committed memory to 704 MiB, the 192 MiB budget plus a 512 MiB reserve for Bun itself. The @@ -306,7 +307,7 @@ limitations state that limit, and reaching it returns `resource_constraint` with `process_memory_limit_bytes`, whatever exit status Bun reports. Observed decoder memory exhaustion returns `resource_constraint`, with the -selected capture, applicable limits and retained process diagnostics. HAR/SAZ's +selected capture, applicable limits and retained process diagnostics. HAR/SAZ/qlog's old-generation heap is fixed; inherited `NODE_OPTIONS` cannot raise it. Record selection happens after complete decoding and does not reduce that workload. Use a smaller capture exported by its producer or another decoder @@ -333,3 +334,108 @@ Command cleanup failures retain the original status and both diagnostic streams; the decoder's ownership preparation receives the same cancellation signal. MCP advertises the same strict named input contract used by the application. Historical inspection delegates argument validation to that shared application boundary so accepted explicit-sensitive declarations also cover invalid argument names and other correction details. The SDK receives raw arguments through its Standard Schema interface; malformed inputs still fail before decoder or filesystem effects and return the normal structured REA error. Protocol-envelope validation remains owned by MCP. + +## Retained aioquic qlog + +Select `format: "qlog"` through `inspect_web_network_capture`, or: + +```bash +rea inspect-web-network-capture /absolute/origin.qlog qlog --json +rea inspect-web-network-capture /absolute/origin.qlog qlog --record 95 --record 0 --json +``` + +The initial profile is **aioquic 1.5.0, JSON qlog 0.3**: one JSON object with +`qlog_format: "JSON"`, `qlog_version: "0.3"`, `traces` containing object events +with `name`, finite numeric `time` and object `data`. JSON-SEQ, JSONL, columnar +array events and other versions are unsupported. Duplicate JSON members, +malformed UTF-8, invalid interpreted QUIC integers and excessive nesting fail +without partial success, including in events outside the selected ordinals. +Well-formed event subsets and missing frame metadata remain inspectable. +Unknown extensions are retained; this is not validation of every qlog schema. + +Choosing the format selects the interpretation profile; the file does not +necessarily report or authenticate an aioquic version. The profile's clock +uses Unix-epoch milliseconds, verified in aioquic 1.5.0's `QuicLoggerTrace` +`log_event`/`encode_time`. The `qlog.clock.basis` and Evidence analysis profile +state that assumption explicitly. Numeric lexemes survive independently of +safe JSON-number projections; decimal spelling does not establish physical +clock precision or monotonicity. + +Every event has a global ordinal from trace/event source traversal and its +original `/traces/T/events/E` pointer. This order is not a global time sort. +Trace index and the selected artifact identify a connection observation; +reported ODCID is preserved without claiming global uniqueness. Streams are +scoped by trace, stream ID and sent/received direction. Control/QPACK streams +are retained, so counting streams does not count HTTP requests. + +`qlog` is a typed facet required exactly for this format. Its scope is +`all-recorded-events-before-selection`: selection changes returned event +records while facts cite the original ordinals across the entire recorded +file. The container retains trace metadata with each events array replaced +by null; there is no single events pointer shared by all traces. + +| Fact | Basis and limits | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Original event and loss declaration | Observed endpoint report; `recovery:packet_lost` does not prove actual network loss. Missing packet number/type remains unknown. | +| STREAM ranges and byte totals | Derived per trace and direction from reported offsets and lengths using exact integer arithmetic. `unique_bytes` unions intervals; `frame_length_sum` includes overlaps; `repeated_bytes` is their difference. Overlap alone does not prove retransmission or equal contents. | +| FIN final sizes | Reported offset plus length where FIN is true; multiple conflicting sizes remain visible. A final size does not establish complete coverage or a received body. | +| Incomplete fact records | Ordinals with omitted/redacted interpretable frame metadata; usable fields in other records survive. An empty list does not establish complete logging or capture. | +| HTTP bodies and causality | `body_bytes: "not-reported"`; lengths and headers are metadata, with no reconstructed body, hash, timing causality or live process attribution. | + +QUIC integer fields interpreted here accept exact nonnegative JSON integer +values through `2^62 - 1`, including equivalent exponent notation. Offsets plus +lengths must fit that range. Derived sums are decimal strings and are not +rounded to JavaScript numbers. A missing offset is unknown, never defaulted to +zero. Ranges can start above zero or contain gaps. Packet headers, HTTP events, +raw lengths and unknown extensions remain producer evidence rather than being +normalized into invented HAR requests. + +The adapter excludes authorization/proxy authorization/cookie/set-cookie +values in known HTTP frame events, reset tokens in transport parameters and +NEW_CONNECTION_ID frames, and NEW_TOKEN frame tokens. Known URL header userinfo +and explicit sensitive values follow the shared exclusions. Ordinary extension +fields and token-looking query text remain retained. Explicit declarations also +cover normalized numeric projections, derived totals and trace pointers; facts +that cannot be safely shown are omitted with incomplete-record coordinates. +No keylog, PCAP, live URL or recorded target is opened by the qlog decoder. + +To repeat the real-provider lane with **local caller-supplied evidence** from +the educational aioquic 1.5.0 experiment (one origin trace, known 32,803-byte +stream coverage and one 1,169-byte repeated range in the loss case): + +```bash +npm run verify:web:qlog -- /absolute/control/origin.qlog /absolute/loss/origin.qlog /absolute/new-report.json +``` + +This lane checks built CLI/stdio MCP parity, SDK-converted JSON Schemas, +artifact digests, original event count, selection scope, retained parent lookup, +credential exclusions and the known control/loss facts. The report is created +exclusively and contains no keys or captured credential values. It is a +fixture-specific verification, not a general loss detector. CI uses synthetic +format inputs for malformed cases and limits; those tests do not establish +producer provenance. PCAP parsing and correlation remain outside this stage. + +### Focused qlog views + +Use MCP `inspect_web_network_capture` with `format: "qlog", detail: "summary"` +for identity, profile, scope and retained-fact counts. Then pass its +`normalized_result.parent_evidence_id` to `inspect_analysis_view` as a +`retained-evidence` source. + +```json +{"kind":"qlog","facet":"traces","offset":0,"limit":10} +{"kind":"qlog","facet":"streams","trace_index":0,"offset":0,"limit":10} +{"kind":"qlog","facet":"contributions","trace_index":0,"stream_id":"0","direction":"sent","offset":0,"limit":20} +``` + +Trace pages omit nested lists and give their counts. Stream pages preserve +exact byte totals but omit contributions, ranges and final-size lists. For +those lists select `contributions`, `ranges` or `final-sizes` with all three +stream selectors. `loss-declarations` and `incomplete-records` need only +`trace_index`. Offsets count rows in that retained facet; defaults are 0 and 64. Missing or ambiguous selectors fail rather than choosing a connection. + +These views page already interpreted facts without decoding again or opening +the source, and work after its deletion. Facts still cover all recorded events +before parent record selection. Their exhaustion does not prove logging or +network completeness. Event preview/complete pages remain available through +kind `capture`, with original numeric lexemes and producer ordinals preserved. diff --git a/package.json b/package.json index f803cc00c..801f72ebf 100644 --- a/package.json +++ b/package.json @@ -278,7 +278,8 @@ "artifacts:error-schema": "node scripts/generate-error-schema.mjs", "test:prepare": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:mcp-catalog artifacts:product-catalog artifacts:managed-evidence", "evidence:check:built": "node scripts/generate-completion-ledger.mjs --check", - "docs:prepare": "node scripts/generate-package-metadata.mjs && turbo run artifacts:product-catalog artifacts:managed-evidence artifacts:error-schema" + "docs:prepare": "node scripts/generate-package-metadata.mjs && turbo run artifacts:product-catalog artifacts:managed-evidence artifacts:error-schema", + "verify:web:qlog": "npm run build:cached && node scripts/verify-qlog-captures.mjs" }, "lint-staged": { "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}": [ diff --git a/scripts/verify-qlog-captures.mjs b/scripts/verify-qlog-captures.mjs new file mode 100644 index 000000000..8d1a16f6c --- /dev/null +++ b/scripts/verify-qlog-captures.mjs @@ -0,0 +1,215 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { isAbsolute } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { parseEvidence } from "../dist/domain/evidence.js"; + +// Caller-supplied producer evidence stays local; no fixture capture or secrets are bundled. +const [controlPath, lossPath, reportPath] = process.argv.slice(2); +if ( + ![controlPath, lossPath, reportPath].every( + (value) => typeof value === "string" && isAbsolute(value), + ) +) + throw new Error( + "Usage: node scripts/verify-qlog-captures.mjs /absolute/control/origin.qlog /absolute/loss/origin.qlog /absolute/new-report.json", + ); +const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url)); +const client = new Client({ name: "qlog-producer-verifier", version: "1" }); +const environment = { + PATH: process.env.PATH ?? "/usr/bin:/bin", + REA_LOG_LEVEL: "silent", + HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", +}; +const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env: environment, + stderr: "pipe", +}); +let diagnostics = ""; +transport.stderr?.on("data", (bytes) => { + if (diagnostics.length < 65536) diagnostics += bytes.toString("utf8"); +}); +const ajv = new Ajv2020({ strict: false, validateFormats: false }); +const measurements = []; +try { + try { + await client.connect(transport); + } catch (cause) { + throw new Error(`MCP startup failed: ${diagnostics}`, { cause }); + } + const tool = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_web_network_capture", + ); + assert.ok(tool?.outputSchema); + for (const schema of [tool.inputSchema, tool.outputSchema]) + assert.ok(ajv.validateSchema(schema)); + const validateOutput = ajv.compile(tool.outputSchema); + for (const [lane, path] of [ + ["control", controlPath], + ["loss", lossPath], + ]) { + const source = await readFile(path); + const raw = JSON.parse(source.toString("utf8")); + const sha256 = createHash("sha256").update(source).digest("hex"); + assert.equal(raw.qlog_format, "JSON"); + assert.equal(raw.qlog_version, "0.3"); + const input = { capture_path: path, format: "qlog" }; + assert.ok(ajv.validate(tool.inputSchema, input)); + const response = await client.callTool({ + name: tool.name, + arguments: input, + }); + assert.notEqual(response.isError, true); + assert.ok( + validateOutput(response.structuredContent), + JSON.stringify(validateOutput.errors), + ); + const mcp = parseEvidence(response.structuredContent); + const completed = await promisify(execFile)( + process.execPath, + [entrypoint, "inspect-web-network-capture", path, "qlog", "--json"], + { env: environment, timeout: 40_000, maxBuffer: 96 * 1024 * 1024 }, + ); + const cli = parseEvidence(JSON.parse(completed.stdout)); + assert.equal(cli.evidence_id, mcp.evidence_id); + assert.deepEqual(cli.normalized_result, mcp.normalized_result); + const capture = mcp.normalized_result; + assert.equal(capture.artifact.sha256, sha256); + assert.equal( + capture.total_records, + raw.traces.reduce((sum, trace) => sum + trace.events.length, 0), + ); + for (const [ordinal, original] of raw.traces + .flatMap((trace) => trace.events) + .entries()) { + const record = capture.records[ordinal]; + assert.equal(record.ordinal, ordinal); + assert.equal(record.reported.name, original.name); + assert.equal(record.reported.time, original.time); + assert.deepEqual(record.binary_fields, []); + assert.ok( + record.numeric_literals.some( + (number) => + number.pointer === "/time" && + Number(number.literal) === original.time, + ), + ); + } + assert.equal(capture.qlog.body_bytes, "not-reported"); + assert.equal(mcp.authority, "historical-reference"); + assert.equal( + mcp.analysis_profile.parameters.qlog_profile, + "aioquic-1.5.0-json-0.3", + ); + const trace = capture.qlog.traces[0]; + assert.equal(trace.vantage_point, "server"); + const stream = trace.streams.find( + (stream) => stream.stream_id === "0" && stream.direction === "sent", + ); + assert.ok(stream); + assert.equal(stream.unique_bytes, "32803"); + assert.equal( + stream.frame_length_sum, + lane === "control" ? "32803" : "33972", + ); + assert.equal(stream.repeated_bytes, lane === "control" ? "0" : "1169"); + assert.equal(trace.loss_declarations.length, lane === "control" ? 0 : 1); + if (lane === "loss") + assert.deepEqual(trace.loss_declarations[0].packet_number, { + state: "reported", + value: "5", + }); + // Structurally-known secrets may not survive in Evidence, including sidecars. + const serialized = JSON.stringify(mcp); + for (const original of raw.traces.flatMap((trace) => trace.events)) { + if ( + original.name === "transport:parameters_set" && + typeof original.data.stateless_reset_token === "string" + ) + assert.ok(!serialized.includes(original.data.stateless_reset_token)); + for (const frame of original.data.frames ?? []) { + if ( + frame.frame_type === "new_connection_id" && + typeof frame.reset_token === "string" + ) + assert.ok(!serialized.includes(frame.reset_token)); + if (frame.frame_type === "new_token" && typeof frame.token === "string") + assert.ok(!serialized.includes(frame.token)); + } + } + const retained = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: mcp.evidence_id }, + view: { kind: "api-surface" }, + }, + }); + assert.equal(retained.isError, true); + assert.equal( + JSON.parse(retained.content.find((item) => item.type === "text").text) + .error.code, + "unsupported_target", + ); + const selected = await client.callTool({ + name: tool.name, + arguments: { ...input, record_ordinals: [capture.total_records - 1, 0] }, + }); + const subset = parseEvidence(selected.structuredContent).normalized_result; + assert.deepEqual( + subset.records.map((record) => record.ordinal), + [capture.total_records - 1, 0], + ); + assert.deepEqual(subset.qlog, capture.qlog); + measurements.push({ + lane, + path, + sha256, + evidence_id: mcp.evidence_id, + profile_digest: mcp.analysis_profile.digest, + events: capture.total_records, + stream_id: stream.stream_id, + unique_bytes: stream.unique_bytes, + frame_length_sum: stream.frame_length_sum, + repeated_bytes: stream.repeated_bytes, + loss_declarations: trace.loss_declarations.length, + cli_mcp_parity: true, + retained_reference: true, + selected_scope_preserved: true, + auth_exclusions: true, + }); + } + await client.ping(); + const report = { + status: "passed", + profile: "aioquic-1.5.0-json-0.3", + measurements, + limitations: [ + "Caller supplies provenance; format selection does not authenticate the producer.", + "These known loopback cases do not establish WAN behavior, other qlog producers, platforms or general completeness.", + "No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.", + ], + }; + await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", { + flag: "wx", + mode: 0o600, + }); + console.log( + JSON.stringify({ + status: report.status, + cases: measurements.length, + report: reportPath, + }), + ); +} finally { + await client.close(); + await transport.close(); +} diff --git a/src/application/WebNetworkCaptureService.ts b/src/application/WebNetworkCaptureService.ts index a571fa8c1..8e29ebb10 100644 --- a/src/application/WebNetworkCaptureService.ts +++ b/src/application/WebNetworkCaptureService.ts @@ -10,6 +10,7 @@ import { AnalysisInputError, AnalysisOutputError, } from "../domain/analysisErrorCore.js"; +import { createAnalysisProfile } from "../domain/analysisProfile.js"; import { createEvidence, type Evidence } from "../domain/evidence.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import { jsonObjectSchema } from "../domain/jsonValue.js"; @@ -175,6 +176,15 @@ export class WebNetworkCaptureService { ? [...limitations, subjectUnavailableReason] : limitations, }, + ...(safeValue.format !== "qlog" + ? {} + : { + analysisProfile: createAnalysisProfile(value.decoder, { + qlog_profile: safeValue.qlog.profile, + clock: safeValue.qlog.clock, + scope: safeValue.qlog.scope, + }), + }), confidence: "observed", authority: "historical-reference", limitations, diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index 51eceb0b5..b525126a5 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -86,7 +86,7 @@ export const historicalCaptureFailure = ( ) return captureMemoryFailure( input, - "The HAR decoder exhausted its fixed V8 heap", + `The ${input.format} decoder exhausted its fixed V8 heap`, cause, ); } diff --git a/src/browser/history/HarJson.ts b/src/browser/history/CaptureJson.ts similarity index 83% rename from src/browser/history/HarJson.ts rename to src/browser/history/CaptureJson.ts index 257630932..59743788a 100644 --- a/src/browser/history/HarJson.ts +++ b/src/browser/history/CaptureJson.ts @@ -2,6 +2,7 @@ import { visit } from "jsonc-parser"; import { LosslessNumber } from "lossless-json"; import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; import { CaptureFormatError } from "./CaptureFormatError.js"; +import { TextDecoder } from "node:util"; type Frame = | { @@ -13,7 +14,10 @@ type Frame = | { kind: "array"; pointer: string; items: unknown[] }; /** Materialize upstream JSON events without duplicate folding, prototype setters or numeric rounding. */ -export const parseHarJson = (text: string): unknown => { +export const parseCaptureJson = ( + text: string, + format: "HAR" | "qlog", +): unknown => { const frames: Frame[] = []; let result: unknown; const valuePointer = (): string => { @@ -29,7 +33,7 @@ export const parseHarJson = (text: string): unknown => { if (frames.length > WEB_NETWORK_CAPTURE_LIMITS.depth) throw new CaptureFormatError( "input-limit", - "HAR exceeds the 64-level complete-evidence nesting budget.", + `${format} exceeds the 64-level complete-evidence nesting budget.`, valuePointer(), ); }; @@ -69,7 +73,7 @@ export const parseHarJson = (text: string): unknown => { if (frame.members.has(key)) throw new CaptureFormatError( "format", - "HAR JSON contains duplicate object members.", + `${format} JSON contains duplicate object members.`, ); frame.pendingKey = key; }, @@ -103,7 +107,7 @@ export const parseHarJson = (text: string): unknown => { throw new TypeError("JSON visitor emitted an unsupported literal."); }, onError: () => { - throw new CaptureFormatError("format", "HAR is malformed JSON."); + throw new CaptureFormatError("format", `${format} is malformed JSON.`); }, }, { @@ -114,3 +118,18 @@ export const parseHarJson = (text: string): unknown => { ); return result; }; + +/** UTF-8 JSON decoding skips one permitted BOM and rejects malformed bytes. */ +export const decodeCaptureText = ( + bytes: Uint8Array, + format: "HAR" | "qlog", +): string => { + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new CaptureFormatError( + "format", + `${format} is not valid UTF-8 JSON.`, + ); + } +}; diff --git a/src/browser/history/CaptureRelease.ts b/src/browser/history/CaptureRelease.ts index 512c2fa6e..77f29df51 100644 --- a/src/browser/history/CaptureRelease.ts +++ b/src/browser/history/CaptureRelease.ts @@ -35,3 +35,11 @@ export const SAZ_CAPTURE_PROVIDER_IDENTITY = { name: "REA Fiddler SAZ capture adapter", version: "1;zip.js@2.15.0;xmldom@0.9.12", } as const; + +/** Fixed offline interpretation profile, not an authenticated producer claim. */ +export const QLOG_CAPTURE_PROVIDER_IDENTITY = { + id: "aioquic-qlog-json", + name: "REA aioquic qlog capture adapter", + version: + "aioquic-profile@1.5.0;qlog@0.3;jsonc-parser@3.3.1;lossless-json@4.3.1", +} as const; diff --git a/src/browser/history/HarCapture.test.ts b/src/browser/history/HarCapture.test.ts index 2eb1fe21a..9614384cc 100644 --- a/src/browser/history/HarCapture.test.ts +++ b/src/browser/history/HarCapture.test.ts @@ -1,7 +1,8 @@ import { createHash } from "node:crypto"; import { expect, it } from "vitest"; import { historicalHar } from "../../../tests/fixtures/historicalHar.js"; -import { decodeHarCapture, decodeHarText } from "./HarCapture.js"; +import { decodeCaptureText } from "./CaptureJson.js"; +import { decodeHarCapture } from "./HarCapture.js"; it.each([ { values: [] }, @@ -563,18 +564,21 @@ it("ignores one leading byte-order mark, as HAR 1.2 requires of readers", () => const text = JSON.stringify(historicalHar()); const body = Buffer.from(text, "utf8"); const marked = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), body]); - expect(decodeHarText(marked)).toBe(text); - expect(decodeHarCapture(decodeHarText(marked), [])).toEqual( - decodeHarCapture(decodeHarText(body), []), + expect(decodeCaptureText(marked, "HAR")).toBe(text); + expect(decodeHarCapture(decodeCaptureText(marked, "HAR"), [])).toEqual( + decodeHarCapture(decodeCaptureText(body, "HAR"), []), ); // A second mark is content, not an encoding signature. expect(() => decodeHarCapture( - decodeHarText(Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), marked])), + decodeCaptureText( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), marked]), + "HAR", + ), [], ), ).toThrow("HAR is malformed JSON."); - expect(() => decodeHarText(Buffer.from([0x7b, 0xff, 0x7d]))).toThrow( - "HAR is not valid UTF-8 JSON.", - ); + expect(() => + decodeCaptureText(Buffer.from([0x7b, 0xff, 0x7d]), "HAR"), + ).toThrow("HAR is not valid UTF-8 JSON."); }); diff --git a/src/browser/history/HarCapture.ts b/src/browser/history/HarCapture.ts index f3a83af15..83b442f5c 100644 --- a/src/browser/history/HarCapture.ts +++ b/src/browser/history/HarCapture.ts @@ -1,7 +1,6 @@ import { createHash } from "node:crypto"; -import { TextDecoder } from "node:util"; import { isSafeNumber, LosslessNumber } from "lossless-json"; -import { parseHarJson } from "./HarJson.js"; +import { parseCaptureJson } from "./CaptureJson.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { WEB_NETWORK_CAPTURE_LIMITS, @@ -26,25 +25,12 @@ type Projection = Pick< "reported" | "numeric_literals" | "binary_fields" | "redactions" >; -/** - * Decode retained HAR bytes as UTF-8. HAR 1.2 lets writers emit a byte-order - * mark and requires readers to ignore it, so one leading BOM is skipped. - */ -export const decodeHarText = (bytes: Uint8Array): string => { - try { - // ignoreBOM: false (the default) is what skips the mark; true keeps it. - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); - } catch { - throw new CaptureFormatError("format", "HAR is not valid UTF-8 JSON."); - } -}; - /** Decode a retained HAR with exact numeric lexemes and unchanged upstream schema validation. */ export const decodeHarCapture = ( text: string, sensitiveValues: readonly string[], ): DecodedCapture => { - const raw = parseHarJson(text); + const raw = parseCaptureJson(text, "HAR"); const validation = projectHar(raw, [], false); createHarValidator()(validation.reported); if ( diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index f17f45a21..7f6e7b83a 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -198,15 +198,14 @@ export class HistoricalCaptureDecoder { OPERATION, "Capture decoder changed the selected upstream profile identity.", ); - result = ok({ - ...reply.value, + const common = { artifact: { path: input.capture_path, sha256: snapshot.sha256, bytes: snapshot.bytes.length, }, - format: input.format, - runtime_attribution: "unknown", + + runtime_attribution: "unknown" as const, limitations: [ ...(adapter.limitations ?? []), "This is retained producer evidence, without live browser transaction IDs, scenario provenance, execution attribution or deployment authenticity. Recorded URLs are never fetched.", @@ -214,7 +213,22 @@ export class HistoricalCaptureDecoder { "Known authentication headers/cookies and explicitly marked literal values are excluded. Unknown extension names do not establish sensitivity.", `A 32 MiB input, 96 MiB reply and 64-level nesting budget bound complete evidence. ${nodeMemoryLimitation()}; mitmproxy decoding uses a 768 MiB Linux address-space limit. Each owned command has a 30-second deadline with independent cleanup.`, ], - }); + }; + if (input.format === "qlog") { + if (!("qlog" in reply.value)) + throw new AnalysisOutputError( + OPERATION, + "Capture decoder omitted qlog transport facts.", + ); + result = ok({ ...reply.value, ...common, format: "qlog" }); + } else { + if ("qlog" in reply.value) + throw new AnalysisOutputError( + OPERATION, + "Capture decoder supplied inapplicable qlog transport facts.", + ); + result = ok({ ...reply.value, ...common, format: input.format }); + } } catch (cause: unknown) { result = err( redactExplicitFailure( @@ -255,10 +269,10 @@ export class HistoricalCaptureDecoder { /** The Node capture decoder runs on this executable; only Node.js enforces its V8 heap flags. */ const nodeMemoryLimitation = (): string => { if (NODE_CAPTURE_HEAP_ENFORCED) - return `HAR/SAZ use a ${String(NODE_CAPTURE_HEAP_LIMITS.oldGenerationMiB)} MiB Node old-generation heap`; + return `HAR/SAZ/qlog use a ${String(NODE_CAPTURE_HEAP_LIMITS.oldGenerationMiB)} MiB Node old-generation heap`; return NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES === null - ? "HAR/SAZ run on Bun, which does not enforce REA's V8 heap flags" - : `HAR/SAZ run on Bun, which does not enforce REA's V8 heap flags, under a ${String(NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES)}-byte Windows Job Object commit limit`; + ? "HAR/SAZ/qlog run on Bun, which does not enforce REA's V8 heap flags" + : `HAR/SAZ/qlog run on Bun, which does not enforce REA's V8 heap flags, under a ${String(NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES)}-byte Windows Job Object commit limit`; }; const replyFailureReason = (cause: unknown): string => { diff --git a/src/browser/history/NodeCaptureAdapter.test.ts b/src/browser/history/NodeCaptureAdapter.test.ts index c341949db..bf2528f5b 100644 --- a/src/browser/history/NodeCaptureAdapter.test.ts +++ b/src/browser/history/NodeCaptureAdapter.test.ts @@ -8,8 +8,11 @@ vi.mock("./CaptureRelease.js", async (original) => ({ NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES: null, })); -it("refuses a SAZ worker when the runtime has no enforceable memory bound", async () => { - await expect( - new NodeCaptureAdapter("saz").command("/owned/request.json"), - ).rejects.toBeInstanceOf(AnalysisCapabilityUnavailableError); -}); +it.each(["saz", "qlog"] as const)( + "refuses a %s worker when the runtime has no enforceable memory bound", + async (format) => { + await expect( + new NodeCaptureAdapter(format).command("/owned/request.json"), + ).rejects.toBeInstanceOf(AnalysisCapabilityUnavailableError); + }, +); diff --git a/src/browser/history/NodeCaptureAdapter.ts b/src/browser/history/NodeCaptureAdapter.ts index f1e98a07b..8f33430ca 100644 --- a/src/browser/history/NodeCaptureAdapter.ts +++ b/src/browser/history/NodeCaptureAdapter.ts @@ -4,6 +4,7 @@ import { NODE_CAPTURE_HEAP_ENFORCED, HAR_CAPTURE_PROVIDER_IDENTITY, SAZ_CAPTURE_PROVIDER_IDENTITY, + QLOG_CAPTURE_PROVIDER_IDENTITY, NODE_CAPTURE_HEAP_LIMITS, NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES, } from "./CaptureRelease.js"; @@ -13,11 +14,13 @@ import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCo export class NodeCaptureAdapter implements HistoricalCaptureFormatAdapter { readonly identity; readonly limitations; - constructor(readonly format: "har" | "saz") { + constructor(readonly format: "har" | "saz" | "qlog") { this.identity = format === "har" ? HAR_CAPTURE_PROVIDER_IDENTITY - : SAZ_CAPTURE_PROVIDER_IDENTITY; + : format === "saz" + ? SAZ_CAPTURE_PROVIDER_IDENTITY + : QLOG_CAPTURE_PROVIDER_IDENTITY; this.limitations = format === "saz" ? [ @@ -30,14 +33,14 @@ export class NodeCaptureAdapter implements HistoricalCaptureFormatAdapter { } async command(requestPath: string) { if ( - this.format === "saz" && + this.format !== "har" && !NODE_CAPTURE_HEAP_ENFORCED && NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES === null ) throw new AnalysisCapabilityUnavailableError( this.identity.id, "inspect_web_network_capture", - "SAZ decoding requires an enforced decoder memory limit. Run REA with Node.js on this host.", + `${this.format} decoding requires an enforced decoder memory limit. Run REA with Node.js on this host.`, ); return { command: process.execPath, diff --git a/src/browser/history/NodeCaptureProcess.ts b/src/browser/history/NodeCaptureProcess.ts index e94295da4..ff45a869a 100644 --- a/src/browser/history/NodeCaptureProcess.ts +++ b/src/browser/history/NodeCaptureProcess.ts @@ -12,22 +12,25 @@ const requestSchema = z.strictObject({ /** Parse untrusted captures in an owned heap and return one complete bounded reply. */ const main = async (): Promise => { - const format = z.enum(["har", "saz"]).parse(process.argv[3]); + const format = z.enum(["har", "saz", "qlog"]).parse(process.argv[3]); const request = requestSchema.parse( JSON.parse(await readFile(process.argv[2] ?? "", "utf8")), ); let reply: unknown; try { let value; - if (format === "har") { - const { decodeHarCapture, decodeHarText } = - await import("./HarCapture.js"); + if (format !== "saz") { + const { decodeCaptureText } = await import("./CaptureJson.js"); + const decoder = + format === "har" + ? (await import("./HarCapture.js")).decodeHarCapture + : (await import("./QlogCapture.js")).decodeQlogCapture; const snapshot = await readStableArtifact( request.snapshot_path, WEB_NETWORK_CAPTURE_LIMITS.inputBytes, ); - value = decodeHarCapture( - decodeHarText(snapshot.bytes), + value = decoder( + decodeCaptureText(snapshot.bytes, format === "har" ? "HAR" : "qlog"), request.sensitive_values, ); } else { diff --git a/src/browser/history/QlogCapture.test.ts b/src/browser/history/QlogCapture.test.ts new file mode 100644 index 000000000..bd67d4e80 --- /dev/null +++ b/src/browser/history/QlogCapture.test.ts @@ -0,0 +1,259 @@ +import { expect, it } from "vitest"; +import { decodeQlogCapture } from "./QlogCapture.js"; +import { decodeCaptureText } from "./CaptureJson.js"; +import { CaptureFormatError } from "./CaptureFormatError.js"; + +// Synthetic format inputs exercise distinct parser semantics, not producer provenance. +const event = (frames: unknown[], name = "transport:packet_sent") => ({ + name, + time: 1791819895000.25, + data: { header: { packet_type: "1RTT", packet_number: 5 }, frames }, +}); +const frame = (offset: number, length: number, fin = false) => ({ + frame_type: "stream", + stream_id: 0, + offset, + length, + fin, +}); +const capture = (events: unknown[], extra: Record = {}) => + JSON.stringify({ + qlog_format: "JSON", + qlog_version: "0.3", + ...extra, + traces: [ + { + common_fields: { ODCID: "same" }, + vantage_point: { name: "aioquic", type: "server" }, + events, + }, + ], + }); +const facts = (text: string, sensitive: string[] = []) => { + const decoded = decodeQlogCapture(text, sensitive); + if (!("qlog" in decoded)) throw new Error("qlog facet required"); + return { decoded, traces: decoded.qlog.traces }; +}; + +it("derives interval unions per trace/direction while retaining event source order and extensions", () => { + const original = JSON.parse( + capture([ + event([frame(10, 10)]), + event([frame(0, 15), frame(30, 5, true)]), + event([frame(0, 4)], "transport:packet_received"), + { + name: "recovery:packet_lost", + time: 1, + data: { packet_number: 5, type: "1RTT" }, + }, + ]), + ); + original.traces.push({ + ...original.traces[0], + events: [event([frame(0, 3)])], + }); + Object.defineProperty(original, "__proto__", { + enumerable: true, + value: { ordinary: true }, + }); + original.extra = "https://example.test/path?token=ordinary"; + const { decoded, traces } = facts(JSON.stringify(original)); + expect(traces[0]?.streams[0]).toMatchObject({ + stream_id: "0", + direction: "sent", + unique_bytes: "25", + frame_length_sum: "30", + repeated_bytes: "5", + covered_ranges: [ + { start: "0", end: "20" }, + { start: "30", end: "35" }, + ], + reported_final_sizes: ["35"], + }); + expect(traces[0]?.streams[1]).toMatchObject({ + direction: "received", + unique_bytes: "4", + }); + expect(traces[1]?.streams[0]).toMatchObject({ unique_bytes: "3" }); + expect(traces[0]?.loss_declarations[0]).toMatchObject({ + confidence: "observed", + packet_number: { state: "reported", value: "5" }, + packet_number_space: "application", + }); + expect(decoded.records.at(-1)?.location).toEqual({ + kind: "json-pointer", + pointer: "/traces/1/events/0", + }); + expect(decoded.container.reported).toMatchObject({ + extra: original.extra, + ["__proto__"]: { ordinary: true }, + }); + expect(Reflect.get(Object.prototype, "ordinary")).toBeUndefined(); +}); + +it("keeps exact 62-bit integers and numeric lexemes without allocating exponent expansion", () => { + const text = capture([event([frame(0, 0, true)])]) + .replace('"stream_id":0', '"stream_id":4611686018427387903') + .replace('"offset":0', '"offset":9007199254740993') + .replace('"length":0', '"length":1e3'); + const { decoded, traces } = facts(text); + expect(traces[0]?.streams[0]).toMatchObject({ + stream_id: "4611686018427387903", + unique_bytes: "1000", + reported_final_sizes: ["9007199254741993"], + }); + expect(decoded.records[0]?.numeric_literals).toContainEqual({ + pointer: "/data/frames/0/length", + producer_type: "json-number", + literal: "1e3", + }); + expect(decoded.records[0]?.reported).toMatchObject({ + data: { frames: [{ stream_id: null, offset: null, length: 1000 }] }, + }); +}); + +it("keeps incomplete fields unknown and FIN without bytes as metadata", () => { + const { traces } = facts( + capture([ + event([{ frame_type: "stream", stream_id: 0, length: 5 }]), + event([frame(10, 0, true)]), + { name: "recovery:packet_lost", time: 1, data: {} }, + { + name: "extension:custom", + time: 0, + data: { body: "ordinary evidence" }, + }, + ]), + ); + expect(traces[0]?.incomplete_fact_records).toEqual([0]); + expect(traces[0]?.streams[0]).toMatchObject({ + unique_bytes: "0", + covered_ranges: [], + reported_final_sizes: ["10"], + }); + expect(traces[0]?.loss_declarations[0]?.packet_number.state).toBe("unknown"); +}); + +it("excludes structural auth material and explicit normalized values without guessing extension secrecy", () => { + const text = capture([ + { + name: "http:frame_parsed", + time: 1, + data: { + frame: { + frame_type: "headers", + headers: [ + { name: "authorization", value: "Bearer credential" }, + { name: ":path", value: "/?token=ordinary" }, + { name: "location", value: "https://user:pass@example.test/a" }, + ], + }, + }, + }, + { + name: "transport:parameters_set", + time: 2, + data: { stateless_reset_token: "reset-credential" }, + }, + event([ + { frame_type: "new_connection_id", reset_token: "reset-frame" }, + { frame_type: "new_token", token: "new-token" }, + frame(0, 100), + ]), + { + name: "extension:custom", + time: 3, + data: { + secret: "caller-selected ordinary value", + reset_token: "extension ordinary", + }, + }, + ]).replace('"length":100', '"length":1e2'); + const { decoded, traces } = facts(text, ["100"]); + const serialized = JSON.stringify(decoded); + for (const hidden of [ + "Bearer credential", + "reset-credential", + "reset-frame", + "new-token", + "100", + "user:pass", + ]) + expect(serialized).not.toContain(hidden); + expect(serialized).toContain("token=ordinary"); + expect(serialized).toContain("caller-selected ordinary value"); + expect(traces[0]?.streams).toEqual([]); + expect(traces[0]?.incomplete_fact_records).toContain(2); +}); + +it("hides marked ancestor keys and safe derived sums while still validating hidden subtrees", () => { + const text = capture([event([frame(0, 3), frame(3, 4)])]); + expect(facts(text, ["7"]).traces[0]?.streams).toEqual([]); + const { decoded, traces } = facts(text, ["events"]); + expect(decoded.records[0]?.reported).toBe(null); + expect(decoded.records[0]?.location.kind).toBe("unknown"); + expect(traces[0]?.location.kind).toBe("unknown"); + expect(traces[0]?.streams).toEqual([]); + expect(() => + facts(text.replace('"length":3', '"length":-3'), ["events"]), + ).toThrow(CaptureFormatError); +}); + +it.each([ + "-1", + "0.5", + "4611686018427387904", + "1e1000000", + "1e-1000000", + '"3"', + "true", +])("rejects malformed or out-of-range STREAM integer %s", (literal) => { + expect(() => + facts( + capture([event([frame(0, 3)])]).replace( + '"length":3', + `"length":${literal}`, + ), + ), + ).toThrow(CaptureFormatError); +}); + +it.each([ + '{"qlog_format":"JSON","qlog_format":"JSON","qlog_version":"0.3","traces":[]}', + capture([]).slice(0, -1), + capture([]) + " {}", + capture([{ name: "invalid", time: "1", data: {} }]), + capture([[1, "transport", "packet_sent", {}]]), + capture([event([{ ...frame(0, 1), fin: "true" }])]), + capture([event([{ ...frame(0, 1), offset: 4611686018427388000 }])]), +])( + "rejects malformed JSON/profile events with no partial success: %s", + (text) => { + expect(() => facts(text)).toThrow(CaptureFormatError); + }, +); + +it("rejects other qlog containers/versions and excessive nesting, accepts empty profiles and UTF-8 BOM", () => { + for (const extra of [{ qlog_format: "JSON-SEQ" }, { qlog_version: "0.4" }]) + expect(() => facts(capture([], extra))).toThrow("Only the aioquic"); + expect(() => + facts( + capture([], { + extension: JSON.parse("[".repeat(65) + "0" + "]".repeat(65)), + }), + ), + ).toThrow("nesting budget"); + expect( + facts('{"qlog_format":"JSON","qlog_version":"0.3","traces":[]}').traces, + ).toEqual([]); + const raw = Buffer.from(capture([])); + expect( + facts( + decodeCaptureText( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), raw]), + "qlog", + ), + ).decoded.total_records, + ).toBe(0); + expect(() => decodeCaptureText(Buffer.from([0xff]), "qlog")).toThrow("UTF-8"); +}); diff --git a/src/browser/history/QlogCapture.ts b/src/browser/history/QlogCapture.ts new file mode 100644 index 000000000..2e8a1f790 --- /dev/null +++ b/src/browser/history/QlogCapture.ts @@ -0,0 +1,503 @@ +import { isSafeNumber, LosslessNumber } from "lossless-json"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import type { QlogCaptureFacts } from "../../domain/qlogCapture.js"; +import { + webNetworkCapturePayloadSchema, + type WebNetworkCaptureRecord, +} from "../../domain/webNetworkCapture.js"; +import type { z } from "zod"; +import { parseCaptureJson } from "./CaptureJson.js"; +import { CaptureFormatError } from "./CaptureFormatError.js"; +import { + CaptureRedaction, + captureObject, + capturePointer, +} from "./CaptureRedaction.js"; +import { QLOG_CAPTURE_PROVIDER_IDENTITY } from "./CaptureRelease.js"; + +const MAX_VARINT = (1n << 62n) - 1n; +const credentials = new Set([ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", +]); +type Projection = Pick< + WebNetworkCaptureRecord, + "reported" | "numeric_literals" | "binary_fields" | "redactions" +>; +type Trace = QlogCaptureFacts["traces"][number]; +type Stream = Trace["streams"][number]; + +/** Inspect the selected aioquic JSON profile without authenticating its producer or interpreting other qlog layouts. */ +export const decodeQlogCapture = ( + text: string, + sensitiveValues: readonly string[], +): z.output => { + const raw = parseCaptureJson(text, "qlog"); + if (!object(raw) || raw.qlog_format !== "JSON" || raw.qlog_version !== "0.3") + throw new CaptureFormatError( + "unsupported", + "Only the aioquic 1.5.0 JSON qlog 0.3 profile is supported.", + ); + if (!Array.isArray(raw.traces)) + return invalid("qlog must contain a traces array.", "/traces"); + const traces = raw.traces.map(validateTrace); + const container = project( + { ...raw, traces: traces.map((trace) => ({ ...trace, events: null })) }, + sensitiveValues, + ); + const records: WebNetworkCaptureRecord[] = []; + const facts: Trace[] = []; + const excluded = (value: string): boolean => + sensitiveValues.some((literal) => value.includes(literal)); + const ancestorsHidden = excluded("traces") || excluded("events"); + for (const [traceIndex, trace] of traces.entries()) { + const tracePointer = `/traces/${traceIndex}`; + const fact = createTraceFact( + trace, + traceIndex, + sensitiveValues, + ancestorsHidden, + ); + const streams = new Map(); + for (const [eventIndex, event] of trace.events.entries()) { + const ordinal = records.length; + const projection: Projection = ancestorsHidden + ? { + reported: null, + numeric_literals: [], + binary_fields: [], + redactions: [{ pointer: "", reason: "explicit-sensitive-value" }], + } + : project(event, sensitiveValues); + records.push({ + ordinal, + location: ancestorsHidden + ? { kind: "unknown", reason: "explicit-sensitive-value" } + : { + kind: "json-pointer", + pointer: `${tracePointer}/events/${eventIndex}`, + }, + ...projection, + limitations: [ + "Record ordinal is derived from trace/event source traversal, not global time order. Qlog lengths and FIN do not expose original HTTP body bytes or establish network completeness.", + ], + }); + collectTransportFacts(projection, ordinal, { fact, streams, excluded }); + } + for (const stream of streams.values()) { + summarizeStream(stream); + const numericValues = [ + stream.frame_length_sum, + stream.unique_bytes, + stream.repeated_bytes, + ...stream.reported_final_sizes, + ...stream.covered_ranges.flatMap((r) => [r.start, r.end]), + ]; + if (numericValues.some(excluded)) + fact.incomplete_fact_records.push( + ...stream.contributions.map((c) => c.record_ordinal), + ); + else fact.streams.push(stream); + } + fact.incomplete_fact_records = [...new Set(fact.incomplete_fact_records)]; + facts.push(fact); + } + return { + decoder: { ...QLOG_CAPTURE_PROVIDER_IDENTITY }, + container: { + reported: container.reported, + numeric_literals: container.numeric_literals, + redactions: container.redactions, + records_pointer: null, + }, + total_records: records.length, + records, + qlog: { + profile: "aioquic-1.5.0-json-0.3", + scope: "all-recorded-events-before-selection", + body_bytes: "not-reported", + clock: { + unit: "milliseconds", + domain: "unix-epoch", + basis: "selected-aioquic-profile-not-producer-authentication", + }, + traces: facts, + }, + }; +}; + +const createTraceFact = ( + trace: Record & { events: unknown[] }, + traceIndex: number, + sensitiveValues: readonly string[], + ancestorsHidden: boolean, +): Trace => { + const tracePointer = `/traces/${traceIndex}`; + const metadata = project( + { ...trace, events: null }, + sensitiveValues, + ).reported; + const meta = !ancestorsHidden && object(metadata) ? metadata : null; + const common = + meta !== null && object(meta.common_fields) ? meta.common_fields : null; + const vantage = + meta !== null && object(meta.vantage_point) + ? meta.vantage_point.type + : null; + return { + trace_index: traceIndex, + location: ancestorsHidden + ? { kind: "unknown", reason: "explicit-sensitive-value" } + : { kind: "json-pointer", pointer: tracePointer }, + event_count: trace.events.length, + odcid: typeof common?.ODCID === "string" ? common.ODCID : null, + vantage_point: + vantage === "client" || vantage === "server" ? vantage : null, + loss_declarations: [], + streams: [], + incomplete_fact_records: [], + }; +}; + +const validateTrace = ( + trace: unknown, + index: number, +): Record & { events: unknown[] } => { + const pointer = `/traces/${index}`; + if (!object(trace) || !Array.isArray(trace.events)) + return invalid("qlog trace must contain an events array.", pointer); + for (const [eventIndex, event] of trace.events.entries()) { + const eventPointer = `${pointer}/events/${eventIndex}`; + if ( + !object(event) || + typeof event.name !== "string" || + !(event.time instanceof LosslessNumber) || + !Number.isFinite(Number(event.time.value)) || + !object(event.data) + ) + invalid( + "qlog events must be objects with name, finite numeric time and object data.", + eventPointer, + ); + // Validate interpretable fields on ORIGINAL values even when a declaration + // subsequently hides them. Missing fields remain meaningful unknowns. + if (event.name === "recovery:packet_lost") + unsigned(event.data.packet_number, `${eventPointer}/data/packet_number`); + if (packetEvent(event.name) && Array.isArray(event.data.frames)) { + for (const [frameIndex, frame] of event.data.frames.entries()) { + if (object(frame) && frame.frame_type === "stream") { + validateStreamFrame( + frame, + `${eventPointer}/data/frames/${frameIndex}`, + ); + } + } + } + } + return { ...trace, events: trace.events }; +}; + +const validateStreamFrame = ( + frame: Record, + p: string, +): void => { + unsigned(frame.stream_id, `${p}/stream_id`); + const offset = unsigned(frame.offset, `${p}/offset`); + const length = unsigned(frame.length, `${p}/length`); + if (offset !== null && length !== null && offset + length > MAX_VARINT) + invalid("qlog stream final offset exceeds the QUIC varint range.", p); + if ( + frame.fin !== undefined && + frame.fin !== null && + typeof frame.fin !== "boolean" + ) + invalid("qlog STREAM fin must be boolean when reported.", `${p}/fin`); +}; + +const collectTransportFacts = ( + projection: Projection, + ordinal: number, + state: { + fact: Trace; + streams: Map; + excluded: (value: string) => boolean; + }, +): void => { + const { fact, streams, excluded } = state; + if (!object(projection.reported) || !object(projection.reported.data)) { + fact.incomplete_fact_records.push(ordinal); + return; + } + const visible = projection.reported; + const data = visible.data; + if (!object(data)) return; + const literals = new Map( + projection.numeric_literals.map((n) => [n.pointer, n.literal]), + ); + const read = (value: unknown, pointer: string): bigint | null => { + const literal = literals.get(pointer); + const number = unsigned( + literal === undefined ? value : new LosslessNumber(literal), + pointer, + ); + return number !== null && excluded(number.toString()) ? null : number; + }; + if (visible.name === "recovery:packet_lost") { + fact.loss_declarations.push(lossDeclaration(data, ordinal, read)); + } + if (!packetEvent(visible.name)) return; + if (!Array.isArray(data.frames)) { + fact.incomplete_fact_records.push(ordinal); + return; + } + for (const [frameIndex, frame] of data.frames.entries()) { + if ( + !object(frame) || + frame.frame_type === undefined || + frame.frame_type === null + ) { + fact.incomplete_fact_records.push(ordinal); + continue; + } + if (frame.frame_type !== "stream") continue; + const p = `/data/frames/${frameIndex}`; + const id = read(frame.stream_id, `${p}/stream_id`), + offset = read(frame.offset, `${p}/offset`), + length = read(frame.length, `${p}/length`); + if (id === null || offset === null || length === null) { + fact.incomplete_fact_records.push(ordinal); + continue; + } + const direction = + visible.name === "transport:packet_sent" ? "sent" : "received"; + const key = `${direction}/${id}`; + let stream = streams.get(key); + if (stream === undefined) { + stream = { + stream_id: id.toString(), + direction, + confidence: "derived", + scope: "observed-stream-frame-ranges", + contributions: [], + covered_ranges: [], + unique_bytes: "0", + frame_length_sum: "0", + repeated_bytes: "0", + reported_final_sizes: [], + }; + streams.set(key, stream); + } + stream.contributions.push({ + record_ordinal: ordinal, + frame_index: frameIndex, + offset: offset.toString(), + length: length.toString(), + fin: typeof frame.fin === "boolean" ? frame.fin : null, + }); + } +}; + +const lossDeclaration = ( + data: Record, + ordinal: number, + read: (value: unknown, pointer: string) => bigint | null, +): Trace["loss_declarations"][number] => { + const pn = read(data.packet_number, "/data/packet_number"); + const type = typeof data.type === "string" ? data.type : null; + return { + record_ordinal: ordinal, + confidence: "observed", + packet_number: + pn === null + ? { + state: "unknown", + reason: "omitted-null-or-redacted", + } + : { state: "reported", value: pn.toString() }, + packet_type: type, + packet_number_space: + type === "initial" + ? "initial" + : type === "handshake" + ? "handshake" + : type === "0RTT" || type === "1RTT" + ? "application" + : null, + }; +}; + +const summarizeStream = (stream: Stream): void => { + const spans = stream.contributions + .map( + (c) => [BigInt(c.offset), BigInt(c.offset) + BigInt(c.length)] as const, + ) + .sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)); + const merged: { start: bigint; end: bigint }[] = []; + for (const [start, end] of spans) { + if (start === end) continue; + const last = merged.at(-1); + if (last !== undefined && start <= last.end) + last.end = end > last.end ? end : last.end; + else merged.push({ start, end }); + } + const sum = stream.contributions.reduce((s, c) => s + BigInt(c.length), 0n); + const unique = merged.reduce((s, r) => s + r.end - r.start, 0n); + stream.covered_ranges = merged.map((r) => ({ + start: r.start.toString(), + end: r.end.toString(), + })); + stream.frame_length_sum = sum.toString(); + stream.unique_bytes = unique.toString(); + stream.repeated_bytes = (sum - unique).toString(); + stream.reported_final_sizes = [ + ...new Set( + stream.contributions + .filter((c) => c.fin === true) + .map((c) => (BigInt(c.offset) + BigInt(c.length)).toString()), + ), + ]; +}; + +const packetEvent = (name: unknown): boolean => + name === "transport:packet_sent" || name === "transport:packet_received"; +const object = (value: unknown): value is Record => + captureObject(value) && !(value instanceof LosslessNumber); +const invalid = (message: string, pointer: string): never => { + throw new CaptureFormatError("format", message, pointer); +}; + +/** Canonicalize JSON integer notation exactly, checking expansion before allocation. */ +const unsigned = (value: unknown, pointer: string): bigint | null => { + if (value === undefined || value === null) return null; + const literal = + value instanceof LosslessNumber + ? value.value + : typeof value === "number" + ? String(value) + : ""; + const m = /^(-?)([0-9]+)(?:\.([0-9]+))?(?:[eE]([+-]?[0-9]+))?$/u.exec( + literal, + ); + if (m === null) + return invalid("qlog QUIC integer must be a JSON number.", pointer); + const digits = `${m[2]}${m[3] ?? ""}`.replace(/^0+/u, ""); + if (digits === "") return 0n; + const exponent = m[4] === undefined ? 0 : Number(m[4]); + const shift = exponent - (m[3]?.length ?? 0); + const count = digits.length + shift; + if (m[1] === "-" || !Number.isSafeInteger(shift) || count > 19 || count <= 0) + return invalid( + "qlog QUIC integer is negative, fractional or outside the 62-bit range.", + pointer, + ); + if (shift < 0 && /[1-9]/u.test(digits.slice(count))) + return invalid("qlog QUIC integer is fractional.", pointer); + const canonical = + shift < 0 ? digits.slice(0, count) : digits + "0".repeat(shift); + const result = BigInt(canonical); + if (result > MAX_VARINT) + return invalid("qlog QUIC integer exceeds the 62-bit range.", pointer); + return result; +}; + +/** Preserve JSON values/lexemes and structurally exclude only transport credentials and declared values. */ +const project = ( + value: unknown, + sensitiveValues: readonly string[], +): Projection => { + const redactor = new CaptureRedaction(sensitiveValues); + const numeric_literals: Projection["numeric_literals"] = []; + const eventName = object(value) ? value.name : null; + const visit = ( + item: unknown, + pointer: string, + parent?: Record, + ): JsonValue => { + const header = + (eventName === "http:frame_created" || + eventName === "http:frame_parsed") && + /^\/data\/frame\/headers\/\d+\/value$/u.test(pointer) && + typeof parent?.name === "string"; + if ( + (header && credentials.has(String(parent?.name).toLowerCase())) || + qlogToken(eventName, pointer, parent) + ) + return redactor.credential(pointer); + if (item instanceof LosslessNumber) { + if ( + redactor.text(item.value, pointer) === null || + (isSafeNumber(item.value) && + redactor.text(String(Number(item.value)), pointer) === null) + ) + return null; + numeric_literals.push({ + pointer, + producer_type: "json-number", + literal: item.value, + }); + return isSafeNumber(item.value) ? Number(item.value) : null; + } + if (typeof item === "string") + return redactor.text( + item, + pointer, + header && + ["location", "referer", "origin"].includes( + String(parent?.name).toLowerCase(), + ), + ); + if (item === null || typeof item === "boolean") return item; + if (Array.isArray(item)) + return item.map((child, index) => + visit(child, capturePointer(pointer, String(index))), + ); + if (object(item)) + return Object.fromEntries( + Object.entries(item).flatMap(([key, child]) => { + if (sensitiveValues.some((literal) => key.includes(literal))) { + redactor.redactions.push({ + pointer, + reason: "explicit-sensitive-value", + scope: "property-name", + }); + return []; + } + return [[key, visit(child, capturePointer(pointer, key), item)]]; + }), + ); + return invalid("qlog parser returned a non-JSON value.", pointer); + }; + return { + reported: visit(value, ""), + numeric_literals, + binary_fields: [], + redactions: redactor.redactions, + }; +}; + +const qlogToken = ( + eventName: unknown, + pointer: string, + parent?: Record, +): boolean => { + if ( + packetEvent(eventName) && + /^\/data\/frames\/\d+\/reset_token$/u.test(pointer) && + parent?.frame_type === "new_connection_id" + ) + return true; + if ( + packetEvent(eventName) && + /^\/data\/frames\/\d+\/token$/u.test(pointer) && + parent?.frame_type === "new_token" + ) + return true; + if ( + eventName === "transport:parameters_set" && + pointer === "/data/stateless_reset_token" + ) + return true; + return false; +}; diff --git a/src/cli/webNetworkCaptureCommands.ts b/src/cli/webNetworkCaptureCommands.ts index 336c79364..655d34bc1 100644 --- a/src/cli/webNetworkCaptureCommands.ts +++ b/src/cli/webNetworkCaptureCommands.ts @@ -17,7 +17,7 @@ export const registerWebNetworkCaptureCommands = ( const service = createWebNetworkCaptureService(environment); cli.command(CLI_COMMANDS.inspectWebNetworkCapture, { description: - "Inspect retained HAR, native mitmproxy, or Fiddler SAZ records without fetching recorded URLs", + "Inspect retained HAR, native mitmproxy, Fiddler SAZ or aioquic qlog records without fetching recorded URLs", args: z.object({ capturePath: z.string().describe("Absolute local capture path"), format: z diff --git a/src/composition/webNetworkCaptures.ts b/src/composition/webNetworkCaptures.ts index 017182500..9d9e8c6cc 100644 --- a/src/composition/webNetworkCaptures.ts +++ b/src/composition/webNetworkCaptures.ts @@ -12,6 +12,7 @@ export const createWebNetworkCaptureService = ( [ new NodeCaptureAdapter("har"), new NodeCaptureAdapter("saz"), + new NodeCaptureAdapter("qlog"), new MitmproxyCaptureAdapter(environment), ], environment, diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 161d9d7b4..06506c83e 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -16,7 +16,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_evm_interface, inspect_managed_artifact, inspect_android_package or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. For layout, JavaScript, native, inventory, WASM, EVM, managed artifact and Android package sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions; occurrence logical_paths matches exact recorded strings (omitted means all available, [] means none). source_complete distinguishes complete inventories from partial source records; page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Native views select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_evm_interface, inspect_managed_artifact, inspect_android_package or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. For layout, JavaScript, native, inventory, WASM, EVM, managed artifact and Android package sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions; occurrence logical_paths matches exact recorded strings (omitted means all available, [] means none). source_complete distinguishes complete inventories from partial source records; page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Native views select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/contracts/webNetworkCaptureToolContracts.ts b/src/contracts/webNetworkCaptureToolContracts.ts index 06420f960..592dd9b54 100644 --- a/src/contracts/webNetworkCaptureToolContracts.ts +++ b/src/contracts/webNetworkCaptureToolContracts.ts @@ -16,7 +16,7 @@ export const WEB_NETWORK_CAPTURE_TOOL_CONTRACTS = [ name: "inspect_web_network_capture", ...toolContractMetadata("inspect_web_network_capture"), description: - "Request detail summary for capture identity and counts; then page retained records with inspect_analysis_view kind capture (preview or complete), avoiding repeated decoding. Default detail complete preserves existing inline delivery. Inspect retained local HAR 1.2, native mitmproxy, or Fiddler SAZ capture records without contacting recorded URLs. Returns original ordinals, pointers/byte ranges/archive member names, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate HAR members are invalid; prototype-named members such as __proto__ are preserved as ordinary own JSON members without prototype mutation. Uses pinned upstream parsers, owned temporary files and bounded offline processes. SAZ preserves HTTP fields, XML metadata and encoded body/message bytes without replay or HAR conversion; encrypted archives are unsupported. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", + "Request detail summary for capture identity and counts; then page retained records with inspect_analysis_view kind capture (preview or complete), avoiding repeated decoding. Default detail complete preserves existing inline delivery. Inspect retained local HAR 1.2, native mitmproxy, Fiddler SAZ or aioquic 1.5.0 JSON qlog 0.3 events without contacting recorded URLs. Returns original ordinals, pointers/byte ranges/archive member names, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate JSON members are invalid; prototype-named members such as __proto__ are preserved as ordinary own JSON members without prototype mutation. Qlog reports trace-scoped STREAM interval unions and endpoint loss declarations; it does not recover bodies, prove network loss or correlate PCAP. Other qlog layouts are unsupported. Uses pinned upstream parsers, owned temporary files and bounded offline processes. SAZ preserves HTTP fields, XML metadata and encoded body/message bytes without replay or HAR conversion; encrypted archives are unsupported. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", kind: "browser-provider", inputSchema: inspectWebNetworkCaptureInputSchema.extend({ detail: analysisDetailSchema, @@ -26,6 +26,14 @@ export const WEB_NETWORK_CAPTURE_TOOL_CONTRACTS = [ webNetworkCaptureSummaryViewResultSchema, ), examples: [ + { + title: "Inspect retained aioquic qlog events", + input: { + capture_path: "/captures/origin.qlog", + format: "qlog", + record_ordinals: [0, 2], + }, + }, { title: "Inspect historical HAR entries", input: { diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 6b3b6bef8..05f879024 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -62,6 +62,11 @@ import { captureSummarySchema, } from "./webNetworkCaptureView.js"; import type { WebNetworkApiSurfaceAccumulator } from "./WebNetworkApiSurfaceAccumulator.js"; +import { + qlogViewRequestSchema, + qlogViewPageSchema, + qlogViewContextSchema, +} from "./qlogCaptureView.js"; import { projectAndroidPackageView, androidPackageSummarySchema, @@ -184,6 +189,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ androidViewRequestSchema, apiSurfaceViewSchema, captureViewRequestSchema, + qlogViewRequestSchema, ]); // The application owns admission and authenticated Evidence parsing. SDK and @@ -403,6 +409,14 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ ...viewResultBase, parent_operation: z.literal("inspect_web_network_capture"), }), + z.strictObject({ + kind: z.literal("qlog"), + view: qlogViewRequestSchema, + page: qlogViewPageSchema, + context: qlogViewContextSchema, + ...viewResultBase, + parent_operation: z.literal("inspect_web_network_capture"), + }), z.strictObject({ kind: z.literal("facet"), view: facetViewSchema, @@ -523,6 +537,7 @@ export type UnsignedAnalysisView = > | Omit, "view_digest"> | Omit, "view_digest"> + | Omit, "view_digest"> | Omit< Extract, "view_digest" diff --git a/src/domain/analysisView/qlogCaptureView.ts b/src/domain/analysisView/qlogCaptureView.ts new file mode 100644 index 000000000..a3bdf88c3 --- /dev/null +++ b/src/domain/analysisView/qlogCaptureView.ts @@ -0,0 +1,248 @@ +import { z } from "zod"; +import { + qlogCaptureFactsSchema, + type QlogCaptureFacts, +} from "../qlogCapture.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { analysisViewInputError } from "./analysisView.js"; +import { pageViewCoverage } from "./analysisViewCoverage.js"; + +const traceSchema = qlogCaptureFactsSchema.shape.traces.element; +const streamSchema = traceSchema.shape.streams.element; +export const qlogViewContextSchema = qlogCaptureFactsSchema.omit({ + traces: true, +}); +export const qlogSummarySchema = qlogViewContextSchema.extend({ + trace_count: z.number().int().nonnegative(), + retained_stream_count: z.number().int().nonnegative(), + loss_declaration_count: z.number().int().nonnegative(), + incomplete_fact_record_count: z.number().int().nonnegative(), +}); + +/** Select retained trace facts without copying event inventories or every contribution. */ +const pageRequestShape = { + kind: z.literal("qlog"), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}; +const traceSelectorShape = { trace_index: z.number().int().nonnegative() }; +const streamSelectorShape = { + ...traceSelectorShape, + stream_id: streamSchema.shape.stream_id, + direction: streamSchema.shape.direction, +}; +export const qlogViewRequestSchema = z.discriminatedUnion("facet", [ + z.strictObject({ ...pageRequestShape, facet: z.literal("traces") }), + z.strictObject({ + ...pageRequestShape, + ...traceSelectorShape, + facet: z.literal("streams"), + }), + z.strictObject({ + ...pageRequestShape, + ...traceSelectorShape, + facet: z.literal("loss-declarations"), + }), + z.strictObject({ + ...pageRequestShape, + ...traceSelectorShape, + facet: z.literal("incomplete-records"), + }), + z.strictObject({ + ...pageRequestShape, + ...streamSelectorShape, + facet: z.literal("contributions"), + }), + z.strictObject({ + ...pageRequestShape, + ...streamSelectorShape, + facet: z.literal("ranges"), + }), + z.strictObject({ + ...pageRequestShape, + ...streamSelectorShape, + facet: z.literal("final-sizes"), + }), +]); +export type QlogViewRequest = z.output; + +const traceSummarySchema = traceSchema + .omit({ + streams: true, + loss_declarations: true, + incomplete_fact_records: true, + }) + .extend({ + stream_count: z.number().int().nonnegative(), + loss_declaration_count: z.number().int().nonnegative(), + incomplete_fact_record_count: z.number().int().nonnegative(), + }); +const streamSummarySchema = streamSchema + .omit({ + contributions: true, + covered_ranges: true, + reported_final_sizes: true, + }) + .extend({ + contribution_count: z.number().int().nonnegative(), + covered_range_count: z.number().int().nonnegative(), + reported_final_size_count: z.number().int().nonnegative(), + }); +/** Each page advertises exactly the retained item contract for its facet. */ +export const qlogViewPageSchema = z.discriminatedUnion("facet", [ + z.strictObject({ + facet: z.literal("traces"), + items: z.array(traceSummarySchema), + }), + z.strictObject({ + facet: z.literal("streams"), + items: z.array(streamSummarySchema), + }), + z.strictObject({ + facet: z.literal("loss-declarations"), + items: traceSchema.shape.loss_declarations, + }), + z.strictObject({ + facet: z.literal("incomplete-records"), + items: traceSchema.shape.incomplete_fact_records, + }), + z.strictObject({ + facet: z.literal("contributions"), + items: streamSchema.shape.contributions, + }), + z.strictObject({ + facet: z.literal("ranges"), + items: streamSchema.shape.covered_ranges, + }), + z.strictObject({ + facet: z.literal("final-sizes"), + items: streamSchema.shape.reported_final_sizes, + }), +]); +type QlogViewPage = z.output; + +export const summarizeQlogCapture = (qlog: QlogCaptureFacts) => { + const { traces, ...context } = qlog; + return { + ...context, + trace_count: traces.length, + retained_stream_count: traces.reduce( + (sum, trace) => sum + trace.streams.length, + 0, + ), + loss_declaration_count: traces.reduce( + (sum, trace) => sum + trace.loss_declarations.length, + 0, + ), + incomplete_fact_record_count: traces.reduce( + (sum, trace) => sum + trace.incomplete_fact_records.length, + 0, + ), + }; +}; + +const summarizeTrace = ({ + streams, + loss_declarations, + incomplete_fact_records, + ...trace +}: QlogCaptureFacts["traces"][number]) => ({ + ...trace, + stream_count: streams.length, + loss_declaration_count: loss_declarations.length, + incomplete_fact_record_count: incomplete_fact_records.length, +}); +const summarizeStream = ({ + contributions, + covered_ranges, + reported_final_sizes, + ...stream +}: QlogCaptureFacts["traces"][number]["streams"][number]) => ({ + ...stream, + contribution_count: contributions.length, + covered_range_count: covered_ranges.length, + reported_final_size_count: reported_final_sizes.length, +}); + +/** Array offsets refer to retained facts, whose scope remains all recorded events. */ +export const projectQlogCapturePage = ( + qlog: QlogCaptureFacts, + view: QlogViewRequest, +): Result< + { + context: z.output; + page: QlogViewPage; + coverage: ReturnType; + }, + AnalysisError +> => { + const invalid = (field: string, message: string) => + err( + analysisViewInputError([ + { path: ["view", field], reason: "invalid_value", message }, + ]), + ); + const slice = (items: readonly T[]) => + items.slice(view.offset, view.offset + view.limit); + let page: QlogViewPage; + let total: number; + if (view.facet === "traces") { + total = qlog.traces.length; + page = { + facet: "traces", + items: slice(qlog.traces).map(summarizeTrace), + }; + } else { + const matches = qlog.traces.filter( + (trace) => trace.trace_index === view.trace_index, + ); + if (matches.length !== 1) + return invalid( + "trace_index", + "Select one unambiguous retained trace index.", + ); + const trace = matches[0]!; + if (view.facet === "streams") { + total = trace.streams.length; + page = { + facet: "streams", + items: slice(trace.streams).map(summarizeStream), + }; + } else if (view.facet === "loss-declarations") { + total = trace.loss_declarations.length; + page = { facet: view.facet, items: slice(trace.loss_declarations) }; + } else if (view.facet === "incomplete-records") { + total = trace.incomplete_fact_records.length; + page = { facet: view.facet, items: slice(trace.incomplete_fact_records) }; + } else { + const matches = trace.streams.filter( + (stream) => + stream.stream_id === view.stream_id && + stream.direction === view.direction, + ); + if (matches.length !== 1) + return invalid( + "stream_id", + "Select one unambiguous retained stream ID and direction within the trace.", + ); + const stream = matches[0]!; + if (view.facet === "contributions") { + total = stream.contributions.length; + page = { facet: view.facet, items: slice(stream.contributions) }; + } else if (view.facet === "ranges") { + total = stream.covered_ranges.length; + page = { facet: view.facet, items: slice(stream.covered_ranges) }; + } else { + total = stream.reported_final_sizes.length; + page = { facet: view.facet, items: slice(stream.reported_final_sizes) }; + } + } + } + const { traces: _traces, ...context } = qlog; + return ok({ + context, + page, + coverage: pageViewCoverage(view.offset, page.items.length, total), + }); +}; diff --git a/src/domain/analysisView/webNetworkCaptureView.ts b/src/domain/analysisView/webNetworkCaptureView.ts index 07495466c..b335e9241 100644 --- a/src/domain/analysisView/webNetworkCaptureView.ts +++ b/src/domain/analysisView/webNetworkCaptureView.ts @@ -7,7 +7,11 @@ import { webNetworkCaptureRecordSchema, webNetworkCaptureBinaryDescriptorSchema, webNetworkCaptureSchema, + webNetworkCaptureArtifactSchema, + webNetworkCaptureDecoderSchema, + WEB_NETWORK_CAPTURE_FORMATS, type WebNetworkCaptureRecord, + type WebNetworkCapture, } from "../webNetworkCapture.js"; import { analysisViewInputError, @@ -19,6 +23,11 @@ import { completeWithinViewCoverage, pageViewCoverage, } from "./analysisViewCoverage.js"; +import { + qlogSummarySchema, + summarizeQlogCapture, + projectQlogCapturePage, +} from "./qlogCaptureView.js"; /** Offsets page the parent's retained record order, preserving producer ordinals. */ export const captureViewRequestSchema = z.strictObject({ @@ -37,12 +46,13 @@ export const captureViewItemsSchema = z.array( /** Count/identity overview; bodies and the archive member inventory stay in the parent. */ export const captureSummarySchema = z.strictObject({ - artifact: webNetworkCaptureSchema.shape.artifact, - decoder: webNetworkCaptureSchema.shape.decoder, - format: webNetworkCaptureSchema.shape.format, + artifact: webNetworkCaptureArtifactSchema, + decoder: webNetworkCaptureDecoderSchema, + format: z.enum(WEB_NETWORK_CAPTURE_FORMATS), total_records: z.number().int().nonnegative(), retained_records: z.number().int().nonnegative(), runtime_attribution: z.literal("unknown"), + qlog: qlogSummarySchema.optional(), }); const object = ( @@ -104,22 +114,10 @@ const preview = (record: WebNetworkCaptureRecord) => { }; }; -/** Page immutable retained facts; no capture file, provider or network is consulted. */ -export const projectWebNetworkCaptureView = ( +/** Authenticate the retained report and its subject at one owning boundary. */ +const parseCaptureParent = ( parent: AnalysisViewParent, - view: AnalysisViewRequest, -): Result => { - if (view.kind !== "summary" && view.kind !== "capture") - return err( - analysisViewInputError([ - { - path: ["view"], - reason: "invalid_value", - message: - "Historical captures support summary, capture record pages, and admitted api-surface views.", - }, - ]), - ); +): Result => { const parsed = webNetworkCaptureSchema.safeParse(parent.normalizedResult); if (!parsed.success) return err( @@ -150,6 +148,32 @@ export const projectWebNetworkCaptureView = ( }, ]), ); + return ok(report); +}; + +/** Page immutable retained facts; no capture file, provider or network is consulted. */ +export const projectWebNetworkCaptureView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if ( + view.kind !== "summary" && + view.kind !== "capture" && + view.kind !== "qlog" + ) + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Historical captures support summary, capture record pages, qlog fact pages and admitted api-surface views.", + }, + ]), + ); + const parsed = parseCaptureParent(parent); + if (!parsed.ok) return parsed; + const report = parsed.value; const shared = { parent_evidence_id: parent.evidenceId, parent_operation: "inspect_web_network_capture" as const, @@ -159,7 +183,7 @@ export const projectWebNetworkCaptureView = ( ...new Set([ ...parent.limitations, ...report.limitations, - "Pages cover the records selected into the parent Evidence, in that selected order; total_records counts the original decoded capture.", + "Capture record pages cover the records selected into the parent Evidence, in that selected order; total_records counts the original decoded capture.", ]), ], unknowns: [ @@ -181,12 +205,40 @@ export const projectWebNetworkCaptureView = ( total_records: report.total_records, retained_records: report.records.length, runtime_attribution: report.runtime_attribution, + ...(report.format === "qlog" + ? { qlog: summarizeQlogCapture(report.qlog) } + : {}), }, coverage: completeWithinViewCoverage( report.records.length, report.records.length, ), }); + if (view.kind === "qlog") { + if (report.format !== "qlog") + return err( + analysisViewInputError([ + { + path: ["view", "kind"], + reason: "invalid_value", + message: "Qlog fact views require retained qlog Evidence.", + }, + ]), + ); + const projected = projectQlogCapturePage(report.qlog, view); + return projected.ok + ? ok({ + kind: "qlog", + view, + ...shared, + ...projected.value, + limitations: [ + ...shared.limitations, + "Qlog fact pages retain all-recorded-events-before-selection scope. Exhausting a page does not establish network completeness, body recovery, retransmission or actual loss.", + ], + }) + : projected; + } const records = report.records.slice(view.offset, view.offset + view.limit); return ok({ kind: "capture", diff --git a/src/domain/qlogCapture.ts b/src/domain/qlogCapture.ts new file mode 100644 index 000000000..575cf2a60 --- /dev/null +++ b/src/domain/qlogCapture.ts @@ -0,0 +1,76 @@ +import { z } from "zod"; + +const decimal = z.string().regex(/^(0|[1-9][0-9]*)$/u); +const unsignedFact = z.discriminatedUnion("state", [ + z.strictObject({ state: z.literal("reported"), value: decimal }), + z.strictObject({ + state: z.literal("unknown"), + reason: z.literal("omitted-null-or-redacted"), + }), +]); +const traceLocation = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("json-pointer"), pointer: z.string() }), + z.strictObject({ + kind: z.literal("unknown"), + reason: z.literal("explicit-sensitive-value"), + }), +]); + +/** Exact, source-bound transport facts; coverage never establishes network completeness. */ +export const qlogCaptureFactsSchema = z.strictObject({ + profile: z.literal("aioquic-1.5.0-json-0.3"), + scope: z.literal("all-recorded-events-before-selection"), + body_bytes: z.literal("not-reported"), + clock: z.strictObject({ + unit: z.literal("milliseconds"), + domain: z.literal("unix-epoch"), + basis: z.literal("selected-aioquic-profile-not-producer-authentication"), + }), + traces: z.array( + z.strictObject({ + trace_index: z.number().int().nonnegative(), + location: traceLocation, + event_count: z.number().int().nonnegative(), + odcid: z.string().nullable(), + vantage_point: z.enum(["client", "server"]).nullable(), + incomplete_fact_records: z.array(z.number().int().nonnegative()), + loss_declarations: z.array( + z.strictObject({ + record_ordinal: z.number().int().nonnegative(), + confidence: z.literal("observed"), + packet_number: unsignedFact, + packet_type: z.string().nullable(), + packet_number_space: z + .enum(["initial", "handshake", "application"]) + .nullable(), + }), + ), + streams: z.array( + z.strictObject({ + stream_id: decimal, + direction: z.enum(["sent", "received"]), + confidence: z.literal("derived"), + scope: z.literal("observed-stream-frame-ranges"), + contributions: z.array( + z.strictObject({ + record_ordinal: z.number().int().nonnegative(), + frame_index: z.number().int().nonnegative(), + offset: decimal, + length: decimal, + fin: z.boolean().nullable(), + }), + ), + covered_ranges: z.array( + z.strictObject({ start: decimal, end: decimal }), + ), + unique_bytes: decimal, + frame_length_sum: decimal, + repeated_bytes: decimal, + reported_final_sizes: z.array(decimal), + }), + ), + }), + ), +}); + +export type QlogCaptureFacts = z.output; diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts index ec98ca496..77752f8e5 100644 --- a/src/domain/webNetworkCapture.ts +++ b/src/domain/webNetworkCapture.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { jsonValueSchema } from "./jsonValue.js"; import { localPathStringSchema } from "./localPath.js"; +import { qlogCaptureFactsSchema } from "./qlogCapture.js"; /** Complete historical evidence budgets; exceeding a budget returns no partial capture. */ export const WEB_NETWORK_CAPTURE_LIMITS = { @@ -10,12 +11,17 @@ export const WEB_NETWORK_CAPTURE_LIMITS = { depth: 64, } as const; -export const WEB_NETWORK_CAPTURE_FORMATS = ["har", "mitmproxy", "saz"] as const; +export const WEB_NETWORK_CAPTURE_FORMATS = [ + "har", + "mitmproxy", + "saz", + "qlog", +] as const; /** Inspect an explicit retained capture without contacting its recorded destinations. */ export const inspectWebNetworkCaptureInputSchema = z.strictObject({ capture_path: localPathStringSchema.describe( - "Absolute local HAR, mitmproxy, or Fiddler SAZ capture path.", + "Absolute local HAR, mitmproxy, Fiddler SAZ or aioquic qlog capture path.", ), format: z.enum(WEB_NETWORK_CAPTURE_FORMATS), record_ordinals: z @@ -123,69 +129,85 @@ export const webNetworkCaptureRecordSchema = z.strictObject({ limitations: z.array(z.string()), }); -/** Decoded records retain unique producer ordinals within their declared coverage. */ -export const webNetworkCapturePayloadSchema = z - .strictObject({ - decoder: z.strictObject({ - id: z.string().min(1), - name: z.string().min(1), - version: z.string().min(1), - }), - container: z.strictObject({ - reported: jsonValueSchema, - numeric_literals: z.array(numberSchema), - redactions: z.array(redactionSchema), - records_pointer: pointerSchema.nullable(), - }), - total_records: z.number().int().nonnegative(), - records: z.array(webNetworkCaptureRecordSchema), - }) - .superRefine((capture, context) => { - if (capture.records.length > capture.total_records) { +/** The selected offline parser profile, independent of authenticated producer identity. */ +export const webNetworkCaptureDecoderSchema = z.strictObject({ + id: z.string().min(1), + name: z.string().min(1), + version: z.string().min(1), +}); + +/** Source coverage is shared by every offline decoder. */ +const payloadShape = { + decoder: webNetworkCaptureDecoderSchema, + container: z.strictObject({ + reported: jsonValueSchema, + numeric_literals: z.array(numberSchema), + redactions: z.array(redactionSchema), + records_pointer: pointerSchema.nullable(), + }), + total_records: z.number().int().nonnegative(), + records: z.array(webNetworkCaptureRecordSchema), +}; +const refineCoverage = ( + capture: { total_records: number; records: { ordinal: number }[] }, + context: z.RefinementCtx, +): void => { + const seen = new Set(); + for (const [index, record] of capture.records.entries()) { + if (record.ordinal >= capture.total_records || seen.has(record.ordinal)) { context.addIssue({ code: "custom", - path: ["records"], - message: "Retained record count exceeds the declared capture total.", + path: ["records", index, "ordinal"], + message: + "Retained ordinals must be unique and within the declared capture total.", }); return; } - const seen = new Set(); - for (const [index, record] of capture.records.entries()) { - if (record.ordinal >= capture.total_records || seen.has(record.ordinal)) { - context.addIssue({ - code: "custom", - path: ["records", index, "ordinal"], - message: - "Retained ordinals must be unique and within the declared capture total.", - }); - return; - } - seen.add(record.ordinal); - } - }); - -/** Historical capture inspection has file identity, producer provenance and explicit unknowns. */ -export const webNetworkCaptureSchema = - webNetworkCapturePayloadSchema.safeExtend({ - artifact: z.strictObject({ - path: z - .string() - .describe( - "Selected artifact path; empty when explicitly excluded from Evidence.", - ), - sha256: z - .string() - .regex(/^[0-9a-f]{64}$/) - .nullable() - .describe( - "Observed artifact SHA-256; null only when explicitly excluded by a sensitive declaration.", - ), - bytes: z.number().int().nonnegative(), + seen.add(record.ordinal); + } +}; +/** Strict alternatives prevent undefined-valued facets from entering JSON Evidence. */ +export const webNetworkCapturePayloadSchema = z + .union([ + z.strictObject({ ...payloadShape, qlog: qlogCaptureFactsSchema }), + z.strictObject(payloadShape), + ]) + .superRefine(refineCoverage); +/** Caller-selected file identity; explicit exclusions remain unknown. */ +export const webNetworkCaptureArtifactSchema = z.strictObject({ + path: z + .string() + .describe( + "Selected artifact path; empty when explicitly excluded from Evidence.", + ), + sha256: z + .string() + .regex(/^[0-9a-f]{64}$/) + .nullable() + .describe("Observed SHA-256; null only when explicitly excluded."), + bytes: z.number().int().nonnegative(), +}); +const artifactShape = { + artifact: webNetworkCaptureArtifactSchema, + runtime_attribution: z.literal("unknown"), + limitations: z.array(z.string()), +}; +/** Qlog transport facts occur exactly on the qlog alternative. */ +export const webNetworkCaptureSchema = z + .discriminatedUnion("format", [ + z.strictObject({ + ...payloadShape, + ...artifactShape, + format: z.enum(["har", "mitmproxy", "saz"]), }), - format: z.enum(WEB_NETWORK_CAPTURE_FORMATS), - runtime_attribution: z.literal("unknown"), - limitations: z.array(z.string()), - }); + z.strictObject({ + ...payloadShape, + ...artifactShape, + format: z.literal("qlog"), + qlog: qlogCaptureFactsSchema, + }), + ]) + .superRefine(refineCoverage); export type InspectWebNetworkCaptureInput = z.output< typeof inspectWebNetworkCaptureInputSchema diff --git a/src/domain/webNetworkCaptureCoordinates.ts b/src/domain/webNetworkCaptureCoordinates.ts index 617dd7607..a975c6b4a 100644 --- a/src/domain/webNetworkCaptureCoordinates.ts +++ b/src/domain/webNetworkCaptureCoordinates.ts @@ -53,8 +53,30 @@ export const excludeCaptureCoordinates = ( numeric_literals: sidecars(value.container.numeric_literals), redactions: sidecars(value.container.redactions), }; + const qlog = + value.format === "qlog" + ? { + ...value.qlog, + traces: value.qlog.traces.map((trace) => { + if ( + trace.location.kind !== "json-pointer" || + !sensitive(trace.location.pointer) + ) + return trace; + excluded = true; + return { + ...trace, + location: { + kind: "unknown" as const, + reason: "explicit-sensitive-value" as const, + }, + }; + }), + } + : undefined; return { ...value, + ...(qlog === undefined ? {} : { qlog }), container, records, limitations: excluded diff --git a/src/server/registerAnalysisViewTool.ts b/src/server/registerAnalysisViewTool.ts index ebd204edd..d427c0cb4 100644 --- a/src/server/registerAnalysisViewTool.ts +++ b/src/server/registerAnalysisViewTool.ts @@ -7,7 +7,10 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "pino"; import { createEvidenceToolHandler } from "./contractToolHandler.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; -import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { + toolRegistrationOptions, + applicationValidatedInputSchema, +} from "./toolRegistrationOptions.js"; /** Bind selected-view projection to its named contract and session Evidence owner. */ export const registerAnalysisViewTool = ( @@ -18,6 +21,7 @@ export const registerAnalysisViewTool = ( withAdmittedAnalysis?: WithAdmittedAnalysis, ): void => { const contract = toolContract("inspect_analysis_view"); + const registration = toolRegistrationOptions(contract); const handler = createEvidenceToolHandler( server, (input) => Promise.resolve(inspectAnalysisView(input, evidenceById)), @@ -27,7 +31,10 @@ export const registerAnalysisViewTool = ( ); server.registerTool( contract.name, - toolRegistrationOptions(contract), + { + ...registration, + inputSchema: applicationValidatedInputSchema(registration.inputSchema), + }, handler(contract), ); }; diff --git a/src/server/registerWebNetworkCaptureTool.ts b/src/server/registerWebNetworkCaptureTool.ts index 8b1203745..3e9759504 100644 --- a/src/server/registerWebNetworkCaptureTool.ts +++ b/src/server/registerWebNetworkCaptureTool.ts @@ -1,5 +1,4 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; -import type { StandardSchemaWithJSON } from "@modelcontextprotocol/server"; import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; import { requireAnalysisViewRetention, @@ -15,7 +14,10 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "pino"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import { createEvidenceToolHandler } from "./contractToolHandler.js"; -import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { + toolRegistrationOptions, + applicationValidatedInputSchema, +} from "./toolRegistrationOptions.js"; /** Bind historical inspection to its named contract and caller-owned Evidence writer. */ export const registerWebNetworkCaptureTool = ( @@ -31,13 +33,7 @@ export const registerWebNetworkCaptureTool = ( // Keep the exact advertised contract; the shared service validates raw input // before effects so caller-declared sensitivity also covers invalid arguments. // SDK pre-validation otherwise emits unprojected issue paths as plain text. - const inputSchema: StandardSchemaWithJSON = { - "~standard": { - ...registration.inputSchema["~standard"], - vendor: "rea-application", - validate: (value: unknown) => ({ value }), - }, - }; + const inputSchema = applicationValidatedInputSchema(registration.inputSchema); const handler = createEvidenceToolHandler( server, async (rawInput, options) => { diff --git a/src/server/toolRegistrationOptions.ts b/src/server/toolRegistrationOptions.ts index 4615971d4..c943be3f2 100644 --- a/src/server/toolRegistrationOptions.ts +++ b/src/server/toolRegistrationOptions.ts @@ -1,9 +1,21 @@ import type { ToolContract } from "../contracts/toolContractTypes.js"; +import type { StandardSchemaWithJSON } from "@modelcontextprotocol/server"; import { toolInputSchemaWithMetadata, toolOutputSchemaWithMetadata, } from "../contracts/toolSchemaMetadata.js"; +/** Preserve advertised contracts while the application owns structured input failures. */ +export const applicationValidatedInputSchema = ( + schema: StandardSchemaWithJSON, +): StandardSchemaWithJSON => ({ + "~standard": { + ...schema["~standard"], + vendor: "rea-application", + validate: (value: unknown) => ({ value }), + }, +}); + /** Project the canonical Zod contracts directly into SDK registration. */ export const toolRegistrationOptions = ( contract: Contract, diff --git a/tests/acceptance/tools/qlogCapture.test.ts b/tests/acceptance/tools/qlogCapture.test.ts new file mode 100644 index 000000000..be67c3a34 --- /dev/null +++ b/tests/acceptance/tools/qlogCapture.test.ts @@ -0,0 +1,457 @@ +import { execFile } from "node:child_process"; +import { writeFile, truncate, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it as runtimeTest } from "vitest"; +import { + NODE_CAPTURE_HEAP_ENFORCED, + NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES, +} from "../../../src/browser/history/CaptureRelease.js"; +import { webNetworkCaptureSummaryViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + webNetworkCaptureSchema, + WEB_NETWORK_CAPTURE_LIMITS, +} from "../../../src/domain/webNetworkCapture.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const hasMemoryBound = + NODE_CAPTURE_HEAP_ENFORCED || + NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES !== null; +const it = runtimeTest.skipIf(!hasMemoryBound); +const cli = fileURLToPath(new URL("../../../scripts/rea.mjs", import.meta.url)); +const event = { + name: "transport:packet_sent", + time: 1791819895000.25, + data: { + header: { packet_type: "1RTT", packet_number: 5 }, + frames: [ + { frame_type: "stream", stream_id: 0, offset: 10, length: 5, fin: true }, + ], + }, +}; +const qlog = (events: unknown[]) => + JSON.stringify({ + qlog_format: "JSON", + qlog_version: "0.3", + traces: [ + { + vantage_point: { name: "aioquic", type: "server" }, + common_fields: { ODCID: "trace-local" }, + events, + }, + ], + }); + +const assertAdditionalQlogFacets = async ( + call: Awaited>["call"], + retained: { kind: string; evidence_id: string }, +): Promise => { + for (const [view, expected] of [ + [ + { kind: "qlog", facet: "traces", offset: 1, limit: 1 }, + [{ trace_index: 1, event_count: 1, stream_count: 1 }], + ], + [ + { kind: "qlog", facet: "loss-declarations", trace_index: 0 }, + [{ record_ordinal: 1, packet_number: { state: "reported", value: "5" } }], + ], + [{ kind: "qlog", facet: "incomplete-records", trace_index: 0 }, []], + [ + { + kind: "qlog", + facet: "ranges", + trace_index: 0, + stream_id: "0", + direction: "sent", + }, + [{ start: "10", end: "15" }], + ], + [ + { + kind: "qlog", + facet: "final-sizes", + trace_index: 0, + stream_id: "0", + direction: "sent", + }, + ["15"], + ], + [{ kind: "qlog", facet: "traces", offset: 99 }, []], + ] as const) { + const response = await call("inspect_analysis_view", { + source: retained, + view, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect( + parseEvidence(response.structuredContent).normalized_result, + ).toMatchObject({ + page: { items: expected }, + coverage: { exhausted: true }, + }); + } +}; + +it("pages original trace-scoped facts after source deletion without conflating selected events or identical stream IDs", async () => { + const root = await createTestTempDirectory("rea-qlog-pages-"); + const path = join(root, "input.qlog"); + const source = JSON.parse( + qlog([ + event, + { + name: "recovery:packet_lost", + time: 1791819895001, + data: { packet_number: 5, type: "1RTT" }, + }, + ]), + ); + source.traces.push({ + ...source.traces[0], + events: [ + { + ...event, + data: { + frames: [ + { + frame_type: "stream", + stream_id: 0, + offset: 100, + length: 3, + fin: true, + }, + ], + }, + }, + ], + }); + await writeFile(path, JSON.stringify(source)); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + detail: "summary", + record_ordinals: [2], + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const summary = webNetworkCaptureSummaryViewResultSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(summary).toMatchObject({ + summary: { + total_records: 3, + retained_records: 1, + qlog: { + trace_count: 2, + retained_stream_count: 2, + loss_declaration_count: 1, + }, + }, + }); + expect(JSON.stringify(summary)).not.toContain('"contributions"'); + if ( + !("parent_evidence_id" in summary) || + typeof summary.parent_evidence_id !== "string" + ) + throw new Error("Missing retained parent"); + const retained = { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }; + const parent = parseEvidence( + ( + await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + record_ordinals: [2], + }) + ).structuredContent, + ); + expect(parent.evidence_id).toBe(summary.parent_evidence_id); + const portablePath = join(root, "parent.json"); + await rm(path); + const pages = []; + for (const trace_index of [0, 1]) { + const page = await call("inspect_analysis_view", { + source: retained, + view: { kind: "qlog", facet: "streams", trace_index, limit: 1 }, + }); + expect(page.isError, JSON.stringify(page)).not.toBe(true); + pages.push(parseEvidence(page.structuredContent).normalized_result); + } + expect(pages[0]).toMatchObject({ + page: { items: [{ stream_id: "0", unique_bytes: "5" }] }, + context: { scope: "all-recorded-events-before-selection" }, + }); + expect(pages[1]).toMatchObject({ + page: { items: [{ stream_id: "0", unique_bytes: "3" }] }, + }); + await assertAdditionalQlogFacets(call, retained); + const view = { + kind: "qlog", + facet: "contributions", + trace_index: 0, + stream_id: "0", + direction: "sent", + offset: 0, + limit: 1, + }; + const contribution = parseEvidence( + (await call("inspect_analysis_view", { source: retained, view })) + .structuredContent, + ); + expect(contribution.normalized_result).toMatchObject({ + page: { + items: [{ record_ordinal: 0, frame_index: 0, offset: "10", length: "5" }], + }, + coverage: { total: 1, exhausted: true }, + }); + await writeFile( + portablePath, + JSON.stringify({ source: { kind: "inline", evidence: parent }, view }), + ); + const cliResult = await execute(process.execPath, [ + cli, + "inspect-analysis-view", + portablePath, + "--json", + ]); + expect(parseEvidence(JSON.parse(cliResult.stdout)).normalized_result).toEqual( + contribution.normalized_result, + ); + for (const invalid of [ + { kind: "qlog", facet: "streams" }, + { kind: "qlog", facet: "streams", trace_index: 99 }, + { ...view, stream_id: "99" }, + { kind: "qlog", facet: "traces", trace_index: 0 }, + ]) + expect( + parseMcpToolError( + await call("inspect_analysis_view", { + source: retained, + view: invalid, + }), + ), + ).toMatchObject({ error: { code: "invalid_request" } }); +}, 60_000); + +runtimeTest.runIf(!hasMemoryBound)( + "refuses qlog when this runtime cannot enforce a decoder memory bound", + async () => { + const { call } = await connectLocalToolsMcp(); + const root = await createTestTempDirectory("rea-qlog-unbounded-"); + const path = join(root, "input.qlog"); + await writeFile(path, qlog([event])); + expect( + parseMcpToolError( + await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + }), + ), + ).toMatchObject({ error: { code: "capability_unavailable" } }); + }, +); + +it("runs built CLI and real stdio SDK with exact evidence parity, selection, retained references and strict schemas", async () => { + const root = await createTestTempDirectory("rea-qlog-public-"); + const path = join(root, "input.qlog"); + await writeFile( + path, + qlog([ + event, + { + name: "recovery:packet_lost", + time: 1791819895001.125, + data: { packet_number: 5, type: "1RTT" }, + }, + ]), + ); + const { call, client } = await connectLocalToolsMcp(); + const input = { capture_path: path, format: "qlog", record_ordinals: [1, 0] }; + const response = await call("inspect_web_network_capture", input); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcp = parseEvidence(response.structuredContent); + const result = await execute(process.execPath, [ + cli, + "inspect-web-network-capture", + path, + "qlog", + "--record", + "1", + "--record", + "0", + "--json", + ]); + const cliEvidence = parseEvidence(JSON.parse(result.stdout)); + expect(cliEvidence.evidence_id).toBe(mcp.evidence_id); + expect(cliEvidence.normalized_result).toEqual(mcp.normalized_result); + expect(mcp.analysis_profile?.parameters).toMatchObject({ + qlog_profile: "aioquic-1.5.0-json-0.3", + }); + expect(mcp.authority).toBe("historical-reference"); + const decoded = webNetworkCaptureSchema.parse(mcp.normalized_result); + expect(decoded.records.map((record) => record.ordinal)).toEqual([1, 0]); + if (decoded.format !== "qlog") throw new Error("qlog format required"); + expect(decoded.qlog.traces[0]?.streams[0]).toMatchObject({ + unique_bytes: "5", + covered_ranges: [{ start: "10", end: "15" }], + }); + const retained = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: mcp.evidence_id }, + view: { kind: "api-surface" }, + }); + // An unsupported view over an existing retained parent proves lookup without pretending qlog is HAR. + expect(parseMcpToolError(retained)).toMatchObject({ + error: { code: "unsupported_target" }, + }); + await client.ping(); + for (const record_ordinals of [[2], [0, 0]]) { + expect( + parseMcpToolError( + await call("inspect_web_network_capture", { + ...input, + record_ordinals, + }), + ), + ).toMatchObject({ error: { code: "invalid_request" } }); + } + const hidden = await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + sensitive_values: ["trace-local", "/traces/0", "input.qlog"], + }); + expect(JSON.stringify(hidden)).not.toContain("trace-local"); + expect(JSON.stringify(hidden)).not.toContain("/traces/0"); + expect(JSON.stringify(hidden)).not.toContain("input.qlog"); + const hiddenEvidence = parseEvidence(hidden.structuredContent); + expect( + webNetworkCaptureSchema.parse(hiddenEvidence.normalized_result).artifact + .path, + ).toBe(""); + const invalid = await call("inspect_web_network_capture", { + ...input, + mysecret: true, + sensitive_values: ["mysecret"], + }); + expect(parseMcpToolError(invalid)).toMatchObject({ + error: { code: "invalid_request" }, + }); + expect(JSON.stringify(invalid)).not.toContain("mysecret"); +}, 60_000); + +it("fails whole-capture validation before selection and survives malformed, unsupported and oversized qlog", async () => { + const root = await createTestTempDirectory("rea-qlog-errors-"); + const path = join(root, "input.qlog"); + const { call, client } = await connectLocalToolsMcp(); + for (const [text, code] of [ + [qlog([event, { ...event, time: "invalid" }]), "invalid_request"], + [qlog([event]).slice(0, -1), "invalid_request"], + [ + qlog([event]).replace('"qlog_version":"0.3"', '"qlog_version":"0.4"'), + "capability_unavailable", + ], + [ + qlog([ + { + ...event, + extension: JSON.parse("[".repeat(65) + "0" + "]".repeat(65)), + }, + ]), + "invalid_request", + ], + ] as const) { + await writeFile(path, text); + expect( + parseMcpToolError( + await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + record_ordinals: [0], + }), + ), + ).toMatchObject({ error: { code } }); + const failed: unknown = await execute(process.execPath, [ + cli, + "inspect-web-network-capture", + path, + "qlog", + "--json", + ]).catch((cause: unknown) => cause); + expect(failed).toMatchObject({ + code: 1, + stdout: expect.stringContaining(`"code":"${code}"`), + }); + } + await truncate(path, WEB_NETWORK_CAPTURE_LIMITS.inputBytes + 1); + expect( + parseMcpToolError( + await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + }), + ), + ).toMatchObject({ error: { code: "invalid_request" } }); + await writeFile(path, qlog([event])); + await client.ping(); + expect( + ( + await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + }) + ).isError, + ).not.toBe(true); +}, 60_000); + +runtimeTest.skipIf(process.versions.bun !== undefined || !hasMemoryBound)( + "contains actual qlog decoder heap exhaustion and keeps MCP usable", + async () => { + const root = await createTestTempDirectory("rea-qlog-capacity-"); + const path = join(root, "dense.qlog"); + const dense = qlog(Array(100_000).fill(event)); + expect(Buffer.byteLength(dense)).toBeLessThan( + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + ); + await writeFile(path, dense); + const { call, client } = await connectLocalToolsMcp(); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + record_ordinals: [0], + }); + expect(parseMcpToolError(response)).toMatchObject({ + error: { + code: "resource_constraint", + details: { + resource: "memory", + reported_limits: { old_generation_heap_mib: 192 }, + }, + }, + }); + const failed: unknown = await execute( + process.execPath, + [cli, "inspect-web-network-capture", path, "qlog", "--json"], + { timeout: 40_000, maxBuffer: 1024 * 1024 }, + ).catch((cause: unknown) => cause); + expect(failed).toMatchObject({ + code: 1, + stdout: expect.stringContaining('"code":"resource_constraint"'), + }); + await writeFile(path, qlog([event])); + await client.ping(); + expect( + ( + await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + }) + ).isError, + ).not.toBe(true); + }, + 60_000, +);