diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 0cebe1649..35cc1e620 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -18,6 +18,7 @@ REA can attach to a user-owned Chrome-family browser through the Chrome DevTools - `compare_web_captures` / `rea compare-web-captures` compares passive captures or exact-step-aligned browser scenarios. Scenario results commit reproducible literal normalization and expose alignment failures plus artifact-level action, screenshot, DOM, accessibility, URL, history, storage, and event differences. Missing or truncated evidence is never treated as equivalence. - `capture_web_screenshot` / `rea capture-web-screenshot` returns a content-addressed visible-viewport PNG. - `compare_web_screenshots` / `rea compare-web-screenshots` performs bounded local PNG pixel comparison without OCR or external services. +- The CLI comparison commands accept each capture, screenshot artifact or normalization policy as inline JSON or a JSON file path. Use files for screenshot artifacts and scenario captures, whose embedded PNG bytes can exceed the host's command-line length limit. - Every surface has equivalent CLI and MCP contracts and returns Evidence provenance. - MCP tools return complete results inline and include the Evidence ID. Session evidence can be exported with `export_evidence_bundle`. diff --git a/src/cli/browserAdvancedCommands.ts b/src/cli/browserAdvancedCommands.ts index 63cbbd756..1eff33f4b 100644 --- a/src/cli/browserAdvancedCommands.ts +++ b/src/cli/browserAdvancedCommands.ts @@ -15,13 +15,14 @@ import { type AnalysisInputIssue, } from "../domain/analysisErrorCore.js"; import { browserCaptureComparisonInputSchema } from "../domain/browserCaptureComparison.js"; +import { projectInputIssues } from "../domain/inputIssueProjection.js"; import { discoverWebMcpToolsInputSchema } from "../domain/webMcpDiscovery.js"; import { captureWebScreenshotInputSchema, compareWebScreenshotsInputSchema, } from "../domain/webScreenshot.js"; import type { JsonValue } from "../domain/jsonValue.js"; -import { safeParseJson } from "../domain/safeJson.js"; +import { parseCliJsonInput } from "../cliJsonInput.js"; import type { Logger } from "pino"; import { CLI_COMMANDS } from "../cliCommandNames.js"; @@ -79,46 +80,39 @@ const registerCaptureDiff = ( description: "Compare two normalized passive captures or recorded browser scenarios", args: z.object({ - beforeJson: z.string().describe("Earlier normalized web capture JSON"), - afterJson: z.string().describe("Later normalized web capture JSON"), + beforeJson: z + .string() + .describe("Earlier normalized web capture JSON or JSON file path"), + afterJson: z + .string() + .describe("Later normalized web capture JSON or JSON file path"), }), options: z.object({ normalizationJson: z .string() .default('{"rules":[]}') .describe( - "Recorded literal normalization policy for browser scenarios", + "Recorded literal normalization policy JSON or JSON file path for browser scenarios", ), }), run: ({ args, options }) => logCliCommand(logger, "compare-web-captures", async () => { - const before = safeParseJson(args.beforeJson); - const after = safeParseJson(args.afterJson); - const normalization = safeParseJson(options.normalizationJson); - if (!before.ok) - return inputError("compare_web_captures", [ - invalidJsonIssue("before"), - ]); - if (!after.ok) - return inputError("compare_web_captures", [ - invalidJsonIssue("after"), - ]); - if (!normalization.ok) - return inputError("compare_web_captures", [ - invalidJsonIssue("normalization"), - ]); + const inputs = await parseJsonArguments("compare_web_captures", { + before: args.beforeJson, + after: args.afterJson, + normalization: options.normalizationJson, + }); + if (!inputs.ok) return inputs.error; + const { before, after, normalization } = inputs.value; const scenarioComparison = browserCaptureComparisonInputSchema.safeParse({ - before_scenario: before.value, - after_scenario: after.value, - normalization: normalization.value, + before_scenario: before, + after_scenario: after, + normalization, }); const parsed = scenarioComparison.success ? scenarioComparison - : browserCaptureComparisonInputSchema.safeParse({ - before: before.value, - after: after.value, - }); + : browserCaptureComparisonInputSchema.safeParse({ before, after }); if (!parsed.success) return inputError("compare_web_captures"); const result = await compareWebCaptureEvidence( new CdpBrowserProvider(), @@ -168,8 +162,12 @@ const registerScreenshotDiff = ( cli.command(CLI_COMMANDS.compareWebScreenshots, { description: "Compare two self-verifying PNG artifact JSON values", args: z.object({ - beforeJson: z.string().describe("Earlier screenshot artifact JSON"), - afterJson: z.string().describe("Later screenshot artifact JSON"), + beforeJson: z + .string() + .describe("Earlier screenshot artifact JSON or JSON file path"), + afterJson: z + .string() + .describe("Later screenshot artifact JSON or JSON file path"), }), options: z.object({ channelThreshold: z @@ -182,12 +180,27 @@ const registerScreenshotDiff = ( }), run: ({ args, options }) => logCliCommand(logger, "compare-web-screenshots", async () => { - const parsed = compareWebScreenshotsInputSchema.safeParse({ - before: parseJson(args.beforeJson), - after: parseJson(args.afterJson), - channel_threshold: options.channelThreshold, + const operation = "compare_web_screenshots"; + const inputs = await parseJsonArguments(operation, { + before: args.beforeJson, + after: args.afterJson, }); - if (!parsed.success) return inputError("compare_web_screenshots"); + if (!inputs.ok) return inputs.error; + const { before, after } = inputs.value; + const input = { + before, + after, + channel_threshold: options.channelThreshold, + }; + const parsed = compareWebScreenshotsInputSchema.safeParse(input); + if (!parsed.success) + return browserCliError( + new AnalysisInputError( + operation, + { cause: parsed.error }, + projectInputIssues(parsed.error.issues, input), + ), + ); const result = await compareWebScreenshotEvidence( new CdpBrowserProvider(), parsed.data, @@ -197,11 +210,40 @@ const registerScreenshotDiff = ( }); }; -const parseJson = (value: string): unknown => { - const parsed = safeParseJson(value); - return parsed.ok ? parsed.value : undefined; +/** + * Parse each named inline JSON argument or local JSON file, like other + * JSON-input commands. Screenshot artifacts and scenario captures embed PNG + * bytes that can exceed the host's command-line length limit. + */ +const parseJsonArguments = async ( + operation: string, + fields: Readonly>, +): Promise< + | { readonly ok: true; readonly value: Readonly> } + | { readonly ok: false; readonly error: JsonValue } +> => { + const values: Record = {}; + for (const [field, value] of Object.entries(fields)) { + const parsed = await parseCliJsonInput(value, operation); + if (!parsed.ok) + return isFileInputFailure(parsed.error) + ? parsed + : { + ok: false, + error: inputError(operation, [invalidJsonIssue(field)]), + }; + values[field] = parsed.value; + } + return { ok: true, value: values }; }; +/** File read and decode failures name the selected path; inline JSON does not. */ +const isFileInputFailure = (error: JsonValue): boolean => + typeof error === "object" && + error !== null && + !Array.isArray(error) && + "input_path" in error; + const invalidJsonIssue = (field: string): AnalysisInputIssue => ({ path: [field], reason: "invalid_format", diff --git a/tests/boundary/cli/browserCli.test.ts b/tests/boundary/cli/browserCli.test.ts index 72c9e851d..bfd167a7e 100644 --- a/tests/boundary/cli/browserCli.test.ts +++ b/tests/boundary/cli/browserCli.test.ts @@ -1,15 +1,12 @@ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; - import { afterEach, describe, expect, it } from "vitest"; +import { runCliJson as runCli } from "../../fixtures/cliJsonProcess.js"; import { startFakeCdpBrowser, type FakeCdpBrowser, type FakeOptions, } from "../../fixtures/fakeCdpBrowser.js"; -const execute = promisify(execFile); const browsers: FakeCdpBrowser[] = []; const INTEGRATION_TEST_TIMEOUT_MS = 60_000; @@ -288,33 +285,6 @@ describe("browser CLI capture parity", () => { ); }); -const runCli = async ( - arguments_: readonly string[], - environment: NodeJS.ProcessEnv, -): Promise => { - try { - const { stdout } = await execute( - process.execPath, - ["scripts/rea.mjs", ...arguments_], - { - cwd: process.cwd(), - env: environment, - maxBuffer: 16 * 1_024 * 1_024, - }, - ); - return JSON.parse(stdout); - } catch (cause: unknown) { - if ( - typeof cause === "object" && - cause !== null && - "stdout" in cause && - typeof cause.stdout === "string" - ) - return JSON.parse(cause.stdout); - throw cause; - } -}; - const screenshotArtifact = (value: unknown): Record => { if ( !isRecord(value) || diff --git a/tests/boundary/cli/browserCompareJsonFiles.test.ts b/tests/boundary/cli/browserCompareJsonFiles.test.ts new file mode 100644 index 000000000..0f067175f --- /dev/null +++ b/tests/boundary/cli/browserCompareJsonFiles.test.ts @@ -0,0 +1,125 @@ +import { randomBytes } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { crc32, deflateSync } from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import { createWebScreenshotArtifact } from "../../../src/domain/webScreenshot.js"; +import { runCliJson as runCli } from "../../fixtures/cliJsonProcess.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const INTEGRATION_TEST_TIMEOUT_MS = 60_000; +// Linux limits one argument to 128 KiB and macOS all arguments to 1 MiB. +const COMMAND_LINE_LIMIT_BYTES = 1024 * 1024; + +const pngChunk = (type: string, data: Buffer): Buffer => { + const typed = Buffer.concat([Buffer.from(type, "latin1"), data]); + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const checksum = Buffer.alloc(4); + checksum.writeUInt32BE(crc32(typed)); + return Buffer.concat([length, typed, checksum]); +}; + +/** An incompressible RGB PNG whose artifact JSON exceeds command-line limits. */ +const noisePng = (width: number, height: number): Buffer => { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header.set([8, 2, 0, 0, 0], 8); + const rows = Array.from({ length: height }, () => + Buffer.concat([Buffer.of(0), randomBytes(width * 3)]), + ); + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", deflateSync(Buffer.concat(rows))), + pngChunk("IEND", Buffer.alloc(0)), + ]); +}; + +describe("browser comparison JSON file inputs", () => { + it( + "compares screenshot artifacts too large for a command-line argument", + async () => { + const root = await createTestTempDirectory("rea-screenshot-files-"); + const artifact = JSON.stringify( + createWebScreenshotArtifact(noisePng(640, 640)), + ); + expect(Buffer.byteLength(artifact)).toBeGreaterThan( + COMMAND_LINE_LIMIT_BYTES, + ); + const before = join(root, "before.json"); + const after = join(root, "after.json"); + await writeFile(before, artifact); + await writeFile(after, artifact); + + expect( + await runCli(["compare-web-screenshots", before, after, "--json"]), + ).toMatchObject({ + operation: "compare_web_screenshots", + normalized_result: { status: "identical", changed_pixels: 0 }, + }); + }, + INTEGRATION_TEST_TIMEOUT_MS, + ); + + it( + "reports which screenshot artifact failed validation", + async () => { + const root = await createTestTempDirectory("rea-screenshot-invalid-"); + const valid = createWebScreenshotArtifact(noisePng(2, 2)); + const before = join(root, "before.json"); + await writeFile(before, JSON.stringify({ ...valid, bytes: 1 })); + + expect( + await runCli([ + "compare-web-screenshots", + before, + JSON.stringify(valid), + "--json", + ]), + ).toMatchObject({ + error: "Browser observation failed", + code: "invalid_request", + details: { + operation: "compare_web_screenshots", + issues: [ + { + path: ["before"], + message: "Screenshot artifact digest or size mismatch", + }, + ], + }, + }); + }, + INTEGRATION_TEST_TIMEOUT_MS, + ); + + it( + "identifies a missing capture file instead of reporting malformed JSON", + async () => { + const root = await createTestTempDirectory("rea-capture-missing-"); + const missing = join(root, "before.json"); + + expect( + await runCli(["compare-web-captures", missing, "{}", "--json"]), + ).toMatchObject({ + code: "invalid_request", + details: { + operation: "compare_web_captures", + issues: [ + { + reason: "invalid_value", + message: expect.stringContaining(`(ENOENT): ${missing}`), + }, + ], + }, + input_path: missing, + input_reason: "read-failed", + }); + }, + INTEGRATION_TEST_TIMEOUT_MS, + ); +}); diff --git a/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts b/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts index 6608ce08d..6340c5c34 100644 --- a/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts +++ b/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts @@ -1,14 +1,11 @@ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; - import { afterEach, expect, it } from "vitest"; +import { runCliJson as runCli } from "../../fixtures/cliJsonProcess.js"; import { startFakeCdpBrowser, type FakeCdpBrowser, } from "../../fixtures/fakeCdpBrowser.js"; -const execute = promisify(execFile); const browsers: FakeCdpBrowser[] = []; afterEach(async () => { @@ -79,30 +76,6 @@ it("reports missing JSON shapes only when body shapes are selected", async () => }); }); -const runCli = async (arguments_: readonly string[]): Promise => { - try { - const { stdout } = await execute( - process.execPath, - ["scripts/rea.mjs", ...arguments_], - { - cwd: process.cwd(), - env: process.env, - maxBuffer: 16 * 1_024 * 1_024, - }, - ); - return JSON.parse(stdout); - } catch (cause: unknown) { - if ( - typeof cause === "object" && - cause !== null && - "stdout" in cause && - typeof cause.stdout === "string" - ) - return JSON.parse(cause.stdout); - throw cause; - } -}; - const normalizedResult = (value: unknown): Record => { if (!isRecord(value) || !isRecord(value.normalized_result)) throw new TypeError("Missing CLI normalized result"); diff --git a/tests/fixtures/cliJsonProcess.ts b/tests/fixtures/cliJsonProcess.ts new file mode 100644 index 000000000..c66c82d4e --- /dev/null +++ b/tests/fixtures/cliJsonProcess.ts @@ -0,0 +1,32 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); + +/** Run the built CLI and parse its JSON stdout, including a failing command's. */ +export const runCliJson = async ( + arguments_: readonly string[], + environment: NodeJS.ProcessEnv = process.env, +): Promise => { + try { + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", ...arguments_], + { + cwd: process.cwd(), + env: environment, + maxBuffer: 16 * 1_024 * 1_024, + }, + ); + return JSON.parse(stdout); + } catch (cause: unknown) { + if ( + typeof cause === "object" && + cause !== null && + "stdout" in cause && + typeof cause.stdout === "string" + ) + return JSON.parse(cause.stdout); + throw cause; + } +};