diff --git a/scripts/fixtures/browser-verifier-site.mjs b/scripts/fixtures/browser-verifier-site.mjs index d5ccb7923..7a5095e52 100644 --- a/scripts/fixtures/browser-verifier-site.mjs +++ b/scripts/fixtures/browser-verifier-site.mjs @@ -7,6 +7,10 @@ export async function startBrowserVerifierSite() { let port = 0; let sessionGeneration = 0; const server = createServer((request, response) => { + if (request.url === "/failed-navigation") { + response.destroy(); + return; + } if (request.url === "/slow-json") { response.setHeader("content-type", "application/json"); response.write('{"pending":'); diff --git a/scripts/lib/browser-scenario-verifier.mjs b/scripts/lib/browser-scenario-verifier.mjs index a41e7cec5..7a56f990c 100644 --- a/scripts/lib/browser-scenario-verifier.mjs +++ b/scripts/lib/browser-scenario-verifier.mjs @@ -1,9 +1,14 @@ +import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { readdir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { browserScenarioSchema } from "../../dist/domain/browserScenario.js"; +import { requireMcpToolError } from "./mcp-verifier-results.mjs"; const execute = promisify(execFile); @@ -142,6 +147,73 @@ export async function runScenarioCli(scenario) { return JSON.parse(stdout); } +/** Verify failed real navigation retains observations through both public adapters. */ +export async function verifyScenarioFailureEvidence(executable, origin) { + const scenario = browserScenarioSchema.parse({ + browser: { mode: "launch", executable_path: executable }, + start_url: { url: `${origin}/failed-navigation` }, + actions: [ + { step_id: "unused", action: "wait_for_timeout", duration_ms: 1 }, + ], + capture: { events: ["network"] }, + }); + const profilesBefore = await scenarioProfiles(); + let cliFailure; + try { + await runScenarioCli(scenario); + assert.fail("The fixture must fail before initial navigation completes"); + } catch (cause) { + assert.equal(typeof cause.stdout, "string"); + cliFailure = JSON.parse(cause.stdout); + } + assertFailedNavigation(cliFailure, scenario.start_url.url); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"], + env: process.env, + stderr: "pipe", + }); + const client = new Client({ + name: "browser-scenario-failure-e2e", + version: "1", + }); + try { + await client.connect(transport); + const result = await client.callTool({ + name: "capture_browser_scenario", + arguments: scenario, + }); + assert.equal(result.isError, true); + assertFailedNavigation(requireMcpToolError(result), scenario.start_url.url); + } finally { + await client.close(); + await transport.close(); + } + const profilesAfter = await scenarioProfiles(); + assert.ok([...profilesAfter].every((profile) => profilesBefore.has(profile))); + return { + mocked: false, + cli: true, + stdio_mcp: true, + failed_navigation_events_retained: true, + profile_cleanup: true, + }; +} + +const assertFailedNavigation = (error, requestedUrl) => { + assert.equal(error?.code, "execution_failure"); + const observation = error.details?.partial_observation; + assert.equal(observation?.kind, "browser-scenario-observation"); + assert.equal(observation.capture.browser.cleanup, "terminated-owned-process"); + assert.deepEqual(observation.capture.steps, []); + assert.ok( + observation.capture.events.items.some( + (event) => + event.kind === "request-failed" && event.url.url === requestedUrl, + ), + ); +}; + /** Snapshot provider-owned profile names for cleanup comparison. */ export async function scenarioProfiles() { return new Set( diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index 58ae384ee..5f05981d6 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -16,7 +16,7 @@ import { listBrowserTargetsInputSchema, } from "../dist/domain/browserObservation.js"; import { observeWebSessionInputSchema } from "../dist/domain/browserSession.js"; -import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiff.js"; +import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiffSchemas.js"; import { captureWebScreenshotInputSchema, compareWebScreenshotsInputSchema, @@ -33,6 +33,7 @@ import { browserScenario, runScenarioCli, scenarioProfiles, + verifyScenarioFailureEvidence, } from "./lib/browser-scenario-verifier.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs"; @@ -294,6 +295,10 @@ try { if ([...profilesAfter].some((entry) => !profilesBefore.has(entry))) throw new Error("Scenario launch retained a temporary browser profile"); assertScenarioCapture(launchedScenario.value); + const scenarioFailure = await verifyScenarioFailureEvidence( + executable, + site.origin, + ); const scenarioResults = JSON.stringify([ attachedScenario, launchedScenario.value, @@ -360,6 +365,7 @@ try { browserScenarioCli: true, browserScenarioAttachCleanup: "disconnected-external", browserScenarioLaunchCleanup: "terminated-owned-process", + scenarioFailure, verified: true, }; } finally { diff --git a/src/browser/BrowserScenarioPartialObservation.ts b/src/browser/BrowserScenarioPartialObservation.ts new file mode 100644 index 000000000..a2e600bcd --- /dev/null +++ b/src/browser/BrowserScenarioPartialObservation.ts @@ -0,0 +1,165 @@ +import type { BrowserScenario } from "../domain/browserScenario.js"; +import type { + BrowserScenarioCaptureData, + BrowserScenarioPartialObservation, +} from "../domain/browserScenarioCapture.js"; +import type { BrowserScenarioStep } from "../domain/browserScenarioCaptureValues.js"; +import type { BrowserScenarioSessionPort } from "./BrowserScenarioSessionPort.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisTimeoutError, +} from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { BrowserObservationError } from "../domain/browserObservationError.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js"; + +const OPERATION = "capture_browser_scenario" as const; + +const browserScenarioLimitations = ( + session: BrowserScenarioSessionPort, +): string[] => [ + "Event sequence records provider receipt order; simultaneous browser causality is not inferred.", + "Network content is retained only when selected; response bytes are browser-decoded, not compressed wire bytes.", + "Request bytes are limited to what Playwright exposes; not_exposed does not establish body absence or multipart file coverage.", + "Network content reads settle within 5 seconds each and never wait for unfinished responses or refetch them.", + "Playwright scenario events do not expose request initiator stacks; receipt order does not prove causality.", + "Storage values are hashed only after declared-secret redaction.", + ...(session.eventLimitations?.() ?? []), + ...(session.mode === "connect" + ? [ + "CDP attachment cannot recover pre-attach events or guarantee launch-time context options.", + ] + : []), +]; + +/** Project one session into the common capture fields used by success and failure. */ +export const browserScenarioCaptureData = (input: { + readonly session: BrowserScenarioSessionPort; + readonly scenario: BrowserScenario; + readonly startedAt: number; + readonly steps: readonly BrowserScenarioStep[]; + readonly cleanup: + | "terminated-owned-process" + | "disconnected-external" + | "incomplete"; + readonly limitations?: readonly string[]; +}): BrowserScenarioCaptureData => ({ + browser: { + mode: input.session.mode, + process_ownership: input.session.processOwnership, + cleanup: input.cleanup, + product: input.session.product, + version: input.session.version, + }, + scenario: { + start_origin: new URL(input.scenario.start_url.url).origin, + action_count: input.scenario.actions.length, + secret_references: input.scenario.secrets + .map(({ secret_id: id }) => id) + .sort(), + network_content: input.scenario.capture.network, + }, + duration_ms: Date.now() - input.startedAt, + steps: [...input.steps], + events: input.session.events(), + limitations: [ + ...browserScenarioLimitations(input.session), + ...(input.limitations ?? []), + ], +}); + +/** Describe the cleanup failure without discarding its resource evidence. */ +export const browserScenarioCleanupObservation = ( + cause: unknown, +): { readonly reason: string; readonly resources: readonly string[] } => + cause instanceof AnalysisError && cause.cleanup !== undefined + ? cause.cleanup + : { + reason: cause instanceof Error ? cause.message : String(cause), + resources: + cause instanceof AnalysisError && cause.cleanupResources.length > 0 + ? cause.cleanupResources + : ["browser_transport"], + }; + +/** Retain operation diagnostics and capture data after cleanup settles. */ +export const browserScenarioOperationFailure = ( + cause: unknown, + partialObservation: BrowserScenarioPartialObservation, + cleanupFailure?: { readonly cause: unknown }, +): AnalysisError => { + const primary = + cause instanceof AnalysisError + ? cause + : new ProviderAdapterError( + PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id, + OPERATION, + { cause }, + ); + const primaryProjection = projectAnalysisError(primary); + const cleanupCause = + cleanupFailure === undefined + ? undefined + : cleanupFailure.cause instanceof AnalysisError + ? cleanupFailure.cause + : new ProviderAdapterError( + PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id, + "close_browser_scenario", + { cause: cleanupFailure.cause }, + ); + const cleanupObservation = + cleanupFailure === undefined + ? undefined + : browserScenarioCleanupObservation(cleanupFailure.cause); + const retainedOptions = { + cause: primary, + partialObservation, + ...(primary.capturedOutput === undefined + ? {} + : { capturedOutput: primary.capturedOutput }), + ...(primary.cleanup === undefined ? {} : { cleanup: primary.cleanup }), + }; + if (cleanupCause === undefined) { + if (primary instanceof AnalysisCancelledError) + return new AnalysisCancelledError(primary.operation, retainedOptions); + if (primary instanceof AnalysisTimeoutError) + return new AnalysisTimeoutError( + primary.operation, + primary.timeoutMs, + retainedOptions, + ); + if (primary instanceof BrowserObservationError) + return new BrowserObservationError(primary.operation, primary.reason, { + ...retainedOptions, + ...(primary.userMessage === undefined + ? {} + : { detail: primary.userMessage }), + }); + } + const cleanup = cleanupObservation ?? primary.cleanup; + return new ProviderAdapterError( + PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id, + OPERATION, + { + ...retainedOptions, + cause: + cleanupCause === undefined + ? primary + : new AggregateError( + [primary, cleanupCause], + "Browser scenario operation and cleanup both failed", + { cause: primary }, + ), + userMessage: primaryProjection.message, + ...(cleanup === undefined ? {} : { cleanup }), + diagnostics: { + primary_error: primaryProjection, + ...(cleanupCause === undefined + ? {} + : { cleanup_error: projectAnalysisError(cleanupCause) }), + }, + }, + ); +}; diff --git a/src/browser/PlaywrightBrowserScenarioProvider.test.ts b/src/browser/PlaywrightBrowserScenarioProvider.test.ts index 96242a844..fd3cd8869 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.test.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.test.ts @@ -9,13 +9,19 @@ import { PlaywrightScenarioBrowserCleanupOwner } from "./PlaywrightScenarioBrows import { browserScenarioSchema, type BrowserScenario, - type BrowserScenarioAction, } from "../domain/browserScenario.js"; +import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js"; import { browserStepArtifactsSchema, type BrowserScenarioEvent, type BrowserStepArtifacts, -} from "../domain/browserScenarioCapture.js"; +} from "../domain/browserScenarioCaptureValues.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, + AnalysisTimeoutError, +} from "../domain/analysisErrorCore.js"; const scenario = ( options: { @@ -70,6 +76,13 @@ const scenario = ( type SnapshotKind = BrowserScenario["capture"]["after_each_step"][number]; +interface FakeSessionFailures { + readonly incomplete?: boolean; + readonly action?: Error; + readonly capture?: { readonly call: number; readonly error: Error }; + readonly close?: Error; +} + const artifactState = ( kind: SnapshotKind, requested: ReadonlySet, @@ -101,13 +114,11 @@ class FakeSession implements BrowserScenarioSessionPort { captureCalls = 0; onPerform: (() => void) | undefined; eventGaps: readonly string[] = []; - private step = 0; + eventItems: readonly BrowserScenarioEvent[] = []; constructor( readonly mode: "launch" | "connect", - private readonly incomplete = false, - private readonly failAction = false, - private readonly failCapture = false, + private readonly failures: FakeSessionFailures = {}, ) { this.processOwnership = mode === "launch" ? "provider-owned" : "external"; } @@ -120,9 +131,7 @@ class FakeSession implements BrowserScenarioSessionPort { return sanitizeBrowserUrl(value); } - setStep(index: number) { - this.step = index; - } + setStep(_index: number) {} nextEventSequence() { return 1; @@ -137,7 +146,11 @@ class FakeSession implements BrowserScenarioSessionPort { readonly dropped: number; readonly items: readonly BrowserScenarioEvent[]; } { - return { retained: 0, dropped: 0, items: [] }; + return { + retained: this.eventItems.length, + dropped: 0, + items: this.eventItems, + }; } eventLimitations(): readonly string[] { @@ -148,7 +161,7 @@ class FakeSession implements BrowserScenarioSessionPort { this.performCalls += 1; this.onPerform?.(); if (signal?.aborted === true) throw new Error("request cancelled"); - if (this.failAction) throw new Error("fixture secret action failure"); + if (this.failures.action !== undefined) throw this.failures.action; } capture( @@ -156,25 +169,40 @@ class FakeSession implements BrowserScenarioSessionPort { _signal?: AbortSignal, ): Promise { this.captureCalls += 1; - if (this.failCapture) return Promise.reject(new Error("capture failed")); + if (this.failures.capture?.call === this.captureCalls) + return Promise.reject(this.failures.capture.error); return Promise.resolve( browserStepArtifactsSchema.parse({ - screenshot: artifactState("screenshot", requested, this.incomplete), - dom: artifactState("dom", requested, this.incomplete), + screenshot: artifactState( + "screenshot", + requested, + this.failures.incomplete === true, + ), + dom: artifactState("dom", requested, this.failures.incomplete === true), accessibility: artifactState( "accessibility", requested, - this.incomplete, + this.failures.incomplete === true, + ), + url: artifactState("url", requested, this.failures.incomplete === true), + history: artifactState( + "history", + requested, + this.failures.incomplete === true, + ), + storage: artifactState( + "storage", + requested, + this.failures.incomplete === true, ), - url: artifactState("url", requested, this.incomplete), - history: artifactState("history", requested, this.incomplete), - storage: artifactState("storage", requested, this.incomplete), }), ); } close() { this.closeCalls += 1; + if (this.failures.close !== undefined) + return Promise.reject(this.failures.close); return Promise.resolve( this.mode === "launch" ? ("terminated-owned-process" as const) @@ -220,7 +248,7 @@ describe("PlaywrightBrowserScenarioProvider", () => { }); it("makes missing and truncated captures ineligible for equality", async () => { - const session = new FakeSession("launch", true); + const session = new FakeSession("launch", { incomplete: true }); const provider = new PlaywrightBrowserScenarioProvider({ open: () => Promise.resolve(session), }); @@ -237,7 +265,9 @@ describe("PlaywrightBrowserScenarioProvider", () => { }); it("records one failed step, cancels later actions, and still cleans up", async () => { - const session = new FakeSession("launch", false, true); + const session = new FakeSession("launch", { + action: new Error("fixture secret action failure"), + }); const provider = new PlaywrightBrowserScenarioProvider({ open: () => Promise.resolve(session), }); @@ -276,17 +306,187 @@ describe("PlaywrightBrowserScenarioProvider", () => { }); expect(session.closeCalls).toBe(1); }); +}); - it("closes the session when initial capture fails", async () => { - const session = new FakeSession("launch", false, false, true); +describe("PlaywrightBrowserScenarioProvider cleanup", () => { + it("returns a cleanup failure with completed observations when close fails", async () => { + const session = new FakeSession("launch", { + action: new Error("fixture secret action failure"), + close: new Error("browser close failed"), + }); const provider = new PlaywrightBrowserScenarioProvider({ open: () => Promise.resolve(session), }); const result = await provider.captureScenario(scenario()); + expect(result.ok).toBe(false); + if (result.ok) return; + const projection = projectAnalysisError(result.error); + expect(projection).toMatchObject({ + code: "cleanup_incomplete", + details: { + cleanup: "incomplete", + resources: ["browser_transport"], + partial_observation: { + kind: "browser-scenario-observation", + capture: { + browser: { cleanup: "incomplete" }, + steps: [ + { step_id: "scenario_start" }, + { step_id: "wait_0", status: "failed" }, + ], + }, + }, + }, + }); expect(session.closeCalls).toBe(1); }); +}); +describe("PlaywrightBrowserScenarioProvider operation failures", () => { + it.each([{ cleanupFails: false }, { cleanupFails: true }])( + "retains typed partial data when operation fails and cleanupFails=$cleanupFails", + async ({ cleanupFails }) => { + const session = new FakeSession("launch", { + capture: { + call: 2, + error: new AnalysisOutputError( + "capture_browser_scenario", + "step capture failed", + { + capturedOutput: { + stdout: "partial stdout", + stderr: "", + truncated: false, + }, + cleanup: { + reason: "capture cleanup is uncertain", + resources: ["capture_handle"], + }, + }, + ), + }, + ...(cleanupFails ? { close: new Error("browser close failed") } : {}), + }); + session.eventItems = [ + { + sequence: 1, + step_index: 0, + kind: "console", + level: "log", + text: "navigation started", + url: sanitizeBrowserUrl("https://app.example.test/"), + }, + ]; + const provider = new PlaywrightBrowserScenarioProvider({ + open: () => Promise.resolve(session), + }); + const result = await provider.captureScenario(scenario()); + + expect(result.ok).toBe(false); + if (result.ok) return; + const projection = projectAnalysisError(result.error); + expect(projection).toMatchObject({ + code: "cleanup_incomplete", + details: { + diagnostics: { + primary_error: { + code: "cleanup_incomplete", + details: { + operation: "capture_browser_scenario", + reason: "step capture failed", + }, + }, + ...(cleanupFails + ? { cleanup_error: { code: "execution_failure" } } + : {}), + }, + partial_observation: { + kind: "browser-scenario-observation", + capture: { + browser: { + cleanup: cleanupFails + ? "incomplete" + : "terminated-owned-process", + }, + steps: [{ step_id: "scenario_start" }], + events: { + items: [ + expect.objectContaining({ text: "navigation started" }), + ], + }, + }, + }, + }, + }); + expect(result.error.capturedOutput).toEqual({ + stdout: "partial stdout", + stderr: "", + truncated: false, + }); + expect(result.error.cleanup).toEqual( + cleanupFails + ? { + reason: "browser close failed", + resources: ["browser_transport"], + } + : { + reason: "capture cleanup is uncertain", + resources: ["capture_handle"], + }, + ); + expect(session.closeCalls).toBe(1); + }, + ); +}); + +describe("PlaywrightBrowserScenarioProvider cancellation and timeout", () => { + it.each([ + { + name: "cancellation", + error: new AnalysisCancelledError("capture_browser_scenario"), + code: "cancelled", + category: "cancelled", + details: { operation: "capture_browser_scenario", cleanup: "complete" }, + }, + { + name: "timeout", + error: new AnalysisTimeoutError("capture_browser_scenario", 250), + code: "provider_timeout", + category: "timeout", + details: { operation: "capture_browser_scenario", timeout_ms: 250 }, + }, + ])( + "preserves $name category and details when cleanup succeeds", + async ({ error, code, category, details }) => { + const session = new FakeSession("launch", { + capture: { call: 1, error }, + }); + const provider = new PlaywrightBrowserScenarioProvider({ + open: () => Promise.resolve(session), + }); + const result = await provider.captureScenario(scenario()); + + expect(result.ok).toBe(false); + if (result.ok) return; + const projection = projectAnalysisError(result.error); + expect(projection).toMatchObject({ + code, + category, + details: { + ...details, + partial_observation: { + kind: "browser-scenario-observation", + capture: { browser: { cleanup: "terminated-owned-process" } }, + }, + }, + }); + expect(session.closeCalls).toBe(1); + }, + ); +}); + +describe("PlaywrightBrowserScenarioProvider request cancellation", () => { it("does not open a session for an already-cancelled request", async () => { let opens = 0; const provider = new PlaywrightBrowserScenarioProvider({ diff --git a/src/browser/PlaywrightBrowserScenarioProvider.ts b/src/browser/PlaywrightBrowserScenarioProvider.ts index accc64d9d..b62d3504f 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.ts @@ -3,22 +3,23 @@ import type { ExecutionOptions, ProviderIdentity, } from "../application/AnalysisProvider.js"; -import type { - BrowserScenario, - BrowserScenarioAction, -} from "../domain/browserScenario.js"; +import type { BrowserScenario } from "../domain/browserScenario.js"; +import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js"; import { browserScenarioCaptureSchema, + type BrowserScenarioCapture, + type BrowserScenarioPartialObservation, +} from "../domain/browserScenarioCapture.js"; +import { classifyBrowserScenarioCompleteness, browserScenarioStepSchema, browserStepArtifactsSchema, - type BrowserScenarioCapture, type BrowserScenarioCompleteness, type BrowserScenarioCompletenessSection, type BrowserScenarioStep, type BrowserScenarioStepOutcome, type BrowserStepArtifacts, -} from "../domain/browserScenarioCapture.js"; +} from "../domain/browserScenarioCaptureValues.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; @@ -27,6 +28,11 @@ import type { BrowserScenarioSessionFactory, BrowserScenarioSessionPort, } from "./BrowserScenarioSessionPort.js"; +import { + browserScenarioCleanupObservation, + browserScenarioOperationFailure, + browserScenarioCaptureData, +} from "./BrowserScenarioPartialObservation.js"; const OPERATION = "capture_browser_scenario" as const; import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js"; @@ -232,7 +238,12 @@ const runScenario = async ( options.signal === undefined ? {} : { signal: options.signal }, ); const steps: BrowserScenarioStep[] = []; - let cleanup: "terminated-owned-process" | "disconnected-external" | undefined; + let operationOutcome: + | { readonly ok: true } + | { + readonly ok: false; + readonly cause: unknown; + } = { ok: true }; try { steps.push( await initialStep({ @@ -255,43 +266,72 @@ const runScenario = async ( steps.push(step); failed ||= step.status !== "completed"; } - } finally { - cleanup = await session.close(); + } catch (cause: unknown) { + operationOutcome = { ok: false, cause }; } - const events = session.events(); + let cleanupOutcome: + | { + readonly ok: true; + readonly cleanup: "terminated-owned-process" | "disconnected-external"; + } + | { readonly ok: false; readonly cause: unknown }; + try { + cleanupOutcome = { ok: true, cleanup: await session.close() }; + } catch (cause: unknown) { + cleanupOutcome = { ok: false, cause }; + } + if (!cleanupOutcome.ok) { + const cleanup = browserScenarioCleanupObservation(cleanupOutcome.cause); + const partialObservation: BrowserScenarioPartialObservation = { + kind: "browser-scenario-observation", + capture: browserScenarioCaptureData({ + session, + scenario, + startedAt, + steps, + cleanup: "incomplete", + limitations: [ + "Scenario observations are partial because browser cleanup could not be confirmed.", + ], + }), + }; + if (!operationOutcome.ok) + throw browserScenarioOperationFailure( + operationOutcome.cause, + partialObservation, + { cause: cleanupOutcome.cause }, + ); + throw new BrowserObservationError(OPERATION, "cleanup_failed", { + cause: cleanupOutcome.cause, + cleanup, + partialObservation, + detail: `Browser scenario cleanup could not be confirmed: ${cleanup.reason}. Collected steps and events are retained in partial_observation. Review the reported resources before retrying.`, + }); + } + if (!operationOutcome.ok) + throw browserScenarioOperationFailure(operationOutcome.cause, { + kind: "browser-scenario-observation", + capture: browserScenarioCaptureData({ + session, + scenario, + startedAt, + steps, + cleanup: cleanupOutcome.cleanup, + limitations: [ + "Scenario capture ended after an operation failed; uncollected steps are not represented.", + ], + }), + }); const completeness = globalCompleteness(scenario, session, steps); return browserScenarioCaptureSchema.parse({ - browser: { - mode: session.mode, - process_ownership: session.processOwnership, - cleanup, - product: session.product, - version: session.version, - }, - scenario: { - start_origin: new URL(scenario.start_url.url).origin, - action_count: scenario.actions.length, - secret_references: scenario.secrets.map(({ secret_id: id }) => id).sort(), - network_content: scenario.capture.network, - }, - duration_ms: Date.now() - startedAt, - steps, - events, + ...browserScenarioCaptureData({ + session, + scenario, + startedAt, + steps, + cleanup: cleanupOutcome.cleanup, + }), completeness, - limitations: [ - "Event sequence records provider receipt order; simultaneous browser causality is not inferred.", - "Network content is retained only when selected; response bytes are browser-decoded, not compressed wire bytes.", - "Request bytes are limited to what Playwright exposes; not_exposed does not establish body absence or multipart file coverage.", - "Network content reads settle within 5 seconds each and never wait for unfinished responses or refetch them.", - "Playwright scenario events do not expose request initiator stacks; receipt order does not prove causality.", - "Storage values are hashed only after declared-secret redaction.", - ...(session.eventLimitations?.() ?? []), - ...(session.mode === "connect" - ? [ - "CDP attachment cannot recover pre-attach events or guarantee launch-time context options.", - ] - : []), - ], }); }; diff --git a/src/browser/PlaywrightScenarioSession.ts b/src/browser/PlaywrightScenarioSession.ts index d1081abbf..1c31ef344 100644 --- a/src/browser/PlaywrightScenarioSession.ts +++ b/src/browser/PlaywrightScenarioSession.ts @@ -1,9 +1,7 @@ import { type BrowserContext, type Page } from "playwright-core"; -import type { - BrowserScenario, - BrowserScenarioAction, -} from "../domain/browserScenario.js"; +import type { BrowserScenario } from "../domain/browserScenario.js"; +import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; import type { BrowserScenarioSessionFactory, @@ -19,6 +17,10 @@ import { performPlaywrightScenarioAction } from "./PlaywrightScenarioActions.js" import { capturePlaywrightStepArtifacts } from "./PlaywrightScenarioArtifacts.js"; import { PlaywrightScenarioEvents } from "./PlaywrightScenarioEvents.js"; import { withPlaywrightExecutionBoundary } from "./PlaywrightExecutionBoundary.js"; +import { + browserScenarioCaptureData, + browserScenarioOperationFailure, +} from "./BrowserScenarioPartialObservation.js"; const OPERATION = "capture_browser_scenario" as const; @@ -135,6 +137,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { const secrets = BrowserScenarioSecrets.resolve(scenario, environment); if (secrets === undefined) throw new BrowserObservationError(OPERATION, "secret_unavailable"); + const startedAt = Date.now(); const opening = openPlaywrightScenarioBrowser(scenario, environment); let opened: OpenedScenarioBrowser; let events: PlaywrightScenarioEvents | undefined; @@ -155,6 +158,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { }); throw cause; } + let session: PlaywrightScenarioSession; try { await withPlaywrightExecutionBoundary( () => initializePage(opened.context, opened.page, scenario, secrets), @@ -170,11 +174,24 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { network: scenario.capture.network, ...(options.signal === undefined ? {} : { signal: options.signal }), }); - const session = new PlaywrightScenarioSession(opened, { + session = new PlaywrightScenarioSession(opened, { mode: scenario.browser.mode, secrets, eventCapture: events, }); + } catch (cause: unknown) { + return failBrowserScenarioOperation( + () => + opened.cleanup.close( + events === undefined + ? undefined + : () => events?.finish() ?? Promise.resolve(), + options.signal, + ), + cause, + ); + } + try { await withPlaywrightExecutionBoundary( () => opened.page.goto(secrets.url(scenario.start_url), { @@ -186,15 +203,33 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { ); return session; } catch (cause: unknown) { - return failBrowserScenarioOperation( - () => - opened.cleanup.close( - events === undefined - ? undefined - : () => events?.finish() ?? Promise.resolve(), - options.signal, - ), + let cleanup: + | "terminated-owned-process" + | "disconnected-external" + | "incomplete"; + let cleanupFailure: { readonly cause: unknown } | undefined; + try { + cleanup = await session.close(); + } catch (failure: unknown) { + cleanup = "incomplete"; + cleanupFailure = { cause: failure }; + } + throw browserScenarioOperationFailure( cause, + { + kind: "browser-scenario-observation", + capture: browserScenarioCaptureData({ + session, + scenario, + startedAt, + steps: [], + cleanup, + limitations: [ + "Scenario navigation failed before the initial state was captured.", + ], + }), + }, + cleanupFailure, ); } } diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 6c9178c55..97638ecd8 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -89,6 +89,7 @@ import type { NativeCallPartialObservation } from "./native/nativeCallPartialObs import type { ElectronActivePartialObservation } from "./javascript/electronActiveObservation.js"; import type { JavaScriptRuntimeObservation } from "./javascript/javascriptRuntimeObservation.js"; import type { ArtifactInventoryPartialObservation } from "./artifactPartialObservation.js"; +import type { BrowserScenarioPartialObservation } from "./browserScenarioCapture.js"; /** Provider-neutral evidence collected before a typed analysis failure. */ export type AnalysisPartialObservation = @@ -96,5 +97,6 @@ export type AnalysisPartialObservation = | NativeCallPartialObservation | FileOffsetPartialObservation | ElectronActivePartialObservation + | BrowserScenarioPartialObservation | JavaScriptRuntimeObservation | ArtifactInventoryPartialObservation; diff --git a/src/domain/browserScenarioCapture.ts b/src/domain/browserScenarioCapture.ts index efd3744ba..87e2626de 100644 --- a/src/domain/browserScenarioCapture.ts +++ b/src/domain/browserScenarioCapture.ts @@ -108,3 +108,25 @@ export const browserScenarioCaptureSchema = z export type BrowserScenarioCapture = z.infer< typeof browserScenarioCaptureSchema >; + +/** Capture fields shared by complete results and retained partial observations. */ +export type BrowserScenarioCaptureData = Omit< + BrowserScenarioCapture, + "browser" | "steps" | "completeness" | "events" +> & { + readonly browser: Omit & { + readonly cleanup: + | BrowserScenarioCapture["browser"]["cleanup"] + | "incomplete"; + }; + readonly steps: readonly BrowserScenarioCapture["steps"][number][]; + readonly events: Omit & { + readonly items: readonly BrowserScenarioCapture["events"]["items"][number][]; + }; +}; + +/** Browser scenario data retained when an operation fails before a complete result. */ +export interface BrowserScenarioPartialObservation { + readonly kind: "browser-scenario-observation"; + readonly capture: BrowserScenarioCaptureData; +}