fix(browser): retain scenario observations across failures
This commit is contained in:
morluto
parent
73e98b2516
commit
614e0c2e74
|
|
@ -7,6 +7,10 @@ export async function startBrowserVerifierSite() {
|
|||
let port = 0;
|
||||
let sessionGeneration = 0;
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/failed-navigation") {
|
||||
response.destroy();
|
||||
return;
|
||||
}
|
||||
if (request.url === "/slow-json") {
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.write('{"pending":');
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
import { browserScenarioSchema } from "../../dist/domain/browserScenario.js";
|
||||
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
|
||||
|
|
@ -142,6 +147,73 @@ export async function runScenarioCli(scenario) {
|
|||
return JSON.parse(stdout);
|
||||
}
|
||||
|
||||
/** Verify failed real navigation retains observations through both public adapters. */
|
||||
export async function verifyScenarioFailureEvidence(executable, origin) {
|
||||
const scenario = browserScenarioSchema.parse({
|
||||
browser: { mode: "launch", executable_path: executable },
|
||||
start_url: { url: `${origin}/failed-navigation` },
|
||||
actions: [
|
||||
{ step_id: "unused", action: "wait_for_timeout", duration_ms: 1 },
|
||||
],
|
||||
capture: { events: ["network"] },
|
||||
});
|
||||
const profilesBefore = await scenarioProfiles();
|
||||
let cliFailure;
|
||||
try {
|
||||
await runScenarioCli(scenario);
|
||||
assert.fail("The fixture must fail before initial navigation completes");
|
||||
} catch (cause) {
|
||||
assert.equal(typeof cause.stdout, "string");
|
||||
cliFailure = JSON.parse(cause.stdout);
|
||||
}
|
||||
assertFailedNavigation(cliFailure, scenario.start_url.url);
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
|
||||
env: process.env,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({
|
||||
name: "browser-scenario-failure-e2e",
|
||||
version: "1",
|
||||
});
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const result = await client.callTool({
|
||||
name: "capture_browser_scenario",
|
||||
arguments: scenario,
|
||||
});
|
||||
assert.equal(result.isError, true);
|
||||
assertFailedNavigation(requireMcpToolError(result), scenario.start_url.url);
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
}
|
||||
const profilesAfter = await scenarioProfiles();
|
||||
assert.ok([...profilesAfter].every((profile) => profilesBefore.has(profile)));
|
||||
return {
|
||||
mocked: false,
|
||||
cli: true,
|
||||
stdio_mcp: true,
|
||||
failed_navigation_events_retained: true,
|
||||
profile_cleanup: true,
|
||||
};
|
||||
}
|
||||
|
||||
const assertFailedNavigation = (error, requestedUrl) => {
|
||||
assert.equal(error?.code, "execution_failure");
|
||||
const observation = error.details?.partial_observation;
|
||||
assert.equal(observation?.kind, "browser-scenario-observation");
|
||||
assert.equal(observation.capture.browser.cleanup, "terminated-owned-process");
|
||||
assert.deepEqual(observation.capture.steps, []);
|
||||
assert.ok(
|
||||
observation.capture.events.items.some(
|
||||
(event) =>
|
||||
event.kind === "request-failed" && event.url.url === requestedUrl,
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
/** Snapshot provider-owned profile names for cleanup comparison. */
|
||||
export async function scenarioProfiles() {
|
||||
return new Set(
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import {
|
|||
listBrowserTargetsInputSchema,
|
||||
} from "../dist/domain/browserObservation.js";
|
||||
import { observeWebSessionInputSchema } from "../dist/domain/browserSession.js";
|
||||
import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiff.js";
|
||||
import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiffSchemas.js";
|
||||
import {
|
||||
captureWebScreenshotInputSchema,
|
||||
compareWebScreenshotsInputSchema,
|
||||
|
|
@ -33,6 +33,7 @@ import {
|
|||
browserScenario,
|
||||
runScenarioCli,
|
||||
scenarioProfiles,
|
||||
verifyScenarioFailureEvidence,
|
||||
} from "./lib/browser-scenario-verifier.mjs";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs";
|
||||
|
|
@ -294,6 +295,10 @@ try {
|
|||
if ([...profilesAfter].some((entry) => !profilesBefore.has(entry)))
|
||||
throw new Error("Scenario launch retained a temporary browser profile");
|
||||
assertScenarioCapture(launchedScenario.value);
|
||||
const scenarioFailure = await verifyScenarioFailureEvidence(
|
||||
executable,
|
||||
site.origin,
|
||||
);
|
||||
const scenarioResults = JSON.stringify([
|
||||
attachedScenario,
|
||||
launchedScenario.value,
|
||||
|
|
@ -360,6 +365,7 @@ try {
|
|||
browserScenarioCli: true,
|
||||
browserScenarioAttachCleanup: "disconnected-external",
|
||||
browserScenarioLaunchCleanup: "terminated-owned-process",
|
||||
scenarioFailure,
|
||||
verified: true,
|
||||
};
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
import type { BrowserScenario } from "../domain/browserScenario.js";
|
||||
import type {
|
||||
BrowserScenarioCaptureData,
|
||||
BrowserScenarioPartialObservation,
|
||||
} from "../domain/browserScenarioCapture.js";
|
||||
import type { BrowserScenarioStep } from "../domain/browserScenarioCaptureValues.js";
|
||||
import type { BrowserScenarioSessionPort } from "./BrowserScenarioSessionPort.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import {
|
||||
AnalysisCancelledError,
|
||||
AnalysisTimeoutError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js";
|
||||
|
||||
const OPERATION = "capture_browser_scenario" as const;
|
||||
|
||||
const browserScenarioLimitations = (
|
||||
session: BrowserScenarioSessionPort,
|
||||
): string[] => [
|
||||
"Event sequence records provider receipt order; simultaneous browser causality is not inferred.",
|
||||
"Network content is retained only when selected; response bytes are browser-decoded, not compressed wire bytes.",
|
||||
"Request bytes are limited to what Playwright exposes; not_exposed does not establish body absence or multipart file coverage.",
|
||||
"Network content reads settle within 5 seconds each and never wait for unfinished responses or refetch them.",
|
||||
"Playwright scenario events do not expose request initiator stacks; receipt order does not prove causality.",
|
||||
"Storage values are hashed only after declared-secret redaction.",
|
||||
...(session.eventLimitations?.() ?? []),
|
||||
...(session.mode === "connect"
|
||||
? [
|
||||
"CDP attachment cannot recover pre-attach events or guarantee launch-time context options.",
|
||||
]
|
||||
: []),
|
||||
];
|
||||
|
||||
/** Project one session into the common capture fields used by success and failure. */
|
||||
export const browserScenarioCaptureData = (input: {
|
||||
readonly session: BrowserScenarioSessionPort;
|
||||
readonly scenario: BrowserScenario;
|
||||
readonly startedAt: number;
|
||||
readonly steps: readonly BrowserScenarioStep[];
|
||||
readonly cleanup:
|
||||
| "terminated-owned-process"
|
||||
| "disconnected-external"
|
||||
| "incomplete";
|
||||
readonly limitations?: readonly string[];
|
||||
}): BrowserScenarioCaptureData => ({
|
||||
browser: {
|
||||
mode: input.session.mode,
|
||||
process_ownership: input.session.processOwnership,
|
||||
cleanup: input.cleanup,
|
||||
product: input.session.product,
|
||||
version: input.session.version,
|
||||
},
|
||||
scenario: {
|
||||
start_origin: new URL(input.scenario.start_url.url).origin,
|
||||
action_count: input.scenario.actions.length,
|
||||
secret_references: input.scenario.secrets
|
||||
.map(({ secret_id: id }) => id)
|
||||
.sort(),
|
||||
network_content: input.scenario.capture.network,
|
||||
},
|
||||
duration_ms: Date.now() - input.startedAt,
|
||||
steps: [...input.steps],
|
||||
events: input.session.events(),
|
||||
limitations: [
|
||||
...browserScenarioLimitations(input.session),
|
||||
...(input.limitations ?? []),
|
||||
],
|
||||
});
|
||||
|
||||
/** Describe the cleanup failure without discarding its resource evidence. */
|
||||
export const browserScenarioCleanupObservation = (
|
||||
cause: unknown,
|
||||
): { readonly reason: string; readonly resources: readonly string[] } =>
|
||||
cause instanceof AnalysisError && cause.cleanup !== undefined
|
||||
? cause.cleanup
|
||||
: {
|
||||
reason: cause instanceof Error ? cause.message : String(cause),
|
||||
resources:
|
||||
cause instanceof AnalysisError && cause.cleanupResources.length > 0
|
||||
? cause.cleanupResources
|
||||
: ["browser_transport"],
|
||||
};
|
||||
|
||||
/** Retain operation diagnostics and capture data after cleanup settles. */
|
||||
export const browserScenarioOperationFailure = (
|
||||
cause: unknown,
|
||||
partialObservation: BrowserScenarioPartialObservation,
|
||||
cleanupFailure?: { readonly cause: unknown },
|
||||
): AnalysisError => {
|
||||
const primary =
|
||||
cause instanceof AnalysisError
|
||||
? cause
|
||||
: new ProviderAdapterError(
|
||||
PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id,
|
||||
OPERATION,
|
||||
{ cause },
|
||||
);
|
||||
const primaryProjection = projectAnalysisError(primary);
|
||||
const cleanupCause =
|
||||
cleanupFailure === undefined
|
||||
? undefined
|
||||
: cleanupFailure.cause instanceof AnalysisError
|
||||
? cleanupFailure.cause
|
||||
: new ProviderAdapterError(
|
||||
PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id,
|
||||
"close_browser_scenario",
|
||||
{ cause: cleanupFailure.cause },
|
||||
);
|
||||
const cleanupObservation =
|
||||
cleanupFailure === undefined
|
||||
? undefined
|
||||
: browserScenarioCleanupObservation(cleanupFailure.cause);
|
||||
const retainedOptions = {
|
||||
cause: primary,
|
||||
partialObservation,
|
||||
...(primary.capturedOutput === undefined
|
||||
? {}
|
||||
: { capturedOutput: primary.capturedOutput }),
|
||||
...(primary.cleanup === undefined ? {} : { cleanup: primary.cleanup }),
|
||||
};
|
||||
if (cleanupCause === undefined) {
|
||||
if (primary instanceof AnalysisCancelledError)
|
||||
return new AnalysisCancelledError(primary.operation, retainedOptions);
|
||||
if (primary instanceof AnalysisTimeoutError)
|
||||
return new AnalysisTimeoutError(
|
||||
primary.operation,
|
||||
primary.timeoutMs,
|
||||
retainedOptions,
|
||||
);
|
||||
if (primary instanceof BrowserObservationError)
|
||||
return new BrowserObservationError(primary.operation, primary.reason, {
|
||||
...retainedOptions,
|
||||
...(primary.userMessage === undefined
|
||||
? {}
|
||||
: { detail: primary.userMessage }),
|
||||
});
|
||||
}
|
||||
const cleanup = cleanupObservation ?? primary.cleanup;
|
||||
return new ProviderAdapterError(
|
||||
PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id,
|
||||
OPERATION,
|
||||
{
|
||||
...retainedOptions,
|
||||
cause:
|
||||
cleanupCause === undefined
|
||||
? primary
|
||||
: new AggregateError(
|
||||
[primary, cleanupCause],
|
||||
"Browser scenario operation and cleanup both failed",
|
||||
{ cause: primary },
|
||||
),
|
||||
userMessage: primaryProjection.message,
|
||||
...(cleanup === undefined ? {} : { cleanup }),
|
||||
diagnostics: {
|
||||
primary_error: primaryProjection,
|
||||
...(cleanupCause === undefined
|
||||
? {}
|
||||
: { cleanup_error: projectAnalysisError(cleanupCause) }),
|
||||
},
|
||||
},
|
||||
);
|
||||
};
|
||||
|
|
@ -9,13 +9,19 @@ import { PlaywrightScenarioBrowserCleanupOwner } from "./PlaywrightScenarioBrows
|
|||
import {
|
||||
browserScenarioSchema,
|
||||
type BrowserScenario,
|
||||
type BrowserScenarioAction,
|
||||
} from "../domain/browserScenario.js";
|
||||
import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
|
||||
import {
|
||||
browserStepArtifactsSchema,
|
||||
type BrowserScenarioEvent,
|
||||
type BrowserStepArtifacts,
|
||||
} from "../domain/browserScenarioCapture.js";
|
||||
} from "../domain/browserScenarioCaptureValues.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import {
|
||||
AnalysisCancelledError,
|
||||
AnalysisOutputError,
|
||||
AnalysisTimeoutError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
|
||||
const scenario = (
|
||||
options: {
|
||||
|
|
@ -70,6 +76,13 @@ const scenario = (
|
|||
|
||||
type SnapshotKind = BrowserScenario["capture"]["after_each_step"][number];
|
||||
|
||||
interface FakeSessionFailures {
|
||||
readonly incomplete?: boolean;
|
||||
readonly action?: Error;
|
||||
readonly capture?: { readonly call: number; readonly error: Error };
|
||||
readonly close?: Error;
|
||||
}
|
||||
|
||||
const artifactState = (
|
||||
kind: SnapshotKind,
|
||||
requested: ReadonlySet<SnapshotKind>,
|
||||
|
|
@ -101,13 +114,11 @@ class FakeSession implements BrowserScenarioSessionPort {
|
|||
captureCalls = 0;
|
||||
onPerform: (() => void) | undefined;
|
||||
eventGaps: readonly string[] = [];
|
||||
private step = 0;
|
||||
eventItems: readonly BrowserScenarioEvent[] = [];
|
||||
|
||||
constructor(
|
||||
readonly mode: "launch" | "connect",
|
||||
private readonly incomplete = false,
|
||||
private readonly failAction = false,
|
||||
private readonly failCapture = false,
|
||||
private readonly failures: FakeSessionFailures = {},
|
||||
) {
|
||||
this.processOwnership = mode === "launch" ? "provider-owned" : "external";
|
||||
}
|
||||
|
|
@ -120,9 +131,7 @@ class FakeSession implements BrowserScenarioSessionPort {
|
|||
return sanitizeBrowserUrl(value);
|
||||
}
|
||||
|
||||
setStep(index: number) {
|
||||
this.step = index;
|
||||
}
|
||||
setStep(_index: number) {}
|
||||
|
||||
nextEventSequence() {
|
||||
return 1;
|
||||
|
|
@ -137,7 +146,11 @@ class FakeSession implements BrowserScenarioSessionPort {
|
|||
readonly dropped: number;
|
||||
readonly items: readonly BrowserScenarioEvent[];
|
||||
} {
|
||||
return { retained: 0, dropped: 0, items: [] };
|
||||
return {
|
||||
retained: this.eventItems.length,
|
||||
dropped: 0,
|
||||
items: this.eventItems,
|
||||
};
|
||||
}
|
||||
|
||||
eventLimitations(): readonly string[] {
|
||||
|
|
@ -148,7 +161,7 @@ class FakeSession implements BrowserScenarioSessionPort {
|
|||
this.performCalls += 1;
|
||||
this.onPerform?.();
|
||||
if (signal?.aborted === true) throw new Error("request cancelled");
|
||||
if (this.failAction) throw new Error("fixture secret action failure");
|
||||
if (this.failures.action !== undefined) throw this.failures.action;
|
||||
}
|
||||
|
||||
capture(
|
||||
|
|
@ -156,25 +169,40 @@ class FakeSession implements BrowserScenarioSessionPort {
|
|||
_signal?: AbortSignal,
|
||||
): Promise<BrowserStepArtifacts> {
|
||||
this.captureCalls += 1;
|
||||
if (this.failCapture) return Promise.reject(new Error("capture failed"));
|
||||
if (this.failures.capture?.call === this.captureCalls)
|
||||
return Promise.reject(this.failures.capture.error);
|
||||
return Promise.resolve(
|
||||
browserStepArtifactsSchema.parse({
|
||||
screenshot: artifactState("screenshot", requested, this.incomplete),
|
||||
dom: artifactState("dom", requested, this.incomplete),
|
||||
screenshot: artifactState(
|
||||
"screenshot",
|
||||
requested,
|
||||
this.failures.incomplete === true,
|
||||
),
|
||||
dom: artifactState("dom", requested, this.failures.incomplete === true),
|
||||
accessibility: artifactState(
|
||||
"accessibility",
|
||||
requested,
|
||||
this.incomplete,
|
||||
this.failures.incomplete === true,
|
||||
),
|
||||
url: artifactState("url", requested, this.failures.incomplete === true),
|
||||
history: artifactState(
|
||||
"history",
|
||||
requested,
|
||||
this.failures.incomplete === true,
|
||||
),
|
||||
storage: artifactState(
|
||||
"storage",
|
||||
requested,
|
||||
this.failures.incomplete === true,
|
||||
),
|
||||
url: artifactState("url", requested, this.incomplete),
|
||||
history: artifactState("history", requested, this.incomplete),
|
||||
storage: artifactState("storage", requested, this.incomplete),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
close() {
|
||||
this.closeCalls += 1;
|
||||
if (this.failures.close !== undefined)
|
||||
return Promise.reject(this.failures.close);
|
||||
return Promise.resolve(
|
||||
this.mode === "launch"
|
||||
? ("terminated-owned-process" as const)
|
||||
|
|
@ -220,7 +248,7 @@ describe("PlaywrightBrowserScenarioProvider", () => {
|
|||
});
|
||||
|
||||
it("makes missing and truncated captures ineligible for equality", async () => {
|
||||
const session = new FakeSession("launch", true);
|
||||
const session = new FakeSession("launch", { incomplete: true });
|
||||
const provider = new PlaywrightBrowserScenarioProvider({
|
||||
open: () => Promise.resolve(session),
|
||||
});
|
||||
|
|
@ -237,7 +265,9 @@ describe("PlaywrightBrowserScenarioProvider", () => {
|
|||
});
|
||||
|
||||
it("records one failed step, cancels later actions, and still cleans up", async () => {
|
||||
const session = new FakeSession("launch", false, true);
|
||||
const session = new FakeSession("launch", {
|
||||
action: new Error("fixture secret action failure"),
|
||||
});
|
||||
const provider = new PlaywrightBrowserScenarioProvider({
|
||||
open: () => Promise.resolve(session),
|
||||
});
|
||||
|
|
@ -276,17 +306,187 @@ describe("PlaywrightBrowserScenarioProvider", () => {
|
|||
});
|
||||
expect(session.closeCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
it("closes the session when initial capture fails", async () => {
|
||||
const session = new FakeSession("launch", false, false, true);
|
||||
describe("PlaywrightBrowserScenarioProvider cleanup", () => {
|
||||
it("returns a cleanup failure with completed observations when close fails", async () => {
|
||||
const session = new FakeSession("launch", {
|
||||
action: new Error("fixture secret action failure"),
|
||||
close: new Error("browser close failed"),
|
||||
});
|
||||
const provider = new PlaywrightBrowserScenarioProvider({
|
||||
open: () => Promise.resolve(session),
|
||||
});
|
||||
const result = await provider.captureScenario(scenario());
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
const projection = projectAnalysisError(result.error);
|
||||
expect(projection).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
cleanup: "incomplete",
|
||||
resources: ["browser_transport"],
|
||||
partial_observation: {
|
||||
kind: "browser-scenario-observation",
|
||||
capture: {
|
||||
browser: { cleanup: "incomplete" },
|
||||
steps: [
|
||||
{ step_id: "scenario_start" },
|
||||
{ step_id: "wait_0", status: "failed" },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(session.closeCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PlaywrightBrowserScenarioProvider operation failures", () => {
|
||||
it.each([{ cleanupFails: false }, { cleanupFails: true }])(
|
||||
"retains typed partial data when operation fails and cleanupFails=$cleanupFails",
|
||||
async ({ cleanupFails }) => {
|
||||
const session = new FakeSession("launch", {
|
||||
capture: {
|
||||
call: 2,
|
||||
error: new AnalysisOutputError(
|
||||
"capture_browser_scenario",
|
||||
"step capture failed",
|
||||
{
|
||||
capturedOutput: {
|
||||
stdout: "partial stdout",
|
||||
stderr: "",
|
||||
truncated: false,
|
||||
},
|
||||
cleanup: {
|
||||
reason: "capture cleanup is uncertain",
|
||||
resources: ["capture_handle"],
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
...(cleanupFails ? { close: new Error("browser close failed") } : {}),
|
||||
});
|
||||
session.eventItems = [
|
||||
{
|
||||
sequence: 1,
|
||||
step_index: 0,
|
||||
kind: "console",
|
||||
level: "log",
|
||||
text: "navigation started",
|
||||
url: sanitizeBrowserUrl("https://app.example.test/"),
|
||||
},
|
||||
];
|
||||
const provider = new PlaywrightBrowserScenarioProvider({
|
||||
open: () => Promise.resolve(session),
|
||||
});
|
||||
const result = await provider.captureScenario(scenario());
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
const projection = projectAnalysisError(result.error);
|
||||
expect(projection).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
diagnostics: {
|
||||
primary_error: {
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
operation: "capture_browser_scenario",
|
||||
reason: "step capture failed",
|
||||
},
|
||||
},
|
||||
...(cleanupFails
|
||||
? { cleanup_error: { code: "execution_failure" } }
|
||||
: {}),
|
||||
},
|
||||
partial_observation: {
|
||||
kind: "browser-scenario-observation",
|
||||
capture: {
|
||||
browser: {
|
||||
cleanup: cleanupFails
|
||||
? "incomplete"
|
||||
: "terminated-owned-process",
|
||||
},
|
||||
steps: [{ step_id: "scenario_start" }],
|
||||
events: {
|
||||
items: [
|
||||
expect.objectContaining({ text: "navigation started" }),
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.error.capturedOutput).toEqual({
|
||||
stdout: "partial stdout",
|
||||
stderr: "",
|
||||
truncated: false,
|
||||
});
|
||||
expect(result.error.cleanup).toEqual(
|
||||
cleanupFails
|
||||
? {
|
||||
reason: "browser close failed",
|
||||
resources: ["browser_transport"],
|
||||
}
|
||||
: {
|
||||
reason: "capture cleanup is uncertain",
|
||||
resources: ["capture_handle"],
|
||||
},
|
||||
);
|
||||
expect(session.closeCalls).toBe(1);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("PlaywrightBrowserScenarioProvider cancellation and timeout", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "cancellation",
|
||||
error: new AnalysisCancelledError("capture_browser_scenario"),
|
||||
code: "cancelled",
|
||||
category: "cancelled",
|
||||
details: { operation: "capture_browser_scenario", cleanup: "complete" },
|
||||
},
|
||||
{
|
||||
name: "timeout",
|
||||
error: new AnalysisTimeoutError("capture_browser_scenario", 250),
|
||||
code: "provider_timeout",
|
||||
category: "timeout",
|
||||
details: { operation: "capture_browser_scenario", timeout_ms: 250 },
|
||||
},
|
||||
])(
|
||||
"preserves $name category and details when cleanup succeeds",
|
||||
async ({ error, code, category, details }) => {
|
||||
const session = new FakeSession("launch", {
|
||||
capture: { call: 1, error },
|
||||
});
|
||||
const provider = new PlaywrightBrowserScenarioProvider({
|
||||
open: () => Promise.resolve(session),
|
||||
});
|
||||
const result = await provider.captureScenario(scenario());
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
const projection = projectAnalysisError(result.error);
|
||||
expect(projection).toMatchObject({
|
||||
code,
|
||||
category,
|
||||
details: {
|
||||
...details,
|
||||
partial_observation: {
|
||||
kind: "browser-scenario-observation",
|
||||
capture: { browser: { cleanup: "terminated-owned-process" } },
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(session.closeCalls).toBe(1);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("PlaywrightBrowserScenarioProvider request cancellation", () => {
|
||||
it("does not open a session for an already-cancelled request", async () => {
|
||||
let opens = 0;
|
||||
const provider = new PlaywrightBrowserScenarioProvider({
|
||||
|
|
|
|||
|
|
@ -3,22 +3,23 @@ import type {
|
|||
ExecutionOptions,
|
||||
ProviderIdentity,
|
||||
} from "../application/AnalysisProvider.js";
|
||||
import type {
|
||||
BrowserScenario,
|
||||
BrowserScenarioAction,
|
||||
} from "../domain/browserScenario.js";
|
||||
import type { BrowserScenario } from "../domain/browserScenario.js";
|
||||
import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
|
||||
import {
|
||||
browserScenarioCaptureSchema,
|
||||
type BrowserScenarioCapture,
|
||||
type BrowserScenarioPartialObservation,
|
||||
} from "../domain/browserScenarioCapture.js";
|
||||
import {
|
||||
classifyBrowserScenarioCompleteness,
|
||||
browserScenarioStepSchema,
|
||||
browserStepArtifactsSchema,
|
||||
type BrowserScenarioCapture,
|
||||
type BrowserScenarioCompleteness,
|
||||
type BrowserScenarioCompletenessSection,
|
||||
type BrowserScenarioStep,
|
||||
type BrowserScenarioStepOutcome,
|
||||
type BrowserStepArtifacts,
|
||||
} from "../domain/browserScenarioCapture.js";
|
||||
} from "../domain/browserScenarioCaptureValues.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
|
|
@ -27,6 +28,11 @@ import type {
|
|||
BrowserScenarioSessionFactory,
|
||||
BrowserScenarioSessionPort,
|
||||
} from "./BrowserScenarioSessionPort.js";
|
||||
import {
|
||||
browserScenarioCleanupObservation,
|
||||
browserScenarioOperationFailure,
|
||||
browserScenarioCaptureData,
|
||||
} from "./BrowserScenarioPartialObservation.js";
|
||||
|
||||
const OPERATION = "capture_browser_scenario" as const;
|
||||
import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js";
|
||||
|
|
@ -232,7 +238,12 @@ const runScenario = async (
|
|||
options.signal === undefined ? {} : { signal: options.signal },
|
||||
);
|
||||
const steps: BrowserScenarioStep[] = [];
|
||||
let cleanup: "terminated-owned-process" | "disconnected-external" | undefined;
|
||||
let operationOutcome:
|
||||
| { readonly ok: true }
|
||||
| {
|
||||
readonly ok: false;
|
||||
readonly cause: unknown;
|
||||
} = { ok: true };
|
||||
try {
|
||||
steps.push(
|
||||
await initialStep({
|
||||
|
|
@ -255,43 +266,72 @@ const runScenario = async (
|
|||
steps.push(step);
|
||||
failed ||= step.status !== "completed";
|
||||
}
|
||||
} finally {
|
||||
cleanup = await session.close();
|
||||
} catch (cause: unknown) {
|
||||
operationOutcome = { ok: false, cause };
|
||||
}
|
||||
const events = session.events();
|
||||
let cleanupOutcome:
|
||||
| {
|
||||
readonly ok: true;
|
||||
readonly cleanup: "terminated-owned-process" | "disconnected-external";
|
||||
}
|
||||
| { readonly ok: false; readonly cause: unknown };
|
||||
try {
|
||||
cleanupOutcome = { ok: true, cleanup: await session.close() };
|
||||
} catch (cause: unknown) {
|
||||
cleanupOutcome = { ok: false, cause };
|
||||
}
|
||||
if (!cleanupOutcome.ok) {
|
||||
const cleanup = browserScenarioCleanupObservation(cleanupOutcome.cause);
|
||||
const partialObservation: BrowserScenarioPartialObservation = {
|
||||
kind: "browser-scenario-observation",
|
||||
capture: browserScenarioCaptureData({
|
||||
session,
|
||||
scenario,
|
||||
startedAt,
|
||||
steps,
|
||||
cleanup: "incomplete",
|
||||
limitations: [
|
||||
"Scenario observations are partial because browser cleanup could not be confirmed.",
|
||||
],
|
||||
}),
|
||||
};
|
||||
if (!operationOutcome.ok)
|
||||
throw browserScenarioOperationFailure(
|
||||
operationOutcome.cause,
|
||||
partialObservation,
|
||||
{ cause: cleanupOutcome.cause },
|
||||
);
|
||||
throw new BrowserObservationError(OPERATION, "cleanup_failed", {
|
||||
cause: cleanupOutcome.cause,
|
||||
cleanup,
|
||||
partialObservation,
|
||||
detail: `Browser scenario cleanup could not be confirmed: ${cleanup.reason}. Collected steps and events are retained in partial_observation. Review the reported resources before retrying.`,
|
||||
});
|
||||
}
|
||||
if (!operationOutcome.ok)
|
||||
throw browserScenarioOperationFailure(operationOutcome.cause, {
|
||||
kind: "browser-scenario-observation",
|
||||
capture: browserScenarioCaptureData({
|
||||
session,
|
||||
scenario,
|
||||
startedAt,
|
||||
steps,
|
||||
cleanup: cleanupOutcome.cleanup,
|
||||
limitations: [
|
||||
"Scenario capture ended after an operation failed; uncollected steps are not represented.",
|
||||
],
|
||||
}),
|
||||
});
|
||||
const completeness = globalCompleteness(scenario, session, steps);
|
||||
return browserScenarioCaptureSchema.parse({
|
||||
browser: {
|
||||
mode: session.mode,
|
||||
process_ownership: session.processOwnership,
|
||||
cleanup,
|
||||
product: session.product,
|
||||
version: session.version,
|
||||
},
|
||||
scenario: {
|
||||
start_origin: new URL(scenario.start_url.url).origin,
|
||||
action_count: scenario.actions.length,
|
||||
secret_references: scenario.secrets.map(({ secret_id: id }) => id).sort(),
|
||||
network_content: scenario.capture.network,
|
||||
},
|
||||
duration_ms: Date.now() - startedAt,
|
||||
steps,
|
||||
events,
|
||||
...browserScenarioCaptureData({
|
||||
session,
|
||||
scenario,
|
||||
startedAt,
|
||||
steps,
|
||||
cleanup: cleanupOutcome.cleanup,
|
||||
}),
|
||||
completeness,
|
||||
limitations: [
|
||||
"Event sequence records provider receipt order; simultaneous browser causality is not inferred.",
|
||||
"Network content is retained only when selected; response bytes are browser-decoded, not compressed wire bytes.",
|
||||
"Request bytes are limited to what Playwright exposes; not_exposed does not establish body absence or multipart file coverage.",
|
||||
"Network content reads settle within 5 seconds each and never wait for unfinished responses or refetch them.",
|
||||
"Playwright scenario events do not expose request initiator stacks; receipt order does not prove causality.",
|
||||
"Storage values are hashed only after declared-secret redaction.",
|
||||
...(session.eventLimitations?.() ?? []),
|
||||
...(session.mode === "connect"
|
||||
? [
|
||||
"CDP attachment cannot recover pre-attach events or guarantee launch-time context options.",
|
||||
]
|
||||
: []),
|
||||
],
|
||||
});
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
import { type BrowserContext, type Page } from "playwright-core";
|
||||
|
||||
import type {
|
||||
BrowserScenario,
|
||||
BrowserScenarioAction,
|
||||
} from "../domain/browserScenario.js";
|
||||
import type { BrowserScenario } from "../domain/browserScenario.js";
|
||||
import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import type {
|
||||
BrowserScenarioSessionFactory,
|
||||
|
|
@ -19,6 +17,10 @@ import { performPlaywrightScenarioAction } from "./PlaywrightScenarioActions.js"
|
|||
import { capturePlaywrightStepArtifacts } from "./PlaywrightScenarioArtifacts.js";
|
||||
import { PlaywrightScenarioEvents } from "./PlaywrightScenarioEvents.js";
|
||||
import { withPlaywrightExecutionBoundary } from "./PlaywrightExecutionBoundary.js";
|
||||
import {
|
||||
browserScenarioCaptureData,
|
||||
browserScenarioOperationFailure,
|
||||
} from "./BrowserScenarioPartialObservation.js";
|
||||
|
||||
const OPERATION = "capture_browser_scenario" as const;
|
||||
|
||||
|
|
@ -135,6 +137,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
|
|||
const secrets = BrowserScenarioSecrets.resolve(scenario, environment);
|
||||
if (secrets === undefined)
|
||||
throw new BrowserObservationError(OPERATION, "secret_unavailable");
|
||||
const startedAt = Date.now();
|
||||
const opening = openPlaywrightScenarioBrowser(scenario, environment);
|
||||
let opened: OpenedScenarioBrowser;
|
||||
let events: PlaywrightScenarioEvents | undefined;
|
||||
|
|
@ -155,6 +158,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
|
|||
});
|
||||
throw cause;
|
||||
}
|
||||
let session: PlaywrightScenarioSession;
|
||||
try {
|
||||
await withPlaywrightExecutionBoundary(
|
||||
() => initializePage(opened.context, opened.page, scenario, secrets),
|
||||
|
|
@ -170,11 +174,24 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
|
|||
network: scenario.capture.network,
|
||||
...(options.signal === undefined ? {} : { signal: options.signal }),
|
||||
});
|
||||
const session = new PlaywrightScenarioSession(opened, {
|
||||
session = new PlaywrightScenarioSession(opened, {
|
||||
mode: scenario.browser.mode,
|
||||
secrets,
|
||||
eventCapture: events,
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
return failBrowserScenarioOperation(
|
||||
() =>
|
||||
opened.cleanup.close(
|
||||
events === undefined
|
||||
? undefined
|
||||
: () => events?.finish() ?? Promise.resolve(),
|
||||
options.signal,
|
||||
),
|
||||
cause,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await withPlaywrightExecutionBoundary(
|
||||
() =>
|
||||
opened.page.goto(secrets.url(scenario.start_url), {
|
||||
|
|
@ -186,15 +203,33 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
|
|||
);
|
||||
return session;
|
||||
} catch (cause: unknown) {
|
||||
return failBrowserScenarioOperation(
|
||||
() =>
|
||||
opened.cleanup.close(
|
||||
events === undefined
|
||||
? undefined
|
||||
: () => events?.finish() ?? Promise.resolve(),
|
||||
options.signal,
|
||||
),
|
||||
let cleanup:
|
||||
| "terminated-owned-process"
|
||||
| "disconnected-external"
|
||||
| "incomplete";
|
||||
let cleanupFailure: { readonly cause: unknown } | undefined;
|
||||
try {
|
||||
cleanup = await session.close();
|
||||
} catch (failure: unknown) {
|
||||
cleanup = "incomplete";
|
||||
cleanupFailure = { cause: failure };
|
||||
}
|
||||
throw browserScenarioOperationFailure(
|
||||
cause,
|
||||
{
|
||||
kind: "browser-scenario-observation",
|
||||
capture: browserScenarioCaptureData({
|
||||
session,
|
||||
scenario,
|
||||
startedAt,
|
||||
steps: [],
|
||||
cleanup,
|
||||
limitations: [
|
||||
"Scenario navigation failed before the initial state was captured.",
|
||||
],
|
||||
}),
|
||||
},
|
||||
cleanupFailure,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ import type { NativeCallPartialObservation } from "./native/nativeCallPartialObs
|
|||
import type { ElectronActivePartialObservation } from "./javascript/electronActiveObservation.js";
|
||||
import type { JavaScriptRuntimeObservation } from "./javascript/javascriptRuntimeObservation.js";
|
||||
import type { ArtifactInventoryPartialObservation } from "./artifactPartialObservation.js";
|
||||
import type { BrowserScenarioPartialObservation } from "./browserScenarioCapture.js";
|
||||
|
||||
/** Provider-neutral evidence collected before a typed analysis failure. */
|
||||
export type AnalysisPartialObservation =
|
||||
|
|
@ -96,5 +97,6 @@ export type AnalysisPartialObservation =
|
|||
| NativeCallPartialObservation
|
||||
| FileOffsetPartialObservation
|
||||
| ElectronActivePartialObservation
|
||||
| BrowserScenarioPartialObservation
|
||||
| JavaScriptRuntimeObservation
|
||||
| ArtifactInventoryPartialObservation;
|
||||
|
|
|
|||
|
|
@ -108,3 +108,25 @@ export const browserScenarioCaptureSchema = z
|
|||
export type BrowserScenarioCapture = z.infer<
|
||||
typeof browserScenarioCaptureSchema
|
||||
>;
|
||||
|
||||
/** Capture fields shared by complete results and retained partial observations. */
|
||||
export type BrowserScenarioCaptureData = Omit<
|
||||
BrowserScenarioCapture,
|
||||
"browser" | "steps" | "completeness" | "events"
|
||||
> & {
|
||||
readonly browser: Omit<BrowserScenarioCapture["browser"], "cleanup"> & {
|
||||
readonly cleanup:
|
||||
| BrowserScenarioCapture["browser"]["cleanup"]
|
||||
| "incomplete";
|
||||
};
|
||||
readonly steps: readonly BrowserScenarioCapture["steps"][number][];
|
||||
readonly events: Omit<BrowserScenarioCapture["events"], "items"> & {
|
||||
readonly items: readonly BrowserScenarioCapture["events"]["items"][number][];
|
||||
};
|
||||
};
|
||||
|
||||
/** Browser scenario data retained when an operation fails before a complete result. */
|
||||
export interface BrowserScenarioPartialObservation {
|
||||
readonly kind: "browser-scenario-observation";
|
||||
readonly capture: BrowserScenarioCaptureData;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue