From 2c1ceabbf1f117daafe166e1134b47737b38e1f6 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Tue, 14 Jul 2026 12:27:13 +0800 Subject: [PATCH] feat: add passive website reverse engineering --- AGENTS.md | 12 +- CONTRIBUTING.md | 4 +- README.md | 42 +- README_ar.md | 21 +- README_ja.md | 7 +- README_ko.md | 7 +- README_zh.md | 7 +- docs/architecture.mermaid | 13 +- docs/browser-observation.md | 121 +++++ docs/error-contract.schema.json | 6 + docs/mcp-prompts.md | 2 +- package.json | 1 + scripts/rea.mjs | 2 +- scripts/verify-package.mjs | 4 + scripts/verify-published-package.mjs | 4 +- scripts/verify-real-browser.mjs | 286 ++++++++++ scripts/verify-real-hopper.mjs | 2 +- skills/rea-analysis/SKILL.md | 21 +- src/application/BrowserEvidence.ts | 61 +++ src/application/BrowserObservationPort.ts | 23 + src/application/BrowserObservationService.ts | 126 +++++ src/application/CapabilityInventory.ts | 10 + src/application/ProjectPermissionStore.ts | 2 + src/browser/CdpBrowserProvider.ts | 229 ++++++++ src/browser/CdpCaptureDocuments.ts | 198 +++++++ src/browser/CdpCaptureEvents.ts | 300 ++++++++++ src/browser/CdpCaptureStorage.ts | 116 ++++ src/browser/CdpCaptureValues.ts | 98 ++++ src/browser/CdpConnection.ts | 252 +++++++++ src/browser/CdpEndpoint.ts | 207 +++++++ src/browser/CdpOptionalCommand.ts | 31 ++ src/browser/CdpPageCapture.ts | 446 +++++++++++++++ src/catalogIdentity.ts | 2 + src/cli.ts | 2 + src/cliBrowserCommands.ts | 173 ++++++ src/cliPolicyCommands.ts | 4 + src/config.ts | 70 +++ src/contracts/browserToolContracts.ts | 92 ++++ src/contracts/errorSchemas.ts | 2 + src/contracts/toolContractTypes.ts | 1 + src/contracts/toolContracts.ts | 8 +- src/domain/browserObservation.ts | 397 ++++++++++++++ src/domain/errors.ts | 55 ++ src/domain/permissionPolicy.ts | 11 + src/generatedPackageMetadata.ts | 4 +- src/main.ts | 7 + src/server/createServer.ts | 10 + src/server/registerBrowserTools.ts | 85 +++ src/server/registerSessionStatusTool.ts | 1 + src/server/registerSessionTools.ts | 3 + tests/browserCli.test.ts | 135 +++++ tests/browserMcp.test.ts | 170 ++++++ tests/browserObservation.test.ts | 126 +++++ tests/cdpBrowserProvider.test.ts | 497 +++++++++++++++++ tests/cdpCaptureDocuments.test.ts | 74 +++ tests/cdpConnection.test.ts | 56 ++ tests/config.test.ts | 54 ++ tests/contractSnapshot.test.ts | 13 +- tests/enhancedTools.test.ts | 2 +- tests/errors.test.ts | 8 +- tests/fixtures/fakeCdpBrowser.ts | 544 +++++++++++++++++++ tests/integration/serverIntegration.test.ts | 8 +- tests/permissionPolicy.test.ts | 39 ++ tests/promptMcp.test.ts | 2 +- tests/readmeConsistency.test.ts | 6 +- tests/runtime.test.ts | 6 +- tests/serverIdentity.test.ts | 15 +- tests/sessionMcp.test.ts | 2 +- tests/setupSkill.test.ts | 2 +- tests/toolContracts.test.ts | 11 +- typedoc.json | 1 + 71 files changed, 5279 insertions(+), 80 deletions(-) create mode 100644 docs/browser-observation.md create mode 100644 scripts/verify-real-browser.mjs create mode 100644 src/application/BrowserEvidence.ts create mode 100644 src/application/BrowserObservationPort.ts create mode 100644 src/application/BrowserObservationService.ts create mode 100644 src/browser/CdpBrowserProvider.ts create mode 100644 src/browser/CdpCaptureDocuments.ts create mode 100644 src/browser/CdpCaptureEvents.ts create mode 100644 src/browser/CdpCaptureStorage.ts create mode 100644 src/browser/CdpCaptureValues.ts create mode 100644 src/browser/CdpConnection.ts create mode 100644 src/browser/CdpEndpoint.ts create mode 100644 src/browser/CdpOptionalCommand.ts create mode 100644 src/browser/CdpPageCapture.ts create mode 100644 src/cliBrowserCommands.ts create mode 100644 src/contracts/browserToolContracts.ts create mode 100644 src/domain/browserObservation.ts create mode 100644 src/server/registerBrowserTools.ts create mode 100644 tests/browserCli.test.ts create mode 100644 tests/browserMcp.test.ts create mode 100644 tests/browserObservation.test.ts create mode 100644 tests/cdpBrowserProvider.test.ts create mode 100644 tests/cdpCaptureDocuments.test.ts create mode 100644 tests/cdpConnection.test.ts create mode 100644 tests/fixtures/fakeCdpBrowser.ts diff --git a/AGENTS.md b/AGENTS.md index 233f67985..dd16617e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,13 +26,15 @@ See [docs/architecture.mermaid](docs/architecture.mermaid) for a visual architec - `src/cli.ts`: one-shot CLI adapter for setup, diagnostics, analysis, and decompilation. - `src/config.ts`: Zod-validated parsing of environment configuration into `AppConfig`. - `src/domain/`: pure, side-effect-free modules. `errors.ts` (tagged error algebra), `result.ts` (`Result`/`ok`/`err`), `hopperValues.ts` (boundary parsers for Hopper JSON), `symbolAnalysis.ts` (Swift/ObjC name parsing). -- `src/contracts/`: caller-visible schemas for 33 direct, 10 enhanced, 5 native, 2 artifact, and 18 session tools; `enhancedInputs.ts` owns enhanced input parsing. +- `src/contracts/`: caller-visible schemas for 33 direct, 10 enhanced, 5 native, 2 artifact, 2 browser, and 18 session tools; `enhancedInputs.ts` owns enhanced input parsing. +- `src/browser/`: loopback CDP discovery, bounded WebSocket transport, exact-origin target authorization, and passive browser observation normalization. - `src/hopper/`: Hopper launch and Unix-socket protocol mechanics. `BridgeLauncher.ts` spawns the Hopper app with the in-process bridge, `HopperClient.ts` correlates request/response over the socket with timeouts and cancellation, `protocol.ts` frames bridge messages. - `bridge/hopper_bridge.py`: runs inside Hopper and adapts declared operations to Hopper's public Python API. Hopper's bundled MCP server is not used. - `src/application/`: shared CLI/MCP session composition, setup and diagnostics, and enhanced workflows. - `src/server/`: MCP request translation. `createServer.ts` assembles the MCP server, `registerOfficialTools.ts`/`registerEnhancedTools.ts` register each tool set, `toolResult.ts` maps `Result` values to MCP content. - `tests/`: Vitest suite. `tests/fixtures/` holds the fake launcher and fake Hopper bridge used as production seams. - `scripts/verify-real-hopper.mjs`: real-Hopper end-to-end verifier. +- `scripts/verify-real-browser.mjs`: real Chrome end-to-end verifier for the passive CDP provider. - `scripts/print-mcp-config.mjs`: prints an MCP server config with absolute paths filled in (`npm run config:print -- /path/to/binary`). ## Build, Test, and Development Commands @@ -49,7 +51,8 @@ See [docs/architecture.mermaid](docs/architecture.mermaid) for a visual architec - `npm run jscpd`: detect duplicate code blocks. - `npm run scan:todos`: scan for TODO, FIXME, and HACK markers. - `npm run verify:hopper`: build and run the real-Hopper verifier with two distinct binaries. -- `npm run verify:package`: pack and test the CLI, setup transaction, skill, and 68-tool target-free MCP server in an isolated environment. +- `npm run verify:browser`: build and run the real Chrome verifier against `REA_BROWSER_EXECUTABLE` or a platform-default Chrome-family executable. +- `npm run verify:package`: pack and test the CLI, setup transaction, skill, and 70-tool target-free MCP server in an isolated environment. - `npm run docs:generate`: generate API documentation from JSDoc comments into `docs/api/` using TypeDoc. - `npm run config:print -- /path/to/binary`: print an MCP server config with absolute paths. - `HOPPER_TARGET_PATH=/path/to/binary npm start`: launch Hopper and run the built stdio MCP server. @@ -62,6 +65,9 @@ Pre-commit hooks via Husky run `oxlint` on staged files before every commit. Use - `HOPPER_LAUNCHER_PATH` (optional): override the Hopper executable path (defaults to `/Applications/Hopper Disassembler.app/Contents/MacOS/hopper`). - `HOPPER_TARGET_KIND` (optional, default `executable`): startup kind for `HOPPER_TARGET_PATH`; dynamic targets are classified from their paths and headers. - `HOPPER_LOADER_ARGS_JSON` (optional): JSON array overriding derived Hopper loader arguments for supported executable targets, e.g. `["-l","Mach-O","--aarch64"]`. +- `REA_BROWSER_OBSERVE_ENABLED` (optional, default `false`): add browser observation authority to the administrator ceiling. +- `REA_BROWSER_CDP_ENDPOINTS_JSON` (optional): approved literal loopback CDP HTTP endpoints. +- `REA_BROWSER_ALLOWED_ORIGINS_JSON` (optional): exact HTTP(S) page origins approved for passive observation. ## Coding Style & Naming Conventions @@ -69,7 +75,7 @@ Use ESM TypeScript, two-space indentation, and Prettier defaults. Keep compiler ## Testing Guidelines -Name tests `*.test.ts`. Use Vitest and production seams (`tests/fixtures/`) rather than module mocks. Domain tests assert pure behavior; adapter tests use the fake launcher/socket; MCP tests connect with the beta.3 client. Preserve the 33 direct, 10 enhanced, 5 native, 2 artifact, and 18 session tool inventory (68 total). Cover malformed input, cancellation, timeouts, process exit, concurrency, limits, and clean shutdown. Real Hopper claims cannot be replaced by mocks — use `npm run verify:hopper` with two distinct binaries for end-to-end verification. +Name tests `*.test.ts`. Use Vitest and production seams (`tests/fixtures/`) rather than module mocks. Domain tests assert pure behavior; adapter tests use the fake launcher/socket; MCP tests connect with the beta.3 client. Preserve the 33 direct, 10 enhanced, 5 native, 2 artifact, 2 browser, and 18 session tool inventory (70 total). Cover malformed input, cancellation, timeouts, process exit, concurrency, limits, and clean shutdown. Real Hopper and browser claims cannot be replaced by mocks; use `npm run verify:hopper` with two distinct binaries and `npm run verify:browser` with Chrome. ## Commit & Pull Request Guidelines diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 713a044e0..ac09b405e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ npm run build npm test ``` -Keep dependencies flowing inward through the existing domain, contracts, Hopper, application, server, and adapter layers. Parse unknown values at process and protocol boundaries, model expected failures with `Result`, and preserve the 33 direct, 10 enhanced, 5 native, 2 artifact, and 18 session tools (68 total) unless a deliberate contract change updates every verifier and snapshot. +Keep dependencies flowing inward through the existing domain, contracts, provider, application, server, and adapter layers. Parse unknown values at process and protocol boundaries, model expected failures with `Result`, and preserve the 33 direct, 10 enhanced, 5 native, 2 artifact, 2 browser, and 18 session tools (70 total) unless a deliberate contract change updates every verifier and snapshot. Before submitting a pull request, run: @@ -70,7 +70,7 @@ Publish the public package: npm publish --access public ``` -After npm registry propagation, verify the published CLI and connect a beta.3 MCP client to the published server to confirm all 68 tools: +After npm registry propagation, verify the published CLI and connect a beta.3 MCP client to the published server to confirm all 70 tools: ```bash npx -y rea-agents --help diff --git a/README.md b/README.md index 973939f88..08f341812 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,11 @@ [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![68 MCP tools](https://img.shields.io/badge/MCP_tools-68-5c4ee5?style=flat-square)](#68-tools-for-investigation) +[![70 MCP tools](https://img.shields.io/badge/MCP_tools-70-5c4ee5?style=flat-square)](#70-tools-for-investigation) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) -[Quick start](#quick-start) · [Current status](#current-status) · [Investigation model](#the-investigation-model) · [68 tools](#68-tools-for-investigation) · [Roadmap](#roadmap) · [How it works](#how-it-works) +[Quick start](#quick-start) · [Current status](#current-status) · [Investigation model](#the-investigation-model) · [70 tools](#70-tools-for-investigation) · [Roadmap](#roadmap) · [How it works](#how-it-works)
@@ -294,15 +294,16 @@ REA handles the app analysis in steps 1–5. The agent performs step 6 with its - Analyze Swift and Objective-C metadata without manually untangling every mangled symbol. - Leave names, comments, and bookmarks in Hopper so human and agent analysis reinforce each other. -## 68 tools for investigation +## 70 tools for investigation -| Tool family | Count | Examples | -| ------------------------- | ----: | ----------------------------------------------------------------------------------------------------------------------- | -| Native inspection | 33 | procedures, pseudocode, assembly, strings, names, segments, callers, callees, xrefs, annotations | -| Investigation workflows | 10 | `binary_overview`, `analyze_function`, `batch_decompile`, `trace_feature`, call graphs, Swift and Objective-C discovery | -| Native macOS utilities | 5 | Mach-O metadata, code signatures, plists, architectures, Swift demangling; Hopper-free and provenance-bearing | -| Artifact graph | 2 | deterministic directory, ZIP/APK/IPA, and ASAR inventory; explicitly selected extraction into an absent owned tree | -| Workspace and observation | 18 | target lifecycle, Evidence v2 bundles, process/artifact/function comparison, evidence-linked residual-unknown lifecycle | +| Tool family | Count | Examples | +| ------------------------- | ----: | ---------------------------------------------------------------------------------------------------------------------------------- | +| Native inspection | 33 | procedures, pseudocode, assembly, strings, names, segments, callers, callees, xrefs, annotations | +| Investigation workflows | 10 | `binary_overview`, `analyze_function`, `batch_decompile`, `trace_feature`, call graphs, Swift and Objective-C discovery | +| Native macOS utilities | 5 | Mach-O metadata, code signatures, plists, architectures, Swift demangling; Hopper-free and provenance-bearing | +| Artifact graph | 2 | deterministic directory, ZIP/APK/IPA, and ASAR inventory; explicitly selected extraction into an absent owned tree | +| Browser observation | 2 | exact-origin CDP page discovery and passive DOM, accessibility, script, resource, network, console, worker, and storage inspection | +| Workspace and observation | 18 | target lifecycle, Evidence v2 bundles, process/artifact/function comparison, evidence-linked residual-unknown lifecycle | The public interface describes what the agent is trying to learn. Providers decide how to answer. macOS utilities handle common semantic inspection without launching Hopper; Hopper handles deeper native analysis; the process harness implements controlled behavioral capture. @@ -311,6 +312,7 @@ The public interface describes what the agent is trying to learn. Providers deci REA is already useful for native application investigation on macOS: - Open Mach-O, ELF, PE, `.app`, ZIP, APK, IPA, ASAR, plist, JavaScript, source-map, and Hopper database targets. +- Attach to a user-owned Chrome-family browser over a configured loopback CDP endpoint, list only pages on approved exact origins, and capture bounded passive web evidence without navigation or JavaScript evaluation. - Traverse content-addressed artifact graphs without extraction; on macOS, read-only DMG traversal additionally requires `native_mount_approved: true` and `REA_ARTIFACT_NATIVE_MOUNT_ENABLED=true`. Materialize only approved occurrences into absent output roots. - Build bounded function dossiers with pseudocode, assembly, CFG edges, comments, calls, references, strings, and names. - Search and trace features across symbols, strings, metadata, references, and call paths. @@ -331,12 +333,27 @@ REA is already useful for native application investigation on macOS: Hopper is the first provider, not the boundary of the project. Some current workflows still require Hopper and macOS; every evidence record identifies the provider and limitations behind its result. +### Website observation with CDP + +REA can inspect an already-running Chrome-family browser that you own. Browser observation is disabled by default and requires a literal loopback CDP endpoint plus exact approved page origins: + +```bash +export REA_BROWSER_OBSERVE_ENABLED=true +export REA_BROWSER_CDP_ENDPOINTS_JSON='["http://127.0.0.1:9222"]' +export REA_BROWSER_ALLOWED_ORIGINS_JSON='["http://127.0.0.1:3000"]' + +rea list-browser-targets http://127.0.0.1:9222 --approved --json +rea inspect-web-page http://127.0.0.1:9222 TARGET_ID --approved --json +``` + +`list_browser_targets` and `inspect_web_page` expose the same Evidence v2 contracts over MCP. Inspection is passive: REA does not evaluate page JavaScript, navigate, click, close the page, or close the browser. It redacts query values and never retains headers, bodies, cookies, storage values, console argument values, or WebSocket payloads. Existing activity before attach is explicitly unavailable. See [Website observation with CDP](docs/browser-observation.md) for browser startup, schemas, limits, and the threat model. + ## Roadmap REA is growing into a toolkit for understanding software across static artifacts and observed behavior. The next capability families are: 1. **Artifact decomposition** — DMG, ASAR, ZIP, packages, universal-binary slices, application resources, embedded frameworks, mobile packages, and artifact graphs. -2. **Web and Electron investigation** — Playwright/CDP capture of DOM, accessibility trees, screenshots, storage, console, IPC, HTTP, WebSocket, routes, and visual or structural differences. +2. **Web and Electron investigation** — extend the shipped passive CDP observation with approved interaction, screenshots, Electron IPC, route discovery, controlled replay, and visual or structural differences. 3. **Deterministic behavior harnesses** — stronger process-tree ownership, protocol fixtures, network policy, filesystem tracing, signals, reconnects, and cross-version comparison. 4. **JavaScript and source recovery** — bundle indexing, AST/module reconstruction, source-map discovery, historical-source matching, and CodeDB-backed cross-references. 5. **Runtime observation** — approval-gated LLDB, Frida, system logs, process and filesystem observers, and native API tracing. @@ -381,8 +398,9 @@ flowchart LR Router --> Hopper["Hopper provider"] Router --> Native["Native macOS provider"] Router --> Artifact["Artifact graph provider"] + Router --> Browser["Browser CDP provider"] Router --> Process["Process capture provider"] - Router -. roadmap .-> More["Browser, dynamic,
and additional static providers"] + Router -. roadmap .-> More["Dynamic and additional
static providers"] Hopper --> Target["Target software"] Process --> Target Native --> Target diff --git a/README_ar.md b/README_ar.md index 16187da35..097053d4c 100644 --- a/README_ar.md +++ b/README_ar.md @@ -10,11 +10,11 @@ [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![68 MCP tools](https://img.shields.io/badge/MCP_tools-68-5c4ee5?style=flat-square)](#منصة-من-68-أداة) +[![70 MCP tools](https://img.shields.io/badge/MCP_tools-70-5c4ee5?style=flat-square)](#منصة-من-70-أداة) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) -[البدء السريع](#البدء-السريع) · [من الملف التنفيذي إلى السلوك](#من-الملف-التنفيذي-إلى-السلوك) · [68 أداة](#منصة-من-68-أداة) · [كيف يعمل](#كيف-يعمل) · [الأسئلة الشائعة](#الأسئلة-الشائعة) +[البدء السريع](#البدء-السريع) · [من الملف التنفيذي إلى السلوك](#من-الملف-التنفيذي-إلى-السلوك) · [70 أداة](#منصة-من-70-أداة) · [كيف يعمل](#كيف-يعمل) · [الأسئلة الشائعة](#الأسئلة-الشائعة)
@@ -155,15 +155,16 @@ rea uninstall --purge-data افحص التحقق من الإدخال، والتشفير، والصلاحيات، والتخزين، ومسارات الأخطاء من دون رفع الملف التنفيذي إلى خدمة بعيدة. -## منصة من 68 أداة +## منصة من 70 أداة -| عائلة الأدوات | العدد | أمثلة | -| --------------------- | ----: | ----------------------------------------------------------------------------------- | -| فحص الملفات التنفيذية | 33 | الدوال، والشيفرة شبه المصدرية، والتعليمات، والسلاسل، والأسماء، والمراجع، والتعليقات | -| التحليل المركب | 10 | النظرة العامة، وفك الترجمة الدفعي، ومخططات الاستدعاء، والمراجع، واكتشاف الأنواع | -| أدوات macOS الأصلية | 5 | بيانات Mach-O والتوقيعات وملفات plist والمعماريات وفك رموز Swift دون تشغيل Hopper | -| رسم القطع الأثرية | 2 | جرد حتمي للدلائل وZIP/APK/IPA وASAR واستخراج معاملاتي محدد صراحة | -| جلسة الملف التنفيذي | 18 | فتح الأهداف وحزم الأدلة ومقارنة العمليات والقطع والدوال وسجل المجهولات المتبقية | +| عائلة الأدوات | العدد | أمثلة | +| --------------------- | ----: | ---------------------------------------------------------------------------------------- | +| فحص الملفات التنفيذية | 33 | الدوال، والشيفرة شبه المصدرية، والتعليمات، والسلاسل، والأسماء، والمراجع، والتعليقات | +| التحليل المركب | 10 | النظرة العامة، وفك الترجمة الدفعي، ومخططات الاستدعاء، والمراجع، واكتشاف الأنواع | +| أدوات macOS الأصلية | 5 | بيانات Mach-O والتوقيعات وملفات plist والمعماريات وفك رموز Swift دون تشغيل Hopper | +| رسم القطع الأثرية | 2 | جرد حتمي للدلائل وZIP/APK/IPA وASAR واستخراج معاملاتي محدد صراحة | +| مراقبة المتصفح | 2 | اكتشاف صفحات CDP ضمن أصول دقيقة وفحص سلبي لـ DOM وإتاحة الوصول والبرامج والشبكة والتخزين | +| جلسة الملف التنفيذي | 18 | فتح الأهداف وحزم الأدلة ومقارنة العمليات والقطع والدوال وسجل المجهولات المتبقية | ## استخدام REA مع وكلاء برمجة آخرين diff --git a/README_ja.md b/README_ja.md index a6c99a25c..b7e37cee4 100644 --- a/README_ja.md +++ b/README_ja.md @@ -10,11 +10,11 @@ [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![68 MCP tools](https://img.shields.io/badge/MCP_tools-68-5c4ee5?style=flat-square)](#68-ツールのワークベンチ) +[![70 MCP tools](https://img.shields.io/badge/MCP_tools-70-5c4ee5?style=flat-square)](#70-ツールのワークベンチ) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) -[クイックスタート](#クイックスタート) · [バイナリから動作へ](#バイナリから動作へ) · [68 ツール](#68-ツールのワークベンチ) · [仕組み](#仕組み) · [FAQ](#faq) +[クイックスタート](#クイックスタート) · [バイナリから動作へ](#バイナリから動作へ) · [70 ツール](#70-ツールのワークベンチ) · [仕組み](#仕組み) · [FAQ](#faq)
@@ -145,7 +145,7 @@ REA は手順 1〜5 のバイナリ解析を処理し、手順 6 はエージェ - Swift / Objective-C のメタデータを解析する。 - Hopper に名前、コメント、ブックマークを残し、人間とエージェントの調査を共有する。 -## 68 ツールのワークベンチ +## 70 ツールのワークベンチ | ツール群 | 数 | 例 | | ---------------------- | --: | ------------------------------------------------------------------------------------------------------------------- | @@ -153,6 +153,7 @@ REA は手順 1〜5 のバイナリ解析を処理し、手順 6 はエージェ | 合成解析 | 10 | `binary_overview`, `analyze_function`, `batch_decompile`, `get_call_graph`, `find_xrefs_to_name`, Swift / ObjC 検出 | | macOS ネイティブ | 5 | Mach-O メタデータ、署名、plist、アーキテクチャ、Swift デマングル。Hopper 起動不要 | | アーティファクトグラフ | 2 | ディレクトリ、ZIP/APK/IPA、ASAR の決定的一覧と明示選択されたトランザクション抽出 | +| ブラウザ観察 | 2 | 完全一致 origin に限定した CDP ページ検出と受動的な DOM、アクセシビリティ、スクリプト、通信、保存領域の調査 | | バイナリセッション | 18 | `open_binary`、`binary_session`、証拠バンドル、プロセス／成果物／関数比較、残余未知レジストリ | ## 他のコーディングエージェントで使う diff --git a/README_ko.md b/README_ko.md index 7e8f1ffd3..a7e0c7e46 100644 --- a/README_ko.md +++ b/README_ko.md @@ -10,11 +10,11 @@ [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![68 MCP tools](https://img.shields.io/badge/MCP_tools-68-5c4ee5?style=flat-square)](#68개-도구로-구성된-워크벤치) +[![70 MCP tools](https://img.shields.io/badge/MCP_tools-70-5c4ee5?style=flat-square)](#70개-도구로-구성된-워크벤치) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) -[빠른 시작](#빠른-시작) · [바이너리에서 동작까지](#바이너리에서-동작까지) · [68개 도구](#68개-도구로-구성된-워크벤치) · [작동 방식](#작동-방식) · [FAQ](#faq) +[빠른 시작](#빠른-시작) · [바이너리에서 동작까지](#바이너리에서-동작까지) · [70개 도구](#70개-도구로-구성된-워크벤치) · [작동 방식](#작동-방식) · [FAQ](#faq)
@@ -145,7 +145,7 @@ REA는 1–5단계의 바이너리 분석을 처리합니다. 6단계는 에이 - Swift 및 Objective-C 메타데이터를 분석합니다. - Hopper에 이름, 주석, 북마크를 남겨 사람과 에이전트의 분석을 연결합니다. -## 68개 도구로 구성된 워크벤치 +## 70개 도구로 구성된 워크벤치 | 도구 그룹 | 수 | 예시 | | ------------------- | --: | -------------------------------------------------------------------------------------------------------------------- | @@ -153,6 +153,7 @@ REA는 1–5단계의 바이너리 분석을 처리합니다. 6단계는 에이 | 합성 분석 | 10 | `binary_overview`, `analyze_function`, `batch_decompile`, `get_call_graph`, `find_xrefs_to_name`, Swift 및 ObjC 탐색 | | macOS 네이티브 도구 | 5 | Mach-O 메타데이터, 코드 서명, plist, 아키텍처, Swift 디맹글링. Hopper 실행 불필요 | | 아티팩트 그래프 | 2 | 디렉터리, ZIP/APK/IPA, ASAR 결정적 인벤토리와 명시적으로 선택한 트랜잭션 추출 | +| 브라우저 관찰 | 2 | 정확한 origin 범위의 CDP 페이지 검색과 수동 DOM, 접근성, 스크립트, 네트워크, 콘솔 및 저장소 조사 | | 바이너리 세션 | 18 | `open_binary`, `binary_session`, 증거 번들, 프로세스·아티팩트·함수 비교, 잔여 미지수 레지스트리 | ## 다른 코딩 에이전트에서 사용하기 diff --git a/README_zh.md b/README_zh.md index f89c64a3b..4bdc6cd9c 100644 --- a/README_zh.md +++ b/README_zh.md @@ -10,11 +10,11 @@ [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![68 MCP tools](https://img.shields.io/badge/MCP_tools-68-5c4ee5?style=flat-square)](#68-个工具组成的工作台) +[![70 MCP tools](https://img.shields.io/badge/MCP_tools-70-5c4ee5?style=flat-square)](#70-个工具组成的工作台) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) -[快速开始](#快速开始) · [从二进制到行为](#从二进制到行为) · [68 个工具](#68-个工具组成的工作台) · [工作原理](#工作原理) · [常见问题](#常见问题) +[快速开始](#快速开始) · [从二进制到行为](#从二进制到行为) · [70 个工具](#70-个工具组成的工作台) · [工作原理](#工作原理) · [常见问题](#常见问题)
@@ -145,7 +145,7 @@ REA 负责第 1–5 步中的二进制分析。第 6 步由智能体使用其常 - 分析 Swift 和 Objective-C 元数据。 - 在 Hopper 中留下名称、注释与书签，使人与智能体的分析互相增强。 -## 68 个工具组成的工作台 +## 70 个工具组成的工作台 | 工具类别 | 数量 | 示例 | | -------------- | ---: | -------------------------------------------------------------------------------------------------------------------- | @@ -153,6 +153,7 @@ REA 负责第 1–5 步中的二进制分析。第 6 步由智能体使用其常 | 组合分析 | 10 | `binary_overview`, `analyze_function`, `batch_decompile`, `get_call_graph`, `find_xrefs_to_name`、Swift 与 ObjC 发现 | | macOS 原生工具 | 5 | Mach-O 元数据、代码签名、plist、架构与 Swift 符号还原，无需启动 Hopper | | 制品图谱 | 2 | 确定性目录、ZIP/APK/IPA 与 ASAR 清单，以及显式选择的事务式提取 | +| 浏览器观察 | 2 | 基于精确来源范围的 CDP 页面发现，以及被动 DOM、无障碍树、脚本、网络、控制台与存储检查 | | 二进制会话 | 18 | `open_binary`、`binary_session`、证据包、进程、制品与函数比较、残余未知项注册表 | ## 与其他编程智能体一起使用 diff --git a/docs/architecture.mermaid b/docs/architecture.mermaid index 39eed561a..60d7b68d5 100644 --- a/docs/architecture.mermaid +++ b/docs/architecture.mermaid @@ -20,8 +20,12 @@ graph TD HOPPER["src/hopper/
Hopper launch & Unix-socket protocol
BridgeLauncher, HopperClient, protocol"] end + subgraph "Browser Provider Layer" + BROWSER["src/browser/
Loopback CDP transport
exact-origin passive observation"] + end + subgraph "Contracts Layer" - CONTRACTS["src/contracts/
Caller-visible tool schemas
68 direct, enhanced, native, artifact, and session tools"] + CONTRACTS["src/contracts/
Caller-visible tool schemas
70 analysis, browser, and session tools"] end subgraph "Domain Layer" @@ -29,11 +33,11 @@ graph TD end subgraph "Configuration" - CONFIG["src/config.ts
Zod-validated env parsing
provider settings and approved roots"] + CONFIG["src/config.ts
Zod-validated env parsing
provider settings, origins, and approved roots"] end subgraph "Tests" - TESTS["tests/
Vitest suite
fixtures/ holds fake launcher & bridge seams"] + TESTS["tests/
Vitest suite
fixtures/ holds fake launcher, bridge, and CDP seams"] end DISPATCH --> CLI @@ -47,9 +51,12 @@ graph TD SERVER --> CONTRACTS SERVER --> APP APP --> HOPPER + APP --> BROWSER HOPPER --> BRIDGE APP --> DOMAIN CONTRACTS --> DOMAIN HOPPER --> DOMAIN + BROWSER --> DOMAIN TESTS -.-> MAIN TESTS -.-> HOPPER + TESTS -.-> BROWSER diff --git a/docs/browser-observation.md b/docs/browser-observation.md new file mode 100644 index 000000000..1728d61d3 --- /dev/null +++ b/docs/browser-observation.md @@ -0,0 +1,121 @@ +# Website observation with CDP + +REA can attach to a user-owned Chrome-family browser through the Chrome DevTools Protocol (CDP) and produce bounded Evidence v2 about an existing page. This is a passive reverse-engineering capability, not a general browser automation or remote-control surface. + +## Shipped surfaces + +- `list_browser_targets` / `rea list-browser-targets` discovers page targets whose current URL matches an approved exact origin. +- `inspect_web_page` / `rea inspect-web-page` captures bounded DOM structure, accessibility nodes, scripts, resources, attach-window network and console metadata, WebSocket frame sizes, workers, quota, and optionally storage key names or script sources. +- Both surfaces return the same provider-neutral normalized results and Evidence v2 provenance. +- MCP session results are retained as `rea://evidence/{evidenceId}` resources. One-shot CLI output is not retained by a long-lived REA session. + +## Start a browser + +Start a separate browser profile with an explicit debugging port. The exact executable name varies by platform and installation: + +```bash +google-chrome \ + --remote-debugging-port=9222 \ + --user-data-dir=/tmp/rea-browser-profile \ + http://127.0.0.1:3000 +``` + +REA does not launch, own, or terminate this browser. Use a dedicated profile and stop it yourself when the investigation is complete. Do not expose the debugging port on a non-loopback interface. + +## Configure authority + +Browser observation is fail-closed by default. Configure the administrator ceiling before starting REA: + +```bash +export REA_BROWSER_OBSERVE_ENABLED=true +export REA_BROWSER_CDP_ENDPOINTS_JSON='["http://127.0.0.1:9222"]' +export REA_BROWSER_ALLOWED_ORIGINS_JSON='["http://127.0.0.1:3000"]' +``` + +`REA_BROWSER_CDP_ENDPOINTS_JSON` accepts at most 16 literal `http://127.0.0.1:PORT` or `http://[::1]:PORT` endpoints. `localhost`, private-LAN addresses, HTTPS endpoints, credentials, paths, queries, fragments, and implicit ports are rejected. + +`REA_BROWSER_ALLOWED_ORIGINS_JSON` accepts at most 32 exact HTTP(S) origins. Paths, credentials, queries, fragments, and wildcards are rejected. An input must stay within both this administrator ceiling and any active project/session/once grant. + +Configuration is reloadable through the existing `SIGHUP` permission-policy path. A request still requires `approved: true`; configuration alone does not make a tool call implicit. + +Inspect the exact effective scope without contacting the browser: + +```bash +rea policy explain browser_observe \ + --origins http://127.0.0.1:9222 \ + --origins http://127.0.0.1:3000 \ + --network loopback \ + --json +``` + +## CLI workflow + +Configured origins are used by default. `--allowed-origins` can request a narrower set that still fits the configured ceiling. + +```bash +rea list-browser-targets http://127.0.0.1:9222 --approved --json +rea inspect-web-page http://127.0.0.1:9222 TARGET_ID \ + --approved \ + --observation-ms 1000 \ + --json +``` + +Script content and storage key names require separate opt-ins: + +```bash +rea inspect-web-page http://127.0.0.1:9222 TARGET_ID \ + --approved \ + --include-script-sources \ + --include-storage-keys \ + --json +``` + +Script content may contain application secrets and becomes part of the returned Evidence. Storage values remain redacted even when key-name capture is approved. + +## MCP input + +```json +{ + "cdp_endpoint": "http://127.0.0.1:9222", + "allowed_origins": ["http://127.0.0.1:3000"], + "target_id": "TARGET_ID_FROM_LIST_BROWSER_TARGETS", + "approved": true, + "observation_ms": 500, + "include_script_sources": false, + "include_storage_keys": false +} +``` + +Call `list_browser_targets` first because target IDs are browser-instance-specific. REA rechecks the selected target's current type and origin immediately before attaching. + +REA establishes an authorized main-frame boundary before enabling Runtime, Debugger, or Network observation, rechecks it before document capture, and validates it again afterward. If the main frame navigates during final capture, REA discards the mixed result and returns `target_changed`. + +## Data minimization + +REA removes sensitive values before normalized event data is retained: + +- URLs retain a bounded path and at most 256 bounded query parameter names, replace every retained query value with `[REDACTED]`, and remove credentials and fragments. +- DOM snapshots retain node types, node names, value lengths, and attribute names, but not text or attribute values. +- Network observations retain method, status, MIME type, size, type, and redacted URLs, but not request/response headers, bodies, post data, cookies, or response content. +- Console observations with an approved stack source retain call type, argument types, timestamp, and redacted source location, but not argument values or descriptions. Events without a source URL are excluded because their origin cannot be established. +- WebSocket observations retain direction, opcode, and payload byte length, but not payload content. +- Storage observations always redact values. Key names, IndexedDB names, and cache names are omitted unless explicitly requested. +- Script metadata is included only when CDP supplies a URL on an allowed origin. URL-less scripts are excluded because their origin cannot be established. Source content is omitted unless explicitly requested and remains subject to per-script and aggregate byte limits. + +Cross-origin frames, resources, scripts, events, and workers are excluded unless their exact origins are also approved. Excluded target details are counted without being exposed. + +## Completeness and limits + +The default observation window is 500 ms and the maximum is 10 seconds. Frames (200), DOM nodes (2,000), accessibility nodes (2,000), scripts (200), resources (2,000), workers (500), and each storage key/name inventory (1,000) have conservative defaults and caller-visible hard bounds. Script source bytes, network events, console events, and WebSocket events are bounded separately. The CLI exposes the same limits as kebab-case options such as `--max-frames` and `--max-storage-keys`. + +A bounded result reports `completeness.status: "truncated"`, named truncated sections, and dropped event counts rather than implying exhaustive capture. Disallowed-origin entries are filtered before collection-limit accounting, so they cannot consume an approved origin's output budget. + +CDP discovery, WebSocket connection, and each command have a 5-second timeout. Version discovery is capped at 64 KiB, target discovery at 2 MiB and 1,000 targets, each WebSocket message at 16 MiB, and correlated pending commands at 128. Malformed unsolicited protocol events poison the connection so later commands fail closed. + +Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Source maps are reported only as redacted declarative URLs and are never fetched. + +## Non-goals and threat model + +The first browser provider deliberately does not expose generic CDP commands, `Runtime.evaluate`, navigation, input, downloads, screenshots, page closure, or browser closure. REA disables the domains it enabled, detaches the target session, and closes only its own WebSocket. + +This feature is not a browser sandbox or network containment mechanism. The attached page and browser continue running with their existing privileges and may make external requests independently of REA. CDP gives deep access to the selected browser profile, so use a dedicated profile, approve only origins you intend to inspect, and treat other same-user processes as outside this boundary. diff --git a/docs/error-contract.schema.json b/docs/error-contract.schema.json index 4c31f236d..39d31b5af 100644 --- a/docs/error-contract.schema.json +++ b/docs/error-contract.schema.json @@ -460,6 +460,7 @@ "type": "array", "items": { "type": "string" } }, + "origins": { "type": "array", "items": { "type": "string" } }, "network": { "type": "string", "enum": ["none", "loopback", "external"] @@ -488,6 +489,7 @@ "type": "array", "items": { "type": "string" } }, + "origins": { "type": "array", "items": { "type": "string" } }, "network": { "type": "string", "enum": ["none", "loopback", "external"] @@ -511,6 +513,10 @@ "type": "array", "items": { "type": "string" } }, + "origins": { + "type": "array", + "items": { "type": "string" } + }, "network": { "type": "string", "enum": ["none", "loopback", "external"] diff --git a/docs/mcp-prompts.md b/docs/mcp-prompts.md index 37bb615bd..ce9d44a78 100644 --- a/docs/mcp-prompts.md +++ b/docs/mcp-prompts.md @@ -1,7 +1,7 @@ # Guided MCP prompts and completion REA exposes six provider-neutral investigation workflows through the MCP -`prompts` capability. They complement the 68 tools; they do not add, remove, or +`prompts` capability. They complement the 70 tools; they do not add, remove, or invoke tools by themselves. ## Prompt inventory diff --git a/package.json b/package.json index b19c45c5b..d4a6310b3 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "test:watch": "vitest", "typecheck": "tsc --noEmit", "verify:hopper": "npm run build && node scripts/verify-real-hopper.mjs", + "verify:browser": "npm run build && node scripts/verify-real-browser.mjs", "verify:hopper:linux": "npm run build && node scripts/verify-real-hopper-linux.mjs", "verify:setup:linux": "npm run build && node scripts/verify-linux-setup-e2e.mjs", "build:fixtures": "node scripts/build-conformance-fixtures.mjs", diff --git a/scripts/rea.mjs b/scripts/rea.mjs index a9ba8206b..695fbc79f 100755 --- a/scripts/rea.mjs +++ b/scripts/rea.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // Route production MCP before importing Incur. Incur owns registration helpers -// such as `mcp add`, while only dist/main.js may serve the 68-tool stdio server. +// such as `mcp add`, while only dist/main.js may serve the 70-tool stdio server. const args = process.argv.slice(2); const { default: packageJson } = await import("../package.json", { with: { type: "json" }, diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index f5dc1fbbc..1f267ed9a 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -130,6 +130,8 @@ try { !help.includes("extract-artifact") || !help.includes("investigate-versions") || !help.includes("import-reference-source") || + !help.includes("list-browser-targets") || + !help.includes("inspect-web-page") || !help.includes("compare") || !llms.includes("decompile") || !llms.includes("function") || @@ -140,6 +142,8 @@ try { !llms.includes("capabilities") || !llms.includes("providers") || !llms.includes("inventory-artifact") || + !llms.includes("list-browser-targets") || + !llms.includes("inspect-web-page") || doctor.healthy !== expectedDoctorHealth || doctorExecution.status !== (expectedDoctorHealth ? 0 : 1) || doctor.checks?.find(({ name }) => name === "hopper")?.ok !== true diff --git a/scripts/verify-published-package.mjs b/scripts/verify-published-package.mjs index 316237969..7b6e0da78 100644 --- a/scripts/verify-published-package.mjs +++ b/scripts/verify-published-package.mjs @@ -22,7 +22,7 @@ transport.stderr?.on("data", (chunk) => { try { await client.connect(transport); const tools = await client.listTools(); - if (tools.tools.length !== 68) + if (tools.tools.length !== 70) throw new Error( `Published MCP exposed ${String(tools.tools.length)} tools`, ); @@ -40,5 +40,5 @@ try { } process.stdout.write( - `${JSON.stringify({ package: "rea-agents", version, mcpTools: 68 })}\n`, + `${JSON.stringify({ package: "rea-agents", version, mcpTools: 70 })}\n`, ); diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs new file mode 100644 index 000000000..835c2141c --- /dev/null +++ b/scripts/verify-real-browser.mjs @@ -0,0 +1,286 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { access, mkdtemp, readFile, rm } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { WebSocketServer } from "ws"; + +import { CdpBrowserProvider } from "../dist/browser/CdpBrowserProvider.js"; +import { + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, +} from "../dist/domain/browserObservation.js"; + +const SECRET_VALUES = [ + "network-secret-value", + "console-secret-value", + "storage-secret-value", + "websocket-secret-value", +]; + +const executable = await browserExecutable(); +const profile = await mkdtemp(join(tmpdir(), "rea-real-browser-")); +const site = await startSite(); +let browser; +try { + browser = spawn( + executable, + [ + "--headless=new", + "--remote-debugging-address=127.0.0.1", + "--remote-debugging-port=0", + `--user-data-dir=${profile}`, + "--no-first-run", + "--no-default-browser-check", + "--disable-background-networking", + "--disable-component-update", + "--disable-sync", + "--metrics-recording-only", + `${site.origin}/app?startup=browser-secret-value`, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + let stderr = ""; + browser.stderr.on("data", (chunk) => { + if (stderr.length < 64 * 1_024) stderr += chunk.toString("utf8"); + }); + const port = await devtoolsPort(profile, browser, () => stderr); + const endpoint = `http://127.0.0.1:${String(port)}`; + const provider = new CdpBrowserProvider(); + const target = await pageTarget(provider, endpoint, site.origin); + const observed = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [site.origin], + approved: true, + target_id: target, + observation_ms: 1_000, + include_storage_keys: true, + }), + ); + if (!observed.ok) throw observed.error; + assertObservation(observed.value, site.origin); + const serialized = JSON.stringify(observed.value); + for (const secret of [...SECRET_VALUES, "browser-secret-value"]) + if (serialized.includes(secret)) + throw new Error( + `Passive browser result retained redacted value: ${secret}`, + ); + + const withSource = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [site.origin], + approved: true, + target_id: target, + observation_ms: 200, + include_script_sources: true, + }), + ); + if (!withSource.ok) throw withSource.error; + const source = withSource.value.scripts.items.find( + (script) => + script.source.included && + script.source.content.includes("reaSourceMarker"), + ); + if (source === undefined) + throw new Error( + "Real Chrome did not return explicitly approved script source", + ); + + process.stdout.write( + `${JSON.stringify({ + browser: observed.value.browser.product, + endpoint, + target, + domNodes: observed.value.dom.nodes.length, + accessibilityNodes: observed.value.accessibility.nodes.length, + scripts: observed.value.scripts.items.length, + networkRequests: observed.value.network.requests.length, + consoleEvents: observed.value.console.events.length, + websocketEvents: observed.value.network.websocket_events.length, + verified: true, + })}\n`, + ); +} finally { + if (browser !== undefined) await stopProcess(browser); + await site.close(); + await rm(profile, { recursive: true, force: true }); +} + +async function browserExecutable() { + const candidates = [ + process.env.REA_BROWSER_EXECUTABLE, + process.argv[2], + "/usr/bin/google-chrome-stable", + "/usr/bin/google-chrome", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + ].filter( + (candidate) => typeof candidate === "string" && candidate.length > 0, + ); + for (const candidate of candidates) { + try { + await access(candidate); + return candidate; + } catch { + // Continue through explicit and platform-default candidates. + } + } + throw new Error( + "No Chrome-family executable found; set REA_BROWSER_EXECUTABLE to run real-browser verification", + ); +} + +async function devtoolsPort(profile, child, stderr) { + const activePort = join(profile, "DevToolsActivePort"); + for (let attempt = 0; attempt < 200; attempt += 1) { + if (child.exitCode !== null) + throw new Error(`Chrome exited before CDP startup: ${stderr()}`); + try { + const [port] = (await readFile(activePort, "utf8")).trim().split("\n"); + const parsed = Number(port); + if (Number.isSafeInteger(parsed) && parsed > 0) return parsed; + } catch { + // Chrome writes this file only after its CDP listener is ready. + } + await delay(25); + } + throw new Error(`Timed out waiting for Chrome CDP startup: ${stderr()}`); +} + +async function pageTarget(provider, endpoint, origin) { + for (let attempt = 0; attempt < 100; attempt += 1) { + const listed = await provider.listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [origin], + approved: true, + }), + ); + if (listed.ok && listed.value.targets.items[0] !== undefined) + return listed.value.targets.items[0].target_id; + await delay(25); + } + throw new Error("Real Chrome did not expose the local test page target"); +} + +function assertObservation(result, origin) { + if (result.target.origin !== origin) + throw new Error("Real Chrome target origin was not preserved exactly"); + if (result.frames.length < 1 || result.dom.nodes.length < 1) + throw new Error( + `Real Chrome DOM/frame observation was empty: ${JSON.stringify({ frames: result.frames.length, domTotal: result.dom.total_nodes, domReturned: result.dom.nodes.length, limitations: result.limitations })}`, + ); + if (result.accessibility.nodes.length < 1) + throw new Error("Real Chrome accessibility observation was empty"); + if (!result.scripts.items.some((script) => script.url.includes("/app.js"))) + throw new Error("Real Chrome script metadata was missing"); + if (!result.resources.some((resource) => resource.url.includes("/app.js"))) + throw new Error("Real Chrome resource metadata was missing"); + if (!result.network.requests.some((request) => request.url.includes("/api"))) + throw new Error( + "Real Chrome attach-window network observation was missing", + ); + if (result.console.events.length < 1) + throw new Error( + "Real Chrome attach-window console observation was missing", + ); + if (result.network.websocket_events.length < 1) + throw new Error("Real Chrome WebSocket metadata was missing"); + if (!result.storage.local_storage_keys.includes("rea-storage-key")) + throw new Error("Real Chrome local-storage key inventory was missing"); + if (!result.storage.indexed_db_names.includes("rea-browser-db")) + throw new Error("Real Chrome IndexedDB name inventory was missing"); + if (!result.storage.cache_names.includes("rea-browser-cache")) + throw new Error("Real Chrome cache name inventory was missing"); +} + +async function startSite() { + let port = 0; + const server = createServer((request, response) => { + if (request.url?.startsWith("/app.js") === true) { + response.setHeader("content-type", "text/javascript"); + response.end(script(port)); + return; + } + if (request.url?.startsWith("/api") === true) { + response.setHeader("content-type", "application/json"); + response.end('{"ok":true,"secret":"response-secret-value"}'); + return; + } + response.setHeader("content-type", "text/html"); + response.end( + 'REA browser verifier

Browser evidence

', + ); + }); + const webSockets = new WebSocketServer({ noServer: true }); + server.on("upgrade", (request, socket, head) => { + webSockets.handleUpgrade(request, socket, head, (webSocket) => { + webSocket.on("message", () => + webSocket.send("websocket-response-secret"), + ); + }); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Local browser verifier did not bind a TCP port"); + port = address.port; + return { + origin: `http://127.0.0.1:${String(port)}`, + async close() { + for (const client of webSockets.clients) client.terminate(); + await new Promise((resolve) => webSockets.close(resolve)); + await new Promise((resolve, reject) => + server.close((error) => + error === undefined ? resolve() : reject(error), + ), + ); + }, + }; +} + +function script(port) { + return ` + export const reaSourceMarker = "approved-source-marker"; + localStorage.setItem("rea-storage-key", "storage-secret-value"); + indexedDB.open("rea-browser-db", 1); + caches.open("rea-browser-cache"); + const observe = () => { + console.log("rea-browser-observation", "console-secret-value"); + fetch("/api?token=network-secret-value", { + headers: { Authorization: "Bearer request-secret-value" } + }); + const socket = new WebSocket("ws://127.0.0.1:${String(port)}/live?token=websocket-url-secret"); + socket.addEventListener("open", () => socket.send("websocket-secret-value")); + socket.addEventListener("message", () => socket.close()); + }; + observe(); + setInterval(observe, 150); +`; +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function stopProcess(child) { + if (child.exitCode !== null) return; + child.kill("SIGTERM"); + const exited = await Promise.race([ + new Promise((resolve) => child.once("exit", () => resolve(true))), + delay(2_000).then(() => false), + ]); + if (!exited) { + child.kill("SIGKILL"); + await new Promise((resolve) => child.once("exit", resolve)); + } +} diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index a173596e4..2914f11a3 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -300,7 +300,7 @@ try { const expectedNames = TOOL_CONTRACTS.map(({ name }) => name).sort(); const actualNames = listed.tools.map(({ name }) => name).sort(); if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { - throw new Error("The real server did not expose the intended 68 tools"); + throw new Error("The real server did not expose the intended 70 tools"); } const options = { timeout }; diff --git a/skills/rea-analysis/SKILL.md b/skills/rea-analysis/SKILL.md index 3f59510f1..4902a925d 100644 --- a/skills/rea-analysis/SKILL.md +++ b/skills/rea-analysis/SKILL.md @@ -1,15 +1,30 @@ --- name: rea-analysis -description: Reverse engineer apps with REA. Explore how features work, then build a version tailored to your project. +description: Reverse engineer apps and websites with REA. Explore how features work, then build a version tailored to your project. metadata: - version: "12" - tool_count: 68 + version: "13" + tool_count: 70 --- # REA Use REA when the user wants to understand how an app or feature works, compare app versions, decompile code, or build a similar feature. +For a website already open in a user-owned Chrome-family browser, use +`list_browser_targets` and `inspect_web_page` instead of opening a binary. Ask +the user to approve the exact page origins and loopback CDP endpoint before the +first call. Browser observation is disabled unless the operator configured +`REA_BROWSER_OBSERVE_ENABLED`, `REA_BROWSER_CDP_ENDPOINTS_JSON`, and +`REA_BROWSER_ALLOWED_ORIGINS_JSON` before starting REA. + +Browser inspection is passive. Never claim that it clicked, navigated, +evaluated page JavaScript, captured prior activity, or contained the page's +network. Query values, headers, bodies, cookies, storage values, console values, +and WebSocket payloads are deliberately absent. Request script sources or +storage key names only when the user separately approves those more sensitive +outputs. Treat truncation and attach-window coverage as explicit limitations, +and cite the returned Evidence v2 IDs. + ## Understand the request Identify the app and what the user wants to understand or build. Do not ask for information they already provided. diff --git a/src/application/BrowserEvidence.ts b/src/application/BrowserEvidence.ts new file mode 100644 index 000000000..edba27982 --- /dev/null +++ b/src/application/BrowserEvidence.ts @@ -0,0 +1,61 @@ +import type { ProviderIdentity } from "./AnalysisProvider.js"; +import { + createEvidence, + type Evidence, + type EvidenceObservation, +} from "../domain/evidence.js"; +import { jsonValueSchema } from "../domain/jsonValue.js"; +import type { + BrowserTargetList, + InspectWebPageInput, + ListBrowserTargetsInput, + WebPageInspection, +} from "../domain/browserObservation.js"; + +type BrowserEvidenceInput = ListBrowserTargetsInput | InspectWebPageInput; +type BrowserEvidenceResult = BrowserTargetList | WebPageInspection; + +/** Create Evidence v2 for a policy-scoped external browser observation. */ +export const createBrowserEvidence = ( + operation: "list_browser_targets" | "inspect_web_page", + input: BrowserEvidenceInput, + result: BrowserEvidenceResult, + provider: ProviderIdentity, +): Evidence => + createEvidence(undefined, provider, { + predicateType: + operation === "list_browser_targets" + ? "rea.browser-target-list/v1" + : "rea.web-page-inspection/v1", + operation, + parameters: browserParameters(input), + result: jsonValueSchema.parse(result), + confidence: "observed", + authority: "external-service", + environment: + "target" in result + ? { + id: `${result.browser.product}@${result.browser.revision}`, + platform: process.platform, + architecture: process.arch, + isolation: "none", + } + : null, + limitations: result.limitations, + }); + +const browserParameters = ( + input: BrowserEvidenceInput, +): EvidenceObservation["parameters"] => ({ + cdp_endpoint: input.cdp_endpoint, + allowed_origins: input.allowed_origins, + ...("target_id" in input + ? { + target_id: input.target_id, + observation_ms: input.observation_ms, + include_script_sources: input.include_script_sources, + include_storage_keys: input.include_storage_keys, + limits: input.limits, + } + : { offset: input.offset, limit: input.limit }), +}); diff --git a/src/application/BrowserObservationPort.ts b/src/application/BrowserObservationPort.ts new file mode 100644 index 000000000..f9e3cb45a --- /dev/null +++ b/src/application/BrowserObservationPort.ts @@ -0,0 +1,23 @@ +import type { AnalysisError } from "../domain/errors.js"; +import type { + BrowserTargetList, + InspectWebPageInput, + ListBrowserTargetsInput, + WebPageInspection, +} from "../domain/browserObservation.js"; +import type { Result } from "../domain/result.js"; +import type { ProviderIdentity } from "./AnalysisProvider.js"; +import type { ExecutionOptions } from "./AnalysisProvider.js"; + +/** Provider-neutral application boundary for passive browser observation. */ +export interface BrowserObservationPort { + identity(): ProviderIdentity; + listTargets( + input: ListBrowserTargetsInput, + options?: ExecutionOptions, + ): Promise>; + inspectPage( + input: InspectWebPageInput, + options?: ExecutionOptions, + ): Promise>; +} diff --git a/src/application/BrowserObservationService.ts b/src/application/BrowserObservationService.ts new file mode 100644 index 000000000..859a33168 --- /dev/null +++ b/src/application/BrowserObservationService.ts @@ -0,0 +1,126 @@ +import type { Evidence } from "../domain/evidence.js"; +import type { + InspectWebPageInput, + ListBrowserTargetsInput, +} from "../domain/browserObservation.js"; +import { + AnalysisCapabilityUnavailableError, + AnalysisProtocolError, + PermissionRequiredError, + type AnalysisError, +} from "../domain/errors.js"; +import { err, ok, type Result } from "../domain/result.js"; +import type { ExecutionOptions } from "./AnalysisProvider.js"; +import type { BrowserObservationPort } from "./BrowserObservationPort.js"; +import type { PermissionAuthority } from "./PermissionAuthority.js"; +import { createBrowserEvidence } from "./BrowserEvidence.js"; + +type ScopedBrowserInput = Pick< + ListBrowserTargetsInput, + "cdp_endpoint" | "allowed_origins" +> & { readonly target_id?: string }; + +/** Authorize and execute one policy-scoped target discovery observation. */ +export const listBrowserTargets = async ( + browser: BrowserObservationPort | undefined, + authority: PermissionAuthority | undefined, + input: ListBrowserTargetsInput, + options: ExecutionOptions = {}, +): Promise> => { + const ready = await prepare( + browser, + authority, + input, + "list_browser_targets", + ); + if (!ready.ok) return ready; + const result = await ready.value.listTargets(input, options); + return result.ok + ? ok( + createBrowserEvidence( + "list_browser_targets", + input, + result.value, + ready.value.identity(), + ), + ) + : result; +}; + +/** Authorize and execute one policy-scoped passive page inspection. */ +export const inspectWebPage = async ( + browser: BrowserObservationPort | undefined, + authority: PermissionAuthority | undefined, + input: InspectWebPageInput, + options: ExecutionOptions = {}, +): Promise> => { + const ready = await prepare(browser, authority, input, "inspect_web_page"); + if (!ready.ok) return ready; + const result = await ready.value.inspectPage(input, options); + return result.ok + ? ok( + createBrowserEvidence( + "inspect_web_page", + input, + result.value, + ready.value.identity(), + ), + ) + : result; +}; + +const prepare = async ( + browser: BrowserObservationPort | undefined, + authority: PermissionAuthority | undefined, + input: ScopedBrowserInput, + operation: "list_browser_targets" | "inspect_web_page", +): Promise> => { + if (authority === undefined) + return err( + new AnalysisCapabilityUnavailableError( + "rea-cdp-browser", + operation, + "browser observation permission policy is not configured", + ), + ); + const authorized = await authority.authorize( + { + capability: "browser_observe", + roots: [], + executables: [], + environment_names: [], + origins: [input.cdp_endpoint, ...input.allowed_origins], + network: browserNetworkScope(input.allowed_origins), + mount: false, + operation_identity: `${operation}:${input.target_id ?? input.cdp_endpoint}`, + }, + "read", + ); + if (!authorized.ok) + return err( + authorized.error instanceof PermissionRequiredError + ? authorized.error + : new AnalysisProtocolError(authorized.error.message, { + cause: authorized.error, + }), + ); + return browser === undefined + ? err( + new AnalysisCapabilityUnavailableError( + "rea-cdp-browser", + operation, + "browser observation provider is not configured", + ), + ) + : ok(browser); +}; + +const browserNetworkScope = ( + allowedOrigins: readonly string[], +): "loopback" | "external" => + allowedOrigins.every((origin) => { + const hostname = new URL(origin).hostname; + return hostname === "127.0.0.1" || hostname === "[::1]"; + }) + ? "loopback" + : "external"; diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 46bdbab0b..95e637665 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -54,6 +54,7 @@ export const buildCapabilityInventory = ( policy: { readonly processCaptureEnabled: boolean; readonly evidenceFileRoots: number; + readonly browserObservationEnabled?: boolean; }, ) => { const status = statusSchema.parse(sessionStatus); @@ -97,6 +98,7 @@ const availabilityFor = ( policy: { readonly processCaptureEnabled: boolean; readonly evidenceFileRoots: number; + readonly browserObservationEnabled?: boolean; }, ): { readonly reason: ToolAvailabilityReason; @@ -113,6 +115,14 @@ const availabilityFor = ( reason: "policy_disabled", remediation: "Configure or grant an evidence_read root.", }; + if (kind === "browser-provider") + return policy.browserObservationEnabled === true + ? { reason: "available", remediation: null } + : { + reason: "policy_disabled", + remediation: + "Enable browser observation and configure exact CDP endpoint and page origins.", + }; if (kind === "session") return { reason: "available", remediation: null }; if (!targetOpen) return { diff --git a/src/application/ProjectPermissionStore.ts b/src/application/ProjectPermissionStore.ts index bfed9ad89..91029236a 100644 --- a/src/application/ProjectPermissionStore.ts +++ b/src/application/ProjectPermissionStore.ts @@ -12,6 +12,7 @@ const grantSchema = z.object({ grant_id: z.string().min(1), capability: z.enum([ "process_capture", + "browser_observe", "evidence_read", "evidence_write", "investigation_input", @@ -26,6 +27,7 @@ const grantSchema = z.object({ roots: z.array(z.string()), executables: z.array(z.string()), environment_names: z.array(z.string()), + origins: z.array(z.string()).optional(), network: z.enum(["none", "loopback", "external"]), mount: z.boolean(), lifetime: z.literal("project"), diff --git a/src/browser/CdpBrowserProvider.ts b/src/browser/CdpBrowserProvider.ts new file mode 100644 index 000000000..67914d98b --- /dev/null +++ b/src/browser/CdpBrowserProvider.ts @@ -0,0 +1,229 @@ +import type { BrowserObservationPort } from "../application/BrowserObservationPort.js"; +import type { ProviderIdentity } from "../application/AnalysisProvider.js"; +import type { ExecutionOptions } from "../application/AnalysisProvider.js"; +import { + browserTargetListSchema, + sanitizeBrowserUrl, + webPageInspectionSchema, + type BrowserTargetList, + type InspectWebPageInput, + type ListBrowserTargetsInput, + type WebPageInspection, +} from "../domain/browserObservation.js"; +import { + AnalysisError, + BrowserObservationError, + ProviderAdapterError, +} from "../domain/errors.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { CdpConnection } from "./CdpConnection.js"; +import { + discoverCdpEndpoint, + type CdpEndpointDiscovery, + type CdpEndpointTarget, +} from "./CdpEndpoint.js"; +import { capturePage } from "./CdpPageCapture.js"; + +const IDENTITY: ProviderIdentity = { + id: "rea-cdp-browser", + name: "REA Chrome DevTools Protocol observation provider", + version: "1", +}; + +/** Passive, origin-scoped browser observation through a user-owned CDP endpoint. */ +export class CdpBrowserProvider implements BrowserObservationPort { + identity(): ProviderIdentity { + return IDENTITY; + } + + async listTargets( + input: ListBrowserTargetsInput, + options: ExecutionOptions = {}, + ): Promise> { + try { + const discovery = await discoverCdpEndpoint( + input.cdp_endpoint, + "list_browser_targets", + options.signal, + ); + const allowedOrigins = new Set(input.allowed_origins); + const selected = selectTargets(discovery, allowedOrigins); + const items = selected.allowed.slice( + input.offset, + input.offset + input.limit, + ); + const nextOffset = input.offset + items.length; + return ok( + browserTargetListSchema.parse({ + schema_version: 1, + browser: discovery.version, + targets: { + items, + offset: input.offset, + limit: input.limit, + total: selected.allowed.length, + next_offset: + nextOffset < selected.allowed.length ? nextOffset : null, + has_more: nextOffset < selected.allowed.length, + }, + excluded: selected.excluded, + limitations: [ + "Only page targets whose current URL matches an approved exact origin are listed.", + ], + }), + ); + } catch (cause: unknown) { + return err(providerError(cause, "list_browser_targets")); + } + } + + async inspectPage( + input: InspectWebPageInput, + options: ExecutionOptions = {}, + ): Promise> { + let connection: CdpConnection | undefined; + let sessionId: string | undefined; + try { + const discovery = await discoverCdpEndpoint( + input.cdp_endpoint, + "inspect_web_page", + options.signal, + ); + const target = authorizeTarget(discovery, input); + connection = await CdpConnection.connect( + discovery.browserWebSocketUrl, + options.signal, + ); + const attached = await connection.send( + "Target.attachToTarget", + { targetId: target.id, flatten: true }, + undefined, + options.signal, + ); + sessionId = attachedSessionId(attached); + const result = await capturePage({ + connection, + sessionId, + discovery, + target, + input, + ...(options.signal === undefined ? {} : { signal: options.signal }), + ...(options.progress === undefined + ? {} + : { progress: options.progress }), + }); + return ok(webPageInspectionSchema.parse(result)); + } catch (cause: unknown) { + return err(providerError(cause, "inspect_web_page")); + } finally { + if (connection !== undefined) { + if (sessionId !== undefined) + await bestEffortCleanup(connection, sessionId); + await connection.close(); + } + } + } +} + +const selectTargets = ( + discovery: CdpEndpointDiscovery, + allowedOrigins: ReadonlySet, +): { + readonly allowed: readonly BrowserTargetList["targets"]["items"][number][]; + readonly excluded: BrowserTargetList["excluded"]; +} => { + const allowed = []; + let disallowedOrigin = 0; + let unsupportedUrl = 0; + let nonPage = 0; + for (const target of discovery.targets) { + if (target.type !== "page") { + nonPage += 1; + continue; + } + const url = sanitizeBrowserUrl(target.url); + if (url.origin === null) { + unsupportedUrl += 1; + continue; + } + if (!allowedOrigins.has(url.origin)) { + disallowedOrigin += 1; + continue; + } + allowed.push({ + target_id: target.id, + type: target.type, + title: target.title.slice(0, 16_384), + url: url.url, + origin: url.origin, + attached: target.attached, + }); + } + allowed.sort((left, right) => left.target_id.localeCompare(right.target_id)); + return { + allowed, + excluded: { + disallowed_origin: disallowedOrigin, + unsupported_url: unsupportedUrl, + non_page: nonPage, + }, + }; +}; + +const authorizeTarget = ( + discovery: CdpEndpointDiscovery, + input: InspectWebPageInput, +): CdpEndpointTarget => { + const target = discovery.targets.find( + (candidate) => candidate.id === input.target_id, + ); + if (target === undefined) + throw new BrowserObservationError("inspect_web_page", "target_not_found"); + const origin = sanitizeBrowserUrl(target.url).origin; + if ( + target.type !== "page" || + origin === null || + !input.allowed_origins.includes(origin) + ) + throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); + return target; +}; + +const attachedSessionId = (value: unknown): string => { + if (typeof value !== "object" || value === null || !("sessionId" in value)) + throw new BrowserObservationError("inspect_web_page", "protocol_error"); + if (typeof value.sessionId !== "string" || value.sessionId.length > 256) + throw new BrowserObservationError("inspect_web_page", "protocol_error"); + return value.sessionId; +}; + +const bestEffortCleanup = async ( + connection: CdpConnection, + sessionId: string, +): Promise => { + for (const method of [ + "Network.disable", + "Debugger.disable", + "Runtime.disable", + "Page.disable", + ]) { + try { + await connection.send(method, {}, sessionId); + } catch { + // Cleanup continues so detach is still attempted. + } + } + try { + await connection.send("Target.detachFromTarget", { sessionId }); + } catch { + // Closing REA's socket is the final non-destructive cleanup boundary. + } +}; + +const providerError = ( + cause: unknown, + operation: "list_browser_targets" | "inspect_web_page", +): AnalysisError => + cause instanceof AnalysisError + ? cause + : new ProviderAdapterError(IDENTITY.id, operation, { cause }); diff --git a/src/browser/CdpCaptureDocuments.ts b/src/browser/CdpCaptureDocuments.ts new file mode 100644 index 000000000..b716dbf17 --- /dev/null +++ b/src/browser/CdpCaptureDocuments.ts @@ -0,0 +1,198 @@ +import type { + InspectWebPageInput, + WebPageInspection, +} from "../domain/browserObservation.js"; +import { + allowedSanitizedUrl, + boundedText, + numberValue, + recordValue, + recordsValue, + requiredRecord, + stringValue, + type UnknownRecord, +} from "./CdpCaptureValues.js"; + +/** Normalize allowed frames while retaining a bounded prefix and total count. */ +export const captureFrames = ( + result: unknown, + allowedOrigins: ReadonlySet, + maximum: number, +): { + readonly total: number; + readonly items: WebPageInspection["frames"]; +} => { + const root = recordValue(requiredRecord(result).frameTree); + if (root === undefined) return { total: 0, items: [] }; + const items: WebPageInspection["frames"] = []; + let total = 0; + for (const tree of walkFrameTrees(root)) { + const frame = recordValue(tree.frame); + const frameId = stringValue(frame?.id); + const sanitized = allowedSanitizedUrl(frame?.url, allowedOrigins); + if ( + frame === undefined || + frameId === undefined || + frameId.length > 256 || + sanitized === undefined || + sanitized.origin === null + ) + continue; + total += 1; + if (items.length >= maximum) continue; + items.push({ + frame_id: frameId, + parent_frame_id: + (stringValue(frame.parentId) ?? "").slice(0, 256) || null, + url: sanitized.url, + origin: sanitized.origin, + }); + } + return { total, items }; +}; + +/** Read the current main-frame URL from an untrusted Page.getFrameTree result. */ +export const mainFrameUrl = (result: unknown): string | undefined => + stringValue( + recordValue(recordValue(requiredRecord(result).frameTree)?.frame)?.url, + ); + +/** Normalize allowed resources while retaining a bounded prefix and total count. */ +export const captureResources = ( + result: unknown, + allowedOrigins: ReadonlySet, + maximum: number, +): { + readonly total: number; + readonly items: WebPageInspection["resources"]; +} => { + const root = recordValue(requiredRecord(result).frameTree); + if (root === undefined) return { total: 0, items: [] }; + const items: WebPageInspection["resources"] = []; + let total = 0; + for (const tree of walkFrameTrees(root)) { + for (const resource of recordsValue(tree.resources)) { + const url = allowedSanitizedUrl(resource.url, allowedOrigins); + if (url === undefined || url.origin === null) continue; + total += 1; + if (items.length >= maximum) continue; + const contentSize = numberValue(resource.contentSize); + items.push({ + url: url.url, + origin: url.origin, + type: (stringValue(resource.type) ?? "Other").slice(0, 100), + mime_type: (stringValue(resource.mimeType) ?? "").slice(0, 256), + content_size: + contentSize === undefined ? null : Math.max(0, contentSize), + }); + } + } + return { total, items }; +}; + +const walkFrameTrees = function* ( + root: UnknownRecord, +): Generator { + const pending = [root]; + while (pending.length > 0) { + const tree = pending.pop(); + if (tree === undefined) return; + const children = recordsValue(tree.childFrames); + for (let index = children.length - 1; index >= 0; index -= 1) { + const child = children[index]; + if (child !== undefined) pending.push(child); + } + yield tree; + } +}; + +export const captureDom = ( + result: unknown, + allowedOrigins: ReadonlySet, + input: InspectWebPageInput, +): { + readonly total: number; + readonly nodes: WebPageInspection["dom"]["nodes"]; +} => { + const root = requiredRecord(result); + const strings = Array.isArray(root.strings) + ? root.strings.map((value) => stringValue(value) ?? "") + : []; + const nodes: WebPageInspection["dom"]["nodes"] = []; + let total = 0; + for (const document of recordsValue(root.documents)) { + const documentUrl = indexedString(strings, document.documentURL); + if (allowedSanitizedUrl(documentUrl, allowedOrigins) === undefined) + continue; + const documentNodes = recordValue(document.nodes); + if (documentNodes === undefined) continue; + const nodeTypes = numberArray(documentNodes.nodeType); + const nodeNames = numberArray(documentNodes.nodeName); + const nodeValues = numberArray(documentNodes.nodeValue); + const parents = numberArray(documentNodes.parentIndex); + const attributes = Array.isArray(documentNodes.attributes) + ? documentNodes.attributes + : []; + const baseIndex = nodes.length; + total += nodeTypes.length; + for ( + let index = 0; + index < nodeTypes.length && nodes.length < input.limits.max_dom_nodes; + index += 1 + ) { + const attributeIndexes = numberArray(attributes[index]); + const parent = Math.trunc(parents[index] ?? -1); + nodes.push({ + index: nodes.length, + parent_index: parent < 0 ? -1 : baseIndex + parent, + node_type: Math.max(0, Math.trunc(nodeTypes[index] ?? 0)), + node_name: indexedString(strings, nodeNames[index]).slice(0, 256), + node_value_length: indexedString(strings, nodeValues[index]).length, + attribute_names: attributeIndexes + .filter((_value, attributeIndex) => attributeIndex % 2 === 0) + .map((value) => indexedString(strings, value).slice(0, 256)) + .slice(0, 200), + }); + } + } + return { total, nodes }; +}; + +export const captureAccessibility = ( + results: readonly unknown[], + maximum: number, +): { + readonly total: number; + readonly nodes: WebPageInspection["accessibility"]["nodes"]; +} => { + const all = results.flatMap((result) => + recordsValue(requiredRecord(result).nodes), + ); + return { + total: all.length, + nodes: all.slice(0, maximum).map((node) => ({ + node_id: (stringValue(node.nodeId) ?? "").slice(0, 256), + parent_id: (stringValue(node.parentId) ?? "").slice(0, 256) || null, + role: axText(node.role), + name: axText(node.name), + description: axText(node.description), + ignored: node.ignored === true, + })), + }; +}; + +const axText = (value: unknown): string | null => + boundedText(recordValue(value)?.value, 1_024); + +const indexedString = (strings: readonly string[], index: unknown): string => { + const integer = numberValue(index); + return integer === undefined ? "" : (strings[Math.trunc(integer)] ?? ""); +}; + +const numberArray = (value: unknown): readonly number[] => + Array.isArray(value) + ? value.flatMap((item) => { + const number = numberValue(item); + return number === undefined ? [] : [number]; + }) + : []; diff --git a/src/browser/CdpCaptureEvents.ts b/src/browser/CdpCaptureEvents.ts new file mode 100644 index 000000000..83da575d8 --- /dev/null +++ b/src/browser/CdpCaptureEvents.ts @@ -0,0 +1,300 @@ +import type { + InspectWebPageInput, + WebPageInspection, +} from "../domain/browserObservation.js"; +import type { CdpEvent } from "./CdpConnection.js"; +import { + allowedSanitizedUrl, + boundedText, + isHttpUrl, + numberValue, + recordValue, + recordsValue, + stringValue, + type UnknownRecord, +} from "./CdpCaptureValues.js"; + +export interface CapturedScript { + readonly scriptId: string; + readonly url: string; + readonly origin: string | null; + readonly hash: string; + readonly length: number; + readonly isModule: boolean; + readonly language: string | null; + readonly sourceMapUrl: string | null; +} + +type NetworkState = WebPageInspection["network"]["requests"][number]; + +/** Bounded event accumulator that drops payload values at ingestion time. */ +export class CdpCaptureEvents { + readonly scripts = new Map(); + readonly network = new Map(); + readonly allowedWebSockets = new Set(); + readonly console: WebPageInspection["console"]["events"] = []; + readonly websockets: WebPageInspection["network"]["websocket_events"] = []; + dropped = 0; + originViolation = false; + navigationDuringCapture = false; + #mainFrameId: string | undefined; + + constructor( + private readonly input: InspectWebPageInput, + private readonly allowedOrigins: ReadonlySet, + ) {} + + beginAuthorizedFrame(frameId: string): void { + this.#mainFrameId = frameId; + this.originViolation = false; + this.navigationDuringCapture = false; + this.dropped = 0; + this.scripts.clear(); + this.network.clear(); + this.allowedWebSockets.clear(); + this.console.length = 0; + this.websockets.length = 0; + } + + beginFinalCapture(frameId: string): void { + this.#mainFrameId = frameId; + this.navigationDuringCapture = false; + } + + ingest(event: CdpEvent): void { + const params = recordValue(event.params); + if (params === undefined) return; + switch (event.method) { + case "Debugger.scriptParsed": + this.#script(params); + break; + case "Network.requestWillBeSent": + this.#request(params); + break; + case "Network.responseReceived": + this.#response(params); + break; + case "Network.loadingFinished": + this.#finished(params); + break; + case "Runtime.consoleAPICalled": + this.#console(params); + break; + case "Page.frameNavigated": + this.#frameNavigated(params); + break; + case "Network.webSocketCreated": + this.#websocketCreated(params); + break; + case "Network.webSocketFrameSent": + this.#websocket(params, "sent"); + break; + case "Network.webSocketFrameReceived": + this.#websocket(params, "received"); + break; + } + } + + #script(params: UnknownRecord): void { + const scriptId = stringValue(params.scriptId); + const rawUrl = stringValue(params.url) ?? ""; + const sanitized = allowedSanitizedUrl(rawUrl, this.allowedOrigins); + if ( + scriptId === undefined || + scriptId.length > 256 || + sanitized === undefined + ) + return; + if ( + this.scripts.size >= this.input.limits.max_scripts && + !this.scripts.has(scriptId) + ) { + this.dropped += 1; + return; + } + this.scripts.set(scriptId, { + scriptId, + url: sanitized.url, + origin: sanitized.origin, + hash: (stringValue(params.hash) ?? "").slice(0, 512), + length: Math.max( + 0, + Math.min( + Number.MAX_SAFE_INTEGER, + Math.trunc(numberValue(params.length) ?? 0), + ), + ), + isModule: params.isModule === true, + language: boundedText(params.scriptLanguage, 100), + sourceMapUrl: this.#sourceMap(params.sourceMapURL), + }); + } + + #sourceMap(value: unknown): string | null { + const sanitized = allowedSanitizedUrl(value, this.allowedOrigins); + return sanitized?.url ?? null; + } + + #request(params: UnknownRecord): void { + const requestId = stringValue(params.requestId); + const request = recordValue(params.request); + const sanitized = allowedSanitizedUrl(request?.url, this.allowedOrigins); + if ( + requestId === undefined || + requestId.length > 256 || + request === undefined || + sanitized === undefined + ) + return; + if ( + this.network.size >= this.input.limits.max_network_events && + !this.network.has(requestId) + ) { + this.dropped += 1; + return; + } + const initiator = recordValue(params.initiator); + const initiatorUrl = allowedSanitizedUrl( + firstCallFrame(initiator)?.url, + this.allowedOrigins, + ); + this.network.set(requestId, { + request_id: requestId, + url: sanitized.url, + origin: sanitized.origin ?? "", + method: (stringValue(request.method) ?? "GET").slice(0, 32), + resource_type: boundedText(params.type, 100), + status: null, + mime_type: null, + encoded_data_length: null, + initiator: { + type: (stringValue(initiator?.type) ?? "other").slice(0, 100), + url: initiatorUrl?.url ?? null, + line: integerOrNull(firstCallFrame(initiator)?.lineNumber), + column: integerOrNull(firstCallFrame(initiator)?.columnNumber), + }, + }); + } + + #response(params: UnknownRecord): void { + const requestId = stringValue(params.requestId); + if (requestId === undefined || requestId.length > 256) return; + const current = this.network.get(requestId); + const response = recordValue(params.response); + if (current === undefined || response === undefined) return; + this.network.set(requestId, { + ...current, + status: numberValue(response.status) ?? null, + mime_type: boundedText(response.mimeType, 256), + }); + } + + #finished(params: UnknownRecord): void { + const requestId = stringValue(params.requestId); + if (requestId === undefined || requestId.length > 256) return; + const current = this.network.get(requestId); + if (current === undefined) return; + this.network.set(requestId, { + ...current, + encoded_data_length: Math.max( + 0, + numberValue(params.encodedDataLength) ?? 0, + ), + }); + } + + #console(params: UnknownRecord): void { + const frame = firstCallFrame(recordValue(params.stackTrace)); + const source = allowedSanitizedUrl(frame?.url, this.allowedOrigins); + if (frame === undefined || source === undefined) return; + if (this.console.length >= this.input.limits.max_console_events) { + this.dropped += 1; + return; + } + this.console.push({ + type: (stringValue(params.type) ?? "unknown").slice(0, 100), + timestamp: numberValue(params.timestamp) ?? 0, + argument_types: recordsValue(params.args) + .map((argument) => + (stringValue(argument.type) ?? "unknown").slice(0, 100), + ) + .slice(0, 100), + url: source?.url ?? null, + line: integerOrNull(frame?.lineNumber), + column: integerOrNull(frame?.columnNumber), + }); + } + + #websocket(params: UnknownRecord, direction: "sent" | "received"): void { + const requestId = stringValue(params.requestId); + if ( + requestId === undefined || + requestId.length > 256 || + (!this.network.has(requestId) && !this.allowedWebSockets.has(requestId)) + ) + return; + if (this.websockets.length >= this.input.limits.max_websocket_events) { + this.dropped += 1; + return; + } + const response = recordValue(params.response); + const payload = stringValue(response?.payloadData) ?? ""; + this.websockets.push({ + request_id: requestId, + direction, + opcode: Math.max(0, Math.trunc(numberValue(response?.opcode) ?? 0)), + payload_bytes: Buffer.byteLength(payload), + }); + } + + #websocketCreated(params: UnknownRecord): void { + const requestId = stringValue(params.requestId); + const rawUrl = stringValue(params.url); + if ( + requestId === undefined || + requestId.length > 256 || + rawUrl === undefined + ) + return; + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + return; + } + if (parsed.protocol === "ws:") parsed.protocol = "http:"; + else if (parsed.protocol === "wss:") parsed.protocol = "https:"; + else return; + if ( + this.allowedOrigins.has(parsed.origin) && + (this.allowedWebSockets.has(requestId) || + this.allowedWebSockets.size < this.input.limits.max_websocket_events) + ) + this.allowedWebSockets.add(requestId); + else if (this.allowedOrigins.has(parsed.origin)) this.dropped += 1; + } + + #frameNavigated(params: UnknownRecord): void { + const frame = recordValue(params.frame); + if (stringValue(frame?.id) !== this.#mainFrameId) return; + this.navigationDuringCapture = true; + const rawUrl = stringValue(frame?.url); + if ( + isHttpUrl(rawUrl) && + allowedSanitizedUrl(rawUrl, this.allowedOrigins) === undefined + ) + this.originViolation = true; + } +} + +const firstCallFrame = ( + value: UnknownRecord | undefined, +): UnknownRecord | undefined => { + const callFrames = recordsValue(value?.callFrames); + return callFrames[0]; +}; + +const integerOrNull = (value: unknown): number | null => { + const number = numberValue(value); + return number === undefined ? null : Math.max(0, Math.trunc(number)); +}; diff --git a/src/browser/CdpCaptureStorage.ts b/src/browser/CdpCaptureStorage.ts new file mode 100644 index 000000000..21f6b32f5 --- /dev/null +++ b/src/browser/CdpCaptureStorage.ts @@ -0,0 +1,116 @@ +import type { + InspectWebPageInput, + WebPageInspection, +} from "../domain/browserObservation.js"; +import { CdpConnection } from "./CdpConnection.js"; +import { optionalCdpCommand } from "./CdpOptionalCommand.js"; +import { + numberValue, + recordValue, + recordsValue, + stringValue, +} from "./CdpCaptureValues.js"; + +/** Capture redacted, bounded storage metadata for one authorized origin. */ +export const captureStorage = async ( + context: { + readonly connection: CdpConnection; + readonly sessionId: string; + readonly input: InspectWebPageInput; + readonly signal?: AbortSignal; + }, + origin: string, + limitations: string[], +): Promise<{ + readonly value: WebPageInspection["storage"]; + readonly truncated: boolean; +}> => { + const quota = recordValue( + await optionalCdpCommand( + context, + "Storage.getUsageAndQuota", + { origin }, + limitations, + ), + ); + const local = context.input.include_storage_keys + ? await storageKeys(context, origin, true, limitations) + : []; + const session = context.input.include_storage_keys + ? await storageKeys(context, origin, false, limitations) + : []; + const indexed = context.input.include_storage_keys + ? stringArray( + recordValue( + await optionalCdpCommand( + context, + "IndexedDB.requestDatabaseNames", + { securityOrigin: origin }, + limitations, + ), + )?.databaseNames, + ) + : []; + const caches = context.input.include_storage_keys + ? recordsValue( + recordValue( + await optionalCdpCommand( + context, + "CacheStorage.requestCacheNames", + { securityOrigin: origin }, + limitations, + ), + )?.caches, + ).flatMap((cache) => { + const name = stringValue(cache.cacheName); + return name === undefined ? [] : [name.slice(0, 1_024)]; + }) + : []; + const maximum = context.input.limits.max_storage_keys; + return { + value: { + origin, + usage_bytes: numberValue(quota?.usage) ?? null, + quota_bytes: numberValue(quota?.quota) ?? null, + local_storage_keys: local.slice(0, maximum), + session_storage_keys: session.slice(0, maximum), + indexed_db_names: indexed.slice(0, maximum), + cache_names: caches.slice(0, maximum), + values_redacted: true, + }, + truncated: [local, session, indexed, caches].some( + (items) => items.length > maximum, + ), + }; +}; + +const storageKeys = async ( + context: Parameters[0], + origin: string, + isLocalStorage: boolean, + limitations: string[], +): Promise => { + const result = recordValue( + await optionalCdpCommand( + context, + "DOMStorage.getDOMStorageItems", + { storageId: { securityOrigin: origin, isLocalStorage } }, + limitations, + ), + ); + return Array.isArray(result?.entries) + ? result.entries.flatMap((entry) => { + if (!Array.isArray(entry)) return []; + const key = stringValue(entry[0]); + return key === undefined ? [] : [key.slice(0, 1_024)]; + }) + : []; +}; + +const stringArray = (value: unknown): readonly string[] => + Array.isArray(value) + ? value.flatMap((item) => { + const text = stringValue(item); + return text === undefined ? [] : [text.slice(0, 1_024)]; + }) + : []; diff --git a/src/browser/CdpCaptureValues.ts b/src/browser/CdpCaptureValues.ts new file mode 100644 index 000000000..439786cff --- /dev/null +++ b/src/browser/CdpCaptureValues.ts @@ -0,0 +1,98 @@ +import { createHash } from "node:crypto"; + +import { + AnalysisCancelledError, + BrowserObservationError, +} from "../domain/errors.js"; +import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; + +export type UnknownRecord = Readonly>; + +const isRecord = (value: unknown): value is UnknownRecord => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const recordValue = (value: unknown): UnknownRecord | undefined => + isRecord(value) ? value : undefined; + +export const recordsValue = (value: unknown): readonly UnknownRecord[] => + Array.isArray(value) + ? value.flatMap((item) => { + const record = recordValue(item); + return record === undefined ? [] : [record]; + }) + : []; + +export const stringValue = (value: unknown): string | undefined => + typeof value === "string" ? value : undefined; + +export const numberValue = (value: unknown): number | undefined => + typeof value === "number" && Number.isFinite(value) ? value : undefined; + +export const requiredRecord = (value: unknown): UnknownRecord => { + const record = recordValue(value); + if (record === undefined) + throw new BrowserObservationError("inspect_web_page", "protocol_error"); + return record; +}; + +export const boundedText = (value: unknown, maximum = 1_024): string | null => { + const text = stringValue(value); + return text === undefined ? null : text.slice(0, maximum); +}; + +export const isHttpUrl = (value: string | undefined): boolean => { + if (value === undefined) return false; + try { + const protocol = new URL(value).protocol; + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +}; + +export const allowedSanitizedUrl = ( + value: unknown, + allowedOrigins: ReadonlySet, +): ReturnType | undefined => { + const text = stringValue(value); + if (text === undefined) return undefined; + const sanitized = sanitizeBrowserUrl(text); + return sanitized.origin !== null && allowedOrigins.has(sanitized.origin) + ? sanitized + : undefined; +}; + +export const sourceResult = ( + source: string, +): { included: true; sha256: string; bytes: number; content: string } => ({ + included: true, + sha256: createHash("sha256").update(source).digest("hex"), + bytes: Buffer.byteLength(source), + content: source, +}); + +export const sourceExcluded = ( + reason: string, +): { included: false; reason: string } => ({ included: false, reason }); + +export const delayWithCancellation = async ( + durationMs: number, + signal?: AbortSignal, +): Promise => { + if (durationMs === 0) return; + if (signal?.aborted === true) + throw new AnalysisCancelledError("inspect_web_page"); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, durationMs); + const onAbort = (): void => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + reject(new AnalysisCancelledError("inspect_web_page")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted === true) onAbort(); + }); +}; diff --git a/src/browser/CdpConnection.ts b/src/browser/CdpConnection.ts new file mode 100644 index 000000000..280134ee5 --- /dev/null +++ b/src/browser/CdpConnection.ts @@ -0,0 +1,252 @@ +import WebSocket, { type RawData } from "ws"; + +import { + AnalysisCancelledError, + AnalysisError, + AnalysisTimeoutError, + BrowserObservationError, +} from "../domain/errors.js"; + +export interface CdpEvent { + readonly method: string; + readonly params: unknown; + readonly sessionId?: string; +} + +interface PendingCommand { + readonly resolve: (value: unknown) => void; + readonly reject: (error: AnalysisError) => void; + readonly timer: ReturnType; + readonly removeAbort: () => void; +} + +const CONNECT_TIMEOUT_MS = 5_000; +const COMMAND_TIMEOUT_MS = 5_000; +const MAX_PAYLOAD_BYTES = 16 * 1_024 * 1_024; +const MAX_PENDING_COMMANDS = 128; + +/** Correlated, bounded JSON-RPC transport for a browser-level CDP socket. */ +export class CdpConnection { + readonly #pending = new Map(); + readonly #listeners = new Set<(event: CdpEvent) => void>(); + #nextId = 1; + #closed = false; + #protocolFailed = false; + + private constructor(private readonly socket: WebSocket) { + socket.on("message", (data) => this.#receive(data)); + socket.on("close", () => this.#failPending("disconnected")); + socket.on("error", () => this.#failPending("disconnected")); + } + + /** Connect to one already-validated loopback browser WebSocket. */ + static async connect( + url: string, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted === true) + throw new AnalysisCancelledError("inspect_web_page"); + const socket = new WebSocket(url, { + handshakeTimeout: CONNECT_TIMEOUT_MS, + maxPayload: MAX_PAYLOAD_BYTES, + perMessageDeflate: false, + }); + await waitForOpen(socket, signal); + return new CdpConnection(socket); + } + + /** Subscribe to validated CDP event envelopes. */ + onEvent(listener: (event: CdpEvent) => void): () => void { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + + /** Execute one bounded command, optionally within a flat target session. */ + async send( + method: string, + params: Readonly> = {}, + sessionId?: string, + signal?: AbortSignal, + ): Promise { + if (this.#protocolFailed) + throw new BrowserObservationError("inspect_web_page", "protocol_error"); + if (this.#closed || this.socket.readyState !== WebSocket.OPEN) + throw new BrowserObservationError("inspect_web_page", "disconnected"); + if (this.#pending.size >= MAX_PENDING_COMMANDS) + throw new BrowserObservationError("inspect_web_page", "payload_limit"); + if (signal?.aborted === true) + throw new AnalysisCancelledError("inspect_web_page"); + const id = this.#nextId; + this.#nextId += 1; + return await new Promise((resolve, reject) => { + const onAbort = (): void => { + const pending = this.#pending.get(id); + if (pending === undefined) return; + clearTimeout(pending.timer); + pending.removeAbort(); + this.#pending.delete(id); + reject(new AnalysisCancelledError("inspect_web_page")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + this.#pending.delete(id); + reject( + new AnalysisTimeoutError("inspect_web_page", COMMAND_TIMEOUT_MS), + ); + }, COMMAND_TIMEOUT_MS); + this.#pending.set(id, { + resolve, + reject, + timer, + removeAbort: () => signal?.removeEventListener("abort", onAbort), + }); + this.socket.send( + JSON.stringify({ + id, + method, + params, + ...(sessionId ? { sessionId } : {}), + }), + (error) => { + if (error === undefined || error === null) return; + const pending = this.#pending.get(id); + if (pending === undefined) return; + this.#complete(id, pending); + reject( + new BrowserObservationError("inspect_web_page", "disconnected", { + cause: error, + }), + ); + }, + ); + }); + } + + /** Close only REA's socket; never close the browser or selected page. */ + async close(): Promise { + if (this.#closed) return; + this.#closed = true; + this.#failPending("disconnected"); + if (this.socket.readyState === WebSocket.CLOSED) return; + await new Promise((resolve) => { + const timer = setTimeout(() => { + this.socket.terminate(); + resolve(); + }, 1_000); + this.socket.once("close", () => { + clearTimeout(timer); + resolve(); + }); + this.socket.close(); + }); + } + + #receive(data: RawData): void { + let message: unknown; + try { + message = JSON.parse(rawText(data)); + } catch { + this.#failPending("protocol_error"); + return; + } + if (!isRecord(message)) { + this.#failPending("protocol_error"); + return; + } + if ("id" in message && typeof message.id === "number") { + this.#receiveResponse(message, message.id); + return; + } + if ("method" in message && typeof message.method === "string") { + const event: CdpEvent = { + method: message.method, + params: "params" in message ? message.params : {}, + ...(typeof message.sessionId === "string" + ? { sessionId: message.sessionId } + : {}), + }; + for (const listener of this.#listeners) listener(event); + } + } + + #receiveResponse(message: Record, id: number): void { + const pending = this.#pending.get(id); + if (pending === undefined) return; + this.#complete(id, pending); + if ("error" in message) { + pending.reject( + new BrowserObservationError("inspect_web_page", "protocol_error"), + ); + return; + } + pending.resolve("result" in message ? message.result : {}); + } + + #complete(id: number, pending: PendingCommand): void { + clearTimeout(pending.timer); + pending.removeAbort(); + this.#pending.delete(id); + } + + #failPending(reason: "disconnected" | "protocol_error"): void { + if (reason === "disconnected") this.#closed = true; + else this.#protocolFailed = true; + for (const [id, pending] of this.#pending) { + this.#complete(id, pending); + pending.reject(new BrowserObservationError("inspect_web_page", reason)); + } + } +} + +const rawText = (data: RawData): string => + Array.isArray(data) + ? Buffer.concat(data).toString("utf8") + : Buffer.isBuffer(data) + ? data.toString("utf8") + : Buffer.from(data).toString("utf8"); + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const waitForOpen = async ( + socket: WebSocket, + signal?: AbortSignal, +): Promise => + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + socket.terminate(); + reject(new AnalysisTimeoutError("inspect_web_page", CONNECT_TIMEOUT_MS)); + }, CONNECT_TIMEOUT_MS); + const onOpen = (): void => { + cleanup(); + resolve(); + }; + const onFailure = (cause: unknown): void => { + cleanup(); + reject( + signal?.aborted === true + ? new AnalysisCancelledError("inspect_web_page") + : new BrowserObservationError( + "inspect_web_page", + "endpoint_unreachable", + { cause }, + ), + ); + }; + const onAbort = (): void => { + socket.terminate(); + onFailure(new AnalysisCancelledError("inspect_web_page")); + }; + const cleanup = (): void => { + clearTimeout(timer); + socket.off("open", onOpen); + socket.off("error", onFailure); + signal?.removeEventListener("abort", onAbort); + }; + socket.once("open", onOpen); + socket.once("error", onFailure); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted === true) onAbort(); + }); diff --git a/src/browser/CdpEndpoint.ts b/src/browser/CdpEndpoint.ts new file mode 100644 index 000000000..d63a5e32b --- /dev/null +++ b/src/browser/CdpEndpoint.ts @@ -0,0 +1,207 @@ +import { request } from "node:http"; + +import { z } from "zod"; + +import { + AnalysisCancelledError, + AnalysisError, + AnalysisTimeoutError, + BrowserObservationError, +} from "../domain/errors.js"; + +const HTTP_TIMEOUT_MS = 5_000; +const MAX_VERSION_BYTES = 64 * 1_024; +const MAX_TARGET_LIST_BYTES = 2 * 1_024 * 1_024; + +const endpointVersionSchema = z.object({ + Browser: z.string().min(1).max(1_024), + "Protocol-Version": z.string().min(1).max(100), + "User-Agent": z.string().max(16_384), + "V8-Version": z.string().max(1_024), + "WebKit-Version": z.string().max(1_024), + webSocketDebuggerUrl: z.string().min(1).max(2_048), +}); + +const endpointTargetSchema = z.object({ + id: z.string().min(1).max(256), + type: z.string().min(1).max(100), + title: z.string().max(16_384).default(""), + url: z.string().max(65_536), + attached: z.boolean().default(false), +}); +const endpointTargetsSchema = z.array(endpointTargetSchema).max(1_000); + +export type CdpEndpointTarget = z.infer; + +export interface CdpEndpointDiscovery { + readonly browserWebSocketUrl: string; + readonly version: { + readonly product: string; + readonly protocol_version: string; + readonly revision: string; + readonly user_agent: string; + readonly js_version: string; + }; + readonly targets: readonly CdpEndpointTarget[]; +} + +type BrowserOperation = "list_browser_targets" | "inspect_web_page"; + +/** Read bounded CDP discovery endpoints without following redirects. */ +export const discoverCdpEndpoint = async ( + endpoint: string, + operation: BrowserOperation, + signal?: AbortSignal, +): Promise => { + const versionInput = await readJson( + new URL("/json/version", endpoint), + MAX_VERSION_BYTES, + operation, + signal, + ); + const targetsInput = await readJson( + new URL("/json/list", endpoint), + MAX_TARGET_LIST_BYTES, + operation, + signal, + ); + const version = parseEndpointValue( + endpointVersionSchema, + versionInput, + operation, + ); + const targets = parseEndpointValue( + endpointTargetsSchema, + targetsInput, + operation, + ); + return { + browserWebSocketUrl: safeBrowserWebSocketUrl( + endpoint, + version.webSocketDebuggerUrl, + operation, + ), + version: { + product: version.Browser, + protocol_version: version["Protocol-Version"], + revision: version["WebKit-Version"], + user_agent: version["User-Agent"], + js_version: version["V8-Version"], + }, + targets, + }; +}; + +const parseEndpointValue = ( + schema: z.ZodType, + input: unknown, + operation: BrowserOperation, +): Output => { + const parsed = schema.safeParse(input); + if (!parsed.success) + throw new BrowserObservationError(operation, "invalid_endpoint_response", { + cause: parsed.error, + }); + return parsed.data; +}; + +const safeBrowserWebSocketUrl = ( + endpoint: string, + reported: string, + operation: BrowserOperation, +): string => { + let parsed: URL; + try { + parsed = new URL(reported); + } catch (cause: unknown) { + throw new BrowserObservationError(operation, "invalid_endpoint_response", { + cause, + }); + } + const trustedEndpoint = new URL(endpoint); + if ( + parsed.protocol !== "ws:" || + parsed.port !== trustedEndpoint.port || + parsed.username !== "" || + parsed.password !== "" || + parsed.search !== "" || + parsed.hash !== "" || + !parsed.pathname.startsWith("/devtools/browser/") + ) + throw new BrowserObservationError(operation, "invalid_endpoint_response"); + parsed.hostname = trustedEndpoint.hostname; + return parsed.href; +}; + +const readJson = async ( + url: URL, + maximumBytes: number, + operation: BrowserOperation, + signal?: AbortSignal, +): Promise => + await new Promise((resolve, reject) => { + if (signal?.aborted === true) { + reject(new AnalysisCancelledError(operation)); + return; + } + const request_ = request( + url, + { method: "GET", signal, headers: { Accept: "application/json" } }, + (response) => { + if (response.statusCode !== 200) { + response.resume(); + reject( + new BrowserObservationError(operation, "invalid_endpoint_response"), + ); + return; + } + const chunks: Buffer[] = []; + let bytes = 0; + response.on("data", (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > maximumBytes) { + response.destroy( + new BrowserObservationError(operation, "payload_limit"), + ); + return; + } + chunks.push(chunk); + }); + response.on("end", () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + } catch (cause: unknown) { + reject( + new BrowserObservationError( + operation, + "invalid_endpoint_response", + { cause }, + ), + ); + } + }); + response.on("error", (cause: unknown) => + reject(endpointFailure(cause, operation, signal)), + ); + }, + ); + request_.setTimeout(HTTP_TIMEOUT_MS, () => + request_.destroy(new AnalysisTimeoutError(operation, HTTP_TIMEOUT_MS)), + ); + request_.on("error", (cause: unknown) => + reject(endpointFailure(cause, operation, signal)), + ); + request_.end(); + }); + +const endpointFailure = ( + cause: unknown, + operation: BrowserOperation, + signal?: AbortSignal, +): AnalysisError => { + if (cause instanceof AnalysisError) return cause; + if (signal?.aborted === true) return new AnalysisCancelledError(operation); + return new BrowserObservationError(operation, "endpoint_unreachable", { + cause, + }); +}; diff --git a/src/browser/CdpOptionalCommand.ts b/src/browser/CdpOptionalCommand.ts new file mode 100644 index 000000000..240c3d338 --- /dev/null +++ b/src/browser/CdpOptionalCommand.ts @@ -0,0 +1,31 @@ +import { BrowserObservationError } from "../domain/errors.js"; +import { CdpConnection } from "./CdpConnection.js"; + +/** Execute an optional CDP method while preserving transport and cancellation failures. */ +export const optionalCdpCommand = async ( + context: { + readonly connection: CdpConnection; + readonly sessionId: string | undefined; + readonly signal?: AbortSignal; + }, + method: string, + parameters: Readonly>, + limitations: string[], +): Promise => { + try { + return await context.connection.send( + method, + parameters, + context.sessionId, + context.signal, + ); + } catch (cause: unknown) { + if ( + !(cause instanceof BrowserObservationError) || + cause.reason !== "protocol_error" + ) + throw cause; + limitations.push(`${method} was unavailable from this browser target.`); + return undefined; + } +}; diff --git a/src/browser/CdpPageCapture.ts b/src/browser/CdpPageCapture.ts new file mode 100644 index 000000000..80b396e20 --- /dev/null +++ b/src/browser/CdpPageCapture.ts @@ -0,0 +1,446 @@ +import type { + InspectWebPageInput, + WebPageInspection, +} from "../domain/browserObservation.js"; +import type { ProgressReporter } from "../application/ProgressReporter.js"; +import { BrowserObservationError } from "../domain/errors.js"; +import type { CdpEndpointDiscovery, CdpEndpointTarget } from "./CdpEndpoint.js"; +import { CdpConnection } from "./CdpConnection.js"; +import { CdpCaptureEvents } from "./CdpCaptureEvents.js"; +import { captureStorage } from "./CdpCaptureStorage.js"; +import { optionalCdpCommand } from "./CdpOptionalCommand.js"; +import { + captureAccessibility, + captureDom, + captureFrames, + captureResources, + mainFrameUrl, +} from "./CdpCaptureDocuments.js"; +import { + allowedSanitizedUrl, + delayWithCancellation, + isHttpUrl, + recordValue, + recordsValue, + requiredRecord, + sourceExcluded, + sourceResult, + stringValue, +} from "./CdpCaptureValues.js"; + +interface CaptureContext { + readonly connection: CdpConnection; + readonly sessionId: string; + readonly discovery: CdpEndpointDiscovery; + readonly target: CdpEndpointTarget; + readonly input: InspectWebPageInput; + readonly signal?: AbortSignal; + readonly progress?: ProgressReporter; +} + +interface CaptureState { + readonly context: CaptureContext; + readonly events: CdpCaptureEvents; + readonly allowedOrigins: ReadonlySet; + readonly limitations: string[]; + readonly truncatedSections: Set; + readonly startedAt: string; +} + +interface CapturedSections { + readonly attachedUrl: string; + readonly frames: WebPageInspection["frames"]; + readonly dom: ReturnType; + readonly accessibility: ReturnType; + readonly scripts: WebPageInspection["scripts"]; + readonly resources: WebPageInspection["resources"]; + readonly workers: WebPageInspection["workers"]; + readonly storage: WebPageInspection["storage"]; +} + +/** Capture and normalize one attached page without evaluating page JavaScript. */ +export const capturePage = async ( + context: CaptureContext, +): Promise => { + const allowedOrigins = new Set(context.input.allowed_origins); + const events = new CdpCaptureEvents(context.input, allowedOrigins); + const removeListener = context.connection.onEvent((event) => + events.ingest(event), + ); + const state: CaptureState = { + context, + events, + allowedOrigins, + limitations: [ + "Observation starts when REA attaches; prior network and console activity is unavailable.", + "Network headers, request bodies, response bodies, cookies, storage values, console values, and WebSocket payloads are not retained.", + "Source maps are reported only as declarative URLs and are not fetched.", + "URL-less scripts and console events without an allowed source URL are excluded because their origin cannot be proven.", + ], + truncatedSections: new Set(), + startedAt: new Date().toISOString(), + }; + try { + return await captureAuthorizedPage(state); + } finally { + removeListener(); + } +}; + +const captureAuthorizedPage = async ( + state: CaptureState, +): Promise => { + const { context, allowedOrigins, limitations, truncatedSections } = state; + const { connection, sessionId, input, signal } = context; + await authorizeObservationWindow(state); + const frameResult = await authorizedFrameTree(context, allowedOrigins); + const attachedUrl = mainFrameUrl(frameResult) ?? ""; + const frameCapture = captureFrames( + frameResult, + allowedOrigins, + input.limits.max_frames, + ); + const frames = frameCapture.items; + if (frameCapture.total > frames.length) truncatedSections.add("frames"); + const captureFrame = frames[0]; + if (captureFrame === undefined) + throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); + state.events.beginFinalCapture(captureFrame.frame_id); + const resourceCapture = captureResources( + await connection.send("Page.getResourceTree", {}, sessionId, signal), + allowedOrigins, + input.limits.max_resources, + ); + if (resourceCapture.total > resourceCapture.items.length) + truncatedSections.add("resources"); + const dom = await capturePageDom(state); + if (dom.total > dom.nodes.length) truncatedSections.add("dom"); + const accessibility = await accessibilityForFrames( + context, + frames, + limitations, + ); + if (accessibility.total > accessibility.nodes.length) + truncatedSections.add("accessibility"); + const scripts = await captureScripts( + context, + state.events, + truncatedSections, + ); + const workerCapture = await captureWorkers( + context, + allowedOrigins, + limitations, + ); + if (workerCapture.total > workerCapture.items.length) + truncatedSections.add("workers"); + const storageCapture = await captureStorage( + context, + frames[0]?.origin ?? new URL(attachedUrl).origin, + limitations, + ); + if (storageCapture.truncated) truncatedSections.add("storage_keys"); + if (state.events.dropped > 0) truncatedSections.add("events"); + const completedFrameResult = await authorizedFrameTree( + context, + allowedOrigins, + ); + const completedUrl = mainFrameUrl(completedFrameResult) ?? ""; + if (state.events.originViolation) + throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); + if (state.events.navigationDuringCapture || completedUrl !== attachedUrl) + throw new BrowserObservationError("inspect_web_page", "target_changed"); + await report(context.progress, 3, "Normalizing browser evidence"); + return normalizedInspection(state, { + attachedUrl, + frames, + dom, + accessibility, + scripts, + resources: resourceCapture.items, + workers: workerCapture.items, + storage: storageCapture.value, + }); +}; + +const authorizeObservationWindow = async ( + state: CaptureState, +): Promise => { + const { context, allowedOrigins, events } = state; + const { connection, sessionId, signal } = context; + await report(context.progress, 1, "Enabling passive CDP domains"); + await connection.send("Page.enable", {}, sessionId, signal); + const initialFrameResult = await authorizedFrameTree(context, allowedOrigins); + const mainFrame = captureFrames(initialFrameResult, allowedOrigins, 1) + .items[0]; + if (mainFrame === undefined) + throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); + events.beginAuthorizedFrame(mainFrame.frame_id); + await enableObservationDomains(connection, sessionId, signal); + await report(context.progress, 2, "Observing page events"); + await delayWithCancellation(context.input.observation_ms, signal); + if (events.originViolation) + throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); +}; + +const capturePageDom = async ( + state: CaptureState, +): Promise> => { + const { context, allowedOrigins } = state; + const result = await context.connection.send( + "DOMSnapshot.captureSnapshot", + { + computedStyles: [], + includePaintOrder: false, + includeDOMRects: false, + }, + context.sessionId, + context.signal, + ); + return captureDom(result, allowedOrigins, context.input); +}; + +const normalizedInspection = ( + state: CaptureState, + captured: CapturedSections, +): WebPageInspection => ({ + schema_version: 1, + browser: state.context.discovery.version, + target: normalizedTarget( + state.context.target, + captured.attachedUrl, + state.allowedOrigins, + ), + capture_window: { + started_at: state.startedAt, + ended_at: new Date().toISOString(), + observation_ms: state.context.input.observation_ms, + }, + completeness: { + status: state.truncatedSections.size === 0 ? "complete" : "truncated", + truncated_sections: [...state.truncatedSections].sort(), + dropped_events: state.events.dropped, + }, + frames: [...captured.frames], + dom: { total_nodes: captured.dom.total, nodes: [...captured.dom.nodes] }, + accessibility: { + total_nodes: captured.accessibility.total, + nodes: [...captured.accessibility.nodes], + }, + scripts: captured.scripts, + resources: [...captured.resources], + network: { + requests: [...state.events.network.values()], + websocket_events: [...state.events.websockets], + coverage_started_at: state.startedAt, + prior_activity_available: false, + }, + console: { + events: [...state.events.console], + coverage_started_at: state.startedAt, + prior_activity_available: false, + }, + workers: [...captured.workers], + storage: captured.storage, + limitations: state.limitations, +}); + +const authorizedFrameTree = async ( + context: CaptureContext, + allowedOrigins: ReadonlySet, +): Promise => { + for (let attempt = 0; attempt < 80; attempt += 1) { + const result = await context.connection.send( + "Page.getFrameTree", + {}, + context.sessionId, + context.signal, + ); + const currentUrl = mainFrameUrl(result); + if (allowedSanitizedUrl(currentUrl, allowedOrigins) !== undefined) + return result; + if (isHttpUrl(currentUrl)) + throw new BrowserObservationError( + "inspect_web_page", + "target_not_allowed", + ); + await delayWithCancellation(25, context.signal); + } + throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); +}; + +const enableObservationDomains = async ( + connection: CdpConnection, + sessionId: string, + signal?: AbortSignal, +): Promise => { + await connection.send("Runtime.enable", {}, sessionId, signal); + await connection.send( + "Debugger.enable", + { maxScriptsCacheSize: 4 * 1_024 * 1_024 }, + sessionId, + signal, + ); + await connection.send( + "Network.enable", + { + maxTotalBufferSize: 4 * 1_024 * 1_024, + maxResourceBufferSize: 1_024 * 1_024, + }, + sessionId, + signal, + ); +}; + +const accessibilityForFrames = async ( + context: CaptureContext, + frames: readonly { readonly frame_id: string }[], + limitations: string[], +): Promise> => { + const nodes: WebPageInspection["accessibility"]["nodes"] = []; + let total = 0; + for (const frame of frames) { + const result = await optionalCdpCommand( + context, + "Accessibility.getFullAXTree", + { depth: 32, frameId: frame.frame_id }, + limitations, + ); + if (result === undefined) continue; + const capture = captureAccessibility( + [result], + Math.max(0, context.input.limits.max_ax_nodes - nodes.length), + ); + total += capture.total; + nodes.push(...capture.nodes); + } + return { total, nodes }; +}; + +const captureScripts = async ( + context: CaptureContext, + events: CdpCaptureEvents, + truncatedSections: Set, +): Promise => { + let totalSourceBytes = 0; + const items: WebPageInspection["scripts"]["items"] = []; + for (const script of events.scripts.values()) { + let source: WebPageInspection["scripts"]["items"][number]["source"] = + sourceExcluded("source capture was not approved"); + if (context.input.include_script_sources) { + if (script.length > context.input.limits.max_script_source_bytes) { + source = sourceExcluded( + "declared script length exceeds per-script limit", + ); + truncatedSections.add("script_sources"); + } else if ( + totalSourceBytes + script.length > + context.input.limits.max_total_script_source_bytes + ) { + source = sourceExcluded("total script source limit reached"); + truncatedSections.add("script_sources"); + } else { + const result = requiredRecord( + await context.connection.send( + "Debugger.getScriptSource", + { scriptId: script.scriptId }, + context.sessionId, + context.signal, + ), + ); + const content = stringValue(result.scriptSource) ?? ""; + const bytes = Buffer.byteLength(content); + if ( + bytes > context.input.limits.max_script_source_bytes || + totalSourceBytes + bytes > + context.input.limits.max_total_script_source_bytes + ) { + source = sourceExcluded( + "actual source exceeds configured byte limits", + ); + truncatedSections.add("script_sources"); + } else { + source = sourceResult(content); + totalSourceBytes += bytes; + } + } + } + items.push({ + script_id: script.scriptId, + url: script.url, + origin: script.origin, + cdp_hash: script.hash, + length: script.length, + is_module: script.isModule, + language: script.language, + source_map_url: script.sourceMapUrl, + source, + }); + } + return { total: events.scripts.size, items }; +}; + +const captureWorkers = async ( + context: CaptureContext, + allowedOrigins: ReadonlySet, + limitations: string[], +): Promise<{ + readonly total: number; + readonly items: WebPageInspection["workers"]; +}> => { + const result = await optionalCdpCommand( + { ...context, sessionId: undefined }, + "Target.getTargets", + {}, + limitations, + ); + const items: WebPageInspection["workers"] = []; + let total = 0; + for (const target of recordsValue(recordValue(result)?.targetInfos)) { + const type = stringValue(target.type) ?? ""; + const url = allowedSanitizedUrl(target.url, allowedOrigins); + if (!type.includes("worker") || url === undefined) continue; + total += 1; + if (items.length >= context.input.limits.max_workers) continue; + items.push({ + target_id: (stringValue(target.targetId) ?? "").slice(0, 256), + type: type.slice(0, 100), + url: url.url, + origin: url.origin, + attached: target.attached === true, + }); + } + return { + total, + items, + }; +}; + +const normalizedTarget = ( + target: CdpEndpointTarget, + currentUrl: string, + allowedOrigins: ReadonlySet, +): WebPageInspection["target"] => { + const url = allowedSanitizedUrl(currentUrl, allowedOrigins); + return { + target_id: target.id, + type: target.type, + title: target.title.slice(0, 16_384), + url: url?.url ?? "[unsupported-url]", + origin: url?.origin ?? "", + attached: target.attached, + }; +}; + +const report = async ( + progress: ProgressReporter | undefined, + completed: number, + message: string, +): Promise => + await progress?.report({ + phase: "browser_observation", + completed, + total: 3, + message, + ...(completed === 3 ? { terminal: true } : {}), + }); diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index 8af3269d2..3778f080b 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -36,6 +36,8 @@ export const CLI_COMMAND_NAMES = [ "capture-process", "compare-process-captures", "policy", + "list-browser-targets", + "inspect-web-page", ] as const; const toolCatalog = TOOL_CONTRACTS.map((contract) => ({ diff --git a/src/cli.ts b/src/cli.ts index a67e4dfb2..ba7f2ec19 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -27,6 +27,7 @@ import { import { projectReferenceSourceImportError } from "./application/ReferenceSourceImportTypes.js"; import { loadConfiguredPermissionAuthority } from "./application/PermissionConfiguration.js"; import { registerPolicyCommands } from "./cliPolicyCommands.js"; +import { registerBrowserCommands } from "./cliBrowserCommands.js"; /** * Build the one-shot Incur CLI without starting Hopper at import time. @@ -73,6 +74,7 @@ export const createCli = (): ReturnType => { registerReferenceSourceCommand(cli, logger); registerProcessCommands(cli, logger); registerPolicyCommands(cli, logger); + registerBrowserCommands(cli, logger); return cli; }; diff --git a/src/cliBrowserCommands.ts b/src/cliBrowserCommands.ts new file mode 100644 index 000000000..b873df441 --- /dev/null +++ b/src/cliBrowserCommands.ts @@ -0,0 +1,173 @@ +import { Cli, z } from "incur"; + +import { + inspectWebPage, + listBrowserTargets, +} from "./application/BrowserObservationService.js"; +import { loadConfiguredPermissionAuthority } from "./application/PermissionConfiguration.js"; +import { CdpBrowserProvider } from "./browser/CdpBrowserProvider.js"; +import { logCliCommand } from "./cliLogging.js"; +import { parseConfig } from "./config.js"; +import { + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, +} from "./domain/browserObservation.js"; +import { AnalysisInputError, projectAnalysisError } from "./domain/errors.js"; +import type { JsonValue } from "./domain/jsonValue.js"; +import type { Logger } from "./logger.js"; + +const browserScopeOptions = { + allowedOrigins: z + .array(z.string().min(1)) + .optional() + .describe( + "Exact origins to observe; defaults to REA_BROWSER_ALLOWED_ORIGINS_JSON", + ), + approved: z.boolean().default(false).describe("Approve passive observation"), +}; + +/** Register CLI equivalents of the passive browser MCP tools. */ +export const registerBrowserCommands = ( + cli: ReturnType, + logger: Logger, +): void => { + registerTargetList(cli, logger); + registerPageInspection(cli, logger); +}; + +const registerTargetList = ( + cli: ReturnType, + logger: Logger, +): void => { + cli.command("list-browser-targets", { + description: "List approved pages from a user-owned loopback CDP browser", + args: z.object({ + endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + }), + options: z.object({ + ...browserScopeOptions, + offset: z.number().int().min(0).default(0), + limit: z.number().int().min(1).max(200).default(100), + }), + run: ({ args, options }) => + logCliCommand(logger, "list-browser-targets", async () => { + const context = await browserContext(); + if (!context.ok) return context.error; + const parsed = listBrowserTargetsInputSchema.safeParse({ + cdp_endpoint: args.endpoint, + allowed_origins: + options.allowedOrigins ?? context.allowedBrowserOrigins, + approved: options.approved, + offset: options.offset, + limit: options.limit, + }); + if (!parsed.success) + return cliError(new AnalysisInputError("list_browser_targets")); + const result = await listBrowserTargets( + context.provider, + context.authority, + parsed.data, + ); + return result.ok ? result.value : cliError(result.error); + }), + }); +}; + +const registerPageInspection = ( + cli: ReturnType, + logger: Logger, +): void => { + cli.command("inspect-web-page", { + description: "Passively inspect one approved page through CDP", + args: z.object({ + endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + targetId: z.string().describe("Target ID from list-browser-targets"), + }), + options: z.object({ + ...browserScopeOptions, + observationMs: z.number().int().min(0).max(10_000).default(500), + includeScriptSources: z.boolean().default(false), + includeStorageKeys: z.boolean().default(false), + maxFrames: z.number().int().min(1).max(1_000).default(200), + maxDomNodes: z.number().int().min(1).max(10_000).default(2_000), + maxAxNodes: z.number().int().min(1).max(10_000).default(2_000), + maxScripts: z.number().int().min(1).max(1_000).default(200), + maxResources: z.number().int().min(1).max(10_000).default(2_000), + maxWorkers: z.number().int().min(1).max(5_000).default(500), + maxStorageKeys: z.number().int().min(1).max(10_000).default(1_000), + maxScriptSourceBytes: z + .number() + .int() + .min(1) + .max(4 * 1_024 * 1_024) + .default(1_024 * 1_024), + maxTotalScriptSourceBytes: z + .number() + .int() + .min(1) + .max(16 * 1_024 * 1_024) + .default(4 * 1_024 * 1_024), + maxNetworkEvents: z.number().int().min(1).max(10_000).default(1_000), + maxConsoleEvents: z.number().int().min(1).max(2_000).default(200), + maxWebsocketEvents: z.number().int().min(1).max(5_000).default(500), + }), + run: ({ args, options }) => + logCliCommand(logger, "inspect-web-page", async () => { + const context = await browserContext(); + if (!context.ok) return context.error; + const parsed = inspectWebPageInputSchema.safeParse({ + cdp_endpoint: args.endpoint, + allowed_origins: + options.allowedOrigins ?? context.allowedBrowserOrigins, + approved: options.approved, + target_id: args.targetId, + observation_ms: options.observationMs, + include_script_sources: options.includeScriptSources, + include_storage_keys: options.includeStorageKeys, + limits: { + max_frames: options.maxFrames, + max_dom_nodes: options.maxDomNodes, + max_ax_nodes: options.maxAxNodes, + max_scripts: options.maxScripts, + max_resources: options.maxResources, + max_workers: options.maxWorkers, + max_storage_keys: options.maxStorageKeys, + max_script_source_bytes: options.maxScriptSourceBytes, + max_total_script_source_bytes: options.maxTotalScriptSourceBytes, + max_network_events: options.maxNetworkEvents, + max_console_events: options.maxConsoleEvents, + max_websocket_events: options.maxWebsocketEvents, + }, + }); + if (!parsed.success) + return cliError(new AnalysisInputError("inspect_web_page")); + const result = await inspectWebPage( + context.provider, + context.authority, + parsed.data, + ); + return result.ok ? result.value : cliError(result.error); + }), + }); +}; + +const browserContext = async () => { + const config = parseConfig(process.env); + if (!config.ok) return { ok: false as const, error: cliError(config.error) }; + const authority = await loadConfiguredPermissionAuthority(config.value); + if (!authority.ok) + return { ok: false as const, error: cliError(authority.error) }; + return { + ok: true as const, + authority: authority.value, + provider: new CdpBrowserProvider(), + allowedBrowserOrigins: config.value.browserAllowedOrigins, + }; +}; + +const cliError = ( + error: Parameters[0], +): JsonValue => ({ + error: "Browser observation failed", + ...projectAnalysisError(error), +}); diff --git a/src/cliPolicyCommands.ts b/src/cliPolicyCommands.ts index d0dc12818..931ff03df 100644 --- a/src/cliPolicyCommands.ts +++ b/src/cliPolicyCommands.ts @@ -16,6 +16,7 @@ import type { Logger } from "./logger.js"; const capabilitySchema = z.enum([ "process_capture", + "browser_observe", "evidence_read", "evidence_write", "investigation_input", @@ -43,6 +44,7 @@ export const registerPolicyCommands = ( root: z.string().optional(), executable: z.string().optional(), environmentNames: z.array(z.string()).default([]), + origins: z.array(z.string()).default([]), network: z.enum(["none", "loopback", "external"]).default("none"), mount: z.boolean().default(false), write: z.boolean().default(false), @@ -102,6 +104,7 @@ const explain = async ( readonly root?: string | undefined; readonly executable?: string | undefined; readonly environmentNames: readonly string[]; + readonly origins: readonly string[]; readonly network: "none" | "loopback" | "external"; readonly mount: boolean; readonly write: boolean; @@ -115,6 +118,7 @@ const explain = async ( roots: options.root === undefined ? [] : [options.root], executables: options.executable === undefined ? [] : [options.executable], environment_names: options.environmentNames, + origins: options.origins, network: options.network, mount: options.mount, operation_identity: `policy_explain:${capability}`, diff --git a/src/config.ts b/src/config.ts index d49ac02c8..d8ab4b457 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,6 +10,10 @@ import type { PermissionCeiling, PermissionGrant, } from "./domain/permissionPolicy.js"; +import { + browserEndpointSchema, + browserOriginSchema, +} from "./domain/browserObservation.js"; const DEFAULT_HOPPER_LAUNCHER_PATH = "/Applications/Hopper Disassembler.app/Contents/MacOS/hopper"; @@ -31,6 +35,9 @@ export interface AppConfig { readonly investigationInputRoots: readonly string[]; readonly analysisSnapshotFilePolicy: EvidenceFilePolicy; readonly referenceSourcePolicy: ReferenceSourcePolicy; + readonly browserObservationEnabled: boolean; + readonly browserCdpEndpoints: readonly string[]; + readonly browserAllowedOrigins: readonly string[]; readonly permissionCeilings: readonly PermissionCeiling[]; readonly administratorPermissionGrants: readonly PermissionGrant[]; readonly permissionProjectRoot: string | undefined; @@ -66,6 +73,9 @@ const environmentSchema = z REA_ANALYSIS_SNAPSHOT_ROOTS_JSON: z.string().default("[]"), REA_REFERENCE_ROOTS_JSON: z.string().default("[]"), REA_REFERENCE_SECRET_PATTERNS_JSON: z.string().default("[]"), + REA_BROWSER_OBSERVE_ENABLED: z.enum(["true", "false"]).default("false"), + REA_BROWSER_CDP_ENDPOINTS_JSON: z.string().default("[]"), + REA_BROWSER_ALLOWED_ORIGINS_JSON: z.string().default("[]"), REA_PERMISSION_PROJECT_ROOT: z.string().min(1).optional(), REA_PERMISSION_PROJECT_STORE: z.string().min(1).optional(), }) @@ -100,6 +110,29 @@ const parseStringArray = ( } }; +const parseBrowserArray = ( + encoded: string, + name: string, + itemSchema: z.ZodType, + maximum: number, +): Result => { + try { + const parsed = z + .array(itemSchema) + .max(maximum) + .safeParse(JSON.parse(encoded)); + return parsed.success + ? ok([...new Set(parsed.data)].sort()) + : err( + new ConfigurationError(`${name} must encode valid browser scopes`, { + cause: parsed.error, + }), + ); + } catch (cause: unknown) { + return err(new ConfigurationError(`${name} must be valid JSON`, { cause })); + } +}; + const filePolicy = (roots: readonly string[]): EvidenceFilePolicy => ({ roots, maxBytes: 64 * 1024 * 1024, @@ -117,6 +150,7 @@ const permissionScope = ( roots, executables: options.executables ?? [], environment_names: options.environment_names ?? [], + ...(options.origins === undefined ? {} : { origins: options.origins }), network: options.network ?? "none", mount: options.mount ?? false, }); @@ -218,6 +252,20 @@ export const parseConfig = ( "REA_REFERENCE_SECRET_PATTERNS_JSON", ); if (!secretPatterns.ok) return secretPatterns; + const browserEndpoints = parseBrowserArray( + parsedEnvironment.data.REA_BROWSER_CDP_ENDPOINTS_JSON, + "REA_BROWSER_CDP_ENDPOINTS_JSON", + browserEndpointSchema, + 16, + ); + if (!browserEndpoints.ok) return browserEndpoints; + const browserOrigins = parseBrowserArray( + parsedEnvironment.data.REA_BROWSER_ALLOWED_ORIGINS_JSON, + "REA_BROWSER_ALLOWED_ORIGINS_JSON", + browserOriginSchema, + 32, + ); + if (!browserOrigins.ok) return browserOrigins; const permissionCeilings = [ ...(parsedEnvironment.data.REA_PROCESS_CAPTURE_ENABLED === "true" ? [ @@ -241,6 +289,14 @@ export const parseConfig = ( permissionScope("snapshot_write", analysisSnapshotRoots.value), permissionScope("artifact_extract", ["/"]), permissionScope("reference_read", referenceRoots.value), + ...(parsedEnvironment.data.REA_BROWSER_OBSERVE_ENABLED === "true" + ? [ + permissionScope("browser_observe", [], { + origins: [...browserEndpoints.value, ...browserOrigins.value], + network: browserNetworkScope(browserOrigins.value), + }), + ] + : []), ...(parsedEnvironment.data.REA_ARTIFACT_NATIVE_MOUNT_ENABLED === "true" ? [permissionScope("native_mount", [], { mount: true })] : []), @@ -276,9 +332,23 @@ export const parseConfig = ( maxDepth: 32, maxPathBytes: 4_096, }, + browserObservationEnabled: + parsedEnvironment.data.REA_BROWSER_OBSERVE_ENABLED === "true", + browserCdpEndpoints: browserEndpoints.value, + browserAllowedOrigins: browserOrigins.value, permissionCeilings, administratorPermissionGrants: administratorGrants(permissionCeilings), permissionProjectRoot: parsedEnvironment.data.REA_PERMISSION_PROJECT_ROOT, permissionProjectStore: parsedEnvironment.data.REA_PERMISSION_PROJECT_STORE, }); }; + +const browserNetworkScope = ( + origins: readonly string[], +): "loopback" | "external" => + origins.every((origin) => { + const hostname = new URL(origin).hostname; + return hostname === "127.0.0.1" || hostname === "[::1]"; + }) + ? "loopback" + : "external"; diff --git a/src/contracts/browserToolContracts.ts b/src/contracts/browserToolContracts.ts new file mode 100644 index 000000000..40d6592f1 --- /dev/null +++ b/src/contracts/browserToolContracts.ts @@ -0,0 +1,92 @@ +import type { ToolContract } from "./toolContracts.js"; +import { evidenceSchema } from "../domain/evidence.js"; +import { + browserTargetListSchema, + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, + webPageInspectionSchema, +} from "../domain/browserObservation.js"; + +const evidenceResult = (schema: typeof browserTargetListSchema) => + evidenceSchema + .omit({ normalized_result: true }) + .extend({ normalized_result: schema }); +const listOutputSchema = evidenceResult(browserTargetListSchema); +const inspectionOutputSchema = evidenceSchema + .omit({ normalized_result: true }) + .extend({ normalized_result: webPageInspectionSchema }); + +const endpoint = "http://127.0.0.1:9222"; +const origin = "https://app.example.test"; + +/** Origin-scoped, passive browser reverse-engineering contracts. */ +export const BROWSER_TOOL_CONTRACTS = [ + { + name: "list_browser_targets", + description: + "List bounded page targets from an approved user-owned loopback Chrome DevTools Protocol endpoint. Only targets whose current URL matches an approved exact origin are returned; URL credentials, query values, and fragments are redacted.", + kind: "browser-provider", + inputSchema: listBrowserTargetsInputSchema, + outputSchema: listOutputSchema, + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: true, + }, + examples: [ + { + title: "List approved browser page targets", + input: { + cdp_endpoint: endpoint, + allowed_origins: [origin], + approved: true, + offset: 0, + limit: 100, + }, + }, + ], + }, + { + name: "inspect_web_page", + description: + "Passively inspect one approved page target through CDP without evaluating JavaScript, navigating, clicking, closing, or mutating the page. Returns bounded DOM structure, accessibility, scripts, resources, attach-window network and console metadata, workers, and redacted storage inventory as Evidence v2.", + kind: "browser-provider", + inputSchema: inspectWebPageInputSchema, + outputSchema: inspectionOutputSchema, + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: true, + }, + examples: [ + { + title: "Inspect one approved browser page", + input: { + cdp_endpoint: endpoint, + allowed_origins: [origin], + approved: true, + target_id: "TARGET_ID_FROM_LIST_BROWSER_TARGETS", + observation_ms: 500, + include_script_sources: false, + include_storage_keys: false, + limits: { + max_frames: 200, + max_dom_nodes: 2_000, + max_ax_nodes: 2_000, + max_scripts: 200, + max_resources: 2_000, + max_workers: 500, + max_storage_keys: 1_000, + max_script_source_bytes: 1_048_576, + max_total_script_source_bytes: 4_194_304, + max_network_events: 1_000, + max_console_events: 200, + max_websocket_events: 500, + }, + }, + }, + ], + }, +] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/errorSchemas.ts b/src/contracts/errorSchemas.ts index cc7c68163..d846eb9c3 100644 --- a/src/contracts/errorSchemas.ts +++ b/src/contracts/errorSchemas.ts @@ -36,6 +36,7 @@ const scopeSchema = z roots: z.array(z.string()), executables: z.array(z.string()), environment_names: z.array(z.string()), + origins: z.array(z.string()).optional(), network: z.enum(["none", "loopback", "external"]), mount: z.boolean(), }) @@ -45,6 +46,7 @@ const missingScopeSchema = z roots: z.array(z.string()).optional(), executables: z.array(z.string()).optional(), environment_names: z.array(z.string()).optional(), + origins: z.array(z.string()).optional(), network: z.enum(["none", "loopback", "external"]).optional(), mount: z.literal(true).optional(), }) diff --git a/src/contracts/toolContractTypes.ts b/src/contracts/toolContractTypes.ts index 762947798..2e8ada42c 100644 --- a/src/contracts/toolContractTypes.ts +++ b/src/contracts/toolContractTypes.ts @@ -22,6 +22,7 @@ export type ToolKind = | "enhanced" | "native-provider" | "artifact-provider" + | "browser-provider" | "session"; /** Single source of truth for a public MCP tool. */ diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index 63fa3ec77..62e29b6d2 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -26,6 +26,7 @@ import { import { TOOL_EXAMPLE_OVERRIDES } from "./toolContractExamples.js"; import { NATIVE_TOOL_CONTRACTS } from "./nativeToolContracts.js"; import { ARTIFACT_TOOL_CONTRACTS } from "./artifactToolContracts.js"; +import { BROWSER_TOOL_CONTRACTS } from "./browserToolContracts.js"; import { closeBinaryInputSchema, openBinaryInputSchema, @@ -512,15 +513,12 @@ export const SESSION_TOOL_CONTRACTS = [ ), ] as const satisfies readonly ToolContract[]; -/** - * Complete ordered public inventory used by registration and verification. - * Keep this collection at 68 tools unless a deliberate contract change updates - * snapshots, package verification, and real-Hopper verification together. - */ +/** Complete ordered public inventory used by registration and verification. */ export const TOOL_CONTRACTS = [ ...OFFICIAL_TOOL_CONTRACTS, ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, + ...BROWSER_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ] as const; diff --git a/src/domain/browserObservation.ts b/src/domain/browserObservation.ts new file mode 100644 index 000000000..a04f57dcd --- /dev/null +++ b/src/domain/browserObservation.ts @@ -0,0 +1,397 @@ +import { z } from "zod"; + +const parseExactOrigin = (value: string): string | undefined => { + let url: URL; + try { + url = new URL(value); + } catch { + return undefined; + } + if ( + (url.protocol !== "http:" && url.protocol !== "https:") || + url.hostname.includes("*") || + url.username !== "" || + url.password !== "" || + url.pathname !== "/" || + url.search !== "" || + url.hash !== "" + ) + return undefined; + return url.origin; +}; + +const MAX_BROWSER_URL_CHARS = 65_536; +const MAX_SANITIZED_BROWSER_URL_CHARS = 131_072; +const MAX_QUERY_PARAMETER_NAMES = 256; +const MAX_QUERY_PARAMETER_NAME_CHARS = 256; + +/** Exact normalized HTTP(S) authority used for browser observation scope. */ +export const browserOriginSchema = z + .string() + .min(1) + .max(2_048) + .transform((value, context) => { + const origin = parseExactOrigin(value); + if (origin === undefined) { + context.addIssue({ + code: "custom", + message: "Expected one exact HTTP(S) origin without a path or wildcard", + }); + return z.NEVER; + } + return origin; + }); + +const isLiteralLoopback = (hostname: string): boolean => + hostname === "127.0.0.1" || hostname === "[::1]" || hostname === "::1"; + +/** Loopback-only HTTP endpoint accepted for a user-owned CDP browser. */ +export const browserEndpointSchema = z + .string() + .min(1) + .max(2_048) + .transform((value, context) => { + let url: URL; + try { + url = new URL(value); + } catch { + context.addIssue({ code: "custom", message: "Invalid CDP endpoint URL" }); + return z.NEVER; + } + if ( + url.protocol !== "http:" || + !isLiteralLoopback(url.hostname) || + url.port === "" || + url.username !== "" || + url.password !== "" || + url.pathname !== "/" || + url.search !== "" || + url.hash !== "" + ) { + context.addIssue({ + code: "custom", + message: + "CDP endpoint must be an explicit-port HTTP URL on 127.0.0.1 or ::1", + }); + return z.NEVER; + } + return url.origin; + }); + +const allowedOriginsSchema = z + .array(browserOriginSchema) + .min(1) + .max(32) + .transform((origins) => [...new Set(origins)].sort()); + +const approvedBrowserInput = { + cdp_endpoint: browserEndpointSchema, + allowed_origins: allowedOriginsSchema, + approved: z + .literal(true) + .describe("Explicit approval for browser observation"), +}; + +/** Public input for bounded discovery of allowed page targets. */ +export const listBrowserTargetsInputSchema = z.object({ + ...approvedBrowserInput, + offset: z.number().int().min(0).default(0), + limit: z.number().int().min(1).max(200).default(100), +}); + +const browserInspectionLimitsSchema = z.object({ + max_frames: z.number().int().min(1).max(1_000).default(200), + max_dom_nodes: z.number().int().min(1).max(10_000).default(2_000), + max_ax_nodes: z.number().int().min(1).max(10_000).default(2_000), + max_scripts: z.number().int().min(1).max(1_000).default(200), + max_resources: z.number().int().min(1).max(10_000).default(2_000), + max_workers: z.number().int().min(1).max(5_000).default(500), + max_storage_keys: z.number().int().min(1).max(10_000).default(1_000), + max_script_source_bytes: z + .number() + .int() + .min(1) + .max(4 * 1_024 * 1_024) + .default(1_024 * 1_024), + max_total_script_source_bytes: z + .number() + .int() + .min(1) + .max(16 * 1_024 * 1_024) + .default(4 * 1_024 * 1_024), + max_network_events: z.number().int().min(1).max(10_000).default(1_000), + max_console_events: z.number().int().min(1).max(2_000).default(200), + max_websocket_events: z.number().int().min(1).max(5_000).default(500), +}); + +const DEFAULT_BROWSER_INSPECTION_LIMITS = { + max_frames: 200, + max_dom_nodes: 2_000, + max_ax_nodes: 2_000, + max_scripts: 200, + max_resources: 2_000, + max_workers: 500, + max_storage_keys: 1_000, + max_script_source_bytes: 1_024 * 1_024, + max_total_script_source_bytes: 4 * 1_024 * 1_024, + max_network_events: 1_000, + max_console_events: 200, + max_websocket_events: 500, +} as const; + +/** Public input for one passive, bounded inspection of an existing page. */ +export const inspectWebPageInputSchema = z + .object({ + ...approvedBrowserInput, + target_id: z.string().trim().min(1).max(256), + observation_ms: z.number().int().min(0).max(10_000).default(500), + include_script_sources: z.boolean().default(false), + include_storage_keys: z.boolean().default(false), + limits: browserInspectionLimitsSchema.default( + DEFAULT_BROWSER_INSPECTION_LIMITS, + ), + }) + .superRefine((input, context) => { + if ( + input.limits.max_script_source_bytes > + input.limits.max_total_script_source_bytes + ) + context.addIssue({ + code: "custom", + path: ["limits", "max_script_source_bytes"], + message: "Per-script source limit cannot exceed the total source limit", + }); + }); + +export type ListBrowserTargetsInput = z.infer< + typeof listBrowserTargetsInputSchema +>; +export type InspectWebPageInput = z.infer; + +const sanitizedBrowserUrlSchema = z.object({ + url: z.string().max(MAX_SANITIZED_BROWSER_URL_CHARS), + origin: z.string().max(2_048).nullable(), + query_parameter_names: z + .array(z.string().max(MAX_QUERY_PARAMETER_NAME_CHARS)) + .max(MAX_QUERY_PARAMETER_NAMES), + redacted: z.boolean(), +}); +export type SanitizedBrowserUrl = z.infer; + +/** Remove credentials and query values before a browser URL becomes durable. */ +export const sanitizeBrowserUrl = (value: string): SanitizedBrowserUrl => { + let parsed: URL; + const wasTruncated = value.length > MAX_BROWSER_URL_CHARS; + try { + parsed = new URL(value.slice(0, MAX_BROWSER_URL_CHARS)); + } catch { + return { + url: "[unsupported-url]", + origin: null, + query_parameter_names: [], + redacted: true, + }; + } + const hadCredentials = parsed.username !== "" || parsed.password !== ""; + const hadFragment = parsed.hash !== ""; + const names = [ + ...new Set( + [...parsed.searchParams.keys()].map((name) => + name.slice(0, MAX_QUERY_PARAMETER_NAME_CHARS), + ), + ), + ] + .sort() + .slice(0, MAX_QUERY_PARAMETER_NAMES); + parsed.username = ""; + parsed.password = ""; + parsed.hash = ""; + parsed.search = ""; + for (const name of names) parsed.searchParams.append(name, "[REDACTED]"); + return { + url: parsed.href, + origin: parsed.origin === "null" ? null : parsed.origin, + query_parameter_names: names, + redacted: wasTruncated || hadCredentials || hadFragment || names.length > 0, + }; +}; + +const browserVersionSchema = z.object({ + product: z.string(), + protocol_version: z.string(), + revision: z.string(), + user_agent: z.string(), + js_version: z.string(), +}); + +const browserTargetSchema = z.object({ + target_id: z.string(), + type: z.string(), + title: z.string(), + url: z.string(), + origin: z.string(), + attached: z.boolean(), +}); + +/** Bounded and policy-filtered browser target discovery result. */ +export const browserTargetListSchema = z.object({ + schema_version: z.literal(1), + browser: browserVersionSchema, + targets: z.object({ + items: z.array(browserTargetSchema), + offset: z.number().int().min(0), + limit: z.number().int().min(1), + total: z.number().int().min(0), + next_offset: z.number().int().min(0).nullable(), + has_more: z.boolean(), + }), + excluded: z.object({ + disallowed_origin: z.number().int().min(0), + unsupported_url: z.number().int().min(0), + non_page: z.number().int().min(0), + }), + limitations: z.array(z.string()), +}); +export type BrowserTargetList = z.infer; + +const browserFrameSchema = z.object({ + frame_id: z.string(), + parent_frame_id: z.string().nullable(), + url: z.string(), + origin: z.string().nullable(), +}); +const browserDomNodeSchema = z.object({ + index: z.number().int().min(0), + parent_index: z.number().int().min(-1), + node_type: z.number().int().min(0), + node_name: z.string(), + node_value_length: z.number().int().min(0), + attribute_names: z.array(z.string()), +}); +const browserAxNodeSchema = z.object({ + node_id: z.string(), + parent_id: z.string().nullable(), + role: z.string().nullable(), + name: z.string().nullable(), + description: z.string().nullable(), + ignored: z.boolean(), +}); +const browserScriptSourceSchema = z.discriminatedUnion("included", [ + z.object({ included: z.literal(false), reason: z.string() }), + z.object({ + included: z.literal(true), + sha256: z.string().regex(/^[a-f0-9]{64}$/u), + bytes: z.number().int().min(0), + content: z.string(), + }), +]); +const browserScriptSchema = z.object({ + script_id: z.string(), + url: z.string(), + origin: z.string().nullable(), + cdp_hash: z.string(), + length: z.number().int().min(0), + is_module: z.boolean(), + language: z.string().nullable(), + source_map_url: z.string().nullable(), + source: browserScriptSourceSchema, +}); +const browserResourceSchema = z.object({ + url: z.string(), + origin: z.string().nullable(), + type: z.string(), + mime_type: z.string(), + content_size: z.number().min(0).nullable(), +}); +const browserNetworkRequestSchema = z.object({ + request_id: z.string(), + url: z.string(), + origin: z.string().nullable(), + method: z.string(), + resource_type: z.string().nullable(), + status: z.number().nullable(), + mime_type: z.string().nullable(), + encoded_data_length: z.number().min(0).nullable(), + initiator: z.object({ + type: z.string(), + url: z.string().nullable(), + line: z.number().int().min(0).nullable(), + column: z.number().int().min(0).nullable(), + }), +}); +const browserConsoleEventSchema = z.object({ + type: z.string(), + timestamp: z.number(), + argument_types: z.array(z.string()), + url: z.string().nullable(), + line: z.number().int().min(0).nullable(), + column: z.number().int().min(0).nullable(), +}); +const browserWebSocketEventSchema = z.object({ + request_id: z.string(), + direction: z.enum(["sent", "received"]), + opcode: z.number().int().min(0), + payload_bytes: z.number().int().min(0), +}); +const browserWorkerSchema = z.object({ + target_id: z.string(), + type: z.string(), + url: z.string(), + origin: z.string().nullable(), + attached: z.boolean(), +}); + +/** Provider-neutral normalized result for one passive web-page observation. */ +export const webPageInspectionSchema = z.object({ + schema_version: z.literal(1), + browser: browserVersionSchema, + target: browserTargetSchema, + capture_window: z.object({ + started_at: z.iso.datetime(), + ended_at: z.iso.datetime(), + observation_ms: z.number().int().min(0), + }), + completeness: z.object({ + status: z.enum(["complete", "truncated"]), + truncated_sections: z.array(z.string()), + dropped_events: z.number().int().min(0), + }), + frames: z.array(browserFrameSchema), + dom: z.object({ + total_nodes: z.number().int().min(0), + nodes: z.array(browserDomNodeSchema), + }), + accessibility: z.object({ + total_nodes: z.number().int().min(0), + nodes: z.array(browserAxNodeSchema), + }), + scripts: z.object({ + total: z.number().int().min(0), + items: z.array(browserScriptSchema), + }), + resources: z.array(browserResourceSchema), + network: z.object({ + requests: z.array(browserNetworkRequestSchema), + websocket_events: z.array(browserWebSocketEventSchema), + coverage_started_at: z.iso.datetime(), + prior_activity_available: z.literal(false), + }), + console: z.object({ + events: z.array(browserConsoleEventSchema), + coverage_started_at: z.iso.datetime(), + prior_activity_available: z.literal(false), + }), + workers: z.array(browserWorkerSchema), + storage: z.object({ + origin: z.string(), + usage_bytes: z.number().min(0).nullable(), + quota_bytes: z.number().min(0).nullable(), + local_storage_keys: z.array(z.string()), + session_storage_keys: z.array(z.string()), + indexed_db_names: z.array(z.string()), + cache_names: z.array(z.string()), + values_redacted: z.literal(true), + }), + limitations: z.array(z.string()), +}); +export type WebPageInspection = z.infer; diff --git a/src/domain/errors.ts b/src/domain/errors.ts index 227ff5921..5f3414cbb 100644 --- a/src/domain/errors.ts +++ b/src/domain/errors.ts @@ -20,6 +20,7 @@ const ANALYSIS_ERROR_TAGS = [ "AnalysisTimeoutError", "ProviderSelectionError", "ProviderAdapterError", + "BrowserObservationError", "ArtifactOperationError", "ProcessCaptureError", "EvidenceIntegrityError", @@ -143,6 +144,27 @@ export class ProviderAdapterError extends AnalysisError { } } +/** A bounded passive browser observation failed at its CDP boundary. */ +export class BrowserObservationError extends AnalysisError { + readonly _tag = "BrowserObservationError"; + + constructor( + readonly operation: "list_browser_targets" | "inspect_web_page", + readonly reason: + | "endpoint_unreachable" + | "invalid_endpoint_response" + | "target_not_found" + | "target_not_allowed" + | "target_changed" + | "protocol_error" + | "disconnected" + | "payload_limit", + options?: ErrorOptions, + ) { + super(`Browser observation ${operation} failed: ${reason}`, options); + } +} + /** Artifact inventory or extraction failed a typed safety boundary. */ export class ArtifactOperationError extends AnalysisError { readonly _tag = "ArtifactOperationError"; @@ -424,6 +446,21 @@ export const projectAnalysisError = ( const errorCode = (error: AnalysisError): AnalysisErrorProjection["code"] => { if (error instanceof PermissionRequiredError) return "permission_required"; + if (error instanceof BrowserObservationError) { + if (error.reason === "payload_limit") return "truncated"; + if ( + error.reason === "target_not_found" || + error.reason === "target_not_allowed" || + error.reason === "target_changed" + ) + return "target_unavailable"; + if ( + error.reason === "endpoint_unreachable" || + error.reason === "disconnected" + ) + return "provider_unavailable"; + return "unreadable_output"; + } if (error instanceof ArtifactOperationError) return error.reason === "integrity" && error.artifactDetails !== undefined ? "artifact_integrity_mismatch" @@ -501,6 +538,8 @@ const errorCode = (error: AnalysisError): AnalysisErrorProjection["code"] => { case "ProviderAdapterError": case "HopperRemoteError": return "execution_failure"; + case "BrowserObservationError": + throw new TypeError("Unhandled specialized browser error"); case "ArtifactOperationError": case "EvidenceFileError": case "InvestigationWorkspaceError": @@ -552,6 +591,8 @@ const errorDetails = ( return { provider_id: error.providerId, operation: error.operation }; if (error instanceof ProviderAdapterError) return { provider_id: error.providerId, operation: error.operation }; + if (error instanceof BrowserObservationError) + return { operation: error.operation, reason: error.reason }; if (error instanceof HopperRemoteError) return { provider_code: error.code, diagnostic_type: error.diagnosticType }; if (error instanceof HopperProcessError && error.failureCode !== undefined) @@ -577,6 +618,7 @@ const scopeDetails = ( roots: [...scope.roots], executables: [...scope.executables], environment_names: [...scope.environment_names], + ...(scope.origins === undefined ? {} : { origins: [...scope.origins] }), network: scope.network, mount: scope.mount, }); @@ -591,6 +633,7 @@ const missingScopeDetails = ( ...(scope.environment_names === undefined ? {} : { environment_names: [...scope.environment_names] }), + ...(scope.origins === undefined ? {} : { origins: [...scope.origins] }), ...(scope.network === undefined ? {} : { network: scope.network }), ...(scope.mount === undefined ? {} : { mount: scope.mount }), }); @@ -616,6 +659,17 @@ const errorCategory = ( if (error instanceof PermissionRequiredError) return "permission_required"; if (error._tag === "ProcessCaptureError") return error.userCategory ?? "execution_failure"; + if (error instanceof BrowserObservationError) { + if (error.reason === "payload_limit") return "truncated"; + if ( + error.reason === "target_not_found" || + error.reason === "target_not_allowed" || + error.reason === "target_changed" || + error.reason === "endpoint_unreachable" || + error.reason === "disconnected" + ) + return "unavailable"; + } if ( error instanceof HopperRemoteError && error.diagnosticType === "authorization" @@ -799,6 +853,7 @@ const KNOWN_ERROR_TAGS = { AnalysisTimeoutError: true, ProviderSelectionError: true, ProviderAdapterError: true, + BrowserObservationError: true, ArtifactOperationError: true, ProcessCaptureError: true, EvidenceIntegrityError: true, diff --git a/src/domain/permissionPolicy.ts b/src/domain/permissionPolicy.ts index afa22af3c..4e62aa666 100644 --- a/src/domain/permissionPolicy.ts +++ b/src/domain/permissionPolicy.ts @@ -5,6 +5,7 @@ import { err, ok, type Result } from "./result.js"; /** Local capabilities governed by the shared REA permission policy. */ export type PermissionCapability = | "process_capture" + | "browser_observe" | "evidence_read" | "evidence_write" | "investigation_input" @@ -25,6 +26,7 @@ export interface PermissionScope { readonly roots: readonly string[]; readonly executables: readonly string[]; readonly environment_names: readonly string[]; + readonly origins?: readonly string[] | undefined; readonly network: PermissionNetwork; readonly mount: boolean; } @@ -50,6 +52,7 @@ export interface MissingPermissionScope { readonly roots?: readonly string[]; readonly executables?: readonly string[]; readonly environment_names?: readonly string[]; + readonly origins?: readonly string[]; readonly network?: PermissionNetwork; readonly mount?: true; } @@ -220,6 +223,9 @@ const normalizeScope = (scope: PermissionScope): PermissionScope => ({ roots: uniqueSorted(scope.roots.map((path) => resolve(path))), executables: uniqueSorted(scope.executables.map((path) => resolve(path))), environment_names: uniqueSorted(scope.environment_names), + ...(scope.origins === undefined + ? {} + : { origins: uniqueSorted(scope.origins) }), network: scope.network, mount: scope.mount, }); @@ -253,6 +259,10 @@ const missingFromScopes = ( const environmentNames = request.environment_names.filter( (name) => !relevant.some((scope) => scope.environment_names.includes(name)), ); + const origins = (request.origins ?? []).filter( + (origin) => + !relevant.some((scope) => (scope.origins ?? []).includes(origin)), + ); const network = request.network === "none" || relevant.some( @@ -270,6 +280,7 @@ const missingFromScopes = ( ...(environmentNames.length === 0 ? {} : { environment_names: environmentNames }), + ...(origins.length === 0 ? {} : { origins }), ...(network === undefined ? {} : { network }), ...(mount === undefined ? {} : { mount }), }; diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index ebca249c2..3c11aee90 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -1,9 +1,9 @@ /** Generated from package.json and package-lock.json; do not edit. */ export const PACKAGE_METADATA = { name: "rea-agents", - version: "1.3.0", + version: "1.4.0", serverSdkVersion: "2.0.0-beta.4", clientSdkVersion: "2.0.0-beta.4", coreSdkVersion: "2.0.0-beta.4", - skillVersion: "12", + skillVersion: "13", } as const; diff --git a/src/main.ts b/src/main.ts index 46d6c85fb..05d06b301 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,6 +15,7 @@ import { createLogger } from "./logger.js"; import { projectAnalysisError } from "./domain/errors.js"; import { readProjectPermissionStore } from "./application/ProjectPermissionStore.js"; import { loadConfiguredPermissionAuthority } from "./application/PermissionConfiguration.js"; +import { CdpBrowserProvider } from "./browser/CdpBrowserProvider.js"; const MCP_CONNECTION_LOST = "REA lost its MCP client connection. Restart REA from your MCP client."; @@ -80,6 +81,7 @@ export const run = async ( } const session = createBinarySession(config.value, logger); + const browserObservation = new CdpBrowserProvider(); if (config.value.hopperTargetPath !== undefined) { const opened = await session.open(config.value.hopperTargetPath, { targetKind: config.value.hopperTargetKind, @@ -126,11 +128,16 @@ export const run = async ( investigationInputRoots: runtimeInvestigationRoots, analysisSnapshotFilePolicy: runtimeSnapshotPolicy, permissionAuthority: permissionAuthority.value, + browserObservation, artifactIntegrityContinueEnabled: () => currentConfig.artifactIntegrityContinueEnabled, availabilityPolicy: () => ({ processCaptureEnabled: currentConfig.processExecutionPolicy.enabled, evidenceFileRoots: currentConfig.evidenceFilePolicy.roots.length, + browserObservationEnabled: + currentConfig.browserObservationEnabled && + currentConfig.browserCdpEndpoints.length > 0 && + currentConfig.browserAllowedOrigins.length > 0, }), }); liveServers.add(server); diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 87d2f2e11..e14dfc392 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -15,6 +15,8 @@ import { registerGuidedPrompts } from "./registerPrompts.js"; import { registerEvidenceResources } from "./registerEvidenceResources.js"; import { createServerIdentity } from "../serverIdentity.js"; import type { PermissionAuthority } from "../application/PermissionAuthority.js"; +import type { BrowserObservationPort } from "../application/BrowserObservationPort.js"; +import { registerBrowserTools } from "./registerBrowserTools.js"; export interface CreateServerOptions { readonly logger?: Logger; @@ -23,10 +25,12 @@ export interface CreateServerOptions { readonly investigationInputRoots?: readonly string[]; readonly analysisSnapshotFilePolicy?: EvidenceFilePolicy; readonly permissionAuthority?: PermissionAuthority; + readonly browserObservation?: BrowserObservationPort; readonly artifactIntegrityContinueEnabled?: () => boolean; readonly availabilityPolicy?: () => { readonly processCaptureEnabled: boolean; readonly evidenceFileRoots: number; + readonly browserObservationEnabled?: boolean; }; } @@ -136,6 +140,12 @@ export const createServer = ( ? {} : { permissionAuthority: options.permissionAuthority }), }); + registerBrowserTools(server, { + logger: toolLogger, + browser: options.browserObservation, + permissionAuthority: options.permissionAuthority, + recordEvidence, + }); registerGuidedPrompts(server, analysis, session); if (session !== undefined) { registerEvidenceResources(server, session); diff --git a/src/server/registerBrowserTools.ts b/src/server/registerBrowserTools.ts new file mode 100644 index 000000000..93e5bb0b8 --- /dev/null +++ b/src/server/registerBrowserTools.ts @@ -0,0 +1,85 @@ +import type { McpServer } from "@modelcontextprotocol/server"; + +import type { BrowserObservationPort } from "../application/BrowserObservationPort.js"; +import type { BinarySessionPort } from "../application/BinarySession.js"; +import type { PermissionAuthority } from "../application/PermissionAuthority.js"; +import { + inspectWebPage, + listBrowserTargets, +} from "../application/BrowserObservationService.js"; +import { BROWSER_TOOL_CONTRACTS } from "../contracts/browserToolContracts.js"; +import { + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, +} from "../domain/browserObservation.js"; +import type { Logger } from "../logger.js"; +import { logToolExecution } from "./toolLogging.js"; +import { toCallToolResult } from "./toolResult.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { mcpProgressReporter } from "./mcpProgress.js"; + +interface BrowserToolRegistration { + readonly logger: Logger; + readonly browser: BrowserObservationPort | undefined; + readonly permissionAuthority: PermissionAuthority | undefined; + readonly recordEvidence: BinarySessionPort["recordEvidence"] | undefined; +} + +/** Register browser tools even when policy/provider availability denies execution. */ +export const registerBrowserTools = ( + server: McpServer, + options: BrowserToolRegistration, +): void => { + const [listContract, inspectContract] = BROWSER_TOOL_CONTRACTS; + server.registerTool( + listContract.name, + toolRegistrationOptions(listContract), + async (input, context) => { + const parsed = listBrowserTargetsInputSchema.parse(input); + const result = await logToolExecution( + options.logger, + listContract.name, + () => + listBrowserTargets( + options.browser, + options.permissionAuthority, + parsed, + { signal: context.mcpReq.signal }, + ), + ); + if (!result.ok) return toCallToolResult(result, listContract); + return evidenceResult(options, listContract, result.value); + }, + ); + server.registerTool( + inspectContract.name, + toolRegistrationOptions(inspectContract), + async (input, context) => { + const parsed = inspectWebPageInputSchema.parse(input); + const result = await logToolExecution( + options.logger, + inspectContract.name, + () => + inspectWebPage(options.browser, options.permissionAuthority, parsed, { + signal: context.mcpReq.signal, + progress: mcpProgressReporter(context), + }), + ); + if (!result.ok) return toCallToolResult(result, inspectContract); + return evidenceResult(options, inspectContract, result.value); + }, + ); +}; + +const evidenceResult = ( + options: BrowserToolRegistration, + contract: (typeof BROWSER_TOOL_CONTRACTS)[number], + evidence: import("../domain/evidence.js").Evidence, +) => { + const recorded = options.recordEvidence?.(evidence); + return recorded !== undefined && !recorded.ok + ? toCallToolResult(recorded, contract) + : toCallToolResult({ ok: true, value: evidence }, contract, { + evidenceResourcesAvailable: recorded !== undefined, + }); +}; diff --git a/src/server/registerSessionStatusTool.ts b/src/server/registerSessionStatusTool.ts index ba57f3658..aba0a6f7d 100644 --- a/src/server/registerSessionStatusTool.ts +++ b/src/server/registerSessionStatusTool.ts @@ -19,6 +19,7 @@ export const registerSessionStatusTool = ( availabilityPolicy: () => { readonly processCaptureEnabled: boolean; readonly evidenceFileRoots: number; + readonly browserObservationEnabled?: boolean; }, ): void => { server.registerTool( diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index cc61a9223..3fab9e7fb 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -432,6 +432,7 @@ interface LifecycleToolRegistration { readonly availabilityPolicy: () => { readonly processCaptureEnabled: boolean; readonly evidenceFileRoots: number; + readonly browserObservationEnabled?: boolean; }; readonly permissionAuthority?: PermissionAuthority; } @@ -592,6 +593,7 @@ export interface SessionToolOptions { readonly availabilityPolicy?: () => { readonly processCaptureEnabled: boolean; readonly evidenceFileRoots: number; + readonly browserObservationEnabled?: boolean; }; } @@ -638,6 +640,7 @@ export const registerSessionTools = ( (() => ({ processCaptureEnabled: processPolicy.enabled, evidenceFileRoots: evidenceFilePolicy.roots.length, + browserObservationEnabled: false, })), ...(options.permissionAuthority === undefined ? {} diff --git a/tests/browserCli.test.ts b/tests/browserCli.test.ts new file mode 100644 index 000000000..364479d37 --- /dev/null +++ b/tests/browserCli.test.ts @@ -0,0 +1,135 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + startFakeCdpBrowser, + type FakeCdpBrowser, +} from "./fixtures/fakeCdpBrowser.js"; + +const execute = promisify(execFile); +const browsers: FakeCdpBrowser[] = []; +const INTEGRATION_TEST_TIMEOUT_MS = 20_000; + +afterEach(async () => { + await Promise.all(browsers.splice(0).map(async (browser) => browser.close())); +}); + +describe("browser CLI parity", () => { + it( + "returns the same Evidence v2 discovery and inspection contracts", + async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const environment = { + ...process.env, + REA_BROWSER_OBSERVE_ENABLED: "true", + REA_BROWSER_CDP_ENDPOINTS_JSON: JSON.stringify([browser.endpoint]), + REA_BROWSER_ALLOWED_ORIGINS_JSON: JSON.stringify([ + browser.allowedOrigin, + ]), + }; + const listed = await runCli( + ["list-browser-targets", browser.endpoint, "--approved", "--json"], + environment, + ); + expect(listed).toMatchObject({ + operation: "list_browser_targets", + provider: { id: "rea-cdp-browser" }, + normalized_result: { + targets: { items: [{ target_id: "allowed-page" }] }, + }, + }); + const inspected = await runCli( + [ + "inspect-web-page", + browser.endpoint, + "allowed-page", + "--approved", + "--observation-ms", + "0", + "--json", + ], + environment, + ); + expect(inspected).toMatchObject({ + operation: "inspect_web_page", + provider: { id: "rea-cdp-browser" }, + normalized_result: { + target: { target_id: "allowed-page" }, + network: { prior_activity_available: false }, + }, + }); + const policy = await runCli( + [ + "policy", + "explain", + "browser_observe", + "--origins", + browser.endpoint, + "--origins", + browser.allowedOrigin, + "--network", + "loopback", + "--json", + ], + environment, + ); + expect(policy).toEqual({ + allowed: true, + grant_id: "administrator:browser_observe", + }); + }, + INTEGRATION_TEST_TIMEOUT_MS, + ); + + it("requires explicit approval before opening the CDP endpoint", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const result = await runCli( + ["list-browser-targets", browser.endpoint, "--json"], + { + ...process.env, + REA_BROWSER_OBSERVE_ENABLED: "true", + REA_BROWSER_CDP_ENDPOINTS_JSON: JSON.stringify([browser.endpoint]), + REA_BROWSER_ALLOWED_ORIGINS_JSON: JSON.stringify([ + browser.allowedOrigin, + ]), + }, + ); + expect(result).toMatchObject({ + error: "Browser observation failed", + code: "invalid_request", + category: "invalid_input", + }); + expect(browser.commands).toHaveLength(0); + }); +}); + +const runCli = async ( + arguments_: readonly string[], + environment: NodeJS.ProcessEnv, +): Promise => { + try { + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", ...arguments_], + { + cwd: process.cwd(), + env: environment, + maxBuffer: 16 * 1_024 * 1_024, + }, + ); + return JSON.parse(stdout); + } catch (cause: unknown) { + if ( + typeof cause === "object" && + cause !== null && + "stdout" in cause && + typeof cause.stdout === "string" + ) + return JSON.parse(cause.stdout); + throw cause; + } +}; diff --git a/tests/browserMcp.test.ts b/tests/browserMcp.test.ts new file mode 100644 index 000000000..948711765 --- /dev/null +++ b/tests/browserMcp.test.ts @@ -0,0 +1,170 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { afterEach, describe, expect, it } from "vitest"; + +import { BinarySession } from "../src/application/BinarySession.js"; +import { loadConfiguredPermissionAuthority } from "../src/application/PermissionConfiguration.js"; +import { CdpBrowserProvider } from "../src/browser/CdpBrowserProvider.js"; +import { parseConfig } from "../src/config.js"; +import { createServer } from "../src/server/createServer.js"; +import { observed } from "./fixtures/analysisExecution.js"; +import { + startFakeCdpBrowser, + type FakeCdpBrowser, +} from "./fixtures/fakeCdpBrowser.js"; + +const resources: Array<{ close(): Promise }> = []; +const browsers: FakeCdpBrowser[] = []; +const INTEGRATION_TEST_TIMEOUT_MS = 20_000; + +afterEach(async () => { + await Promise.all( + resources.splice(0).map(async (resource) => resource.close()), + ); + await Promise.all(browsers.splice(0).map(async (browser) => browser.close())); +}); + +describe("browser MCP tools", () => { + it( + "exposes CLI-equivalent Evidence v2 and session resources", + async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const connected = await connectBrowser(browser); + + const tools = await connected.client.listTools(); + expect(tools.tools).toHaveLength(70); + expect(tools.tools.map(({ name }) => name)).toEqual( + expect.arrayContaining(["list_browser_targets", "inspect_web_page"]), + ); + const status = await connected.client.callTool({ + name: "binary_session", + arguments: {}, + }); + expect(status.structuredContent).toMatchObject({ + result: { + tool_availability: expect.arrayContaining([ + expect.objectContaining({ + name: "inspect_web_page", + available: true, + reason: "available", + }), + ]), + }, + }); + const listed = await connected.client.callTool({ + name: "list_browser_targets", + arguments: { + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + approved: true, + }, + }); + expect(listed.isError).not.toBe(true); + expect(listed.structuredContent).toMatchObject({ + operation: "list_browser_targets", + authority: "external-service", + confidence: "observed", + subject: null, + provider: { id: "rea-cdp-browser", version: "1" }, + normalized_result: { + targets: { items: [{ target_id: "allowed-page" }] }, + }, + }); + const inspected = await connected.client.callTool({ + name: "inspect_web_page", + arguments: { + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + }, + }); + expect(inspected.isError).not.toBe(true); + expect(inspected.structuredContent).toMatchObject({ + operation: "inspect_web_page", + normalized_result: { + target: { target_id: "allowed-page" }, + network: { prior_activity_available: false }, + console: { prior_activity_available: false }, + }, + }); + const evidenceId = evidenceIdOf(inspected.structuredContent); + const evidence = await connected.client.readResource({ + uri: `rea://evidence/${evidenceId}`, + }); + expect(evidence.contents[0]).toEqual( + expect.objectContaining({ text: expect.stringContaining(evidenceId) }), + ); + }, + INTEGRATION_TEST_TIMEOUT_MS, + ); + + it("denies origins outside the administrator ceiling before CDP attach", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const connected = await connectBrowser(browser); + const denied = await connected.client.callTool({ + name: "inspect_web_page", + arguments: { + cdp_endpoint: browser.endpoint, + allowed_origins: ["https://unapproved.example.test"], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + }, + }); + expect(denied.isError).toBe(true); + expect(denied.structuredContent).toMatchObject({ + error: { + code: "permission_required", + details: { + missing: { origins: ["https://unapproved.example.test"] }, + }, + }, + }); + expect(browser.commands).toHaveLength(0); + }); +}); + +const connectBrowser = async (browser: FakeCdpBrowser) => { + const config = parseConfig({ + REA_BROWSER_OBSERVE_ENABLED: "true", + REA_BROWSER_CDP_ENDPOINTS_JSON: JSON.stringify([browser.endpoint]), + REA_BROWSER_ALLOWED_ORIGINS_JSON: JSON.stringify([browser.allowedOrigin]), + }); + if (!config.ok) throw config.error; + const authority = await loadConfiguredPermissionAuthority(config.value); + if (!authority.ok) throw authority.error; + const session = new BinarySession(() => ({ + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(), + })); + const server = createServer(session, session, { + browserObservation: new CdpBrowserProvider(), + permissionAuthority: authority.value, + availabilityPolicy: () => ({ + processCaptureEnabled: false, + evidenceFileRoots: 0, + browserObservationEnabled: true, + }), + }); + const client = new Client({ name: "browser-mcp-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push(client, server, session); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { client }; +}; + +const evidenceIdOf = (value: unknown): string => { + if ( + typeof value !== "object" || + value === null || + !("evidence_id" in value) || + typeof value.evidence_id !== "string" + ) + throw new TypeError("Missing browser evidence ID"); + return value.evidence_id; +}; diff --git a/tests/browserObservation.test.ts b/tests/browserObservation.test.ts new file mode 100644 index 000000000..2d6d41fa6 --- /dev/null +++ b/tests/browserObservation.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; + +import { + browserEndpointSchema, + browserOriginSchema, + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, + sanitizeBrowserUrl, +} from "../src/domain/browserObservation.js"; + +describe("browser observation contracts", () => { + it("normalizes exact HTTP origins without accepting broader URL scopes", () => { + expect(browserOriginSchema.parse("https://Example.COM:443")).toBe( + "https://example.com", + ); + expect(browserOriginSchema.parse("http://example.com:80")).toBe( + "http://example.com", + ); + for (const value of [ + "https://*.example.com", + "https://example.com/path", + "https://example.com/?token=secret", + "https://user:pass@example.com", + "file:///tmp/page.html", + ]) + expect(browserOriginSchema.safeParse(value).success, value).toBe(false); + }); + + it("accepts only literal loopback HTTP CDP endpoints", () => { + expect(browserEndpointSchema.parse("http://127.0.0.1:9222")).toBe( + "http://127.0.0.1:9222", + ); + expect(browserEndpointSchema.parse("http://[::1]:9222")).toBe( + "http://[::1]:9222", + ); + for (const value of [ + "http://localhost:9222", + "http://192.168.1.2:9222", + "https://127.0.0.1:9222", + "http://127.0.0.1:9222/json/list", + "http://user:pass@127.0.0.1:9222", + ]) + expect(browserEndpointSchema.safeParse(value).success, value).toBe(false); + }); + + it("applies conservative bounded defaults to public tool input", () => { + expect( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: ["https://app.example.test"], + approved: true, + }), + ).toEqual({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: ["https://app.example.test"], + approved: true, + offset: 0, + limit: 100, + }); + expect( + inspectWebPageInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: ["https://app.example.test"], + target_id: "page-1", + approved: true, + }), + ).toMatchObject({ + observation_ms: 500, + include_script_sources: false, + include_storage_keys: false, + limits: { + max_frames: 200, + max_dom_nodes: 2_000, + max_ax_nodes: 2_000, + max_scripts: 200, + max_resources: 2_000, + max_workers: 500, + max_storage_keys: 1_000, + max_script_source_bytes: 1_048_576, + max_total_script_source_bytes: 4_194_304, + max_network_events: 1_000, + max_console_events: 200, + max_websocket_events: 500, + }, + }); + }); + + it("removes credentials, query values, and fragments from observed URLs", () => { + expect( + sanitizeBrowserUrl( + "https://user:pass@app.example.test/path?token=secret&mode=full#part", + ), + ).toEqual({ + url: "https://app.example.test/path?mode=%5BREDACTED%5D&token=%5BREDACTED%5D", + origin: "https://app.example.test", + query_parameter_names: ["mode", "token"], + redacted: true, + }); + }); + + it("bounds retained URL and query-name metadata", () => { + const longName = `a${"x".repeat(400)}`; + const parameters = [ + `${longName}=secret`, + ...Array.from( + { length: 300 }, + (_value, index) => `k${String(index).padStart(3, "0")}=secret`, + ), + ].join("&"); + const sanitized = sanitizeBrowserUrl( + `https://app.example.test/path?${parameters}`, + ); + expect(sanitized.query_parameter_names).toHaveLength(256); + expect( + sanitized.query_parameter_names.every((name) => name.length <= 256), + ).toBe(true); + expect(sanitized.url).not.toContain("secret"); + + const oversized = sanitizeBrowserUrl( + `https://app.example.test/${"p".repeat(70_000)}`, + ); + expect(oversized.origin).toBe("https://app.example.test"); + expect(oversized.url.length).toBeLessThanOrEqual(131_072); + expect(oversized.redacted).toBe(true); + }); +}); diff --git a/tests/cdpBrowserProvider.test.ts b/tests/cdpBrowserProvider.test.ts new file mode 100644 index 000000000..5db899f8c --- /dev/null +++ b/tests/cdpBrowserProvider.test.ts @@ -0,0 +1,497 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { CdpBrowserProvider } from "../src/browser/CdpBrowserProvider.js"; +import { + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, + webPageInspectionSchema, +} from "../src/domain/browserObservation.js"; +import { + startFakeCdpBrowser, + type FakeCdpBrowser, +} from "./fixtures/fakeCdpBrowser.js"; + +describe("CdpBrowserProvider", () => { + const browsers: FakeCdpBrowser[] = []; + + afterEach(async () => { + await Promise.all( + browsers.splice(0).map(async (browser) => browser.close()), + ); + }); + + it("lists only exact-origin pages and sanitizes URLs", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const provider = new CdpBrowserProvider(); + const result = await provider.listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + approved: true, + }), + ); + + if (!result.ok) throw result.error; + expect(result.value.targets.items).toEqual([ + expect.objectContaining({ + target_id: "allowed-page", + origin: browser.allowedOrigin, + url: `${browser.allowedOrigin}/app?token=%5BREDACTED%5D`, + }), + ]); + expect(result.value.excluded).toEqual({ + disallowed_origin: 1, + unsupported_url: 1, + non_page: 1, + }); + expect(JSON.stringify(result.value)).not.toContain("forbidden"); + expect(JSON.stringify(result.value)).not.toContain("page-secret"); + }); + + it("captures bounded passive evidence without retaining sensitive values", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const provider = new CdpBrowserProvider(); + const result = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + include_storage_keys: true, + }), + ); + + if (!result.ok) throw result.error; + expect(() => webPageInspectionSchema.parse(result.value)).not.toThrow(); + expect(result.value.frames).toHaveLength(1); + expect(result.value.dom.nodes).toHaveLength(2); + expect(result.value.dom.nodes[1]?.attribute_names).toEqual(["token"]); + expect(result.value.accessibility.nodes[0]?.name).toBe("Submit report"); + expect(result.value.scripts.items).toHaveLength(1); + expect(result.value.scripts.items[0]?.source).toEqual({ + included: false, + reason: "source capture was not approved", + }); + expect(result.value.resources).toHaveLength(1); + expect(result.value.network.requests).toHaveLength(1); + expect(result.value.network.websocket_events).toEqual([ + { + request_id: "websocket-1", + direction: "sent", + opcode: 1, + payload_bytes: Buffer.byteLength("websocket-secret"), + }, + ]); + expect(result.value.console.events[0]?.argument_types).toEqual(["string"]); + expect(result.value.workers).toHaveLength(1); + expect(result.value.storage).toEqual( + expect.objectContaining({ + local_storage_keys: ["public-key"], + session_storage_keys: ["public-key"], + indexed_db_names: ["app-db"], + cache_names: ["assets-v1"], + values_redacted: true, + }), + ); + const serialized = JSON.stringify(result.value); + for (const secret of [ + "page-secret", + "frame-secret", + "dom-secret", + "forbidden", + "resource-secret", + "script-secret", + "inline-secret", + "map-secret", + "network-secret", + "request-secret", + "request-body-secret", + "response-secret", + "websocket-secret", + "websocket-url-secret", + "console-secret", + "unknown-origin-console-secret", + "unknown-console-value-secret", + "storage-secret", + "secret-id", + ]) + expect(serialized).not.toContain(secret); + const methods = browser.commands.map((command) => command.method); + expect(methods).toContain("Target.detachFromTarget"); + expect(methods).not.toContain("Browser.close"); + expect(methods).not.toContain("Target.closeTarget"); + expect(methods).not.toContain("Runtime.evaluate"); + }); + + it("includes allowed script source only after explicit approval", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const provider = new CdpBrowserProvider(); + const result = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + include_script_sources: true, + }), + ); + + if (!result.ok) throw result.error; + expect(result.value.scripts.items[0]?.source).toEqual( + expect.objectContaining({ + included: true, + content: "export const observed = 'source-secret';", + bytes: 40, + }), + ); + expect(browser.commands.map((command) => command.method)).toContain( + "Debugger.getScriptSource", + ); + }); + + it("rejects missing and disallowed page targets before attaching", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const provider = new CdpBrowserProvider(); + for (const targetId of ["missing", "disallowed-page"]) { + const result = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: targetId, + approved: true, + observation_ms: 0, + }), + ); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error).toMatchObject({ + _tag: "BrowserObservationError", + reason: + targetId === "missing" ? "target_not_found" : "target_not_allowed", + }); + } + expect(browser.commands).toHaveLength(0); + }); + + it("returns typed errors for malformed discovery and disconnects", async () => { + const malformed = await startFakeCdpBrowser({ malformedDiscovery: true }); + browsers.push(malformed); + const malformedResult = await new CdpBrowserProvider().listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: malformed.endpoint, + allowed_origins: [malformed.allowedOrigin], + approved: true, + }), + ); + expect(malformedResult).toMatchObject({ + ok: false, + error: { + _tag: "BrowserObservationError", + reason: "invalid_endpoint_response", + }, + }); + + const disconnecting = await startFakeCdpBrowser({ + closeOnMethod: "Page.getFrameTree", + }); + browsers.push(disconnecting); + const disconnectedResult = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: disconnecting.endpoint, + allowed_origins: [disconnecting.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + }), + ); + expect(disconnectedResult).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "disconnected" }, + }); + }); + + it("reports configured truncation without over-reading DOM or script source", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const result = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + include_script_sources: true, + limits: { + max_frames: 1, + max_dom_nodes: 1, + max_ax_nodes: 1, + max_scripts: 1, + max_resources: 1, + max_workers: 1, + max_storage_keys: 1, + max_script_source_bytes: 10, + max_total_script_source_bytes: 10, + max_network_events: 1, + max_console_events: 1, + max_websocket_events: 1, + }, + }), + ); + if (!result.ok) throw result.error; + expect(result.value.completeness).toMatchObject({ + status: "truncated", + truncated_sections: ["dom", "script_sources"], + }); + expect(result.value.dom).toMatchObject({ total_nodes: 2 }); + expect(result.value.dom.nodes).toHaveLength(1); + expect(result.value.scripts.items[0]?.source).toMatchObject({ + included: false, + reason: "declared script length exceeds per-script limit", + }); + expect(browser.commands.map(({ method }) => method)).not.toContain( + "Debugger.getScriptSource", + ); + }); + + it("bounds frames, resources, workers, accessibility, and storage inventories", async () => { + const browser = await startFakeCdpBrowser({ extraCollections: true }); + browsers.push(browser); + const result = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + include_storage_keys: true, + limits: { + max_frames: 1, + max_dom_nodes: 2, + max_ax_nodes: 1, + max_scripts: 1, + max_resources: 1, + max_workers: 1, + max_storage_keys: 1, + max_script_source_bytes: 10, + max_total_script_source_bytes: 10, + max_network_events: 1, + max_console_events: 1, + max_websocket_events: 1, + }, + }), + ); + if (!result.ok) throw result.error; + expect(result.value.completeness).toMatchObject({ + status: "truncated", + truncated_sections: [ + "accessibility", + "frames", + "resources", + "storage_keys", + "workers", + ], + }); + expect(result.value.frames).toHaveLength(1); + expect(result.value.resources).toHaveLength(1); + expect(result.value.workers).toHaveLength(1); + expect(result.value.accessibility).toMatchObject({ total_nodes: 2 }); + expect(result.value.accessibility.nodes).toHaveLength(1); + expect(result.value.storage.local_storage_keys).toHaveLength(1); + expect(result.value.storage.session_storage_keys).toHaveLength(1); + expect(result.value.storage.indexed_db_names).toHaveLength(1); + expect(result.value.storage.cache_names).toHaveLength(1); + }); + + it("degrades optional domains but propagates protocol and payload failures", async () => { + const optional = await startFakeCdpBrowser({ + unsupportedMethods: ["Accessibility.getFullAXTree"], + }); + browsers.push(optional); + const degraded = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: optional.endpoint, + allowed_origins: [optional.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + }), + ); + if (!degraded.ok) throw degraded.error; + expect(degraded.value.accessibility.nodes).toEqual([]); + expect(degraded.value.limitations).toContain( + "Accessibility.getFullAXTree was unavailable from this browser target.", + ); + + for (const [options, reason] of [ + [{ oversizedDiscovery: true }, "payload_limit"], + [{ invalidBrowserWebSocket: true }, "invalid_endpoint_response"], + ] as const) { + const failed = await startFakeCdpBrowser(options); + browsers.push(failed); + const result = await new CdpBrowserProvider().listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: failed.endpoint, + allowed_origins: [failed.allowedOrigin], + approved: true, + }), + ); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error).toMatchObject({ + _tag: "BrowserObservationError", + reason, + }); + } + + const malformed = await startFakeCdpBrowser({ + malformedMessageOnMethod: "Page.getFrameTree", + }); + browsers.push(malformed); + const protocol = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: malformed.endpoint, + allowed_origins: [malformed.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + }), + ); + expect(protocol).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "protocol_error" }, + }); + + const malformedEvent = await startFakeCdpBrowser({ + malformedEventOnMethod: "Debugger.enable", + }); + browsers.push(malformedEvent); + const eventProtocol = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: malformedEvent.endpoint, + allowed_origins: [malformedEvent.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + }), + ); + expect(eventProtocol).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "protocol_error" }, + }); + }); + + it("cancels observation and still detaches without closing the page", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const controller = new AbortController(); + const pending = new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 1_000, + }), + { signal: controller.signal }, + ); + setTimeout(() => controller.abort(), 20); + const result = await pending; + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + const methods = browser.commands.map(({ method }) => method); + expect(methods).toContain("Target.detachFromTarget"); + expect(methods).not.toContain("Target.closeTarget"); + expect(methods).not.toContain("Browser.close"); + }); + + it("waits for navigation commit and rechecks the attached main-frame origin", async () => { + const transitioning = await startFakeCdpBrowser({ + transitionalFrameReads: 2, + }); + browsers.push(transitioning); + const input = (browser: FakeCdpBrowser) => + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + approved: true, + observation_ms: 0, + }); + const committed = await new CdpBrowserProvider().inspectPage( + input(transitioning), + ); + if (!committed.ok) throw committed.error; + expect(committed.value.target.origin).toBe(transitioning.allowedOrigin); + expect( + transitioning.commands.filter( + ({ method }) => method === "Page.getFrameTree", + ), + ).toHaveLength(5); + + const navigated = await startFakeCdpBrowser({ + attachedFrameUrl: "https://unapproved.example.test/after-attach", + }); + browsers.push(navigated); + const denied = await new CdpBrowserProvider().inspectPage(input(navigated)); + expect(denied).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "target_not_allowed" }, + }); + const methods = navigated.commands.map(({ method }) => method); + expect(methods).toContain("Target.detachFromTarget"); + expect(methods).not.toContain("DOMSnapshot.captureSnapshot"); + expect(methods).not.toContain("Storage.getUsageAndQuota"); + + const duringObservation = await startFakeCdpBrowser({ + navigateDuringObservationUrl: + "https://unapproved.example.test/during-observation", + }); + browsers.push(duringObservation); + const interrupted = await new CdpBrowserProvider().inspectPage( + input(duringObservation), + ); + expect(interrupted).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "target_not_allowed" }, + }); + expect( + duringObservation.commands.map(({ method }) => method), + ).not.toContain("Accessibility.getFullAXTree"); + + const captureOptions: { navigateDuringCaptureUrl?: string } = {}; + const duringCapture = await startFakeCdpBrowser(captureOptions); + captureOptions.navigateDuringCaptureUrl = `${duringCapture.allowedOrigin}/changed`; + browsers.push(duringCapture); + const discarded = await new CdpBrowserProvider().inspectPage( + input(duringCapture), + ); + expect(discarded).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "target_changed" }, + }); + expect(duringCapture.commands.map(({ method }) => method)).toContain( + "Target.detachFromTarget", + ); + + const crossOriginCapture = await startFakeCdpBrowser({ + navigateDuringCaptureUrl: + "https://unapproved.example.test/during-capture", + }); + browsers.push(crossOriginCapture); + const rejectedCapture = await new CdpBrowserProvider().inspectPage( + input(crossOriginCapture), + ); + expect(rejectedCapture).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "target_not_allowed" }, + }); + }); +}); diff --git a/tests/cdpCaptureDocuments.test.ts b/tests/cdpCaptureDocuments.test.ts new file mode 100644 index 000000000..1df0ec5e5 --- /dev/null +++ b/tests/cdpCaptureDocuments.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; + +import { + captureDom, + captureFrames, + captureResources, +} from "../src/browser/CdpCaptureDocuments.js"; +import { inspectWebPageInputSchema } from "../src/domain/browserObservation.js"; + +const origin = "https://app.example.test"; + +describe("CDP document capture", () => { + it("walks deeply nested frame trees iteratively and retains a bounded prefix", () => { + let frameTree: Record = { + frame: { id: "frame-0", url: `${origin}/0` }, + resources: [{ url: `${origin}/0.js`, type: "Script" }], + }; + for (let index = 1; index < 2_000; index += 1) { + frameTree = { + frame: { + id: `frame-${String(index)}`, + url: `${origin}/${String(index)}`, + }, + resources: [{ url: `${origin}/${String(index)}.js`, type: "Script" }], + childFrames: [frameTree], + }; + } + const result = { frameTree }; + const frames = captureFrames(result, new Set([origin]), 3); + const resources = captureResources(result, new Set([origin]), 2); + expect(frames).toMatchObject({ total: 2_000 }); + expect(frames.items).toHaveLength(3); + expect(resources).toMatchObject({ total: 2_000 }); + expect(resources.items).toHaveLength(2); + }); + + it("rebases parent indexes when allowed DOM documents are combined", () => { + const input = inspectWebPageInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: [origin], + target_id: "page-1", + approved: true, + }); + const snapshot = { + strings: [`${origin}/main`, `${origin}/frame`, "#document", "DIV", ""], + documents: [ + { + documentURL: 0, + nodes: { + nodeType: [9, 1], + nodeName: [2, 3], + nodeValue: [4, 4], + parentIndex: [-1, 0], + attributes: [[], []], + }, + }, + { + documentURL: 1, + nodes: { + nodeType: [9, 1], + nodeName: [2, 3], + nodeValue: [4, 4], + parentIndex: [-1, 0], + attributes: [[], []], + }, + }, + ], + }; + const capture = captureDom(snapshot, new Set([origin]), input); + expect(capture.nodes.map((node) => node.parent_index)).toEqual([ + -1, 0, -1, 2, + ]); + }); +}); diff --git a/tests/cdpConnection.test.ts b/tests/cdpConnection.test.ts new file mode 100644 index 000000000..a957f6bad --- /dev/null +++ b/tests/cdpConnection.test.ts @@ -0,0 +1,56 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { CdpConnection } from "../src/browser/CdpConnection.js"; +import { + startFakeCdpBrowser, + type FakeCdpBrowser, +} from "./fixtures/fakeCdpBrowser.js"; + +describe("CDP connection", () => { + const browsers: FakeCdpBrowser[] = []; + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all( + browsers.splice(0).map(async (browser) => browser.close()), + ); + }); + + it("correlates concurrent command responses over a real WebSocket", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const connection = await CdpConnection.connect(browser.browserWebSocketUrl); + try { + const [attached, frames] = await Promise.all([ + connection.send("Target.attachToTarget"), + connection.send("Page.getFrameTree"), + ]); + expect(attached).toMatchObject({ sessionId: "session-1" }); + expect(frames).toMatchObject({ + frameTree: { frame: { id: "frame-main" } }, + }); + } finally { + await connection.close(); + } + }); + + it("returns a typed timeout for an unresponsive CDP command", async () => { + const browser = await startFakeCdpBrowser({ hangOnMethod: "Page.enable" }); + browsers.push(browser); + const connection = await CdpConnection.connect(browser.browserWebSocketUrl); + vi.useFakeTimers(); + try { + const pending = connection.send("Page.enable"); + const assertion = expect(pending).rejects.toMatchObject({ + _tag: "AnalysisTimeoutError", + operation: "inspect_web_page", + timeoutMs: 5_000, + }); + await vi.advanceTimersByTimeAsync(5_000); + await assertion; + } finally { + vi.useRealTimers(); + await connection.close(); + } + }); +}); diff --git a/tests/config.test.ts b/tests/config.test.ts index 369e0bafb..575ef99c2 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -26,6 +26,9 @@ describe("runtime configuration", () => { maxDepth: 32, maxPathBytes: 4_096, }); + expect(result.value.browserObservationEnabled).toBe(false); + expect(result.value.browserCdpEndpoints).toEqual([]); + expect(result.value.browserAllowedOrigins).toEqual([]); } }); @@ -157,4 +160,55 @@ describe("runtime configuration", () => { ).toBe(false); }, ); + + it("builds an exact-origin browser observation ceiling only when enabled", () => { + const result = parseConfig({ + REA_BROWSER_OBSERVE_ENABLED: "true", + REA_BROWSER_CDP_ENDPOINTS_JSON: '["http://127.0.0.1:9222"]', + REA_BROWSER_ALLOWED_ORIGINS_JSON: + '["https://app.example.test", "http://127.0.0.1:3000"]', + }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value.permissionCeilings).toContainEqual({ + capability: "browser_observe", + roots: [], + executables: [], + environment_names: [], + origins: [ + "http://127.0.0.1:9222", + "http://127.0.0.1:3000", + "https://app.example.test", + ], + network: "external", + mount: false, + }); + expect(result.value.administratorPermissionGrants).toContainEqual( + expect.objectContaining({ + capability: "browser_observe", + lifetime: "administrator", + }), + ); + }); + + it.each([ + '["http://localhost:9222"]', + '["http://192.168.1.5:9222"]', + '["https://127.0.0.1:9222"]', + ])("rejects unsafe browser CDP endpoint scopes: %s", (encoded) => { + expect(parseConfig({ REA_BROWSER_CDP_ENDPOINTS_JSON: encoded }).ok).toBe( + false, + ); + }); + + it.each([ + '["https://*.example.test"]', + '["https://app.example.test/private"]', + '["https://user:pass@app.example.test"]', + '["file:///tmp/application.html"]', + ])("rejects non-exact browser origin scopes: %s", (encoded) => { + expect(parseConfig({ REA_BROWSER_ALLOWED_ORIGINS_JSON: encoded }).ok).toBe( + false, + ); + }); }); diff --git a/tests/contractSnapshot.test.ts b/tests/contractSnapshot.test.ts index e8897606c..6de2985c0 100644 --- a/tests/contractSnapshot.test.ts +++ b/tests/contractSnapshot.test.ts @@ -8,6 +8,7 @@ import { } from "../src/contracts/toolContracts.js"; import { NATIVE_TOOL_CONTRACTS } from "../src/contracts/nativeToolContracts.js"; import { ARTIFACT_TOOL_CONTRACTS } from "../src/contracts/artifactToolContracts.js"; +import { BROWSER_TOOL_CONTRACTS } from "../src/contracts/browserToolContracts.js"; import { enhancedOutputSchemas, officialOutputSchemas, @@ -34,8 +35,9 @@ describe("tool contract surface", () => { ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, + ...BROWSER_TOOL_CONTRACTS, ]; - expect(contracts).toHaveLength(50); + expect(contracts).toHaveLength(52); for (const contract of contracts) { const inputSchema = z.toJSONSchema(contract.inputSchema, { target: "draft-07", @@ -50,7 +52,9 @@ describe("tool contract surface", () => { expect(contract.annotations).toMatchObject({ idempotentHint: true, }); - expect(contract.annotations.openWorldHint).toBe(false); + expect(contract.annotations.openWorldHint).toBe( + contract.kind === "browser-provider", + ); expect(typeof contract.annotations.readOnlyHint).toBe("boolean"); expect(typeof contract.annotations.destructiveHint).toBe("boolean"); expect(contract.examples).toHaveLength(1); @@ -136,15 +140,16 @@ describe("tool contract surface", () => { } }); - it("publishes no unconstrained output-schema holes across all 68 tools", () => { + it("publishes no unconstrained output-schema holes across all 70 tools", () => { const contracts = [ ...OFFICIAL_TOOL_CONTRACTS, ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, + ...BROWSER_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ]; - expect(contracts).toHaveLength(68); + expect(contracts).toHaveLength(70); for (const contract of contracts) { const schema = z.toJSONSchema(contract.outputSchema, { target: "draft-07", diff --git a/tests/enhancedTools.test.ts b/tests/enhancedTools.test.ts index f7cceb4f5..f48f7228e 100644 --- a/tests/enhancedTools.test.ts +++ b/tests/enhancedTools.test.ts @@ -183,7 +183,7 @@ describe("enhanced MCP tools", () => { it("lists the complete 33 plus 10 surface", async () => { const client = await connect(); const listed = await client.listTools(); - expect(listed.tools).toHaveLength(50); + expect(listed.tools).toHaveLength(52); expect( listed.tools .map(({ name }) => name) diff --git a/tests/errors.test.ts b/tests/errors.test.ts index afe287c47..c49f7d7b7 100644 --- a/tests/errors.test.ts +++ b/tests/errors.test.ts @@ -9,6 +9,7 @@ import { AnalysisTimeoutError, ArtifactOperationError, BinaryTargetError, + BrowserObservationError, ConfigurationError, EvidenceFileError, EvidenceIntegrityError, @@ -88,6 +89,11 @@ describe("analysis error projection", () => { ProviderAdapterError: new ProviderAdapterError("fixture", "overview", { cause: secretCause, }), + BrowserObservationError: new BrowserObservationError( + "inspect_web_page", + "endpoint_unreachable", + { cause: secretCause }, + ), ArtifactOperationError: new ArtifactOperationError( "inventory_artifact", "integrity", @@ -147,7 +153,7 @@ describe("analysis error projection", () => { } satisfies Readonly>; const errors = Object.values(byTag); const projected = errors.map(projectAnalysisError); - expect(projected).toHaveLength(25); + expect(projected).toHaveLength(26); const serialized = JSON.stringify(projected); for (const hidden of [ "secret-token", diff --git a/tests/fixtures/fakeCdpBrowser.ts b/tests/fixtures/fakeCdpBrowser.ts new file mode 100644 index 000000000..6183a728f --- /dev/null +++ b/tests/fixtures/fakeCdpBrowser.ts @@ -0,0 +1,544 @@ +import { createServer, type IncomingMessage } from "node:http"; + +import { WebSocket, WebSocketServer } from "ws"; + +interface FakeCdpCommand { + readonly id: number; + readonly method: string; + readonly params: Readonly>; + readonly sessionId?: string; +} + +export interface FakeCdpBrowser { + readonly endpoint: string; + readonly browserWebSocketUrl: string; + readonly allowedOrigin: string; + readonly commands: readonly FakeCdpCommand[]; + close(): Promise; +} + +interface FakeOptions { + readonly malformedDiscovery?: boolean; + readonly oversizedDiscovery?: boolean; + readonly invalidBrowserWebSocket?: boolean; + readonly malformedMessageOnMethod?: string; + readonly malformedEventOnMethod?: string; + readonly closeOnMethod?: string; + readonly hangOnMethod?: string; + readonly unsupportedMethods?: readonly string[]; + readonly transitionalFrameReads?: number; + readonly attachedFrameUrl?: string; + readonly navigateDuringObservationUrl?: string; + readonly navigateDuringCaptureUrl?: string; + readonly extraCollections?: boolean; +} + +/** Start a real HTTP/WebSocket fake at the same seams as a user-owned browser. */ +export const startFakeCdpBrowser = async ( + options: FakeOptions = {}, +): Promise => { + const commands: FakeCdpCommand[] = []; + const sockets = new Set(); + let frameTreeReads = 0; + let port = 0; + const http = createServer((request, response) => { + response.setHeader("content-type", "application/json"); + if (request.url === "/json/version") { + response.end( + options.oversizedDiscovery === true + ? " ".repeat(65 * 1_024) + : options.malformedDiscovery === true + ? "{not-json" + : JSON.stringify({ + Browser: "FakeChrome/1.0", + "Protocol-Version": "1.3", + "User-Agent": "FakeChrome", + "V8-Version": "13.0", + "WebKit-Version": "fake-revision", + webSocketDebuggerUrl: + options.invalidBrowserWebSocket === true + ? `ws://localhost:${String(port)}/devtools/page/not-browser` + : `ws://localhost:${String(port)}/devtools/browser/fake`, + }), + ); + return; + } + if (request.url === "/json/list") { + response.end(JSON.stringify(targets(port, options))); + return; + } + response.statusCode = 404; + response.end("{}"); + }); + const webSockets = new WebSocketServer({ noServer: true }); + http.on("upgrade", (request, socket, head) => { + if (request.url !== "/devtools/browser/fake") { + socket.destroy(); + return; + } + webSockets.handleUpgrade(request, socket, head, (webSocket) => + webSockets.emit("connection", webSocket, request), + ); + }); + webSockets.on("connection", (socket: WebSocket, request: IncomingMessage) => { + void request; + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("message", (raw) => { + const command = parseCommand(raw.toString()); + commands.push(command); + if (options.closeOnMethod === command.method) { + socket.close(); + return; + } + if (options.hangOnMethod === command.method) return; + if (options.malformedMessageOnMethod === command.method) { + socket.send("{not-json"); + return; + } + if (options.unsupportedMethods?.includes(command.method) === true) { + socket.send( + JSON.stringify({ + id: command.id, + error: { code: -32_601, message: "Method not found" }, + }), + ); + return; + } + if (command.method === "Page.getFrameTree") frameTreeReads += 1; + socket.send( + JSON.stringify({ + id: command.id, + result: resultFor(command, port, options, frameTreeReads), + }), + ); + if (options.malformedEventOnMethod === command.method) + socket.send("{not-json"); + emitEvents(socket, command, port, options); + }); + }); + await new Promise((resolve, reject) => { + http.once("error", reject); + http.listen(0, "127.0.0.1", () => resolve()); + }); + const address = http.address(); + if (address === null || typeof address === "string") + throw new Error("Fake CDP server did not bind a TCP address"); + port = address.port; + const endpoint = `http://127.0.0.1:${String(port)}`; + return { + endpoint, + browserWebSocketUrl: `ws://127.0.0.1:${String(port)}/devtools/browser/fake`, + allowedOrigin: endpoint, + commands, + async close() { + for (const socket of sockets) socket.terminate(); + await new Promise((resolve) => webSockets.close(() => resolve())); + await new Promise((resolve, reject) => + http.close((error) => + error === undefined ? resolve() : reject(error), + ), + ); + }, + }; +}; + +const targets = ( + port: number, + options: FakeOptions = {}, +): readonly Record[] => [ + { + id: "allowed-page", + type: "page", + title: "Inspectable application", + url: `http://127.0.0.1:${String(port)}/app?token=page-secret#fragment`, + attached: false, + webSocketDebuggerUrl: `ws://localhost:${String(port)}/devtools/page/allowed-page`, + }, + { + id: "disallowed-page", + type: "page", + title: "Must not leak", + url: "https://private.example.test/app?token=forbidden", + attached: false, + }, + { + id: "unsupported-page", + type: "page", + title: "Internal page", + url: "chrome://settings/", + attached: false, + }, + { + id: "worker-1", + type: "service_worker", + title: "Worker", + url: `http://127.0.0.1:${String(port)}/worker.js?secret=worker`, + attached: false, + }, + ...(options.extraCollections === true + ? [ + { + id: "worker-2", + type: "shared_worker", + title: "Second worker", + url: `http://127.0.0.1:${String(port)}/worker-2.js`, + attached: false, + }, + ] + : []), +]; + +const parseCommand = (text: string): FakeCdpCommand => { + const value: unknown = JSON.parse(text); + if (typeof value !== "object" || value === null) + throw new TypeError("Expected CDP command object"); + if (!("id" in value) || typeof value.id !== "number") + throw new TypeError("Expected CDP command id"); + if (!("method" in value) || typeof value.method !== "string") + throw new TypeError("Expected CDP method"); + const params = + "params" in value && isRecord(value.params) ? value.params : {}; + return { + id: value.id, + method: value.method, + params, + ...("sessionId" in value && typeof value.sessionId === "string" + ? { sessionId: value.sessionId } + : {}), + }; +}; + +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const resultFor = ( + command: FakeCdpCommand, + port: number, + options: FakeOptions, + frameTreeReads: number, +): Readonly> => { + switch (command.method) { + case "Target.attachToTarget": + return { sessionId: "session-1" }; + case "Page.getFrameTree": + return frameTree( + port, + frameTreeReads <= (options.transitionalFrameReads ?? 0) + ? ":" + : options.attachedFrameUrl, + options.extraCollections === true, + ); + case "Page.getResourceTree": + return resourceTree(port, options.extraCollections === true); + case "DOMSnapshot.captureSnapshot": + return domSnapshot(port); + case "Accessibility.getFullAXTree": + return accessibilityTree(options.extraCollections === true); + case "Debugger.getScriptSource": + return { scriptSource: "export const observed = 'source-secret';" }; + case "Target.getTargets": + return { targetInfos: targets(port, options).map(endpointTargetToInfo) }; + case "Storage.getUsageAndQuota": + return { usage: 42, quota: 1_024, usageBreakdown: [] }; + case "DOMStorage.getDOMStorageItems": + return { + entries: + options.extraCollections === true + ? [ + ["public-key", "storage-secret"], + ["second-key", "second-secret"], + ] + : [["public-key", "storage-secret"]], + }; + case "IndexedDB.requestDatabaseNames": + return { + databaseNames: + options.extraCollections === true + ? ["app-db", "second-db"] + : ["app-db"], + }; + case "CacheStorage.requestCacheNames": + return { + caches: [ + { cacheName: "assets-v1", cacheId: "secret-id" }, + ...(options.extraCollections === true + ? [{ cacheName: "assets-v2", cacheId: "second-secret-id" }] + : []), + ], + }; + default: + return {}; + } +}; + +const endpointTargetToInfo = ( + target: Readonly>, +): Readonly> => ({ + targetId: target.id, + type: target.type, + title: target.title, + url: target.url, + attached: target.attached, +}); + +const frameTree = ( + port: number, + overrideUrl?: string, + extraCollections = false, +): Readonly> => ({ + frameTree: { + frame: { + id: "frame-main", + loaderId: "loader-main", + url: + overrideUrl ?? + `http://127.0.0.1:${String(port)}/app?token=frame-secret`, + }, + childFrames: [ + ...(extraCollections + ? [ + { + frame: { + id: "frame-child", + parentId: "frame-main", + loaderId: "loader-child", + url: `http://127.0.0.1:${String(port)}/child`, + }, + }, + ] + : []), + { + frame: { + id: "frame-private", + parentId: "frame-main", + loaderId: "loader-private", + url: "https://private.example.test/frame?secret=forbidden", + }, + }, + ], + }, +}); + +const resourceTree = ( + port: number, + extraCollections = false, +): Readonly> => ({ + frameTree: { + frame: { + id: "frame-main", + loaderId: "loader-main", + url: `http://127.0.0.1:${String(port)}/app`, + }, + resources: [ + { + url: `http://127.0.0.1:${String(port)}/app.js?api_key=resource-secret`, + type: "Script", + mimeType: "text/javascript", + contentSize: 128, + }, + ...(extraCollections + ? [ + { + url: `http://127.0.0.1:${String(port)}/app.css`, + type: "Stylesheet", + mimeType: "text/css", + contentSize: 64, + }, + ] + : []), + { + url: "https://private.example.test/private.js?secret=forbidden", + type: "Script", + mimeType: "text/javascript", + }, + ], + }, +}); + +const domSnapshot = (port: number): Readonly> => { + const strings = [ + `http://127.0.0.1:${String(port)}/app`, + "#document", + "", + "HTML", + "token", + "dom-secret", + "https://private.example.test/frame", + "PRIVATE-TEXT", + ]; + return { + strings, + documents: [ + { + documentURL: 0, + nodes: { + nodeType: [9, 1], + nodeName: [1, 3], + nodeValue: [2, 2], + parentIndex: [-1, 0], + attributes: [[], [4, 5]], + }, + }, + { + documentURL: 6, + nodes: { + nodeType: [3], + nodeName: [1], + nodeValue: [7], + parentIndex: [-1], + attributes: [[]], + }, + }, + ], + }; +}; + +const accessibilityTree = ( + extraCollections = false, +): Readonly> => ({ + nodes: [ + { + nodeId: "ax-1", + ignored: false, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Submit report" }, + description: { type: "computedString", value: "Sends the form" }, + }, + ...(extraCollections + ? [ + { + nodeId: "ax-2", + parentId: "ax-1", + ignored: false, + role: { type: "role", value: "link" }, + name: { type: "computedString", value: "Second action" }, + }, + ] + : []), + ], +}); + +const emitEvents = ( + socket: WebSocket, + command: FakeCdpCommand, + port: number, + options: FakeOptions, +): void => { + if (command.method === "Debugger.enable") { + event(socket, "Debugger.scriptParsed", command.sessionId, { + scriptId: "script-allowed", + url: `http://127.0.0.1:${String(port)}/app.js?token=script-secret`, + hash: "cdp-hash", + length: 40, + isModule: true, + scriptLanguage: "JavaScript", + sourceMapURL: `http://127.0.0.1:${String(port)}/app.js.map?token=map-secret`, + }); + event(socket, "Debugger.scriptParsed", command.sessionId, { + scriptId: "script-private", + url: "https://private.example.test/private.js?secret=forbidden", + hash: "private-hash", + length: 100, + isModule: false, + }); + event(socket, "Debugger.scriptParsed", command.sessionId, { + scriptId: "script-inline", + url: "", + hash: "inline-secret", + length: 100, + isModule: false, + }); + } + if (command.method === "Network.enable") { + if (options.navigateDuringObservationUrl !== undefined) + event(socket, "Page.frameNavigated", command.sessionId, { + frame: { + id: "frame-main", + url: options.navigateDuringObservationUrl, + }, + }); + const url = `http://127.0.0.1:${String(port)}/api?token=network-secret`; + event(socket, "Network.requestWillBeSent", command.sessionId, { + requestId: "request-1", + type: "Fetch", + request: { + url, + method: "POST", + headers: { Authorization: "Bearer request-secret" }, + postData: "request-body-secret", + }, + initiator: { + type: "script", + callFrames: [ + { url: `${url}#caller-secret`, lineNumber: 3, columnNumber: 5 }, + ], + }, + }); + event(socket, "Network.responseReceived", command.sessionId, { + requestId: "request-1", + response: { + status: 200, + mimeType: "application/json", + headers: { "Set-Cookie": "response-secret" }, + }, + }); + event(socket, "Network.loadingFinished", command.sessionId, { + requestId: "request-1", + encodedDataLength: 321, + }); + event(socket, "Network.webSocketCreated", command.sessionId, { + requestId: "websocket-1", + url: `ws://127.0.0.1:${String(port)}/live?token=websocket-url-secret`, + }); + event(socket, "Network.webSocketFrameSent", command.sessionId, { + requestId: "websocket-1", + response: { opcode: 1, payloadData: "websocket-secret" }, + }); + } + if (command.method === "Runtime.enable") { + event(socket, "Runtime.consoleAPICalled", command.sessionId, { + type: "log", + timestamp: 123, + args: [{ type: "string", value: "console-secret" }], + stackTrace: { + callFrames: [ + { + url: `http://127.0.0.1:${String(port)}/app.js?secret=console-url`, + lineNumber: 7, + columnNumber: 9, + }, + ], + }, + }); + event(socket, "Runtime.consoleAPICalled", command.sessionId, { + type: "unknown-origin-console-secret", + timestamp: 124, + args: [{ type: "string", value: "unknown-console-value-secret" }], + }); + } + if ( + command.method === "DOMSnapshot.captureSnapshot" && + options.navigateDuringCaptureUrl !== undefined + ) + event(socket, "Page.frameNavigated", command.sessionId, { + frame: { + id: "frame-main", + url: options.navigateDuringCaptureUrl, + }, + }); +}; + +const event = ( + socket: WebSocket, + method: string, + sessionId: string | undefined, + params: Readonly>, +): void => + socket.send( + JSON.stringify({ + method, + params, + ...(sessionId === undefined ? {} : { sessionId }), + }), + ); diff --git a/tests/integration/serverIntegration.test.ts b/tests/integration/serverIntegration.test.ts index 3de17cddd..e42faba03 100644 --- a/tests/integration/serverIntegration.test.ts +++ b/tests/integration/serverIntegration.test.ts @@ -127,7 +127,7 @@ describe("full MCP integration with multi-tool sequences", () => { }); }); - it("preserves the complete 68-tool inventory with a session", async () => { + it("preserves the complete 70-tool inventory with a session", async () => { const session = new BinarySession( (_path) => ({ @@ -150,7 +150,7 @@ describe("full MCP integration with multi-tool sequences", () => { await client.connect(clientTransport); const listed = await client.listTools(); - expect(listed.tools).toHaveLength(68); + expect(listed.tools).toHaveLength(70); const names = listed.tools.map((t) => t.name); expect(names).toContain("open_binary"); expect(names).toContain("close_binary"); @@ -353,12 +353,12 @@ describe("full MCP integration with multi-tool sequences", () => { }); }); - it("preserves the 50-tool inventory without a session", async () => { + it("preserves the 52-tool inventory without a session", async () => { const client = await connect({ execute: () => Promise.resolve(ok(null)), }); const listed = await client.listTools(); - expect(listed.tools).toHaveLength(50); + expect(listed.tools).toHaveLength(52); const names = listed.tools.map((t) => t.name); expect(names).toContain("binary_overview"); expect(names).toContain("batch_decompile"); diff --git a/tests/permissionPolicy.test.ts b/tests/permissionPolicy.test.ts index b3e372211..c1ffc1e34 100644 --- a/tests/permissionPolicy.test.ts +++ b/tests/permissionPolicy.test.ts @@ -88,6 +88,45 @@ describe("permission policy", () => { }); }); + it("authorizes browser observation only for exact declared origins", () => { + const scope = { + capability: "browser_observe" as const, + roots: [], + executables: [], + environment_names: [], + origins: ["http://127.0.0.1:9222", "https://app.example.test"], + network: "external" as const, + mount: false, + }; + const granted = grantPermission(createPermissionPolicy([scope]), { + ...scope, + grant_id: "browser-session", + lifetime: "session", + operation_identity: null, + expires_at: null, + }); + expect(granted.ok).toBe(true); + if (!granted.ok) return; + + expect( + evaluatePermission(granted.value, { + ...scope, + operation_identity: "inspect:page-1", + }), + ).toMatchObject({ allowed: true }); + expect( + evaluatePermission(granted.value, { + ...scope, + origins: ["http://127.0.0.1:9222", "https://cdn.example.test"], + operation_identity: "inspect:page-2", + }), + ).toMatchObject({ + allowed: false, + reason: "outside_administrator_ceiling", + missing: { origins: ["https://cdn.example.test"] }, + }); + }); + it("consumes once grants and applies revocation and ceiling reload immediately", () => { const ceiling = { capability: "evidence_write" as const, diff --git a/tests/promptMcp.test.ts b/tests/promptMcp.test.ts index 69fd8b28d..a95fc6c16 100644 --- a/tests/promptMcp.test.ts +++ b/tests/promptMcp.test.ts @@ -54,7 +54,7 @@ describe("guided prompts over MCP", () => { ), ); } - expect((await client.listTools()).tools).toHaveLength(68); + expect((await client.listTools()).tools).toHaveLength(70); const result = await client.getPrompt({ name: "investigate_feature", arguments: { diff --git a/tests/readmeConsistency.test.ts b/tests/readmeConsistency.test.ts index 1d57b0237..27e73e0ab 100644 --- a/tests/readmeConsistency.test.ts +++ b/tests/readmeConsistency.test.ts @@ -10,6 +10,7 @@ import { } from "../src/contracts/toolContracts.js"; import { NATIVE_TOOL_CONTRACTS } from "../src/contracts/nativeToolContracts.js"; import { ARTIFACT_TOOL_CONTRACTS } from "../src/contracts/artifactToolContracts.js"; +import { BROWSER_TOOL_CONTRACTS } from "../src/contracts/browserToolContracts.js"; const readmes = [ "README.md", @@ -24,14 +25,15 @@ const expectedToolCounts = [ ENHANCED_TOOL_CONTRACTS.length, NATIVE_TOOL_CONTRACTS.length, ARTIFACT_TOOL_CONTRACTS.length, + BROWSER_TOOL_CONTRACTS.length, SESSION_TOOL_CONTRACTS.length, ] as const; const toolCountsFromReadme = (content: string, path: string): number[] => { const lines = content.split(/\r?\n/u); - const headingIndex = lines.findIndex((line) => /^## .*68/u.test(line)); + const headingIndex = lines.findIndex((line) => /^## .*70/u.test(line)); if (headingIndex === -1) - throw new Error(`Missing 68-tool heading in ${path}`); + throw new Error(`Missing 70-tool heading in ${path}`); const counts: number[] = []; for (const line of lines.slice(headingIndex + 1)) { diff --git a/tests/runtime.test.ts b/tests/runtime.test.ts index bbb48f542..60a14c8ea 100644 --- a/tests/runtime.test.ts +++ b/tests/runtime.test.ts @@ -10,7 +10,7 @@ const fixturePath = fileURLToPath( ); describe("production stdio runtime", () => { - it("starts the built entrypoint, lists 68 tools, calls one, and shuts down", async () => { + it("starts the built entrypoint, lists 70 tools, calls one, and shuts down", async () => { const transport = new StdioClientTransport({ command: process.execPath, args: [mainPath], @@ -32,7 +32,7 @@ describe("production stdio runtime", () => { try { await client.connect(transport); const tools = await client.listTools(); - expect(tools.tools).toHaveLength(68); + expect(tools.tools).toHaveLength(70); const result = await client.callTool({ name: "current_document", arguments: {}, @@ -78,7 +78,7 @@ describe("production stdio runtime", () => { try { await client.connect(transport); - expect((await client.listTools()).tools).toHaveLength(68); + expect((await client.listTools()).tools).toHaveLength(70); } finally { await client.close(); await transport.close(); diff --git a/tests/serverIdentity.test.ts b/tests/serverIdentity.test.ts index 1530f98bb..3f60fe480 100644 --- a/tests/serverIdentity.test.ts +++ b/tests/serverIdentity.test.ts @@ -24,11 +24,11 @@ describe("server and catalog identity", () => { }); expect(PRODUCT_IDENTITY.packageVersion).toBe(packageJson.version); expect(SDK_IDENTITY.server).toBe("2.0.0-beta.4"); - expect(CLI_COMMAND_NAMES).toHaveLength(28); - expect(new Set(CLI_COMMAND_NAMES).size).toBe(28); + expect(CLI_COMMAND_NAMES).toHaveLength(30); + expect(new Set(CLI_COMMAND_NAMES).size).toBe(30); expect(CATALOG_IDENTITY.counts).toEqual({ - cli_commands: 28, - mcp_tools: 68, + cli_commands: 30, + mcp_tools: 70, mcp_prompts: 6, mcp_resources: 2, mcp_resource_templates: 7, @@ -162,7 +162,7 @@ describe("server and catalog identity", () => { expect(status.structuredContent).toMatchObject({ result: { server_identity: { - catalog: { counts: { mcp_tools: 68, cli_commands: 28 } }, + catalog: { counts: { mcp_tools: 70, cli_commands: 30 } }, alignment: { state: "mcp_server_restart_required" }, }, tool_availability: expect.arrayContaining([ @@ -176,6 +176,11 @@ describe("server and catalog identity", () => { available: false, reason: "policy_disabled", }), + expect.objectContaining({ + name: "inspect_web_page", + available: false, + reason: "policy_disabled", + }), ]), client_features: { elicitation_form: false, diff --git a/tests/sessionMcp.test.ts b/tests/sessionMcp.test.ts index ece3a8239..6bc756416 100644 --- a/tests/sessionMcp.test.ts +++ b/tests/sessionMcp.test.ts @@ -68,7 +68,7 @@ describe("target-free MCP lifecycle", () => { expect(text(before)).toBe( "No app is open. Ask the user which app to investigate, then call open_binary with its local path.", ); - expect((await mcp.listTools()).tools).toHaveLength(68); + expect((await mcp.listTools()).tools).toHaveLength(70); const deniedCapture = await mcp.callTool({ name: "capture_process_scenario", arguments: { diff --git a/tests/setupSkill.test.ts b/tests/setupSkill.test.ts index 00320f952..9e7d33747 100644 --- a/tests/setupSkill.test.ts +++ b/tests/setupSkill.test.ts @@ -27,7 +27,7 @@ describe("canonical skill transaction", () => { expect(await readFile(`${destination}.rea.backup`, "utf8")).toBe( "stale managed skill\n", ); - expect(await readFile(destination, "utf8")).toContain('version: "12"'); + expect(await readFile(destination, "utf8")).toContain('version: "13"'); expect(await readFile(destination, "utf8")).toContain( `tool_count: ${String(TOOL_CONTRACTS.length)}`, ); diff --git a/tests/toolContracts.test.ts b/tests/toolContracts.test.ts index d0ff3f8e0..fecebb73f 100644 --- a/tests/toolContracts.test.ts +++ b/tests/toolContracts.test.ts @@ -8,13 +8,18 @@ import { } from "../src/contracts/toolContracts.js"; import { ARTIFACT_TOOL_CONTRACTS } from "../src/contracts/artifactToolContracts.js"; import { NATIVE_TOOL_CONTRACTS } from "../src/contracts/nativeToolContracts.js"; +import { BROWSER_TOOL_CONTRACTS } from "../src/contracts/browserToolContracts.js"; describe("tool contract inventory", () => { - it("publishes 50 analysis contracts and eighteen session tools", () => { + it("publishes 52 analysis contracts and eighteen session tools", () => { expect(OFFICIAL_TOOL_CONTRACTS).toHaveLength(33); expect(ENHANCED_TOOL_CONTRACTS).toHaveLength(10); expect(NATIVE_TOOL_CONTRACTS).toHaveLength(5); expect(ARTIFACT_TOOL_CONTRACTS).toHaveLength(2); + expect(BROWSER_TOOL_CONTRACTS.map(({ name }) => name)).toEqual([ + "list_browser_targets", + "inspect_web_page", + ]); expect(SESSION_TOOL_CONTRACTS.map(({ name }) => name)).toEqual([ "open_binary", "close_binary", @@ -35,8 +40,8 @@ describe("tool contract inventory", () => { "update_unknown", "verify_unknown_resolution", ]); - expect(TOOL_CONTRACTS).toHaveLength(68); - expect(new Set(TOOL_CONTRACTS.map(({ name }) => name)).size).toBe(68); + expect(TOOL_CONTRACTS).toHaveLength(70); + expect(new Set(TOOL_CONTRACTS.map(({ name }) => name)).size).toBe(70); }); it("retains documented enhanced-tool limits at the input boundary", () => { diff --git a/typedoc.json b/typedoc.json index a64d7c90f..818089fc3 100644 --- a/typedoc.json +++ b/typedoc.json @@ -7,6 +7,7 @@ "src/domain", "src/contracts", "src/hopper", + "src/browser", "src/application", "src/server" ],