From 3f80b75708f88e51c64eb0122c178b96789c53b3 Mon Sep 17 00:00:00 2001 From: Riley Gramlich <101842921+rileygramlich@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:52:47 -0600 Subject: [PATCH 01/18] fix(source-maps): keep sources when sourcesContent length differs (#2873) ECMA-426 DecodeSourceMapSources reads a content slot only while its index is within sourcesContent, so a shorter array leaves trailing sources without content and a longer array adds no sources. Local application analysis and browser collection rejected any length mismatch and dropped every original from the map. The content helper now checks only entry types. Browser collection drops its separate check because the leaf schema already validates every supplied entry, and the raw records are still budgeted before parsing. Fixes #2821 --- .../javascript/JavaScriptArtifactAnalysis.ts | 4 +- src/domain/sourceMapContents.test.ts | 20 +++++--- src/domain/sourceMapContents.ts | 15 +++--- src/javascript/sourceMaps/SourceMapFormat.ts | 13 ------ .../browser/webSourceMapFetcher.test.ts | 45 +++++++++++++++++- .../javascriptArtifactReconstruction.test.ts | 46 ++++++++++++++++++- 6 files changed, 114 insertions(+), 29 deletions(-) diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index e61d3ae94..6b48b448c 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -553,10 +553,10 @@ const collectSourceMapOriginals = ( if (!Array.isArray(names)) return invalidSourceMap(file, "Source map has no sources array."); const contents = map.sourcesContent; - if (!hasValidSourceMapContents(names.length, contents)) + if (!hasValidSourceMapContents(contents)) return invalidSourceMap( file, - "Source map sourcesContent must contain one string or null entry per source.", + "Source map sourcesContent must be an array of strings or null entries.", ); if (map.sourceRoot != null && typeof map.sourceRoot !== "string") return invalidSourceMap( diff --git a/src/domain/sourceMapContents.test.ts b/src/domain/sourceMapContents.test.ts index 06f07197b..dd80f39fb 100644 --- a/src/domain/sourceMapContents.test.ts +++ b/src/domain/sourceMapContents.test.ts @@ -2,10 +2,18 @@ import { expect, it } from "vitest"; import { hasValidSourceMapContents } from "./sourceMapContents.js"; -it("accepts absent map-level source contents and aligned nullable slots", () => { - expect(hasValidSourceMapContents(2, undefined)).toBe(true); - expect(hasValidSourceMapContents(2, null)).toBe(true); - expect(hasValidSourceMapContents(2, [null, "retained source"])).toBe(true); - expect(hasValidSourceMapContents(2, ["retained source"])).toBe(false); - expect(hasValidSourceMapContents(2, [null, 42])).toBe(false); +it("accepts absent map-level source contents and nullable string slots", () => { + expect(hasValidSourceMapContents(undefined)).toBe(true); + expect(hasValidSourceMapContents(null)).toBe(true); + expect(hasValidSourceMapContents([null, "retained source"])).toBe(true); + expect(hasValidSourceMapContents([null, 42])).toBe(false); + expect(hasValidSourceMapContents("retained source")).toBe(false); +}); + +it("leaves sourcesContent length to source decoding (ECMA-426 DecodeSourceMapSources)", () => { + expect(hasValidSourceMapContents([])).toBe(true); + expect(hasValidSourceMapContents(["retained source"])).toBe(true); + expect( + hasValidSourceMapContents(["a", null, "unused trailing content"]), + ).toBe(true); }); diff --git a/src/domain/sourceMapContents.ts b/src/domain/sourceMapContents.ts index 5d55810a9..c6a621f1a 100644 --- a/src/domain/sourceMapContents.ts +++ b/src/domain/sourceMapContents.ts @@ -1,12 +1,15 @@ -/** Validate optional source-map content against its source inventory. */ -export const hasValidSourceMapContents = ( - sourceCount: number, - contents: unknown, -): boolean => +/** + * Validate optional source-map content against its source inventory. + * + * ECMA-426 DecodeSourceMapSources reads a content slot only while its index is + * within `sourcesContent`, so a shorter array leaves trailing sources without + * content and a longer array contributes no extra sources. Only the entry types + * are checked here. + */ +export const hasValidSourceMapContents = (contents: unknown): boolean => contents === undefined || contents === null || (Array.isArray(contents) && - contents.length === sourceCount && contents.every( (content: unknown) => content === null || typeof content === "string", )); diff --git a/src/javascript/sourceMaps/SourceMapFormat.ts b/src/javascript/sourceMaps/SourceMapFormat.ts index f8975c9b6..a2edcab52 100644 --- a/src/javascript/sourceMaps/SourceMapFormat.ts +++ b/src/javascript/sourceMaps/SourceMapFormat.ts @@ -1,6 +1,5 @@ import { z } from "zod"; import { WEB_SOURCE_MAP_LIMITS } from "../../domain/webSourceLocation.js"; -import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js"; import { sourceMapJsonText } from "../../domain/sourceMapEnvelope.js"; import type { EncodedSourceMapXInput } from "@jridgewell/trace-mapping"; @@ -267,18 +266,6 @@ const inspectLeaf = ( consumeRecords(budget, Array.isArray(rawNames) ? rawNames.length : 0); consumeRecords(budget, Array.isArray(rawContents) ? rawContents.length : 0); options.check?.(); - if ( - options.profile === "browser-collection" && - !hasValidSourceMapContents( - Array.isArray(rawSources) ? rawSources.length : 0, - rawContents, - ) - ) - throw new SourceMapFormatFailure( - "format", - "Source-map sourcesContent must align with its source declarations.", - ); - options.check?.(); const leaf = leafSchema.parse(node.value); const layout = inspectMappingLayout(leaf.mappings, options.check); consumeRecords(budget, layout.segments + layout.lines); diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 6dcabb75b..5fab66f58 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -810,7 +810,6 @@ describe("web source-map sourcesContent validation", () => { it.each([ ["a scalar sourcesContent value", { sourcesContent: "source" }], ["a non-string, non-null source entry", { sourcesContent: [42] }], - ["a sourcesContent array with the wrong length", { sourcesContent: [] }], ])("reports %s as invalid", async (_description, content) => { const result = await fetchWebSourceMaps([request], input(), undefined, { fetch: () => @@ -841,6 +840,50 @@ describe("web source-map sourcesContent validation", () => { }); }); + it.each([ + ["shorter", ["import './a.ts';"]], + ["longer", ["import './a.ts';", null, "unused trailing content"]], + ])( + "keeps every source declaration when sourcesContent is %s than sources", + async (_length, sourcesContent) => { + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => + Promise.resolve( + new Response( + JSON.stringify({ + version: 3, + sources: ["a.ts", "b.ts"], + sourcesContent, + names: [], + mappings: "", + }), + { status: 200 }, + ), + ), + }); + + expect(result.items[0]).toMatchObject({ + status: "included", + limitation: null, + original_sources: [ + { + source_index: 0, + declaration_index: 0, + source: `${origin}/assets/a.ts`, + artifact: { text: "import './a.ts';" }, + }, + { + source_index: 1, + declaration_index: 1, + source: `${origin}/assets/b.ts`, + artifact: null, + }, + ], + }); + expect(result.items[0]?.original_sources).toHaveLength(2); + }, + ); + it("accepts null source-map source contents", async () => { const result = await fetchWebSourceMaps([request], input(), undefined, { fetch: () => diff --git a/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts index aabfdd19b..1254d11dd 100644 --- a/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts @@ -135,7 +135,6 @@ it("retains every source declared by every included local source map", async () it.each([ ["a non-array sourcesContent value", { sourcesContent: "source" }], ["a non-string, non-null source entry", { sourcesContent: [42] }], - ["a source-content array with the wrong length", { sourcesContent: [] }], ])( "reports %s as an invalid local source map", async (_description, content) => { @@ -210,6 +209,51 @@ it("accepts omitted and explicitly null source-map source contents", async () => expect(result.graph.coverage.status).toBe("complete"); }); +it.each([ + ["shorter", ["export const answer = 42;"]], + ["longer", ["export const answer = 42;", null, "unused trailing content"]], +])( + "keeps every source declaration when sourcesContent is %s than sources", + async (length, sourcesContent) => { + const root = await createTestTempDirectory( + `rea-javascript-${length}-source-content-`, + ); + await writeFile( + join(root, "bundle.js.map"), + JSON.stringify({ + version: 3, + sources: ["a.ts", "b.ts"], + sourcesContent, + names: [], + mappings: "AAAA;ACAA", + }), + ); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + const originals = result.graph.nodes.filter( + ({ identity }) => identity.strategy === "source-map-original", + ); + + expect( + originals + .map(({ identity, observations }) => ({ + original_source: + "original_source" in identity ? identity.original_source : null, + source_index: observations[0]?.properties.source_index, + content_available: observations[0]?.properties.content_available, + })) + .sort( + (left, right) => + Number(left.source_index) - Number(right.source_index), + ), + ).toEqual([ + { original_source: "a.ts", source_index: 0, content_available: true }, + { original_source: "b.ts", source_index: 1, content_available: false }, + ]); + expect(result.graph.coverage.status).toBe("complete"); + }, +); + it("retains unnamed source declarations without shifting later source evidence", async () => { const root = await createTestTempDirectory("rea-javascript-unnamed-source-"); const text = JSON.stringify({ From d4f0c5e664e26fe7ed1566dcc4b85ea9b57d52da Mon Sep 17 00:00:00 2001 From: Gefrezim Date: Tue, 13 Oct 2026 16:53:35 +0100 Subject: [PATCH 02/18] fix(cli): start on Windows when USERPROFILE is empty (#2345) * fix(cli): start on Windows when USERPROFILE is empty Node's Windows home lookup throws ERR_SYSTEM_ERROR (uv_os_homedir ENOENT) for an empty USERPROFILE instead of consulting the account profile. Incur resolves the home while it is imported, so every command, including `rea --version`, stopped with a raw stack trace before REA ran. Treat the empty selection as unset in the launcher, before any runtime import, and cover startup with a compiled-CLI regression. Co-Authored-By: Claude Opus 5.5 * fix(process): report no remaining time after a provider deadline fires Node timers can fire while performance.now() is still a fraction of a millisecond short of the monotonic deadline. ProviderStartupDeadline then reported interruption "timeout" and an aborted signal while remainingMs() returned ceil(0.x) = 1, so polling loops guarded by remainingMs() > 0 (Hopper and Ghidra startup) could run past an elapsed deadline, and the real-timer test "aborts a bounded wait when its real monotonic deadline expires" failed intermittently in CI (expected 1 to be 0). Once the deadline's own timer has classified a timeout, report zero remaining time. A deterministic regression test fires the timer with the monotonic clock 0.4 ms short of the deadline. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- src/process/ProviderDeadline.test.ts | 22 ++++++++++++++ src/process/ProviderDeadline.ts | 3 ++ tests/boundary/cli/emptyHomeStartup.test.ts | 32 +++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 tests/boundary/cli/emptyHomeStartup.test.ts diff --git a/src/process/ProviderDeadline.test.ts b/src/process/ProviderDeadline.test.ts index 2ecd6402c..ca83d98ab 100644 --- a/src/process/ProviderDeadline.test.ts +++ b/src/process/ProviderDeadline.test.ts @@ -179,3 +179,25 @@ it("releases the timer and listener on dispose with a pending wait", async () => vi.useRealTimers(); } }); + +// Preserve the authored fractional-clock control while following the current +// owning deadline's re-arm contract for an early native timer callback. +it("reports zero after the re-armed deadline reaches a fractional early callback", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const now = vi.spyOn(performance, "now").mockReturnValue(1_000); + const deadline = new ProviderStartupDeadline(40); + try { + now.mockReturnValue(1_039.6); + await vi.advanceTimersByTimeAsync(40); + expect(deadline.signal.aborted).toBe(false); + expect(deadline.remainingMs()).toBe(1); + now.mockReturnValue(1_040); + await vi.advanceTimersByTimeAsync(1); + expect(deadline.signal.aborted).toBe(true); + expect(deadline.interruption).toBe("timeout"); + expect(deadline.remainingMs()).toBe(0); + } finally { + deadline.dispose(); + vi.useRealTimers(); + } +}); diff --git a/src/process/ProviderDeadline.ts b/src/process/ProviderDeadline.ts index 83e24a807..7681ecf62 100644 --- a/src/process/ProviderDeadline.ts +++ b/src/process/ProviderDeadline.ts @@ -79,6 +79,9 @@ export class ProviderStartupDeadline { /** Milliseconds still available to the next startup phase. */ remainingMs(): number { + // Node timers can fire a fraction of a millisecond before the monotonic + // deadline, so an elapsed timer must not report time left to spend. + if (this.#interruption === "timeout") return 0; return Math.max(0, Math.ceil(this.#deadlineAt - performance.now())); } diff --git a/tests/boundary/cli/emptyHomeStartup.test.ts b/tests/boundary/cli/emptyHomeStartup.test.ts new file mode 100644 index 000000000..f1c73734f --- /dev/null +++ b/tests/boundary/cli/emptyHomeStartup.test.ts @@ -0,0 +1,32 @@ +import { readFile } from "node:fs/promises"; + +import { expect } from "vitest"; +import { z } from "zod"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const packageSchema = z.object({ version: z.string() }); + +// Node's Windows home lookup throws for an empty USERPROFILE, so a dependency +// that resolves the home during import used to stop every command there. +cliTest("starts when the selected home is empty", async ({ cli }) => { + const { version } = packageSchema.parse( + JSON.parse(await readFile("package.json", "utf8")), + ); + const cwd = await createTestTempDirectory("rea-empty-home-startup-"); + const environment = { HOME: "", USERPROFILE: "" }; + + const reported = await cli.run({ + arguments: ["--version"], + cwd, + environment, + }); + expect(reported.stderr).toBe(""); + expect(reported.exitCode).toBe(0); + expect(reported.stdout.trim()).toBe(version); + + const help = await cli.run({ arguments: ["--help"], cwd, environment }); + expect(help.stderr).not.toContain("uv_os_homedir"); + expect(help.exitCode).toBe(0); +}); From 72b9ab4af16a4f19383aa6dde4b12f25fab7f59f Mon Sep 17 00:00:00 2001 From: hhang <17309584214@163.com> Date: Tue, 13 Oct 2026 23:59:49 +0800 Subject: [PATCH 03/18] fix(javascript): retain configuration reads before member access (#2756) Squash merge of https://github.com/morluto/rea/pull/2756. Original pull request submitted by @xu91102. Co-authored-by: hhang <169027046+xu91102@users.noreply.github.com> --- .../javascriptSemanticDataEffectHelpers.ts | 9 --- .../javascriptSemanticDataEffects.ts | 11 +-- .../javascriptConfigurationReadsCli.test.ts | 78 +++++++++++++++++++ 3 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 tests/acceptance/applications/javascriptConfigurationReadsCli.test.ts diff --git a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts index 5485fcde9..0f7a6d942 100644 --- a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts +++ b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts @@ -275,12 +275,3 @@ export const dataEffectMemberCallee = ( t.isMemberExpression(node.callee) || t.isOptionalMemberExpression(node.callee) ? node.callee : null; - -/** Read a parent member's object when present. */ -export const dataEffectMemberObject = ( - node: t.Node | null, -): t.Expression | t.Super | null => - node !== null && - (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) - ? node.object - : null; diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index 445cd2e5e..607843aab 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -24,7 +24,6 @@ import { dataEffectArgumentBindingId as argumentBindingId, dataEffectLiteralString as literalString, dataEffectMemberCallee as memberCallee, - dataEffectMemberObject as memberObject, dataEffectPrimitive as primitiveValue, isSemanticProcessObject as isProcessObject, outerDataEffectBinding as outerAssignedBinding, @@ -83,8 +82,8 @@ const collectConfigurations = ( context: TraversalContext, ): ConfigurationCandidate[] => { const output: ConfigurationCandidate[] = []; - traverseWithContext(program, context, (node, parent) => { - const details = configurationDetails(node, parent, context); + traverseWithContext(program, context, (node) => { + const details = configurationDetails(node, context); if (details === null) return; const operation: JavaScriptSemanticConfigurationOperation = { configId: `config:${details.kind}:${String(node.start ?? -1)}:${String(node.end ?? -1)}`, @@ -104,17 +103,13 @@ const collectConfigurations = ( const configurationDetails = ( node: t.Node, - parent: t.Node | null, context: TraversalContext, ): { readonly kind: "environment" | "argv" | "file"; readonly key: string | null; readonly resolution: "complete" | "partial"; } | null => { - if ( - (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) && - !(memberObject(parent) === node) - ) { + if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { const processMember = processConfigurationMember(node, context.state); if (processMember !== null) return processMember; } diff --git a/tests/acceptance/applications/javascriptConfigurationReadsCli.test.ts b/tests/acceptance/applications/javascriptConfigurationReadsCli.test.ts new file mode 100644 index 000000000..8b9ba1abc --- /dev/null +++ b/tests/acceptance/applications/javascriptConfigurationReadsCli.test.ts @@ -0,0 +1,78 @@ +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains configuration sources consumed by member access through %s", + async (surface, { cli, processes }) => { + const root = await createTestTempDirectory("rea-config-reads-"); + await writeFixtureFiles(root, { + "entry.mjs": [ + "const length = process.env.REA_ENDPOINT.length;", + "const endpoint = process.env.REA_ENDPOINT;", + "const argumentLength = process.argv[2].length;", + "const mode = process.argv[3].trim();", + "const optional = process.env.REA_MISSING?.length;", + "function shadow(process) { return process.env.LOCAL.length + process.argv[2].length; }", + 'const local = { env: { FAKE: "fake" }, argv: [0, 1, "abc"] };', + 'const control = shadow({ env: { LOCAL: "local" }, argv: [0, 1, "abc"] });', + "console.log(JSON.stringify([length, endpoint, argumentLength, mode, optional, control, local.env.FAKE.length]));", + ].join("\n"), + }); + const control = await processes.run( + process.execPath, + [join(root, "entry.mjs"), "abc", " mode "], + { env: { REA_ENDPOINT: "endpoint" }, timeoutMs: 15_000 }, + ); + expect(control.exitCode, control.stderr).toBe(0); + expect(JSON.parse(control.stdout)).toEqual([ + 8, + "endpoint", + 3, + "mode", + null, + 8, + 4, + ]); + const graph = (await analyzeJavaScriptDirectory(surface, cli, root)) + .semantic_graph; + const sources = graph.nodes.filter((node) => node.kind === "config-source"); + expect( + sources + .map((node) => [node.properties.source_kind, node.properties.key]) + .sort(), + ).toEqual([ + ["argv", "2"], + ["argv", "3"], + ["environment", "REA_ENDPOINT"], + ["environment", "REA_ENDPOINT"], + ["environment", "REA_MISSING"], + ]); + expect(sources.every((node) => node.properties.value === null)).toBe(true); + const endpoint = graph.nodes.find( + (node) => node.kind === "binding" && node.label === "endpoint", + ); + expect(endpoint).toBeDefined(); + for (const source of sources) { + const bindingTargets = graph.relations + .filter((relation) => relation.source_node_id === source.node_id) + .flatMap((relation) => + graph.nodes.filter( + (node) => + node.node_id === relation.target_node_id && + node.kind === "binding", + ), + ); + expect(bindingTargets.map((node) => node.node_id)).toEqual( + source.identity.source_range?.start.line === 2 + ? [endpoint?.node_id] + : [], + ); + } + }, +); From 7e3a19863835215491b55fba7127a78bad6a7709 Mon Sep 17 00:00:00 2001 From: Ganesh Date: Tue, 13 Oct 2026 16:06:45 +0000 Subject: [PATCH 04/18] fix(javascript): keep logical assignment results uncertain for unknown values (#2788) * fix(javascript): keep logical assignment results uncertain for unknown values * test(javascript): cover exact uninitialized logical assignments Source inspection and targeted formatting only. Tests, builds, lint, typechecks and provider workloads were not run at the user's explicit request. --------- Co-authored-by: Ganesh2325 <184471511+Ganesh2325@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- ...emanticAnalysis.logical_assignment.test.ts | 96 +++++++++++++++++++ .../javascript/javascriptSemanticAnalysis.ts | 19 +++- 2 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 src/domain/javascript/javascriptSemanticAnalysis.logical_assignment.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.logical_assignment.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.logical_assignment.test.ts new file mode 100644 index 000000000..255376cc1 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysis.logical_assignment.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +const observe = (source: string) => { + const ir = analyzeJavaScriptSemantics(source); + return { + binding: ir.bindings.find((item) => item.name === "x")?.value, + returned: ir.callables.find((item) => item.name === "read")?.returnSites[0] + ?.value, + }; +}; + +describe("JavaScript semantic analysis: logical assignment", () => { + it.each(["||=", "??=", "&&="])( + "keeps a parameter uncertain after %s", + (operator) => { + const { binding, returned } = observe( + `function read(x) { x ${operator} 2; return x; }`, + ); + expect(binding?.status).toBe("unknown"); + expect(returned?.status).toBe("unknown"); + }, + ); + + it.each(["||=", "??=", "&&="])( + "keeps a destructured parameter uncertain after %s", + (operator) => { + const { binding, returned } = observe( + `function read({ x }) { x ${operator} 2; return x; }`, + ); + expect(binding?.status).toBe("unknown"); + expect(returned?.status).toBe("unknown"); + }, + ); + + it.each(["||=", "??=", "&&="])( + "keeps a catch binding uncertain after %s", + (operator) => { + const { binding, returned } = observe( + `function read() { try { run(); } catch (x) { x ${operator} 2; return x; } }`, + ); + expect(binding?.status).toBe("unknown"); + expect(returned?.status).toBe("unknown"); + }, + ); + + it.each([ + ["let", "||="], + ["let", "??="], + ["var", "||="], + ["var", "??="], + ])( + "keeps an uninitialized %s local exact after %s", + (declaration, operator) => { + const { binding, returned } = observe( + `function read() { ${declaration} x; x ${operator} 2; return x; }`, + ); + expect(binding).toEqual({ status: "literal", value: 2 }); + expect(returned).toEqual({ status: "literal", value: 2 }); + }, + ); + + it("keeps an unconditional parameter assignment exact", () => { + const { binding, returned } = observe( + "function read(x) { x = 2; return x; }", + ); + expect(binding).toEqual({ status: "literal", value: 2 }); + expect(returned).toEqual({ status: "literal", value: 2 }); + }); + + it("keeps an unchanged parameter unknown", () => { + expect(observe("function read(x) { return x; }").returned?.status).toBe( + "unknown", + ); + }); + + it("reproduces the issue #2646 inputs without an exact literal", () => { + for (const operator of ["||=", "??="]) { + const { binding, returned } = observe( + `function read(x) { x ${operator} 2; return x; }`, + ); + expect(binding).not.toEqual({ status: "literal", value: 2 }); + expect(returned).not.toEqual({ status: "literal", value: 2 }); + } + const readOr = (x: number) => { + x ||= 2; + return x; + }; + const readNullish = (x: number) => { + x ??= 2; + return x; + }; + expect(readOr(5)).toBe(5); + expect(readNullish(5)).toBe(5); + }); +}); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index a3045fbd8..119fbdd45 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -919,6 +919,21 @@ const bindImportEquals = ( }); }; +// A parameter or catch binding starts with a caller-supplied value that a +// conditional logical assignment can retain, so its RHS is not exact. +const hasIncomingValue = ( + identifier: t.Identifier, + scope: JavaScriptSemanticScopeState, + state: JavaScriptSemanticAnalysisState, +): boolean => { + const kind = resolveSemanticBindingFromScope( + scope, + identifier.name, + state, + )?.kind; + return kind === "parameter" || kind === "catch"; +}; + const bindAssignment = ( node: t.Node, scope: JavaScriptSemanticScopeState, @@ -929,8 +944,8 @@ const bindAssignment = ( addAssignment( node.left, node.operator === "=" || - node.operator === "||=" || - node.operator === "??=" + ((node.operator === "||=" || node.operator === "??=") && + !hasIncomingValue(node.left, scope, state)) ? node.right : node, scope, From df3d96da801976fdbe37a726c2f4f84af8ffbf90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:31:27 +0800 Subject: [PATCH 05/18] fix: repair the static-check failures on the default branch (#2895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: repair the static-check failures on the default branch The default branch fails its own required `Static checks` job three ways, so every pull request reports that job as failed regardless of its own change. - `npm run format:check` reports `src/domain/functionComparison.test.ts` and `src/ida/IdaOperationRunner.ts`. - `npm run lint` reports a duplicate `import { join } from "node:path"` in `tests/boundary/mcp/ghidraInventoryMcp.test.ts` (lines 4 and 17). - `npm run typecheck` reports `format: "devin"` missing from the client format union in `src/application/SupportedClients.ts`, and an unchecked index in `tests/boundary/cli/clientProfileLifecycle.test.ts`, whose pair array is inferred as `string[][]`. Apply the pinned formatter, drop the duplicate import, list the `"devin"` format that its own definition already uses, and mark the pair array `as const`. Validation: `npm run check:fast` and `npm run format:check` pass. * fix(setup): keep Devin on the JSON configuration dialect --------- Co-authored-by: 吴佳浩 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/application/SupportedClients.ts | 2 +- src/domain/functionComparison.test.ts | 4 +++- src/ida/IdaOperationRunner.ts | 3 ++- tests/boundary/cli/clientProfileLifecycle.test.ts | 2 +- tests/boundary/mcp/ghidraInventoryMcp.test.ts | 1 - 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 7c3811a23..55a651a52 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -539,7 +539,7 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ configPath: ({ home, platform, env }: ClientPathContext) => join(devinDirectory({ home, platform, env }), "mcp_config.json"), markerPath: devinDirectory, - format: "devin", + format: "json", }, { name: "opencode", diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index 7170f098b..16f8282c9 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -175,7 +175,9 @@ describe("function comparison normalized identity", () => { it("retains unresolved call details alongside missing facet coverage", () => { const value: Record = { ...dossier("return 0;", "0x1000"), - unresolved_calls: [{ address: "0x1010", reason: "Unknown indirect target" }], + unresolved_calls: [ + { address: "0x1010", reason: "Unknown indirect target" }, + ], }; delete value.facet_availability; const legacy = observe("a", functionDossierSchema.parse(value), "ida"); diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index 647b06e52..576dac1e7 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -503,7 +503,8 @@ export class IdaOperationRunner { items: [], availability: { available: false, - reason: "Upstream xrefs do not establish direct caller relationships.", + reason: + "Upstream xrefs do not establish direct caller relationships.", }, coverage: { status: "unknown", diff --git a/tests/boundary/cli/clientProfileLifecycle.test.ts b/tests/boundary/cli/clientProfileLifecycle.test.ts index 8a71b669d..1ceaf49cd 100644 --- a/tests/boundary/cli/clientProfileLifecycle.test.ts +++ b/tests/boundary/cli/clientProfileLifecycle.test.ts @@ -290,7 +290,7 @@ cliTest("uninstall names each retained skill directory", async ({ cli }) => { for (const [client, directory] of [ ["shared", shared], ["claude_code", claude], - ]) { + ] as const) { expect(skill?.detail).toContain( `${client} (${directory}): retained (Skill manifest is not REA-owned.)`, ); diff --git a/tests/boundary/mcp/ghidraInventoryMcp.test.ts b/tests/boundary/mcp/ghidraInventoryMcp.test.ts index ecd331816..ac42b9388 100644 --- a/tests/boundary/mcp/ghidraInventoryMcp.test.ts +++ b/tests/boundary/mcp/ghidraInventoryMcp.test.ts @@ -14,7 +14,6 @@ import { } from "./ghidraMcpHarness.js"; import { spawn } from "node:child_process"; import { mkdir } from "node:fs/promises"; -import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { GhidraClient } from "../../../src/ghidra/GhidraClient.js"; From 501c5d45fe60cae4a6337f38fc417f86d13e1ae8 Mon Sep 17 00:00:00 2001 From: ShitK <80999801+ShitK@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:33:14 +0800 Subject: [PATCH 06/18] fix(managed): share native verification input diagnostics (#2901) --- .../managed/managedWorkflowToolContracts.ts | 32 +------ src/contracts/workflowInputContracts.test.ts | 92 ++++++++++++++++--- .../managedNativeVerificationSchemas.ts | 5 +- 3 files changed, 86 insertions(+), 43 deletions(-) diff --git a/src/contracts/managed/managedWorkflowToolContracts.ts b/src/contracts/managed/managedWorkflowToolContracts.ts index 3842c3814..519c2e125 100644 --- a/src/contracts/managed/managedWorkflowToolContracts.ts +++ b/src/contracts/managed/managedWorkflowToolContracts.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { evidenceSchema } from "../../domain/evidence.js"; +import { managedNativeVerificationInputSchema } from "../../domain/managed/managedNativeVerificationSchemas.js"; import { managedReconstructionImportInputSchema } from "../../domain/managed/managedReconstruction.js"; import type { ToolContract } from "../toolContractTypes.js"; @@ -34,35 +35,6 @@ export const managedReconstructionReferenceInputSchema = static_members: evidenceSchema, }); -/** Inline Evidence inputs for managed/native verification. */ -export const managedNativeVerificationReferenceInputSchema = z - .strictObject({ - managed_boundaries: evidenceSchema, - native_observations: z - .array(evidenceSchema) - .min(1) - .describe("Native function or export Evidence records"), - }) - .superRefine((input, context) => { - const ids = new Set([input.managed_boundaries.evidence_id]); - for (const [index, evidence] of input.native_observations.entries()) { - if (evidence.evidence_id === input.managed_boundaries.evidence_id) - context.addIssue({ - code: "custom", - path: ["native_observations", index], - message: - "Native observation Evidence must be distinct from managed boundary Evidence", - }); - if (ids.has(evidence.evidence_id)) - context.addIssue({ - code: "custom", - path: ["native_observations", index], - message: "Native observation Evidence IDs must be unique", - }); - ids.add(evidence.evidence_id); - } - }); - const managedApplicationGraphReferenceFacts = {} as const; const managedApplicationGraphReferenceShape = z.strictObject({ @@ -146,7 +118,7 @@ export const MANAGED_WORKFLOW_TOOL_CONTRACTS = [ description: "Verify managed P/Invoke/native-boundary declarations against authenticated native export or function Evidence without executing managed code or translating managed metadata tokens into native addresses. The workflow preserves declaration-only, verified, inferred, contradicted, and unresolved states.", kind: "application", - inputSchema: managedNativeVerificationReferenceInputSchema, + inputSchema: managedNativeVerificationInputSchema, outputSchema: nativeVerificationOutputSchema, examples: [ { diff --git a/src/contracts/workflowInputContracts.test.ts b/src/contracts/workflowInputContracts.test.ts index 32e1ddf02..237469487 100644 --- a/src/contracts/workflowInputContracts.test.ts +++ b/src/contracts/workflowInputContracts.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; +import { managedNativeVerificationInputSchema } from "../domain/managed/managedNativeVerificationSchemas.js"; +import { toolContract } from "./toolContracts.js"; +import { toolInputSchemaWithMetadata } from "./toolSchemaMetadata.js"; + import { traceJavaScriptSemanticsRequestSchema } from "./javascript/applicationWorkflowInputContracts.js"; import { JAVASCRIPT_FEATURE_TRACE_EXAMPLE } from "./javascript/javascriptApplicationWorkflowExamples.js"; import { @@ -10,7 +14,6 @@ import { import { compareManagedMembersReferenceInputSchema, managedApplicationGraphReferenceInputSchema, - managedNativeVerificationReferenceInputSchema, } from "./managed/managedWorkflowToolContracts.js"; describe("workflow input contracts", () => { @@ -41,17 +44,6 @@ describe("workflow input contracts", () => { }).success, ).toBe(false); - expect( - managedNativeVerificationReferenceInputSchema.safeParse({ - managed_boundaries: - MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries, - native_observations: [ - MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0], - MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0], - ], - }).success, - ).toBe(false); - expect( managedApplicationGraphReferenceInputSchema.safeParse({ managed_artifact: MANAGED_APPLICATION_GRAPH_EXAMPLE.managed_members, @@ -62,4 +54,80 @@ describe("workflow input contracts", () => { managedApplicationGraphReferenceInputSchema.safeParse({}).success, ).toBe(false); }); + + it.each([ + { + name: "one observation reusing the boundary", + observations: [MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries], + issues: [ + { + path: ["native_observations", 0], + message: + "Native observation Evidence must be distinct from managed boundary Evidence", + }, + ], + }, + { + name: "repeated native observations", + observations: [ + MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0], + MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0], + ], + issues: [ + { + path: ["native_observations", 1], + message: "Native observation Evidence IDs must be unique", + }, + ], + }, + { + name: "repeated observations reusing the boundary", + observations: [ + MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries, + MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries, + ], + issues: [ + { + path: ["native_observations", 0], + message: + "Native observation Evidence must be distinct from managed boundary Evidence", + }, + { + path: ["native_observations", 1], + message: + "Native observation Evidence must be distinct from managed boundary Evidence", + }, + { + path: ["native_observations", 1], + message: "Native observation Evidence IDs must be unique", + }, + ], + }, + { + name: "distinct native observations", + observations: MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations, + issues: [], + }, + ])( + "preserves managed/native diagnostics for $name", + async ({ observations, issues }) => { + const input = { + managed_boundaries: + MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries, + native_observations: observations, + }; + const contract = toolContract("verify_managed_native_boundaries"); + for (const schema of [ + managedNativeVerificationInputSchema, + contract.inputSchema, + toolInputSchemaWithMetadata(contract), + ]) { + const result = await schema["~standard"].validate(input); + expect( + result.issues?.map(({ path, message }) => ({ path, message })) ?? [], + ).toEqual(issues); + if (issues.length === 0) expect(result).toEqual({ value: input }); + } + }, + ); }); diff --git a/src/domain/managed/managedNativeVerificationSchemas.ts b/src/domain/managed/managedNativeVerificationSchemas.ts index 8e543bdab..16afe4048 100644 --- a/src/domain/managed/managedNativeVerificationSchemas.ts +++ b/src/domain/managed/managedNativeVerificationSchemas.ts @@ -92,7 +92,10 @@ export type PinvokeVerification = z.infer; export const managedNativeVerificationInputSchema = z .strictObject({ managed_boundaries: evidenceSchema, - native_observations: z.array(evidenceSchema).min(1), + native_observations: z + .array(evidenceSchema) + .min(1) + .describe("Native function or export Evidence records"), }) .superRefine((input, context) => { const ids = new Set(); From 0c9ee56c2f819d122f12cb5872c5db76cfee1169 Mon Sep 17 00:00:00 2001 From: ShitK <80999801+ShitK@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:34:06 +0800 Subject: [PATCH 07/18] fix(native): accept declared facts in dispatch name inventory (#2906) --- .../EnhancedNativeInvestigation.test.ts | 141 ++++++++++++++++-- .../NativeDispatchMetadataInspection.ts | 8 +- ....01-lists-the-complete-target-open.test.ts | 11 +- 3 files changed, 145 insertions(+), 15 deletions(-) diff --git a/src/application/native/EnhancedNativeInvestigation.test.ts b/src/application/native/EnhancedNativeInvestigation.test.ts index e81dcb8b1..1e1b517b6 100644 --- a/src/application/native/EnhancedNativeInvestigation.test.ts +++ b/src/application/native/EnhancedNativeInvestigation.test.ts @@ -23,6 +23,20 @@ import type { Result } from "../../domain/result.js"; const target = "a".repeat(64); const provider = { id: "fixture", name: "Fixture", version: "1" }; +const methodName = "-[BuildViewController buildTapped:]"; +const methodSymbol = { + primary: true, + dynamic: false, + external: false, + type: "function", + source: "user_defined", +}; +const methodInventoryEntry = { + address: "0x1000", + value: methodName, + symbol: methodSymbol, +}; + const evidence = [ { kind: "interface_builder_resource" as const, @@ -258,8 +272,12 @@ describe("native UI action trace", () => { }, }); }); - it("builds the evidence path from the active app and native provider", async () => { - const tools = new EnhancedTools(testAnalysis()); + it("builds the evidence path from provider names with declared symbol facts", async () => { + const tools = new EnhancedTools( + testAnalysis({ + list_names: () => execution("list_names", [methodInventoryEntry]), + }), + ); const result = await tools.execute("trace_native_ui_action", { action: "buildTapped:", }); @@ -392,18 +410,117 @@ describe("bounded native UI tracing", () => { error: { _tag: "AnalysisInputError" }, }); }); +}); - it("binds dispatch metadata to the active target digest", async () => { - const result = await new EnhancedTools(testAnalysis()).execute( - "inspect_native_dispatch_metadata", - { max_records: 10 }, - ); +describe("native dispatch inventory", () => { + it.each([ + { label: "plain value", entry: { address: "0x1000", value: methodName } }, + { label: "value with symbol facts", entry: methodInventoryEntry }, + { + label: "legacy name with symbol facts", + entry: { address: "0x1000", name: methodName, symbol: methodSymbol }, + }, + ])( + "binds $label inventory to the target without overstating dispatch coverage", + async ({ entry }) => { + const result = await new EnhancedTools( + testAnalysis({ + list_names: () => execution("list_names", [entry]), + }), + ).execute("inspect_native_dispatch_metadata", { max_records: 10 }); - expect(result).toMatchObject({ - ok: true, - value: { target_sha256: target }, - }); - }); + expect(result).toMatchObject({ + ok: true, + value: { + target_sha256: target, + provider, + result: { + objc_dispatch_implementations: [ + { + class_name: "BuildViewController", + selector: "buildTapped:", + implementation_address: "0x1000", + decode: { status: "partial", reason: "symbol_name_only" }, + }, + ], + coverage: expect.arrayContaining([ + expect.objectContaining({ + facet: "objc_dispatch_implementations", + status: "partial", + }), + expect.objectContaining({ + facet: "swift_dispatch_tables", + status: "unsupported", + }), + expect.objectContaining({ + facet: "relative_pointers", + status: "unsupported", + }), + ]), + }, + }, + }); + }, + ); + + it.each([ + { + label: "malformed symbol flag", + entry: { + ...methodInventoryEntry, + symbol: { ...methodSymbol, primary: "true" }, + }, + }, + { + label: "unknown symbol source", + entry: { + ...methodInventoryEntry, + symbol: { ...methodSymbol, source: "unknown" }, + }, + }, + { + label: "incomplete symbol", + entry: { ...methodInventoryEntry, symbol: { primary: true } }, + }, + { + label: "malformed legacy symbol", + entry: { address: "0x1000", name: methodName, symbol: null }, + }, + { + label: "malformed address", + entry: { ...methodInventoryEntry, address: 4096 }, + }, + { + label: "malformed value with a legacy name", + entry: { ...methodInventoryEntry, value: 42, name: methodName }, + }, + { + label: "undeclared field", + entry: { address: "0x1000", value: methodName, unexpected: true }, + }, + ])( + "rejects $label inventory in both dispatch consumers", + async ({ entry }) => { + const tools = new EnhancedTools( + testAnalysis({ + list_names: () => execution("list_names", [entry]), + }), + ); + for (const [operation, parameters] of [ + ["inspect_native_dispatch_metadata", { max_records: 10 }], + ["trace_native_ui_action", { action: "buildTapped:" }], + ] as const) { + expect(await tools.execute(operation, parameters)).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisOutputError", + operation: "list_names", + reason: "provider returned an invalid inventory", + }, + }); + } + }, + ); }); describe("native UI query outcomes", () => { diff --git a/src/application/native/NativeDispatchMetadataInspection.ts b/src/application/native/NativeDispatchMetadataInspection.ts index 85c1a69c3..e72d17dbf 100644 --- a/src/application/native/NativeDispatchMetadataInspection.ts +++ b/src/application/native/NativeDispatchMetadataInspection.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { addressedValue } from "../../contracts/toolOutputSchemaPrimitives.js"; import { nativeDispatchMetadataResultSchema, inspectNativeDispatchMetadata, @@ -36,8 +37,11 @@ export const inspectNativeDispatch = async ( const names = z .array( z.union([ - z.strictObject({ address: z.string(), value: z.string() }), - z.strictObject({ address: z.string(), name: z.string() }), + addressedValue.strict(), + addressedValue + .omit({ value: true }) + .extend({ name: z.string() }) + .strict(), ]), ) .safeParse(execution.value.result); diff --git a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts index 62246a85c..add15a6b2 100644 --- a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts +++ b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts @@ -104,7 +104,16 @@ const fixturePort = (): AnalysisOperationPort => ({ "0x11": "_OBJC_CLASS_$_Fixture", "0x12": "_OBJC_PROTOCOL_$_FixtureDelegate", "0x13": "entry", - }), + }).map((entry) => ({ + ...entry, + symbol: { + primary: true, + dynamic: false, + external: false, + type: "label", + source: "analysis", + }, + })), ), ); case "decode_interface_builder": From 62c8b3da1b00260a5ef97d088737dafa0a7ea542 Mon Sep 17 00:00:00 2001 From: all-black-493 Date: Tue, 13 Oct 2026 19:36:37 +0300 Subject: [PATCH 08/18] feat(browser): attach passive tools to browsers without HTTP discovery (#2346) * feat(browser): attach passive tools to browsers without HTTP discovery Accept a literal loopback browser WebSocket, ws://127.0.0.1:PORT/devtools/browser/ID, as the cdp_endpoint of the six passive browser tools. Discovery then reads Browser.getVersion and Target.getTargets over that socket, and the target session adopts the same socket, so a browser that asks for consent per connection prompts once per call. The socket identifier gates access, so Evidence records the endpoint as ws://127.0.0.1:PORT/devtools/browser/[REDACTED], and runtime reconciliation accepts that recorded form. HTTP endpoints and their Evidence are unchanged. Electron, web runtime, scenario and Inspector endpoints stay HTTP-only. Refs #2288 * test(browser): match browser-socket captures against HTTP discovery Check in the real Chrome lane that DevToolsActivePort names the same browser socket as /json/version, and that inspecting the selected page through that socket leaves DOM structure and scripts unchanged compared with an inspection through HTTP discovery. Refs #2288 * fix(browser): bound browser-socket handshake and report discovery timeouts Run the browser-socket WebSocket handshake and both discovery commands under one discovery deadline, so a port that accepts the connection but never completes the upgrade times out instead of waiting forever. Report a reached deadline as a timeout without carrying the internal abort as its cause, so CLI and MCP project provider_timeout rather than cancelled. Caller cancellation is still reported as cancelled. Refs #2288 --- docs/browser-observation.md | 51 +++- scripts/lib/browser-socket-e2e.mjs | 100 ++++++++ scripts/verify-real-browser.mjs | 9 + src/application/BrowserEvidence.ts | 2 +- src/browser/CdpBrowserProvider.ts | 230 ++++++++--------- src/browser/CdpEndpoint.ts | 103 ++++++++ src/browser/CdpTargetSession.ts | 67 +++++ src/cli/browserAdvancedCommands.ts | 8 +- src/cli/browserCommands.ts | 16 +- src/domain/browserObservation.test.ts | 54 ++++ src/domain/browserObservation.ts | 127 ++++++++-- src/domain/browserSession.ts | 4 +- ...riptRuntimeReconciliationCaptureParsing.ts | 14 +- src/domain/webMcpDiscovery.ts | 4 +- src/domain/webScreenshot.ts | 4 +- .../cdpBrowserProvider.browserSocket.test.ts | 232 ++++++++++++++++++ ...cdpBrowserProvider.targetInventory.test.ts | 11 +- tests/boundary/cli/browserCli.test.ts | 48 ++++ tests/boundary/mcp/browserMcp.test.ts | 41 ++++ tests/fixtures/fakeCdpBrowser.ts | 3 + tests/fixtures/fakeCdpBrowserResults.ts | 11 + tests/fixtures/fakeCdpBrowserTypes.ts | 2 + 22 files changed, 961 insertions(+), 180 deletions(-) create mode 100644 scripts/lib/browser-socket-e2e.mjs create mode 100644 tests/boundary/browser/cdpBrowserProvider.browserSocket.test.ts diff --git a/docs/browser-observation.md b/docs/browser-observation.md index dca1f994b..72dde900b 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -107,6 +107,47 @@ first navigation. For a dedicated investigation profile, add then describe an unloaded page; REA continues to require an allowed, committed URL. See [the startup investigation](https://github.com/morluto/rea/issues/1384#issuecomment-6093781686). +### Browsers without HTTP discovery + +Some browsers expose a loopback browser WebSocket but no `/json/version` or +`/json/list`. Chrome 144 and later can enable remote debugging on a running +browser from `chrome://inspect/#remote-debugging`; this mode is reported to +serve only the browser socket. Since Chrome 136, `--remote-debugging-port` is +ignored for the default profile, so this is the remaining way to inspect a page +in an existing profile. + +For such a browser, pass the browser socket instead of the HTTP endpoint: + +```bash +rea list-browser-targets ws://127.0.0.1:9222/devtools/browser/ +``` + +Chrome writes the port and the `/devtools/browser/` path to the +`DevToolsActivePort` file in its user data directory. REA does not read that +file for you. + +- Only `list_browser_targets`, `inspect_web_page`, `analyze_web_bundle`, + `observe_web_session`, `discover_webmcp_tools`, and `capture_web_screenshot` + accept a browser socket. Website runtime attribution, Electron inspection, + and scenario attach mode still require HTTP discovery. +- Each call opens one WebSocket and uses it for discovery + (`Browser.getVersion`, `Target.getTargets`) and for the selected page. + Discovery must finish within the same 10-second deadline as HTTP discovery. + Chrome may ask you to approve each connection. +- The browser identifier controls access to the socket. Evidence records the + endpoint as `ws://127.0.0.1:9222/devtools/browser/[REDACTED]`. +- `browser.revision` is the socket's `Browser.getVersion` value, such as + `@e533e98…`. HTTP discovery reports `WebKit-Version`, such as + `537.36 (@e533e98…)`. +- `Target.getTargets` reports whether each target is attached. Chrome's + `/json/list` can omit it, so HTTP discovery may report `attached` as `null`. +- An existing profile can hold many unrelated pages. Target listing returns + each eligible page's URL and title, so pass `--allowed-origins` to list only + the origins you are investigating. + +A browser driven only through `--remote-debugging-pipe` or a vendor's private +automation channel has neither endpoint, and REA cannot attach to it. + ## Request boundary CDP connections enforce the shared 64 MiB limit per WebSocket message. A larger @@ -116,9 +157,13 @@ decoded strings or JSON objects. Discovery supplies a literal loopback CDP endpoint and returns all eligible page targets unless an exact HTTP(S) origin filter narrows the result. Follow-up -inspection requests select one target; their default scope is its current origin. `localhost`, private-LAN -addresses, HTTPS CDP endpoints, credentials, paths, queries, fragments, and -implicit ports are not valid CDP endpoints. Origin filters contain only scheme, +inspection requests select one target; their default scope is its current origin. +The endpoint is either `http://127.0.0.1:PORT` or a +[browser socket](#browsers-without-http-discovery), +`ws://127.0.0.1:PORT/devtools/browser/ID`; `[::1]` may replace `127.0.0.1`. +`localhost`, private-LAN addresses, HTTPS and `wss:` endpoints, credentials, +queries, fragments, implicit ports, and any other path are not valid CDP +endpoints. Origin filters contain only scheme, host, and port; paths, credentials, queries, fragments, and wildcards are not valid origins. diff --git a/scripts/lib/browser-socket-e2e.mjs b/scripts/lib/browser-socket-e2e.mjs new file mode 100644 index 000000000..6e54e3d18 --- /dev/null +++ b/scripts/lib/browser-socket-e2e.mjs @@ -0,0 +1,100 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, +} from "../../dist/domain/browserObservation.js"; +import { compareWebCapturesInputSchema } from "../../dist/domain/webCaptureDiffSchemas.js"; +import { artifactCliEvidence } from "./artifact-e2e.mjs"; + +/** Verify browser-socket discovery and inspection against HTTP discovery. */ +export async function verifyBrowserSocketTransport({ + provider, + profile, + endpoint, + target, + origin, +}) { + const [port, path] = ( + await readFile(join(profile, "DevToolsActivePort"), "utf8") + ).split("\n"); + const socket = `ws://127.0.0.1:${port}${path}`; + const version = await (await fetch(`${endpoint}/json/version`)).json(); + assert.equal( + new URL(version.webSocketDebuggerUrl).pathname, + path, + "DevToolsActivePort and /json/version named different browser sockets", + ); + const list = async (cdpEndpoint) => { + const listed = await provider.listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: cdpEndpoint, + allowed_origins: [origin], + }), + ); + if (!listed.ok) throw listed.error; + return listed.value; + }; + const throughHttp = await list(endpoint); + const throughSocket = await list(socket); + // Target.getTargets reports attachment state that /json/list may omit. + const withoutAttachment = (targets) => + targets.map(({ attached: _attached, ...rest }) => rest); + assert.ok(throughSocket.targets.length > 0, "Browser socket listed no page"); + assert.deepEqual( + withoutAttachment(throughSocket.targets), + withoutAttachment(throughHttp.targets), + ); + assert.ok( + throughSocket.targets.every( + ({ attached }) => typeof attached === "boolean", + ), + "Browser socket omitted target attachment state", + ); + assert.equal(throughSocket.browser.product, throughHttp.browser.product); + + const evidence = await artifactCliEvidence("inspect-web-page", socket, [ + target, + "--allowed-origins", + origin, + "--observation-ms", + "100", + ]); + assert.equal(evidence.normalized_result.target.target_id, target); + assert.ok( + evidence.normalized_result.dom.nodes.length > 0, + "Browser-socket inspection returned no DOM nodes", + ); + assert.equal( + evidence.parameters.cdp_endpoint, + `ws://127.0.0.1:${port}/devtools/browser/[REDACTED]`, + ); + assert.ok( + !JSON.stringify(evidence).includes(path), + "Browser-socket Evidence retained the socket identifier", + ); + const throughHttpInspection = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [origin], + target_id: target, + observation_ms: 100, + }), + ); + if (!throughHttpInspection.ok) throw throughHttpInspection.error; + const compared = await provider.compareCaptures( + compareWebCapturesInputSchema.parse({ + before: { inspection: throughHttpInspection.value }, + after: { inspection: evidence.normalized_result }, + }), + ); + if (!compared.ok) throw compared.error; + assert.equal(compared.value.dimensions.dom_structure.status, "unchanged"); + assert.equal(compared.value.dimensions.scripts.status, "unchanged"); + return { + targets: throughSocket.targets.length, + revision: throughSocket.browser.revision, + }; +} diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index 27c0c5e52..158877bca 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -45,6 +45,7 @@ import { verifyBrowserNetworkEvidence } from "./lib/browser-network-e2e.mjs"; import { verifyBrowserScriptExport } from "./lib/browser-script-export-e2e.mjs"; import { verifyBrowserModules } from "./lib/browser-module-e2e.mjs"; import { verifyBrowserDomDestinations } from "./lib/browser-dom-destinations-e2e.mjs"; +import { verifyBrowserSocketTransport } from "./lib/browser-socket-e2e.mjs"; import { verifyBrowserCaptureMetadataBudget } from "./lib/browser-capture-metadata-budget-e2e.mjs"; import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs"; import { verifyScenarioEnvironment } from "./lib/browser-scenario-environment-e2e.mjs"; @@ -234,6 +235,13 @@ try { pageProxy = await startPageCdpProxy(endpoint); await verifyPageScopedTransport(provider, pageProxy, site.origin); + const browserSocketTransport = await verifyBrowserSocketTransport({ + provider, + profile, + endpoint, + target, + origin: site.origin, + }); const attachedScenarioInput = browserScenario( { mode: "connect", @@ -377,6 +385,7 @@ try { bundle_cli_and_stdio_mcp: true, sessionEvents: session.value.timeline.length, pageScopedTransport: true, + browserSocketTransport, screenshotBytes: screenshot.value.artifact.bytes, largeScreenshot, popupEvents, diff --git a/src/application/BrowserEvidence.ts b/src/application/BrowserEvidence.ts index d864fac9d..495f1f197 100644 --- a/src/application/BrowserEvidence.ts +++ b/src/application/BrowserEvidence.ts @@ -131,7 +131,7 @@ const browserParameters = ( } const observedOrigin = browserResultTargetOrigin(result); const scope = { - cdp_endpoint: input.cdp_endpoint, + cdp_endpoint: input.cdp_endpoint.recorded, allowed_origins: input.allowed_origins.length > 0 ? input.allowed_origins diff --git a/src/browser/CdpBrowserProvider.ts b/src/browser/CdpBrowserProvider.ts index f4d898a68..744a82983 100644 --- a/src/browser/CdpBrowserProvider.ts +++ b/src/browser/CdpBrowserProvider.ts @@ -49,13 +49,14 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import { - discoverCdpEndpoint, hasCdpTargetWebSocket, type CdpEndpointDiscovery, type CdpEndpointTarget, } from "./CdpEndpoint.js"; import { + attachCdpTargetSession, closeCdpTargetSession, + openCdpEndpoint, openCdpTargetSession, type CdpTargetSession, } from "./CdpTargetSession.js"; @@ -82,11 +83,13 @@ export class CdpBrowserProvider implements BrowserObservationPort { options: ExecutionOptions = {}, ): Promise> { try { - const discovery = await discoverCdpEndpoint( + const endpoint = await openCdpEndpoint( input.cdp_endpoint, "list_browser_targets", options.signal, ); + await endpoint.connection?.close(); + const { discovery } = endpoint; const selected = selectTargets( discovery, input.allowed_origins.length === 0 @@ -169,45 +172,19 @@ export class CdpBrowserProvider implements BrowserObservationPort { input: ObserveWebSessionInput, options: ExecutionOptions = {}, ): Promise> { - let targetSession: CdpTargetSession | undefined; try { - const discovery = await discoverCdpEndpoint( - input.cdp_endpoint, - "observe_web_session", - options.signal, - ); - const target = authorizeTarget(discovery, input); - const scopedInput = scopeToTargetOrigin(input, target); - targetSession = await openCdpTargetSession( - discovery, - target, - "observe_web_session", - options.signal, - ); return ok( webObservationSessionSchema.parse( - await observeCdpSession({ - connection: targetSession.connection, - sessionId: targetSession.sessionId, - discovery, - target, - input: scopedInput, - ...(options.signal === undefined ? {} : { signal: options.signal }), - ...(options.progress === undefined - ? {} - : { progress: options.progress }), - }), + await withTargetSession( + input, + "observe_web_session", + options, + observeCdpSession, + ), ), ); } catch (cause: unknown) { return err(providerError(cause, "observe_web_session")); - } finally { - if (targetSession !== undefined) - await closeCdpTargetSession( - targetSession, - CLEANUP_DOMAINS, - options.signal, - ); } } @@ -215,45 +192,19 @@ export class CdpBrowserProvider implements BrowserObservationPort { input: DiscoverWebMcpToolsInput, options: ExecutionOptions = {}, ): Promise> { - let targetSession: CdpTargetSession | undefined; try { - const discovery = await discoverCdpEndpoint( - input.cdp_endpoint, - "discover_webmcp_tools", - options.signal, - ); - const target = authorizeTarget(discovery, input); - const scopedInput = scopeToTargetOrigin(input, target); - targetSession = await openCdpTargetSession( - discovery, - target, - "discover_webmcp_tools", - options.signal, - ); return ok( webMcpDiscoverySchema.parse( - await discoverWebMcp({ - connection: targetSession.connection, - sessionId: targetSession.sessionId, - discovery, - target, - input: scopedInput, - ...(options.signal === undefined ? {} : { signal: options.signal }), - ...(options.progress === undefined - ? {} - : { progress: options.progress }), - }), + await withTargetSession( + input, + "discover_webmcp_tools", + options, + discoverWebMcp, + ), ), ); } catch (cause: unknown) { return err(providerError(cause, "discover_webmcp_tools")); - } finally { - if (targetSession !== undefined) - await closeCdpTargetSession( - targetSession, - CLEANUP_DOMAINS, - options.signal, - ); } } @@ -273,45 +224,19 @@ export class CdpBrowserProvider implements BrowserObservationPort { input: CaptureWebScreenshotInput, options: ExecutionOptions = {}, ): Promise> { - let targetSession: CdpTargetSession | undefined; try { - const discovery = await discoverCdpEndpoint( - input.cdp_endpoint, - "capture_web_screenshot", - options.signal, - ); - const target = authorizeTarget(discovery, input); - const scopedInput = scopeToTargetOrigin(input, target); - targetSession = await openCdpTargetSession( - discovery, - target, - "capture_web_screenshot", - options.signal, - ); return ok( webScreenshotSchema.parse( - await captureCdpScreenshot({ - connection: targetSession.connection, - sessionId: targetSession.sessionId, - discovery, - target, - input: scopedInput, - ...(options.signal === undefined ? {} : { signal: options.signal }), - ...(options.progress === undefined - ? {} - : { progress: options.progress }), - }), + await withTargetSession( + input, + "capture_web_screenshot", + options, + captureCdpScreenshot, + ), ), ); } catch (cause: unknown) { return err(providerError(cause, "capture_web_screenshot")); - } finally { - if (targetSession !== undefined) - await closeCdpTargetSession( - targetSession, - CLEANUP_DOMAINS, - options.signal, - ); } } @@ -336,44 +261,83 @@ export class CdpBrowserProvider implements BrowserObservationPort { options: ExecutionOptions, operation: "inspect_web_page" | "analyze_web_bundle", ): Promise { - let targetSession: CdpTargetSession | undefined; - try { - const discovery = await discoverCdpEndpoint( - input.cdp_endpoint, - operation, - options.signal, - ); - const target = authorizeTarget(discovery, input); - const scopedInput = scopeToTargetOrigin(input, target); - targetSession = await openCdpTargetSession( - discovery, - target, - operation, - options.signal, - ); - return await capturePage({ - connection: targetSession.connection, - sessionId: targetSession.sessionId, - operation, - discovery, - target, - input: scopedInput, - ...(options.signal === undefined ? {} : { signal: options.signal }), - ...(options.progress === undefined - ? {} - : { progress: options.progress }), - }); - } finally { - if (targetSession !== undefined) - await closeCdpTargetSession( - targetSession, - CLEANUP_DOMAINS, - options.signal, - ); - } + return await withTargetSession( + input, + operation, + options, + async (session) => await capturePage({ ...session, operation }), + ); } } +/** Facts and transport handed to one target operation. */ +interface CdpTargetOperationSession { + readonly connection: CdpTargetSession["connection"]; + readonly sessionId: CdpTargetSession["sessionId"]; + readonly discovery: CdpEndpointDiscovery; + readonly target: CdpEndpointTarget; + readonly input: Input; + readonly signal?: AbortSignal; + readonly progress?: NonNullable; +} + +/** Authorize one discovered target, run work in its session, and close REA's socket. */ +const withTargetSession = async < + Input extends Pick< + InspectWebPageInput, + "cdp_endpoint" | "target_id" | "allowed_origins" + >, + Output, +>( + input: Input, + operation: BrowserObservationOperation, + options: ExecutionOptions, + work: (session: CdpTargetOperationSession) => Promise, +): Promise => { + const endpoint = await openCdpEndpoint( + input.cdp_endpoint, + operation, + options.signal, + ); + let targetSession: CdpTargetSession | undefined; + try { + const target = authorizeTarget(endpoint.discovery, input); + const scopedInput = scopeToTargetOrigin(input, target); + targetSession = + endpoint.connection === undefined + ? await openCdpTargetSession( + endpoint.discovery, + target, + operation, + options.signal, + ) + : await attachCdpTargetSession( + endpoint.connection, + target, + operation, + options.signal, + ); + return await work({ + connection: targetSession.connection, + sessionId: targetSession.sessionId, + discovery: endpoint.discovery, + target, + input: scopedInput, + ...(options.signal === undefined ? {} : { signal: options.signal }), + ...(options.progress === undefined ? {} : { progress: options.progress }), + }); + } finally { + if (targetSession !== undefined) + await closeCdpTargetSession( + targetSession, + CLEANUP_DOMAINS, + options.signal, + ); + // An unadopted browser socket is closed here; closing is idempotent. + await endpoint.connection?.close(); + } +}; + const selectTargets = ( discovery: CdpEndpointDiscovery, allowedOrigins: ReadonlySet | undefined, diff --git a/src/browser/CdpEndpoint.ts b/src/browser/CdpEndpoint.ts index cf78df183..3db49d28c 100644 --- a/src/browser/CdpEndpoint.ts +++ b/src/browser/CdpEndpoint.ts @@ -9,6 +9,7 @@ import type { BrowserObservationOperation } from "../domain/browserObservationEr import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; import { urlDeclaresQueryOrFragment } from "../domain/urlComponents.js"; import { safeParseJson } from "../domain/safeJson.js"; +import type { CdpConnection } from "./CdpConnection.js"; const endpointVersionSchema = z.object({ Browser: z.string().min(1), @@ -40,6 +41,26 @@ interface CdpDiscoveryResponseLimits { readonly timeoutMs: number; } +const browserSocketVersionSchema = z.object({ + protocolVersion: z.string().min(1), + product: z.string().min(1), + revision: z.string(), + userAgent: z.string(), + jsVersion: z.string(), +}); + +const browserSocketTargetsSchema = z.object({ + targetInfos: z.array( + z.object({ + targetId: z.string().min(1), + type: z.string().min(1), + title: z.string().default(""), + url: z.string(), + attached: z.boolean().optional(), + }), + ), +}); + /** Validated direct CDP WebSocket bound to one discovered page target. */ interface CdpPageWebSocketEndpoint { readonly scope: "page"; @@ -124,6 +145,88 @@ export const discoverCdpEndpoint = async ( }; }; +/** + * Run browser-socket discovery work, including its WebSocket handshake, under + * the HTTP discovery deadline. Reaching the deadline is a timeout, not a + * caller cancellation, so the internal abort is not carried as its cause. + */ +export const withCdpDiscoveryDeadline = async ( + operation: BrowserObservationOperation, + signal: AbortSignal | undefined, + work: (deadlineSignal: AbortSignal) => Promise, + timeoutMs: number = CDP_DISCOVERY_RESPONSE_LIMITS.timeoutMs, +): Promise => { + const deadline = new AbortController(); + const timer = setTimeout(() => deadline.abort(), timeoutMs); + try { + return await work( + signal === undefined + ? deadline.signal + : AbortSignal.any([signal, deadline.signal]), + ); + } catch (cause: unknown) { + if (deadline.signal.aborted && signal?.aborted !== true) + throw new BrowserObservationError(operation, "timeout", { + detail: `CDP browser-socket discovery exceeded its ${String(timeoutMs)} ms deadline.`, + }); + throw cause; + } finally { + clearTimeout(timer); + } +}; + +/** + * Read discovery facts over a browser socket that has no HTTP discovery. + * Each reply is bounded by the connection's protocol message limit; callers + * apply the discovery deadline through `signal`. + */ +export const discoverCdpBrowserSocket = async ( + connection: CdpConnection, + url: string, + operation: BrowserObservationOperation, + signal?: AbortSignal, +): Promise => { + const versionInput = await connection.send( + "Browser.getVersion", + {}, + undefined, + signal, + ); + const targetsInput = await connection.send( + "Target.getTargets", + {}, + undefined, + signal, + ); + const version = parseCdpEndpointValue( + browserSocketVersionSchema, + versionInput, + operation, + ); + const targets = parseCdpEndpointValue( + browserSocketTargetsSchema, + targetsInput, + operation, + ); + return { + webSocket: { scope: "browser", url }, + version: { + product: version.product, + protocol_version: version.protocolVersion, + revision: version.revision, + user_agent: version.userAgent, + js_version: version.jsVersion, + }, + targets: targets.targetInfos.map((target) => ({ + id: target.targetId, + type: target.type, + title: sanitizeTargetTitle(target.title, target.url), + url: target.url, + attached: target.attached ?? null, + })), + }; +}; + const sanitizeTargetTitle = (value: string, targetUrl: string): string => { let url: URL; try { diff --git a/src/browser/CdpTargetSession.ts b/src/browser/CdpTargetSession.ts index 3e1f6eb43..7355e142e 100644 --- a/src/browser/CdpTargetSession.ts +++ b/src/browser/CdpTargetSession.ts @@ -1,8 +1,12 @@ +import type { BrowserObservationEndpoint } from "../domain/browserObservation.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js"; import { CdpConnection } from "./CdpConnection.js"; import { cdpTargetWebSocket, + discoverCdpBrowserSocket, + discoverCdpEndpoint, + withCdpDiscoveryDeadline, type CdpEndpointDiscovery, type CdpEndpointTarget, } from "./CdpEndpoint.js"; @@ -13,6 +17,59 @@ export interface CdpTargetSession { readonly sessionId: string | undefined; } +/** Discovery for one caller-selected endpoint and the browser socket that read it. */ +export interface CdpOpenedEndpoint { + readonly discovery: CdpEndpointDiscovery; + /** + * Browser socket kept for the target session, because a browser may ask + * the user to approve each socket. HTTP discovery opens no socket. + */ + readonly connection: CdpConnection | undefined; +} + +/** + * Discover one passive browser endpoint through its selected transport. A + * browser socket's handshake and discovery share one discovery deadline. + */ +export const openCdpEndpoint = async ( + endpoint: BrowserObservationEndpoint, + operation: BrowserObservationOperation, + signal?: AbortSignal, + timeoutMs?: number, +): Promise => { + if (endpoint.transport === "http_discovery") + return { + discovery: await discoverCdpEndpoint(endpoint.url, operation, signal), + connection: undefined, + }; + return await withCdpDiscoveryDeadline( + operation, + signal, + async (deadlineSignal) => { + const connection = await CdpConnection.connect( + endpoint.url, + operation, + deadlineSignal, + ); + try { + return { + discovery: await discoverCdpBrowserSocket( + connection, + endpoint.url, + operation, + deadlineSignal, + ), + connection, + }; + } catch (cause: unknown) { + await connection.close(); + throw cause; + } + }, + timeoutMs, + ); +}; + /** Open one authorized target through its supported browser or page transport. */ export const openCdpTargetSession = async ( discovery: CdpEndpointDiscovery, @@ -29,6 +86,16 @@ export const openCdpTargetSession = async ( limits, ); if (webSocket.scope === "page") return { connection, sessionId: undefined }; + return await attachCdpTargetSession(connection, target, operation, signal); +}; + +/** Attach one target through an open browser socket, which the session then owns. */ +export const attachCdpTargetSession = async ( + connection: CdpConnection, + target: CdpEndpointTarget, + operation: BrowserObservationOperation, + signal?: AbortSignal, +): Promise => { try { const attached = await connection.send( "Target.attachToTarget", diff --git a/src/cli/browserAdvancedCommands.ts b/src/cli/browserAdvancedCommands.ts index 50c9ce446..dc0c268d1 100644 --- a/src/cli/browserAdvancedCommands.ts +++ b/src/cli/browserAdvancedCommands.ts @@ -46,7 +46,9 @@ const registerWebMcp = ( cli.command(CLI_COMMANDS.discoverWebMcpTools, { description: "Passively discover page-declared WebMCP tools", args: z.object({ - endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + endpoint: z + .string() + .describe("Loopback CDP HTTP endpoint or browser WebSocket URL"), targetId: z.string().describe("Target ID from list-browser-targets"), }), options: z.object({ @@ -150,7 +152,9 @@ const registerScreenshot = ( description: "Capture one visible page viewport within the supplied origin scope", args: z.object({ - endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + endpoint: z + .string() + .describe("Loopback CDP HTTP endpoint or browser WebSocket URL"), targetId: z.string().describe("Target ID from list-browser-targets"), }), options: z.object({ diff --git a/src/cli/browserCommands.ts b/src/cli/browserCommands.ts index aae6714d2..e9043ec80 100644 --- a/src/cli/browserCommands.ts +++ b/src/cli/browserCommands.ts @@ -42,7 +42,9 @@ const registerTargetList = ( description: "List page targets from a selected loopback CDP browser, optionally filtered by origin", args: z.object({ - endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + endpoint: z + .string() + .describe("Loopback CDP HTTP endpoint or browser WebSocket URL"), }), options: z.object({ ...browserScopeOptions }), run: ({ args, options }) => @@ -75,7 +77,9 @@ const registerPageInspection = ( description: "Passively inspect one page through CDP, defaulting to its current origin", args: z.object({ - endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + endpoint: z + .string() + .describe("Loopback CDP HTTP endpoint or browser WebSocket URL"), targetId: z.string().describe("Target ID from list-browser-targets"), }), options: browserPageInspectionOptions, @@ -118,7 +122,9 @@ const registerBundleAnalysis = ( description: "Capture and statically analyze a page bundle, defaulting to the selected target origin", args: z.object({ - endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + endpoint: z + .string() + .describe("Loopback CDP HTTP endpoint or browser WebSocket URL"), targetId: z.string().describe("Target ID from list-browser-targets"), }), options: z.object({ @@ -162,7 +168,9 @@ const registerObservationSession = ( cli.command(CLI_COMMANDS.observeWebSession, { description: "Observe user-driven page navigation during a chosen window", args: z.object({ - endpoint: z.string().describe("Configured loopback CDP HTTP endpoint"), + endpoint: z + .string() + .describe("Loopback CDP HTTP endpoint or browser WebSocket URL"), targetId: z.string().describe("Target ID from list-browser-targets"), }), options: z.object({ diff --git a/src/domain/browserObservation.test.ts b/src/domain/browserObservation.test.ts index c5eb73287..6bcf09f68 100644 --- a/src/domain/browserObservation.test.ts +++ b/src/domain/browserObservation.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest"; import { browserEndpointSchema, + browserObservationEndpointSchema, browserOriginSchema, + recordedBrowserObservationEndpointSchema, sanitizeEndpointCandidate, sanitizeBrowserUrl, } from "./browserObservation.js"; @@ -45,6 +47,58 @@ describe("browser observation contracts", () => { ]) expect(browserEndpointSchema.safeParse(value).success, value).toBe(false); }); + + it("accepts a loopback browser socket and records it without its identifier", () => { + expect( + browserObservationEndpointSchema.parse("http://127.0.0.1:9222/"), + ).toEqual({ + transport: "http_discovery", + url: "http://127.0.0.1:9222", + recorded: "http://127.0.0.1:9222", + }); + expect( + browserObservationEndpointSchema.parse( + "ws://[::1]:9222/devtools/browser/0b1c-socket-id", + ), + ).toEqual({ + transport: "browser_websocket", + url: "ws://[::1]:9222/devtools/browser/0b1c-socket-id", + recorded: "ws://[::1]:9222/devtools/browser/[REDACTED]", + }); + for (const value of [ + "ws://127.0.0.1/devtools/browser/id", + "ws://localhost:9222/devtools/browser/id", + "wss://127.0.0.1:9222/devtools/browser/id", + "ws://user:pass@127.0.0.1:9222/devtools/browser/id", + "ws://127.0.0.1:9222/devtools/browser/id?token=x", + "ws://127.0.0.1:9222/devtools/browser/id#fragment", + "ws://127.0.0.1:9222/devtools/browser/id?", + "ws://127.0.0.1:9222/devtools/browser/id#", + "ws://127.0.0.1:9222/devtools/browser/", + "ws://127.0.0.1:9222/devtools/browser/id/extra", + "ws://127.0.0.1:9222/devtools/page/id", + ]) + expect( + browserObservationEndpointSchema.safeParse(value).success, + value, + ).toBe(false); + }); + + it("accepts recorded browser sockets only in their redacted form", () => { + for (const value of [ + "http://127.0.0.1:9222", + "ws://127.0.0.1:9222/devtools/browser/[REDACTED]", + ]) + expect( + recordedBrowserObservationEndpointSchema.safeParse(value).success, + value, + ).toBe(true); + expect( + recordedBrowserObservationEndpointSchema.safeParse( + "ws://127.0.0.1:9222/devtools/browser/0b1c-socket-id", + ).success, + ).toBe(false); + }); }); describe("browser observation sensitive surfaces and retention", () => { diff --git a/src/domain/browserObservation.ts b/src/domain/browserObservation.ts index f4b8984b6..e15265305 100644 --- a/src/domain/browserObservation.ts +++ b/src/domain/browserObservation.ts @@ -43,46 +43,125 @@ export const browserOriginSchema = z export const isLiteralLoopbackHostname = (hostname: string): boolean => hostname === "127.0.0.1" || hostname === "[::1]" || hostname === "::1"; +const HTTP_ENDPOINT_MESSAGE = + "CDP endpoint must be an explicit-port HTTP URL on 127.0.0.1 or ::1"; +const BROWSER_SOCKET_PATH = "/devtools/browser/"; +const REDACTED_BROWSER_SOCKET_PATH = `${BROWSER_SOCKET_PATH}[REDACTED]`; + +/** Normalized origin of an accepted loopback CDP HTTP discovery endpoint. */ +const httpDiscoveryOrigin = (url: URL): string | undefined => + url.protocol === "http:" && + isLiteralLoopbackHostname(url.hostname) && + url.port !== "" && + url.username === "" && + url.password === "" && + url.pathname === "/" && + !urlDeclaresQueryOrFragment(url) + ? url.origin + : undefined; + +/** Scheme and authority of an accepted loopback CDP browser WebSocket. */ +const browserSocketAuthority = (url: URL): string | undefined => { + const id = url.pathname.slice(BROWSER_SOCKET_PATH.length); + return url.protocol === "ws:" && + isLiteralLoopbackHostname(url.hostname) && + url.port !== "" && + url.username === "" && + url.password === "" && + !urlDeclaresQueryOrFragment(url) && + url.pathname.startsWith(BROWSER_SOCKET_PATH) && + id.length > 0 && + !id.includes("/") + ? `${url.protocol}//${url.host}` + : undefined; +}; + /** Loopback-only HTTP endpoint accepted for a user-owned CDP browser. */ export const browserEndpointSchema = z .string() .min(1) .transform((value, context) => { - let url: URL; - try { - url = new URL(value); - } catch (cause: unknown) { - // Invalid input is reported through the zod issue. - void cause; + const url = URL.parse(value); + const origin = url === null ? undefined : httpDiscoveryOrigin(url); + if (origin !== undefined) return origin; + context.addIssue({ + code: "custom", + message: + url === null ? "Invalid CDP endpoint URL" : HTTP_ENDPOINT_MESSAGE, + }); + return z.NEVER; + }); + +/** + * Caller-selected endpoint for passive browser observation. A browser socket + * skips HTTP discovery; its identifier gates access, so `recorded` omits it. + */ +export interface BrowserObservationEndpoint { + readonly transport: "http_discovery" | "browser_websocket"; + readonly url: string; + readonly recorded: string; +} + +/** Loopback CDP HTTP discovery origin or browser WebSocket URL. */ +export const browserObservationEndpointSchema = z + .string() + .min(1) + .describe( + "Literal loopback CDP endpoint: http://127.0.0.1:PORT, or ws://127.0.0.1:PORT/devtools/browser/ID.", + ) + .transform((value, context): BrowserObservationEndpoint => { + const url = URL.parse(value); + if (url === null) { context.addIssue({ code: "custom", message: "Invalid CDP endpoint URL" }); return z.NEVER; } - if ( - url.protocol !== "http:" || - !isLiteralLoopbackHostname(url.hostname) || - url.port === "" || - url.username !== "" || - url.password !== "" || - url.pathname !== "/" || - urlDeclaresQueryOrFragment(url) - ) { - context.addIssue({ - code: "custom", - message: - "CDP endpoint must be an explicit-port HTTP URL on 127.0.0.1 or ::1", - }); - return z.NEVER; - } - return url.origin; + const origin = httpDiscoveryOrigin(url); + if (origin !== undefined) + return { transport: "http_discovery", url: origin, recorded: origin }; + const authority = browserSocketAuthority(url); + if (authority !== undefined) + return { + transport: "browser_websocket", + url: url.href, + recorded: `${authority}${REDACTED_BROWSER_SOCKET_PATH}`, + }; + context.addIssue({ + code: "custom", + message: + url.protocol === "ws:" + ? "CDP browser WebSocket must be ws://127.0.0.1:PORT/devtools/browser/ID or ws://[::1]:PORT/devtools/browser/ID" + : HTTP_ENDPOINT_MESSAGE, + }); + return z.NEVER; }); +/** Evidence form of a passive browser endpoint, with any socket identifier redacted. */ +export const recordedBrowserObservationEndpointSchema = z + .string() + .min(1) + .refine( + (value) => { + const url = URL.parse(value); + return ( + url !== null && + (httpDiscoveryOrigin(url) !== undefined || + (browserSocketAuthority(url) !== undefined && + url.pathname === REDACTED_BROWSER_SOCKET_PATH)) + ); + }, + { + message: + "Recorded CDP endpoint must be a loopback HTTP origin or a redacted browser WebSocket URL", + }, + ); + export const browserAllowedOriginsSchema = z .array(browserOriginSchema) .transform((origins) => Array.from(new Set(origins)).sort()) .default([]); const browserInput = { - cdp_endpoint: browserEndpointSchema, + cdp_endpoint: browserObservationEndpointSchema, allowed_origins: browserAllowedOriginsSchema, }; diff --git a/src/domain/browserSession.ts b/src/domain/browserSession.ts index 17ecfa8db..c2f00e9a2 100644 --- a/src/domain/browserSession.ts +++ b/src/domain/browserSession.ts @@ -2,14 +2,14 @@ import { z } from "zod"; import { browserAllowedOriginsSchema, - browserEndpointSchema, + browserObservationEndpointSchema, } from "./browserObservation.js"; import { browserCompletenessSchema } from "./browserCompleteness.js"; import { browserVersionSchema } from "./browserObservationSchemas.js"; /** Input for a navigation-aware browser observation window. */ export const observeWebSessionInputSchema = z.strictObject({ - cdp_endpoint: browserEndpointSchema, + cdp_endpoint: browserObservationEndpointSchema, allowed_origins: browserAllowedOriginsSchema, target_id: z.string().trim().min(1), observation_ms: z.number().int().min(1).default(10_000), diff --git a/src/domain/javascript/javascriptRuntimeReconciliationCaptureParsing.ts b/src/domain/javascript/javascriptRuntimeReconciliationCaptureParsing.ts index baba3395d..be428b06c 100644 --- a/src/domain/javascript/javascriptRuntimeReconciliationCaptureParsing.ts +++ b/src/domain/javascript/javascriptRuntimeReconciliationCaptureParsing.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { browserAllowedOriginsSchema, browserEndpointSchema, + recordedBrowserObservationEndpointSchema, } from "../browserObservation.js"; import { webPageInspectionSchema, @@ -424,14 +425,17 @@ const assertRuntimeParameters = ( ): void => { const common = { target_id: z.string().trim().min(1), - cdp_endpoint: browserEndpointSchema, include_script_sources: z.boolean(), }; if (expected.kind === "browser") { const parameters = parseAtPath( (value) => z - .object({ ...common, allowed_origins: browserAllowedOriginsSchema }) + .object({ + ...common, + cdp_endpoint: recordedBrowserObservationEndpointSchema, + allowed_origins: browserAllowedOriginsSchema, + }) .passthrough() .parse(value), evidence.parameters, @@ -455,7 +459,11 @@ const assertRuntimeParameters = ( return; } const parameters = parseAtPath( - (value) => z.object(common).passthrough().parse(value), + (value) => + z + .object({ ...common, cdp_endpoint: browserEndpointSchema }) + .passthrough() + .parse(value), evidence.parameters, [...path, "parameters"], ); diff --git a/src/domain/webMcpDiscovery.ts b/src/domain/webMcpDiscovery.ts index f1a3f3b12..0924f5b78 100644 --- a/src/domain/webMcpDiscovery.ts +++ b/src/domain/webMcpDiscovery.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { browserCompletenessSchema } from "./browserCompleteness.js"; import { browserAllowedOriginsSchema, - browserEndpointSchema, + browserObservationEndpointSchema, } from "./browserObservation.js"; import { browserVersionSchema } from "./browserObservationSchemas.js"; import { jsonValueSchema } from "./jsonValue.js"; @@ -11,7 +11,7 @@ import { digestSchema, prefixedDigestSchema } from "./../domain/digests.js"; /** Input for passive discovery of page-registered WebMCP tools. */ export const discoverWebMcpToolsInputSchema = z.strictObject({ - cdp_endpoint: browserEndpointSchema, + cdp_endpoint: browserObservationEndpointSchema, allowed_origins: browserAllowedOriginsSchema, target_id: z.string().trim().min(1), observation_ms: z.number().int().min(0).default(100), diff --git a/src/domain/webScreenshot.ts b/src/domain/webScreenshot.ts index 57ee35161..d477bdaeb 100644 --- a/src/domain/webScreenshot.ts +++ b/src/domain/webScreenshot.ts @@ -5,7 +5,7 @@ import { z } from "zod"; import { browserCompletenessSchema } from "./browserCompleteness.js"; import { browserAllowedOriginsSchema, - browserEndpointSchema, + browserObservationEndpointSchema, } from "./browserObservation.js"; import { browserVersionSchema } from "./browserObservationSchemas.js"; @@ -34,7 +34,7 @@ export type WebScreenshotArtifact = z.infer; /** Input for one read-only visible-viewport screenshot. */ export const captureWebScreenshotInputSchema = z.strictObject({ - cdp_endpoint: browserEndpointSchema, + cdp_endpoint: browserObservationEndpointSchema, allowed_origins: browserAllowedOriginsSchema, target_id: z.string().trim().min(1), }); diff --git a/tests/boundary/browser/cdpBrowserProvider.browserSocket.test.ts b/tests/boundary/browser/cdpBrowserProvider.browserSocket.test.ts new file mode 100644 index 000000000..f04b8210d --- /dev/null +++ b/tests/boundary/browser/cdpBrowserProvider.browserSocket.test.ts @@ -0,0 +1,232 @@ +import { createServer, type Socket } from "node:net"; + +import { describe, expect, it, vi } from "vitest"; + +import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; +import { openCdpEndpoint } from "../../../src/browser/CdpTargetSession.js"; +import { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { + browserObservationEndpointSchema, + inspectWebPageInputSchema, + listBrowserTargetsInputSchema, +} from "../../../src/domain/browserObservation.js"; +import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; +import { trackBrowser } from "./cdpBrowserProvider.support.js"; + +describe("CdpBrowserProvider: browser socket without HTTP discovery", () => { + it("discovers and inspects a page through one browser socket", async () => { + const browser = await startFakeCdpBrowser(); + trackBrowser(browser); + const provider = new CdpBrowserProvider(); + const throughHttp = await provider.listTargets( + listBrowserTargetsInputSchema.parse({ cdp_endpoint: browser.endpoint }), + ); + if (!throughHttp.ok) throw throughHttp.error; + const httpRequests = browser.httpRequests.length; + const openedBefore = browser.openedConnectionCount(); + + const listed = await provider.listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: browser.browserWebSocketUrl, + }), + ); + if (!listed.ok) throw listed.error; + expect(listed.value.targets).toEqual(throughHttp.value.targets); + expect(listed.value.excluded).toEqual(throughHttp.value.excluded); + expect(listed.value.browser).toEqual({ + ...throughHttp.value.browser, + revision: "@fake-revision", + }); + + const commandsBefore = browser.commands.length; + const inspected = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.browserWebSocketUrl, + target_id: "allowed-page", + observation_ms: 0, + }), + ); + if (!inspected.ok) throw inspected.error; + expect(inspected.value.target.origin).toBe(browser.allowedOrigin); + expect( + browser.commands + .slice(commandsBefore, commandsBefore + 3) + .map(({ method, params }) => ({ method, params })), + ).toEqual([ + { method: "Browser.getVersion", params: {} }, + { method: "Target.getTargets", params: {} }, + { + method: "Target.attachToTarget", + params: { targetId: "allowed-page", flatten: true }, + }, + ]); + expect(browser.httpRequests).toHaveLength(httpRequests); + expect(browser.openedConnectionCount()).toBe(openedBefore + 2); + await vi.waitFor(() => expect(browser.activeConnectionCount()).toBe(0)); + }); +}); + +describe("CdpBrowserProvider: browser socket failures and cleanup", () => { + it("closes an unadopted browser socket when the target is missing", async () => { + const browser = await startFakeCdpBrowser(); + trackBrowser(browser); + const result = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.browserWebSocketUrl, + target_id: "missing", + observation_ms: 0, + }), + ); + + expect(result).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "target_not_found" }, + }); + expect(browser.commands.map(({ method }) => method)).toEqual([ + "Browser.getVersion", + "Target.getTargets", + ]); + await vi.waitFor(() => expect(browser.activeConnectionCount()).toBe(0)); + }); + + it("rejects malformed browser-socket discovery and closes the socket", async () => { + const browser = await startFakeCdpBrowser({ + commandResult: ({ method }) => + method === "Browser.getVersion" + ? { product: "FakeChrome/1.0" } + : undefined, + }); + trackBrowser(browser); + const result = await new CdpBrowserProvider().listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: browser.browserWebSocketUrl, + }), + ); + + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "BrowserObservationError", + operation: "list_browser_targets", + reason: "invalid_endpoint_response", + }, + }); + await vi.waitFor(() => expect(browser.activeConnectionCount()).toBe(0)); + }); + + it("reports a refused browser socket as unreachable", async () => { + const browser = await startFakeCdpBrowser(); + trackBrowser(browser); + const result = await new CdpBrowserProvider().listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: browser.browserWebSocketUrl.replace( + /fake$/u, + "not-this-browser", + ), + }), + ); + + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "BrowserObservationError", + reason: "endpoint_unreachable", + }, + }); + expect(JSON.stringify(result)).not.toContain("not-this-browser"); + }); + + it("cancels browser-socket discovery and releases the socket", async () => { + const browser = await startFakeCdpBrowser({ + hangOnMethod: "Target.getTargets", + }); + trackBrowser(browser); + const controller = new AbortController(); + const pending = new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.browserWebSocketUrl, + target_id: "allowed-page", + observation_ms: 0, + }), + { signal: controller.signal }, + ); + await vi.waitFor(() => + expect(browser.commands.map(({ method }) => method)).toContain( + "Target.getTargets", + ), + ); + controller.abort(); + + expect(await pending).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError", operation: "inspect_web_page" }, + }); + expect(browser.commands.map(({ method }) => method)).not.toContain( + "Target.attachToTarget", + ); + await vi.waitFor(() => expect(browser.activeConnectionCount()).toBe(0)); + }); +}); + +/** Expect a discovery timeout that projects publicly as a provider timeout. */ +const expectDiscoveryTimeout = async (url: string): Promise => { + let failure: unknown; + try { + await openCdpEndpoint( + browserObservationEndpointSchema.parse(url), + "list_browser_targets", + undefined, + 50, + ); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toMatchObject({ + _tag: "BrowserObservationError", + reason: "timeout", + }); + if (!(failure instanceof AnalysisError)) + throw new Error("Expected a typed discovery failure"); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "provider_timeout", + category: "timeout", + }); +}; + +describe("CdpBrowserProvider: browser socket discovery deadline", () => { + it("times out a discovery reply that never arrives and closes the socket", async () => { + const browser = await startFakeCdpBrowser({ + hangOnMethod: "Target.getTargets", + }); + trackBrowser(browser); + await expectDiscoveryTimeout(browser.browserWebSocketUrl); + await vi.waitFor(() => expect(browser.activeConnectionCount()).toBe(0)); + }); + + it("times out a WebSocket handshake that never completes", async () => { + const sockets = new Set(); + // Read the upgrade request but never answer it; reading lets the + // client's close be observed here. + const server = createServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.resume(); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Expected a TCP listener address"); + try { + await expectDiscoveryTimeout( + `ws://127.0.0.1:${String(address.port)}/devtools/browser/stalled`, + ); + await vi.waitFor(() => expect(sockets.size).toBe(0)); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); +}); diff --git a/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.ts b/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.ts index 81fe60759..9b2053d39 100644 --- a/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.ts @@ -3,6 +3,7 @@ import { createServer } from "node:http"; import { describe, expect, it } from "vitest"; import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; +import { listBrowserTargetsInputSchema } from "../../../src/domain/browserObservation.js"; describe("CDP target inventory", () => { it("returns every allowed target within the bounded discovery payload", async () => { @@ -51,10 +52,12 @@ describe("CDP target inventory", () => { port = address.port; const endpoint = `http://127.0.0.1:${String(port)}`; - const result = await new CdpBrowserProvider().listTargets({ - cdp_endpoint: endpoint, - allowed_origins: ["https://allowed.example"], - }); + const result = await new CdpBrowserProvider().listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: ["https://allowed.example"], + }), + ); if (!result.ok) throw result.error; expect(result.value.targets).toHaveLength(targets.length); diff --git a/tests/boundary/cli/browserCli.test.ts b/tests/boundary/cli/browserCli.test.ts index 5f8193fce..351d07512 100644 --- a/tests/boundary/cli/browserCli.test.ts +++ b/tests/boundary/cli/browserCli.test.ts @@ -183,6 +183,54 @@ it( INTEGRATION_TEST_TIMEOUT_MS, ); +it( + "browser CLI attaches through a browser socket and records it redacted", + async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const inspected = await runCli( + [ + "inspect-web-page", + browser.browserWebSocketUrl, + "allowed-page", + "--observation-ms", + "0", + "--json", + ], + process.env, + ); + expect(inspected).toMatchObject({ + operation: "inspect_web_page", + parameters: { + cdp_endpoint: browser.browserWebSocketUrl.replace( + /fake$/u, + "[REDACTED]", + ), + }, + normalized_result: { target: { target_id: "allowed-page" } }, + }); + expect(JSON.stringify(inspected)).not.toContain("/devtools/browser/fake"); + + const refused = await runCli( + [ + "list-browser-targets", + browser.browserWebSocketUrl.replace(/fake$/u, "not-this-browser"), + "--json", + ], + process.env, + ); + expect(refused).toMatchObject({ + code: "provider_unavailable", + details: { + operation: "list_browser_targets", + reason: "endpoint_unreachable", + }, + }); + expect(JSON.stringify(refused)).not.toContain("not-this-browser"); + }, + INTEGRATION_TEST_TIMEOUT_MS, +); + describe("scenario comparison CLI diagnostics", () => { const rule = { rule_id: "domain", diff --git a/tests/boundary/mcp/browserMcp.test.ts b/tests/boundary/mcp/browserMcp.test.ts index a8a780d61..b13d17df5 100644 --- a/tests/boundary/mcp/browserMcp.test.ts +++ b/tests/boundary/mcp/browserMcp.test.ts @@ -424,6 +424,47 @@ it("delivers the source-map response URL through authenticated MCP Evidence", as ).toBe(true); }); +it("records a browser-socket endpoint redacted and reconciles its Evidence", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const connected = await connectBrowser(); + const tool = (await connected.client.listTools()).tools.find( + ({ name }) => name === "inspect_web_page", + ); + expect(tool?.inputSchema.properties?.cdp_endpoint).toMatchObject({ + description: expect.stringContaining("/devtools/browser/ID"), + }); + const inspected = await connected.client.callTool({ + name: "inspect_web_page", + arguments: { + cdp_endpoint: browser.browserWebSocketUrl, + target_id: "allowed-page", + observation_ms: 0, + include_script_sources: true, + }, + }); + expect(inspected.isError).not.toBe(true); + expect(inspected.structuredContent).toMatchObject({ + parameters: { + cdp_endpoint: browser.browserWebSocketUrl.replace(/fake$/u, "[REDACTED]"), + }, + }); + expect(JSON.stringify(inspected.structuredContent)).not.toContain( + "/devtools/browser/fake", + ); + const reconciled = await connected.client.callTool({ + name: "reconcile_javascript_runtime", + arguments: { + static_layers: JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE.static_layers, + runtime_observations: [evidenceFor(inspected.structuredContent)], + }, + }); + expect(reconciled.isError).not.toBe(true); + expect(reconciled.structuredContent).toMatchObject({ + normalized_result: { summary: { runtime_scripts: 1 } }, + }); +}); + it("does not attach to a target outside the request's allowed origin scope", async () => { const browser = await startFakeCdpBrowser(); browsers.push(browser); diff --git a/tests/fixtures/fakeCdpBrowser.ts b/tests/fixtures/fakeCdpBrowser.ts index 191b4569c..aa476a58b 100644 --- a/tests/fixtures/fakeCdpBrowser.ts +++ b/tests/fixtures/fakeCdpBrowser.ts @@ -28,6 +28,7 @@ export const startFakeCdpBrowser = async ( referer: string | undefined; }[] = []; const sockets = new Set(); + let openedConnections = 0; let frameTreeReads = 0; let port = 0; const http = createServer((request, response) => { @@ -85,6 +86,7 @@ export const startFakeCdpBrowser = async ( }); webSockets.on("connection", (socket: WebSocket, request: IncomingMessage) => { const directPage = request.url?.startsWith("/devtools/page/") === true; + openedConnections += 1; sockets.add(socket); socket.on("close", () => sockets.delete(socket)); socket.on("message", (raw) => { @@ -142,6 +144,7 @@ export const startFakeCdpBrowser = async ( allowedOrigin: endpoint, commands, activeConnectionCount: () => sockets.size, + openedConnectionCount: () => openedConnections, emitRawMessage(message) { for (const socket of sockets) socket.send(message); }, diff --git a/tests/fixtures/fakeCdpBrowserResults.ts b/tests/fixtures/fakeCdpBrowserResults.ts index 908db238e..4a489227f 100644 --- a/tests/fixtures/fakeCdpBrowserResults.ts +++ b/tests/fixtures/fakeCdpBrowserResults.ts @@ -24,6 +24,8 @@ export const resultFor = ( return responseBodyResult(options); case "Page.captureScreenshot": return screenshotResult(); + case "Browser.getVersion": + return browserVersionResult(); case "Target.getTargets": return { targetInfos: targets(port, options).map(endpointTargetToInfo) }; case "Storage.getUsageAndQuota": @@ -66,6 +68,15 @@ export const resultFor = ( } }; +/** Chrome's browser-socket reply shape, mirroring the fake's `/json/version`. */ +const browserVersionResult = () => ({ + protocolVersion: "1.3", + product: "FakeChrome/1.0", + revision: "@fake-revision", + userAgent: "FakeChrome", + jsVersion: "13.0", +}); + const attachToTargetResult = (options: FakeOptions) => ({ sessionId: options.invalidAttachedSession ? "" : "session-1", }); diff --git a/tests/fixtures/fakeCdpBrowserTypes.ts b/tests/fixtures/fakeCdpBrowserTypes.ts index 07a882a1e..e75e0dc48 100644 --- a/tests/fixtures/fakeCdpBrowserTypes.ts +++ b/tests/fixtures/fakeCdpBrowserTypes.ts @@ -13,6 +13,8 @@ export interface FakeCdpBrowser { readonly commands: readonly FakeCdpCommand[]; /** Actual WebSocket connections still held by the wire fixture. */ activeConnectionCount(): number; + /** WebSocket upgrades accepted over the fixture's lifetime. */ + openedConnectionCount(): number; /** Deliver actual malformed wire data without bypassing the production transport parser. */ emitRawMessage(message: string): void; emitEvent(event: { From c6631814e0b70a324c48fda4acef12541dd30cf5 Mon Sep 17 00:00:00 2001 From: Praveen Yadav <153702595+parbin-12@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:07:39 +0530 Subject: [PATCH 09/18] docs(website): document Apple Silicon Cairo setup for local preview (#1930) * docs(website): document Apple Silicon Cairo setup for local preview * docs(website): scope Apple Silicon Cairo fix to preview process * docs(website): avoid shared-prefix rewrite and clobbering backup * docs(website): keep shared Intel libs intact with same-shell Cairo lookup * docs(website): enter ARM shell before exporting Cairo lookup * docs(website): use ARM brew path and unprotected Python for Cairo * docs(website): reinstall requirements after ARM venv rebuild * docs(website): document DejaVu font dependency * docs(website): show ARM Python selection for venv rebuild --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- website/README.md | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/website/README.md b/website/README.md index f09c7f603..6c260affd 100644 --- a/website/README.md +++ b/website/README.md @@ -33,8 +33,43 @@ After editing source HTML or a catalog, run `python3 scripts/prepare-website.py` to rebuild the preview. `serve` prepares the site once when it starts. Keep the virtual environment active when preparing assets. CairoSVG needs the native Cairo library; the sharing card uses DejaVu Sans. On Debian or Ubuntu, -install them with `sudo apt-get install libcairo2 fonts-dejavu-core`. For other -systems, see [CairoSVG's installation instructions](https://cairosvg.org/documentation/#installation). +install them with `sudo apt-get install libcairo2 fonts-dejavu-core`. On an +Apple Silicon Mac, enter an ARM shell first with `/usr/bin/arch -arm64 /bin/zsh`. Install +the ARM build with `/opt/homebrew/bin/brew install cairo pkg-config`. Install +the sharing-card font with `/opt/homebrew/bin/brew install --cask font-dejavu`. +The card in `public/assets/social-card.svg` requests DejaVu Sans, so Cairo output +uses a fallback font without it. Verify with `fc-match "DejaVu Sans"`. The result +should name DejaVu. A plain +`brew` call can select the Intel brew when `/usr/local/bin` comes first or the +terminal runs under Rosetta. Check that +`/opt/homebrew/lib/libcairo.dylib` reports `arm64`. An old Intel build in +`/usr/local/lib` shadows the ARM build. The preview then fails with +`incompatible architecture (have 'x86_64', need 'arm64')`. Keep +`/usr/local/lib` intact. Do not move its files and do not link ARM files into +it. That breaks Intel programs that use the same names. Use an unprotected ARM +Python, not Apple `/usr/bin/python3`. A venv from the Apple interpreter stays +protected and purges `DYLD` at launch. Deactivate any active venv, confirm the +shell reports `arm64`, then recreate the venv with the ARM interpreter by its +full path: + +```sh +deactivate || true +/usr/bin/arch +/opt/homebrew/bin/brew install python3 +/opt/homebrew/bin/python3 -m venv website/.venv --clear +source website/.venv/bin/activate +python3 -m pip install -r website/requirements.txt +``` + +A rebuilt venv is empty +without this step. Export `DYLD_LIBRARY_PATH=/opt/homebrew/lib` and +`DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib` inside the ARM shell. Verify +with `python3 -c "import ctypes.util; +print(ctypes.util.find_library('cairo'))"`. The result should point at +`/opt/homebrew`. Run the preview from that same shell with +`python3 scripts/website.py serve` and no `arch` prefix. A `DYLD` export +before `arch` gets purged and the preview fails again. For other systems, see +[CairoSVG's installation instructions](https://cairosvg.org/documentation/#installation). CI installs both packages explicitly before rendering the card. To test the GitHub Pages prefix, use From 93e29d242ae60a3d2daec355e7a31b4a0bdd4234 Mon Sep 17 00:00:00 2001 From: To3akaRin <131595336+To3akaRin@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:37:54 +0800 Subject: [PATCH 10/18] feat(cli): summarize committed reference-source inventories (#2768) --- docs/cli.md | 28 +++ src/cli/utilityCommands.ts | 48 ++++- src/domain/referenceSourceSummary.test.ts | 104 ++++++++++ src/domain/referenceSourceSummary.ts | 93 +++++++++ .../cli/referenceSourceSummary.test.ts | 190 ++++++++++++++++++ 5 files changed, 461 insertions(+), 2 deletions(-) create mode 100644 src/domain/referenceSourceSummary.test.ts create mode 100644 src/domain/referenceSourceSummary.ts create mode 100644 tests/boundary/cli/referenceSourceSummary.test.ts diff --git a/docs/cli.md b/docs/cli.md index 6d81ec7c1..c43ec2181 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -214,8 +214,36 @@ Import an older source tree to compare with the current artifact: ```bash rea import-reference-source /absolute/path/to/source +rea import-reference-source /absolute/path/to/source --detail summary --outline-depth 2 --json ``` +The default (`--detail full`) returns the complete `HistoricalSourceGraph`. +Choose `--detail summary` for a `HistoricalSourceSummary` of that same committed +inventory: its `HistoricalSourceManifest` contains the full graph and root +digests, with provenance, VCS facts, languages, manifest paths, entry and +relationship counts, parse-failure counts, file/directory states, exclusions +and limitations. The summary is a presentation view, not a graph accepted by +source-to-bundle comparison; keep the full output for that workflow. + +`--outline-depth` requires `--detail summary` and accepts a nonnegative safe +integer. Depth 1 shows root children, depth 2 (the default) also shows their +children, and depth 0 omits all outline entries. Paths retain canonical Unicode +code-point order. `outline.omitted_entry_count` counts only inventoried entries +hidden by the chosen depth; it does not count unexamined children of excluded or +unreadable directories, or change `inventory_state`. Exclusions and source +limitations remain separate from this presentation omission. + +`file_sizes.known_bytes` is the exact sum of known inventoried file sizes as +decimal text, so large sums cannot lose integer precision. Unknown sizes count +in `file_sizes.unknown_file_count`; excluded or unexamined paths are not included +in that total. A zero known-byte sum therefore does not establish an empty source +tree. Full output retains individual file hashes, relationships and parse-failure +diagnostics. Summary mode performs the same import and cleanup, then projects +the committed result without rereading source files. It reduces the selected +output, not the amount of source examined. A shallow tree or many exclusions can +still produce a large summary; depth 0 removes the outline but preserves the +other facts. No fixed output-size or runtime improvement is promised. + The import records hashes and metadata for the supplied files, separately from observations of the current app. For a directory inside a Git checkout, it records the HEAD of the nearest enclosing repository; whether the working diff --git a/src/cli/utilityCommands.ts b/src/cli/utilityCommands.ts index 2a9d05a4c..a5da57ac5 100644 --- a/src/cli/utilityCommands.ts +++ b/src/cli/utilityCommands.ts @@ -2,6 +2,7 @@ import { z } from "incur"; import type { DirectAnalysis } from "../composition/directAnalysis.js"; import type { ReferenceSourcePolicy } from "../domain/referenceSourcePolicy.js"; +import { createHistoricalSourceSummary } from "../domain/referenceSourceSummary.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { importReferenceSource } from "../application/ReferenceSourceImport.js"; @@ -241,11 +242,48 @@ const registerReferenceSourceCommand = ( .string() .describe("Local source directory to import as historical reference"), }), - run: ({ args }) => + options: z.object({ + detail: z + .enum(["full", "summary"]) + .default("full") + .describe( + "Complete committed graph or compact historical-source summary", + ), + outlineDepth: z + .number() + .int() + .nonnegative() + .safe() + .optional() + .describe( + "Summary outline path depth (default: 2; 0 omits entries); requires --detail summary", + ), + }), + run: ({ args, options }) => logCliCommand( logger, "import-reference-source", async () => { + if ( + options.outlineDepth !== undefined && + options.detail !== "summary" + ) + return { + error: "Import failed", + ...projectAnalysisError( + new AnalysisInputError( + "import-reference-source", + { userMessage: "--outline-depth requires --detail summary" }, + [ + { + path: ["outlineDepth"], + reason: "invalid_value", + message: "--outline-depth requires --detail summary", + }, + ], + ), + ), + }; const config = parseConfig(environment); if (!config.ok) return { @@ -255,6 +293,9 @@ const registerReferenceSourceCommand = ( return runReferenceSourceImport( args.root, config.value.referenceSourcePolicy, + options.detail === "summary" + ? (options.outlineDepth ?? 2) + : undefined, ); }, isReferenceSourceImportCliFailure, @@ -265,6 +306,7 @@ const registerReferenceSourceCommand = ( const runReferenceSourceImport = async ( root: string, policy: ReferenceSourcePolicy, + outlineDepth: number | undefined, ): Promise => { const resources = new ArtifactResourceScope(); let outcome: @@ -346,7 +388,9 @@ const runReferenceSourceImport = async ( cause: closeOutcome.cause, }), }; - return imported.value; + return outlineDepth === undefined + ? imported.value + : createHistoricalSourceSummary(imported.value, outlineDepth); }; const mergeReferenceSourceCleanup = ( diff --git a/src/domain/referenceSourceSummary.test.ts b/src/domain/referenceSourceSummary.test.ts new file mode 100644 index 000000000..a0d23b0e1 --- /dev/null +++ b/src/domain/referenceSourceSummary.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; + +import { + createHistoricalSourceGraph, + createHistoricalSourceManifest, + type HistoricalSourceGraphInput, +} from "./referenceSourceGraph.js"; +import { createHistoricalSourceSummary } from "./referenceSourceSummary.js"; + +const input = ( + entries: HistoricalSourceGraphInput["entries"], +): HistoricalSourceGraphInput => ({ + schema: "HistoricalSourceGraph", + authority: "historical-reference", + root_alias: "$REFERENCE_ROOT", + inventory_state: "unknown", + entries, + relationships: [], + parse_failures: [], + exclusions: [], + languages: [], + manifests: [], + vcs: null, + provenance: { importer: "fixture", importer_version: null, caller: "test" }, + limitations: ["Inventory coverage is unknown"], +}); + +describe("committed historical-source summaries", () => { + it("preserves unknown coverage, sizes, directories and missing symlinks without mutating the graph", () => { + const graph = createHistoricalSourceGraph( + input([ + { + path: "a", + kind: "directory", + tree_state: "unknown", + classifications: ["unknown"], + limitations: ["Not enumerated"], + }, + { + path: "b", + kind: "file", + sha256: null, + size: null, + content_state: "unreadable", + language: null, + classifications: ["unknown"], + limitations: ["Not readable"], + }, + { + path: "c", + kind: "symlink", + target: "missing", + target_state: "missing", + classifications: ["unknown"], + limitations: [], + }, + ]), + ); + const original = structuredClone(graph); + const summary = createHistoricalSourceSummary(graph, 1); + expect(summary).toMatchObject({ + manifest: createHistoricalSourceManifest(original), + inventory_state: "unknown", + vcs: null, + provenance: original.provenance, + file_sizes: { known_bytes: "0", unknown_file_count: 1 }, + counts: { entries: 3, files: 1, directories: 1, symlinks: 1 }, + directory_tree_states: { unknown: 1, enumerated: 0 }, + file_content_states: { unreadable: 1, hashed: 0 }, + limitations: original.limitations, + outline: { + entries: [ + original.entries[0], + expect.objectContaining({ size: null, content_state: "unreadable" }), + original.entries[2], + ], + omitted_entry_count: 0, + }, + }); + expect(summary.outline.entries[1]).not.toHaveProperty("sha256"); + expect(graph).toEqual(original); + }); + + it("sums known bytes losslessly beyond the JSON safe integer range", () => { + const graph = createHistoricalSourceGraph( + input( + ["a", "b"].map((path) => ({ + path, + kind: "file", + sha256: "a".repeat(64), + size: Number.MAX_SAFE_INTEGER, + content_state: "hashed", + language: null, + classifications: ["unknown"], + limitations: [], + })), + ), + ); + expect(createHistoricalSourceSummary(graph, 0)).toMatchObject({ + file_sizes: { known_bytes: "18014398509481982", unknown_file_count: 0 }, + outline: { entries: [], omitted_entry_count: 2 }, + }); + }); +}); diff --git a/src/domain/referenceSourceSummary.ts b/src/domain/referenceSourceSummary.ts new file mode 100644 index 000000000..e7e99e3fa --- /dev/null +++ b/src/domain/referenceSourceSummary.ts @@ -0,0 +1,93 @@ +import { + createHistoricalSourceManifest, + type HistoricalSourceGraph, +} from "./referenceSourceGraph.js"; + +type Entry = HistoricalSourceGraph["entries"][number]; + +const outlineEntry = (entry: Entry) => { + if (entry.kind !== "file") return entry; + return { + path: entry.path, + kind: entry.kind, + language: entry.language, + size: entry.size, + content_state: entry.content_state, + classifications: entry.classifications, + limitations: entry.limitations, + }; +}; + +/** Project one committed inventory without rereading or reinterpreting source. */ +export const createHistoricalSourceSummary = ( + graph: HistoricalSourceGraph, + outlineDepth: number, +) => { + const manifest = createHistoricalSourceManifest(graph); + const counts = { + entries: graph.entries.length, + files: 0, + directories: 0, + symlinks: 0, + relationships: graph.relationships.length, + parse_failures: graph.parse_failures.length, + exclusions: graph.exclusions.length, + }; + const contentStates = { + hashed: 0, + "redacted-secret": 0, + excluded: 0, + unreadable: 0, + unknown: 0, + }; + const treeStates = { + enumerated: 0, + partial: 0, + excluded: 0, + unreadable: 0, + unknown: 0, + }; + let knownBytes = 0n; + let unknownSizeFiles = 0; + const outline: ReturnType[] = []; + for (const entry of graph.entries) { + if (entry.kind === "file") { + counts.files += 1; + contentStates[entry.content_state] += 1; + if (entry.size === null) unknownSizeFiles += 1; + else knownBytes += BigInt(entry.size); + } else if (entry.kind === "directory") { + counts.directories += 1; + treeStates[entry.tree_state] += 1; + } else counts.symlinks += 1; + // Canonical entries already use Unicode code-point order; filtering preserves it. + if (entry.path.split("/").length <= outlineDepth) + outline.push(outlineEntry(entry)); + } + return { + schema: "HistoricalSourceSummary" as const, + authority: graph.authority, + root_alias: graph.root_alias, + manifest, + inventory_state: graph.inventory_state, + provenance: graph.provenance, + vcs: graph.vcs, + languages: graph.languages, + manifests: graph.manifests, + counts, + file_content_states: contentStates, + directory_tree_states: treeStates, + file_sizes: { + // Decimal text keeps large totals exact; this covers only known inventoried sizes. + known_bytes: knownBytes.toString(), + unknown_file_count: unknownSizeFiles, + }, + exclusions: graph.exclusions, + limitations: graph.limitations, + outline: { + depth: outlineDepth, + entries: outline, + omitted_entry_count: graph.entries.length - outline.length, + }, + }; +}; diff --git a/tests/boundary/cli/referenceSourceSummary.test.ts b/tests/boundary/cli/referenceSourceSummary.test.ts new file mode 100644 index 000000000..aed8cb871 --- /dev/null +++ b/tests/boundary/cli/referenceSourceSummary.test.ts @@ -0,0 +1,190 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect } from "vitest"; + +import { + createHistoricalSourceManifest, + parseHistoricalSourceGraph, +} from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +describe.skipIf(process.platform === "win32")( + "compiled historical-source summaries", + () => { + cliTest( + "selects a committed summary without changing complete output", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-summary-"); + await mkdir(join(root, "src", "nested"), { recursive: true }); + await Promise.all([ + writeFile(join(root, "package.json"), '{"name":"fixture"}\n'), + writeFile( + join(root, "src", "main.js"), + 'import "./nested/value.js";\n', + ), + writeFile( + join(root, "src", "nested", "value.js"), + "export const value = 1;\n", + ), + writeFile(join(root, "\uE000.js"), "export const first = 1;\n"), + writeFile(join(root, "\u{10000}.js"), "export const last = 2;\n"), + ]); + const run = (...options: string[]) => + cli.run({ + arguments: ["import-reference-source", root, "--json", ...options], + cwd: root, + }); + const complete = await run(); + expect(complete.exitCode).toBe(0); + const graph = parseHistoricalSourceGraph(complete.json); + expect((await run("--detail", "full")).stdout).toBe(complete.stdout); + for (const depth of [undefined, 0, 1, 10]) { + const result = await run( + "--detail", + "summary", + ...(depth === undefined ? [] : ["--outline-depth", String(depth)]), + ); + expect(result.exitCode).toBe(0); + const selected = graph.entries.filter( + (entry) => entry.path.split("/").length <= (depth ?? 2), + ); + expect(result.json).toMatchObject({ + schema: "HistoricalSourceSummary", + authority: graph.authority, + root_alias: graph.root_alias, + manifest: createHistoricalSourceManifest(graph), + inventory_state: graph.inventory_state, + provenance: graph.provenance, + vcs: graph.vcs, + languages: graph.languages, + manifests: graph.manifests, + counts: { + entries: 7, + files: 5, + directories: 2, + symlinks: 0, + relationships: 1, + parse_failures: 0, + exclusions: 0, + }, + outline: { + depth: depth ?? 2, + entries: selected.map(({ path, kind }) => + expect.objectContaining({ path, kind }), + ), + omitted_entry_count: graph.entries.length - selected.length, + }, + }); + expect(result.json).not.toHaveProperty("relationships"); + } + }, + ); + + cliTest( + "retains exclusions and parse coverage independently of a hidden outline", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-partial-summary-", + ); + await Promise.all([ + writeFile(join(root, ".gitignore"), "ignored.txt\n"), + writeFile(join(root, "ignored.txt"), "ignored\n"), + writeFile(join(root, "broken.js"), "export const = ;\n"), + ]); + const result = await cli.run({ + arguments: [ + "import-reference-source", + root, + "--detail", + "summary", + "--outline-depth", + "0", + "--json", + ], + cwd: root, + }); + expect(result.exitCode).toBe(0); + expect(result.json).toMatchObject({ + inventory_state: "partial", + counts: { parse_failures: 1, exclusions: 1 }, + exclusions: [ + { + path: "ignored.txt", + reason: "project-ignored", + pattern: "ignored.txt", + }, + ], + outline: { depth: 0, entries: [], omitted_entry_count: 2 }, + }); + }, + ); + }, +); + +describe("compiled historical-source summary input boundaries", () => { + cliTest( + "documents detail and depth in public command help", + async ({ cli }) => { + const result = await cli.run({ + arguments: ["import-reference-source", "--help"], + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("--detail"); + expect(result.stdout).toContain("--outline-depth"); + expect(result.stdout).toContain("requires --detail summary"); + }, + ); + + for (const options of [ + ["--detail", "summry"], + ["--detail", "summary", "--outline-depth", "-1"], + ["--detail", "summary", "--outline-depth", "1.5"], + ["--detail", "summary", "--outline-depth", "NaN"], + ["--detail", "summary", "--outline-depth", "9007199254740992"], + ["--outline-depth", "1"], + ["--detail", "full", "--outline-depth", "1"], + ]) { + cliTest(`rejects ${options.join(" ")}`, async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-summary-input-", + ); + const result = await cli.run({ + arguments: ["import-reference-source", root, ...options, "--json"], + cwd: root, + }); + expect(result.exitCode).toBe(1); + expect(result.stdout).not.toContain("HistoricalSourceGraph"); + expect(result.stdout).not.toContain("HistoricalSourceSummary"); + expect(result.stdout).toMatch(/detail|outline/u); + }); + } + + cliTest( + "preserves the missing-root failure in summary mode", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-summary-missing-", + ); + const result = await cli.run({ + arguments: [ + "import-reference-source", + join(root, "missing"), + "--detail", + "summary", + "--json", + ], + cwd: root, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + error: "Import failed", + category: "invalid_input", + message: expect.stringContaining( + "Reference source root could not be resolved", + ), + }); + }, + ); +}); From 1b6c899b031421dcf2d725a63e67d99375028d6e Mon Sep 17 00:00:00 2001 From: Lin Gan <190970720@qq.com> Date: Wed, 14 Oct 2026 00:42:03 +0800 Subject: [PATCH 11/18] docs: update Chinese README guides (#2119) Update Chinese README docs Refresh the Chinese README guidance to mention IDA as a supported native analysis backend, clarify that static JavaScript analysis does not require native engines, and document global CLI installation/update steps for Bun (`bun add --global`, `bunx` usage, minimum version 1.2.17). The capability table also now includes Go binary and WASM artifact support. Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/readme/README_zh-TW.md | 10 +++++++--- docs/readme/README_zh.md | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/readme/README_zh-TW.md b/docs/readme/README_zh-TW.md index 44956b8eb..1ddbbd26e 100644 --- a/docs/readme/README_zh-TW.md +++ b/docs/readme/README_zh-TW.md @@ -51,7 +51,7 @@ REA 將 AI 代理連接到分析工具，用於檢查原生二進位檔案、JavaScript 和 Electron 應用、.NET 程式集以及網站。你也可以在終端中使用同樣的工具。分析在本機執行，結果包含支撐各項結論的證據和相關限制。 -設定流程會向 AI 代理註冊 REA，並安裝對應的工作流程指引。原生分析可以使用已有的 Hopper 或 Ghidra；設定流程也可以在你核准後安裝 Hopper。靜態 JavaScript 分析不需要這兩個引擎。 +設定流程會向 AI 代理註冊 REA，並安裝對應的工作流程指引。原生分析可以使用已有的 Hopper、Ghidra 或 IDA；設定流程也可以在你核准後安裝 Hopper。靜態 JavaScript 分析不需要原生分析引擎。 > **[訪問 REA 網站](https://rea.tools/zh-TW/)**，查看設定說明、圖解指南和真實案例。 @@ -100,12 +100,14 @@ rea --help REA 更新較快，新版本經常包含錯誤修復。請將安裝的版本保持最新。 -通過 npm 安裝的 CLI 使用： +通過 `npm install --global` 或 `bun add --global` 安裝的 CLI 使用： ```bash rea update ``` +Bun 全域更新需要 Bun 1.2.17 或更新版本。如果使用較舊的 Bun 版本，請先手動升級 Bun，再執行 `rea update`。 + 要更新 AI 代理註冊和 skill，請執行更新命令輸出的設定命令。 如果使用 `npx`，通過以下命令更新 AI 代理設定： @@ -114,7 +116,7 @@ rea update npx rea-agents@latest setup ``` -檢查設定變更並重新啟動 AI 代理。單次 CLI 操作可使用 `npx rea-agents@latest`，後接所需命令。 +檢查設定變更並重新啟動 AI 代理。單次 CLI 操作可使用 `npx rea-agents@latest`，後接所需命令。使用 Bun 時，用 `bunx rea-agents@latest` 代替 `npx rea-agents@latest`。 ## REA 的工作原理 @@ -134,6 +136,8 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | --------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | | 原生二進位檔案 | 偽程式碼、組合語言、字串、符號、呼叫和引用 | Hopper、Ghidra 或 IDA；[原生分析](https://rea.tools/zh-TW/guides/native/) | | 離線 ELF 佈局 | 節、段、原始符號/重定位資訊及靜態防護機制候選項 | Linux x64 上由呼叫方提供的 pwntools；[二進位診斷](../binary-diagnostics.md) | +| Go 二進位檔案 | 內嵌編譯器版本、模組、替換項和建置設定 | 內建靜態 ELF/PE/thin Mach-O 檢查；無需安裝 Go | +| WASM 工件 | 驗證、節、精確的匯入/匯出 WAT 形式及解碼後的 WAT | 呼叫方提供的 WABT 1.0.42；`inspect_wasm_artifact` | | EVM 位元組碼 | 分派選擇器、位元組偏移、推斷的參數和狀態可變性 | Linux x64 上的本地原始位元組/十六進位輸入載體；[離線 EVM 指南](../evm-bytecode.md) | | 已記錄的 Linux 崩潰 | 原始 note 記錄、每個已記錄執行緒的暫存器/訊號及可選的對映候選項 | Linux x64 上由呼叫方提供的 pwntools；可選的 GDB/pwndbg；[已記錄崩潰](../recorded-crashes.md) | | JavaScript / Electron | 模組、匯入、source map、路由、IPC 和原生附加元件關係 | Node.js 和 npm；[應用分析](https://rea.tools/zh-TW/guides/javascript/) | diff --git a/docs/readme/README_zh.md b/docs/readme/README_zh.md index 85e51651f..6e70fd3c3 100644 --- a/docs/readme/README_zh.md +++ b/docs/readme/README_zh.md @@ -51,7 +51,7 @@ REA 将智能体连接到分析工具，用于检查原生二进制文件、JavaScript 和 Electron 应用、.NET 程序集以及网站。你也可以在终端中使用同样的工具。分析在本机运行，结果包含支撑各项结论的证据和相关限制。 -设置流程会向智能体注册 REA，并安装匹配的工作流指引。原生分析可以使用已有的 Hopper 或 Ghidra；设置流程也可以在你批准后安装 Hopper。静态 JavaScript 分析不需要这两个引擎。 +设置流程会向智能体注册 REA，并安装匹配的工作流指引。原生分析可以使用已有的 Hopper、Ghidra 或 IDA；设置流程也可以在你批准后安装 Hopper。静态 JavaScript 分析不需要原生分析引擎。 > **[访问 REA 网站](https://rea.tools/zh-CN/)**，查看设置说明、图解指南和真实案例。 @@ -100,12 +100,14 @@ rea --help REA 更新较快，新版本经常包含错误修复。请保持安装的版本为最新。 -通过 npm 安装的 CLI 使用： +通过 `npm install --global` 或 `bun add --global` 安装的 CLI 使用： ```bash rea update ``` +Bun 全局更新需要 Bun 1.2.17 或更高版本。如果使用较旧的 Bun 版本，请先手动升级 Bun，再运行 `rea update`。 + 要更新智能体注册和 skill，请运行更新命令输出的设置命令。 如果使用 `npx`，通过以下命令更新智能体设置： @@ -114,7 +116,7 @@ rea update npx rea-agents@latest setup ``` -检查设置变更并重启智能体。单次 CLI 操作可使用 `npx rea-agents@latest`，后接所需命令。 +检查设置变更并重启智能体。单次 CLI 操作可使用 `npx rea-agents@latest`，后接所需命令。使用 Bun 时，用 `bunx rea-agents@latest` 代替 `npx rea-agents@latest`。 ## REA 的工作原理 @@ -134,6 +136,8 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | --------------------- | ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | | 原生二进制文件 | 伪代码、汇编、字符串、符号、调用和引用 | Hopper、Ghidra 或 IDA；[原生分析](https://rea.tools/zh-CN/guides/native/) | | 离线 ELF 布局 | 节、段、原始符号/重定位信息及静态防护机制候选项 | Linux x64 上由调用方提供的 pwntools；[二进制诊断](../binary-diagnostics.md) | +| Go 二进制文件 | 内嵌编译器版本、模块、替换项和构建设置 | 内置静态 ELF/PE/thin Mach-O 检查；无需安装 Go | +| WASM 工件 | 验证、节、精确的导入/导出 WAT 形式及解码后的 WAT | 调用方提供的 WABT 1.0.42；`inspect_wasm_artifact` | | EVM 字节码 | 分派选择器、字节偏移、推断的参数和状态可变性 | Linux x64 上的本地原始字节/十六进制输入载体；[离线 EVM 指南](../evm-bytecode.md) | | 已记录的 Linux 崩溃 | 原始 note 记录、每个已记录线程的寄存器/信号及可选的映射候选项 | Linux x64 上由调用方提供的 pwntools；可选的 GDB/pwndbg；[已记录崩溃](../recorded-crashes.md) | | JavaScript / Electron | 模块、导入、source map、路由、IPC 和原生扩展关系 | Node.js 和 npm；[应用分析](https://rea.tools/zh-CN/guides/javascript/) | From 69bd81f7f2768b22fb75caa753f4b21a8e5d249d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:56:43 +0800 Subject: [PATCH 12/18] fix: isolate Linux inventory and repair DEX schema regex (#2910) * fix(ci): isolate Linux Hopper process fixture inventory * fix(android): make DEX descriptor pattern portable --- scripts/lib/clean-linux-hopper-runtime.d.mts | 1 + scripts/lib/clean-linux-hopper-runtime.mjs | 7 ++--- src/domain/android/androidAnalysis.ts | 2 +- .../cleanLinuxHopperRuntime.test.ts | 26 ++++++++++++++++--- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/scripts/lib/clean-linux-hopper-runtime.d.mts b/scripts/lib/clean-linux-hopper-runtime.d.mts index dd1010aa1..d2177fad4 100644 --- a/scripts/lib/clean-linux-hopper-runtime.d.mts +++ b/scripts/lib/clean-linux-hopper-runtime.d.mts @@ -1,3 +1,4 @@ export function assertCleanLinuxHopperRuntime( launcherPaths?: readonly string[], + procRoot?: string, ): Promise; diff --git a/scripts/lib/clean-linux-hopper-runtime.mjs b/scripts/lib/clean-linux-hopper-runtime.mjs index 27a306fce..fbe15e0d3 100644 --- a/scripts/lib/clean-linux-hopper-runtime.mjs +++ b/scripts/lib/clean-linux-hopper-runtime.mjs @@ -16,6 +16,7 @@ export const assertCleanLinuxHopperRuntime = async ( ? [] : [process.env.HOPPER_LAUNCHER_PATH]), ], + procRoot = "/proc", ) => { const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]); if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout)) @@ -32,17 +33,17 @@ export const assertCleanLinuxHopperRuntime = async ( // /proc exposes the executed file even when argv[0] names a removed symlink. // Compare file identity, not a display path or a process name. let inspectionError; - for (const entry of await readdir("/proc", { withFileTypes: true })) { + for (const entry of await readdir(procRoot, { withFileTypes: true })) { if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue; let executable; try { - executable = await stat(`/proc/${entry.name}/exe`, { bigint: true }); + executable = await stat(`${procRoot}/${entry.name}/exe`, { bigint: true }); } catch (error) { if (error?.code === "ENOENT" || error?.code === "ESRCH") continue; // Other users' executables may be unreadable. The setup lane launches // providers as the caller; unreadable caller-owned processes fail closed. if (error?.code === "EACCES") { - const owner = await stat(`/proc/${entry.name}`).catch((cause) => { + const owner = await stat(`${procRoot}/${entry.name}`).catch((cause) => { if (cause?.code === "ENOENT") return undefined; throw cause; }); diff --git a/src/domain/android/androidAnalysis.ts b/src/domain/android/androidAnalysis.ts index c1ec76ba8..caa3bf5ee 100644 --- a/src/domain/android/androidAnalysis.ts +++ b/src/domain/android/androidAnalysis.ts @@ -102,7 +102,7 @@ const textCoverage = z.strictObject({ export const dexDescriptorSchema = z .string() .regex( - /^\((?:\[*(?:[ZBSCIJFD]|L[^;()]+;))*\)(?:V|\[*(?:[ZBSCIJFD]|L[^;()]+;))$/u, + /^\((?:\[*(?:[ZBSCIJFD]|L[^;\x28\x29]+;))*\)(?:V|\[*(?:[ZBSCIJFD]|L[^;\x28\x29]+;))$/u, ); const method = z.strictObject({ diff --git a/tests/process-global/cleanLinuxHopperRuntime.test.ts b/tests/process-global/cleanLinuxHopperRuntime.test.ts index f6b9fc677..8b7be17da 100644 --- a/tests/process-global/cleanLinuxHopperRuntime.test.ts +++ b/tests/process-global/cleanLinuxHopperRuntime.test.ts @@ -1,6 +1,13 @@ import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { chmod, copyFile, readFile, symlink, unlink } from "node:fs/promises"; +import { + chmod, + copyFile, + mkdir, + readFile, + symlink, + unlink, +} from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -24,6 +31,8 @@ it const directory = await createTestTempDirectory("rea-clean-runtime-test-"); const launcher = join(directory, "Hopper"); const alternate = join(directory, "Hopper-alternate"); + const procRoot = join(directory, "proc"); + await mkdir(procRoot); const alternateMode = mode === "alternate launcher"; if (alternateMode) { await copyFile("/usr/bin/sleep", launcher); @@ -32,7 +41,10 @@ it } // A rejection must come from the fixture we are about to launch. An // existing Xvfb would otherwise mask a broken executable-identity check. - await assertCleanLinuxHopperRuntime(alternateMode ? [launcher] : []); + await assertCleanLinuxHopperRuntime( + alternateMode ? [launcher] : [], + procRoot, + ); const command = alternateMode ? alternate : process.execPath; const arguments_ = alternateMode ? ["30"] : ["-e", childCode]; const parentCode = ` @@ -59,6 +71,11 @@ it const { stdout } = await parent; const { pid } = JSON.parse(stdout) as { pid: number }; orphanPid = pid; + if (alternateMode) { + const processEntry = join(procRoot, String(pid)); + await mkdir(processEntry); + await symlink(`/proc/${String(pid)}/exe`, join(processEntry, "exe")); + } const { stdout: coordinates } = await execFileAsync("ps", [ "-p", String(pid), @@ -92,7 +109,10 @@ it await unlink(alternate); } await expect( - assertCleanLinuxHopperRuntime(alternateMode ? [launcher] : []), + assertCleanLinuxHopperRuntime( + alternateMode ? [launcher] : [], + procRoot, + ), ).rejects.toThrow("Clean Linux runner retained a Hopper demo process"); // Observation must not terminate the fixture, even though it resembles Xvfb. const { stdout: alive } = await execFileAsync("ps", [ From f1045001f03ca0ddb2247769e3b4d1fadedaf630 Mon Sep 17 00:00:00 2001 From: R <115410287+JFSAS@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:57:42 +0800 Subject: [PATCH 13/18] fix(browser): retain optional source-map requires (#2886) Carry the require callee from dependency extraction into lexical scope validation so optional calls remain visible without admitting shadowed loaders. Fixes #2883 --- src/browser/WebSourceMapFetcher.ts | 15 ++-- .../browser/optionalSourceMapRequire.test.ts | 69 +++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 tests/boundary/browser/optionalSourceMapRequire.test.ts diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index c4bae782a..5c8f86ea0 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -743,8 +743,8 @@ const originalModuleEdges = ( check(); const dependency = originalDependency(node); if (dependency === null) return; - const { kind, specifier } = dependency; - if (kind === "require" && t.isCallExpression(node)) { + const { kind, specifier, requireCallee } = dependency; + if (requireCallee !== undefined) { unboundRequires ??= new Set( analyzeParsedJavaScriptReferences(parsed, "require") .filter( @@ -756,7 +756,7 @@ const originalModuleEdges = ( `${String(location.start.line)}:${String(location.start.column)}`, ), ); - const location = node.callee.loc?.start; + const location = requireCallee.loc?.start; if ( location === undefined || !unboundRequires.has( @@ -1039,6 +1039,7 @@ const originalDependency = ( ): { readonly kind: ParsedSourceMapItem["original_module_edges"][number]["kind"]; readonly specifier: string; + readonly requireCallee?: t.Identifier; } | null => { if ( t.isImportDeclaration(node) || @@ -1051,11 +1052,15 @@ const originalDependency = ( if (t.isImportExpression(node) && t.isStringLiteral(node.source)) return { kind: "dynamic_import", specifier: node.source.value }; if ( - t.isCallExpression(node) && + (t.isCallExpression(node) || t.isOptionalCallExpression(node)) && t.isIdentifier(node.callee, { name: "require" }) && t.isStringLiteral(node.arguments[0]) ) - return { kind: "require", specifier: node.arguments[0].value }; + return { + kind: "require", + specifier: node.arguments[0].value, + requireCallee: node.callee, + }; // `import x = require("m")` is only legal at module top level and always // refers to the host loader, so it needs no unbound-`require` check. The // `moduleReference` is an entity name for a local alias instead, which diff --git a/tests/boundary/browser/optionalSourceMapRequire.test.ts b/tests/boundary/browser/optionalSourceMapRequire.test.ts new file mode 100644 index 000000000..e7e861d77 --- /dev/null +++ b/tests/boundary/browser/optionalSourceMapRequire.test.ts @@ -0,0 +1,69 @@ +import { createHash } from "node:crypto"; +import { createServer } from "node:http"; + +import { expect, it } from "vitest"; + +import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; +import { analyzeWebBundleInputSchema } from "../../../src/domain/webBundleAnalysis.js"; + +it("retains unshadowed optional require dependencies from HTTP source-map originals", async () => { + const source = [ + 'require("./normal.cjs");', + 'require?.("./optional.cjs");', + 'function local(require) { require?.("./shadowed.cjs"); }', + "require?.(selectedModule);", + ].join("\n"); + const text = JSON.stringify({ + version: 3, + names: [], + mappings: "AAAA", + sources: ["main.cjs"], + sourcesContent: [source], + }); + const server = createServer((_request, response) => response.end(text)); + try { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (address === null || typeof address === "string") + throw new TypeError("Expected a TCP listener address"); + const origin = `http://127.0.0.1:${String(address.port)}`; + const url = `${origin}/app.js.map`; + const result = await fetchWebSourceMaps( + [{ scriptKey: `scr_${"1".repeat(64)}`, declaredUrl: url, fetchUrl: url }], + analyzeWebBundleInputSchema.parse({ + cdp_endpoint: origin, + target_id: "fixture", + allowed_origins: [origin], + fetch_source_maps: true, + }), + ); + expect(result.items[0]).toMatchObject({ + status: "included", + artifact: { + text, + bytes: Buffer.byteLength(text), + sha256: createHash("sha256").update(text).digest("hex"), + }, + original_sources: [{ artifact: { text: source } }], + original_module_edges: ["./normal.cjs", "./optional.cjs"].map( + (specifier) => ({ + from_source: `${origin}/main.cjs`, + kind: "require", + specifier, + resolved_source: new URL(specifier, origin).href, + }), + ), + }); + } finally { + server.closeAllConnections(); + if (server.listening) + await new Promise((resolve, reject) => { + server.close((error) => + error === undefined ? resolve() : reject(error), + ); + }); + } +}); From c8ef1f916cb0f94a39d74320f58398ba1bc2b17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Kalayc=C4=B1?= Date: Tue, 13 Oct 2026 20:04:55 +0300 Subject: [PATCH 14/18] docs(readme): sync Turkish translation with current README (#2777) docs(readme): sync Turkish translation --- docs/readme/README_tr.md | 81 ++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index dc33a4a38..8463c300d 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -51,7 +51,7 @@ Bir uygulamada kendi ürününüze eklemek istediğiniz bir özellik mi gördün REA, ajanınızı yerel makine kodu içeren ikili dosyaları, JavaScript ve Electron uygulamalarını, .NET derlemelerini ve web sitelerini inceleyen araçlara bağlar. Aynı araçları terminalden de kullanabilirsiniz. Analiz yerel olarak çalışır; sonuçlar her çıkarımın dayandığı kanıtları ve sınırlamaları içerir. -Kurulum, REA'yı ajanınıza kaydeder ve uyumlu iş akışı talimatlarını yükler. Yerel makine kodu analizi mevcut bir Hopper veya Ghidra kurulumunu kullanabilir; kurulum işlemi, onayınızla isteğe bağlı olarak Hopper da yükleyebilir. Statik JavaScript analizi bu iki motora da ihtiyaç duymaz. +Kurulum, REA'yı ajanınıza kaydeder ve uyumlu iş akışı talimatlarını yükler. Yerel makine kodu analizi mevcut bir Hopper, Ghidra veya IDA kurulumunu kullanabilir; kurulum işlemi, onayınızla isteğe bağlı olarak Hopper da yükleyebilir. Statik JavaScript analizi bu yerel analiz motorlarından hiçbirine ihtiyaç duymaz. > Kurulum talimatları, görselli rehberler ve gerçek örnek çalışmalar için **[REA web sitesini ziyaret edin](https://rea.tools/tr/)**. @@ -100,12 +100,14 @@ Yerel makine kodu analizi için önce bir sağlayıcı yapılandırın. Yerel an REA hızla değişir ve yeni sürümler sık sık hata düzeltmeleri içerir. Kurulumunuzu güncel tutun. -npm ile yüklenen CLI için: +`npm install --global` veya `bun add --global` ile yüklenen CLI için: ```bash rea update ``` +Bun ile yapılan global güncellemeler için Bun 1.2.17 veya üzeri gerekir. Daha eski bir Bun sürümünü kullanıyorsanız `rea update` komutunu çalıştırmadan önce Bun'u manuel olarak yükseltin. + Ajan kayıtlarını ve skill'i güncellemek için güncellemenin çıktısında gösterilen kurulum komutunu çalıştırın. `npx` kullanıyorsanız ajan kurulumunu şu komutla güncelleyin: @@ -114,7 +116,7 @@ Ajan kayıtlarını ve skill'i güncellemek için güncellemenin çıktısında npx rea-agents@latest setup ``` -Kurulum değişikliklerini inceleyin ve ajanınızı yeniden başlatın. Tek seferlik CLI işlemleri için `npx rea-agents@latest` komutunun ardından istediğiniz komutu yazın. +Kurulum değişikliklerini inceleyin ve ajanınızı yeniden başlatın. Tek seferlik CLI işlemleri için `npx rea-agents@latest` komutunun ardından istediğiniz komutu yazın. Bun kullanıyorsanız `npx rea-agents@latest` yerine `bunx rea-agents@latest` kullanın. ## REA nasıl çalışır @@ -130,33 +132,36 @@ Ajanınız, hedefi incelemek ve ilgili kodu izlemek için MCP üzerinden REA'yı REA; Node.js 22.x (>=22.19), 24.x (>=24.11) veya 26+ ve npm gerektirir. Ek araçlar ve desteklenen ana bilgisayar sistemleri hedefe bağlıdır: -| Hedef | REA'nın döndürdükleri | Gereksinimler ve rehber | -| --------------------------------------- | ----------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | -| Yerel makine kodu içeren ikili dosyalar | Sözde kod, assembly kodu, dizeler, semboller, çağrılar ve referanslar | Hopper, Ghidra veya IDA; [yerel kod analizi](https://rea.tools/tr/guides/native/) | -| Çevrimdışı ELF yapısı | Bölümler, segmentler, özgün semboller/yeniden konumlandırmalar ve statik analizden olası güvenlik önlemleri | Linux x64 üzerinde çağıranın sağladığı pwntools; [ikili dosya tanılama](../binary-diagnostics.md) | -| EVM bayt kodu | Yönlendirme seçicileri, bayt ofsetleri, çıkarımla belirlenen argümanlar ve durum değiştirilebilirliği | Linux x64 üzerinde yerel ham bayt/onaltılık girdi; [çevrimdışı EVM rehberi](../evm-bytecode.md) | -| Kaydedilmiş Linux çökmeleri | Ham note kayıtları, kaydedilen her iş parçacığının yazmaçları/sinyalleri ve isteğe bağlı eşleme adayları | Linux x64 üzerinde çağıranın sağladığı pwntools; isteğe bağlı GDB/pwndbg; [kaydedilmiş çökmeler](../recorded-crashes.md) | -| JavaScript / Electron | Modüller, içe aktarmalar, kaynak haritaları, rotalar, IPC ve yerel eklenti ilişkileri | Node.js ve npm; [uygulama analizi](https://rea.tools/tr/guides/javascript/) | -| Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](https://rea.tools/tr/guides/browser/) | -| Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR; mitmproxy'nin kendi biçimindeki yakalamalar için Linux üzerinde mitmdump; [ağ yakalama rehberi](../web-network-captures.md) | -| Paket yakalamaları | Uç noktalar, DNS, TLS SNI, protokol dağılımı ve HTTP istekleri | Çağıranın sağladığı tshark ve capinfos; [pcap rehberi](../pcap-analysis.md) | -| .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](../managed-code-analysis.md) | -| Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS/Windows x64 üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](../android-analysis.md) | -| Android cihazları | Cihazlar, paketler, süreçler, günlükler, dökümler, ekran görüntüleri ve APK/dosya aktarımları | Çağıranın sağladığı adb (emülatör veya cihaz); [ADB rehberi](../adb-device-analysis.md) | -| JEB projeleri | Motor destekli proje birimleri, artefakt özetleri ve tür/metot sözdekodu | Çağıranın başlattığı, MCP sunan bir JEB istemcisi, REA_JEB_MCP_URL ile seçilir; [JEB rehberi](../jeb-analysis.md) | -| Android kaynakları | Çözülmüş manifest, dize tabloları, diller ve sürüm bilgileri | Çağıranın sağladığı Apktool başlatıcısı; [kaynak rehberi](../apktool-resource-analysis.md) | -| Flutter uygulamaları | Dart anlık görüntü karması, ABI başına kitaplık özetleri ve motor derleme kökeni | Yok (statik APK çözümleme); [Flutter rehberi](../flutter-analysis.md) | -| Donanım yazılımı | Bölgeler, çıkarma sonuçları ve yerel kod analizine devirler | Linux üzerinde Binwalk / Unblob; [donanım yazılımı rehberi](../firmware-analysis.md) | -| Kaydedilmiş paketler | Seçilen alanlar, yinelenen değerler, filtreler ve çözümleme kuralları | Linux: TShark 4.x + prlimit; [yakalama kılavuzu](../packet-captures.md) | -| Kaydedilmiş mantık sinyalleri | Kod çözücü açıklamaları, örnek aralıkları ve tanılama | Linux: sigrok-cli 0.7.2 + prlimit; [sinyal kılavuzu](../signal-recordings.md) | +| Hedef | REA'nın döndürdükleri | Gereksinimler ve rehber | +| --------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Yerel makine kodu içeren ikili dosyalar | Sözde kod, assembly kodu, dizeler, semboller, çağrılar ve referanslar | Hopper, Ghidra veya IDA; [yerel kod analizi](https://rea.tools/tr/guides/native/) | +| Çevrimdışı ELF yapısı | Bölümler, segmentler, özgün semboller/yeniden konumlandırmalar ve statik analizden olası güvenlik önlemleri | Linux x64 üzerinde çağıranın sağladığı pwntools; [ikili dosya tanılama](../binary-diagnostics.md) | +| Go ikili dosyaları | Gömülü compiler sürümü, modüller, replacement'lar ve build ayarları | Yerleşik statik ELF/PE/thin Mach-O incelemesi; Go kurulumu gerekmez | +| WASM artefaktı | Doğrulama, bölümler, tam import/export WAT biçimleri ve decode edilmiş WAT | Çağıranın sağladığı WABT 1.0.42; `inspect_wasm_artifact` | +| EVM bayt kodu | Yönlendirme seçicileri, bayt ofsetleri, çıkarımla belirlenen argümanlar ve durum değiştirilebilirliği | Linux x64 üzerinde yerel ham bayt/onaltılık girdi; [çevrimdışı EVM rehberi](../evm-bytecode.md) | +| SQLite veritabanları | Tablolar, sütunlar, index'ler, view'lar, trigger'lar ve seçilen sıradan tablo satırları | Yerel database/WAL snapshot'ı; native SQLite sınırlarıyla Node.js 24.x >=24.15 veya 26+ | +| Kaydedilmiş Linux çökmeleri | Ham note kayıtları, kaydedilen her iş parçacığının yazmaçları/sinyalleri ve isteğe bağlı eşleme adayları | Linux x64 üzerinde çağıranın sağladığı pwntools; isteğe bağlı GDB/pwndbg; [kaydedilmiş çökmeler](../recorded-crashes.md) | +| JavaScript / Electron | Modüller, içe aktarmalar, kaynak haritaları, rotalar, IPC ve yerel eklenti ilişkileri | Node.js ve npm; [uygulama analizi](https://rea.tools/tr/guides/javascript/) | +| Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](https://rea.tools/tr/guides/browser/) | +| Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR veya Fiddler Everywhere SAZ; Linux'ta native mitmproxy için mitmdump; [ağ yakalama rehberi](../web-network-captures.md) | +| .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](../managed-code-analysis.md) | +| Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS/Windows x64 üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](../android-analysis.md) | +| Android cihazları | Cihazlar, paketler, süreçler, günlükler, dökümler, ekran görüntüleri ve APK/dosya aktarımları | Çağıranın sağladığı adb (emülatör veya cihaz); [ADB rehberi](../adb-device-analysis.md) | +| JEB projeleri | Motor destekli proje birimleri, artefakt özetleri ve tür/metot sözdekodu | Çağıranın başlattığı, MCP sunan bir JEB istemcisi, REA_JEB_MCP_URL ile seçilir; [JEB rehberi](../jeb-analysis.md) | +| Android kaynakları | Çözülmüş manifest, dize tabloları, diller ve sürüm bilgileri | Çağıranın sağladığı Apktool başlatıcısı; [kaynak rehberi](../apktool-resource-analysis.md) | +| Flutter uygulamaları | Dart anlık görüntü karması, ABI başına kitaplık özetleri ve motor derleme kökeni | Yok (statik APK çözümleme); [Flutter rehberi](../flutter-analysis.md) | +| Donanım yazılımı | Bölgeler, çıkarma sonuçları ve yerel kod analizine devirler | Linux üzerinde Binwalk / Unblob; [donanım yazılımı rehberi](../firmware-analysis.md) | +| Kaydedilmiş paketler | Seçilen alanlar, yinelenen değerler, filtreler ve çözümleme kuralları | Linux: TShark 4.x + prlimit; [yakalama kılavuzu](../packet-captures.md) | +| Paket özetleri | Uç noktalar, DNS, TLS SNI, protokol dağılımı ve HTTP istekleri | Çağıranın sağladığı tshark ve capinfos (Linux üzerinde doğrulandı); [pcap rehberi](../pcap-analysis.md) | +| Kaydedilmiş mantık sinyalleri | Kod çözücü açıklamaları, örnek aralıkları ve tanılama | Linux: sigrok-cli 0.7.2 + prlimit; [sinyal kılavuzu](../signal-recordings.md) | | Paketler ve kaynaklar | Dosya envanterleri, özet değerleri, plist'ler, Apple paket yapısı ve çıkarılmış kaynaklar | [Artefakt ve JavaScript rehberi](../javascript-artifact-reconstruction.md), [Apple uygulamaları](../apple-application-analysis.md) | -| Süreç davranışı | Terminal çıktısı, etkileşimler, sonlanma ve dosya sistemi gözlemleri, çalıştırma karşılaştırmaları | Yerel PTY ile Linux/macOS; [süreç yakalama](../process-capture.md) | +| Süreç davranışı | Terminal çıktısı, etkileşimler, sonlanma ve dosya sistemi gözlemleri, çalıştırma karşılaştırmaları | Yerel PTY ile Linux/macOS; [süreç yakalama](../process-capture.md) | Statik JavaScript ve .NET incelemesi, uygulamayı çalıştırmadan sağlanan dosyaları okur. Çalışma zamanı yakalaması, seçilen hedefi kullanıcı izinlerinizle çalıştırır veya onunla etkileşime girer; her çalışma zamanı rehberi etkilerini açıklar. -Yerel makine kodu içeren ikili dosya biçimleri ve ana bilgisayar desteği sağlayıcıya göre değişir. [Hopper ve Ghidra kurulumu](../installation.md#hopper), [IDA rehberi](../ida-provider.md) ve [Windows için deneysel Ghidra desteği](../windows-ghidra-p0.md) belgelerine bakın. Ghidra ayrıca [16 bit DOS analizini](../ghidra-dos.md) destekler. Sağlayıcı seçimi için [CLI rehberine](../cli.md#choose-a-provider) bakın. En son npm sürümünden sonra eklenen özellikler için [sürümlerde kullanılabilirliği](../installation.md#released-package-and-main) kontrol edin. +Yerel makine kodu içeren ikili dosya biçimleri ve ana bilgisayar desteği sağlayıcıya göre değişir. [Hopper ve Ghidra kurulumu](../installation.md#hopper), isteğe bağlı native MCP bridge'i de açıklayan [IDA rehberi](../ida-provider.md) ve [Windows için deneysel Ghidra desteği](../windows-ghidra-p0.md) belgelerine bakın. Ghidra ayrıca [16 bit DOS analizini](../ghidra-dos.md) destekler. Büyük ikili dosyalar için sağlayıcının başlatma zaman aşımını `REA_GHIDRA_STARTUP_TIMEOUT_MS` veya `REA_HOPPER_STARTUP_TIMEOUT_MS` ile artırın. Sağlayıcı seçimi için [CLI rehberine](../cli.md#choose-a-provider) bakın. En son npm sürümünden sonra eklenen özellikler için [sürümlerde kullanılabilirliği](../installation.md#released-package-and-main) kontrol edin. ## Örnek çalışmalar @@ -280,9 +285,9 @@ Sorumlu kullanım yönergeleri için [DISCLAIMER.md](../../DISCLAIMER.md) dosyas REA; yasal araştırma, eğitim, birlikte çalışabilirlik, yazılımın korunması ve sahibi olduğunuz ya da inceleme yetkiniz bulunan yazılımların analizi için tasarlanmıştır. -The project's position on illegal or unauthorized use covers all REA -versions, distribution channels, and platforms; each copy retains its -applicable licenses. +Projenin yasa dışı veya yetkisiz kullanıma ilişkin tutumu tüm REA sürümleri, +dağıtım kanalları ve platformlar için geçerlidir; her kopya geçerli lisanslarını +korur. ## Katkıda bulunma @@ -294,14 +299,16 @@ Geliştirme ortamı ve kontroller için [CONTRIBUTING.md](../../CONTRIBUTING.md) [MIT](../../LICENSE) -REA-authored code is licensed under MIT. Third-party material retains its own -licenses and copyright notices; see [Third-party notices](../../THIRD_PARTY_NOTICES.md) -for the dependency inventory, preserved license texts, and external-tool terms. +REA tarafından yazılan kod MIT lisansı altında lisanslanır. Üçüncü taraf +materyaller kendi lisanslarını ve telif hakkı bildirimlerini korur; bağımlılık +envanteri, korunan lisans metinleri ve harici araç koşulları için [Üçüncü taraf +bildirimleri](../../THIRD_PARTY_NOTICES.md) dosyasına bakın. -**You must legally obtain and be authorized to use commercial tools**, including -IDA Pro / Hex-Rays decompilers, Hopper, and JEB, under their applicable licenses. -REA grants no tool license, key, or activation entitlement. Trials and demos -remain subject to vendor terms, including user, feature, automation, and output -sharing restrictions. Cracked copies, stolen keys, unauthorized key sharing, -and bypassing license restrictions are not authorized. This guidance does not -amend MIT or third-party licenses. +**IDA Pro / Hex-Rays decompiler'ları, Hopper ve JEB dahil olmak üzere ticari +araçları yasal olarak edinmeli ve ilgili lisanslar kapsamında kullanma +yetkiniz olmalıdır.** REA hiçbir araç lisansı, anahtarı veya etkinleştirme hakkı +vermez. Deneme sürümleri ve demolar; kullanıcı, özellik, otomasyon ve çıktı +paylaşımı kısıtlamaları dahil olmak üzere satıcı koşullarına tabidir. Kırılmış +kopyalar, çalınmış anahtarlar, yetkisiz anahtar paylaşımı ve lisans +kısıtlamalarını aşma yetkili kullanım kapsamında değildir. Bu yönlendirme MIT +veya üçüncü taraf lisanslarını değiştirmez. From f652709cac54441f401e90b6a604997c9321d70d Mon Sep 17 00:00:00 2001 From: Shubham Ahuja <157806066+shubhamahuja9999@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:35:14 +0530 Subject: [PATCH 15/18] fix(cli): keep file URLs out of operator-relative path resolution (#2882) CLI path normalization resolved every nonblank selected string against the working directory, so executable_path "file:///opt/electron" became an invented local path and failed with ENOENT for that location instead of the field's local-path requirement. Leave `file:` and `scheme://` references unchanged so the shared schema rejects them as non-local paths. One-letter prefixes stay paths, so Windows drive paths and relative names containing a colon resolve as before. Fixes #2577 Co-authored-by: Claude Opus 5.5 --- src/cliJsonInput.ts | 17 +++++++-- src/cliJsonPaths.test.ts | 34 +++++++++++++++++ .../applications/electronInvalidPaths.test.ts | 37 +++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.ts index 4dd38a90e..cf1b6ea72 100644 --- a/src/cliJsonInput.ts +++ b/src/cliJsonInput.ts @@ -65,12 +65,19 @@ const accessDeniedSystemCode = ( : undefined; }; +/** + * A `file:` URL or `scheme://` URL is not an operator-relative filesystem path. + * One-letter prefixes stay paths so Windows drive paths resolve normally. + */ +const URL_REFERENCE = /^(?:file:|[A-Za-z][A-Za-z0-9+.-]+:\/\/)/iu; + /** * Resolve named string fields of CLI-supplied JSON against the operator * working directory. Each entry is a key path from the document root; only - * fields holding nonblank strings are resolved, everything else passes through - * untouched so shared schemas can reject blank paths. Shared MCP contracts reject - * relative paths, so CLI workflows resolve operator-relative values before validation. + * fields holding nonblank, non-URL strings are resolved, everything else passes + * through untouched so shared schemas can reject blank paths and URLs. Shared MCP + * contracts reject relative paths, so CLI workflows resolve operator-relative + * values before validation. */ export const resolveCliJsonPaths = ( value: unknown, @@ -94,7 +101,9 @@ const resolveJsonPath = ( if (tail.length === 0) return [ key, - typeof entry === "string" && entry.trim().length > 0 + typeof entry === "string" && + entry.trim().length > 0 && + !URL_REFERENCE.test(entry) ? resolve(entry) : entry, ]; diff --git a/src/cliJsonPaths.test.ts b/src/cliJsonPaths.test.ts index 936bfacea..ed9e18642 100644 --- a/src/cliJsonPaths.test.ts +++ b/src/cliJsonPaths.test.ts @@ -56,4 +56,38 @@ describe("CLI JSON path normalization", () => { }); expect(input.browser.executable_path).toBe(" browser "); }); + + it.each([ + "file:///opt/electron", + "FILE:electron", + "https://example.test/app", + ])("keeps URL %j distinguishable from a local path", (value) => { + const input = { + executable_path: value, + application_path: resolve("/synthetic/app"), + }; + const normalized = resolveCliJsonPaths(input, paths); + expect(normalized).toEqual(input); + const parsed = electronActiveObservationInputSchema.safeParse(normalized); + expect(parsed.success).toBe(false); + if (parsed.success) throw new Error("Expected URL path rejection"); + expect(parsed.error.issues).toEqual([ + expect.objectContaining({ + path: ["executable_path"], + message: expect.stringContaining( + "must be absolute local filesystem paths", + ), + }), + ]); + }); + + it.each([ + "bin/electron", + "release:v1/electron", + "C:\\Electron\\electron.exe", + ])("keeps resolving local path %j", (value) => { + expect(resolveCliJsonPaths({ executable_path: value }, paths)).toEqual({ + executable_path: resolve(value), + }); + }); }); diff --git a/tests/acceptance/applications/electronInvalidPaths.test.ts b/tests/acceptance/applications/electronInvalidPaths.test.ts index d74c04b2b..e7d5935e2 100644 --- a/tests/acceptance/applications/electronInvalidPaths.test.ts +++ b/tests/acceptance/applications/electronInvalidPaths.test.ts @@ -67,3 +67,40 @@ cliTest( expect(session.isError, JSON.stringify(session)).not.toBe(true); }, ); + +cliTest( + "rejects a file URL path field without resolving an invented local path", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-electron-file-url-"); + const application = join(root, "main.js"); + await writeFile(application, "module.exports = {};\n"); + const response = await cli.run({ + arguments: [ + "capture-electron-scenario", + JSON.stringify({ + executable_path: "file:///opt/electron", + application_path: application, + }), + "--format", + "json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(response.exitCode).toBe(1); + expect(response.json).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["executable_path"], + message: expect.stringContaining( + "must be absolute local filesystem paths", + ), + }, + ], + }, + }); + expect(response.stdout).not.toContain("ENOENT"); + expect(response.stdout).not.toContain("file:/opt/electron"); + }, +); From 1d5f95dc4aff3ffadd06e922128606ea0fb669cc Mon Sep 17 00:00:00 2001 From: Chaoqi <136579807+Chaoqi31@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:06:55 -0400 Subject: [PATCH 16/18] fix(native): parse renamed dyld_info re-exports (#2364) * fix(native): parse renamed dyld_info re-exports dyld_info prints a re-export whose name differs from the source library's symbol as `[re-export] _foo_renamed (_foo from libfoo)`. The export parser only accepted `(from libfoo)`, so inspect_macho reported a nonexistent export named `_foo_renamed (_foo from libfoo)` with no source, and the nm row for `_foo_renamed` survived the name-based merge as a second export with reexport null. Accept the optional source symbol, trying the plain `(from lib)` form first so a library name containing ` from ` keeps its text. The row now reports `_foo_renamed` re-exported from libfoo and replaces the nm duplicate. The export schema has no field for the source symbol name, so exports coverage records a limitation when a renamed re-export is present. The golden fixture is a dyld_info capture of a synthetic dylib linked with -Wl,-alias,_foo,_foo_renamed. * fix(native): split dyld re-export suffix without nested quantifiers Replace the (?:(.+?)\s+)??from lazy-optional group with a linear string scan so malformed lines cannot backtrack and a library path containing " from " still keeps its whole text via the plain-form-first check. * test(native): cover nested re-export library names * fix(native): retain reexports missing from nm inventory --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: morluto <17176810+morluto@users.noreply.github.com> --- src/native/NativeMachoInspection.ts | 5 + src/native/parsers/dyldInfo.ts | 85 ++++++++++++++++- .../native/nativeInventory.golden.test.ts | 92 +++++++++++++++++++ .../native-macos/dyld-inventory/README.md | 13 +++ .../dyld-inventory/renamed-reexports.txt | 5 + 5 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/native-macos/dyld-inventory/renamed-reexports.txt diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index bb1966c3d..8453f38a8 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -266,6 +266,11 @@ const normalizeMacho = ( ]), exports: covered(exports, false, [ "Merged nm/dyld_info results may be incomplete for stripped binaries.", + ...(dyldExports.some(({ sourceSymbol }) => sourceSymbol !== undefined) + ? [ + "Renamed re-exports are listed under their exported name; the source library's symbol name is not reported.", + ] + : []), ]), segments: covered( load.segments.map((segment) => ({ diff --git a/src/native/parsers/dyldInfo.ts b/src/native/parsers/dyldInfo.ts index f08dc8bdd..9742c66a5 100644 --- a/src/native/parsers/dyldInfo.ts +++ b/src/native/parsers/dyldInfo.ts @@ -4,6 +4,8 @@ interface ParsedDyldSymbol { readonly weak: boolean | null; readonly reexport: boolean | null; readonly source: string | null; + /** The source library's name for a symbol it re-exports under another name. */ + readonly sourceSymbol?: string; } /** Decode Apple dyld inventory rows, projecting export offsets to VM addresses. */ @@ -33,15 +35,23 @@ export const parseDyldSymbols = ( ]; return []; } - const reexport = /^\[re-export\]\s+(.+?)(?:\s+\(from (.+)\))?$/u.exec(line); - if (reexport?.[1] !== undefined) + // dyld_info prints `_sym (from lib)`, or `_new (_old from lib)` when the + // source library names the symbol differently. Split the trailing + // parenthesized suffix with string scans so a library path containing + // " from ", or a source symbol containing parentheses such as + // `operator()`, cannot pull the match into nested lazy quantifiers. + const reexport = parseReexportRow(line, knownExportNames); + if (reexport !== null) return [ { - name: reexport[1], + name: reexport.name, address: null, weak: null, reexport: true, - source: reexport[2] ?? null, + source: reexport.source, + ...(reexport.sourceSymbol === undefined + ? {} + : { sourceSymbol: reexport.sourceSymbol }), }, ]; const exported = /^(0x[\da-f]+)\s+(.+?)(?:\s+\[([^\]]+)\])?$/iu.exec(line); @@ -78,3 +88,70 @@ export const parseDyldSymbols = ( const canonicalHex = (value: string | undefined): string | null => value === undefined ? null : `0x${BigInt(value).toString(16)}`; + +interface ReexportRow { + readonly name: string; + readonly source: string | null; + readonly sourceSymbol?: string; +} + +/** + * Split a `[re-export]` row without nested lazy quantifiers. + * + * Returns null when the line is not a re-export row so callers fall through + * to the plain export parser. Known names disambiguate nested parentheses; + * when that inventory is incomplete, retain the suffix-based fallback. + */ +const parseReexportRow = ( + line: string, + knownNames: ReadonlySet, +): ReexportRow | null => { + const prefix = /^\[re-export\]\s+/u.exec(line); + if (prefix === null) return null; + const rest = line.slice(prefix[0].length); + if (rest.length === 0) return null; + if (!rest.endsWith(")")) return { name: rest, source: null }; + const candidates: ReexportRow[] = []; + for (const match of rest.matchAll(/\s+\(/gu)) { + const name = rest.slice(0, match.index); + if (!knownNames.has(name)) continue; + const parsed = reexportSuffix( + name, + rest.slice(match.index + match[0].length, -1), + ); + if (parsed !== null) candidates.push(parsed); + } + // Exact nm identities establish a boundary even when names contain parentheses. + // Competing known interpretations remain unknown rather than creating a new name. + if (candidates.length > 0) + return candidates.length === 1 ? candidates[0]! : null; + if (knownNames.has(rest)) return { name: rest, source: null }; + let depth = 0; + for (let index = rest.length - 1; index >= 0; index--) { + if (rest[index] === ")") depth++; + else if (rest[index] === "(" && --depth === 0) { + if (index === 0 || !/\s/u.test(rest[index - 1]!)) break; + const name = rest.slice(0, index).trimEnd(); + return ( + reexportSuffix(name, rest.slice(index + 1, -1)) ?? { + name: rest, + source: null, + } + ); + } + } + return { name: rest, source: null }; +}; + +const reexportSuffix = (name: string, inner: string): ReexportRow | null => { + if (name.length === 0) return null; + if (inner.startsWith("from ") && inner.length > 5) + return { name, source: inner.slice(5) }; + const separator = /\s+from /u.exec(inner); + if (separator === null) return null; + const sourceSymbol = inner.slice(0, separator.index); + const source = inner.slice(separator.index + separator[0].length); + return sourceSymbol.length > 0 && source.length > 0 + ? { name, source, sourceSymbol } + : null; +}; diff --git a/tests/boundary/providers/native/nativeInventory.golden.test.ts b/tests/boundary/providers/native/nativeInventory.golden.test.ts index 5ac80e778..934d24137 100644 --- a/tests/boundary/providers/native/nativeInventory.golden.test.ts +++ b/tests/boundary/providers/native/nativeInventory.golden.test.ts @@ -225,6 +225,79 @@ it("retains native inventory facts from captured Apple tool output", async () => }); }); +it("reports a renamed re-export once under its exported name", async () => { + const client = new NativeMacOSProvider( + {}, + new FixtureRunner({ + "dyld_info:-exports": await fixture( + "dyld-inventory/renamed-reexports.txt", + ), + nm: "_foo_renamed\n_own\n", + }), + "darwin", + ).createClient(machoTarget("/private/fixture")); + const execution = await client.execute("inspect_macho", {}); + if (!execution.ok) throw execution.error; + expect(execution.value.result).toMatchObject({ + exports: { + items: [ + { + name: "_foo_renamed", + address: null, + weak: null, + reexport: true, + source: "libfoo", + }, + { name: "_own", reexport: false, source: null }, + ], + limitations: [ + "Merged nm/dyld_info results may be incomplete for stripped binaries.", + "Renamed re-exports are listed under their exported name; the source library's symbol name is not reported.", + ], + }, + }); +}); + +it("splits re-export rows at the source library", () => { + // dyld_info -exports rows for synthetic dylibs: a source symbol named + // `operator()`, and a library whose install name contains ` from `. + expect( + parseDyldSymbols( + [ + "[re-export] _call_renamed (operator() from libfoo)", + "[re-export] _g (from lib from foo)", + "[re-export] _nested (from lib (from foo))", + ].join("\n"), + "exports", + null, + new Set(["_call_renamed", "_g", "_nested"]), + ), + ).toEqual([ + { + name: "_call_renamed", + address: null, + weak: null, + reexport: true, + source: "libfoo", + sourceSymbol: "operator()", + }, + { + name: "_g", + address: null, + weak: null, + reexport: true, + source: "lib from foo", + }, + { + name: "_nested", + address: null, + weak: null, + reexport: true, + source: "lib (from foo)", + }, + ]); +}); + it("retains symbol names containing spaces in dyld inventory rows", () => { expect( parseDyldSymbols(" 0x0000 _entry with space (from fixture)", "imports"), @@ -449,6 +522,25 @@ it("leaves an ambiguous short and literal name to the caller's nm inventory", () ).toEqual([]); }); +it("retains unmatched re-export rows when the known export inventory is incomplete", () => { + expect( + parseDyldSymbols( + "[re-export] _g (from libfoo)", + "exports", + null, + new Set(["_own"]), + ), + ).toEqual([ + { + name: "_g", + address: null, + weak: null, + reexport: true, + source: "libfoo", + }, + ]); +}); + it("retains annotation parsing without nm name evidence", () => { expect(parseDyldSymbols("0x120 _weak [weak-def]", "exports")).toEqual([ { diff --git a/tests/fixtures/native-macos/dyld-inventory/README.md b/tests/fixtures/native-macos/dyld-inventory/README.md index af4744fe8..2c9b27743 100644 --- a/tests/fixtures/native-macos/dyld-inventory/README.md +++ b/tests/fixtures/native-macos/dyld-inventory/README.md @@ -37,6 +37,19 @@ import rows from `xcrun dyld_info -imports /usr/lib/libobjc.A.dylib`. `xcrun dyld_info -exports /usr/lib/libc++.1.dylib`. These are separate library captures used to check their respective row formats. +`renamed-reexports.txt` is `xcrun dyld_info -exports` for a dylib that +re-exports `_foo` from `libfoo` under the name `_foo_renamed`: + +```sh +printf 'int foo(void){return 1;}\n' > /tmp/foo.c +printf 'int own(void){return 2;}\n' > /tmp/re.c +xcrun clang -dynamiclib /tmp/foo.c -o /tmp/libfoo.dylib -install_name @rpath/libfoo.dylib +xcrun clang -dynamiclib /tmp/re.c -o /tmp/libre.dylib -install_name @rpath/libre.dylib \ + /tmp/libfoo.dylib -Wl,-alias,_foo,_foo_renamed +xcrun dyld_info -exports /tmp/libre.dylib +xcrun nm -gjU /tmp/libre.dylib +``` + `codesign-missing.txt` is stderr from `codesign -d --verbose=4` on a nonexistent path (exit 1). `codesign-unsigned.txt` is stderr from the same command on a copy of the fixture after `codesign --remove-signature` (exit 1). Only the latter is diff --git a/tests/fixtures/native-macos/dyld-inventory/renamed-reexports.txt b/tests/fixtures/native-macos/dyld-inventory/renamed-reexports.txt new file mode 100644 index 000000000..f401afcf5 --- /dev/null +++ b/tests/fixtures/native-macos/dyld-inventory/renamed-reexports.txt @@ -0,0 +1,5 @@ +fixture [arm64]: + -exports: + offset symbol + [re-export] _foo_renamed (_foo from libfoo) + 0x00000308 _own From ebaaa79ca02bde4a73013a2a9a781d0b07a57ee4 Mon Sep 17 00:00:00 2001 From: Luis <86675342+LuisTaffelli@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:09:19 -0300 Subject: [PATCH 17/18] test(http3): add real MITM capture verification lab (#1997) * test(http3): add real MITM capture verification lab * test(http3): register the optional verification command * fix(http3): preserve the latest verification attempt owner * test(cli): admit the dyld slice fixture on x64 hosts * test(http3): verify exact producer TLS connection facts * test(http3): verify readable capture fields against observed bytes * fix(comparison): recheck historical calls-only changes (#2223) * fix(comparison): preserve incomplete function call evidence Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty. Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls. * fix(comparison): recheck historical calls-only changes --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> * fix(android): make DEX descriptor pattern portable * fix(ci): isolate Linux Hopper process fixture inventory * fix(http3): retain MCP startup diagnostics --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Lucien <164857476+dakjdakd@users.noreply.github.com> --- docs/http3-capture-lab.md | 183 ++++++++ package.json | 1 + scripts/fixtures/http3-lab-requirements.txt | 10 + scripts/fixtures/http3-lab.py | 416 +++++++++++++++++ scripts/fixtures/http3-lab.test.py | 56 +++ scripts/fixtures/observe-mitmproxy-http3.py | 64 +++ scripts/lib/http3-capture-assertions.mjs | 243 ++++++++++ scripts/lib/http3-capture-assertions.test.mjs | 163 +++++++ scripts/lib/http3-verifier-lifecycle.mjs | 143 ++++++ scripts/lib/http3-verifier-lifecycle.test.mjs | 433 ++++++++++++++++++ scripts/verify-http3-network-captures.mjs | 408 +++++++++++++++++ 11 files changed, 2120 insertions(+) create mode 100644 docs/http3-capture-lab.md create mode 100644 scripts/fixtures/http3-lab-requirements.txt create mode 100644 scripts/fixtures/http3-lab.py create mode 100644 scripts/fixtures/http3-lab.test.py create mode 100644 scripts/fixtures/observe-mitmproxy-http3.py create mode 100644 scripts/lib/http3-capture-assertions.mjs create mode 100644 scripts/lib/http3-capture-assertions.test.mjs create mode 100644 scripts/lib/http3-verifier-lifecycle.mjs create mode 100644 scripts/lib/http3-verifier-lifecycle.test.mjs create mode 100644 scripts/verify-http3-network-captures.mjs diff --git a/docs/http3-capture-lab.md b/docs/http3-capture-lab.md new file mode 100644 index 000000000..b4a41ebb4 --- /dev/null +++ b/docs/http3-capture-lab.md @@ -0,0 +1,183 @@ +# Verify real HTTP/3 MITM captures + +This optional Linux laboratory checks native mitmproxy captures through REA's +public CLI and MCP. It changes no runtime contracts or adapters and establishes +only the specific producer, mode and clients exercised here. + +## Prepare a disposable guest + +The exercised image is [Omarchy 4.0.4, build 440](https://nightly.omarchy.org/stable/4.0.4/440/omarchy-4.0.4.qcow2), +SHA-256 `43412b896fe3d730e373a51a8acf99343d7c3e6dee78aa9f9bd7fca181127264`. +Download it without installing guest packages on the host, check its digest, +and use a **new qcow2 overlay** with that file as a read-only backing image. +The matching [boot artifacts](https://nightly.omarchy.org/stable/4.0.4/440/unpacked/) +include the kernel, initrd, cmdline and upstream vm-boot recipe. Use a fresh SSH +key, loopback forwarding and guest disk; do not reuse another investigation's +writable overlay or host trust configuration. Shut the guest down before +switching overlays. Preserve the image and boot artifact hashes with evidence. + +This sealed guest already supplies cURL 8.22.0 with OpenSSL 3.6.4, ngtcp2 1.25.0, +nghttp3 1.18.0 and nghttp2 1.70.0. **No guest rolling package upgrade is needed.** +Run `curl --version` and verify HTTP2 and HTTP3 features before starting. A stock +cURL from another distribution may omit HTTP/3. For a different guest, building +cURL requires a compatible TLS library, ngtcp2 and nghttp3; follow the +[official build instructions](https://curl.se/docs/http3.html), install to a guest +private prefix, and record exact source revisions and build inputs. That alternative +has not been exercised by this laboratory; it is not the pinned reproduction path. + +Use a fresh VM overlay over a sealed Linux image. Use user-mode networking with +SSH forwarding bound to host loopback; keep host directories unmounted. Install +and build dependencies inside the guest, then remove its default route before +generating traffic. Host trust, routes and installed packages stay unchanged. + +Record the image digest, REA source revision and these dependency versions: + +- REA's `.nvmrc`, `package.json#packageManager` and locked npm dependencies. +- Official Linux x86_64 mitmproxy **12.2.3** standalone archive. SHA-256: + `2e95286b618fa6fd33e5e62a78c2e5112571d85f42ec2bac29b97ee242bdb5c5`. +- A private Python virtual environment using the complete pinned graph in + `scripts/fixtures/http3-lab-requirements.txt`. Save `pip freeze` and build/download logs. +- OpenSSL and cURL advertising HTTP3 and HTTP2 in `curl --version`. + +The verified guest used Python 3.14.7, cURL 8.22.0, ngtcp2 1.25.0 and nghttp3 +1.18.0. Other dependency profiles need verification. Back up the guest's +`/etc/hosts` before adding `127.0.0.1 lab.test`. Reserve guest TCP/UDP loopback +ports 18443 and 19443. These changes belong only to the disposable guest. + +From the guest REA checkout, use its pinned Node/npm toolchain: + +```sh +npm ci +npm run build:cached +python3 -m venv /absolute/lab/venv +/absolute/lab/venv/bin/pip install --no-deps -r scripts/fixtures/http3-lab-requirements.txt +/absolute/lab/venv/bin/pip check +/absolute/lab/venv/bin/pip freeze > /absolute/lab/python-requirements.txt +``` + +Before removing the guest default route, download the official mitmproxy archive +and verify it, then extract to the guest's private tools directory: + +```sh +mkdir -p /absolute/lab/tools +curl --fail --location https://downloads.mitmproxy.org/12.2.3/mitmproxy-12.2.3-linux-x86_64.tar.gz \ + --output /absolute/lab/mitmproxy.tar.gz +printf '%s %s\n' 2e95286b618fa6fd33e5e62a78c2e5112571d85f42ec2bac29b97ee242bdb5c5 \ + /absolute/lab/mitmproxy.tar.gz | sha256sum --check --strict +tar -xzf /absolute/lab/mitmproxy.tar.gz -C /absolute/lab/tools +/absolute/lab/tools/mitmdump --version +``` + +Pin the REA revision with `git rev-parse HEAD` or archive SHA-256. Build using +the exact `.nvmrc` and npm version; do not copy a different checkout's `dist`. +For stronger offline dependency reproduction, preserve downloaded Python +archives, their digests and built wheels along with the image and toolchain. +The requirement versions pin the runtime graph, not every transient build tool; +source builds and alternate interpreters are not guaranteed byte-identical. + +## Run the narrow profile + +Select absolute executables and a new evidence directory. The producer refuses +to reuse a directory. No certificates are installed in a system trust store; +clients and proxy receive explicit paths to freshly generated test CAs. + +```sh +REA_MITMDUMP_COMMAND=/absolute/lab/tools/mitmdump \ +REA_HTTP3_PYTHON_COMMAND=/absolute/lab/venv/bin/python \ +npm run verify:web:http3 -- /absolute/evidence/run-1 +``` + +The producer sends three concurrent binary POSTs on one HTTP/3 connection to the +same URL through mitmproxy reverse mode to an aioquic HTTP/3 origin. The origin +waits for all complete requests, then dispatches responses in reverse order of +completion at the origin. QUIC stream send/receive orders need not coincide. A cURL HTTP/3-only +request checks a second client implementation. Mitmproxy's real save addon +writes `normal.mitm`; no native flows are synthesized. + +A second proxy run disables HTTP/3. cURL `--http3` falls back to HTTP/2, reaching +an HTTP/1.1 TLS test origin. `fallback.mitm` must retain those lower protocols, +so a successful HTTP request cannot masquerade as an HTTP/3 success. + +## Acceptance and evidence + +Each execution creates `verifications//` with its own diagnostics. +`verification-latest.json` belongs to the most recently registered attempt and +changes atomically from `running` to `passed` or `failed`. Registration and +the ownership check/replacement use the same SQLite transaction lock. An older +attempt finishing later keeps its own reports without replacing the marker. +The `.verification-lock.sqlite` pathname must remain in place while attempts +may run; SQLite releases the lock on connection close or process death. +Lock acquisition waits for the current owner without a timeout or lease +stealing. The verifier has per-command deadlines, but no overall run deadline; +a live stalled lock owner can therefore delay publication until it exits. +Only a successful attempt writes its own `verification.json`, after +all cleanup and input-integrity checks. `result.json` also records failures and +the original error separately from cleanup errors. Older successes remain in +identified history; a legacy root `verification.json` is archived before a rerun. +A killed verifier may leave `running`, which must never be interpreted as success. + +Capture inputs live in `captures/`. To inspect them again without new traffic: + +```sh +REA_MITMDUMP_COMMAND=/absolute/lab/tools/mitmdump \ +npm run verify:web:http3 -- /absolute/evidence/run-1 --inspect-existing +``` + +Acceptance for the current run: + +- Every positive native record reports HTTP/3, UDP, ALPN h3 and successful TLS + timestamps on both legs. Connection IDs, producer TLS labels, exact TLS setup + timestamps and connection errors match the UUID-matched producer observation + on each leg. Endpoint handshakes independently observe QUIC v1. +- Native flow UUIDs remain distinct. Three endpoint streams share a connection + on each leg. Origin dispatch reverses completed receipt. Client and capture + ordering is observed, while association is verified independent of their order. +- CLI and MCP agree. Request/response bytes, lengths, SHA-256, ordered duplicate + headers, method, scheme, authority and path match independent observations. + Readable header names and values, request URL fields and ALPN also match; + non-UTF-8 header bytes retain a null display value rather than replacement text. +- Actual MCP advertised schemas validate, Evidence preserves source digests and + provider provenance, and the complete returned Evidence matches the retained MCP session records. +- The fallback capture reports client HTTP/2 over TCP and origin HTTP/1.1, with + cURL independently reporting version 2. Input files remain unchanged. +- Every cleanup action is attempted on success and failure, and final lineage + contains no live descendants. Recorded producer groups are inspected separately + after token-verified cleanup, including after their launcher has exited. + +Retain captures, endpoint/producer JSONL, proxy diagnostics, cURL output, manifest, +CLI/MCP output, advertised schema and session bundle. Export public certificates +and dependency/source provenance separately. Do not commit evidence, private +keys, dependencies or VM disks. Shut down the guest after exporting evidence. + +The reverse proxy can preserve authority `lab.test:18443` while recording origin +port 19443. Mitmproxy's serialized flow/connection UUIDs are not stream IDs. +Endpoint stream IDs are scoped observations; this test invents no cross-leg +mapping. Mitmproxy 12.2.3's `QUICv1` label alone does not prove negotiation: the +fixture checks the negotiated `_quic._version` in pinned aioquic 1.2.0. + +Coverage is native capture, Linux reverse mode, QUIC v1 and these clients. +Empty/absent/incomplete/redacted body semantics, cancellations, resets, +certificate failures, malformed captures, HAR and qlog are separate increments. +Local/WireGuard, browsers, QUIC v2, migration, 0-RTT, mTLS, replay, sustained load +and loss tolerance remain unverified by this profile. Passing it does not +establish general HTTP/3 support in REA. + +Official references: [mitmproxy protocol support](https://github.com/mitmproxy/mitmproxy/blob/v12.2.3/docs/src/content/concepts/protocols.md), +[HTTP/3 implementation](https://github.com/mitmproxy/mitmproxy/blob/v12.2.3/mitmproxy/proxy/layers/http/_http3.py), +[serialized HTTP state](https://github.com/mitmproxy/mitmproxy/blob/v12.2.3/mitmproxy/http.py), +[aioquic configuration](https://github.com/aiortc/aioquic/blob/1.2.0/src/aioquic/quic/configuration.py). + +## Verifier regressions + +After building REA, run these inside the same isolated guest: + +```sh +node --test scripts/lib/http3-verifier-lifecycle.test.mjs scripts/lib/http3-capture-assertions.test.mjs +/absolute/lab/venv/bin/python scripts/fixtures/http3-lab.test.py +``` + +These cover different QUIC stream completion orders, stale success reports, +multiple failed closes, separately owned process groups and partially acquired +resources, plus producer TLS fact mismatches on both legs and altered readable +fields with unchanged binary evidence. They test verifier +semantics; they do not add HTTP/3 provider coverage. diff --git a/package.json b/package.json index b042eb106..19e978205 100644 --- a/package.json +++ b/package.json @@ -266,6 +266,7 @@ "verify:browser:source-maps": "npm run build:cached && node scripts/verify-browser-source-maps.mjs", "verify:browser:runtime": "npm run build:cached && node scripts/verify-browser-runtime.mjs", "verify:web:network-captures": "npm run build:cached && node scripts/verify-web-network-captures.mjs", + "verify:web:http3": "npm run build:cached && node scripts/verify-http3-network-captures.mjs", "verify:binary:layout": "npm run build:cached && node scripts/verify-binary-layout.mjs", "verify:go:binary": "npm run build:cached && node scripts/verify-go-binary.mjs", "verify:wasm:package": "npm run build:cached && node scripts/verify-packaged-wasm-artifact.mjs", diff --git a/scripts/fixtures/http3-lab-requirements.txt b/scripts/fixtures/http3-lab-requirements.txt new file mode 100644 index 000000000..57e2af74a --- /dev/null +++ b/scripts/fixtures/http3-lab-requirements.txt @@ -0,0 +1,10 @@ +# Exercised runtime graph: CPython 3.14.7, Linux x86_64 guest. +aioquic==1.2.0 +attrs==26.1.0 +certifi==2026.7.22 +cffi==2.1.1 +cryptography==50.0.2 +pycparser==3.11 +pylsqpack==0.3.24 +pyOpenSSL==26.4.0 +service-identity==26.1.0 diff --git a/scripts/fixtures/http3-lab.py b/scripts/fixtures/http3-lab.py new file mode 100644 index 000000000..0f1ed7480 --- /dev/null +++ b/scripts/fixtures/http3-lab.py @@ -0,0 +1,416 @@ +"""Real loopback HTTP/3 laboratory using aioquic and caller-supplied mitmdump. + +Run inside a disposable Linux guest. All native records are written by +mitmproxy's real save addon, not constructed fixtures. Endpoint stream IDs are +observations scoped to that endpoint; no MITM stream mapping is claimed. +""" + +import argparse +import asyncio +import base64 +import hashlib +import json +import os +import signal +import ssl +import subprocess +import time +from pathlib import Path + +import aioquic +from aioquic.asyncio import QuicConnectionProtocol, connect, serve +from aioquic.h3.connection import H3Connection +from aioquic.h3.events import DataReceived, HeadersReceived +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.events import ConnectionTerminated, HandshakeCompleted, StreamReset +from aioquic.quic.packet import QuicProtocolVersion + + +def binary(value): + return {"base64": base64.b64encode(value).decode("ascii"), "bytes": len(value), + "sha256": hashlib.sha256(value).hexdigest()} + + +def pairs(headers): + return [[binary(name), binary(value)] for name, value in headers] + + +class Journal: + def __init__(self, root): + self.root = root + self.events = [] + self.requests = {} + self.responses = {} + self.tasks = set() + self.children = [] + self.writers = [] + + def emit(self, event, **data): + value = {"event": event, "monotonic": time.monotonic(), **data} + self.events.append(value) + with (self.root / "endpoints.jsonl").open("a") as handle: + handle.write(json.dumps(value) + "\n") + + def task(self, coroutine): + task = asyncio.create_task(coroutine) + self.tasks.add(task) + return task + + + +class Endpoint(QuicConnectionProtocol): + def __init__(self, *args, journal, endpoint, **kwargs): + super().__init__(*args, **kwargs) + self.journal, self.endpoint = journal, endpoint + self.http = H3Connection(self._quic) + self.messages = {} + self.waiters = {} + self.concurrent = [] + self.identity = f"{endpoint}-{len(journal.events)}" + + def quic_event_received(self, event): + if isinstance(event, HandshakeCompleted): + # This is the negotiated runtime value in pinned aioquic 1.2.0, + # not configuration.supported_versions or mitmproxy's QUICv1 label. + self.journal.emit("handshake", endpoint=self.endpoint, connection=self.identity, + alpn=event.alpn_protocol, quic_version=int(self._quic._version)) + if isinstance(event, (StreamReset, ConnectionTerminated)): + self.journal.emit(type(event).__name__, endpoint=self.endpoint, + connection=self.identity, error_code=event.error_code, + stream_id=getattr(event, "stream_id", None), + reason=getattr(event, "reason_phrase", None)) + targets = self.waiters.values() if isinstance(event, ConnectionTerminated) else [self.waiters.get(event.stream_id)] + for waiter in targets: + if waiter is not None and not waiter.done(): + waiter.set_exception(ConnectionError(type(event).__name__)) + for item in self.http.handle_event(event): + if isinstance(item, (HeadersReceived, DataReceived)): + message = self.messages.setdefault(item.stream_id, {"headers": [], "body": bytearray()}) + if isinstance(item, HeadersReceived): + message["headers"].extend(item.headers) + else: + message["body"].extend(item.data) + if item.stream_ended: + self.completed(item.stream_id, message) + + def completed(self, stream_id, message): + body = bytes(message["body"]) + if self.endpoint == "client": + self.journal.emit("response-received", endpoint="client", connection=self.identity, + stream_id=stream_id, headers=pairs(message["headers"]), body=binary(body)) + self.waiters[stream_id].set_result(message) + return + headers = dict(message["headers"]) + case = headers.get(b"x-lab-case", b"curl").decode("ascii") + self.journal.requests[case] = {"headers": pairs(message["headers"]), "body": binary(body)} + self.journal.emit("request-received", endpoint="origin", connection=self.identity, + stream_id=stream_id, case=case, **self.journal.requests[case]) + if case.startswith("parallel-"): + self.concurrent.append((case, stream_id, body)) + if len(self.concurrent) == 3: + self.journal.task(self.answer_parallel()) + else: + self.answer(case, stream_id, b"\x00\xfeanswer:" + case.encode() + b":" + body[::-1]) + + async def answer_parallel(self): + for case, stream_id, body in reversed(self.concurrent): + self.answer(case, stream_id, b"\x00\xfeanswer:" + case.encode() + b":" + body[::-1]) + await asyncio.sleep(0.15) + + def answer(self, case, stream_id, body): + headers = [(b":status", b"200"), (b"content-type", b"application/octet-stream"), + (b"content-length", str(len(body)).encode()), + (b"x-lab-case", case.encode()), (b"x-duplicate", b"one"), (b"x-duplicate", b"two")] + self.journal.responses[case] = {"headers": pairs(headers), "body": binary(body), "finished": True} + self.journal.emit("response-sent", endpoint="origin", connection=self.identity, + stream_id=stream_id, case=case, **self.journal.responses[case]) + self.http.send_headers(stream_id, headers) + self.http.send_data(stream_id, body, end_stream=True) + self.transmit() + + def request(self, case, body): + stream_id = self._quic.get_next_available_stream_id() + headers = [(b":method", b"POST"), (b":scheme", b"https"), + (b":authority", b"lab.test:18443"), (b":path", b"/same?token=ordinary&x=%2F"), + (b"x-lab-case", case.encode()), (b"content-type", b"application/octet-stream"), + (b"content-length", str(len(body)).encode()), + (b"x-duplicate", b"first"), (b"x-duplicate", b"second")] + self.waiters[stream_id] = asyncio.get_running_loop().create_future() + self.journal.emit("request-sent", endpoint="client", connection=self.identity, + stream_id=stream_id, case=case, headers=pairs(headers), body=binary(body), finished=True) + self.http.send_headers(stream_id, headers) + self.http.send_data(stream_id, body, end_stream=True) + self.transmit() + return stream_id, self.waiters[stream_id] + + + +class Proxy: + def __init__(self, args, root, name, http3=True): + self.args, self.root, self.name, self.http3 = args, root, name, http3 + self.process = None + self.log = None + + async def start(self): + self.log = (self.root / f"{self.name}.proxy.log").open("wb") + command = [self.args.mitmdump, "--mode", "reverse:https://lab.test:19443@127.0.0.1:18443", + "--set", f"confdir={self.root / 'proxy-config'}", "--set", "connection_strategy=lazy", + "--set", "keep_host_header=true", "--set", "store_streamed_bodies=false", + "--set", f"rea_lab_log={self.root / (self.name + '.producer.jsonl')}", + "-w", str(self.root / (self.name + '.mitm')), + "-s", str(Path(__file__).with_name("observe-mitmproxy-http3.py"))] + command.extend(["--set", f"ssl_verify_upstream_trusted_ca={self.root / 'origin-ca.pem'}"]) + if not self.http3: + command.extend(["--set", "http3=false"]) + self.process = subprocess.Popen(command, stdout=self.log, stderr=subprocess.STDOUT, + start_new_session=True) + lease = {"runId": os.environ["REA_PROCESS_RUN_ID"], "leaderPid": self.process.pid, + "processGroupId": self.process.pid} + with (self.root / "process-groups.jsonl").open("a") as handle: + handle.write(json.dumps(lease) + "\n") + deadline = time.monotonic() + 15 + journal = self.root / f"{self.name}.producer.jsonl" + while time.monotonic() < deadline: + if self.process.poll() is not None: + raise RuntimeError(f"Proxy exited: {self.log.name}") + if journal.exists() and '"event": "ready"' in journal.read_text(): + return self + await asyncio.sleep(0.05) + raise TimeoutError(f"Proxy readiness: {self.log.name}") + + async def stop(self): + if self.process is not None and self.process.poll() is None: + # The standalone mitmdump distribution has a bootloader parent. + # Signal its owned session together, including the interpreter child. + os.killpg(self.process.pid, signal.SIGINT) + try: + await asyncio.wait_for(asyncio.to_thread(self.process.wait), 10) + except TimeoutError: + os.killpg(self.process.pid, signal.SIGKILL) + await asyncio.to_thread(self.process.wait) + raise + if self.process is not None: + for _ in range(40): + if group_is_empty(self.process.pid): + return + await asyncio.sleep(0.05) + raise RuntimeError(f"Owned proxy group still live: {self.process.pid}") + + async def close_log(self): + if self.log is not None: + self.log.close() + + +def certificates(root): + for name in ["origin"]: + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(root / f"{name}-key.pem"), "-out", str(root / f"{name}-ca.pem"), + "-days", "2", "-subj", "/CN=lab.test", "-addext", "subjectAltName=DNS:lab.test", + "-addext", "basicConstraints=critical,CA:TRUE"], check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def client_config(root): + config = QuicConfiguration(is_client=True, alpn_protocols=["h3"], supported_versions=[QuicProtocolVersion.VERSION_1], + server_name="lab.test", idle_timeout=3) + config.load_verify_locations(cafile=str(root / "proxy-config/mitmproxy-ca-cert.pem")) + return config + + +def connection(root, journal): + return connect("127.0.0.1", 18443, configuration=client_config(root), + create_protocol=lambda *a, **kw: Endpoint(*a, journal=journal, endpoint="client", **kw)) + + +async def curl_request(root, journal, name, version, path="/curl", maximum=8): + child = await asyncio.create_subprocess_exec("curl", version, "--silent", "--show-error", + "--max-time", str(maximum), "--cacert", str(root / "proxy-config/mitmproxy-ca-cert.pem"), + "--resolve", "lab.test:18443:127.0.0.1", "--noproxy", "*", "-H", f"x-lab-case: {name}", + "--output", str(root / (name + ".body")), "--dump-header", str(root / (name + ".headers")), + "--write-out", "%{json}", "https://lab.test:18443" + path, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + journal.children.append(child) + out, err = await child.communicate() + (root / (name + ".json")).write_bytes(out) + (root / (name + ".stderr")).write_bytes(err) + return child.returncode, json.loads(out) + + +async def tcp_origin(reader, writer, journal): + journal.writers.append(writer) + try: + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 5) + case = "tcp" + for line in head.split(b"\r\n")[1:]: + if line.lower().startswith(b"x-lab-case:"): + case = line.split(b":", 1)[1].strip().decode("ascii") + body = b"\x00\xfftcp-origin" + journal.emit("tcp-request", case=case, alpn=writer.get_extra_info("ssl_object").selected_alpn_protocol(), + headers=binary(head), body=binary(body)) + writer.write(b"HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: " + + str(len(body)).encode() + b"\r\nconnection: close\r\n\r\n" + body) + await writer.drain() + finally: + writer.close() + await writer.wait_closed() + + +def group_is_empty(group): + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() + except FileNotFoundError: + continue + if fields[0] != "Z" and int(fields[2]) == group: + return False + return True + + +async def attempt_all(actions): + failures = [] + for name, action in actions: + try: + await asyncio.wait_for(action(), 12) + except BaseException as error: + failures.append((name, error)) + return failures + + +async def close_resource(resource): + resource.close() + + +async def finish_child(child): + if child.returncode is None: + child.terminate() + try: + await asyncio.wait_for(child.wait(), 3) + except TimeoutError: + child.kill() + await child.wait() + + +async def finish_tasks(journal): + for task in journal.tasks: + if not task.done(): + task.cancel() + results = await asyncio.gather(*journal.tasks, return_exceptions=True) + failures = [error for error in results if isinstance(error, BaseException) + and not isinstance(error, asyncio.CancelledError)] + if failures: + raise BaseExceptionGroup("Endpoint tasks failed", failures) + + +async def run(args): + if aioquic.__version__ != "1.2.0": + raise RuntimeError("This laboratory requires aioquic 1.2.0") + root = Path(args.output).resolve() + root.mkdir(mode=0o700) + os.umask(0o077) + journal = Journal(root) + loop = asyncio.get_running_loop() + asynchronous_errors = [] + def unexpected_exception(active_loop, context): + asynchronous_errors.append(context.get("message", "Unexpected asynchronous error")) + active_loop.default_exception_handler(context) + loop.set_exception_handler(unexpected_exception) + certificates(root) + server = tcp_server = None + proxies = [] + scenarios = [] + primary_error = None + cleanup_errors = [] + try: + config = QuicConfiguration(is_client=False, alpn_protocols=["h3"], + supported_versions=[QuicProtocolVersion.VERSION_1], idle_timeout=3) + config.load_cert_chain(str(root / "origin-ca.pem"), str(root / "origin-key.pem")) + server = await serve("127.0.0.1", 19443, configuration=config, + create_protocol=lambda *a, **kw: Endpoint(*a, journal=journal, endpoint="origin", **kw)) + tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls_context.load_cert_chain(str(root / "origin-ca.pem"), str(root / "origin-key.pem")) + tls_context.set_alpn_protocols(["http/1.1"]) + tcp_server = await asyncio.start_server(lambda r, w: journal.task(tcp_origin(r, w, journal)), + "127.0.0.1", 19443, ssl=tls_context) + proxy = Proxy(args, root, "normal") + proxies.append(proxy) + await proxy.start() + async with connection(root, journal) as client: + requests = [client.request(f"parallel-{i}", bytes([0, 255, i]) + bytes(range(256))) for i in range(3)] + await asyncio.wait_for(asyncio.gather(*(future for _, future in requests)), 10) + # cURL supplies an independently implemented HTTP/3 smoke case. + code, result = await curl_request(root, journal, "curl", "--http3-only") + if code != 0: + raise RuntimeError("HTTP/3 cURL failed; inspect curl.stderr") + if result["http_version"] != "3": + raise AssertionError("cURL silently downgraded") + await proxy.stop() + scenarios.append({"name": "normal", "capture": "normal.mitm"}) + + proxy = Proxy(args, root, "fallback", http3=False) + proxies.append(proxy) + await proxy.start() + code, result = await curl_request(root, journal, "fallback", "--http3", path="/fallback") + if code != 0 or result["http_version"] != "2": + raise AssertionError(f"Fallback control: {result}") + await proxy.stop() + scenarios.append({"name": "fallback", "capture": "fallback.mitm"}) + + except BaseException as error: + primary_error = error + finally: + actions = [] + for proxy in proxies: + actions.extend([(f"proxy-{proxy.name}", proxy.stop), + (f"proxy-log-{proxy.name}", proxy.close_log)]) + if server is not None: + actions.append(("quic-server", lambda: close_resource(server))) + if tcp_server is not None: + actions.extend([("tcp-listener-close", lambda: close_resource(tcp_server)), + ("tcp-listener-wait", tcp_server.wait_closed)]) + for index, writer in enumerate(journal.writers): + actions.extend([(f"tcp-writer-close-{index}", lambda w=writer: close_resource(w)), + (f"tcp-writer-wait-{index}", writer.wait_closed)]) + actions.append(("endpoint-tasks", lambda: finish_tasks(journal))) + for index, child in enumerate(journal.children): + actions.append((f"curl-child-{index}", lambda c=child: finish_child(c))) + cleanup_errors = await attempt_all(actions) + # This runs after every close attempt, including failed setup. + groups = [] + async def audit_groups(): + for proxy in proxies: + if proxy.process is not None: + groups.append({"group": proxy.process.pid, "empty": group_is_empty(proxy.process.pid)}) + if any(not item["empty"] for item in groups): + raise RuntimeError("Proxy groups remain live") + cleanup_errors.extend(await attempt_all([("process-groups", audit_groups)])) + try: + (root / "producer-cleanup.json").write_text(json.dumps({ + "primary_error": None if primary_error is None else str(primary_error), + "cleanup_errors": [{"action": name, "type": type(error).__name__, "message": str(error)} + for name, error in cleanup_errors], "groups": groups, + "asynchronous_errors": asynchronous_errors}, indent=2)) + except BaseException as error: + cleanup_errors.append(("cleanup-report", error)) + if primary_error is not None: + for name, error in cleanup_errors: + primary_error.add_note(f"Cleanup {name}: {error}") + raise primary_error + if cleanup_errors: + raise BaseExceptionGroup("HTTP/3 producer cleanup failed", [error for _, error in cleanup_errors]) + if asynchronous_errors: + raise RuntimeError(f"Unexpected asynchronous errors: {asynchronous_errors}") + (root / "manifest.json").write_text(json.dumps({"profile": "http3-minimal-v1", "aioquic": aioquic.__version__, + "scenarios": scenarios, "requests": journal.requests, "responses": journal.responses, + "events": journal.events, "limitations": [ + "Endpoint stream IDs are scoped observations, not mitmproxy serialized IDs or mappings.", + "The fixture reads negotiated _version from pinned aioquic 1.2.0.", + "Only QUIC v1, Linux reverse proxy and the selected endpoints are exercised."]}, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--mitmdump", required=True) + parser.add_argument("--output", required=True) + asyncio.run(run(parser.parse_args())) diff --git a/scripts/fixtures/http3-lab.test.py b/scripts/fixtures/http3-lab.test.py new file mode 100644 index 000000000..3bc8c1b7e --- /dev/null +++ b/scripts/fixtures/http3-lab.test.py @@ -0,0 +1,56 @@ +"""Lifecycle regressions; these tests do not establish HTTP/3 provider support.""" +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("http3_lab", Path(__file__).with_name("http3-lab.py")) +lab = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lab) + + +class CleanupTests(unittest.IsolatedAsyncioTestCase): + async def test_all_actions_run_after_multiple_close_failures(self): + attempted = [] + async def fail(): + attempted.append("failed-close") + raise RuntimeError("close failed") + async def audit(): + attempted.append("audit") + failures = await lab.attempt_all([("one", fail), ("two", fail), ("audit", audit)]) + self.assertEqual(attempted, ["failed-close", "failed-close", "audit"]) + self.assertEqual([name for name, _ in failures], ["one", "two"]) + + async def test_partial_setup_keeps_primary_error_and_audits_after_close_failure(self): + original_serve = lab.serve + closed = [] + class ClosingServer: + def __init__(self, server): + self.server = server + def close(self): + self.server.close() + closed.append(True) + raise RuntimeError("injected close failure") + async def serve(*args, **kwargs): + return ClosingServer(await original_serve(*args, **kwargs)) + async def fail_tcp(*args, **kwargs): + raise RuntimeError("injected primary setup failure") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "capture" + with patch.object(lab, "serve", serve), patch.object(lab.asyncio, "start_server", fail_tcp): + with self.assertRaisesRegex(RuntimeError, "injected primary setup failure") as caught: + await lab.run(SimpleNamespace(output=str(root), mitmdump="unused")) + self.assertEqual(closed, [True]) + self.assertIn("injected close failure", " ".join(caught.exception.__notes__)) + report = json.loads((root / "producer-cleanup.json").read_text()) + self.assertEqual(report["primary_error"], "injected primary setup failure") + self.assertEqual(report["cleanup_errors"][0]["action"], "quic-server") + self.assertEqual(report["groups"], []) + self.assertFalse((root / "manifest.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/fixtures/observe-mitmproxy-http3.py b/scripts/fixtures/observe-mitmproxy-http3.py new file mode 100644 index 000000000..003025a75 --- /dev/null +++ b/scripts/fixtures/observe-mitmproxy-http3.py @@ -0,0 +1,64 @@ +"""Test-only observation of real flows; never synthesizes flow or stream IDs.""" + +import base64 +import hashlib +import json +from pathlib import Path + +from mitmproxy import ctx + + +def bytes_view(value): + if value is None: + return None + return {"base64": base64.b64encode(value).decode("ascii"), + "bytes": len(value), "sha256": hashlib.sha256(value).hexdigest()} + + +class Observer: + def load(self, loader): + loader.add_option("rea_lab_log", str, "", "HTTP/3 verifier observations") + + def emit(self, event, **data): + with Path(ctx.options.rea_lab_log).open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"event": event, **data}) + "\n") + + def running(self): + self.emit("ready") + + def connection(self, conn): + return {"id": conn.id, "transport_protocol": conn.transport_protocol, + "alpn": bytes_view(conn.alpn), "tls_version": conn.tls_version, + "timestamp_tls_setup": conn.timestamp_tls_setup, + "error": conn.error} + + def tls_established_client(self, data): + self.emit("tls-established-client", connection=self.connection(data.conn)) + + def tls_established_server(self, data): + self.emit("tls-established-server", connection=self.connection(data.conn)) + + def flow(self, event, flow): + def message(message): + if message is None: + return None + return {"http_version": message.http_version, + "content": bytes_view(message.raw_content), + "timestamp_end": message.timestamp_end} + self.emit(event, id=flow.id, case=flow.request.headers.get("x-lab-case", "curl"), + request=message(flow.request), response=message(flow.response), + error=None if flow.error is None else flow.error.msg, + client_conn=self.connection(flow.client_conn), + server_conn=self.connection(flow.server_conn)) + + def request(self, flow): + self.flow("request", flow) + + def response(self, flow): + self.flow("response", flow) + + def error(self, flow): + self.flow("error", flow) + + +addons = [Observer()] diff --git a/scripts/lib/http3-capture-assertions.mjs b/scripts/lib/http3-capture-assertions.mjs new file mode 100644 index 000000000..ca08d32cc --- /dev/null +++ b/scripts/lib/http3-capture-assertions.mjs @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { isUtf8 } from "node:buffer"; +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +// The native decoder displays strict UTF-8 and retains invalid bytes as null text. +const textOf = (expected) => { + const value = Buffer.from(expected.base64, "base64"); + return isUtf8(value) ? value.toString("utf8") : null; +}; +const field = (record, pointer) => + record.binary_fields.find((item) => item.pointer === pointer); +const bytes = (record, pointer) => { + const value = field(record, pointer); + assert.equal( + value?.state, + "retained", + `Missing retained bytes at ${pointer}`, + ); + const decoded = Buffer.from(value.content_base64, "base64"); + assert.equal(value.bytes, decoded.length); + assert.equal(value.sha256, digest(decoded)); + return decoded; +}; +const binaryEqual = (record, pointer, expected) => { + const decoded = bytes(record, pointer); + assert.deepEqual(decoded, Buffer.from(expected.base64, "base64"), pointer); + assert.equal(field(record, pointer).sha256, expected.sha256); + assert.equal(field(record, pointer).bytes, expected.bytes); +}; +const caseOf = (record) => + record.reported.request.headers.find( + ([name]) => name === "x-lab-case", + )?.[1] ?? "curl"; +const byCase = (value, name) => { + const selected = value.records.filter((record) => caseOf(record) === name); + assert.equal( + selected.length, + 1, + `Expected exactly one producer record for ${name}`, + ); + return selected[0]; +}; +const connectionH3 = (record, side) => { + const conn = record.reported[side]; + assert.equal(conn.transport_protocol, "udp"); + assert.equal(bytes(record, `/${side}/alpn`).toString(), "h3"); + assert.equal(conn.alpn, "h3", `${side}.alpn readable value`); + assert.equal(typeof conn.timestamp_tls_setup, "number"); +}; + +function assertHeaders(record, side, headers) { + const regular = headers.filter( + ([name]) => !Buffer.from(name.base64, "base64").toString().startsWith(":"), + ); + assert.equal(record.reported[side].headers.length, regular.length); + for (const [index, pair] of regular.entries()) { + binaryEqual(record, `/${side}/headers/${index}/0`, pair[0]); + binaryEqual(record, `/${side}/headers/${index}/1`, pair[1]); + assert.deepEqual( + record.reported[side].headers[index], + pair.map(textOf), + `${side}.headers[${index}] readable values`, + ); + } +} + +export function assertEndpointHeaders(actual, expected) { + // The proxy rebuilds pseudo fields from its HTTP state. Compare their values + // by name while retaining the exact order and duplicates of regular headers. + const partition = (headers) => { + const pseudo = new Map(); + const regular = []; + for (const pair of headers) { + const name = Buffer.from(pair[0].base64, "base64").toString(); + if (name.startsWith(":")) { + assert.equal(pseudo.has(name), false); + pseudo.set(name, pair); + } else regular.push(pair); + } + return { pseudo, regular }; + }; + assert.deepEqual(partition(actual), partition(expected)); +} + +export function requireHttp3(record) { + for (const side of ["request", "response"]) + assert.equal(record.reported[side].http_version, "HTTP/3"); + connectionH3(record, "client_conn"); + connectionH3(record, "server_conn"); +} + +export function verifyNormalRecords(normal, manifest, finalProducer) { + for (const record of normal.records) { + const name = caseOf(record); + const producer = finalProducer.find( + (event) => event.id === record.reported.id, + ); + assert.ok(producer, `No actual producer observation for ${name}`); + assert.equal(record.reported.error, null); + assert.equal(producer.error, null); + assert.equal(record.reported.response.status_code, 200); + requireHttp3(record); + for (const side of ["request", "response"]) { + assert.equal( + record.reported[side].http_version, + producer[side].http_version, + ); + binaryEqual(record, `/${side}/content`, producer[side].content); + assertHeaders( + record, + side, + manifest[side === "request" ? "requests" : "responses"][name].headers, + ); + } + // Native UUIDs must be retained as UUIDs; the producer exports no stream mapping. + for (const side of ["client_conn", "server_conn"]) + for (const key of ["id", "tls_version", "timestamp_tls_setup", "error"]) + assert.equal( + record.reported[side][key], + producer[side][key], + `${side}.${key} differs from producer observation ${producer.id}`, + ); + binaryEqual(record, "/request/content", manifest.requests[name].body); + binaryEqual(record, "/response/content", manifest.responses[name].body); + const pseudo = { + ":method": "method", + ":scheme": "scheme", + ":authority": "authority", + ":path": "path", + }; + for (const [headerName, headerValue] of manifest.requests[name].headers) { + const decodedName = Buffer.from(headerName.base64, "base64").toString(); + if (pseudo[decodedName]) { + binaryEqual(record, `/request/${pseudo[decodedName]}`, headerValue); + assert.equal( + record.reported.request[pseudo[decodedName]], + textOf(headerValue), + `request.${pseudo[decodedName]} readable value`, + ); + } + } + } +} + +export { bytes, caseOf, byCase }; + +export function verifyConcurrency(normal, manifest) { + const parallelNames = ["parallel-0", "parallel-1", "parallel-2"]; + const sent = manifest.events.filter( + (event) => + event.event === "request-sent" && parallelNames.includes(event.case), + ); + const received = manifest.events.filter( + (event) => + event.event === "request-received" && parallelNames.includes(event.case), + ); + const responses = manifest.events.filter( + (event) => + event.event === "response-sent" && parallelNames.includes(event.case), + ); + assert.equal(new Set(sent.map((event) => event.connection)).size, 1); + assert.equal(new Set(received.map((event) => event.connection)).size, 1); + assert.equal(new Set(sent.map((event) => event.stream_id)).size, 3); + assert.equal(new Set(received.map((event) => event.stream_id)).size, 3); + assert.ok( + Math.max(...received.map((event) => event.monotonic)) < + responses[0].monotonic, + ); + assert.deepEqual( + responses.map((event) => event.case), + received.map((event) => event.case).reverse(), + ); + for (const side of ["client_conn", "server_conn"]) + assert.equal( + new Set( + parallelNames.map((name) => byCase(normal, name).reported[side].id), + ).size, + 1, + ); + const normalConnections = new Set( + [...sent, ...received].map((event) => event.connection), + ); + for (const connection of normalConnections) { + const handshakes = manifest.events.filter( + (event) => event.event === "handshake" && event.connection === connection, + ); + assert.equal(handshakes.length, 1); + assert.equal(handshakes[0].alpn, "h3"); + assert.equal(handshakes[0].quic_version, 1); + } + assert.equal( + new Set(normal.records.map((record) => record.reported.id)).size, + 4, + ); + assert.equal(sent.length, 3); + const receivedByClient = manifest.events.filter( + (event) => + event.event === "response-received" && + sent.some( + (request) => + request.connection === event.connection && + request.stream_id === event.stream_id, + ), + ); + assert.equal(receivedByClient.length, 3); + for (const request of sent) { + const response = receivedByClient.find( + (event) => + event.connection === request.connection && + event.stream_id === request.stream_id, + ); + assert.ok( + response, + "Each sent request needs an independently observed client response", + ); + assert.deepEqual(response.body, manifest.responses[request.case].body); + assert.deepEqual(request.body, manifest.requests[request.case].body); + assertEndpointHeaders( + request.headers, + manifest.requests[request.case].headers, + ); + assertEndpointHeaders( + response.headers, + manifest.responses[request.case].headers, + ); + } + return { + sent: sent.map((event) => event.case), + completed_at_origin: received.map((event) => event.case), + dispatched_at_origin: responses.map((event) => event.case), + completed_at_client: receivedByClient.map( + (event) => + sent.find( + (request) => + request.connection === event.connection && + request.stream_id === event.stream_id, + ).case, + ), + saved_in_capture: normal.records + .filter((record) => parallelNames.includes(caseOf(record))) + .map(caseOf), + }; +} diff --git a/scripts/lib/http3-capture-assertions.test.mjs b/scripts/lib/http3-capture-assertions.test.mjs new file mode 100644 index 000000000..bc7e3c602 --- /dev/null +++ b/scripts/lib/http3-capture-assertions.test.mjs @@ -0,0 +1,163 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { test } from "node:test"; +import { verifyNormalRecords } from "./http3-capture-assertions.mjs"; + +function connectionFactsFixture() { + const body = Buffer.from([0, 255, 128, 1]); + const view = (bytes) => ({ + base64: bytes.toString("base64"), + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + const producer = { + id: "flow-id", + error: null, + request: { http_version: "HTTP/3", content: view(body) }, + response: { http_version: "HTTP/3", content: view(body) }, + client_conn: { + id: "client-id", + transport_protocol: "udp", + tls_version: "QUICv1", + timestamp_tls_setup: 1_700_000_000.125678, + error: null, + }, + server_conn: { + id: "server-id", + transport_protocol: "udp", + tls_version: "QUICv1", + timestamp_tls_setup: 1_700_000_000.987654, + error: null, + }, + }; + const reported = structuredClone(producer); + const requestFields = { + method: "POST", + scheme: "https", + authority: "lab.test:18443", + path: "/echo?case=readable", + }; + Object.assign(reported.request, requestFields); + const headers = [ + [Buffer.from("x-repeat"), Buffer.from("first-α")], + [Buffer.from("x-repeat"), Buffer.from("second-β")], + [Buffer.from("x-binary"), Buffer.from([255])], + ]; + for (const side of ["request", "response"]) + reported[side].headers = [ + ["x-repeat", "first-α"], + ["x-repeat", "second-β"], + ["x-binary", null], + ]; + reported.client_conn.alpn = "h3"; + reported.server_conn.alpn = "h3"; + reported.response.status_code = 200; + const binary_fields = [ + ["/request/content", body], + ["/response/content", body], + ["/client_conn/alpn", Buffer.from("h3")], + ["/server_conn/alpn", Buffer.from("h3")], + ...Object.entries(requestFields).map(([name, value]) => [ + `/request/${name}`, + Buffer.from(value), + ]), + ...["request", "response"].flatMap((side) => + headers.flatMap((pair, index) => + pair.map((value, part) => [`/${side}/headers/${index}/${part}`, value]), + ), + ), + ].map(([pointer, bytes]) => { + const { base64, ...facts } = view(bytes); + return { + pointer, + representation: "producer-bytes", + state: "retained", + content_base64: base64, + ...facts, + }; + }); + return { + normal: { records: [{ reported, binary_fields }] }, + manifest: { + requests: { + curl: { + headers: [ + ...Object.entries(requestFields).map(([name, value]) => [ + view(Buffer.from(`:${name}`)), + view(Buffer.from(value)), + ]), + ...headers.map((pair) => pair.map(view)), + ], + body: view(body), + }, + }, + responses: { + curl: { + headers: headers.map((pair) => pair.map(view)), + body: view(body), + }, + }, + }, + producer: [producer], + }; +} + +test("positive assertions retain exact producer connection facts on both legs", () => { + const { normal, manifest, producer } = connectionFactsFixture(); + verifyNormalRecords(normal, manifest, producer); +}); + +for (const [field, value] of [ + ["method", "GET"], + ["scheme", "http"], + ["authority", "other.test:18443"], + ["path", "/different"], +]) + test(`positive assertions reject changed readable request ${field}`, () => { + const { normal, manifest, producer } = connectionFactsFixture(); + normal.records[0].reported.request[field] = value; + assert.throws(() => verifyNormalRecords(normal, manifest, producer), { + code: "ERR_ASSERTION", + }); + }); + +for (const side of ["client_conn", "server_conn"]) + test(`positive assertions reject null readable ${side} ALPN`, () => { + const { normal, manifest, producer } = connectionFactsFixture(); + normal.records[0].reported[side].alpn = null; + assert.throws(() => verifyNormalRecords(normal, manifest, producer), { + code: "ERR_ASSERTION", + }); + }); + +for (const side of ["request", "response"]) + for (const [index, part, value] of [ + [0, 0, "x-changed"], + [0, 1, "changed-first"], + [1, 1, "changed-second"], + [2, 1, "�"], + ]) + test(`positive assertions reject changed readable ${side} header ${index}/${part}`, () => { + const { normal, manifest, producer } = connectionFactsFixture(); + normal.records[0].reported[side].headers[index][part] = value; + assert.throws(() => verifyNormalRecords(normal, manifest, producer), { + code: "ERR_ASSERTION", + }); + }); + +for (const side of ["client_conn", "server_conn"]) + for (const [field, value] of [ + ["tls_version", null], + ["tls_version", "changed-producer-label"], + ["error", "changed-connection-error"], + ["error", undefined], + ["timestamp_tls_setup", 0], + ["timestamp_tls_setup", 1_700_000_000], + ]) + test(`positive assertions reject ${side}.${field} changed to ${String(value)}`, () => { + const { normal, manifest, producer } = connectionFactsFixture(); + normal.records[0].reported[side][field] = value; + assert.throws(() => verifyNormalRecords(normal, manifest, producer), { + code: "ERR_ASSERTION", + }); + }); diff --git a/scripts/lib/http3-verifier-lifecycle.mjs b/scripts/lib/http3-verifier-lifecycle.mjs new file mode 100644 index 000000000..d083bf8d6 --- /dev/null +++ b/scripts/lib/http3-verifier-lifecycle.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { setTimeout } from "node:timers/promises"; +import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js"; +import { observeOwnedProcessGroups } from "../../dist/process/ProcessOwnershipObservation.js"; + +export const describeError = (error) => ({ + name: error?.name ?? "Error", + message: error?.message ?? String(error), + ...(error?.stack ? { stack: error.stack } : {}), +}); + +export async function attemptAll(actions) { + const failures = []; + for (const [name, action] of actions) { + try { + await action(); + } catch (error) { + failures.push({ action: name, ...describeError(error) }); + } + } + return failures; +} + +async function latest(root, report) { + const temporary = join(root, `.verification-${report.run_id}.tmp`); + await writeFile(temporary, JSON.stringify(report, null, 2), { flag: "wx" }); + await rename(temporary, join(root, "verification-latest.json")); +} + +async function withLatestLock(root, action) { + // Keep this pathname stable: removing it could split concurrent lock owners. + // SQLite releases its OS-backed lock when the connection/process closes. + const database = new DatabaseSync(join(root, ".verification-lock.sqlite")); + try { + for (;;) { + try { + database.exec("BEGIN EXCLUSIVE"); + break; + } catch (error) { + if ( + error.code !== "ERR_SQLITE_ERROR" || + (error.errcode !== 5 && error.errcode !== 6) + ) + throw error; + // Yield while another process (or async operation here) owns the lock. + // A synchronous busy timeout would block that operation's fs callbacks. + await setTimeout(10); + } + } + return await action(); + } finally { + database.close(); + } +} + +export async function prepareAttempt(root, inspectExisting, run) { + if (!inspectExisting) await mkdir(root, { mode: 0o700 }); + const directory = join(root, "verifications", run.run_id); + await mkdir(directory, { recursive: true, mode: 0o700 }); + await withLatestLock(root, async () => { + // Preserve older reports as history, without leaving an ambiguous success + // at the capture root. Registration is the shared marker's ordering point. + try { + await rename( + join(root, "verification.json"), + join(directory, "legacy-verification.json"), + ); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + await latest(root, { + run_id: run.run_id, + status: "running", + started_at: new Date().toISOString(), + directory, + }); + }); + return { directory, captureRoot: join(root, "captures") }; +} + +export async function publishAttempt(root, directory, report) { + await writeFile( + join(directory, "result.json"), + JSON.stringify(report, null, 2), + { flag: "wx" }, + ); + if (report.status === "passed") + await writeFile( + join(directory, "verification.json"), + JSON.stringify(report, null, 2), + { flag: "wx" }, + ); + await withLatestLock(root, async () => { + const current = JSON.parse( + await readFile(join(root, "verification-latest.json"), "utf8"), + ); + if (current.run_id !== report.run_id) return; + // Ownership check and replacement share one lock with attempt registration. + // An older attempt still keeps its reports, but cannot replace this marker. + await latest(root, { + run_id: report.run_id, + status: report.status, + directory, + finished_at: report.finished_at, + }); + }); +} + +export async function finishProducerGroups(captureRoot, runId) { + let leases; + try { + leases = (await readFile(join(captureRoot, "process-groups.jsonl"), "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + } catch (error) { + if (error.code === "ENOENT") return { groups: [], observation: [] }; + throw error; + } + for (const lease of leases) assert.equal(lease.runId, runId); + const failures = await attemptAll( + leases.map((lease) => [ + `producer-group-${lease.processGroupId}`, + async () => { + const result = await cleanupOwnedProcessGroup(lease); + assert.equal(result.cleaned, true, JSON.stringify(result)); + }, + ]), + ); + // Inspect even when one close failed; the groups can outlive their bootloader + // parent and disappear from a plain descendant-only observation. + const observation = [ + ...(await observeOwnedProcessGroups( + runId, + leases.map((lease) => lease.processGroupId), + )), + ]; + return { groups: leases, failures, observation }; +} diff --git a/scripts/lib/http3-verifier-lifecycle.test.mjs b/scripts/lib/http3-verifier-lifecycle.test.mjs new file mode 100644 index 000000000..fc7c3f5d2 --- /dev/null +++ b/scripts/lib/http3-verifier-lifecycle.test.mjs @@ -0,0 +1,433 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { setTimeout } from "node:timers/promises"; +import { DatabaseSync } from "node:sqlite"; +import { + attemptAll, + prepareAttempt, + publishAttempt, + finishProducerGroups, +} from "./http3-verifier-lifecycle.mjs"; +import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js"; +import { runOwnedCommand } from "../../dist/process/OwnedCommand.js"; +import { verifyConcurrency } from "./http3-capture-assertions.mjs"; + +async function waitForFile(path) { + const signal = AbortSignal.timeout(5_000); + for (;;) { + try { + await readFile(path); + return; + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + await setTimeout(10, undefined, { signal }); + } +} + +function startAttemptWorker(root, report) { + const worker = ` + import { readFile, writeFile } from "node:fs/promises"; + import { join } from "node:path"; + import { setTimeout } from "node:timers/promises"; + import { prepareAttempt, publishAttempt } from ${JSON.stringify( + new URL("./http3-verifier-lifecycle.mjs", import.meta.url).href, + )}; + const [root, serialized] = process.argv.slice(1); + const report = JSON.parse(serialized); + const attempt = await prepareAttempt(root, true, report); + await writeFile(join(root, report.run_id + ".ready"), ""); + for (;;) { + try { await readFile(join(root, report.run_id + ".finish")); break; } + catch (error) { if (error.code !== "ENOENT") throw error; } + await setTimeout(10); + } + await publishAttempt(root, attempt.directory, report); + `; + return runOwnedCommand( + { + command: process.execPath, + arguments: [ + "--input-type=module", + "--eval", + worker, + root, + JSON.stringify(report), + ], + cwd: root, + hostEnvironment: process.env, + runId: randomUUID(), + }, + { timeoutMs: 10_000, diagnosticBytes: 4096 }, + ).then( + (result) => ({ result }), + (error) => ({ error }), + ); +} + +test("cleanup continues through failed closes and reaches its audit", async () => { + const attempted = []; + const failures = await attemptAll( + ["client", "transport", "audit"].map((name) => [ + name, + async () => { + attempted.push(name); + if (name !== "audit") throw new Error(`${name} failed`); + }, + ]), + ); + assert.deepEqual(attempted, ["client", "transport", "audit"]); + assert.deepEqual( + failures.map((failure) => failure.action), + ["client", "transport"], + ); +}); + +test("a failed reinspection cannot leave the earlier success as the current result", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-http3-results-")); + try { + await writeFile( + join(root, "verification.json"), + '{"status":"passed","run_id":"legacy"}', + ); + const first = await prepareAttempt(root, true, { run_id: "first" }); + await publishAttempt(root, first.directory, { + run_id: "first", + status: "passed", + }); + const next = await prepareAttempt(root, true, { run_id: "next" }); + assert.equal( + JSON.parse(await readFile(join(root, "verification-latest.json"))).status, + "running", + ); + await publishAttempt(root, next.directory, { + run_id: "next", + status: "failed", + error: { message: "new failure" }, + }); + assert.equal( + JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id, + "next", + ); + assert.equal( + JSON.parse(await readFile(join(root, "verification-latest.json"))).status, + "failed", + ); + await assert.rejects(readFile(join(next.directory, "verification.json")), { + code: "ENOENT", + }); + await assert.rejects(readFile(join(root, "verification.json")), { + code: "ENOENT", + }); + assert.equal( + JSON.parse(await readFile(join(first.directory, "verification.json"))) + .status, + "passed", + ); + assert.equal( + JSON.parse( + await readFile(join(first.directory, "legacy-verification.json")), + ).run_id, + "legacy", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("separate overlapping attempts finishing in reverse order retain the newer failure", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-http3-overlap-")); + const commands = []; + const failed = { + run_id: "newer", + status: "failed", + error: { message: "newer inspection failed" }, + cleanup_failures: [{ action: "transport-close", message: "close failed" }], + }; + let primaryError; + const start = (report) => { + const command = startAttemptWorker(root, report); + commands.push(command); + return command; + }; + try { + const older = start({ run_id: "older", status: "passed" }); + await waitForFile(join(root, "older.ready")); + const newer = start(failed); + await waitForFile(join(root, "newer.ready")); + await writeFile(join(root, "newer.finish"), ""); + const newerResult = await newer; + if (newerResult.error) throw newerResult.error; + assert.equal(newerResult.result.exitCode, 0); + assert.equal(newerResult.result.signal, null); + const marker = await readFile(join(root, "verification-latest.json")); + await writeFile(join(root, "older.finish"), ""); + const olderResult = await older; + if (olderResult.error) throw olderResult.error; + assert.equal(olderResult.result.exitCode, 0); + assert.equal(olderResult.result.signal, null); + assert.deepEqual( + await readFile(join(root, "verification-latest.json")), + marker, + ); + assert.deepEqual( + JSON.parse(await readFile(join(root, "verifications/newer/result.json"))), + failed, + ); + await assert.rejects( + readFile(join(root, "verifications/newer/verification.json")), + { + code: "ENOENT", + }, + ); + assert.equal( + JSON.parse( + await readFile(join(root, "verifications/older/verification.json")), + ).status, + "passed", + ); + } catch (error) { + primaryError = error; + } finally { + // Release both barriers even after an assertion/setup failure. Each command + // retains native supervision through termination before the root is removed. + const failures = await attemptAll( + ["older", "newer"].map((name) => [ + name, + () => writeFile(join(root, name + ".finish"), ""), + ]), + ); + const outcomes = await Promise.all(commands); + const retained = outcomes.filter( + ({ error }) => error?.cleanupFailure || error?.cleanupResult, + ); + if (retained.length) + primaryError = new AggregateError( + [primaryError, ...retained.map(({ error }) => error)].filter(Boolean), + `Worker cleanup failed; retained test evidence at ${root}`, + { cause: primaryError }, + ); + else + failures.push( + ...(await attemptAll([ + ["root", () => rm(root, { recursive: true, force: true })], + ])), + ); + if (failures.length) + primaryError = new AggregateError( + [primaryError, ...failures].filter(Boolean), + "Test cleanup failed", + { cause: primaryError }, + ); + } + if (primaryError) throw primaryError; +}); + +test("publication waits for the lock before reading the current owner", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-")); + let database; + let publication; + let primaryError; + try { + const older = await prepareAttempt(root, true, { run_id: "older" }); + database = new DatabaseSync(join(root, ".verification-lock.sqlite")); + database.exec("BEGIN EXCLUSIVE"); + publication = publishAttempt(root, older.directory, { + run_id: "older", + status: "passed", + }); + await waitForFile(join(older.directory, "verification.json")); + const marker = join(root, "verification-latest.json"); + assert.equal(JSON.parse(await readFile(marker)).status, "running"); + // This connection represents another marker writer. Its replacement must + // be observed only after the blocked publisher acquires the same lock. + const newer = { run_id: "newer", status: "failed" }; + await writeFile(marker, JSON.stringify(newer)); + database.close(); + database = undefined; + await publication; + assert.deepEqual(JSON.parse(await readFile(marker)), newer); + } catch (error) { + primaryError = error; + } finally { + const failures = await attemptAll([ + ["lock", () => database?.close()], + [ + "publication", + async () => { + if (publication) await publication; + }, + ], + ["root", () => rm(root, { recursive: true, force: true })], + ]); + if (failures.length) + primaryError = new AggregateError( + [primaryError, ...failures].filter(Boolean), + "Test cleanup failed", + { cause: primaryError }, + ); + } + if (primaryError) throw primaryError; +}); + +test("a failed marker read releases the lock and preserves the attempt report", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-failure-")); + try { + const older = await prepareAttempt(root, true, { run_id: "older" }); + await writeFile(join(root, "verification-latest.json"), "{"); + const report = { + run_id: "older", + status: "failed", + error: { message: "inspection failed" }, + }; + await assert.rejects( + publishAttempt(root, older.directory, report), + SyntaxError, + ); + assert.deepEqual( + JSON.parse(await readFile(join(older.directory, "result.json"))), + report, + ); + const next = await prepareAttempt(root, true, { run_id: "next" }); + await publishAttempt(root, next.directory, { + run_id: "next", + status: "passed", + }); + assert.equal( + JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id, + "next", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("dispatch reverses origin completion; client and capture may arrive in other orders", () => { + const names = ["parallel-0", "parallel-1", "parallel-2"]; + const header = (text) => ({ base64: Buffer.from(text).toString("base64") }); + const messages = Object.fromEntries( + names.map((name) => [ + name, + { + headers: [[header("x-lab-case"), header(name)]], + body: { base64: header(name).base64 }, + }, + ]), + ); + const event = (kind, index, connection, monotonic) => ({ + event: kind, + case: names[index], + stream_id: index * 4, + connection, + monotonic, + ...messages[names[index]], + }); + const manifest = { + requests: messages, + responses: messages, + events: [ + ...[0, 1, 2].map((i) => event("request-sent", i, "client", i)), + ...[1, 2, 0].map((i) => event("request-received", i, "origin", 10)), + ...[0, 2, 1].map((i) => event("response-sent", i, "origin", 20)), + ...[2, 0, 1].map((i) => event("response-received", i, "client", 30)), + ...["client", "origin"].map((connection) => ({ + event: "handshake", + connection, + alpn: "h3", + quic_version: 1, + })), + ], + }; + const record = (name) => ({ + reported: { + id: name, + request: { headers: [["x-lab-case", name]] }, + client_conn: { id: "client-uuid" }, + server_conn: { id: "origin-uuid" }, + }, + }); + const normal = { + records: [...[1, 0, 2].map((i) => record(names[i])), record("curl")], + }; + const orders = verifyConcurrency(normal, manifest); + assert.deepEqual(orders.dispatched_at_origin, [ + "parallel-0", + "parallel-2", + "parallel-1", + ]); + manifest.events = manifest.events.filter( + (item) => item.event !== "response-sent", + ); + manifest.events.push( + ...[2, 1, 0].map((i) => event("response-sent", i, "origin", 20)), + ); + assert.throws(() => verifyConcurrency(normal, manifest), { + code: "ERR_ASSERTION", + }); +}); + +test("cleanup verifies and retires a separately owned producer group", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-http3-group-")); + const runId = randomUUID(); + const child = spawn("/usr/bin/sleep", ["60"], { + detached: true, + env: { ...process.env, REA_PROCESS_RUN_ID: runId }, + stdio: "ignore", + }); + let primaryError; + let cleaned = false; + try { + await once(child, "spawn"); + await writeFile( + join(root, "process-groups.jsonl"), + JSON.stringify({ + runId, + leaderPid: child.pid, + processGroupId: child.pid, + }) + "\n", + ); + const result = await finishProducerGroups(root, runId); + assert.deepEqual(result.failures, []); + assert.equal(result.observation[0][1].state, "empty"); + } catch (error) { + primaryError = error; + } finally { + const failures = await attemptAll([ + [ + "process", + async () => { + const result = await cleanupOwnedProcessGroup({ + runId, + leaderPid: child.pid, + processGroupId: child.pid, + }); + assert.equal( + result.cleaned, + true, + `Retained ownership evidence at ${root}: ${JSON.stringify(result)}`, + ); + cleaned = true; + }, + ], + [ + "root", + async () => { + if (cleaned) await rm(root, { recursive: true, force: true }); + }, + ], + ]); + if (failures.length) + primaryError = new AggregateError( + [primaryError, ...failures].filter(Boolean), + "Test cleanup failed", + { cause: primaryError }, + ); + } + if (primaryError) throw primaryError; +}); diff --git a/scripts/verify-http3-network-captures.mjs b/scripts/verify-http3-network-captures.mjs new file mode 100644 index 000000000..2b3493821 --- /dev/null +++ b/scripts/verify-http3-network-captures.mjs @@ -0,0 +1,408 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { isAbsolute, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; +import { toolContract } from "../dist/contracts/toolContracts.js"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { + bytes, + byCase, + verifyNormalRecords, + requireHttp3, + verifyConcurrency, +} from "./lib/http3-capture-assertions.mjs"; +import { + attemptAll, + describeError, + prepareAttempt, + publishAttempt, + finishProducerGroups, +} from "./lib/http3-verifier-lifecycle.mjs"; +import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; + +const root = process.argv[2]; +const inspectExisting = process.argv[3] === "--inspect-existing"; +const mitmdump = process.env.REA_MITMDUMP_COMMAND; +const python = process.env.REA_HTTP3_PYTHON_COMMAND; +assert.equal( + process.platform, + "linux", + "HTTP/3 native verification requires Linux", +); +assert.ok( + root && isAbsolute(root), + "Select an absolute, new evidence directory", +); +assert.ok( + mitmdump && isAbsolute(mitmdump), + "Select caller-supplied mitmdump 12.2.3", +); +if (!inspectExisting) + assert.ok( + python && isAbsolute(python), + "Select a Python environment with aioquic 1.2.0", + ); +const run = createVerifierRun(); +const cwd = fileURLToPath(new URL("..", import.meta.url)); +const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url)); +const environment = Object.fromEntries( + Object.entries(process.env).filter(([, value]) => typeof value === "string"), +); +const client = new Client({ + name: "real-http3-capture-verifier", + version: "1", +}); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env: environment, + stderr: "pipe", +}); +const contract = toolContract("inspect_web_network_capture"); +const maxMcpStderrBytes = 64 * 1024; +let mcpStderr = Buffer.alloc(0); +transport.stderr?.on("data", (chunk) => { + mcpStderr = Buffer.concat([mcpStderr, chunk]); + if (mcpStderr.length > maxMcpStderrBytes) + mcpStderr = mcpStderr.subarray(-maxMcpStderrBytes); +}); +const checks = []; +const retainedEvidence = []; +const originals = new Map(); +let directory; +let captureRoot = join(root, "captures"); +let primaryError; +let verifier; +let producerAudit; +let cleanupFailures = []; +let validateOutput; + +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const readJson = async (path) => JSON.parse(await readFile(path, "utf8")); +const readLines = async (path) => + (await readFile(path, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + +async function cli(input, name) { + const command = await runOwnedCommand( + { + command: process.execPath, + arguments: [ + entrypoint, + "inspect-web-network-capture", + input.capture_path, + input.format, + "--json", + ], + cwd, + hostEnvironment: environment, + runId: run.run_id, + }, + { + timeoutMs: 40_000, + diagnosticBytes: 8 * 1024 * 1024, + }, + ); + assert.notEqual(command.diagnosticTruncated, true); + await writeFile( + join(directory, `${name}.cli.stdout.json`), + command.stdout.text, + ); + await writeFile( + join(directory, `${name}.cli.stderr.log`), + command.stderr.text, + ); + const value = JSON.parse(command.stdout.text); + return contract.outputSchema.parse(value); +} + +async function mcp(input, name) { + const reply = await client.callTool({ + name: contract.name, + arguments: input, + }); + await writeFile( + join(directory, `${name}.mcp.reply.json`), + JSON.stringify(reply, null, 2), + ); + const value = JSON.parse(mcpTextValue(reply)); + assert.notEqual(reply.isError, true, mcpTextValue(reply)); + assert.ok( + validateOutput(reply.structuredContent), + JSON.stringify(validateOutput.errors), + ); + const evidence = contract.outputSchema.parse(value); + assert.deepEqual(reply.structuredContent, value); + retainedEvidence.push(evidence); + return evidence; +} + +async function inspect(name) { + const input = { + capture_path: join(captureRoot, `${name}.mitm`), + format: "mitmproxy", + }; + const cliResult = await cli(input, name); + const mcpResult = await mcp(input, name); + assert.deepEqual(cliResult.normalized_result, mcpResult.normalized_result); + for (const evidence of [cliResult, mcpResult]) { + assert.equal(evidence.authority, "historical-reference"); + assert.equal(evidence.confidence, "observed"); + assert.equal(evidence.provider.version, "12.2.3"); + assert.equal(evidence.normalized_result.runtime_attribution, "unknown"); + assert.deepEqual(evidence.provider, evidence.normalized_result.decoder); + assert.equal( + evidence.subject.digest.sha256, + digest(await readFile(input.capture_path)), + ); + } + checks.push({ + case: name, + outcome: "passed", + records: mcpResult.normalized_result.records.length, + }); + return mcpResult.normalized_result; +} + +try { + ({ directory, captureRoot } = await prepareAttempt( + root, + inspectExisting, + run, + )); + if (!inspectExisting) { + const produced = await runOwnedCommand( + { + command: python, + arguments: [ + fileURLToPath(new URL("./fixtures/http3-lab.py", import.meta.url)), + "--mitmdump", + mitmdump, + "--output", + captureRoot, + ], + cwd, + hostEnvironment: environment, + runId: run.run_id, + }, + { timeoutMs: 120_000, diagnosticBytes: 1024 * 1024 }, + ); + await writeFile( + join(directory, "producer.stdout.log"), + produced.stdout.text, + ); + await writeFile( + join(directory, "producer.stderr.log"), + produced.stderr.text, + ); + } + const manifest = await readJson(join(captureRoot, "manifest.json")); + assert.equal(manifest.aioquic, "1.2.0"); + assert.equal(manifest.profile, "http3-minimal-v1"); + await client.connect(transport); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === contract.name, + ); + assert.ok(advertised?.outputSchema, "Missing named advertised schema"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + assert.ok(ajv.validateSchema(advertised.inputSchema)); + assert.ok(ajv.validateSchema(advertised.outputSchema)); + validateOutput = ajv.compile(advertised.outputSchema); + const inputValidator = ajv.compile(advertised.inputSchema); + assert.ok( + inputValidator({ + capture_path: join(captureRoot, "normal.mitm"), + format: "mitmproxy", + }), + ); + assert.equal( + inputValidator({ + capture_path: join(captureRoot, "normal.mitm"), + format: "mitmproxy", + stream_id: 0, + }), + false, + ); + await writeFile( + join(directory, "advertised-tool.json"), + JSON.stringify(advertised, null, 2), + ); + + for (const name of ["normal", "fallback"]) + originals.set(name, await readFile(join(captureRoot, `${name}.mitm`))); + const normal = await inspect("normal"); + assert.equal(normal.total_records, 4); + const finalProducer = ( + await readLines(join(captureRoot, "normal.producer.jsonl")) + ).filter((event) => ["response", "error"].includes(event.event)); + verifyNormalRecords(normal, manifest, finalProducer); + const orders = verifyConcurrency(normal, manifest); + assert.equal( + (await readJson(join(captureRoot, "curl.json"))).http_version, + "3", + ); + assert.deepEqual( + await readFile(join(captureRoot, "curl.body")), + bytes(byCase(normal, "curl"), "/response/content"), + ); + checks.push({ + case: "concurrency-bytes", + outcome: "passed", + parallel_streams: 3, + orders, + }); + const fallback = await inspect("fallback"); + assert.equal(fallback.total_records, 1); + const record = byCase(fallback, "fallback"); + assert.throws( + () => requireHttp3(record), + { code: "ERR_ASSERTION" }, + "The same positive HTTP/3 check must reject the fallback record", + ); + assert.equal(record.reported.client_conn.transport_protocol, "tcp"); + assert.equal(record.reported.server_conn.transport_protocol, "tcp"); + assert.equal(record.reported.request.http_version, "HTTP/2.0"); + assert.equal(record.reported.response.http_version, "HTTP/1.1"); + assert.equal(record.reported.server_conn.alpn, "http/1.1"); + assert.equal( + (await readJson(join(captureRoot, "fallback.json"))).http_version, + "2", + ); + checks.push({ + case: "fallback-detected", + outcome: "passed", + client_http: "2", + origin_http: "1.1", + }); + + const bundleReply = await client.callTool({ + name: "get_evidence_bundle", + arguments: {}, + }); + assert.notEqual(bundleReply.isError, true, mcpTextValue(bundleReply)); + const bundle = JSON.parse(mcpTextValue(bundleReply)).result; + const records = bundle.records; + assert.ok(Array.isArray(records), "Evidence bundle omitted its records"); + for (const evidence of retainedEvidence) { + const retained = records.find( + (record) => record.evidence_id === evidence.evidence_id, + ); + assert.deepEqual( + retained, + evidence, + `MCP must retain the complete Evidence for ${evidence.evidence_id}`, + ); + } + await writeFile( + join(directory, "session-bundle.reply.json"), + JSON.stringify(bundleReply, null, 2), + ); + checks.push({ + case: "mcp-session-retention", + outcome: "passed", + records: retainedEvidence.length, + }); +} catch (error) { + primaryError = error; +} finally { + cleanupFailures = await attemptAll([ + ["client-close", () => client.close()], + ["transport-close", () => transport.close()], + [ + "producer-groups", + async () => { + if (!inspectExisting) { + producerAudit = await finishProducerGroups(captureRoot, run.run_id); + assert.deepEqual(producerAudit.failures ?? [], []); + for (const [, observation] of producerAudit.observation) + assert.equal( + observation.state, + "empty", + JSON.stringify(observation), + ); + } + }, + ], + ...[...originals].map(([name, original]) => [ + `input-immutability-${name}`, + async () => { + assert.deepEqual( + await readFile(join(captureRoot, `${name}.mitm`)), + original, + ); + }, + ]), + [ + "mcp-stderr-diagnostics", + async () => { + if (directory) + await writeFile(join(directory, "mcp.stderr.log"), mcpStderr); + }, + ], + [ + "failed-command-diagnostics", + async () => { + if (directory && primaryError?.snapshot) { + await writeFile( + join(directory, "failed-command.stdout.log"), + primaryError.snapshot.stdout.text, + ); + await writeFile( + join(directory, "failed-command.stderr.log"), + primaryError.snapshot.stderr.text, + ); + } + }, + ], + [ + "final-lineage", + async () => { + verifier = await completeVerifierRun(run); + assert.equal(verifier.process_lineage.status, "verified"); + assert.deepEqual(verifier.process_lineage.descendants, []); + }, + ], + ]); +} +const report = { + run_id: run.run_id, + status: primaryError || cleanupFailures.length ? "failed" : "passed", + finished_at: new Date().toISOString(), + ...(primaryError + ? { + error: describeError(primaryError), + mcp_stderr_log: "mcp.stderr.log", + } + : {}), + cleanup_failures: cleanupFailures, + checks, + verifier, + producer_audit: producerAudit, + scope: + "Real Linux reverse HTTP/3, mitmproxy 12.2.3, aioquic 1.2.0 and guest cURL", + limitations: [ + "No local/WireGuard, browser, QUIC v2, migration, 0-RTT, mTLS or replay claim.", + "No HAR, interruption, certificate-failure or body-completeness coverage in this profile.", + ], +}; +if (directory) { + try { + await publishAttempt(root, directory, report); + } catch (error) { + console.error("Could not publish this attempt's result:", error); + primaryError ??= error; + } +} +if (primaryError) throw primaryError; +if (cleanupFailures.length) + throw new AggregateError(cleanupFailures, "HTTP/3 verifier cleanup failed"); +console.log(JSON.stringify(report, null, 2)); From 8fef2aa64ec1ee742e8b70b825938d30a0ea7319 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:52:57 +0800 Subject: [PATCH 18/18] fix(ci): enforce merge verification and unblock superseded runs (#2938) * fix(ci): repair inherited formatting failures * fix(ci): release superseded PR concurrency slots * docs: require current-base checks before maintainer merges * docs: preserve administrator merge override --- .github/workflows/ci.yml | 7 +-- .github/workflows/pr-title.yml | 2 +- CONTRIBUTING.md | 5 ++ docs/readme/README_tr.md | 46 +++++++++---------- scripts/lib/clean-linux-hopper-runtime.mjs | 4 +- .../filesystem/ciRequiredLanes.test.ts | 4 +- 6 files changed, 39 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11d5dc30b..69c757b3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ env: jobs: changes: name: Classify changes - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: contents: read @@ -777,7 +777,8 @@ jobs: required: name: CI required - if: ${{ always() }} + # Report failed dependencies, but release superseded PR concurrency slots. + if: ${{ !cancelled() }} needs: [ changes, @@ -804,7 +805,7 @@ jobs: nativeaot, website, ] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index abe98bb48..bb2715b3e 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -14,7 +14,7 @@ concurrency: jobs: pr-title: name: Conventional PR title - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 2 steps: - name: Validate title for Release Please diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28e8788bd..96d8010ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -181,6 +181,11 @@ The separate GitHub Pages host uses only the manual ## Development feedback and PR verification +For routine merges, wait for the required checks to pass for the current PR +revision against the current `main`. Branch protection requires up-to-date +branches; administrators retain an override for deliberate maintainer recovery. +If `main` is broken, prioritize a focused repair before updating affected PRs. + Select checks for the behavior changed. For a source edit: ```bash diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index 8463c300d..152c81819 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -132,30 +132,30 @@ Ajanınız, hedefi incelemek ve ilgili kodu izlemek için MCP üzerinden REA'yı REA; Node.js 22.x (>=22.19), 24.x (>=24.11) veya 26+ ve npm gerektirir. Ek araçlar ve desteklenen ana bilgisayar sistemleri hedefe bağlıdır: -| Hedef | REA'nın döndürdükleri | Gereksinimler ve rehber | -| --------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| Yerel makine kodu içeren ikili dosyalar | Sözde kod, assembly kodu, dizeler, semboller, çağrılar ve referanslar | Hopper, Ghidra veya IDA; [yerel kod analizi](https://rea.tools/tr/guides/native/) | -| Çevrimdışı ELF yapısı | Bölümler, segmentler, özgün semboller/yeniden konumlandırmalar ve statik analizden olası güvenlik önlemleri | Linux x64 üzerinde çağıranın sağladığı pwntools; [ikili dosya tanılama](../binary-diagnostics.md) | -| Go ikili dosyaları | Gömülü compiler sürümü, modüller, replacement'lar ve build ayarları | Yerleşik statik ELF/PE/thin Mach-O incelemesi; Go kurulumu gerekmez | -| WASM artefaktı | Doğrulama, bölümler, tam import/export WAT biçimleri ve decode edilmiş WAT | Çağıranın sağladığı WABT 1.0.42; `inspect_wasm_artifact` | -| EVM bayt kodu | Yönlendirme seçicileri, bayt ofsetleri, çıkarımla belirlenen argümanlar ve durum değiştirilebilirliği | Linux x64 üzerinde yerel ham bayt/onaltılık girdi; [çevrimdışı EVM rehberi](../evm-bytecode.md) | -| SQLite veritabanları | Tablolar, sütunlar, index'ler, view'lar, trigger'lar ve seçilen sıradan tablo satırları | Yerel database/WAL snapshot'ı; native SQLite sınırlarıyla Node.js 24.x >=24.15 veya 26+ | -| Kaydedilmiş Linux çökmeleri | Ham note kayıtları, kaydedilen her iş parçacığının yazmaçları/sinyalleri ve isteğe bağlı eşleme adayları | Linux x64 üzerinde çağıranın sağladığı pwntools; isteğe bağlı GDB/pwndbg; [kaydedilmiş çökmeler](../recorded-crashes.md) | -| JavaScript / Electron | Modüller, içe aktarmalar, kaynak haritaları, rotalar, IPC ve yerel eklenti ilişkileri | Node.js ve npm; [uygulama analizi](https://rea.tools/tr/guides/javascript/) | -| Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](https://rea.tools/tr/guides/browser/) | -| Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR veya Fiddler Everywhere SAZ; Linux'ta native mitmproxy için mitmdump; [ağ yakalama rehberi](../web-network-captures.md) | -| .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](../managed-code-analysis.md) | -| Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS/Windows x64 üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](../android-analysis.md) | -| Android cihazları | Cihazlar, paketler, süreçler, günlükler, dökümler, ekran görüntüleri ve APK/dosya aktarımları | Çağıranın sağladığı adb (emülatör veya cihaz); [ADB rehberi](../adb-device-analysis.md) | -| JEB projeleri | Motor destekli proje birimleri, artefakt özetleri ve tür/metot sözdekodu | Çağıranın başlattığı, MCP sunan bir JEB istemcisi, REA_JEB_MCP_URL ile seçilir; [JEB rehberi](../jeb-analysis.md) | -| Android kaynakları | Çözülmüş manifest, dize tabloları, diller ve sürüm bilgileri | Çağıranın sağladığı Apktool başlatıcısı; [kaynak rehberi](../apktool-resource-analysis.md) | -| Flutter uygulamaları | Dart anlık görüntü karması, ABI başına kitaplık özetleri ve motor derleme kökeni | Yok (statik APK çözümleme); [Flutter rehberi](../flutter-analysis.md) | -| Donanım yazılımı | Bölgeler, çıkarma sonuçları ve yerel kod analizine devirler | Linux üzerinde Binwalk / Unblob; [donanım yazılımı rehberi](../firmware-analysis.md) | -| Kaydedilmiş paketler | Seçilen alanlar, yinelenen değerler, filtreler ve çözümleme kuralları | Linux: TShark 4.x + prlimit; [yakalama kılavuzu](../packet-captures.md) | -| Paket özetleri | Uç noktalar, DNS, TLS SNI, protokol dağılımı ve HTTP istekleri | Çağıranın sağladığı tshark ve capinfos (Linux üzerinde doğrulandı); [pcap rehberi](../pcap-analysis.md) | -| Kaydedilmiş mantık sinyalleri | Kod çözücü açıklamaları, örnek aralıkları ve tanılama | Linux: sigrok-cli 0.7.2 + prlimit; [sinyal kılavuzu](../signal-recordings.md) | +| Hedef | REA'nın döndürdükleri | Gereksinimler ve rehber | +| --------------------------------------- | ----------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Yerel makine kodu içeren ikili dosyalar | Sözde kod, assembly kodu, dizeler, semboller, çağrılar ve referanslar | Hopper, Ghidra veya IDA; [yerel kod analizi](https://rea.tools/tr/guides/native/) | +| Çevrimdışı ELF yapısı | Bölümler, segmentler, özgün semboller/yeniden konumlandırmalar ve statik analizden olası güvenlik önlemleri | Linux x64 üzerinde çağıranın sağladığı pwntools; [ikili dosya tanılama](../binary-diagnostics.md) | +| Go ikili dosyaları | Gömülü compiler sürümü, modüller, replacement'lar ve build ayarları | Yerleşik statik ELF/PE/thin Mach-O incelemesi; Go kurulumu gerekmez | +| WASM artefaktı | Doğrulama, bölümler, tam import/export WAT biçimleri ve decode edilmiş WAT | Çağıranın sağladığı WABT 1.0.42; `inspect_wasm_artifact` | +| EVM bayt kodu | Yönlendirme seçicileri, bayt ofsetleri, çıkarımla belirlenen argümanlar ve durum değiştirilebilirliği | Linux x64 üzerinde yerel ham bayt/onaltılık girdi; [çevrimdışı EVM rehberi](../evm-bytecode.md) | +| SQLite veritabanları | Tablolar, sütunlar, index'ler, view'lar, trigger'lar ve seçilen sıradan tablo satırları | Yerel database/WAL snapshot'ı; native SQLite sınırlarıyla Node.js 24.x >=24.15 veya 26+ | +| Kaydedilmiş Linux çökmeleri | Ham note kayıtları, kaydedilen her iş parçacığının yazmaçları/sinyalleri ve isteğe bağlı eşleme adayları | Linux x64 üzerinde çağıranın sağladığı pwntools; isteğe bağlı GDB/pwndbg; [kaydedilmiş çökmeler](../recorded-crashes.md) | +| JavaScript / Electron | Modüller, içe aktarmalar, kaynak haritaları, rotalar, IPC ve yerel eklenti ilişkileri | Node.js ve npm; [uygulama analizi](https://rea.tools/tr/guides/javascript/) | +| Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](https://rea.tools/tr/guides/browser/) | +| Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR veya Fiddler Everywhere SAZ; Linux'ta native mitmproxy için mitmdump; [ağ yakalama rehberi](../web-network-captures.md) | +| .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](../managed-code-analysis.md) | +| Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS/Windows x64 üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](../android-analysis.md) | +| Android cihazları | Cihazlar, paketler, süreçler, günlükler, dökümler, ekran görüntüleri ve APK/dosya aktarımları | Çağıranın sağladığı adb (emülatör veya cihaz); [ADB rehberi](../adb-device-analysis.md) | +| JEB projeleri | Motor destekli proje birimleri, artefakt özetleri ve tür/metot sözdekodu | Çağıranın başlattığı, MCP sunan bir JEB istemcisi, REA_JEB_MCP_URL ile seçilir; [JEB rehberi](../jeb-analysis.md) | +| Android kaynakları | Çözülmüş manifest, dize tabloları, diller ve sürüm bilgileri | Çağıranın sağladığı Apktool başlatıcısı; [kaynak rehberi](../apktool-resource-analysis.md) | +| Flutter uygulamaları | Dart anlık görüntü karması, ABI başına kitaplık özetleri ve motor derleme kökeni | Yok (statik APK çözümleme); [Flutter rehberi](../flutter-analysis.md) | +| Donanım yazılımı | Bölgeler, çıkarma sonuçları ve yerel kod analizine devirler | Linux üzerinde Binwalk / Unblob; [donanım yazılımı rehberi](../firmware-analysis.md) | +| Kaydedilmiş paketler | Seçilen alanlar, yinelenen değerler, filtreler ve çözümleme kuralları | Linux: TShark 4.x + prlimit; [yakalama kılavuzu](../packet-captures.md) | +| Paket özetleri | Uç noktalar, DNS, TLS SNI, protokol dağılımı ve HTTP istekleri | Çağıranın sağladığı tshark ve capinfos (Linux üzerinde doğrulandı); [pcap rehberi](../pcap-analysis.md) | +| Kaydedilmiş mantık sinyalleri | Kod çözücü açıklamaları, örnek aralıkları ve tanılama | Linux: sigrok-cli 0.7.2 + prlimit; [sinyal kılavuzu](../signal-recordings.md) | | Paketler ve kaynaklar | Dosya envanterleri, özet değerleri, plist'ler, Apple paket yapısı ve çıkarılmış kaynaklar | [Artefakt ve JavaScript rehberi](../javascript-artifact-reconstruction.md), [Apple uygulamaları](../apple-application-analysis.md) | -| Süreç davranışı | Terminal çıktısı, etkileşimler, sonlanma ve dosya sistemi gözlemleri, çalıştırma karşılaştırmaları | Yerel PTY ile Linux/macOS; [süreç yakalama](../process-capture.md) | +| Süreç davranışı | Terminal çıktısı, etkileşimler, sonlanma ve dosya sistemi gözlemleri, çalıştırma karşılaştırmaları | Yerel PTY ile Linux/macOS; [süreç yakalama](../process-capture.md) | Statik JavaScript ve .NET incelemesi, uygulamayı çalıştırmadan sağlanan dosyaları okur. Çalışma zamanı yakalaması, seçilen hedefi kullanıcı izinlerinizle çalıştırır veya onunla etkileşime girer; her çalışma zamanı rehberi etkilerini açıklar. diff --git a/scripts/lib/clean-linux-hopper-runtime.mjs b/scripts/lib/clean-linux-hopper-runtime.mjs index fbe15e0d3..4d6bc8163 100644 --- a/scripts/lib/clean-linux-hopper-runtime.mjs +++ b/scripts/lib/clean-linux-hopper-runtime.mjs @@ -37,7 +37,9 @@ export const assertCleanLinuxHopperRuntime = async ( if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue; let executable; try { - executable = await stat(`${procRoot}/${entry.name}/exe`, { bigint: true }); + executable = await stat(`${procRoot}/${entry.name}/exe`, { + bigint: true, + }); } catch (error) { if (error?.code === "ENOENT" || error?.code === "ESRCH") continue; // Other users' executables may be unreadable. The setup lane launches diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index da6c43186..4c23844e7 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -365,7 +365,9 @@ it("keeps fast feedback separate from complete qualification and release proof", expect(ci.concurrency["cancel-in-progress"]).toBe( "${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}", ); - expect(ci.jobs.required?.if).toBe("${{ always() }}"); + // Failed dependencies still reach the gate, but superseded PR runs must + // release their concurrency slot without waiting for another runner. + expect(ci.jobs.required?.if).toBe("${{ !cancelled() }}"); const shards = ci.jobs["test-shard"]?.steps ?? []; const fast = shards.find((step) => step.run?.includes("npm run test:run")); const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));