From e1cd292aa6e8f5f13441d408c66126782c8dbb0d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:38:39 +0800 Subject: [PATCH 01/27] fix(adb): admit canonical device states and preserve malformed output --- src/adb/AdbDeviceOutput.test.ts | 22 ++++++++++++-- src/adb/AdbDeviceOutput.ts | 39 +++++++++--------------- src/adb/AdbProvider.test.ts | 46 +++++++++++++++++++++++++++++ src/domain/adb/adbDeviceAnalysis.ts | 21 ++++++++++++- 4 files changed, 100 insertions(+), 28 deletions(-) diff --git a/src/adb/AdbDeviceOutput.test.ts b/src/adb/AdbDeviceOutput.test.ts index fe11c2a26..12b82ab84 100644 --- a/src/adb/AdbDeviceOutput.test.ts +++ b/src/adb/AdbDeviceOutput.test.ts @@ -84,6 +84,24 @@ describe("parseAdbDevicesOutput", () => { ]); }); + it("accepts complete multiword states and rejects state prefixes", () => { + const output = [ + "serial-no-permission no permissions usb:1-1", + "serial-bad-device deviceXYZ ", + "serial-bad-offline offline-ish", + "serial-bad-auth unauthorizedXYZ", + ].join("\n"); + const { devices, unparsedLines } = parseAdbDevicesOutput(output); + expect(devices.map(({ serial, state }) => [serial, state])).toEqual([ + ["serial-no-permission", "no permissions"], + ]); + expect(unparsedLines).toEqual([ + "serial-bad-device deviceXYZ ", + "serial-bad-offline offline-ish", + "serial-bad-auth unauthorizedXYZ", + ]); + }); + it("returns unrecognized lines instead of dropping them", () => { const output = [ "List of devices attached", @@ -194,10 +212,10 @@ describe("parsePackageListOutput", () => { it("keeps unrecognized lines for partial coverage reporting", () => { const { packages, unparsedLines } = parsePackageListOutput( - "package:/data/app/x/base.apk=com.ok\ngarbage line\n", + "package:/data/app/x/base.apk=com.ok\n garbage line \n", ); expect(packages).toHaveLength(1); - expect(unparsedLines).toEqual(["garbage line"]); + expect(unparsedLines).toEqual([" garbage line "]); }); }); diff --git a/src/adb/AdbDeviceOutput.ts b/src/adb/AdbDeviceOutput.ts index 3380de608..8d3c74f5f 100644 --- a/src/adb/AdbDeviceOutput.ts +++ b/src/adb/AdbDeviceOutput.ts @@ -10,11 +10,15 @@ import { adbHostFileName, type AdbHostFileNameReason, } from "./AdbHostFileName.js"; +import { + ADB_DEVICE_STATES, + type AdbDeviceState, +} from "../domain/adb/adbDeviceAnalysis.js"; /** One device exactly as `adb devices -l` reports it. */ export interface AdbDeviceEntry { readonly serial: string; - readonly state: string; + readonly state: AdbDeviceState; readonly transport: "usb" | "tcp" | "unknown"; readonly product: string | null; readonly model: string | null; @@ -24,25 +28,6 @@ export interface AdbDeviceEntry { readonly kind_basis: "emulator_serial_prefix" | "usb_transport" | "none"; } -/** - * Device states adb prints after the serial, longest first so two-word - * states match before their single-word prefixes. A line whose remainder - * starts with none of them is not a device line. - */ -const DEVICE_STATES = [ - "no permissions", - "unauthorized", - "authorizing", - "offline", - "recovery", - "sideload", - "connecting", - "bootloader", - "device", - "unknown", - "host", -] as const; - const DEVICE_LINE = /^(\S+)\s+(.*)$/u; const metadataValue = (remainder: string, key: string): string | null => { @@ -73,11 +58,15 @@ export const parseAdbDevicesOutput = ( continue; } const match = DEVICE_LINE.exec(line); - const state = DEVICE_STATES.find((candidate) => - match?.[2]?.startsWith(candidate), - ); + const state = ADB_DEVICE_STATES.find((candidate) => { + const remainder = match?.[2]; + if (remainder === undefined || !remainder.startsWith(candidate)) + return false; + const next = remainder[candidate.length]; + return next === undefined || /\s/u.test(next); + }); if (match === null || state === undefined) { - if (line.trim() !== "") unparsedLines.push(line); + if (rawLine.trim() !== "") unparsedLines.push(rawLine); continue; } const serial = match[1]!; @@ -192,7 +181,7 @@ export const parsePackageListOutput = ( if (line === "") continue; const match = PACKAGE_LINE.exec(line); if (match === null) { - unparsedLines.push(line); + unparsedLines.push(rawLine); continue; } packages.push({ diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index d1a153848..b89ab9dfd 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -121,6 +121,52 @@ fi`, ]); }); +it("refuses device listings with an invalid state token", async () => { + const malformedLine = "serial deviceXYZ "; + const { binary } = await writeAdbStub( + () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "devices" ]; then + printf 'List of devices attached\\n${malformedLine}\\n' + exit 0 +fi`, + ); + const result = await providerOver(binary).execute({ + operation: "list_adb_devices", + input: {}, + }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error._tag).toBe("AnalysisProtocolError"); + expect(result.error.message).toContain(malformedLine); +}); + +it("lists rescue and detached transports alongside online devices", async () => { + const { binary } = await writeAdbStub( + () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "devices" ]; then + printf 'List of devices attached\\n' + printf 'serial-online device transport_id:1\\n' + printf 'serial-rescue rescue transport_id:2\\n' + printf 'serial-detached detached transport_id:3\\n' + exit 0 +fi`, + ); + const result = await evidenceResult(providerOver(binary), { + operation: "list_adb_devices", + input: {}, + }); + expect(result.devices).toEqual( + expect.arrayContaining([ + expect.objectContaining({ serial: "serial-online", state: "device" }), + expect.objectContaining({ serial: "serial-rescue", state: "rescue" }), + expect.objectContaining({ + serial: "serial-detached", + state: "detached", + }), + ]), + ); +}); + it("inspects a device through the getprop whitelist", async () => { const { binary } = await writeAdbStub( () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi diff --git a/src/domain/adb/adbDeviceAnalysis.ts b/src/domain/adb/adbDeviceAnalysis.ts index 17cdc223e..c524ebd02 100644 --- a/src/domain/adb/adbDeviceAnalysis.ts +++ b/src/domain/adb/adbDeviceAnalysis.ts @@ -30,6 +30,25 @@ const packageNameSchema = z "Package name must be dot-separated Java identifiers", ); +/** Text states emitted by adb's ConnectionState formatter. */ +export const ADB_DEVICE_STATES = [ + "no permissions", + "unauthorized", + "authorizing", + "offline", + "recovery", + "sideload", + "connecting", + "bootloader", + "device", + "unknown", + "host", + "rescue", + "detached", +] as const; +export const adbDeviceStateSchema = z.enum(ADB_DEVICE_STATES); +export type AdbDeviceState = (typeof ADB_DEVICE_STATES)[number]; + const keyStringSchema = z.string().min(1).max(256); const keyPattern = /^[A-Za-z0-9._\x2d]+$/u; @@ -412,7 +431,7 @@ const client = z.strictObject({ const device = z.strictObject({ serial: z.string().min(1), - state: z.string().min(1), + state: adbDeviceStateSchema, transport: z.enum(["usb", "tcp", "unknown"]), product: z.string().nullable(), model: z.string().nullable(), From 1b953021eadf4cac09e20be388d79536903633b3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:39:06 +0800 Subject: [PATCH 02/27] fix(javascript): centralize module syntax and separate erased type dependencies --- docs/javascript-application-workflows.md | 8 + .../ReferenceSourceImportEntries.ts | 6 +- src/application/ReferenceSourceImportTypes.ts | 3 - .../JavaScriptArtifactPathResolution.ts | 22 +- .../JavaScriptModuleRelationships.ts | 14 +- .../javascript/javascriptModuleDeclaration.ts | 60 +++++ .../javascript/javascriptModuleSpecifier.ts | 6 + ...riptSemanticAnalysis.require_links.test.ts | 62 +++++ .../javascript/javascriptSemanticAnalysis.ts | 6 + .../javascriptSemanticProjection.ts | 47 +++- .../javascript/javascriptSemanticReturns.ts | 1 + .../javascript/javascriptStaticAnalysis.ts | 20 +- src/domain/referenceSourceImportParsing.ts | 14 +- src/domain/webModuleImports.test.ts | 11 +- src/domain/webModuleImports.ts | 37 ++- .../JavaScriptModuleLookupCli.test.ts | 238 +++++++++++------- .../javascriptModuleRelationships.test.ts | 54 ++++ .../filesystem/referenceSourceImport.test.ts | 38 +++ 18 files changed, 513 insertions(+), 134 deletions(-) create mode 100644 src/domain/javascript/javascriptModuleDeclaration.ts create mode 100644 src/domain/javascript/javascriptModuleSpecifier.ts diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index 2d2d9e64a..1e9e7fe19 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -23,6 +23,14 @@ artifact digest and structural graph ID. Semantic tracing requires the semantic relation graph and reports when it is unavailable rather than treating missing data as an empty graph. +Explicit TypeScript `import type` and `export type` declarations remain source +facts and do not create runtime import or export relationships. Type export names +are reported separately from value export names. Inline `type` specifiers can +leave an empty import or re-export with side effects under +[`verbatimModuleSyntax`](https://www.typescriptlang.org/tsconfig/verbatimModuleSyntax.html); +REA retains those dependency candidates and reports the unavailable compiler +options rather than assuming the dependency is erased. + The semantic graph wire format stores shared non-location provenance once in its required `evidence_contexts` table. Each node, relation, fingerprint, and unknown retains its exact `evidence.location` and names the owning context with diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 3254d7cfb..a6e962118 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -12,12 +12,14 @@ import { detectReferenceSourceLanguage, } from "../domain/referenceSourceClassification.js"; import type { HistoricalSourceGraphInput } from "../domain/referenceSourceGraph.js"; -import { parseReferenceSourceImports } from "../domain/referenceSourceImportParsing.js"; +import { + parseReferenceSourceImports, + PARSEABLE_REFERENCE_SOURCE_LANGUAGES, +} from "../domain/referenceSourceImportParsing.js"; import type { ReferenceSourceEntry, ReferenceSourceRead, } from "../reference/ReferenceSourceReaderTypes.js"; -import { PARSEABLE_REFERENCE_SOURCE_LANGUAGES } from "./ReferenceSourceImportTypes.js"; import { typescriptEmittedSourceCandidates } from "./TypeScriptEmittedSourceCandidates.js"; export interface ParsedReferenceSourceEntries { diff --git a/src/application/ReferenceSourceImportTypes.ts b/src/application/ReferenceSourceImportTypes.ts index 9395dc5c0..f8474315a 100644 --- a/src/application/ReferenceSourceImportTypes.ts +++ b/src/application/ReferenceSourceImportTypes.ts @@ -87,6 +87,3 @@ export const DEFAULT_REFERENCE_SOURCE_IGNORE_PATTERNS = [ ".coverage", "*.log", ] as const; - -export const PARSEABLE_REFERENCE_SOURCE_LANGUAGES: ReadonlySet = - new Set(["JavaScript", "TypeScript", "JSX", "TSX"]); diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.ts index b3d769af6..3ea23ab11 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts @@ -20,6 +20,7 @@ import { lookupExactPackageImport, } from "../../domain/javascript/javascriptPackageImportAliases.js"; import { typescriptEmittedSourceCandidates } from "../TypeScriptEmittedSourceCandidates.js"; +import { isRelativeJavaScriptModuleSpecifier } from "../../domain/javascript/javascriptModuleSpecifier.js"; type ArtifactPathResolutionContext = | "package-entrypoint" @@ -166,7 +167,7 @@ const resolvePath = (input: ResolveArtifactPathInput): CandidateResolution => { ); const isModuleUrlPath = input.context === "module-specifier" && - (input.declaredPath.startsWith(".") || + (isRelativeJavaScriptModuleSpecifier(input.declaredPath) || input.declaredPath.startsWith("/") || fileUrlPath(input.declaredPath) !== undefined); const emittedSourceCandidates = @@ -229,7 +230,10 @@ const contextualCandidate = ( return unresolvedCandidate("external", [ "URL and Node builtin schemes are outside static artifact module resolution.", ]); - if (!declared.startsWith(".") && !declared.startsWith("/")) + if ( + !isRelativeJavaScriptModuleSpecifier(declared) && + !declared.startsWith("/") + ) return bareModuleCandidate(input); if (input.moduleKind !== "require") { const decoded = decodeModuleUrlPath(stripQueryAndFragment(declared)); @@ -275,7 +279,10 @@ const contextualCandidate = ( "URL", ); } - return relativeCandidate(input.sourcePath, declared); + const candidate = relativeCandidate(input.sourcePath, declared); + return context === "module-specifier" && input.moduleKind === "import" + ? exactUrlPathCandidate(declared, candidate, "module URL") + : candidate; }; const decodeModuleUrlPath = ( @@ -310,9 +317,12 @@ const bareModuleCandidate = ( return unresolvedCandidate("external", [ "The bare specifier is a Node builtin or invalid package name.", ]); - if (input.moduleKind === "import" && request.packageName.includes("%")) + if ( + input.moduleKind === "import" && + (request.packageName.startsWith(".") || request.packageName.includes("%")) + ) return unresolvedCandidate("rejected", [ - "Percent signs are not admitted in ES module package names.", + "ES module package names cannot start with a dot or contain percent signs.", ]); const self = packageSelfCandidate(input, request); if (self !== null) return self; @@ -735,7 +745,7 @@ const htmlCandidate = ( const exactUrlPathCandidate = ( declared: string, candidate: string, - kind: "HTML" | "URL", + kind: "HTML" | "URL" | "module URL", ): string | CandidateResolution => { if (!urlPathIsDirectory(declared)) return candidate; const confined = confineCandidate(candidate); diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.ts index 874377810..514a9bb7c 100644 --- a/src/application/javascript/JavaScriptModuleRelationships.ts +++ b/src/application/javascript/JavaScriptModuleRelationships.ts @@ -81,8 +81,13 @@ export const addJavaScriptSourceModules = ( if (asset === undefined || semantic === null || program === undefined) continue; const exports = semantic.ir.moduleLinks - .flatMap(({ exportedName }) => - exportedName === null ? [] : [exportedName], + .flatMap(({ exportedName, kind }) => + exportedName === null || kind === "type-import" ? [] : [exportedName], + ) + .sort(compareUnicodeCodePoints); + const typeExports = semantic.ir.moduleLinks + .flatMap(({ exportedName, kind }) => + exportedName === null || kind !== "type-import" ? [] : [exportedName], ) .sort(compareUnicodeCodePoints); const retainedExports = [...new Set(exports)]; @@ -105,8 +110,12 @@ export const addJavaScriptSourceModules = ( import_relationships: semantic.ir.moduleLinks.filter( ({ kind }) => kind === "import" || kind === "require", ).length, + type_relationships: semantic.ir.moduleLinks.filter( + ({ kind }) => kind === "type-import", + ).length, export_relationships: exports.length, export_names: retainedExports, + type_export_names: [...new Set(typeExports)], omitted_export_names: 0, }, evidence: astObservationEvidence({ @@ -152,6 +161,7 @@ export function* addJavaScriptModuleRelationshipsSteps( for (const link of semantic.ir.moduleLinks) { // One module can export thousands of bindings, each with return shapes. yield; + if (link.kind === "type-import") continue; const input = { context, file, semantic, source, link }; if (isExportLink(link)) addExportRelationship(input, omissions); else if (link.specifier !== null) diff --git a/src/domain/javascript/javascriptModuleDeclaration.ts b/src/domain/javascript/javascriptModuleDeclaration.ts new file mode 100644 index 000000000..d7a8d6754 --- /dev/null +++ b/src/domain/javascript/javascriptModuleDeclaration.ts @@ -0,0 +1,60 @@ +import * as t from "@babel/types"; + +/** Whether TypeScript explicitly erases this entire import declaration. */ +export const isTypeOnlyImportDeclaration = ( + node: t.ImportDeclaration, +): boolean => node.importKind === "type" || node.importKind === "typeof"; + +/** Whether TypeScript explicitly erases this entire export declaration. */ +export const isTypeOnlyExportDeclaration = ( + node: t.ExportNamedDeclaration | t.ExportAllDeclaration, +): boolean => + node.exportKind === "type" || + (t.isExportNamedDeclaration(node) && + node.declaration !== null && + (t.isTSInterfaceDeclaration(node.declaration) || + t.isTSTypeAliasDeclaration(node.declaration) || + t.isTSDeclareFunction(node.declaration))); + +/** Whether this import binding is explicitly type-only. */ +export const isTypeOnlyImportSpecifier = (node: t.ImportSpecifier): boolean => + node.importKind === "type" || node.importKind === "typeof"; + +/** Whether this exported binding is explicitly type-only. */ +export const isTypeOnlyExportSpecifier = (node: t.ExportSpecifier): boolean => + node.exportKind === "type"; + +/** A regular import whose bindings are all erased but may retain side effects. */ +export const hasOnlyInlineTypeImportSpecifiers = ( + node: t.ImportDeclaration, +): boolean => + !isTypeOnlyImportDeclaration(node) && + node.specifiers.length > 0 && + node.specifiers.every( + (specifier) => + t.isImportSpecifier(specifier) && isTypeOnlyImportSpecifier(specifier), + ); + +/** A regular re-export whose bindings are all erased but may retain side effects. */ +export const hasOnlyInlineTypeExportSpecifiers = ( + node: t.ExportNamedDeclaration, +): boolean => + !isTypeOnlyExportDeclaration(node) && + node.source != null && + node.specifiers.length > 0 && + node.specifiers.every( + (specifier) => + t.isExportSpecifier(specifier) && isTypeOnlyExportSpecifier(specifier), + ); + +/** Whether compiler options leave an inline type-only declaration's side effect unknown. */ +export const hasUncertainTypeOnlyModuleSideEffect = ( + program: t.Program, +): boolean => + program.body.some( + (node) => + (t.isImportDeclaration(node) && + hasOnlyInlineTypeImportSpecifiers(node)) || + (t.isExportNamedDeclaration(node) && + hasOnlyInlineTypeExportSpecifiers(node)), + ); diff --git a/src/domain/javascript/javascriptModuleSpecifier.ts b/src/domain/javascript/javascriptModuleSpecifier.ts new file mode 100644 index 000000000..f52f081dc --- /dev/null +++ b/src/domain/javascript/javascriptModuleSpecifier.ts @@ -0,0 +1,6 @@ +/** Relative module syntax differs from an arbitrary dot-prefixed filename. */ +export const isRelativeJavaScriptModuleSpecifier = (value: string): boolean => + value === "." || + value === ".." || + value.startsWith("./") || + value.startsWith("../"); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.require_links.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.require_links.test.ts index 333ad4f9c..a0d93c3c3 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.require_links.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.require_links.test.ts @@ -22,6 +22,11 @@ describe("TypeScript import-equals links", () => { localName: "dependency", exportedName: null, }), + expect.objectContaining({ + kind: "type-import", + specifier: "./types", + localName: "types", + }), ]); }); @@ -51,6 +56,57 @@ describe("TypeScript import-equals links", () => { }); }); +describe("TypeScript type-only module links", () => { + it("retains declarations without runtime bindings or callable claims", () => { + const ir = analyzeJavaScriptSemantics(` + import type { Whole } from "./whole"; + import { type Inline } from "./inline"; + import { type Erased, value } from "./mixed"; + export type { Reexported } from "./reexported"; + export { type InlineExport } from "./inline-export"; + export type LocalType = string; + export interface LocalInterface { value: string } + function hiddenFunction() { return 1; } + export type { hiddenFunction }; + `); + + const externalLinks = ir.moduleLinks + .filter(({ specifier }) => specifier !== null) + .map( + ({ kind, specifier, importedName, exportedName }) => + [kind, specifier, importedName, exportedName] as const, + ); + expect(externalLinks).toEqual([ + ["type-import", "./whole", "Whole", null], + ["type-import", "./inline", "Inline", null], + ["type-import", "./mixed", "Erased", null], + ["import", "./mixed", "value", null], + ["type-import", "./reexported", "Reexported", "Reexported"], + ["type-import", "./inline-export", "InlineExport", "InlineExport"], + ]); + + const typeLinks = ir.moduleLinks + .filter(({ kind }) => kind === "type-import") + .map( + ({ specifier, localName, exportedName, callableId }) => + [specifier, localName, exportedName, callableId] as const, + ); + expect(typeLinks).toEqual([ + ["./whole", "Whole", null, null], + ["./inline", "Inline", null, null], + ["./mixed", "Erased", null, null], + ["./reexported", null, "Reexported", null], + ["./inline-export", null, "InlineExport", null], + [null, "LocalType", "LocalType", null], + [null, "LocalInterface", "LocalInterface", null], + [null, "hiddenFunction", "hiddenFunction", null], + ]); + expect(ir.limitations).toContain( + "Compiler options are unavailable; inline type-only imports or re-exports may preserve a module side-effect dependency.", + ); + }); +}); + describe("CommonJS dependency links", () => { it("retains an import-equals binding and its module origin", () => { const ir = analyzeJavaScriptSemantics(` @@ -76,6 +132,12 @@ describe("CommonJS dependency links", () => { importedName: null, localName: "typed", }), + expect.objectContaining({ + kind: "type-import", + specifier: "type-only", + importedName: null, + localName: "TypeOnly", + }), ]); expect(ir.references.filter(({ name }) => name === "typed")).toHaveLength( 1, diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index b56e1a112..35a8b7dc9 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -43,6 +43,7 @@ import { import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js"; import { indexJavaScriptAmbientWritesSteps } from "./javascriptSemanticAmbientWrites.js"; +import { hasUncertainTypeOnlyModuleSideEffect } from "./javascriptModuleDeclaration.js"; import { collectSemanticReturns, resolveSemanticModuleCallables, @@ -200,6 +201,11 @@ export function* analyzeParsedJavaScriptSemanticsSteps( "The parser recovered from syntax errors; affected bindings are partial.", ] : []), + ...(hasUncertainTypeOnlyModuleSideEffect(file.program) + ? [ + "Compiler options are unavailable; inline type-only imports or re-exports may preserve a module side-effect dependency.", + ] + : []), "Values and aliases were recovered from inert syntax only; no JavaScript was executed.", "Return sites include only direct callable returns; nested callable returns remain separate.", "Local call, argument, return, and closure relations are static candidates and do not prove runtime invocation.", diff --git a/src/domain/javascript/javascriptSemanticProjection.ts b/src/domain/javascript/javascriptSemanticProjection.ts index 7e4663efe..9d252a0d8 100644 --- a/src/domain/javascript/javascriptSemanticProjection.ts +++ b/src/domain/javascript/javascriptSemanticProjection.ts @@ -28,6 +28,12 @@ import { propertyName, semanticStaticPropertyKey, } from "./javascriptAstValues.js"; +import { + isTypeOnlyExportDeclaration, + isTypeOnlyExportSpecifier, + isTypeOnlyImportDeclaration, + isTypeOnlyImportSpecifier, +} from "./javascriptModuleDeclaration.js"; interface CollectCallableInput { readonly node: t.Node; @@ -75,7 +81,7 @@ export const collectSemanticModuleLink = ( addModuleLink( state, { - kind: "re-export", + kind: isTypeOnlyExportDeclaration(node) ? "type-import" : "re-export", specifier: node.source.value, importedName: "*", localName: null, @@ -108,12 +114,19 @@ const collectImportEquals = ( node: t.TSImportEqualsDeclaration, state: JavaScriptSemanticAnalysisState, ): void => { - const specifier = semanticImportEqualsSpecifier(node); + const moduleReference = node.moduleReference; + const specifier = + semanticImportEqualsSpecifier(node) ?? + (node.importKind === "type" && + t.isTSExternalModuleReference(moduleReference) && + t.isStringLiteral(moduleReference.expression) + ? moduleReference.expression.value + : null); if (specifier === null) return; addModuleLink( state, { - kind: "require", + kind: node.importKind === "type" ? "type-import" : "require", specifier, importedName: null, localName: node.id.name, @@ -287,7 +300,7 @@ const collectImports = ( addModuleLink( state, { - kind: "import", + kind: isTypeOnlyImportDeclaration(node) ? "type-import" : "import", specifier: node.source.value, importedName: null, localName: null, @@ -300,7 +313,12 @@ const collectImports = ( addModuleLink( state, { - kind: "import", + kind: + isTypeOnlyImportDeclaration(node) || + (t.isImportSpecifier(specifier) && + isTypeOnlyImportSpecifier(specifier)) + ? "type-import" + : "import", specifier: node.source.value, importedName: t.isImportDefaultSpecifier(specifier) ? "default" @@ -319,12 +337,13 @@ const collectNamedExports = ( node: t.ExportNamedDeclaration, state: JavaScriptSemanticAnalysisState, ): void => { + const declarationTypeOnly = isTypeOnlyExportDeclaration(node); if (node.declaration !== null && node.declaration !== undefined) for (const localName of declarationNames(node.declaration)) addModuleLink( state, { - kind: "export", + kind: declarationTypeOnly ? "type-import" : "export", specifier: null, importedName: null, localName, @@ -342,7 +361,14 @@ const collectNamedExports = ( addModuleLink( state, { - kind: node.source == null ? "export" : "re-export", + kind: + declarationTypeOnly || + (t.isExportSpecifier(specifier) && + isTypeOnlyExportSpecifier(specifier)) + ? "type-import" + : node.source == null + ? "export" + : "re-export", specifier: node.source?.value ?? null, importedName: t.isExportNamespaceSpecifier(specifier) ? "*" @@ -745,6 +771,13 @@ const declarationNames = (declaration: t.Declaration): string[] => { t.isIdentifier(declaration.id) ) return [declaration.id.name]; + if ( + (t.isTSInterfaceDeclaration(declaration) || + t.isTSTypeAliasDeclaration(declaration) || + t.isTSDeclareFunction(declaration)) && + t.isIdentifier(declaration.id) + ) + return [declaration.id.name]; if ( t.isTSImportEqualsDeclaration(declaration) && semanticImportEqualsSpecifier(declaration) !== null diff --git a/src/domain/javascript/javascriptSemanticReturns.ts b/src/domain/javascript/javascriptSemanticReturns.ts index 189a1385e..a3ca92ead 100644 --- a/src/domain/javascript/javascriptSemanticReturns.ts +++ b/src/domain/javascript/javascriptSemanticReturns.ts @@ -101,6 +101,7 @@ export const resolveSemanticModuleCallables = ( bindings.map(({ bindingId, provenance }) => [bindingId, provenance]), ); return state.moduleLinks.map((link) => { + if (link.kind === "type-import") return { ...link, callableId: null }; // Named exports follow their lexical binding, including every assignment. // A declaration's ID or a callable's display name cannot prove that link. if (link.localName === null) diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index 4d10638bc..c34462498 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -53,6 +53,11 @@ import { import { createJavaScriptRpcScan } from "./javascriptRpcEndpoints.js"; import { semanticImportEqualsSpecifier } from "./javascriptSemanticProjection.js"; +import { + hasUncertainTypeOnlyModuleSideEffect, + isTypeOnlyExportDeclaration, + isTypeOnlyImportDeclaration, +} from "./javascriptModuleDeclaration.js"; /** Parse one JavaScript artifact and recover static structure only. */ export const analyzeJavaScriptStaticSource = ( @@ -154,6 +159,11 @@ const finalizeStaticAnalysis = ( : [ "One or more static keys, expressions, or Electron boundary values were dynamic and remain unknown.", ]), + ...(hasUncertainTypeOnlyModuleSideEffect(file.program) + ? [ + "Compiler options are unavailable; inline type-only imports or re-exports may preserve a module side-effect dependency.", + ] + : []), ...(accumulator.references.some( ({ value }) => value.specifier !== null && ELECTRON_MODULE.test(value.specifier), @@ -246,14 +256,20 @@ const inspectNode = ( addReference(findings, { node, kind: "require", specifier }); } else if ( (t.isImportDeclaration(node) || t.isExportAllDeclaration(node)) && - node.source !== undefined + node.source !== undefined && + !(t.isImportDeclaration(node) && isTypeOnlyImportDeclaration(node)) && + !(t.isExportAllDeclaration(node) && isTypeOnlyExportDeclaration(node)) ) addReference(findings, { node, kind: "static-import", specifier: node.source.value, }); - else if (t.isExportNamedDeclaration(node) && node.source != null) + else if ( + t.isExportNamedDeclaration(node) && + node.source != null && + !isTypeOnlyExportDeclaration(node) + ) addReference(findings, { node, kind: "static-import", diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts index 998e368c4..18f76a358 100644 --- a/src/domain/referenceSourceImportParsing.ts +++ b/src/domain/referenceSourceImportParsing.ts @@ -11,6 +11,7 @@ import { parserPluginsForPath, } from "./javascript/javascriptSourceParser.js"; import { unshadowedReferenceSourceRequireCalls } from "./referenceSourceRequireBindings.js"; +import { isRelativeJavaScriptModuleSpecifier } from "./javascript/javascriptModuleSpecifier.js"; import { isCallExpression, isExportAllDeclaration, @@ -60,7 +61,8 @@ export interface ReferenceSourceImportParseResult { readonly limitations: readonly string[]; } -const codeSourceLanguages = new Set(["JavaScript", "TypeScript", "JSX", "TSX"]); +export const PARSEABLE_REFERENCE_SOURCE_LANGUAGES: ReadonlySet = + new Set(["JavaScript", "TypeScript", "JSX", "TSX"]); const safeModuleName = (node: Node | null | undefined): string | undefined => { if (isStringLiteral(node)) return node.value; @@ -70,9 +72,11 @@ const safeModuleName = (node: Node | null | undefined): string | undefined => { const resolveSpecifier = ( specifier: string, ): ReferenceSourceImportResolution => { - if (specifier.startsWith(".") || specifier.startsWith("/")) return "internal"; - if (specifier.startsWith("node:") || specifier.startsWith("data:")) - return "external"; + if ( + isRelativeJavaScriptModuleSpecifier(specifier) || + specifier.startsWith("/") + ) + return "internal"; return "external"; }; @@ -314,7 +318,7 @@ export const parseReferenceSourceImports = ( bytes: Uint8Array, language: string | null, ): ReferenceSourceImportParseResult => { - if (language !== null && !codeSourceLanguages.has(language)) + if (language !== null && !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language)) return { relationships: [], parse_failures: [], limitations: [] }; let source: string; diff --git a/src/domain/webModuleImports.test.ts b/src/domain/webModuleImports.test.ts index 26db977a1..c0bcabdbf 100644 --- a/src/domain/webModuleImports.test.ts +++ b/src/domain/webModuleImports.test.ts @@ -25,11 +25,18 @@ describe("native module syntax observations", () => { item.expression, ); }); - it("excludes erased TypeScript imports/exports and bundler inference", () => { + it("excludes whole type declarations and retains inline type dependency syntax", () => { const result = collectWebModuleImports( 'import type {T} from "types"; import {type X} from "also-types"; export type {T} from "export-types"; export {type Y} from "only-types"; import {type T, value} from "mixed"; __webpack_require__.e(4); __vitePreload(() => 1, ["chunk.js"]);', ); - expect(result.imports.map((item) => item.specifier)).toEqual(["mixed"]); + expect(result.imports.map((item) => item.specifier)).toEqual([ + "also-types", + "only-types", + "mixed", + ]); + expect(result.diagnostics).toContain( + "Compiler options are unavailable; inline type-only imports or re-exports may preserve a module side-effect dependency.", + ); }); it("uses UTF-16 source locations after astral characters", () => { const source = 'const emoji = "😀";\nimport "./x.js";'; diff --git a/src/domain/webModuleImports.ts b/src/domain/webModuleImports.ts index 026868955..0f9846871 100644 --- a/src/domain/webModuleImports.ts +++ b/src/domain/webModuleImports.ts @@ -4,8 +4,13 @@ import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal. import { parseJavaScriptSource } from "./javascript/javascriptSourceParser.js"; import type { WebModuleImport } from "./webModuleTrace.js"; import type { ExportedWebScript } from "./webScriptExport.js"; +import { + hasUncertainTypeOnlyModuleSideEffect, + isTypeOnlyExportDeclaration, + isTypeOnlyImportDeclaration, +} from "./javascript/javascriptModuleDeclaration.js"; -/** Collect native import syntax, excluding erased type-only references. */ +/** Collect module declarations, omitting whole-statement type-only forms. */ export const collectWebModuleImports = ( source: string, ): { @@ -55,35 +60,18 @@ export const collectWebModuleImports = ( traverseJavaScriptAst(ast, { enter: (node) => { if (node.type === "ImportDeclaration") { - if (node.importKind === "type" || node.importKind === "typeof") return; - if ( - node.specifiers.length > 0 && - node.specifiers.every( - (s) => - s.type === "ImportSpecifier" && - (s.importKind === "type" || s.importKind === "typeof"), - ) - ) - return; + if (isTypeOnlyImportDeclaration(node)) return; add("static-import", node.source); } else if ( node.type === "ExportAllDeclaration" || node.type === "ExportNamedDeclaration" ) { if ( - node.exportKind === "type" || + isTypeOnlyExportDeclaration(node) || node.source === null || node.source === undefined ) return; - if ( - node.type === "ExportNamedDeclaration" && - node.specifiers.length > 0 && - node.specifiers.every( - (s) => s.type === "ExportSpecifier" && s.exportKind === "type", - ) - ) - return; add("re-export", node.source); } else if (node.type === "ImportExpression") add("dynamic-import", node.source); @@ -99,7 +87,14 @@ export const collectWebModuleImports = ( return { imports, state: "parsed", - diagnostics: ast.errors?.map((error) => error.message) ?? [], + diagnostics: [ + ...(ast.errors?.map((error) => error.message) ?? []), + ...(hasUncertainTypeOnlyModuleSideEffect(ast.program) + ? [ + "Compiler options are unavailable; inline type-only imports or re-exports may preserve a module side-effect dependency.", + ] + : []), + ], }; }; diff --git a/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts b/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts index cd045a00a..3a7fad35f 100644 --- a/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts +++ b/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts @@ -29,6 +29,14 @@ const createModuleLookupFixture = async () => { 'import "ancestor-empty";', 'import "ancestor-blocked";', 'import "ancestor-sub/parser";', + 'import "./.hidden.js";', + 'import "./..pkg.js";', + 'import "./exact.js/.";', + 'import "./exact.js/child/..";', + 'import ".hidden.js";', + 'import "..pkg.js";', + 'import ".";', + 'import "..";', "void fallback;", ].join("\n"), "esm/extensionless.js": "module.exports = 1;", @@ -36,6 +44,10 @@ const createModuleLookupFixture = async () => { "esm/explicit.js.js": "module.exports = 3;", "esm/exact.js": "module.exports = 4;", "esm/actual.cjs": "module.exports = 5;", + "esm/.hidden.js": "module.exports = 'local hidden';", + "esm/..pkg.js": "module.exports = 'local dot package';", + "node_modules/.hidden.js/index.js": "module.exports = 'hidden package';", + "node_modules/..pkg.js/index.js": "module.exports = 'dot package';", "esm/typed.mts": 'import value from "./typed-dep.js"; void value;', "esm/typed-dep.ts": "export default 42;", "cjs/entry.cjs": [ @@ -54,8 +66,15 @@ const createModuleLookupFixture = async () => { 'require("ancestor-empty");', 'require("ancestor-blocked");', 'require("ancestor-sub/parser");', + 'require(".hidden.js");', + 'require("./.hidden.js");', + 'require(".");', + 'require("..");', ].join("\n"), "cjs/directory/index.cjs": "module.exports = 6;", + "cjs/index.js": "module.exports = 'current directory';", + "index.js": "module.exports = 'parent directory';", + "cjs/.hidden.js": "module.exports = 'local hidden';", "cjs/extensionless.cjs": "module.exports = 1;", "cjs/feature.cjs": "module.exports = 2;", "cjs/module-only.mjs": "export default 42;", @@ -79,7 +98,7 @@ const createModuleLookupFixture = async () => { "node_modules/ancestor-blocked/index.js": "module.exports = 9;", "node_modules/ancestor-sub/parser.js": "module.exports = 10;", "esm/runtime-control.mjs": [ - 'const specifiers = ["./extensionless", "./directory", "./explicit.js", "./exact.js", "./actual.cjs?cache=1", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser"];', + 'const specifiers = ["./extensionless", "./directory", "./explicit.js", "./exact.js", "./actual.cjs?cache=1", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser", "./.hidden.js", "./..pkg.js", "./exact.js/.", "./exact.js/child/..", ".hidden.js", "..pkg.js", ".", ".."];', "const results = [];", "for (const specifier of specifiers) {", " try { const value = (await import(specifier)).default; results.push({ specifier, ok: true, value }); }", @@ -88,7 +107,7 @@ const createModuleLookupFixture = async () => { "console.log(JSON.stringify(results));", ].join("\n"), "cjs/runtime-control.cjs": [ - 'const specifiers = ["./extensionless", "./directory", "./feature", "./feature.cjs", "./module-only", "./valid-directory", "./default-js", "./default-json", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser"];', + 'const specifiers = ["./extensionless", "./directory", "./feature", "./feature.cjs", "./module-only", "./valid-directory", "./default-js", "./default-json", "empty-main", "nested-main", "file-only", "ancestor-empty", "ancestor-blocked", "ancestor-sub/parser", ".hidden.js", "./.hidden.js", ".", ".."];', "const results = [];", "for (const specifier of specifiers) {", " try { const value = require(specifier); results.push({ specifier, ok: true, value }); }", @@ -109,6 +128,110 @@ const createModuleLookupFixture = async () => { return root; }; +type RuntimeControlResult = { + readonly specifier: string; + readonly ok: boolean; + readonly code?: string; + readonly value?: unknown; +}; + +const expectNodeRuntimeControls = ( + esmOutput: string, + cjsOutput: string, +): void => { + const esmResults = JSON.parse(esmOutput) as RuntimeControlResult[]; + const cjsResults = JSON.parse(cjsOutput) as RuntimeControlResult[]; + expect(esmResults).toEqual([ + { + specifier: "./extensionless", + ok: false, + code: "ERR_MODULE_NOT_FOUND", + }, + { + specifier: "./directory", + ok: false, + code: "ERR_UNSUPPORTED_DIR_IMPORT", + }, + { + specifier: "./explicit.js", + ok: false, + code: "ERR_MODULE_NOT_FOUND", + }, + { specifier: "./exact.js", ok: true, value: 4 }, + { specifier: "./actual.cjs?cache=1", ok: true, value: 5 }, + { specifier: "empty-main", ok: true, value: 42 }, + { specifier: "nested-main", ok: false, code: "ERR_MODULE_NOT_FOUND" }, + { specifier: "file-only", ok: false, code: "ERR_MODULE_NOT_FOUND" }, + { specifier: "ancestor-empty", ok: false, code: "ERR_MODULE_NOT_FOUND" }, + { + specifier: "ancestor-blocked", + ok: false, + code: "ERR_MODULE_NOT_FOUND", + }, + { + specifier: "ancestor-sub/parser", + ok: false, + code: "ERR_MODULE_NOT_FOUND", + }, + { specifier: "./.hidden.js", ok: true, value: "local hidden" }, + { specifier: "./..pkg.js", ok: true, value: "local dot package" }, + { + specifier: "./exact.js/.", + ok: false, + code: "ERR_UNSUPPORTED_DIR_IMPORT", + }, + { + specifier: "./exact.js/child/..", + ok: false, + code: "ERR_UNSUPPORTED_DIR_IMPORT", + }, + { + specifier: ".hidden.js", + ok: false, + code: "ERR_INVALID_MODULE_SPECIFIER", + }, + { + specifier: "..pkg.js", + ok: false, + code: "ERR_INVALID_MODULE_SPECIFIER", + }, + { specifier: ".", ok: false, code: "ERR_UNSUPPORTED_DIR_IMPORT" }, + { specifier: "..", ok: false, code: "ERR_UNSUPPORTED_DIR_IMPORT" }, + ]); + expect(cjsResults).toEqual([ + { + specifier: "./extensionless", + ok: false, + code: "MODULE_NOT_FOUND", + }, + { + specifier: "./directory", + ok: false, + code: "MODULE_NOT_FOUND", + }, + { + specifier: "./feature", + ok: false, + code: "MODULE_NOT_FOUND", + }, + { specifier: "./feature.cjs", ok: true, value: 2 }, + { specifier: "./module-only", ok: false, code: "MODULE_NOT_FOUND" }, + { specifier: "./valid-directory", ok: true, value: 42 }, + { specifier: "./default-js", ok: true, value: 3 }, + { specifier: "./default-json", ok: true, value: { value: 4 } }, + { specifier: "empty-main", ok: true, value: 42 }, + { specifier: "nested-main", ok: false, code: "MODULE_NOT_FOUND" }, + { specifier: "file-only", ok: true, value: 7 }, + { specifier: "ancestor-empty", ok: true, value: 8 }, + { specifier: "ancestor-blocked", ok: false, code: "MODULE_NOT_FOUND" }, + { specifier: "ancestor-sub/parser", ok: true, value: 10 }, + { specifier: ".hidden.js", ok: true, value: "hidden package" }, + { specifier: "./.hidden.js", ok: true, value: "local hidden" }, + { specifier: ".", ok: true, value: "current directory" }, + { specifier: "..", ok: true, value: "parent directory" }, + ]); +}; + cliTest.for(["CLI", "stdio MCP"] as const)( "matches Node module lookup for ESM and CommonJS through %s", async (surface, { cli }) => { @@ -123,79 +246,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)( timeout: 5_000, }), ]); - const esmResults = JSON.parse(esmControl.stdout) as Array<{ - readonly specifier: string; - readonly ok: boolean; - readonly code?: string; - readonly value?: unknown; - }>; - const cjsResults = JSON.parse(cjsControl.stdout) as Array<{ - readonly specifier: string; - readonly ok: boolean; - readonly code?: string; - readonly value?: unknown; - }>; - expect(esmResults).toEqual([ - { - specifier: "./extensionless", - ok: false, - code: "ERR_MODULE_NOT_FOUND", - }, - { - specifier: "./directory", - ok: false, - code: "ERR_UNSUPPORTED_DIR_IMPORT", - }, - { - specifier: "./explicit.js", - ok: false, - code: "ERR_MODULE_NOT_FOUND", - }, - { specifier: "./exact.js", ok: true, value: 4 }, - { specifier: "./actual.cjs?cache=1", ok: true, value: 5 }, - { specifier: "empty-main", ok: true, value: 42 }, - { specifier: "nested-main", ok: false, code: "ERR_MODULE_NOT_FOUND" }, - { specifier: "file-only", ok: false, code: "ERR_MODULE_NOT_FOUND" }, - { specifier: "ancestor-empty", ok: false, code: "ERR_MODULE_NOT_FOUND" }, - { - specifier: "ancestor-blocked", - ok: false, - code: "ERR_MODULE_NOT_FOUND", - }, - { - specifier: "ancestor-sub/parser", - ok: false, - code: "ERR_MODULE_NOT_FOUND", - }, - ]); - expect(cjsResults).toEqual([ - { - specifier: "./extensionless", - ok: false, - code: "MODULE_NOT_FOUND", - }, - { - specifier: "./directory", - ok: false, - code: "MODULE_NOT_FOUND", - }, - { - specifier: "./feature", - ok: false, - code: "MODULE_NOT_FOUND", - }, - { specifier: "./feature.cjs", ok: true, value: 2 }, - { specifier: "./module-only", ok: false, code: "MODULE_NOT_FOUND" }, - { specifier: "./valid-directory", ok: true, value: 42 }, - { specifier: "./default-js", ok: true, value: 3 }, - { specifier: "./default-json", ok: true, value: { value: 4 } }, - { specifier: "empty-main", ok: true, value: 42 }, - { specifier: "nested-main", ok: false, code: "MODULE_NOT_FOUND" }, - { specifier: "file-only", ok: true, value: 7 }, - { specifier: "ancestor-empty", ok: true, value: 8 }, - { specifier: "ancestor-blocked", ok: false, code: "MODULE_NOT_FOUND" }, - { specifier: "ancestor-sub/parser", ok: true, value: 10 }, - ]); + expectNodeRuntimeControls(esmControl.stdout, cjsControl.stdout); const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); const expectations = [ @@ -211,6 +262,14 @@ cliTest.for(["CLI", "stdio MCP"] as const)( ["import", "ancestor-empty", null, "not-found"], ["import", "ancestor-blocked", null, "not-found"], ["import", "ancestor-sub/parser", null, "not-found"], + ["import", "./.hidden.js", "esm/.hidden.js", "resolved"], + ["import", "./..pkg.js", "esm/..pkg.js", "resolved"], + ["import", ".hidden.js", null, "rejected"], + ["import", "..pkg.js", null, "rejected"], + ["import", ".", null, "not-found"], + ["import", "..", null, "not-found"], + ["import", "./exact.js/.", null, "not-found"], + ["import", "./exact.js/child/..", null, "not-found"], ["require", "./extensionless", null, "not-found"], ["require", "./directory", null, "not-found"], ["require", "./feature", null, "not-found"], @@ -242,19 +301,30 @@ cliTest.for(["CLI", "stdio MCP"] as const)( "node_modules/ancestor-sub/parser.js", "resolved", ], + ["require", ".hidden.js", "node_modules/.hidden.js/index.js", "resolved"], + ["require", "./.hidden.js", "cjs/.hidden.js", "resolved"], + ["require", ".", "cjs/index.js", "resolved"], + ["require", "..", "index.js", "resolved"], ] as const; - for (const [kind, specifier, resolvedPath, status] of expectations) - expect(graph.edges).toContainEqual( + for (const [kind, specifier, resolvedPath, status] of expectations) { + const matchingProperties = graph.edges + .filter((edge) => edge.relation === "imports") + .map((edge) => edge.properties) + .filter( + (properties) => + "module_link_kind" in properties && + properties.module_link_kind === kind && + properties.specifier === specifier, + ); + expect(matchingProperties).toContainEqual( expect.objectContaining({ - relation: "imports", - properties: expect.objectContaining({ - module_link_kind: kind, - specifier, - resolved_path: resolvedPath, - resolution_status: status, - }), + module_link_kind: kind, + specifier, + resolved_path: resolvedPath, + resolution_status: status, }), ); + } }, ); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts index be51b9bb4..9b4df9992 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts @@ -15,6 +15,60 @@ import { } from "../../../support/javascriptApplicationFixture.js"; describe("TypeScript source module loader semantics", () => { + it("keeps whole type-only declarations out of runtime module relationships", async () => { + const root = await createTestTempDirectory("rea-type-module-links-"); + await writeFixtureFiles(root, { + "main.ts": ` + import type { Whole } from "./whole"; + export type { Remote } from "./remote"; + import { type Inline } from "./inline"; + export type LocalType = string; + export interface LocalInterface { value: string } + `, + "whole.ts": "export interface Whole { value: string }", + "remote.ts": "export interface Remote { value: string }", + "inline.ts": "export interface Inline { value: string }", + }); + + const { graph } = await reconstructJavaScriptFixture(root); + const source = findSourceModule(graph, "main.ts"); + const sourceProperties = source?.observations.find( + ({ properties }) => properties.semantic_role === "source-module", + )?.properties; + + expect(sourceProperties).toMatchObject({ + import_relationships: 0, + type_relationships: 5, + export_names: [], + type_export_names: ["LocalInterface", "LocalType", "Remote"], + }); + expect( + graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && + ["./whole", "./remote"].includes(String(properties.specifier)), + ), + ).toEqual([]); + expect( + graph.edges.find( + ({ source_node_id, relation, properties }) => + source_node_id === source?.node_id && + relation === "imports" && + properties.kind === "static-import" && + properties.specifier === "./inline", + ), + ).toBeDefined(); + expect( + source?.observations + .flatMap(({ evidence }) => evidence.limitations) + .join(" "), + ).toContain( + "inline type-only imports or re-exports may preserve a module side-effect dependency", + ); + expect(findExportNode(graph, "main.ts", "LocalType")).toBeUndefined(); + expect(findExportNode(graph, "main.ts", "LocalInterface")).toBeUndefined(); + }); + it("uses emitted TypeScript loader semantics without changing dynamic imports", async () => { const root = await createTestTempDirectory("rea-cts-module-loader-"); await writeFixtureFiles(root, { diff --git a/tests/boundary/filesystem/referenceSourceImport.test.ts b/tests/boundary/filesystem/referenceSourceImport.test.ts index 75eab7acb..8443dfbdd 100644 --- a/tests/boundary/filesystem/referenceSourceImport.test.ts +++ b/tests/boundary/filesystem/referenceSourceImport.test.ts @@ -378,6 +378,44 @@ describe("reference source path selection", () => { }); describe("reference source rooted module specifiers", () => { + it("keeps dot-prefixed bare specifiers external while resolving explicit relative paths", async () => { + const root = await createTestTempDirectory("rea-reference-dot-prefix-"); + await Promise.all([ + writeFile( + join(root, "main.js"), + 'import ".hidden.js"; import "..pkg.js"; import "./.hidden.js"; require(".hidden.js");\n', + ), + writeFile(join(root, ".hidden.js"), "export {};\n"), + writeFile(join(root, "..pkg.js"), "export {};\n"), + ]); + const result = await importTree(root); + if (!result.ok) throw result.error; + expect(result.value.relationships).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + to: ".hidden.js", + kind: "imports", + resolution: "external", + }), + expect.objectContaining({ + to: "..pkg.js", + kind: "imports", + resolution: "external", + }), + expect.objectContaining({ + to: ".hidden.js", + kind: "imports", + resolution: "internal", + }), + expect.objectContaining({ + to: ".hidden.js", + kind: "requires", + resolution: "external", + }), + ]), + ); + }); + it("does not rebase an absolute import onto a coincidentally matching source member", async () => { const root = await createTestTempDirectory("rea-reference-rooted-"); await mkdir(join(root, "src", "outside"), { recursive: true }); From 26bffb94bf80355c35e22a98a8fc629c6210307e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:39:49 +0800 Subject: [PATCH 03/27] perf(evidence): reuse authenticated comparisons through aggregation --- src/domain/changedBehavior.test.ts | 42 +++--- src/domain/changedBehavior.ts | 133 ++++++++++++------ src/server/registerInvestigationTools.ts | 30 ++-- .../mcp/processEvidenceIdentityMcp.test.ts | 17 ++- 4 files changed, 146 insertions(+), 76 deletions(-) diff --git a/src/domain/changedBehavior.test.ts b/src/domain/changedBehavior.test.ts index cc19c4c50..6d99e828f 100644 --- a/src/domain/changedBehavior.test.ts +++ b/src/domain/changedBehavior.test.ts @@ -22,6 +22,9 @@ const source = (digit: string): Evidence => const left = source("1"); const right = source("2"); +const aggregate = (comparisons: readonly unknown[]) => + findChangedBehavior({ comparisons }); + const comparison = ( operation: "compare_process_captures" | "compare_artifacts", result: JsonValue, @@ -136,9 +139,9 @@ const expectInvalidSummary = (result: ChangedBehaviorResult): void => { describe("changed behavior", () => { it("accepts process comparison Evidence", () => { - const result = findChangedBehavior([ + const result = aggregate([ comparison("compare_process_captures", processResult()), - ]); + ]).result; expect(result.behavior_status).toBe("observed_unchanged"); }); @@ -152,7 +155,7 @@ describe("changed behavior", () => { }), ); - expect(findChangedBehavior([evidence])).toMatchObject({ + expect(aggregate([evidence]).result).toMatchObject({ behavior_status: "observed_changed", summary: { observed_changes: 2 }, findings: { @@ -172,7 +175,7 @@ describe("changed behavior", () => { "compare_process_captures", processResult({ status: "changed", terminal: "changed" }), ); - const result = findChangedBehavior([evidence]); + const result = aggregate([evidence]).result; expect(changedBehaviorResultSchema.parse(result)).toMatchObject({ behavior_status: "observed_changed", summary: { observed_changes: 1, static_candidates: 0 }, @@ -193,7 +196,7 @@ describe("changed behavior", () => { it("keeps static differences as candidates, never runtime observations", () => { const evidence = comparison("compare_artifacts", artifactResult()); - const result = findChangedBehavior([evidence]); + const result = aggregate([evidence]).result; expect(result).toMatchObject({ behavior_status: "unknown", summary: { observed_changes: 0, static_candidates: 1 }, @@ -217,17 +220,17 @@ describe("changed behavior", () => { "compare_process_captures", processResult({ status: "unknown", exit: "unknown" }), ); - const result = findChangedBehavior([changed, unknown]); + const result = aggregate([changed, unknown]).result; expect(result).toMatchObject({ behavior_status: "unknown", findings: { items: expect.any(Array) }, }); - expect(findChangedBehavior([changed, unknown])).toEqual(result); + expect(aggregate([changed, unknown]).result).toEqual(result); }); it("uses complete inline artifact deltas as static candidates", () => { const complete = comparison("compare_artifacts", artifactResult()); - expect(findChangedBehavior([complete]).findings.items).toEqual( + expect(aggregate([complete]).result.findings.items).toEqual( expect.arrayContaining([ expect.objectContaining({ dimension: "artifact:main.js" }), ]), @@ -236,7 +239,7 @@ describe("changed behavior", () => { it("rejects bundles, duplicate comparisons, and malformed exact results", () => { const valid = comparison("compare_process_captures", processResult()); - expect(() => findChangedBehavior([valid, valid])).toThrow( + expect(() => aggregate([valid, valid])).toThrow( /duplicate comparison Evidence/u, ); const bundle = createEvidence( @@ -252,25 +255,25 @@ describe("changed behavior", () => { evidenceLinks: [left.evidence_id, right.evidence_id], }, ); - expect(() => findChangedBehavior([bundle])).toThrow( + expect(() => aggregate([bundle])).toThrow( /requires process, artifact, or function/u, ); const malformed = comparison("compare_process_captures", { status: "unchanged", }); - expect(() => findChangedBehavior([malformed])).toThrow(); + expect(() => aggregate([malformed])).toThrow(); const contradictory = comparison( "compare_process_captures", processResult({ status: "unchanged", terminal: "changed" }), ); - expect(() => findChangedBehavior([contradictory])).toThrow( + expect(() => aggregate([contradictory])).toThrow( /contradicts its dimensions/u, ); const danglingNested = comparison( "compare_artifacts", artifactResult([`ev_${"f".repeat(64)}`, right.evidence_id]), ); - expect(() => findChangedBehavior([danglingNested])).toThrow( + expect(() => aggregate([danglingNested])).toThrow( /outside its top-level closure/u, ); }); @@ -299,10 +302,15 @@ describe("changed behavior aggregation", () => { ), ); - const result = findChangedBehavior(comparisons); + const result = aggregate(comparisons); - expect(result.summary.observed_changes).toBe(comparisons.length); - expect(result.evidence_links).toHaveLength(comparisons.length * 201 + 2); - expect(result.findings.items[0]?.evidence_links).toHaveLength(203); + expect(result.result.summary.observed_changes).toBe(comparisons.length); + expect(result.result.evidence_links).toHaveLength( + comparisons.length * 201 + 2, + ); + expect(result.result.findings.items[0]?.evidence_links).toHaveLength(203); + expect(result.comparisonEvidenceIds).toEqual( + comparisons.map(({ evidence_id }) => evidence_id), + ); }); }); diff --git a/src/domain/changedBehavior.ts b/src/domain/changedBehavior.ts index b785cdd23..56e9b6ad9 100644 --- a/src/domain/changedBehavior.ts +++ b/src/domain/changedBehavior.ts @@ -91,6 +91,22 @@ export const changedBehaviorResultSchema = z export type ChangedBehaviorResult = z.infer; type Finding = z.infer; type RuntimeStatus = z.infer["status"]; +type ParsedComparison = + | { + readonly operation: "compare_process_captures"; + readonly evidence: Evidence; + readonly result: z.infer; + } + | { + readonly operation: "compare_functions"; + readonly evidence: Evidence; + readonly result: z.infer; + } + | { + readonly operation: "compare_artifacts"; + readonly evidence: Evidence; + readonly result: z.infer; + }; const EXPECTED_PROVIDERS = { compare_process_captures: { @@ -117,24 +133,37 @@ const acceptsPredicate = ( ): boolean => expected.some((predicate) => predicate === actual); /** Find observed runtime changes and separately report static change candidates. */ -export const findChangedBehavior = ( - comparisonsInput: unknown, -): ChangedBehaviorResult => { - const parsedInput = changedBehaviorInputSchema.parse({ - comparisons: comparisonsInput, - }); - const comparisons = parsedInput.comparisons; +interface FoundChangedBehavior { + readonly result: ChangedBehaviorResult; + /** Authenticated IDs retained from the comparisons used to compute result. */ + readonly comparisonEvidenceIds: readonly string[]; +} + +export const findChangedBehavior = (input: unknown): FoundChangedBehavior => { + // The public input schema remains available to clients; this internal + // admission only checks the list shape because parseEvidence below owns + // validation and authentication of each retained Evidence item. + const { comparisons } = z + .strictObject({ comparisons: z.array(z.unknown()).min(1) }) + .parse(input); const evidence = comparisons.map(parseComparisonEvidence); assertUnique( - evidence.map(({ evidence_id: id }) => id), + evidence.map(({ evidence: { evidence_id: id } }) => id), "comparison Evidence", ); const findings = evidence.flatMap(findingsFor).sort(compareFindings); - const runtimeStatuses = evidence.flatMap(runtimeStatus); - const links = uniqueSorted( - evidence.flatMap((item) => [item.evidence_id, ...item.evidence_links]), + const runtimeStatuses = evidence.flatMap((comparison) => + comparison.operation === "compare_process_captures" + ? [comparison.result.status] + : [], ); - return changedBehaviorResultSchema.parse({ + const links = uniqueSorted( + evidence.flatMap(({ evidence: item }) => [ + item.evidence_id, + ...item.evidence_links, + ]), + ); + const result = changedBehaviorResultSchema.parse({ behavior_status: behaviorStatus(runtimeStatuses), summary: { observed_changes: count(findings, "observed_change"), @@ -147,7 +176,7 @@ export const findChangedBehavior = ( }, evidence_links: links, limitations: uniqueSorted([ - ...evidence.flatMap(({ limitations }) => limitations), + ...evidence.flatMap(({ evidence: item }) => item.limitations), ...(evidence.some( ({ operation }) => operation !== "compare_process_captures", ) @@ -160,9 +189,15 @@ export const findChangedBehavior = ( : []), ]), }); + return { + result, + comparisonEvidenceIds: evidence.map( + ({ evidence: { evidence_id } }) => evidence_id, + ), + }; }; -const parseComparisonEvidence = (input: unknown): Evidence => { +const parseComparisonEvidence = (input: unknown): ParsedComparison => { const evidence = parseEvidence(input); if (!(evidence.operation in EXPECTED_PROVIDERS)) throw new TypeError( @@ -198,12 +233,17 @@ const parseComparisonEvidence = (input: unknown): Evidence => { throw new TypeError( "Comparison Evidence must cite both selected source observations", ); - parseResult(evidence); - return evidence; + return parseResult(evidence, operation); }; -const parseResult = (evidence: Evidence): void => { - if (evidence.operation === "compare_process_captures") { +const parseResult = ( + evidence: Evidence, + operation: + | "compare_process_captures" + | "compare_artifacts" + | "compare_functions", +): ParsedComparison => { + if (operation === "compare_process_captures") { const result = processCaptureComparisonSchema.parse( evidence.normalized_result, ); @@ -214,9 +254,9 @@ const parseResult = (evidence: Evidence): void => { throw new TypeError( "Process comparison status contradicts its dimensions", ); - return; + return { operation, evidence, result }; } - if (evidence.operation === "compare_functions") { + if (operation === "compare_functions") { const result = functionComparisonResultSchema.parse( evidence.normalized_result, ); @@ -224,7 +264,7 @@ const parseResult = (evidence: Evidence): void => { evidence, result.dimensions.flatMap(({ evidence_links: links }) => links), ); - return; + return { operation, evidence, result }; } const result = artifactComparisonResultSchema.parse( evidence.normalized_result, @@ -233,6 +273,7 @@ const parseResult = (evidence: Evidence): void => { evidence, result.changes.flatMap(({ evidence_links: links }) => links), ); + return { operation, evidence, result }; }; const assertNestedLinks = ( @@ -246,18 +287,21 @@ const assertNestedLinks = ( ); }; -const findingsFor = (evidence: Evidence): Finding[] => { - if (evidence.operation === "compare_process_captures") - return processFindings(evidence); - if (evidence.operation === "compare_functions") - return functionFindings(evidence); - return artifactFindings(evidence); +const findingsFor = (comparison: ParsedComparison): Finding[] => { + if (comparison.operation === "compare_process_captures") + return processFindings(comparison); + if (comparison.operation === "compare_functions") + return functionFindings(comparison); + return artifactFindings(comparison); }; -const processFindings = (evidence: Evidence): Finding[] => { - const result = processCaptureComparisonSchema.parse( - evidence.normalized_result, - ); +const processFindings = ( + comparison: Extract< + ParsedComparison, + { readonly operation: "compare_process_captures" } + >, +): Finding[] => { + const { evidence, result } = comparison; const dimensions = [ ["terminal", result.terminal, "runtime"], ["interaction", result.interaction, "runtime"], @@ -283,10 +327,13 @@ const processFindings = (evidence: Evidence): Finding[] => { ); }; -const functionFindings = (evidence: Evidence): Finding[] => { - const result = functionComparisonResultSchema.parse( - evidence.normalized_result, - ); +const functionFindings = ( + comparison: Extract< + ParsedComparison, + { readonly operation: "compare_functions" } + >, +): Finding[] => { + const { evidence, result } = comparison; return result.dimensions.flatMap((dimension) => dimension.status === "unchanged" ? [] @@ -307,10 +354,13 @@ const functionFindings = (evidence: Evidence): Finding[] => { ); }; -const artifactFindings = (evidence: Evidence): Finding[] => { - const result = artifactComparisonResultSchema.parse( - evidence.normalized_result, - ); +const artifactFindings = ( + comparison: Extract< + ParsedComparison, + { readonly operation: "compare_artifacts" } + >, +): Finding[] => { + const { evidence, result } = comparison; const changes = result.changes.map((change) => makeFinding({ evidence, @@ -367,11 +417,6 @@ const makeFinding = (input: { limitations: uniqueSorted(input.limitations), }); -const runtimeStatus = (evidence: Evidence): RuntimeStatus[] => - evidence.operation === "compare_process_captures" - ? [processCaptureComparisonSchema.parse(evidence.normalized_result).status] - : []; - const behaviorStatus = ( statuses: readonly RuntimeStatus[], ): ChangedBehaviorResult["behavior_status"] => { diff --git a/src/server/registerInvestigationTools.ts b/src/server/registerInvestigationTools.ts index d5a2a4706..02e1c52d0 100644 --- a/src/server/registerInvestigationTools.ts +++ b/src/server/registerInvestigationTools.ts @@ -21,7 +21,10 @@ import { RECONSTRUCTION_PROVIDER, STATIC_RUNTIME_PROVIDER, } from "../application/InvestigationProviders.js"; -import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { + applicationValidatedInputSchema, + toolRegistrationOptions, +} from "./toolRegistrationOptions.js"; import { verifyReconstruction } from "../domain/reconstructionVerification.js"; /** Register Evidence-composed differential investigation workflows. */ @@ -52,30 +55,29 @@ const registerChangedBehavior = ( session: BinarySessionPort, contract: ReturnType>, ): void => { + const registration = toolRegistrationOptions(contract); server.registerTool( contract.name, - toolRegistrationOptions(contract), + { + ...registration, + inputSchema: applicationValidatedInputSchema(registration.inputSchema), + }, async (input, context) => { - const closure = evidenceClosure( - session, - comparisonClosure(input.comparisons), - ); - if (!closure.ok) - return server.delivery.toCallToolResult(closure, contract); - const links = closure.value; const computed = await runDerivedOperation(context, contract.name, () => - findChangedBehavior(input.comparisons), + findChangedBehavior(input), ); if (!computed.ok) return server.delivery.toCallToolResult(computed, contract); - const result = computed.value; + const { result, comparisonEvidenceIds } = computed.value; + const closure = evidenceClosure(session, result.evidence_links); + if (!closure.ok) + return server.delivery.toCallToolResult(closure, contract); + const links = closure.value; const evidence = createEvidence(undefined, CHANGED_BEHAVIOR_PROVIDER, { predicateType: "rea.changed-behavior", operation: contract.name, parameters: { - comparison_evidence_ids: input.comparisons.map( - ({ evidence_id: id }) => id, - ), + comparison_evidence_ids: [...comparisonEvidenceIds], }, result: jsonValueSchema.parse(result), confidence: "derived", diff --git a/tests/boundary/mcp/processEvidenceIdentityMcp.test.ts b/tests/boundary/mcp/processEvidenceIdentityMcp.test.ts index fb20dd79b..993f1df69 100644 --- a/tests/boundary/mcp/processEvidenceIdentityMcp.test.ts +++ b/tests/boundary/mcp/processEvidenceIdentityMcp.test.ts @@ -309,7 +309,9 @@ it("separately rejects a tampered digest before admission to any consumer or ses }; const digestMismatch = /semantic identifier does not match/u; expect(() => parseEvidence(tampered)).toThrow(digestMismatch); - expect(() => findChangedBehavior([tampered])).toThrow(digestMismatch); + expect(() => findChangedBehavior({ comparisons: [tampered] })).toThrow( + digestMismatch, + ); expect(() => correlateStaticAndRuntime(correlationInput(tampered))).toThrow( digestMismatch, ); @@ -336,6 +338,19 @@ it("separately rejects a tampered digest before admission to any consumer or ses } }); +it("maps malformed changed-behavior Evidence to an invalid request before closure lookup", async () => { + const { client, comparison } = await connectedComparison(); + const malformed = { ...comparison, evidence_links: undefined } as unknown; + const response = await client.callTool({ + name: "find_changed_behavior", + arguments: { comparisons: [malformed] }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response)).toMatchObject({ + error: { code: "invalid_request" }, + }); +}); + it("keeps withheld ownership unknown through CLI and MCP comparison", async () => { const { client, right } = await connectedComparison(); const left = captureEvidence("left", { From 12303812a8d70f198c99c7aeb0df3aaa8166543c Mon Sep 17 00:00:00 2001 From: Rohit Ghumare <48523873+rohitg00@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:15:03 +0530 Subject: [PATCH 04/27] fix(evidence): admit bundle imports across session transitions (#2303) --- src/server/registerSessionRecordTools.ts | 78 +++++---- .../mcp/evidenceImportLifecycle.test.ts | 155 ++++++++++++++++++ 2 files changed, 202 insertions(+), 31 deletions(-) create mode 100644 tests/boundary/mcp/evidenceImportLifecycle.test.ts diff --git a/src/server/registerSessionRecordTools.ts b/src/server/registerSessionRecordTools.ts index a41d60cac..68daab23b 100644 --- a/src/server/registerSessionRecordTools.ts +++ b/src/server/registerSessionRecordTools.ts @@ -12,6 +12,8 @@ import { err, ok } from "../domain/result.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { inspectEvidenceBundle } from "../application/investigation/InspectEvidenceBundle.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { withAdmittedAnalysis } from "./analysisAdmission.js"; +import { runAdmittedToolOperation } from "./admittedToolOperation.js"; interface EvidenceToolRegistration { readonly server: EvidenceMcpServer; @@ -92,40 +94,54 @@ const registerImportEvidenceTool = ({ resources, importContract, }: EvidenceToolRegistration): void => { + const admission = withAdmittedAnalysis({ kind: "session", session }); server.registerTool( importContract.name, toolRegistrationOptions(importContract), - async (input, context) => { - const path = input.path; - const signal = context.mcpReq.signal; - const loaded = await readImmutableEvidenceBundle(path, signal, resources); - if (!loaded.ok) - return server.delivery.toCallToolResult(loaded, importContract); - const retainedUnknownRevisions = new Set( - bundleForSerialization(session).unknowns.map((unknown) => - unknownRevisionKey(unknown), - ), - ); - if (signal.aborted) - return server.delivery.toCallToolResult( - err(new AnalysisCancelledError("import_evidence_bundle")), - importContract, - ); - const imported = session.importEvidenceBundle(loaded.value); - return imported.ok - ? server.delivery.toCallToolResult( - ok({ - imported: imported.value, - unknowns_added: loaded.value.unknowns.filter( - (unknown) => - !retainedUnknownRevisions.has(unknownRevisionKey(unknown)), - ).length, - total: bundleForSerialization(session).records.length, - }), - importContract, - ) - : server.delivery.toCallToolResult(imported, importContract); - }, + async (input, context) => + runAdmittedToolOperation( + server, + admission, + importContract.name, + context.mcpReq.signal, + async () => { + const path = input.path; + const signal = context.mcpReq.signal; + const loaded = await readImmutableEvidenceBundle( + path, + signal, + resources, + ); + if (!loaded.ok) + return server.delivery.toCallToolResult(loaded, importContract); + const retainedUnknownRevisions = new Set( + bundleForSerialization(session).unknowns.map((unknown) => + unknownRevisionKey(unknown), + ), + ); + if (signal.aborted) + return server.delivery.toCallToolResult( + err(new AnalysisCancelledError("import_evidence_bundle")), + importContract, + ); + const imported = session.importEvidenceBundle(loaded.value); + return imported.ok + ? server.delivery.toCallToolResult( + ok({ + imported: imported.value, + unknowns_added: loaded.value.unknowns.filter( + (unknown) => + !retainedUnknownRevisions.has( + unknownRevisionKey(unknown), + ), + ).length, + total: bundleForSerialization(session).records.length, + }), + importContract, + ) + : server.delivery.toCallToolResult(imported, importContract); + }, + ), ); }; diff --git a/tests/boundary/mcp/evidenceImportLifecycle.test.ts b/tests/boundary/mcp/evidenceImportLifecycle.test.ts new file mode 100644 index 000000000..f0371ef89 --- /dev/null +++ b/tests/boundary/mcp/evidenceImportLifecycle.test.ts @@ -0,0 +1,155 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, onTestFinished, vi } from "vitest"; + +import { createEvidence } from "../../../src/domain/evidence.js"; +import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { + createDeferred, + createTestBinarySession, +} from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const connectImportSession = async () => { + const root = await createTestTempDirectory("rea-mcp-import-lifecycle-"); + const session = createTestBinarySession(() => { + throw new Error("Evidence import must not start a binary provider"); + }); + const server = createServer({ kind: "session", session }); + const client = new Client({ + name: "evidence-import-lifecycle", + version: "1", + }); + const readCompleted = createDeferred(); + const releaseRead = createDeferred(); + const originalFileClose = OwnedFileHandle.prototype.close; + vi.spyOn(OwnedFileHandle.prototype, "close").mockImplementationOnce( + async function (this: OwnedFileHandle) { + await originalFileClose.call(this); + readCompleted.resolve(); + await releaseRead.promise; + }, + ); + const closeStarted = createDeferred(); + const originalSessionClose = session.close.bind(session); + const events: string[] = []; + vi.spyOn(session, "close").mockImplementation((...args) => { + const closing = originalSessionClose(...args); + closeStarted.resolve(); + return closing.then((result) => { + events.push("close"); + return result; + }); + }); + const originalImport = session.importEvidenceBundle.bind(session); + vi.spyOn(session, "importEvidenceBundle").mockImplementation((bundle) => { + const imported = originalImport(bundle); + events.push("import"); + return imported; + }); + onTestFinished(async () => { + releaseRead.resolve(); + vi.restoreAllMocks(); + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const laterImportDispatched = createDeferred(); + let importsDispatched = 0; + const send = clientTransport.send.bind(clientTransport); + vi.spyOn(clientTransport, "send").mockImplementation( + async (message, ...args) => { + await send(message, ...args); + if ( + "method" in message && + message.method === "tools/call" && + message.params?.name === "import_evidence_bundle" + ) { + importsDispatched += 1; + if (importsDispatched === 2) laterImportDispatched.resolve(); + } + }, + ); + await server.connect(serverTransport); + await client.connect(clientTransport); + const importBundle = async (operation: string) => { + const record = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { operation, parameters: {}, result: { retained: operation } }, + ); + const path = join(root, `${operation}.json`); + await writeFile(path, JSON.stringify(createEvidenceBundle([record]))); + return client.callTool({ + name: "import_evidence_bundle", + arguments: { path }, + }); + }; + return { + client, + session, + events, + importBundle, + readCompleted: readCompleted.promise, + closeStarted: closeStarted.promise, + laterImportDispatched: laterImportDispatched.promise, + releaseRead: () => releaseRead.resolve(), + }; +}; + +it("drains a file import before close_binary clears the session", async () => { + const fixture = await connectImportSession(); + const importing = fixture.importBundle("before_close"); + await fixture.readCompleted; + const closing = fixture.client.callTool({ + name: "close_binary", + arguments: {}, + }); + await fixture.closeStarted; + await fixture.client.ping(); + fixture.releaseRead(); + const [imported, closed] = await Promise.all([importing, closing]); + + expect(imported.isError, JSON.stringify(imported)).not.toBe(true); + expect(imported.structuredContent).toMatchObject({ + result: { imported: 1, unknowns_added: 0, total: 1 }, + }); + expect(closed.isError, JSON.stringify(closed)).not.toBe(true); + expect(fixture.events).toEqual(["import", "close"]); + expect(fixture.session.exportEvidenceBundle().records).toEqual([]); + await fixture.client.ping(); +}); + +it("admits a later import after close_binary without retaining the earlier bundle", async () => { + const fixture = await connectImportSession(); + const earlier = fixture.importBundle("before_close"); + await fixture.readCompleted; + const closing = fixture.client.callTool({ + name: "close_binary", + arguments: {}, + }); + await fixture.closeStarted; + const later = fixture.importBundle("after_close"); + await fixture.laterImportDispatched; + await fixture.client.ping(); + fixture.releaseRead(); + const results = await Promise.all([earlier, closing, later]); + + for (const result of results) + expect(result.isError, JSON.stringify(result)).not.toBe(true); + expect(results[2]?.structuredContent).toMatchObject({ + result: { imported: 1, unknowns_added: 0, total: 1 }, + }); + expect(fixture.events).toEqual(["import", "close", "import"]); + expect( + fixture.session + .exportEvidenceBundle() + .records.map((record) => record.operation), + ).toEqual(["after_close"]); + await fixture.client.ping(); +}); From 1a1b67c8885e9ff33f7a06ed2e027ebfdb83e084 Mon Sep 17 00:00:00 2001 From: Aurantium <3550271410@qq.com> Date: Mon, 12 Oct 2026 23:46:28 -0700 Subject: [PATCH 05/27] fix(javascript): preserve POSIX runtime path case on Windows (#2422) Use POSIX path semantics for recorded POSIX roots when reconciling runtime Evidence on Windows. Preserve Windows mapping behavior and cover implicit and explicit file roots through the reconciliation service. Assisted-by: OpenAI GPT-6 Co-authored-by: Aurantium-47 <280805072+Aurantium-47@users.noreply.github.com> --- .../javascriptRuntimeStaticCandidates.ts | 4 +- .../javascriptApplicationWorkflows.test.ts | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts index 1f02c820d..810ab7f5e 100644 --- a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts +++ b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts @@ -1,4 +1,4 @@ -import { posix, relative, win32 } from "node:path"; +import { posix, win32 } from "node:path"; import type { ApplicationNode } from "./javascriptApplicationGraphSchemas.js"; import type { JavaScriptApplicationGraph } from "./javascriptApplicationGraph.js"; @@ -216,7 +216,7 @@ const pathBelowRoot = (root: string, value: string): string | null => { const remainder = rootStyle === "windows" ? win32.relative(root, value) - : relative(root, value); + : posix.relative(root, value); const portableRemainder = remainder.replaceAll("\\", "/"); if ( remainder === "" || diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index 17c3f0048..b79407821 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -21,6 +21,7 @@ import { type ApplicationVersionComparisonResult, } from "../../../src/domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; import { javascriptRuntimeReconciliationResultSchema } from "../../../src/domain/javascript/javascriptRuntimeReconciliationSchemas.js"; +import { electronPageInspectionSchema } from "../../../src/domain/javascript/electronObservation.js"; import { compareJavaScriptApplicationVersions } from "../../../src/domain/javascript/javascriptApplicationVersionComparison.js"; import { createJavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; import { traceApplicationFeature } from "../../../src/domain/javascript/javascriptFeatureTrace.js"; @@ -444,6 +445,73 @@ describe("JavaScript application workflow error diagnostics", () => { }); describe("JavaScript runtime reconciliation local paths", () => { + it.each([ + { + name: "case-sensitive POSIX input root", + root: null, + path: "/applications/Example.app/Contents/Resources/app/renderer.js", + status: "unknown", + }, + { + name: "case-sensitive POSIX mapping root", + root: "/runtime/App", + path: "/runtime/app/renderer.js", + status: "unknown", + }, + { + name: "matching POSIX mapping root", + root: "/runtime/App", + path: "/runtime/App/renderer.js", + status: "matched", + }, + { + name: "case-insensitive Windows mapping root", + root: String.raw`C:\runtime\App`, + path: String.raw`c:\runtime\app\renderer.js`, + status: "matched", + }, + ])("uses $name on every host", ({ root, path, status }) => { + const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; + const original = base.runtime_observations[0]!; + const inspection = electronPageInspectionSchema.parse( + original.normalized_result, + ); + const runtime = createEvidence(undefined, original.provider, { + predicateType: original.predicate_type, + operation: original.operation, + parameters: { ...original.parameters, include_script_sources: false }, + result: { + ...inspection, + scripts: { + ...inspection.scripts, + items: inspection.scripts.items.map((script) => ({ + ...script, + file_path: path, + source: { included: false, reason: "not-requested" }, + })), + }, + }, + confidence: original.confidence, + authority: original.authority, + }); + const reconciled = reconcileJavaScriptRuntimeEvidence({ + static_layers: base.static_layers.map((layer) => ({ + ...layer, + runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], + })), + runtime_observations: [runtime], + }); + if (!reconciled.ok) throw reconciled.error; + const result = javascriptRuntimeReconciliationResultSchema.parse( + reconciled.value.normalized_result, + ); + expect( + result.reconciliations.find( + ({ entity_kind }) => entity_kind === "script", + ), + ).toMatchObject({ status }); + }); + it("resolves relative file mapping roots before deriving result Evidence", () => { const absoluteRoot = resolve(process.cwd(), "runtime-cache-fixture"); const relativeRoot = relative(process.cwd(), absoluteRoot); From c6acd19b6d842699ab807d9526dc762c743686f4 Mon Sep 17 00:00:00 2001 From: Mykola Dementiev Date: Tue, 13 Oct 2026 08:48:17 +0200 Subject: [PATCH 06/27] fix(ida): stop the private input of a universal Mach-O at the end of the selected slice (#2291) * test(ida): expect the private input of a universal Mach-O to end at the selected slice Cover a universal file larger than one read chunk with another slice after the selected one, for both a middle slice and the first slice. * fix(ida): stop the private input of a universal Mach-O at the end of the selected slice A read chunk that starts after the slice produced a negative end offset, and Buffer.subarray counts a negative bound from the end, so the head of the following slice was written to the private input and its digest. --- src/ida/IdaWorkspace.ts | 6 +- src/ida/IdaWorkspaceThinning.test.ts | 93 +++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/src/ida/IdaWorkspace.ts b/src/ida/IdaWorkspace.ts index 0b3c36231..c7d9e642c 100644 --- a/src/ida/IdaWorkspace.ts +++ b/src/ida/IdaWorkspace.ts @@ -85,8 +85,10 @@ export class IdaWorkspace { slice === undefined ? position + bytes.length : Math.min(position + bytes.length, slice.offset + slice.size); - const admitted = bytes.subarray(start - position, end - position); - if (admitted.length > 0) { + // A chunk after the slice has end < position, and a negative subarray + // bound would count from the chunk's end instead of being empty. + if (end > start) { + const admitted = bytes.subarray(start - position, end - position); inputHash.update(admitted); await input.write(admitted); } diff --git a/src/ida/IdaWorkspaceThinning.test.ts b/src/ida/IdaWorkspaceThinning.test.ts index 7feca3ddc..7d83f089b 100644 --- a/src/ida/IdaWorkspaceThinning.test.ts +++ b/src/ida/IdaWorkspaceThinning.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { parseBinaryTarget } from "../application/BinaryTargetResolver.js"; import { parseExecutableHeader } from "../domain/binaryTarget.js"; -import { fat } from "../domain/binaryTarget.fixture.js"; +import { fat, thinMach } from "../domain/binaryTarget.fixture.js"; import { createIdaTarget, RecordingIdaMcp, @@ -35,6 +35,45 @@ const universalTarget = async (hostArchitecture: "arm64" | "x64") => { return { bytes, target: resolved.value }; }; +/** One slice of a synthetic universal file: CPU type, offset and size. */ +interface Layout { + readonly cpu: number; + readonly offset: number; + readonly size: number; +} + +/** + * Write a universal file whose slices span more than one 64 KiB read, each + * filled with its own byte so bytes outside the selected slice are visible. + */ +const largeUniversalTarget = async ( + layout: readonly Layout[], + hostArchitecture: "arm64" | "x64", +) => { + const directory = await createTestTempDirectory("rea-ida-thin-large-"); + roots.push(directory); + const path = join(directory, "universal"); + const total = Math.max(...layout.map(({ offset, size }) => offset + size)); + const bytes = Buffer.alloc(total); + bytes.writeUInt32BE(0xcafebabe, 0); + bytes.writeUInt32BE(layout.length, 4); + layout.forEach(({ cpu, offset, size }, index) => { + const entry = 8 + index * 20; + bytes.writeUInt32BE(cpu, entry); + bytes.writeUInt32BE(offset, entry + 8); + bytes.writeUInt32BE(size, entry + 12); + bytes.writeUInt32BE(14, entry + 16); + bytes.fill(0x11 * (index + 1), offset, offset + size); + thinMach(0xfeedfacf, cpu).copy(bytes, offset); + }); + await writeFile(path, bytes); + const resolved = await parseBinaryTarget(path, { hostArchitecture }); + if (!resolved.ok) throw resolved.error; + if (resolved.value.format !== "mach-o-universal") + throw new Error("the universal fixture did not resolve as a FAT container"); + return { bytes, target: resolved.value }; +}; + describe("IDA workspace input thinning", () => { it.skipIf(process.platform === "win32")( "thins the selected arm64 slice into the private headless input", @@ -78,6 +117,58 @@ describe("IDA workspace input thinning", () => { }, ); + it.skipIf(process.platform === "win32")( + "copies exactly the selected slice when later slices follow it across read boundaries", + async () => { + const { bytes, target } = await largeUniversalTarget( + [ + { cpu: 0x01000007, offset: 4096, size: 8480 }, + { cpu: 0x0100000c, offset: 16384, size: 33464 }, + { cpu: 0x00000007, offset: 65536, size: 33464 }, + ], + "arm64", + ); + const workspace = await IdaWorkspace.allocate(target); + roots.push(workspace.directory); + await workspace.copyInput(); + + const input = await readFile(workspace.inputPath); + const { offset, size } = target.universalSlice; + expect(size, "the arm64 slice is selected").toBe(33464); + expect(input.length, "no byte after the slice is copied").toBe(size); + expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true); + expect(workspace.inputSha256).toBe( + createHash("sha256") + .update(bytes.subarray(offset, offset + size)) + .digest("hex"), + ); + await workspace.remove(); + }, + ); + + it.skipIf(process.platform === "win32")( + "copies exactly the first slice when it is selected and others follow", + async () => { + const { bytes, target } = await largeUniversalTarget( + [ + { cpu: 0x01000007, offset: 16384, size: 37456 }, + { cpu: 0x0100000c, offset: 65536, size: 33464 }, + ], + "x64", + ); + const workspace = await IdaWorkspace.allocate(target); + roots.push(workspace.directory); + await workspace.copyInput(); + + const input = await readFile(workspace.inputPath); + const { offset, size } = target.universalSlice; + expect(size, "the x86_64 slice is selected").toBe(37456); + expect(input.length, "no byte after the slice is copied").toBe(size); + expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true); + await workspace.remove(); + }, + ); + it("keeps a whole-file copy for thin targets", async () => { const { root, target } = await createIdaTarget("arm64"); roots.push(root); From 2c6a4fac4ff3477ac14f3d780894c5b390880607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:48:26 +0800 Subject: [PATCH 07/27] fix(process): accept a trace identifier the comparison result can represent (#2488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trace specification owns the declared identifier vocabulary and the comparison result echoes those identifiers verbatim into raw_trace[].event_id, matched_variant and diagnostic event_ids. The result module restated the shared IDENTIFIER_PATTERN by hand with a 64-character cap, so a specification with a longer identifier validated and then its own comparison result failed to parse: compareProcessTraces threw a raw ZodError at the caller instead of returning a comparison. Export the identifier schema from the specification as the single owner and reuse it in the evaluation module, removing the capped copy. IDENTIFIER_PATTERN has no length bound and every other consumer uses it unbounded, so the duplicate was the outlier. Removing the bound rather than adding one keeps the advertised trace_spec input contract unchanged; identifiers of 64 characters or fewer behave exactly as before. Verified with 48 process tests, 24 boundary CLI/MCP trace tests, check:fast and verify:test-discovery on macOS arm64. The regression test failed with the ZodError before the change and passes after it. Co-authored-by: 吴佳浩 --- .../process/processTraceComparison.test.ts | 28 +++++++++++++++ src/domain/process/processTraceEvaluation.ts | 9 +++-- .../process/processTraceSpecification.ts | 35 ++++++++++++------- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/domain/process/processTraceComparison.test.ts b/src/domain/process/processTraceComparison.test.ts index caeaf7999..2207352e3 100644 --- a/src/domain/process/processTraceComparison.test.ts +++ b/src/domain/process/processTraceComparison.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { parseProcessCapture } from "./processCaptureParsing.js"; import type { ProcessTraceSpecification } from "./processTraceSpecification.js"; +import { processTraceSpecificationSchema as specificationSchema } from "./processTraceSpecification.js"; import { processTraceComparisonResultSchema } from "./processTraceEvaluation.js"; import { capture, @@ -100,6 +101,33 @@ describe("generic capture trace comparison", () => { right: { matched_variant: "two" }, }); }); + + it("compares a declared identifier the comparison result can represent", () => { + // The specification owns the trace identifier vocabulary and the result + // echoes those identifiers verbatim, so both must accept the same values + // rather than handing the caller a raw schema error for its own input. + const eventId = `e${"0".repeat(64)}`; + const specification: ProcessTraceSpecification = { + events: [ + { + id: eventId, + source: "terminal_raw", + exact: terminal, + cardinality: { kind: "required" }, + }, + ], + language: { + kind: "finite_traces", + variants: [{ id: "only", trace: [eventId] }], + }, + }; + expect(specificationSchema.safeParse(specification).success).toBe(true); + const left = capture(values(["terminal", "process"])); + expect(compareProcessTraces(left, left, specification)).toMatchObject({ + verdict: "equivalent", + left: { matched_variant: "only", raw_trace: [{ event_id: eventId }] }, + }); + }); }); describe("process trace constraints and evidence sufficiency", () => { diff --git a/src/domain/process/processTraceEvaluation.ts b/src/domain/process/processTraceEvaluation.ts index 51a2a15f8..7c232a480 100644 --- a/src/domain/process/processTraceEvaluation.ts +++ b/src/domain/process/processTraceEvaluation.ts @@ -13,13 +13,12 @@ import { import { canonicalTraceJson, processTraceCardinalityBounds, + processTraceIdentifierSchema, type ProcessTraceSpecification, } from "./processTraceSpecification.js"; -const identifierSchema = z.string().regex(/^[A-Za-z][A-Za-z0-9._\x2d]{0,63}$/u); - const matchedEventSchema = z.strictObject({ - event_id: identifierSchema, + event_id: processTraceIdentifierSchema, location: processObservationLocationSchema, }); @@ -32,7 +31,7 @@ const sideResultSchema = z.union([ z.strictObject({ ...sideResultShape, status: z.literal("pass"), - matched_variant: identifierSchema.nullable(), + matched_variant: processTraceIdentifierSchema.nullable(), }), z.strictObject({ ...sideResultShape, @@ -53,7 +52,7 @@ const diagnosticSchema = z.strictObject({ ]), side: z.enum(["left", "right"]), message: z.string(), - event_ids: z.array(identifierSchema), + event_ids: z.array(processTraceIdentifierSchema), locations: z.array(processObservationLocationSchema), }); diff --git a/src/domain/process/processTraceSpecification.ts b/src/domain/process/processTraceSpecification.ts index 09c14cf10..4e28774e0 100644 --- a/src/domain/process/processTraceSpecification.ts +++ b/src/domain/process/processTraceSpecification.ts @@ -9,7 +9,16 @@ import { processObservationSourceSchema, } from "./processObservation.js"; -const identifierSchema = z.string().regex(IDENTIFIER_PATTERN); +/** + * A declared trace, event or variant identifier. The specification owns this + * vocabulary and the comparison result echoes those identifiers verbatim, so + * the result module must reuse this schema rather than restate the pattern: + * two bounds on the same identifier would reject the result of an accepted + * specification. + */ +export const processTraceIdentifierSchema = z + .string() + .regex(IDENTIFIER_PATTERN); const cardinalitySchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("required") }), @@ -31,7 +40,7 @@ const cardinalitySchema = z.discriminatedUnion("kind", [ ]); const eventDeclarationSchema = z.strictObject({ - id: identifierSchema, + id: processTraceIdentifierSchema, source: processObservationSourceSchema, exact: jsonValueSchema, ignore_fields: z @@ -56,29 +65,31 @@ const partialOrderLanguageSchema = z.strictObject({ happens_before: z .array( z.strictObject({ - before: identifierSchema, - after: identifierSchema, + before: processTraceIdentifierSchema, + after: processTraceIdentifierSchema, }), ) .default([]), not_before: z .array( z.strictObject({ - event: identifierSchema, - anchor: identifierSchema, + event: processTraceIdentifierSchema, + anchor: processTraceIdentifierSchema, }), ) .default([]), unordered_groups: z - .array(z.strictObject({ events: z.array(identifierSchema).min(2) })) + .array( + z.strictObject({ events: z.array(processTraceIdentifierSchema).min(2) }), + ) .default([]), - prefix: z.array(identifierSchema).default([]), - suffix: z.array(identifierSchema).default([]), + prefix: z.array(processTraceIdentifierSchema).default([]), + suffix: z.array(processTraceIdentifierSchema).default([]), }); const finiteTraceTokenSchema = z.union([ - identifierSchema, - z.strictObject({ unordered: z.array(identifierSchema).min(2) }), + processTraceIdentifierSchema, + z.strictObject({ unordered: z.array(processTraceIdentifierSchema).min(2) }), ]); const finiteTraceLanguageSchema = z.strictObject({ @@ -86,7 +97,7 @@ const finiteTraceLanguageSchema = z.strictObject({ variants: z .array( z.strictObject({ - id: identifierSchema, + id: processTraceIdentifierSchema, trace: z.array(finiteTraceTokenSchema), }), ) From ab62ee9dcf98e4717d0b3445330f7cfdfe50c9f4 Mon Sep 17 00:00:00 2001 From: Stefano Rizzello Date: Tue, 13 Oct 2026 08:49:34 +0200 Subject: [PATCH 08/27] fix(windows): preserve caller affinity only within one processor group (#2202) * fix(windows): preserve caller affinity only within one processor group `spawnProcess` read the caller's affinity mask and applied it to the suspended child unconditionally. A mask names processors relative to one processor group, and Windows assigns each new process to a group of its own choosing, so on hosts with more than 64 logical processors the child is often placed in another group. `SetProcessAffinityMask` then rejects the mask with ERROR_INVALID_PARAMETER (87) and the spawn fails, making every owned provider process start intermittently on such hosts. Affinity is a nicety, not an authority or isolation control, so it is now applied only when the caller and the suspended child both occupy the same single processor group. A child placed in another group keeps that group's default affinity and the spawn proceeds. Job Object membership, kill-on-owner-close policy, owner normalization and priority preservation are unchanged, and the mask failure remains fatal within a shared group, where it is not expected. Single-group hosts, including every host with 64 or fewer logical processors, are unaffected: `GetProcessGroupAffinity` reports one group for both processes and the same mask is applied as before. The optional process-boundary fixture gains an `affinity` observation, and the native verification lane asserts that an owned spawn succeeds and that the child's mask matches the caller's whenever both share a single group. Fixes #2181 Co-Authored-By: Claude Opus 5 * fix(windows): refuse to widen a pinned caller's affinity across groups Addresses review feedback on the first commit. An unconstrained caller loses nothing when Windows places an owned child in another processor group, so that spawn proceeds with the group's default. A caller deliberately pinned to fewer processors than its group holds is a different case: silently accepting the default would hand the provider tree the whole group, which `docs/installation.md` tells operators to rely on affinity to prevent. Such a spawn now fails naming the constraint instead. `GetProcessAffinityMask` already reports the group's active set, which distinguishes the two without another query. Selecting the caller's group at creation through PROC_THREAD_ATTRIBUTE_GROUP_AFFINITY was tried first and rejected: `CreateProcessW` fails with ERROR_INVALID_PARAMETER on a host with a single processor group, both alongside PROC_THREAD_ATTRIBUTE_JOB_LIST and on its own, so it cannot carry the ordinary path. The boundary fixture's affinity observation now reports a process id's own mask rather than its own, because a separate probe process is assigned its own group and cannot stand in for the process that spawns the child. Masks and the group's active set cross as fixed-width hexadecimal strings: a mask can set bits above the safe-integer range, where `JSON.parse` would round two different masks to one value. The verification lane asserts the mask is inherited exactly when caller and child share a group, and otherwise that the caller was unconstrained. Co-Authored-By: Claude Opus 5 * docs(windows): record what owned-process affinity does not preserve A mask covering its group's whole active set does not distinguish a caller the system placed in that group from one that selected the group deliberately, so an owned child can still be placed elsewhere with its processor count bounded and its group placement lost. Treating that case as constrained instead is not available: on a multi-group host an ordinary process is assigned a group by the system and its mask is that group's active set, so refusing it would fail every owned spawn there. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- docs/windows-ghidra-p0.md | 8 +++-- native/windows/README.md | 20 ++++++++++--- native/windows/src/process.cc | 30 ++++++++++++++++++- scripts/verify-windows-native.mjs | 36 +++++++++++++++++++++++ tests/fixtures/windows/processBoundary.cc | 36 +++++++++++++++++++++++ 5 files changed, 123 insertions(+), 7 deletions(-) diff --git a/docs/windows-ghidra-p0.md b/docs/windows-ghidra-p0.md index d14add07c..9cd3bb6d8 100644 --- a/docs/windows-ghidra-p0.md +++ b/docs/windows-ghidra-p0.md @@ -153,8 +153,12 @@ the private runtime, isolating Ghidra preferences, logs, and caches. Process creation assigns a suspended child atomically to an owned Job Object. Membership and kill-on-owner-close policy are verified before resume. Breakaway is disabled. Only the child's default file owner is normalized; -the caller token stays unchanged. Caller CPU affinity and priority are -preserved. Close, cancellation, timeout, startup failure, and provider failure +the caller token stays unchanged. Caller priority is preserved, and caller CPU +affinity whenever the caller and the child share a processor group. A child that +Windows places in another group keeps that group's default when the caller asked +for no bound of its own; a caller pinned to fewer processors than its group +holds is refused there rather than widened. +Close, cancellation, timeout, startup failure, and provider failure terminate the job and verify all members have settled before runtime cleanup. Cleanup removes admitted objects by handle and unlinks reparse entries without following their targets. Abrupt owner death kills the job but can leave diff --git a/native/windows/README.md b/native/windows/README.md index c63540140..e2543260f 100644 --- a/native/windows/README.md +++ b/native/windows/README.md @@ -69,10 +69,22 @@ without this lane reports Windows native controls as unavailable. limits only for its JavaScript helpers when running on Bun, which ignores their V8 heap flags; Node.js launches are unchanged. - Only the suspended child's default file owner is normalized. Caller token - privileges and ownership remain unchanged. Caller CPU affinity and process - priority are preserved in the child. Environment names are sorted and - deduplicated using Windows ordinal comparison, preserving their native Unicode - semantics and caller-selected values. + privileges and ownership remain unchanged. Caller process priority is + preserved in the child, and caller CPU affinity whenever both occupy the same + processor group. Windows chooses a new process's group itself and rejects a + mask belonging to another group, so a child placed elsewhere keeps that + group's default when the caller asked for no bound of its own. A caller + deliberately pinned to fewer processors than its group holds is a constraint + such a child cannot express, and is refused rather than widened to that + group's default. A mask covering its group's whole active set does not + distinguish a caller Windows placed in that group from one that selected the + group deliberately, so such a child can still be placed elsewhere: its + processor count stays bounded, its group placement is not preserved, and + neither that case nor the refusal above has a verification lane, because both + need a host whose processors span more than one group. + Environment names are sorted and deduplicated using Windows + ordinal comparison, preserving their native Unicode semantics and + caller-selected values. - Cleanup walks locked directory objects, removes files by handle, and unlinks reparse entries without traversing their targets. Failed removal retains the root and ancestor leases for retry; a partially removed runtime accepts only diff --git a/native/windows/src/process.cc b/native/windows/src/process.cc index ede2b5622..80a4d0ac5 100644 --- a/native/windows/src/process.cc +++ b/native/windows/src/process.cc @@ -68,6 +68,18 @@ static ULONGLONG jobMemoryLimit(napi_env env, napi_value value) { return limit; } +// An affinity mask is meaningful only inside one processor group, so only a +// process occupying exactly one group has a group a child can inherit. A +// process whose threads span groups, or whose membership cannot be read, has no +// single group. +static bool singleProcessorGroup(HANDLE process, USHORT& group) { + USHORT count = 1; + std::array groups{}; + if (!GetProcessGroupAffinity(process, &count, groups.data()) || count != 1) return false; + group = groups[0]; + return true; +} + static void pipe(Handle& read, Handle& write, bool parentReads = true) { SECURITY_ATTRIBUTES attributes{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; HANDLE rawRead, rawWrite; @@ -168,11 +180,27 @@ std::unique_ptr spawnProcess(const std::wstring& command, const std::ws TOKEN_OWNER owner{reinterpret_cast(tokenUser.data())->User.Sid}; require(SetTokenInformation(token.get(), TokenOwner, &owner, sizeof(owner)), "Set suspended process file owner failed", command); + // A mask names processors inside one processor group, and Windows assigns a + // new process to a group of its own choosing, so a mask belonging to another + // group is rejected (ERROR_INVALID_PARAMETER). An unconstrained caller, whose + // mask is its group's whole active set, asks for no bound the child's own + // group cannot give, so such a child keeps that group's default and the spawn + // proceeds. A narrower mask is a deliberate CPU constraint that a child in + // another group cannot express, and silently widening it would hand the + // provider tree the whole group, so that spawn fails naming the constraint. DWORD_PTR affinity = 0, systemAffinity = 0; require(GetProcessAffinityMask(GetCurrentProcess(), &affinity, &systemAffinity), "Read caller process affinity failed", command); - if (affinity != 0) + USHORT callerGroup = 0, childGroup = 0; + const bool sharedGroup = singleProcessorGroup(GetCurrentProcess(), callerGroup) && + singleProcessorGroup(result->process.get(), childGroup) && + callerGroup == childGroup; + if (affinity != 0 && sharedGroup) require(SetProcessAffinityMask(result->process.get(), affinity), "Preserve caller process affinity failed", command); + else + require(affinity == 0 || affinity == systemAffinity, + "Caller CPU affinity cannot bound an owned process in another processor group", command, + ERROR_INVALID_PARAMETER); const DWORD priority = GetPriorityClass(GetCurrentProcess()); require(priority != 0 && SetPriorityClass(result->process.get(), priority), "Preserve caller process priority failed", command); diff --git a/scripts/verify-windows-native.mjs b/scripts/verify-windows-native.mjs index 3db2613ad..8623081f9 100644 --- a/scripts/verify-windows-native.mjs +++ b/scripts/verify-windows-native.mjs @@ -662,6 +662,42 @@ setInterval(()=>{},1000);\n`, unicodeNamesPreserved: true, }); report.processes.windowsEnvironmentSemantics = true; + // Observed through this process's own id: a separate probe process would + // get its own processor group, so it cannot stand in for the process that + // spawns the child. + const callerAffinity = JSON.parse( + execFileSync(boundaryFixture, ["affinity", String(process.pid)], { + encoding: "utf8", + }), + ); + const affinityChild = native.call("process_spawn", [ + boundaryFixture, + `${quote(boundaryFixture)} affinity`, + workspace, + environment, + ]); + const observedAffinity = await wait(affinityChild); + native.call("process_close", [affinityChild.handle]); + assert.equal(observedAffinity.exitCode, 0); + const childAffinity = JSON.parse(observedAffinity.output); + // Masks cross as hexadecimal strings because a mask can exceed the + // safe-integer range. Sharing one processor group makes the caller's mask + // applicable, so it must be inherited exactly. Windows placing the child in + // another group is only permitted when the caller asked for no bound of its + // own, which its group's whole active set establishes; a narrower mask has + // to fail the spawn rather than widen to that group's default. + const sharedGroup = + callerAffinity.singleProcessorGroup && + childAffinity.singleProcessorGroup && + callerAffinity.processorGroup === childAffinity.processorGroup; + if (sharedGroup) + assert.equal(childAffinity.affinityMask, callerAffinity.affinityMask); + else assert.equal(callerAffinity.affinityMask, callerAffinity.systemMask); + report.processes.callerAffinityBoundsOwnedProcessOrRefusesToWiden = true; + report.callerAffinity = { + ...callerAffinity, + sharedGroupWithChild: sharedGroup, + }; } const ownerFixture = join(workspace, "owner.mjs"); const ownerPidFile = join(workspace, "crash-child.pid"); diff --git a/tests/fixtures/windows/processBoundary.cc b/tests/fixtures/windows/processBoundary.cc index 18dcb2dbd..05ee218e8 100644 --- a/tests/fixtures/windows/processBoundary.cc +++ b/tests/fixtures/windows/processBoundary.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,41 @@ int wmain(int count, wchar_t** args) { value(L"REA_\u00df") == expectedSharp && value(L"REA_SS") == L"double" ? "true" : "false"); return 0; } + if ((count == 2 || count == 3) && std::wstring(args[1]) == L"affinity") { + // Without a process id this observes itself, which is what an owned child + // reports. With one it observes that process, so a caller's own affinity is + // read from the process that spawns the child rather than from a separate + // probe Windows may place in another processor group. + HANDLE opened = nullptr; + if (count == 3) { + opened = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, + static_cast(std::wcstoul(args[2], nullptr, 10))); + if (!opened) return 9; + } + const HANDLE target = opened ? opened : GetCurrentProcess(); + DWORD_PTR affinity = 0, systemAffinity = 0; + USHORT groupCount = 1; + USHORT groups[1]{}; + const bool read = GetProcessAffinityMask(target, &affinity, &systemAffinity) != 0; + const bool single = read && GetProcessGroupAffinity(target, &groupCount, groups) && groupCount == 1; + if (opened) CloseHandle(opened); + if (!read) return 8; + // A mask can set bits above JavaScript's safe-integer range, so masks cross + // as fixed-width hexadecimal strings and are compared without rounding. The + // group's own active set distinguishes an unconstrained process from one + // deliberately pinned to fewer processors. + auto mask = [](DWORD_PTR value, char* text) { + std::snprintf(text, 19, "0x%08lX%08lX", static_cast(value >> 32), + static_cast(value & 0xFFFFFFFFu)); + return text; + }; + char affinityText[19]{}, systemText[19]{}; + std::printf("{\"affinityMask\":\"%s\",\"systemMask\":\"%s\"," + "\"singleProcessorGroup\":%s,\"processorGroup\":%u}\n", + mask(affinity, affinityText), mask(systemAffinity, systemText), + single ? "true" : "false", single ? static_cast(groups[0]) : 0u); + return 0; + } if (count == 4 && std::wstring(args[1]) == L"reparse-directory") { // Only the conformance lane's disposable, nonempty source directory is used. const auto substitute = L"\\??\\" + std::wstring(args[3]); From 68ecf3df8c666fdbe1ced8089c64340af614d1c3 Mon Sep 17 00:00:00 2001 From: chAngE <2815456271@qq.com> Date: Tue, 13 Oct 2026 14:49:47 +0800 Subject: [PATCH 09/27] fix(browser): exclude sensitive binary lengths from historical captures (#2477) Co-authored-by: chAngE <172472630+chAng-L19@users.noreply.github.com> --- bridge/mitmproxy/capture.py | 2 +- .../fixtures/generate-mitmproxy-capture.py | 14 +++ src/browser/history/HarCapture.ts | 3 +- .../browser/historicalWebCapture.test.ts | 88 +++++++++++++++++++ 4 files changed, 105 insertions(+), 2 deletions(-) diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.py index fa0535c02..dcd90e745 100644 --- a/bridge/mitmproxy/capture.py +++ b/bridge/mitmproxy/capture.py @@ -93,7 +93,7 @@ def project_record(state, secrets): safe_bytes, credential_url = transport_value(value, path, transport_url) content_base64 = base64.b64encode(value).decode("ascii") digest = hashlib.sha256(value).hexdigest() - hidden = any(secret in value or secret in safe_bytes for secret in byte_secrets) or any(secret in content_base64 or secret in digest for secret in secrets) + hidden = any(secret in value or secret in safe_bytes for secret in byte_secrets) or any(secret in content_base64 or secret in digest or secret in str(len(value)) for secret in secrets) safe_text = None try: safe_text = safe_bytes.decode("utf-8", errors="strict") diff --git a/scripts/fixtures/generate-mitmproxy-capture.py b/scripts/fixtures/generate-mitmproxy-capture.py index 4be3dd4a7..880c5e690 100644 --- a/scripts/fixtures/generate-mitmproxy-capture.py +++ b/scripts/fixtures/generate-mitmproxy-capture.py @@ -45,6 +45,20 @@ class Generator: assert "ows-secret" not in json.dumps(projected), projected if isinstance(value, bytes): assert projected["binary_fields"][-1]["state"] == "redacted", projected + body = b"A" * 12345 + state = {"response": {"content": body}} + control = adapter.project_record(state, []) + field = control["binary_fields"][0] + assert field["state"] == "retained" and field["bytes"] == len(body), control + for literal in ["12345", "2345"]: + # Only the generated byte count matches this exclusion. + assert literal not in control["reported"]["response"]["content"] + assert literal not in field["content_base64"] and literal not in field["sha256"] + projected = adapter.project_record(state, [literal]) + assert projected["reported"]["response"]["content"] is None, projected + assert projected["binary_fields"] == [{"pointer": "/response/content", "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None}], projected + assert {"pointer": "/response/content", "reason": "explicit-sensitive-value"} in projected["redactions"], projected + assert literal not in json.dumps(projected), projected (root / "native-error-classifications.json").write_text(json.dumps({"passed": len(classifications) + 2})) first = http.HTTPFlow(connection.Client(peername=("127.0.0.1", 1), sockname=("127.0.0.1", 2)), connection.Server(address=("example.test", 80))) first.id = "original-producer-id" diff --git a/src/browser/history/HarCapture.ts b/src/browser/history/HarCapture.ts index 83b442f5c..81c5ffb15 100644 --- a/src/browser/history/HarCapture.ts +++ b/src/browser/history/HarCapture.ts @@ -144,7 +144,8 @@ const projectHar = ( (literal) => item.includes(literal) || contentBase64.includes(literal) || - sha256.includes(literal), + sha256.includes(literal) || + String(bytes.length).includes(literal), ); binary_fields.push( hidden diff --git a/tests/boundary/browser/historicalWebCapture.test.ts b/tests/boundary/browser/historicalWebCapture.test.ts index a0a9f10e2..781172cd3 100644 --- a/tests/boundary/browser/historicalWebCapture.test.ts +++ b/tests/boundary/browser/historicalWebCapture.test.ts @@ -7,6 +7,9 @@ import { execFile } from "node:child_process"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { parseEvidence } from "../../../src/domain/evidence.js"; +import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const entrypoint = fileURLToPath( new URL("../../../scripts/rea.mjs", import.meta.url), ); @@ -41,6 +44,91 @@ const cli = async ( }; import { historicalHar } from "../../fixtures/historicalHar.js"; +it("excludes declared binary lengths through CLI, MCP and retained capture pages", async () => { + const root = await createTestTempDirectory("rea-har-byte-length-"); + const path = join(root, "capture.har"); + const fixture = historicalHar(); + const entry = fixture.log.entries[0]; + if (entry === undefined) throw new Error("fixture missing"); + const body = Buffer.alloc(12_345, "A"); + const content = body.toString("base64"); + const digest = createHash("sha256").update(body).digest("hex"); + await writeFile( + path, + JSON.stringify({ + log: { + ...fixture.log, + entries: [ + { + ...entry, + response: { + ...entry.response, + content: { + size: 99, + mimeType: "text/plain", + encoding: "base64", + text: content, + }, + }, + }, + ], + }, + }), + ); + const { call } = await connectLocalToolsMcp(); + const args = { capture_path: path, format: "har" }; + const control = await call("inspect_web_network_capture", args); + expect(control.isError).not.toBe(true); + const retained = parseEvidence(control.structuredContent); + const controlCapture = webNetworkCaptureSchema.parse( + retained.normalized_result, + ); + expect(controlCapture.records[0]?.binary_fields[0]).toMatchObject({ + state: "retained", + bytes: body.length, + content_base64: content, + sha256: digest, + }); + for (const literal of ["12345", "2345"]) { + // Only the derived count matches, so this cannot pass via another exclusion. + expect(content).not.toContain(literal); + expect(digest).not.toContain(literal); + const fromCli = await cli(path, "har", ["--sensitive-value", literal]); + expect(fromCli.ok, fromCli.stdout).toBe(true); + const evidence = parseEvidence(JSON.parse(fromCli.stdout)); + const response = await call("inspect_web_network_capture", { + ...args, + sensitive_values: [literal], + }); + expect(response.isError).not.toBe(true); + const fromMcp = parseEvidence(response.structuredContent); + expect(fromMcp.normalized_result).toEqual(evidence.normalized_result); + const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); + expect(capture.records[0]?.reported).toMatchObject({ + response: { content: { size: 99, text: null } }, + }); + expect(capture.records[0]?.binary_fields[0]).toEqual({ + pointer: "/response/content/text", + representation: "har-base64-content", + state: "redacted", + bytes: null, + content_base64: null, + sha256: null, + }); + for (const detail of ["preview", "complete"]) { + const page = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: fromMcp.evidence_id }, + view: { kind: "capture", detail }, + }); + expect(page.isError).not.toBe(true); + const result = parseEvidence(page.structuredContent); + expect(result.normalized_result).toMatchObject({ + items: [{ binary_fields: [{ state: "redacted", bytes: null }] }], + }); + } + } +}, 60_000); + it("reads actual upstream HAR packages in an owned process and retains selected producer order", async () => { const root = await mkdtemp(join(tmpdir(), "rea-history-har-")); onTestFinished(() => rm(root, { recursive: true, force: true })); From e74e4e951bf59c42b25779b45d1c7b8dde6b618f Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:50:44 -0400 Subject: [PATCH 10/27] fix(screenshot): reject invalid PNG header dimensions (#2472) --- src/domain/pngHeader.test.ts | 21 +++++++++++++ src/domain/pngHeader.ts | 8 ++++- src/native/NativeUiObservation.test.ts | 31 +++++++++++++++++++ .../cdpScreenshotDocumentIdentity.test.ts | 31 +++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/domain/pngHeader.test.ts b/src/domain/pngHeader.test.ts index e8df531bc..676a832e0 100644 --- a/src/domain/pngHeader.test.ts +++ b/src/domain/pngHeader.test.ts @@ -33,3 +33,24 @@ it("rejects incomplete headers, invalid signatures and zero dimensions", () => { expect(readPngHeaderDimensions(zero)).toBeUndefined(); } }); + +it.each([0, 12, 14, 0xffff_ffff])("rejects IHDR data length %s", (length) => { + const bytes = Buffer.from(png); + bytes.writeUInt32BE(length, 8); + expect(readPngHeaderDimensions(bytes)).toBeUndefined(); +}); + +it.each([16, 20])("rejects out-of-range PNG dimension at %s", (offset) => { + const bytes = Buffer.from(png); + bytes.writeUInt32BE(0x8000_0000, offset); + expect(readPngHeaderDimensions(bytes)).toBeUndefined(); +}); + +it("accepts the largest dimension representable by the PNG specification", () => { + const bytes = Buffer.from(png); + bytes.writeUInt32BE(0x7fff_ffff, 16); + expect(readPngHeaderDimensions(bytes)).toEqual({ + width: 0x7fff_ffff, + height: 1, + }); +}); diff --git a/src/domain/pngHeader.ts b/src/domain/pngHeader.ts index 58cdd7d3f..5ab8c7c3d 100644 --- a/src/domain/pngHeader.ts +++ b/src/domain/pngHeader.ts @@ -8,10 +8,16 @@ export const readPngHeaderDimensions = ( if ( bytes.byteLength < 24 || !bytes.subarray(0, 8).equals(PNG_SIGNATURE) || + bytes.readUInt32BE(8) !== 13 || !bytes.subarray(12, 16).equals(IHDR_TYPE) ) return undefined; const width = bytes.readUInt32BE(16); const height = bytes.readUInt32BE(20); - return width === 0 || height === 0 ? undefined : { width, height }; + return width === 0 || + height === 0 || + width > 0x7fff_ffff || + height > 0x7fff_ffff + ? undefined + : { width, height }; }; diff --git a/src/native/NativeUiObservation.test.ts b/src/native/NativeUiObservation.test.ts index d2238bd01..527afd8db 100644 --- a/src/native/NativeUiObservation.test.ts +++ b/src/native/NativeUiObservation.test.ts @@ -35,6 +35,37 @@ const validPng = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jv4sAAAAASUVORK5CYII=", "base64", ); +it.each([ + { offset: 8, value: 12 }, + { offset: 16, value: 0x8000_0000 }, + { offset: 20, value: 0x8000_0000 }, +])( + "rejects invalid IHDR metadata with a matching digest", + async ({ offset, value }) => { + const bytes = Buffer.from(validPng); + bytes.writeUInt32BE(value, offset); + const screenshot = { + mime_type: "image/png", + base64: bytes.toString("base64"), + sha256: createHash("sha256").update(bytes).digest("hex"), + width: bytes.readUInt32BE(16), + height: bytes.readUInt32BE(20), + }; + const result = await observeNativeUi( + target, + "observe_native_ui", + { ...scope, screenshot: true }, + { + environment: {}, + invoke: async () => ({ + ok: true, + result: { ...snapshot, screenshot }, + }), + }, + ); + expect(result.ok).toBe(false); + }, +); describe("native UI screenshot validation", () => { it("rejects malformed or mismatched screenshot bytes from the helper", async () => { const screenshot = { diff --git a/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts b/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts index 012ad9663..b3fcbc592 100644 --- a/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts +++ b/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts @@ -3,6 +3,37 @@ import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; import { captureWebScreenshotInputSchema } from "../../../src/domain/webScreenshot.js"; import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; describe("screenshot committed document identity", () => { + it.each([ + { offset: 8, value: 14 }, + { offset: 16, value: 0x8000_0000 }, + { offset: 20, value: 0x8000_0000 }, + ])("rejects invalid screenshot IHDR metadata", async ({ offset, value }) => { + const bytes = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jv4sAAAAASUVORK5CYII=", + "base64", + ); + bytes.writeUInt32BE(value, offset); + const browser = await startFakeCdpBrowser({ + commandResult: ({ method }) => + method === "Page.captureScreenshot" + ? { data: bytes.toString("base64") } + : undefined, + }); + onTestFinished(async () => { + await browser.close(); + }); + const result = await new CdpBrowserProvider().captureScreenshot( + captureWebScreenshotInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + }), + ); + expect(result).toMatchObject({ + ok: false, + error: { reason: "protocol_error" }, + }); + }); it.each(["loader-main", "loader-reloaded"])( "distinguishes %s for the same URL", async (screenshotDocumentLoader) => { From 6a04f922e16ad4d171ba0c07e32238efdaf43e2d Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:51:23 -0400 Subject: [PATCH 11/27] fix(javascript): parse explicit CommonJS wrapper syntax (#2475) --- .../javascript/javascriptSourceParser.ts | 59 +++++++++++++------ ...eferenceSourceTypeScriptExtensions.test.ts | 25 ++++++++ .../process/javascriptSourceModes.test.ts | 39 ++++++++++++ 3 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 tests/boundary/process/javascriptSourceModes.test.ts diff --git a/src/domain/javascript/javascriptSourceParser.ts b/src/domain/javascript/javascriptSourceParser.ts index ce5619ead..630d5e9bf 100644 --- a/src/domain/javascript/javascriptSourceParser.ts +++ b/src/domain/javascript/javascriptSourceParser.ts @@ -37,10 +37,19 @@ export const parserPluginsForPath = (sourcePath?: string): ParserPlugin[] => { : ["decorators-legacy", typescript]; }; -/** `.mjs` and `.mts` are modules by extension; other paths leave the mode to syntax. */ +/** `.mjs` and `.mts` explicitly select module mode. */ export const isExplicitModuleSourcePath = (path?: string): boolean => /\.m[jt]s$/iu.test(path ?? ""); +const runtimeSourceTypeForPath = ( + path?: string, +): "commonjs" | "module" | "unambiguous" => + /\.cjs$/iu.test(path ?? "") + ? "commonjs" + : isExplicitModuleSourcePath(path) + ? "module" + : "unambiguous"; + export interface RecoveredJavaScriptParse { readonly file: ParsedJavaScriptSource; /** Errors that apply only if a source of unknown mode is an ES module. */ @@ -58,25 +67,25 @@ export const parseRecoveredJavaScript = ( sourcePath: string | undefined, options: Omit, ): RecoveredJavaScriptParse => { - // `.cjs` is CommonJS by extension, so module-only errors cannot apply. - if (/\.cjs$/iu.test(sourcePath ?? "")) + const sourceType = runtimeSourceTypeForPath(sourcePath); + if (sourceType !== "unambiguous") { + const normalizedOptions = { ...options }; + if (sourceType === "commonjs") { + // Babel owns these wrapper semantics in CommonJS mode and rejects + // explicit script overrides, including the reference-import options. + delete normalizedOptions.allowAwaitOutsideFunction; + delete normalizedOptions.allowReturnOutsideFunction; + delete normalizedOptions.allowNewTargetOutsideFunction; + } return { file: parse(source, { - ...options, - sourceType: "script", - errorRecovery: true, - }), - moduleOnlyErrors: [], - }; - if (isExplicitModuleSourcePath(sourcePath)) - return { - file: parse(source, { - ...options, - sourceType: "module", + ...normalizedOptions, + sourceType, errorRecovery: true, }), moduleOnlyErrors: [], }; + } const file = parse(source, { ...options, sourceType: "unambiguous", @@ -98,18 +107,34 @@ export const parseRecoveredJavaScript = ( }; }; -/** Parse JavaScript or TypeScript once without attaching comments to AST nodes. */ +/** Parse inert JavaScript or TypeScript facts without attaching comments to AST nodes. */ export const parseJavaScriptSource = ( source: string, sourcePath?: string, ): ParsedJavaScriptSource | null => { try { - return parse(source, { - sourceType: "unambiguous", + const runtimeSourceType = runtimeSourceTypeForPath(sourcePath); + const commonjs = runtimeSourceType === "commonjs"; + const options = { errorRecovery: true, attachComment: false, plugins: parserPluginsForPath(sourcePath), + }; + const file = parse(source, { + ...options, + // Inert artifact analysis retains module declarations even in .cjs assets. + // Infer their syntax here; reference imports keep filename-mode diagnostics. + sourceType: commonjs ? "unambiguous" : runtimeSourceType, + allowReturnOutsideFunction: commonjs, + allowNewTargetOutsideFunction: commonjs, }); + // Unambiguous mode retains module-only diagnostics after choosing a script. + // The .cjs extension establishes wrapper mode, so obtain its own diagnostics. + return commonjs && + file.program.sourceType === "script" && + file.errors.length > 0 + ? parse(source, { ...options, sourceType: "commonjs" }) + : file; } catch (cause: unknown) { // Unparseable source is represented by the null return. void cause; diff --git a/tests/boundary/filesystem/referenceSourceTypeScriptExtensions.test.ts b/tests/boundary/filesystem/referenceSourceTypeScriptExtensions.test.ts index d537a1dca..7b3d8189c 100644 --- a/tests/boundary/filesystem/referenceSourceTypeScriptExtensions.test.ts +++ b/tests/boundary/filesystem/referenceSourceTypeScriptExtensions.test.ts @@ -6,6 +6,31 @@ import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { importReferenceSource } from "../../support/referenceSourceResourceScope.js"; +it("imports valid CommonJS wrapper syntax without inventing a parse failure", async () => { + const root = await createTestTempDirectory("rea-reference-commonjs-mode-"); + await Promise.all([ + writeFile( + join(root, "main.cjs"), + 'void new.target; require("./dep.cjs"); return;\n', + ), + writeFile(join(root, "dep.cjs"), "module.exports = 1;\n"), + ]); + const imported = await importReferenceSource({ + root, + caller: "native-commonjs-source", + policy: { secretPatterns: [] }, + }); + if (!imported.ok) throw imported.error; + expect(imported.value.parse_failures).toEqual([]); + expect(imported.value.relationships).toContainEqual({ + from_path: "main.cjs", + to: "dep.cjs", + kind: "requires", + resolution: "internal", + parse_state: "parsed", + }); +}); + it.each([ ["main.ts", "dep.js", "dep.ts"], ["main.mts", "dep.mjs", "dep.mts"], diff --git a/tests/boundary/process/javascriptSourceModes.test.ts b/tests/boundary/process/javascriptSourceModes.test.ts new file mode 100644 index 000000000..64c15a2d8 --- /dev/null +++ b/tests/boundary/process/javascriptSourceModes.test.ts @@ -0,0 +1,39 @@ +import { expect } from "vitest"; + +import { parseJavaScriptSource } from "../../../src/domain/javascript/javascriptSourceParser.js"; +import { processTest } from "../../support/process/processFixture.js"; + +for (const source of [ + "return;", + "void new.target;", + "var await = 1; return;", + "with ({ value: 1 }) { void value; } void new.target;", +]) { + processTest( + `accepts Node CommonJS syntax while retaining module errors: ${source}`, + async ({ processes }) => { + const commonjs = await processes.run( + process.execPath, + ["--check", "--input-type=commonjs"], + { input: source, timeoutMs: 5_000 }, + ); + expect(commonjs.exitCode).toBe(0); + const module = await processes.run( + process.execPath, + ["--check", "--input-type=module"], + { input: source, timeoutMs: 5_000 }, + ); + expect(module.exitCode).not.toBe(0); + + const parsed = parseJavaScriptSource(source, "main.cjs"); + expect(parsed).not.toBeNull(); + expect(parsed?.errors).toEqual([]); + expect( + parseJavaScriptSource(source, "main.mjs")?.errors.length, + ).toBeGreaterThan(0); + expect( + parseJavaScriptSource(source, "main.mts")?.errors.length, + ).toBeGreaterThan(0); + }, + ); +} From a07ceaec36346a7c73e66efe35c8af4654029c84 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:51:29 -0400 Subject: [PATCH 12/27] fix(config): reject NUL bytes in Hopper loader arguments (#2474) --- src/config.test.ts | 12 ++++++++++++ src/config/parsers.ts | 13 +++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/config.test.ts b/src/config.test.ts index eb784802f..67db7e111 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -3,6 +3,18 @@ import { describe, expect, it } from "vitest"; import { parseConfig } from "./config/parseConfig.js"; import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; +it("reports a configuration failure for a NUL in Hopper loader arguments", () => { + const result = parseConfig({ + HOPPER_LOADER_ARGS_JSON: JSON.stringify(["--option", "value\0suffix"]), + }); + if (result.ok) + throw new Error("NUL argv cannot be passed to a child process"); + expect(result.error).toMatchObject({ + settings: [{ setting: "HOPPER_LOADER_ARGS_JSON" }], + }); + expect(result.error.message).toContain("NUL"); +}); + describe("runtime configuration", () => { it("allows target-free startup and applies runtime defaults", () => { const empty = parseConfig({}); diff --git a/src/config/parsers.ts b/src/config/parsers.ts index fc842f884..52feae622 100644 --- a/src/config/parsers.ts +++ b/src/config/parsers.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { ConfigurationError } from "../domain/configurationErrors.js"; +import { nulFreeStringSchema } from "../domain/localPath.js"; import { err, ok, type Result } from "../domain/result.js"; import { safeParseJson } from "../domain/safeJson.js"; @@ -20,7 +21,8 @@ export const parseLoaderArgs = ( return parseConfiguredStringArray( encoded, "HOPPER_LOADER_ARGS_JSON", - z.string(), + nulFreeStringSchema, + "must encode an array of strings without NUL bytes", ); }; @@ -28,6 +30,7 @@ const parseConfiguredStringArray = ( encoded: string, setting: string, itemSchema: z.ZodString, + requirement = "must encode an array of strings", ): Result => { const decoded = safeParseJson(encoded); if (!decoded.ok) @@ -37,13 +40,7 @@ const parseConfiguredStringArray = ( const parsed = z.array(itemSchema).safeParse(decoded.value); return parsed.success ? ok(parsed.data) - : err( - configuredArrayError( - setting, - "must encode an array of strings", - parsed.error, - ), - ); + : err(configuredArrayError(setting, requirement, parsed.error)); }; const configuredArrayError = ( From f6775a36694a6a9c620f4248638e8056a6cbcb66 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:51:55 +0800 Subject: [PATCH 13/27] fix(setup): preserve current recovery snapshots and decoding failures --- docs/installation.md | 4 + src/application/RegularFileRead.ts | 65 ++-- src/application/Setup.recovery.test.ts | 34 ++ src/application/SetupClientConfiguration.ts | 175 +++++++++- src/application/SetupClients.ts | 18 +- src/application/SetupHost.ts | 8 +- src/application/SetupPlan.ts | 2 +- src/application/SetupTypes.ts | 10 + src/application/Uninstall.ts | 12 +- .../clientConfigurationDiagnosticsCli.test.ts | 17 +- .../clientConfigurationTransactions.test.ts | 80 ++++- .../setupClientConfigCleanupOwnership.test.ts | 227 +++++++----- .../setupConfigurationRollback.test.ts | 325 ++++++++++++++++++ 13 files changed, 835 insertions(+), 142 deletions(-) create mode 100644 tests/process-global/setupConfigurationRollback.test.ts diff --git a/docs/installation.md b/docs/installation.md index 0a8041898..3bd4e0928 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -57,6 +57,10 @@ Hopper installation or any file write. Declining or pressing Ctrl-C makes no changes. Agent configuration writes preserve unrelated entries and comments, create backups, use atomic replacement, and verify their result. Setup and uninstall retain an existing `.rea.backup` rather than replacing the first snapshot. +If that snapshot differs from the current configuration, setup previews and +preserves a verified `.rea.backup..recovery` copy of the current bytes. +Both snapshots remain available; a failed rollback reports the current recovery +copy's path and the restore failure. Setup also retains valid environment overrides in REA's existing server entry, such as `REA_MCP_INPUT_SCHEMA_PROFILE` and `REA_MCP_MAX_RESPONSE_BYTES`. Detected provider settings replace only matching environment keys. diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index f09f4c192..ee4d6170a 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -19,6 +19,17 @@ export interface RegularFileReadOptions { readonly resources?: ArtifactResourceScope; } +export class InvalidUtf8FileError extends TypeError { + readonly _tag = "InvalidUtf8FileError"; + constructor( + readonly path: string, + cause: unknown, + ) { + super(`File is not valid UTF-8: ${path}`, { cause }); + this.name = this._tag; + } +} + export type RegularFileReadOutcome = | { readonly kind: "completed"; readonly value: Value } | { readonly kind: "failed"; readonly cause: unknown }; @@ -142,7 +153,34 @@ const withRegularFileOwned = async ( export const readRegularFile = ( path: string, options: RegularFileReadOptions = {}, -): Promise => +): Promise => readRegularFileContent(path, (bytes) => bytes, options); + +/** Read valid UTF-8 without replacing source bytes or stripping an initial BOM. */ +export const readRegularFileText = ( + path: string, + options: RegularFileReadOptions = {}, +): Promise => + readRegularFileContent( + path, + (bytes) => { + try { + return new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true, + }).decode(bytes); + } catch (cause: unknown) { + if (!(cause instanceof TypeError)) throw cause; + throw new InvalidUtf8FileError(path, cause); + } + }, + options, + ); + +const readRegularFileContent = ( + path: string, + decode: (bytes: Buffer) => Value, + options: RegularFileReadOptions, +): Promise => withRegularFile( path, async (handle, stats) => { @@ -153,30 +191,7 @@ export const readRegularFile = ( ); if (bytes === undefined || bytes.length !== stats.size) throw new RegularFileChangedError(path); - return bytes; + return decode(bytes); }, options, ); - -/** Read one admitted regular file as UTF-8 without waiting on a pipe. */ -export const readRegularFileText = async ( - path: string, - options: RegularFileReadOptions = {}, -): Promise => { - try { - return (await readRegularFile(path, options)).toString("utf8"); - } catch (cause: unknown) { - if ( - cause instanceof RegularFileCleanupFailure && - cause.outcome.kind === "completed" && - Buffer.isBuffer(cause.outcome.value) - ) - throw new RegularFileCleanupFailure( - cause.path, - cause.owner, - { kind: "completed", value: cause.outcome.value.toString("utf8") }, - cause.cleanupCause, - ); - throw cause; - } -}; diff --git a/src/application/Setup.recovery.test.ts b/src/application/Setup.recovery.test.ts index 951ef3a54..f29c9d375 100644 --- a/src/application/Setup.recovery.test.ts +++ b/src/application/Setup.recovery.test.ts @@ -54,6 +54,40 @@ describe("setup workflow", () => { expect(result.remediation).toContain("could not be updated"); }); + it("reports a failed config rollback with its destination and recovery backup", async () => { + const host = new FakeSetupHost(); + host.hopper = "/Applications/Hopper"; + host.clients = [{ name: "codex", configPath: "/fixture/codex.json" }]; + host.clientResults.set("codex", { + status: "failed", + reason: "rollback", + backupPath: "/fixture/codex.json.rea.backup", + failure: { + kind: "restore", + path: "/fixture/codex.json", + detail: "permission denied", + }, + }); + + const result = await runSetup( + { ...options(true), clientIds: ["codex"] }, + host, + ); + + expect(result.clients.codex).toEqual({ + status: "failed", + reason: "rollback", + backupPath: "/fixture/codex.json.rea.backup", + failure: { + kind: "restore", + path: "/fixture/codex.json", + detail: "permission denied", + }, + }); + expect(result.remediation).toContain("permission denied"); + expect(result.remediation).toContain("/fixture/codex.json.rea.backup"); + }); + it("explains skill installation recovery", async () => { const host = new FakeSetupHost(); host.hopper = "/Applications/Hopper"; diff --git a/src/application/SetupClientConfiguration.ts b/src/application/SetupClientConfiguration.ts index bfab80cbf..981b829a4 100644 --- a/src/application/SetupClientConfiguration.ts +++ b/src/application/SetupClientConfiguration.ts @@ -19,6 +19,7 @@ import { } from "./ClientConfigurationDocument.js"; import { dshDocument, DshCordisDocument } from "./DshCordisPatch.js"; import { constants as fsConstants } from "node:fs"; +import { createHash } from "node:crypto"; import { copyFile, mkdir, rm, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -29,6 +30,7 @@ import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js"; import { MCP_STARTUP_POLICY } from "../mcpStartupPolicy.js"; import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js"; import { + readRegularFile, readRegularFileText, RegularFileCleanupFailure, } from "./RegularFileRead.js"; @@ -40,6 +42,26 @@ import type { } from "./SetupTypes.js"; import type { SetupClient } from "./SupportedClients.js"; +export class ClientConfigurationCleanupRollbackFailure extends RegularFileCleanupFailure { + constructor( + failure: RegularFileCleanupFailure, + readonly configurationFailure: Extract< + ClientConfigurationResult, + { readonly status: "failed"; readonly reason: "rollback" } + >, + ) { + super( + failure.path, + failure.owner, + failure.outcome, + new AggregateError( + [failure.cleanupCause, new Error(configurationFailure.failure.detail)], + `Configuration readback cleanup and rollback failed: ${configurationFailure.failure.path}`, + ), + ); + } +} + /** Configure one supported client's native stdio MCP registration shape. */ export const configureClientConfiguration = ( client: SetupClient, @@ -167,11 +189,23 @@ const persistClientConfiguration = async ({ }: ClientConfigurationUpdate): Promise => { const backupPath = original === undefined ? undefined : `${client.configPath}.rea.backup`; - if ( - backupPath !== undefined && - !(await preserveConfigBackup(transactionPath, backupPath)) - ) - return { status: "failed", reason: "backup" }; + let recoveryBackupPath: string | undefined; + if (backupPath !== undefined && original !== undefined) { + const selection = await selectCurrentRecoveryBackup(backupPath, original); + if (!selection.ok) return { status: "failed", reason: "backup" }; + if ( + !(await preserveConfigBackup(transactionPath, selection.value.backupPath)) + ) + return { status: "failed", reason: "backup" }; + const verified = await selectCurrentRecoveryBackup(backupPath, original); + if ( + !verified.ok || + verified.value.backupPath !== selection.value.backupPath || + !verified.value.present + ) + return { status: "failed", reason: "backup" }; + recoveryBackupPath = verified.value.backupPath; + } // A native OpenCode V2 entry replaces REA's legacy V1 entry, which would // otherwise conflict with it. const legacyPath = legacyClientServerPath( @@ -239,19 +273,39 @@ const persistClientConfiguration = async ({ !clientConfigurationValuesEqual(readback.servers[name], entry), ) ) { - await restoreConfig(transactionPath, original); - return { status: "failed", reason: "readback" }; + return rollbackOrReadbackFailure( + transactionPath, + original, + recoveryBackupPath, + ); } } catch (cause: unknown) { - if (cause instanceof RegularFileCleanupFailure) throw cause; + if (cause instanceof RegularFileCleanupFailure) { + const rollback = await restoreConfig(transactionPath, original); + if (!rollback.ok) + throw new ClientConfigurationCleanupRollbackFailure(cause, { + status: "failed", + reason: "rollback", + ...(recoveryBackupPath === undefined + ? {} + : { backupPath: recoveryBackupPath }), + failure: rollback.error, + }); + throw cause; + } // Readback failure restores the transaction before reporting. void cause; - await restoreConfig(transactionPath, original); - return { status: "failed", reason: "readback" }; + return rollbackOrReadbackFailure( + transactionPath, + original, + recoveryBackupPath, + ); } return { status: "configured", - ...(backupPath === undefined ? {} : { backupPath }), + ...(recoveryBackupPath === undefined + ? {} + : { backupPath: recoveryBackupPath }), }; }; @@ -406,9 +460,18 @@ export const inspectClientConfiguration = async ( "The existing configuration is malformed. Repair it before setup applies any other changes.", }; } + const backupSelection = await selectCurrentRecoveryBackup( + `${client.configPath}.rea.backup`, + original, + ); + if (!backupSelection.ok) + return { + status: "invalid", + remediation: backupSelection.error.detail, + }; return { status: "update", - backupPath: `${client.configPath}.rea.backup`, + backupPath: backupSelection.value.backupPath, }; }; @@ -473,19 +536,101 @@ const preserveConfigBackup = async ( } }; +interface RecoveryBackupSelection { + readonly backupPath: string; + readonly present: boolean; +} + +interface RecoveryBackupSelectionFailure { + readonly kind: "unreadable" | "occupied"; + readonly path: string; + readonly detail: string; +} + +const selectCurrentRecoveryBackup = async ( + firstBackupPath: string, + original: string, +): Promise> => { + const expected = Buffer.from(original, "utf8"); + let firstBackup: Buffer | undefined; + try { + firstBackup = await readRegularFile(firstBackupPath, { + symlinks: "reject", + }); + } catch (cause: unknown) { + if (cause instanceof RegularFileCleanupFailure) throw cause; + if (!isMissing(cause)) + return err({ + kind: "unreadable", + path: firstBackupPath, + detail: `The recovery backup could not be checked: ${errorDetail(cause)}`, + }); + } + if (firstBackup === undefined) + return ok({ backupPath: firstBackupPath, present: false }); + if (firstBackup.equals(expected)) + return ok({ backupPath: firstBackupPath, present: true }); + + const digest = createHash("sha256").update(original, "utf8").digest("hex"); + const recoveryPath = `${firstBackupPath}.${digest}.recovery`; + let recovery: Buffer | undefined; + try { + recovery = await readRegularFile(recoveryPath, { symlinks: "reject" }); + } catch (cause: unknown) { + if (cause instanceof RegularFileCleanupFailure) throw cause; + if (!isMissing(cause)) + return err({ + kind: "unreadable", + path: recoveryPath, + detail: `The recovery backup could not be checked: ${errorDetail(cause)}`, + }); + } + if (recovery !== undefined && !recovery.equals(expected)) + return err({ + kind: "occupied", + path: recoveryPath, + detail: `The recovery path already contains different data: ${recoveryPath}`, + }); + return ok({ backupPath: recoveryPath, present: recovery !== undefined }); +}; + const restoreConfig = async ( path: string, original: string | undefined, -): Promise => { +): Promise< + Result< + void, + { readonly kind: "restore"; readonly path: string; readonly detail: string } + > +> => { try { if (original === undefined) await rm(path, { force: true }); else await writeFile(path, original, { encoding: "utf8", mode: 0o600 }); + return ok(undefined); } catch (cause: unknown) { - // The backup remains available for the remediation reported by setup. - void cause; + return err({ kind: "restore", path, detail: errorDetail(cause) }); } }; +const rollbackOrReadbackFailure = async ( + targetPath: string, + original: string | undefined, + backupPath: string | undefined, +): Promise => { + const rollback = await restoreConfig(targetPath, original); + return rollback.ok + ? { status: "failed", reason: "readback" } + : { + status: "failed", + reason: "rollback", + ...(backupPath === undefined ? {} : { backupPath }), + failure: rollback.error, + }; +}; + +const errorDetail = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + /** Whether REA's entry matches, no legacy entry remains, and no client disable list suppresses it. */ const registrationCurrent = ( parsed: ClientConfigurationDocument, diff --git a/src/application/SetupClients.ts b/src/application/SetupClients.ts index bbeeedf3f..d86d04b9e 100644 --- a/src/application/SetupClients.ts +++ b/src/application/SetupClients.ts @@ -6,15 +6,19 @@ import type { } from "./SetupTypes.js"; import type { SetupClient } from "./SupportedClients.js"; -const failedConfigurationMessage = ( - reason: "path" | "backup" | "write" | "readback", +export const failedConfigurationMessage = ( + result: Extract, ): string => { - if (reason === "path") + if (result.reason === "path") return "Agent configuration path could not be safely verified. Check its permissions and, if it is a symbolic link, verify that the link resolves to a regular file owned by the current user, then rerun setup."; - if (reason === "backup") + if (result.reason === "backup") return "Agent configuration could not be backed up, so no change was made. Check file permissions, then rerun setup."; - if (reason === "write") + if (result.reason === "write") return "Agent configuration could not be updated. Check file permissions, then rerun setup."; + if (result.reason === "rollback") + return result.backupPath === undefined + ? `Agent configuration rollback failed at ${result.failure.path}: ${result.failure.detail}; the failed first-run file may remain. Remove or repair it before rerunning setup.` + : `Agent configuration rollback failed at ${result.failure.path}: ${result.failure.detail}. Inspect the preserved backup at ${result.backupPath} and restore the intended configuration, then rerun setup.`; return "Agent configuration could not be verified after writing. Repair the configuration file or restore its `.rea.backup`, then rerun setup."; }; @@ -40,12 +44,12 @@ export const configureDetectedClients = async (options: { ); options.clients[client.name] = result; if (result.status === "failed") { - failure ??= failedConfigurationMessage(result.reason); + failure ??= failedConfigurationMessage(result); options.onProgress?.({ actionId, label, state: "failed", - detail: failedConfigurationMessage(result.reason), + detail: failedConfigurationMessage(result), }); } else if (result.status === "configured") { options.appliedActions.push(`configured_${client.name}`); diff --git a/src/application/SetupHost.ts b/src/application/SetupHost.ts index 9e2ae95f1..7ac6828fd 100644 --- a/src/application/SetupHost.ts +++ b/src/application/SetupHost.ts @@ -33,9 +33,11 @@ import { } from "./SetupSkill.js"; import { clientConfigurationAligned, + ClientConfigurationCleanupRollbackFailure, configureClientConfiguration, inspectClientConfiguration, } from "./SetupClientConfiguration.js"; +import { failedConfigurationMessage } from "./SetupClients.js"; import { setupInstallFailure } from "./SetupInstallFailure.js"; import { RegularFileCleanupFailure, @@ -275,8 +277,12 @@ const runClientConfigurationOperation = async ( cleanup.outcome.kind === "failed" ? cleanup.outcome.cause : cause.cleanupCause; + const rollbackMessage = + cause instanceof ClientConfigurationCleanupRollbackFailure + ? ` ${failedConfigurationMessage(cause.configurationFailure)}` + : ""; throw configurationCleanupError( - `Could not safely access client configuration at ${cause.path}: ${errorMessage(primary)}`, + `Could not safely access client configuration at ${cause.path}: ${errorMessage(primary)}.${rollbackMessage}`, cause, cleanup.cleanup, ); diff --git a/src/application/SetupPlan.ts b/src/application/SetupPlan.ts index 83402d1a4..a78697c80 100644 --- a/src/application/SetupPlan.ts +++ b/src/application/SetupPlan.ts @@ -320,5 +320,5 @@ const clientConfigurationDetail = ( ? " If this plan installs Hopper, REA will add HOPPER_LAUNCHER_PATH using the launcher path reported by that installation." : " The HOPPER_LAUNCHER_PATH value shown above is the current fallback; if this plan installs Hopper, REA will replace it with the launcher path reported by that installation." : ""; - return `Add the REA MCP registration for ${client}; preserve unrelated configuration. Command: ${command.map((part) => JSON.stringify(part)).join(" ")}.${environmentDetail}${hopperDetail}`; + return `Add the REA MCP registration for ${client}; preserve unrelated configuration. The backup path reflects the configuration inspected for this plan; if that file changes before execution, setup will select a recovery path for its current contents. Command: ${command.map((part) => JSON.stringify(part)).join(" ")}.${environmentDetail}${hopperDetail}`; }; diff --git a/src/application/SetupTypes.ts b/src/application/SetupTypes.ts index da58abbff..241e0f6dd 100644 --- a/src/application/SetupTypes.ts +++ b/src/application/SetupTypes.ts @@ -26,6 +26,16 @@ export type ClientConfigurationResult = | { readonly status: "failed"; readonly reason: "path" | "backup" | "write" | "readback"; + } + | { + readonly status: "failed"; + readonly reason: "rollback"; + readonly backupPath?: string; + readonly failure: { + readonly kind: "restore"; + readonly path: string; + readonly detail: string; + }; }; /** Read-only state used to block unsafe setup plans before any mutation. */ diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index c410280bc..a7f051a49 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -25,7 +25,11 @@ import { import { isOwnedClientRegistrationCommand } from "./ClientRegistrationIdentity.js"; import { dshDocument } from "./DshCordisPatch.js"; import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js"; -import { readRegularFileText } from "./RegularFileRead.js"; +import { + InvalidUtf8FileError, + RegularFileCleanupFailure, + readRegularFileText, +} from "./RegularFileRead.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { manualRegistrationRemediation, @@ -269,13 +273,17 @@ const readClientConfiguration = async ( try { original = await fileSystem.readText(transactionPath); } catch (cause: unknown) { + const readFailure = + cause instanceof RegularFileCleanupFailure ? cause.cause : cause; return itemRead( isMissing(cause) ? item(client.name, "skipped", "Configuration does not exist.") : item( client.name, "failed", - "Configuration could not be read. Check file permissions, then rerun uninstall.", + readFailure instanceof InvalidUtf8FileError + ? "Configuration is not valid UTF-8 and was not changed. Repair it, then rerun uninstall." + : "Configuration could not be read. Check file permissions, then rerun uninstall.", ), ); } diff --git a/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts b/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts index d9363988f..425e0e610 100644 --- a/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts +++ b/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts @@ -6,6 +6,17 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; const cases = [ + { + client: "cursor", + path: ".cursor/mcp.json", + text: "malformed UTF-8", + bytes: Buffer.concat([ + Buffer.from('{"label":"'), + Buffer.from([0xff]), + Buffer.from('"}'), + ]), + diagnostic: "not valid UTF-8", + }, { client: "cursor", path: ".cursor/mcp.json", @@ -92,7 +103,9 @@ for (const scenario of cases) ); const config = join(home, scenario.path); await mkdir(dirname(config), { recursive: true }); - await writeFile(config, scenario.text); + const original = + "bytes" in scenario ? scenario.bytes : Buffer.from(scenario.text); + await writeFile(config, original); const environment = { HOME: home, USERPROFILE: home, @@ -146,7 +159,7 @@ for (const scenario of cases) }); if (!("syntax" in scenario)) expect(uninstall.stdout).not.toContain("not valid JSON"); - expect(await readFile(config, "utf8")).toBe(scenario.text); + expect(await readFile(config)).toEqual(original); expect(await readdir(dirname(config))).toEqual([ scenario.path.split("/").at(-1), ]); diff --git a/tests/boundary/filesystem/clientConfigurationTransactions.test.ts b/tests/boundary/filesystem/clientConfigurationTransactions.test.ts index 477085381..507b7137e 100644 --- a/tests/boundary/filesystem/clientConfigurationTransactions.test.ts +++ b/tests/boundary/filesystem/clientConfigurationTransactions.test.ts @@ -1,13 +1,91 @@ import { chmod, readFile, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parse as parseToml } from "smol-toml"; -import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; +import { + configureClientConfiguration, + inspectClientConfiguration, +} from "../../../src/application/SetupClientConfiguration.js"; import { systemUninstallHost } from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +it("accepts a literal UTF-8 replacement character without changing its value", async () => { + const root = await createTestTempDirectory("rea-client-config-utf8-"); + const configPath = join(root, "config.json"); + const original = Buffer.concat([ + Buffer.from('{"label":"'), + Buffer.from("\uFFFD"), + Buffer.from('"}'), + ]); + await writeFile(configPath, original); + const result = await configureClientConfiguration( + { name: "cursor", format: "json", configPath }, + {}, + ["rea", "mcp"], + ); + expect(result.status).toBe("configured"); + expect(await readFile(`${configPath}.rea.backup`)).toEqual(original); + expect(JSON.parse(await readFile(configPath, "utf8")).label).toBe("\uFFFD"); +}); + +it("previews and creates a current recovery backup while preserving the first backup", async () => { + const root = await createTestTempDirectory("rea-client-current-recovery-"); + const configPath = join(root, "config.json"); + const firstOriginal = '{"keep":"first"}\n'; + const currentOriginal = '{"keep":"current"}\n'; + const client = { name: "cursor", format: "json", configPath } as const; + await writeFile(configPath, currentOriginal); + await writeFile(`${configPath}.rea.backup`, firstOriginal); + + const recoveryPath = `${configPath}.rea.backup.${createHash("sha256") + .update(currentOriginal, "utf8") + .digest("hex")}.recovery`; + expect(await inspectClientConfiguration(client, {}, ["rea", "mcp"])).toEqual({ + status: "update", + backupPath: recoveryPath, + }); + + expect( + await configureClientConfiguration(client, {}, ["rea", "mcp"]), + ).toEqual({ status: "configured", backupPath: recoveryPath }); + expect(await readFile(`${configPath}.rea.backup`, "utf8")).toBe( + firstOriginal, + ); + expect(await readFile(recoveryPath, "utf8")).toBe(currentOriginal); +}); + +it("blocks configuration when the planned content-addressed backup is occupied", async () => { + const root = await createTestTempDirectory("rea-client-recovery-conflict-"); + const configPath = join(root, "config.json"); + const firstOriginal = '{"keep":"first"}\n'; + const currentOriginal = '{"keep":"current"}\n'; + const client = { name: "cursor", format: "json", configPath } as const; + await writeFile(configPath, currentOriginal); + await writeFile(`${configPath}.rea.backup`, firstOriginal); + const recoveryPath = `${configPath}.rea.backup.${createHash("sha256") + .update(currentOriginal, "utf8") + .digest("hex")}.recovery`; + await writeFile(recoveryPath, "occupied by other data"); + + expect( + await inspectClientConfiguration(client, {}, ["rea", "mcp"]), + ).toMatchObject({ + status: "invalid", + remediation: expect.stringContaining(recoveryPath), + }); + expect( + await configureClientConfiguration(client, {}, ["rea", "mcp"]), + ).toEqual({ status: "failed", reason: "backup" }); + expect(await readFile(configPath, "utf8")).toBe(currentOriginal); + expect(await readFile(`${configPath}.rea.backup`, "utf8")).toBe( + firstOriginal, + ); + expect(await readFile(recoveryPath, "utf8")).toBe("occupied by other data"); +}); + describe("client configuration write failures", () => { it.skipIf(process.platform === "win32" || process.getuid?.() === 0).each([ ["json", configureClientConfiguration], diff --git a/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts b/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts index 3bf33c312..ab2b5e5a7 100644 --- a/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts +++ b/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts @@ -11,6 +11,7 @@ import { } from "../../../src/application/SetupHost.js"; import { RegularFileCleanupFailure } from "../../../src/application/RegularFileRead.js"; import { configureDetectedClients } from "../../../src/application/SetupClients.js"; +import { systemUninstallHost } from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; vi.mock("node:fs/promises", async (importOriginal) => { @@ -31,6 +32,61 @@ beforeEach(async () => { afterEach(() => openMock.mockReset()); +const holdReadHandle = (path: string, reason: string) => { + const closeAttempts: object[] = []; + let released = false; + openMock.mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) !== path) return handle; + const nativeClose = handle.close.bind(handle); + vi.spyOn(handle, "close").mockImplementation(async () => { + closeAttempts.push(handle); + if (!released) throw new Error(reason); + await nativeClose(); + }); + return handle; + }); + return { + closeAttempts, + release: () => { + released = true; + }, + }; +}; + +it("retains invalid UTF-8 diagnostics and the uninstall read owner when close fails", async () => { + const home = await createTestTempDirectory("rea-uninstall-utf8-close-"); + const path = join(home, "client.json"); + const original = Buffer.from([0xff]); + await writeFile(path, original); + const { closeAttempts, release } = holdReadHandle( + path, + "held uninstall handle", + ); + const host = systemUninstallHost(home, undefined, { HOME: home }, "linux"); + try { + expect( + await host.removeClient({ + name: "cursor", + format: "json", + configPath: path, + }), + ).toMatchObject({ + status: "failed", + detail: expect.stringContaining("not valid UTF-8"), + }); + expect(await readFile(path)).toEqual(original); + expect(closeAttempts).toHaveLength(2); + release(); + expect(await host.close?.()).toBeUndefined(); + expect(closeAttempts).toHaveLength(3); + expect(closeAttempts[2]).toBe(closeAttempts[0]); + } finally { + release(); + await host.close?.(); + } +}); + describe("ZCode fallback cleanup ownership", () => { it.each(["preflight", "configure"])( "retains a failed ZCode fallback close during %s until host cleanup", @@ -49,19 +105,10 @@ describe("ZCode fallback cleanup ownership", () => { await mkdir(dirname(fallback), { recursive: true }); const original = '{"mcpServers":{"other":{"command":"other"}}}\n'; await writeFile(fallback, original); - const closeAttempts: object[] = []; - let allowClose = false; - openMock.mockImplementation(async (...args) => { - const handle = await actualFs.open(...args); - if (String(args[0]) !== fallback) return handle; - const nativeClose = handle.close.bind(handle); - vi.spyOn(handle, "close").mockImplementation(async () => { - closeAttempts.push(handle); - if (!allowClose) throw new Error("held fallback handle"); - await nativeClose(); - }); - return handle; - }); + const { closeAttempts, release } = holdReadHandle( + fallback, + "held fallback handle", + ); try { const action = operation === "preflight" @@ -79,12 +126,12 @@ describe("ZCode fallback cleanup ownership", () => { await expect(readFile(client.configPath)).rejects.toMatchObject({ code: "ENOENT", }); - allowClose = true; + release(); expect(await host.close?.()).toBeUndefined(); expect(closeAttempts).toHaveLength(3); expect(closeAttempts[2]).toBe(closeAttempts[0]); } finally { - allowClose = true; + release(); await host.close?.(); } }, @@ -92,83 +139,87 @@ describe("ZCode fallback cleanup ownership", () => { }); describe("setup client configuration cleanup ownership", () => { - it("stops client configuration and retries the same alignment handle from host close", async () => { - const home = await createTestTempDirectory("rea-setup-config-close-"); - const environment = { - HOME: home, - XDG_CONFIG_HOME: join(home, "xdg"), - }; - const host = systemSetupHost( - { platform: "linux", homeDirectory: home } as DoctorHost, - environment, - ); - const client = (await host.supportedClients()).find( - ({ name }) => name === "vscode", - ); - if (client === undefined) throw new Error("VS Code setup client missing"); - await mkdir(dirname(client.configPath), { recursive: true }); - const original = Buffer.from('{"servers": {"other": {}}}\n'); - await writeFile(client.configPath, original); + it.each([ + ["valid UTF-8", Buffer.from('{"servers": {"other": {}}}\n')], + ["malformed UTF-8", Buffer.from([0xff])], + ] as const)( + "stops configuration and retains the %s read owner until host close", + async (encoding, original) => { + const home = await createTestTempDirectory("rea-setup-config-close-"); + const environment = { + HOME: home, + XDG_CONFIG_HOME: join(home, "xdg"), + }; + const host = systemSetupHost( + { platform: "linux", homeDirectory: home } as DoctorHost, + environment, + ); + const client = (await host.supportedClients()).find( + ({ name }) => name === "vscode", + ); + if (client === undefined) throw new Error("VS Code setup client missing"); + await mkdir(dirname(client.configPath), { recursive: true }); + await writeFile(client.configPath, original); - const closeAttempts: object[] = []; - let allowClose = false; - openMock.mockImplementation(async (...args) => { - const handle = await actualFs.open(...args); - if (String(args[0]) !== client.configPath) return handle; - const nativeClose = handle.close.bind(handle); - vi.spyOn(handle, "close").mockImplementation(async () => { - closeAttempts.push(handle); - if (!allowClose) throw new Error("test holds the config handle open"); - await nativeClose(); - }); - return handle; - }); - const configure = vi.spyOn(host, "configureClient"); + const { closeAttempts, release } = holdReadHandle( + client.configPath, + "test holds the config handle open", + ); + const configure = vi.spyOn(host, "configureClient"); - try { - const failure = await filterClientsNeedingConfigure(host, [client], {}, [ - "rea", - "mcp", - ]) - .then((detectedClients) => - configureDetectedClients({ - host, - detectedClients, - providerEnvironment: {}, - command: ["rea", "mcp"], - clients: {}, - appliedActions: [], - }), + try { + const failure = await filterClientsNeedingConfigure( + host, + [client], + {}, + ["rea", "mcp"], ) - .then( - () => undefined, - (cause: unknown) => cause, - ); - expect(failure).toBeInstanceOf(ConfigurationError); - expect(failure).toMatchObject({ - _tag: "ConfigurationError", - cause: expect.any(RegularFileCleanupFailure), - cleanup: { - resources: [client.configPath], - reason: expect.stringContaining("test holds the config handle open"), - }, - }); - expect(configure).not.toHaveBeenCalled(); - expect(closeAttempts).toHaveLength(2); - expect(closeAttempts[0]).toBe(closeAttempts[1]); - expect(await readFile(client.configPath)).toEqual(original); + .then((detectedClients) => + configureDetectedClients({ + host, + detectedClients, + providerEnvironment: {}, + command: ["rea", "mcp"], + clients: {}, + appliedActions: [], + }), + ) + .then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(ConfigurationError); + if (encoding === "malformed UTF-8") + expect(failure).toMatchObject({ + cause: { + outcome: { kind: "failed", cause: expect.any(TypeError) }, + }, + }); + expect(failure).toMatchObject({ + _tag: "ConfigurationError", + cause: expect.any(RegularFileCleanupFailure), + cleanup: { + resources: [client.configPath], + reason: expect.stringContaining( + "test holds the config handle open", + ), + }, + }); + expect(configure).not.toHaveBeenCalled(); + expect(closeAttempts).toHaveLength(2); + expect(closeAttempts[0]).toBe(closeAttempts[1]); + expect(await readFile(client.configPath)).toEqual(original); - allowClose = true; - const close = host.close; - if (close === undefined) throw new Error("Setup host close is missing"); - expect(await close()).toBeUndefined(); - expect(closeAttempts).toHaveLength(3); - expect(closeAttempts[2]).toBe(closeAttempts[0]); - } finally { - allowClose = true; - await host.close?.(); - } - }); + release(); + expect(await host.close?.()).toBeUndefined(); + expect(closeAttempts).toHaveLength(3); + expect(closeAttempts[2]).toBe(closeAttempts[0]); + } finally { + release(); + await host.close?.(); + } + }, + ); it("waits for an admitted configuration read and rejects later admissions", async () => { const home = await createTestTempDirectory("rea-setup-config-inflight-"); diff --git a/tests/process-global/setupConfigurationRollback.test.ts b/tests/process-global/setupConfigurationRollback.test.ts new file mode 100644 index 000000000..cf6504c32 --- /dev/null +++ b/tests/process-global/setupConfigurationRollback.test.ts @@ -0,0 +1,325 @@ +import { open, readFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; + +import { afterEach, expect, it, onTestFinished, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + readbackFailed: false, + rejectRestore: false, + rejectRemove: false, + removeCopiedRecovery: false, + initialCleanupFailure: undefined as unknown, + readbackCleanupFailure: undefined as unknown, +})); + +afterEach(() => { + state.readbackFailed = false; + state.rejectRestore = false; + state.rejectRemove = false; + state.removeCopiedRecovery = false; + state.initialCleanupFailure = undefined; + state.readbackCleanupFailure = undefined; +}); + +vi.mock("node:fs/promises", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + rm: ( + ...arguments_: Parameters + ): ReturnType => { + if (state.rejectRemove && state.readbackFailed) + return Promise.reject(new Error("injected restore remove failure")); + return original.rm(...arguments_); + }, + writeFile: ( + ...arguments_: Parameters + ): ReturnType => { + if (state.rejectRestore && state.readbackFailed) + return Promise.reject(new Error("injected restore write failure")); + return original.writeFile(...arguments_); + }, + copyFile: async ( + ...arguments_: Parameters + ): Promise => { + await original.copyFile(...arguments_); + const destination = arguments_[1]; + if ( + state.removeCopiedRecovery && + typeof destination === "string" && + destination.endsWith(".recovery") + ) + await original.rm(destination); + }, + }; +}); + +vi.mock("write-file-atomic", async (importOriginal) => { + const original = await importOriginal<{ + default: typeof import("write-file-atomic"); + }>(); + return { + ...original, + default: async (...arguments_: Parameters) => { + if (state.initialCleanupFailure !== undefined) + throw new Error("unexpected config write before initial read"); + const result = await original.default(...arguments_); + state.readbackFailed = true; + return result; + }, + }; +}); + +vi.mock("../../src/application/RegularFileRead.js", async (importOriginal) => { + const original = + await importOriginal< + typeof import("../../src/application/RegularFileRead.js") + >(); + return { + ...original, + readRegularFileText: ( + ...arguments_: Parameters + ) => + state.initialCleanupFailure !== undefined + ? Promise.reject(state.initialCleanupFailure) + : state.readbackFailed && state.readbackCleanupFailure !== undefined + ? Promise.reject(state.readbackCleanupFailure) + : state.readbackFailed + ? Promise.reject(new Error("injected readback failure")) + : original.readRegularFileText(...arguments_), + }; +}); + +import { configureClientConfiguration } from "../../src/application/SetupClientConfiguration.js"; +import { RegularFileCleanupFailure } from "../../src/application/RegularFileRead.js"; +import { OwnedFileHandle } from "../../src/filesystem/OwnedFileHandle.js"; +import type { DoctorHost } from "../../src/application/Doctor.js"; +import { systemSetupHost } from "../../src/application/SetupHost.js"; +import { ConfigurationError } from "../../src/domain/configurationErrors.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +const ownReadHandle = async (path: string) => { + const owner = new OwnedFileHandle(await open(path, "r")); + onTestFinished(() => owner.close()); + return owner; +}; + +it("reports a failed config rollback and retains its backup path", async () => { + state.readbackFailed = false; + state.rejectRestore = false; + state.rejectRemove = false; + state.initialCleanupFailure = undefined; + state.readbackCleanupFailure = undefined; + const root = await createTestTempDirectory("rea-config-rollback-"); + const path = join(root, "client.json"); + const original = '{"keep":true}\n'; + await import("node:fs/promises").then(({ writeFile }) => + writeFile(path, original), + ); + state.rejectRestore = true; + + const result = await configureClientConfiguration( + { name: "cursor", configPath: path, format: "json" }, + {}, + ["rea", "mcp"], + ); + + expect(result).toEqual({ + status: "failed", + reason: "rollback", + backupPath: `${path}.rea.backup`, + failure: { + kind: "restore", + path, + detail: "injected restore write failure", + }, + }); + expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(original); + expect(JSON.parse(await readFile(path, "utf8"))).toMatchObject({ + mcpServers: { rea: { command: "rea", args: ["mcp"] } }, + }); +}); + +it("preserves the first backup and reports a verified backup of the current original", async () => { + state.readbackFailed = false; + state.rejectRestore = false; + state.rejectRemove = false; + state.initialCleanupFailure = undefined; + state.readbackCleanupFailure = undefined; + const root = await createTestTempDirectory("rea-config-stale-backup-"); + const path = join(root, "client.json"); + const firstOriginal = '{"keep":"first"}\n'; + const currentOriginal = '{"keep":"current"}\n'; + await import("node:fs/promises").then(async ({ writeFile }) => { + await writeFile(path, currentOriginal); + await writeFile(`${path}.rea.backup`, firstOriginal); + }); + state.rejectRestore = true; + + const result = await configureClientConfiguration( + { name: "cursor", configPath: path, format: "json" }, + {}, + ["rea", "mcp"], + ); + state.rejectRestore = false; + + const recoveryPath = `${path}.rea.backup.${createHash("sha256") + .update(currentOriginal, "utf8") + .digest("hex")}.recovery`; + expect(result).toMatchObject({ + status: "failed", + reason: "rollback", + backupPath: recoveryPath, + failure: { + kind: "restore", + path, + detail: "injected restore write failure", + }, + }); + expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(firstOriginal); + expect(await readFile(recoveryPath, "utf8")).toBe(currentOriginal); +}); + +it("does not write config when a recovery copy disappears before verification", async () => { + state.readbackFailed = false; + state.rejectRestore = false; + state.rejectRemove = false; + state.removeCopiedRecovery = false; + state.initialCleanupFailure = undefined; + state.readbackCleanupFailure = undefined; + const root = await createTestTempDirectory("rea-config-vanished-recovery-"); + const path = join(root, "client.json"); + const firstOriginal = '{"keep":"first"}\n'; + const currentOriginal = '{"keep":"current"}\n'; + await import("node:fs/promises").then(async ({ writeFile }) => { + await writeFile(path, currentOriginal); + await writeFile(`${path}.rea.backup`, firstOriginal); + }); + state.removeCopiedRecovery = true; + + const result = await configureClientConfiguration( + { name: "cursor", configPath: path, format: "json" }, + {}, + ["rea", "mcp"], + ); + state.removeCopiedRecovery = false; + + expect(result).toEqual({ status: "failed", reason: "backup" }); + expect(await readFile(path, "utf8")).toBe(currentOriginal); + expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(firstOriginal); +}); + +it("does not roll back or remove config when its initial read cleanup fails", async () => { + state.readbackFailed = false; + state.rejectRestore = false; + state.rejectRemove = false; + state.readbackCleanupFailure = undefined; + const root = await createTestTempDirectory("rea-config-initial-cleanup-"); + const path = join(root, "client.json"); + const original = '{"keep":true}\n'; + await import("node:fs/promises").then(({ writeFile }) => + writeFile(path, original), + ); + const failure = new RegularFileCleanupFailure( + path, + await ownReadHandle(path), + { kind: "completed", value: original }, + new Error("injected initial close failure"), + ); + state.initialCleanupFailure = failure; + + await expect( + configureClientConfiguration( + { name: "cursor", configPath: path, format: "json" }, + {}, + ["rea", "mcp"], + ), + ).rejects.toBe(failure); + expect(await readFile(path, "utf8")).toBe(original); +}); + +it("reports first-run rollback removal failure without a backup", async () => { + state.readbackFailed = false; + state.rejectRestore = false; + state.rejectRemove = false; + state.initialCleanupFailure = undefined; + state.readbackCleanupFailure = undefined; + const root = await createTestTempDirectory("rea-config-first-run-rollback-"); + const path = join(root, "client.json"); + state.rejectRemove = true; + + const result = await configureClientConfiguration( + { name: "cursor", configPath: path, format: "json" }, + {}, + ["rea", "mcp"], + ); + state.rejectRemove = false; + + expect(result).toMatchObject({ + status: "failed", + reason: "rollback", + failure: { + kind: "restore", + path, + detail: "injected restore remove failure", + }, + }); + expect(result).not.toHaveProperty("backupPath"); + expect(JSON.parse(await readFile(path, "utf8"))).toMatchObject({ + mcpServers: { rea: { command: "rea", args: ["mcp"] } }, + }); +}); + +it("retains readback cleanup ownership when rollback also fails", async () => { + state.readbackFailed = false; + state.rejectRestore = false; + state.rejectRemove = false; + state.initialCleanupFailure = undefined; + const root = await createTestTempDirectory("rea-config-readback-cleanup-"); + const path = join(root, "client.json"); + const original = '{"keep":true}\n'; + await import("node:fs/promises").then(({ writeFile }) => + writeFile(path, original), + ); + const owner = await ownReadHandle(path); + const cleanupFailure = new RegularFileCleanupFailure( + path, + owner, + { kind: "completed", value: "" }, + new Error("injected readback close failure"), + ); + state.readbackCleanupFailure = cleanupFailure; + state.rejectRestore = true; + + const host = systemSetupHost( + { platform: "linux", homeDirectory: root } as DoctorHost, + { HOME: root }, + ); + let caught: unknown; + try { + await host.configureClient( + { name: "cursor", configPath: path, format: "json" }, + {}, + ["rea", "mcp"], + ); + } catch (cause: unknown) { + caught = cause; + } finally { + await host.close?.(); + } + + if (!(caught instanceof ConfigurationError)) throw caught; + expect(caught.message).toContain("injected restore write failure"); + expect(caught.message).toContain(`${path}.rea.backup`); + const cleanupOwnerError = caught.cause; + if (!(cleanupOwnerError instanceof RegularFileCleanupFailure)) + throw new Error("Configuration failure lost its read owner"); + expect(cleanupOwnerError).toMatchObject({ + path, + outcome: { kind: "completed", value: "" }, + cleanupCause: expect.any(AggregateError), + }); + expect(cleanupOwnerError.owner).toBe(owner); + expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(original); +}); From 15463bdbe605b9dd11da5dbba031f99cd1ec9190 Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 13 Oct 2026 02:52:48 -0400 Subject: [PATCH 14/27] fix(adb): count requested permissions that the device marks restricted (#2359) Android 10 and 11 print restricted entries of the requested-permissions list as "NAME: restricted=true". The parser took the first such line as the end of the list, so the count stopped there while coverage still read complete. Co-authored-by: Claude Opus 5.5 --- src/adb/AdbProvider.test.ts | 28 ++++++++++++++++++++++++++++ src/adb/AdbShellOutput.ts | 6 +++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index b89ab9dfd..ae81353c7 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -626,6 +626,34 @@ it("projects package details from dumpsys package output", async () => { }); }); +// Android 10 and 11 append ": restricted=true" to hard- and soft-restricted +// entries of the requested-permissions list (Settings.dumpPackageLPr). +it("counts requested permissions that the device marks as restricted", async () => { + const { binary } = await writeAdbStub( + shellStub({ + "dumpsys:package": `Packages: + Package [com.example] (abcdef): + userId=10234 + versionName=1.2.3 + requested permissions: + android.permission.INTERNET + android.permission.WRITE_EXTERNAL_STORAGE: restricted=true + android.permission.CAMERA + install permissions: + android.permission.INTERNET: granted=true +`, + }), + ); + const result = await evidenceResult(providerOver(binary), { + operation: "inspect_adb_package", + input: { serial: "emulator-5554", package: "com.example" }, + }); + expect(result).toMatchObject({ + requested_permissions_count: 3, + coverage: "complete", + }); +}); + it("reports an uninstalled package from dumpsys refusal output", async () => { const { binary } = await writeAdbStub( shellStub({ "dumpsys:package": "Unable to find package: com.absent\n" }), diff --git a/src/adb/AdbShellOutput.ts b/src/adb/AdbShellOutput.ts index f3323e7b2..726e6164d 100644 --- a/src/adb/AdbShellOutput.ts +++ b/src/adb/AdbShellOutput.ts @@ -236,7 +236,11 @@ export const parseDumpsysPackageOutput = ( inPermissions = true; continue; } - if (inPermissions && /^\s{2,}[A-Za-z0-9._]+$/.test(line)) { + // Android 10 and 11 mark restricted entries as "NAME: restricted=true". + if ( + inPermissions && + /^\s{2,}[A-Za-z0-9._]+(?:: restricted=true)?$/u.test(line) + ) { permissions += 1; continue; } From b9ee921560dcf4590c2b13a1cc92c49c93e10675 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:54:18 -0400 Subject: [PATCH 15/27] fix(process): preserve nested path identities during normalization (#2468) --- src/domain/process/processScenario.ts | 3 + .../capture/ProcessNormalization.test.ts | 44 ++++++++++++++ src/process/capture/ProcessNormalization.ts | 48 +++++++++++---- .../processCaptureNormalization.test.ts | 60 ++++++++++++++++++- 4 files changed, 141 insertions(+), 14 deletions(-) diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts index a87ffd86e..e98d7a537 100644 --- a/src/domain/process/processScenario.ts +++ b/src/domain/process/processScenario.ts @@ -128,6 +128,9 @@ export const processComparisonContract = ( : {}), limits: scenario.limits, normalization: scenario.normalization, + ...(scenario.normalization.paths + ? { path_normalization_version: "longest-original-paths-v1" } + : {}), ...(scenario.normalization.ports ? { port_normalization_version: "contextual-endpoints-v2" } : {}), diff --git a/src/process/capture/ProcessNormalization.test.ts b/src/process/capture/ProcessNormalization.test.ts index 9c9ba1951..e5ef61062 100644 --- a/src/process/capture/ProcessNormalization.test.ts +++ b/src/process/capture/ProcessNormalization.test.ts @@ -266,3 +266,47 @@ it("normalizes representative process identities and preserves null, zero, and u "/program child 42020 parent 41010 group 43030 session 44040 listen: selected", ); }); + +it("normalizes nested paths once using the most specific selected identity", () => { + const declared = parseProcessScenario({ + executable: "/workspace/project/bin/program", + working_directory: "/workspace/project", + filesystem_observation_paths: [ + "/workspace/project/data", + "/workspace/project/data/cache", + ], + normalization: { paths: true, pids: false, ports: false, patterns: [] }, + }); + expect( + normalizeProcessText( + "/workspace/project/bin/program /workspace/project/data/cache/item /workspace/project/data/result /workspace/project/log /workspace/tmp", + declared, + "/workspace", + rootPid, + ), + ).toBe( + " /item /result /log /tmp", + ); +}); + +it("treats path metacharacters literally while preferring nested observed roots", () => { + const declared = parseProcessScenario({ + executable: "/selected/[tool]+", + working_directory: "/selected", + filesystem_observation_paths: [ + "/selected/(data)", + "/selected/(data)/cache", + ], + normalization: { paths: true, pids: false, ports: false, patterns: [] }, + }); + expect( + normalizeProcessText( + "/selected/[tool]+ /selected/(data)/cache/item /selected/(data)/file /selected/log /selected/toolll", + declared, + "/tmp", + rootPid, + ), + ).toBe( + " /item /file /log /toolll", + ); +}); diff --git a/src/process/capture/ProcessNormalization.ts b/src/process/capture/ProcessNormalization.ts index 9364ec95b..74db31b1c 100644 --- a/src/process/capture/ProcessNormalization.ts +++ b/src/process/capture/ProcessNormalization.ts @@ -17,19 +17,8 @@ export const normalizeProcessText = ( pid: number, ): string => { let normalized = value; - if (scenario.normalization.paths) { - normalized = normalized.replaceAll(temporaryRoot, ""); - normalized = normalized.replaceAll( - scenario.working_directory, - "", - ); - normalized = normalized.replaceAll(scenario.executable, ""); - for (const [index, root] of scenario.filesystem_observation_paths.entries()) - normalized = normalized.replaceAll( - root, - ``, - ); - } + if (scenario.normalization.paths) + normalized = normalizePathTokens(normalized, scenario, temporaryRoot); if (scenario.normalization.ports) normalized = normalizePortTokens(normalized); if (scenario.normalization.pids) @@ -44,6 +33,39 @@ export const normalizeProcessText = ( return normalized; }; +/** Match original paths once, preferring the most specific selected identity. */ +const normalizePathTokens = ( + value: string, + scenario: ProcessScenario, + temporaryRoot: string, +): string => { + const aliases = new Map(); + const selected: readonly (readonly [string, string])[] = [ + [temporaryRoot, ""], + [scenario.working_directory, ""], + [scenario.executable, ""], + ...scenario.filesystem_observation_paths.map( + (root, index): readonly [string, string] => [ + root, + ``, + ], + ), + ]; + for (const [path, alias] of selected) + if (path !== "" && !aliases.has(path)) aliases.set(path, alias); + const paths = [...aliases.keys()].sort( + (left, right) => right.length - left.length, + ); + if (paths.length === 0) return value; + const pattern = paths + .map((path) => path.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + .join("|"); + return value.replace( + new RegExp(pattern, "gu"), + (path) => aliases.get(path) ?? path, + ); +}; + // A port may end a sentence, but a period followed by a letter, digit, or // underscore in any script continues a decimal, version, or host name and is // not a port boundary. diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 965dc2af5..2837ccf40 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -1,4 +1,4 @@ -import { readFile } from "node:fs/promises"; +import { mkdir, readFile, symlink } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; @@ -202,3 +202,61 @@ itWithLinuxCaptureCapability.each([false, true])( }, 20_000, ); + +itWithCaptureCapability( + "preserves selected nested path identities in actual PTY Evidence", + async () => { + const root = await createTestTempDirectory("rea-process-nested-paths-"); + const executable = join(root, "selected-node"); + const observed = join(root, "data"); + await symlink(process.execPath, executable); + await mkdir(observed); + const original = `${executable}\n${observed}/result\n${root}/log`; + const scenario = parseProcessScenario({ + executable, + working_directory: root, + filesystem_observation_paths: [observed], + arguments: ["-e", `process.stdout.write(${JSON.stringify(original)})`], + normalization: { paths: true, pids: false, ports: false, patterns: [] }, + settle_ms: 100, + }); + const result = await captureProcessScenario(scenario); + if (!result.ok) throw result.error; + const capture = parseProcessCapture( + createProcessCaptureEvidence(scenario, result.value).normalized_result, + ); + expect(capture.frames.map(({ data }) => data).join("")).toBe( + "\r\n/result\r\n/log", + ); + expect( + capture.frames.map(({ raw_data, data }) => raw_data ?? data).join(""), + ).toBe(original.replaceAll("\n", "\r\n")); + expect(capture.manifest.comparison_contract).toHaveProperty( + "path_normalization_version", + "longest-original-paths-v1", + ); + const legacyContract = { ...capture.manifest.comparison_contract }; + delete legacyContract.path_normalization_version; + const legacy = parseProcessCapture({ + ...capture, + manifest: { + ...capture.manifest, + comparison_contract: legacyContract, + comparison_contract_sha256: digestProcessCommitment(legacyContract), + }, + }); + expect(() => compareProcessCaptures(legacy, capture)).toThrow( + expect.objectContaining({ + issues: [ + expect.objectContaining({ expected: ["path_normalization_version"] }), + ], + }), + ); + expect(capture.exit).toMatchObject({ code: 0, reason: "exited" }); + expect(capture.cleanup).toMatchObject({ + owned_process_group: "verified", + temporary_root: "removed", + }); + }, + 20_000, +); From 9c34c091b3b8c47f8bd18d6f0321efde2dedb92a Mon Sep 17 00:00:00 2001 From: Olorin <59321404+Olorinm@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:55:25 +0800 Subject: [PATCH 16/27] feat(captures): discover installed protocol and field definitions (#2496) * feat(captures): discover installed protocol and field definitions * test(captures): isolate the direct glossary oracle profile * test(captures): accept padded kernel limit reports * fix(captures): detach every retained glossary column * fix(captures): describe every field discovery CLI option * fix(captures): retain discovery operation across cleanup retries --------- Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- .../skills/reverse-engineer-anything/SKILL.md | 3 + .github/workflows/real-packet-capture.yml | 2 +- docs/packet-captures.md | 46 +++ scripts/verify-packet-capture.mjs | 153 +++++++- src/application/CapabilityInventory.ts | 2 +- .../captures/PacketCaptureService.ts | 70 +++- src/captures/TsharkCaptureProvider.ts | 342 +++++++++++------- src/captures/parseFieldDiscovery.ts | 92 +++++ src/cli/captureCommands.ts | 93 +++-- src/cliCommandNames.ts | 1 + .../captures/captureToolContracts.ts | 20 + src/contracts/toolEffects.ts | 5 + src/domain/captures/packetFields.ts | 100 +++++ src/process/ArtifactAnalysisResources.ts | 45 ++- src/server/registerCaptureTools.ts | 13 + .../boundary/captures/captureProvider.test.ts | 164 ++++++++- .../boundary/captures/fieldDiscovery.test.ts | 67 ++++ tests/boundary/cli/captureCleanup.test.ts | 9 + tests/boundary/mcp/packetCaptureMcp.test.ts | 87 +++-- 19 files changed, 1095 insertions(+), 219 deletions(-) create mode 100644 src/captures/parseFieldDiscovery.ts create mode 100644 src/domain/captures/packetFields.ts create mode 100644 tests/boundary/captures/fieldDiscovery.test.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index f8ec45268..48bc8b7df 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -76,6 +76,9 @@ explicit path or endpoint and do not need it. Facts retain all-recorded-events-before-selection scope. Other qlog layouts and PCAP correlation are unsupported. - User-owned Electron runtime already open: `list_electron_targets`. +- Packet-field discovery: `inspect_packet_fields` searches installed protocols + and field names/types before choosing fields for `inspect_packet_capture`. + Query/page the relevant records; registrations do not establish observed traffic. - Recorded packet captures: `inspect_packet_capture` with selected TShark fields, display filters and decode-as rules. Requires Linux, util-linux prlimit and `REA_TSHARK_COMMAND`; see [capture guide](https://github.com/morluto/rea/blob/main/docs/packet-captures.md). diff --git a/.github/workflows/real-packet-capture.yml b/.github/workflows/real-packet-capture.yml index f481ed00d..b5e9f9201 100644 --- a/.github/workflows/real-packet-capture.yml +++ b/.github/workflows/real-packet-capture.yml @@ -51,7 +51,7 @@ jobs: - run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tshark util-linux python3 - run: npm ci - run: npm run build:cached - - run: npm run test:focused -- src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts + - run: npm run test:focused -- tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts - run: node scripts/verify-packet-capture.mjs env: REA_TSHARK_COMMAND: /usr/bin/tshark diff --git a/docs/packet-captures.md b/docs/packet-captures.md index e8ce3ae92..32cedb11b 100644 --- a/docs/packet-captures.md +++ b/docs/packet-captures.md @@ -128,3 +128,49 @@ Each provider admits one inspection at a time, including acquisition and cleanup An occupied provider returns a retryable capacity error before acquiring another file or worker. Closing the provider stops new admission and drains accepted work; failed cleanup remains owned and can be retried. + +## Discover protocols and field names + +`inspect_packet_fields` / `inspect-packet-fields` queries the installed TShark +registration glossary before selecting fields for capture inspection: + +```sh +rea inspect-packet-fields --query usb +rea inspect-packet-fields --kind fields --protocol usb --query idVendor +rea inspect-packet-fields --kind fields --protocol udp --all +``` + +`kind` defaults to `protocols`; `fields` selects field definitions. `protocol` +selects an exact protocol abbreviation (the protocol itself or its child fields). +`query` is a case-insensitive literal substring in name, abbreviation or field +description. `offset` skips matching records; `limit` defaults to 50. MCP +`limit: null` or CLI `--all` requests all remaining matches within the resource +budget. `--all` and `--limit` conflict. The window reports examined records of +the selected kind before filters, matching records, returned records and +`has_more`; it never counts capture packets. + +Returned records preserve field type, parent protocol, display base, bitmask, +description and original glossary line number. Empty producer strings remain +empty; protocol records have null field-only properties. Repeated abbreviations +remain separate registrations. A listed field is an installation capability, not +proof that the field appears in a selected recording. + +The adapter uses the documented `tshark -G fields` grammar under its existing +private empty profile and Linux process limits. TShark 4.6.9's full glossary was +measured at 22,303,197 bytes; discovery admits at most **32 MiB** combined command +output (capture inspection remains at **4 MiB**). Parsing walks lines once and +constructs objects only for the selected page. Returned strings are copied so +retained pages do not retain the complete glossary string. Before constructing a selected +record it charges six times the source UTF-8 bytes plus 512 bytes against an +**8 MiB** representation budget, covering JSON escaping and record bookkeeping. +If that budget would be exceeded, narrow the query or page size; results are not +silently truncated. This is a conservative admission budget, not measured heap +usage. Parent pipe buffers and producer per-process limits remain distinct. + +Tool/limiter identity, version, stderr and selected arguments are retained in +Evidence. No capture, interface or device is opened. Discovery shares capture +analysis's admission slot and cleanup owner. Each call rereads the glossary; +installed plugins, libraries and global configuration remain trusted and are not +fully fingerprinted. There is no cache or snapshot-consistent pagination claim. +The real Linux lane checks public CLI/MCP queries against the direct glossary, +including USB vendor-ID typing, negative queries and paging. diff --git a/scripts/verify-packet-capture.mjs b/scripts/verify-packet-capture.mjs index 2b959e602..9993117c2 100644 --- a/scripts/verify-packet-capture.mjs +++ b/scripts/verify-packet-capture.mjs @@ -1,8 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; +import { estimateTokenCount } from "tokenx"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; -import { readFile, writeFile, readdir } from "node:fs/promises"; +import { readFile, writeFile, readdir, mkdir, rm } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; @@ -91,6 +92,20 @@ const source = pcap(1, [ // LINKTYPE_BLUETOOTH_HCI_H4 = 187; H4 command packet, HCI Reset opcode 0x0c03, no parameters. const bluetoothSource = pcap(187, [Buffer.from("01030c00", "hex")]); let cases = 0; +const discoveryMeasurements = []; +const inheritedAddressSpace = (await readFile("/proc/self/limits", "utf8")) + .split("\n") + .find((line) => line.startsWith("Max address space ")) + ?.trim() + .match(/^Max address space\s+(\S+)\s+(\S+)\s+bytes$/u); +assert.ok(inheritedAddressSpace); +const expectedAddressSpace = Math.min( + 1073741824, + ...inheritedAddressSpace + .slice(1) + .map((value) => (value === "unlimited" ? Infinity : Number(value))), +); + try { await writeFile(path, source); await writeFile(bluetooth, bluetoothSource); @@ -121,14 +136,16 @@ try { .digest("hex"), ); assert.equal(report.hardware_acquisition, "not-performed"); - assert.equal(report.limits.address_space_bytes, 1073741824); + assert.equal(report.limits.address_space_bytes, expectedAddressSpace); assert.equal(report.tool.configured_path, process.env.REA_TSHARK_COMMAND); assert.equal( report.resource_limiter.configured_path, process.env.REA_TSHARK_PRLIMIT_COMMAND ?? "/usr/bin/prlimit", ); assert.ok( - report.resource_limiter.arguments.includes("--as=1073741824:1073741824"), + report.resource_limiter.arguments.includes( + `--as=${expectedAddressSpace}:${expectedAddressSpace}`, + ), ); assert.equal(report.configuration, "private-empty-personal-profile"); const flags = Object.entries(input) @@ -313,6 +330,133 @@ try { true, ); cases++; + const discovery = (await client.listTools()).tools.find( + ({ name }) => name === "inspect_packet_fields", + ); + assert.ok(discovery?.outputSchema); + assert.ok(ajv.validateSchema(discovery.inputSchema)); + assert.ok(ajv.validateSchema(discovery.outputSchema)); + const nativeProfile = join(root.path, "native-profile"); + await mkdir(nativeProfile); + const native = ( + await execute(process.env.REA_TSHARK_COMMAND, ["-G", "fields"], { + env: { ...environment, WIRESHARK_CONFIG_DIR: nativeProfile }, + timeout: 30000, + maxBuffer: 32 * 1024 * 1024, + }) + ).stdout; + await rm(nativeProfile, { recursive: true }); + const nativeLines = native.split(/\r?\n/); + if (nativeLines.at(-1) === "") nativeLines.pop(); + for (const input of [ + { query: "usb" }, + { limit: 1 }, + { offset: 1, limit: 2 }, + { kind: "fields", protocol: "usb", query: "idVendor" }, + { kind: "fields", protocol: "udp", limit: null }, + { kind: "fields", query: "rea_no_such_field" }, + ]) { + const started = performance.now(); + const response = await client.callTool({ + name: discovery.name, + arguments: input, + }); + assert.notEqual(response.isError, true, JSON.stringify(response)); + assert.ok( + ajv.validate(discovery.outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ); + const evidence = parseEvidence(response.structuredContent); + const report = evidence.normalized_result; + const kind = input.kind ?? "protocols"; + const query = (input.query ?? "").toLowerCase(); + const expectedMatches = nativeLines.flatMap((line, index) => { + const columns = line.split("\t"); + if ((kind === "protocols") !== (columns[0] === "P")) return []; + if ( + input.protocol !== undefined && + (columns[0] === "P" ? columns[2] : columns[4]) !== input.protocol + ) + return []; + if ( + ![ + columns[1], + columns[2], + ...(columns[0] === "F" ? [columns[7]] : []), + ].some((value) => value.toLowerCase().includes(query)) + ) + return []; + return [{ sourceLine: index + 1, columns }]; + }); + const offset = input.offset ?? 0; + const limit = input.limit === null ? undefined : (input.limit ?? 50); + const expectedPage = expectedMatches.slice( + offset, + limit === undefined ? undefined : offset + limit, + ); + assert.deepEqual( + report.records.map((record) => record.source_line), + expectedPage.map((record) => record.sourceLine), + ); + assert.deepEqual(report.output_window, { + examined: nativeLines.filter((line) => + line.startsWith(kind === "protocols" ? "P\t" : "F\t"), + ).length, + matching: expectedMatches.length, + returned: expectedPage.length, + has_more: + Math.max(0, expectedMatches.length - offset) > expectedPage.length, + }); + discoveryMeasurements.push({ + input, + elapsed_ms: +(performance.now() - started).toFixed(2), + response_bytes: Buffer.byteLength(JSON.stringify(response)), + estimated_tokens_one_tool_request_response: estimateTokenCount( + JSON.stringify(discovery) + + JSON.stringify(input) + + JSON.stringify(response), + ), + }); + for (const record of report.records) { + const columns = nativeLines[record.source_line - 1].split("\t"); + assert.equal(record.name, columns[1]); + assert.equal(record.abbreviation, columns[2]); + assert.equal(record.field_type, columns[0] === "F" ? columns[3] : null); + assert.equal( + record.parent_protocol, + columns[0] === "F" ? columns[4] : null, + ); + assert.equal(record.description, columns[0] === "F" ? columns[7] : null); + assert.equal(record.display_base, columns[0] === "F" ? columns[5] : null); + assert.equal(record.bitmask, columns[0] === "F" ? columns[6] : null); + } + if (input.query === "idVendor") + assert.ok( + report.records.some( + (row) => + row.abbreviation === "usb.idVendor" && + row.field_type === "FT_UINT16", + ), + ); + if (input.query === "usb") + assert.ok(report.records.some((row) => row.abbreviation === "usb")); + if (input.query === "rea_no_such_field") + assert.equal(report.output_window.matching, 0); + const flags = Object.entries(input).flatMap(([key, value]) => + value === null ? ["--all"] : [`--${key}`, String(value)], + ); + const cli = JSON.parse( + ( + await execute( + process.execPath, + [entrypoint, "inspect-packet-fields", ...flags, "--format", "json"], + { env: environment, timeout: 60000, maxBuffer: 16 * 1024 * 1024 }, + ) + ).stdout, + ); + assert.deepEqual(cli.normalized_result, report); + cases++; + } await client.close(); assert.deepEqual((await readdir(root.path)).sort(), [ "bluetooth.pcap", @@ -329,6 +473,9 @@ console.log( { status: "passed", cases, + discovery_measurements: discoveryMeasurements, + measurement_scope: + "Warm MCP server, fresh TShark processes; tokenx estimates include one advertised tool/request/response, not billed usage. Model-driven selection, CPU and peak process-family memory are unmeasured; no cache/speedup claim.", host: `${process.platform}/${process.arch}`, scope: "real TShark CLI/MCP; synthetic Ethernet/IPv4/UDP/DNS and Bluetooth HCI PCAP; no hardware acquisition", diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 34b418029..94518db85 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -397,7 +397,7 @@ const workflowAvailabilityFor = ({ remediation: "On Linux x64, set absolute REA_PWNTOOLS_PYTHON to caller-supplied Python with pwntools 4.15.0. No active binary target is required.", }; - if (name === "inspect_packet_capture") + if (name === "inspect_packet_capture" || name === "inspect_packet_fields") return policy.packetCaptureEnabled === true ? { reason: "available", remediation: null } : { diff --git a/src/application/captures/PacketCaptureService.ts b/src/application/captures/PacketCaptureService.ts index f2098b49c..e2877fa3f 100644 --- a/src/application/captures/PacketCaptureService.ts +++ b/src/application/captures/PacketCaptureService.ts @@ -1,3 +1,9 @@ +import { + inspectPacketFieldsInputSchema, + packetFieldsCatalogSchema, + type InspectPacketFieldsInput, +} from "../../domain/captures/packetFields.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; import { isAbsolute } from "node:path"; import type { AnalysisExecution, @@ -24,6 +30,10 @@ export interface PacketCapturePort { input: InspectPacketCaptureInput, options?: ExecutionOptions, ): Promise>; + inspectFields( + input: InspectPacketFieldsInput, + options?: ExecutionOptions, + ): Promise>; close(): Promise; } @@ -74,14 +84,58 @@ export class PacketCaptureService { "Capture analyzer returned malformed data or changed the selected artifact identity/options.", ), ); - const evidence = createEvidence(execution.subject, execution.provider, { - operation, - parameters: input.data, - result: report.data, - confidence: "observed", - locations: execution.locations, - limitations: execution.limitations, - }); + return this.#evidence(operation, input.data, execution, options); + } + async inspectFields( + raw: unknown, + options?: ExecutionOptions, + ): Promise> { + const operation = "inspect_packet_fields"; + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(operation)); + const input = inspectPacketFieldsInputSchema.safeParse(raw); + if (!input.success) + return err( + analysisInputErrorFromIssues(operation, input.error.issues, raw, { + cause: input.error, + }), + ); + const decoded = await this.provider.inspectFields(input.data, options); + if (!decoded.ok) return decoded; + const execution = decoded.value; + const report = packetFieldsCatalogSchema.safeParse(execution.result); + if ( + !report.success || + JSON.stringify(report.data.selection) !== JSON.stringify(input.data) || + execution.subject?.path !== report.data.tool.path || + execution.subject.sha256 !== report.data.tool.sha256 + ) + return err( + new AnalysisOutputError( + operation, + "Field discovery returned malformed data or changed the selected installation/options.", + ), + ); + return this.#evidence(operation, input.data, execution, options); + } + #evidence( + operation: string, + parameters: Readonly>, + execution: AnalysisExecution, + options?: ExecutionOptions, + ): Result { + const evidence = createEvidence( + execution.subject ?? undefined, + execution.provider, + { + operation, + parameters, + result: execution.result, + confidence: "observed", + locations: execution.locations, + limitations: execution.limitations, + }, + ); return options?.signal?.aborted ? err( new AnalysisCancelledError(operation, { diff --git a/src/captures/TsharkCaptureProvider.ts b/src/captures/TsharkCaptureProvider.ts index 13cadd774..d22515929 100644 --- a/src/captures/TsharkCaptureProvider.ts +++ b/src/captures/TsharkCaptureProvider.ts @@ -1,3 +1,12 @@ +import { + type InspectPacketFieldsInput, + packetFieldsCatalogSchema, +} from "../domain/captures/packetFields.js"; +import { + parseFieldDiscovery, + FieldDiscoveryBudgetError, + FIELD_RECORD_BUDGET, +} from "./parseFieldDiscovery.js"; import { readLinuxResourceLimits, type LinuxResourceLimits, @@ -91,17 +100,31 @@ export class TsharkCaptureProvider implements PacketCapturePort { options?: ExecutionOptions, ): Promise> { return this.#resources.run( - () => this.#inspect(input, options), + () => this.#executeRequest(input, options), options?.signal, ); } + inspectFields( + input: InspectPacketFieldsInput, + options?: ExecutionOptions, + ): Promise> { + return this.#resources.run( + () => this.#executeRequest(input, options), + options?.signal, + "inspect_packet_fields", + ); + } close(): Promise { return this.#resources.close(); } - async #inspect( - input: InspectPacketCaptureInput, + async #executeRequest( + input: InspectPacketCaptureInput | InspectPacketFieldsInput, options?: ExecutionOptions, ): Promise> { + const operation = "path" in input ? OPERATION : "inspect_packet_fields"; + // TShark 4.6.9 emits a 22.3 MB glossary; keep capture output capped at 4 MiB. + const limits = + "path" in input ? LIMITS : { ...LIMITS, output_bytes: 32 * 1024 * 1024 }; let phase: | "configuration" | "artifact" @@ -109,12 +132,12 @@ export class TsharkCaptureProvider implements PacketCapturePort { | "producer" | "integrity" = "configuration"; let capturedOutput: AnalysisCapturedOutput | undefined; - let resourcePath = input.path; + let resourcePath = "path" in input ? input.path : "TShark field glossary"; let effectiveLimits: LinuxResourceLimits | undefined; let root: ArtifactAnalysisRuntime | undefined; let result: Result; try { - if (options?.signal?.aborted) throw new AnalysisCancelledError(OPERATION); + if (options?.signal?.aborted) throw new AnalysisCancelledError(operation); await this.#resources.release(); if (this.platform !== "linux") throw new Error( @@ -141,7 +164,7 @@ export class TsharkCaptureProvider implements PacketCapturePort { 64 * 1024 * 1024, options?.signal, ); - effectiveLimits = await this.#resources.readLimits(LIMITS); + effectiveLimits = await this.#resources.readLimits(limits); const limitArguments = [ `--as=${effectiveLimits.address_space_bytes}:${effectiveLimits.address_space_bytes}`, `--fsize=${effectiveLimits.file_bytes}:${effectiveLimits.file_bytes}`, @@ -152,7 +175,7 @@ export class TsharkCaptureProvider implements PacketCapturePort { ]; phase = "workspace"; root = await this.createRuntime(); - this.#resources.retain(root, undefined); + this.#resources.retain(root, undefined, operation); const snapshotPath = join(root.path, "capture"); const configurationPath = join(root.path, "configuration"); await mkdir(configurationPath, { mode: 0o700 }); @@ -170,8 +193,8 @@ export class TsharkCaptureProvider implements PacketCapturePort { }, }, { - timeoutMs: LIMITS.command_timeout_ms, - diagnosticBytes: LIMITS.output_bytes, + timeoutMs: limits.command_timeout_ms, + diagnosticBytes: limits.output_bytes, acceptNonZeroExit: true, }, { @@ -200,24 +223,24 @@ export class TsharkCaptureProvider implements PacketCapturePort { ); if (currentLimiter.sha256 !== limiterIdentity.sha256) throw new AnalysisArtifactChangedError( - OPERATION, + operation, limiter, "Configured resource limiter changed during analysis.", { capturedOutput }, ); if (current.sha256 !== tool.sha256) throw new AnalysisArtifactChangedError( - OPERATION, + operation, command, "Configured TShark executable changed during analysis.", { capturedOutput }, ); phase = previousPhase; - resourcePath = input.path; + resourcePath = "path" in input ? input.path : command; if (output.exitCode !== 0) throw new ProviderAdapterError( TSHARK_PROVIDER_IDENTITY.id, - OPERATION, + operation, { diagnostics: { exit_code: output.exitCode ?? null, @@ -238,86 +261,15 @@ export class TsharkCaptureProvider implements PacketCapturePort { throw new Error( "Expected TShark 4.x; inspect the selected installation.", ); - phase = "artifact"; - const selected = await this.#resources.read( - input.path, - LIMITS.input_bytes, - options?.signal, - ); - phase = "workspace"; - await writeFile(snapshotPath, selected.bytes, { - flag: "wx", - mode: 0o400, - }); - phase = "producer"; - const args = [ - "-n", - "-r", - "capture", - "-T", - "json", - "--no-duplicate-keys", - "--temp-dir=.", - ...(input.two_pass ? ["-2"] : []), - ...(input.packet_limit === null - ? [] - : ["-c", String(input.packet_limit)]), - ...(input.display_filter === null ? [] : ["-Y", input.display_filter]), - ...input.decode_as.flatMap((value) => ["-d", value]), - ...[...new Set(["frame.number", ...input.fields])].flatMap((field) => [ - "-e", - field, - ]), - ]; - const output = await execute(args); - phase = "integrity"; - resourcePath = snapshotPath; - if ( - (await this.#resources.read(snapshotPath, LIMITS.input_bytes)) - .sha256 !== selected.sha256 - ) - throw new AnalysisArtifactChangedError( - OPERATION, - snapshotPath, - "TShark changed the private capture copy.", - capturedOutput === undefined ? {} : { capturedOutput }, - ); - phase = "producer"; - resourcePath = input.path; - const matchingPackets = parsePacketOutput( - output.stdout.text, - input.fields, - ); - const start = Math.min(input.match_offset, matchingPackets.length); - const end = - input.match_limit === null - ? matchingPackets.length - : start + Math.min(input.match_limit, matchingPackets.length - start); - const packets = matchingPackets.slice(start, end); - const report = packetCaptureSchema.parse({ - artifact: { - path: input.path, - sha256: selected.sha256, - bytes: selected.bytes.length, - }, - selection: input, - packets, - output_window: { - matching_packets: matchingPackets.length, - returned_packets: packets.length, - has_more: end < matchingPackets.length, - }, - read_scope: - input.packet_limit === null ? "to-eof" : "up-to-input-packet-limit", - configuration: "private-empty-personal-profile", - diagnostics: output.stderr.text, + const metadata = (args: readonly string[], diagnostics: string) => ({ + diagnostics, tool: { configured_path: configured, path: command, sha256: tool.sha256, version_banner: version.stdout.text, version_diagnostics: version.stderr.text, - arguments: args, + arguments: [...args], }, resource_limiter: { configured_path: configuredLimiter, @@ -325,33 +277,136 @@ export class TsharkCaptureProvider implements PacketCapturePort { sha256: limiterIdentity.sha256, arguments: limitArguments, }, - limits: { ...LIMITS, ...effectiveLimits }, - hardware_acquisition: "not-performed", + limits: { ...limits, ...effectiveLimits }, + hardware_acquisition: "not-performed" as const, }); - result = ok( - createAnalysisExecution(report, TSHARK_PROVIDER_IDENTITY, { - subject: { + if (!("path" in input)) { + phase = "producer"; + const args = ["-G", "fields"]; + const output = await execute(args); + const report = packetFieldsCatalogSchema.parse({ + selection: input, + ...parseFieldDiscovery(output.stdout.text, input), + ...metadata(args, output.stderr.text), + limits: { + ...limits, + ...effectiveLimits, + record_budget_bytes: FIELD_RECORD_BUDGET, + }, + configuration: "private-empty-personal-profile", + }); + result = ok( + createAnalysisExecution(report, TSHARK_PROVIDER_IDENTITY, { + subject: { path: command, sha256: tool.sha256, format: "file" }, + limitations: [ + "Protocol and field registrations describe the selected installation, not traffic observed in a capture. Duplicate abbreviations remain distinct producer records with source line numbers.", + "Each request probes again without caching. The private empty personal profile does not fingerprint or isolate installed libraries, global configuration or plugins; they remain trusted.", + "No capture file, interface or device is opened. Resource ceilings do not constitute an OS sandbox. Glossary output is capped at 32 MiB and selected record representation at 8 MiB before expansion.", + ], + }), + ); + } else { + phase = "artifact"; + const selected = await this.#resources.read( + input.path, + LIMITS.input_bytes, + options?.signal, + ); + phase = "workspace"; + await writeFile(snapshotPath, selected.bytes, { + flag: "wx", + mode: 0o400, + }); + phase = "producer"; + const args = [ + "-n", + "-r", + "capture", + "-T", + "json", + "--no-duplicate-keys", + "--temp-dir=.", + ...(input.two_pass ? ["-2"] : []), + ...(input.packet_limit === null + ? [] + : ["-c", String(input.packet_limit)]), + ...(input.display_filter === null + ? [] + : ["-Y", input.display_filter]), + ...input.decode_as.flatMap((value) => ["-d", value]), + ...[...new Set(["frame.number", ...input.fields])].flatMap( + (field) => ["-e", field], + ), + ]; + const output = await execute(args); + phase = "integrity"; + resourcePath = snapshotPath; + if ( + (await this.#resources.read(snapshotPath, LIMITS.input_bytes)) + .sha256 !== selected.sha256 + ) + throw new AnalysisArtifactChangedError( + operation, + snapshotPath, + "TShark changed the private capture copy.", + capturedOutput === undefined ? {} : { capturedOutput }, + ); + phase = "producer"; + resourcePath = "path" in input ? input.path : command; + const matchingPackets = parsePacketOutput( + output.stdout.text, + input.fields, + ); + const start = Math.min(input.match_offset, matchingPackets.length); + const end = + input.match_limit === null + ? matchingPackets.length + : start + + Math.min(input.match_limit, matchingPackets.length - start); + const packets = matchingPackets.slice(start, end); + const report = packetCaptureSchema.parse({ + artifact: { path: input.path, sha256: selected.sha256, - format: "file", + bytes: selected.bytes.length, }, - locations: [{ kind: "artifact-path", path: input.path }], - limitations: [ - "Fields are TShark dissector observations under the selected filter/decode-as/pass settings, not independently verified protocol semantics. Missing fields are absent from the producer output, not known zero or empty values.", - "Packet limits count input packets before display filtering. Filtered results do not establish whole-capture or device traffic absence. Two-pass reassembly cannot recover bytes missing from the recording or beyond the read limit.", - "Repeated field values remain ordered string arrays; raw packet bytes are included only when explicitly selected as a supported field. Export index/score metadata are omitted. All stderr, including warnings on exit zero, is retained.", - "A private empty personal configuration is used. Installed libraries, global configuration and dissector plugins are trusted and not fully fingerprinted. File-only analysis and disabled name resolution do not constitute an OS sandbox.", - "Linux prlimit enforces soft and hard ceilings before execution, preserving stricter inherited limits: at most 1 GiB address space per process, 64 MiB per file, 30 CPU seconds and no core dumps. Effective limits are reported. Resident input is capped at 32 MiB, each command at 30 wall-clock seconds and combined output at 4 MiB. Output overflow fails. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted.", - ], - }), - ); + selection: input, + packets, + output_window: { + matching_packets: matchingPackets.length, + returned_packets: packets.length, + has_more: end < matchingPackets.length, + }, + read_scope: + input.packet_limit === null ? "to-eof" : "up-to-input-packet-limit", + configuration: "private-empty-personal-profile", + ...metadata(args, output.stderr.text), + }); + result = ok( + createAnalysisExecution(report, TSHARK_PROVIDER_IDENTITY, { + subject: { + path: input.path, + sha256: selected.sha256, + format: "file", + }, + locations: [{ kind: "artifact-path", path: input.path }], + limitations: [ + "Fields are TShark dissector observations under the selected filter/decode-as/pass settings, not independently verified protocol semantics. Missing fields are absent from the producer output, not known zero or empty values.", + "Packet limits count input packets before display filtering. Filtered results do not establish whole-capture or device traffic absence. Two-pass reassembly cannot recover bytes missing from the recording or beyond the read limit.", + "Repeated field values remain ordered string arrays; raw packet bytes are included only when explicitly selected as a supported field. Export index/score metadata are omitted. All stderr, including warnings on exit zero, is retained.", + "A private empty personal configuration is used. Installed libraries, global configuration and dissector plugins are trusted and not fully fingerprinted. File-only analysis and disabled name resolution do not constitute an OS sandbox.", + "Linux prlimit enforces soft and hard ceilings before execution, preserving stricter inherited limits: at most 1 GiB address space per process, 64 MiB per file, 30 CPU seconds and no core dumps. Effective limits are reported. Resident input is capped at 32 MiB, each command at 30 wall-clock seconds and combined output at 4 MiB. Output overflow fails. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted.", + ], + }), + ); + } } catch (cause) { if ( root !== undefined && cause instanceof OwnedCommandFailure && cause.cleanupOwner !== undefined ) - this.#resources.retain(root, cause.cleanupOwner); + this.#resources.retain(root, cause.cleanupOwner, operation); result = err( this.#failure( cause, @@ -360,18 +415,23 @@ export class TsharkCaptureProvider implements PacketCapturePort { root?.path ?? null, effectiveLimits, capturedOutput, + operation, + limits, ), ); } - return this.#resources.finish(result, (execution) => - createEvidence(execution.subject ?? undefined, execution.provider, { - operation: OPERATION, - parameters: input, - result: execution.result, - confidence: "observed", - locations: execution.locations, - limitations: execution.limitations, - }), + return this.#resources.finish( + result, + (execution) => + createEvidence(execution.subject ?? undefined, execution.provider, { + operation, + parameters: input, + result: execution.result, + confidence: "observed", + locations: execution.locations, + limitations: execution.limitations, + }), + operation, ); } #failure( @@ -386,6 +446,11 @@ export class TsharkCaptureProvider implements PacketCapturePort { workspacePath: string | null, effectiveLimits: LinuxResourceLimits | undefined, observedOutput: AnalysisCapturedOutput | undefined, + operation = OPERATION, + limits: { + readonly output_bytes: number; + readonly command_timeout_ms: number; + } = LIMITS, ): AnalysisError { if (cause instanceof AnalysisError) return cause; if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) @@ -393,15 +458,34 @@ export class TsharkCaptureProvider implements PacketCapturePort { cause.cleanup, cause, observedOutput, + operation, ); if ( (cause instanceof Error && cause.name === "AbortError") || (cause instanceof ArtifactReaderFailure && cause.reason === "cancelled") ) - return new AnalysisCancelledError(OPERATION, { cause }); + return new AnalysisCancelledError(operation, { cause }); + if (cause instanceof FieldDiscoveryBudgetError) + return new AnalysisResourceConstraintError( + operation, + "memory", + cause.message, + { record_budget_bytes: FIELD_RECORD_BUDGET }, + { + cause, + ...(observedOutput === undefined + ? {} + : { capturedOutput: observedOutput }), + }, + ); const reason = cause instanceof Error ? cause.message : String(cause); if (cause instanceof OwnedCommandFailure) - return this.#resources.commandFailure(cause, LIMITS, effectiveLimits); + return this.#resources.commandFailure( + cause, + limits, + effectiveLimits, + operation, + ); if (phase === "workspace") { const code = cause instanceof Error && @@ -411,13 +495,13 @@ export class TsharkCaptureProvider implements PacketCapturePort { : null; if (code === "ENOSPC" || code === "EDQUOT" || code === "EFBIG") return new AnalysisResourceConstraintError( - OPERATION, + operation, "file-size", `Private analysis workspace failed (${code}): ${reason}`, { workspace_path: workspacePath, system_code: code }, { cause }, ); - return new ProviderAdapterError(TSHARK_PROVIDER_IDENTITY.id, OPERATION, { + return new ProviderAdapterError(TSHARK_PROVIDER_IDENTITY.id, operation, { cause, userMessage: `Cannot prepare the private analysis workspace: ${reason}`, diagnostics: { @@ -440,12 +524,12 @@ export class TsharkCaptureProvider implements PacketCapturePort { ["limit", "integrity", "path"].includes(cause.reason) ) return new AnalysisArtifactChangedError( - OPERATION, + operation, path, `Post-run identity verification failed for ${path}: ${reason}`, detail, ); - return new ProviderAdapterError(TSHARK_PROVIDER_IDENTITY.id, OPERATION, { + return new ProviderAdapterError(TSHARK_PROVIDER_IDENTITY.id, operation, { ...detail, diagnostics: { stage: "integrity", resource_path: path, reason }, }); @@ -453,10 +537,13 @@ export class TsharkCaptureProvider implements PacketCapturePort { if (phase === "configuration") return new AnalysisCapabilityUnavailableError( TSHARK_PROVIDER_IDENTITY.id, - OPERATION, + operation, reason, { cause, + ...(observedOutput === undefined + ? {} + : { capturedOutput: observedOutput }), userMessage: `${reason} On Linux, configure TShark 4.x through REA_TSHARK_COMMAND and util-linux prlimit through REA_TSHARK_PRLIMIT_COMMAND (default /usr/bin/prlimit).`, }, ); @@ -466,14 +553,14 @@ export class TsharkCaptureProvider implements PacketCapturePort { cause.reason === "limit" ) return new AnalysisResourceConstraintError( - OPERATION, + operation, "file-size", reason, { input_bytes: LIMITS.input_bytes }, { cause }, ); if (cause instanceof ArtifactReaderFailure && cause.reason === "integrity") - return new AnalysisArtifactChangedError(OPERATION, path, reason, { + return new AnalysisArtifactChangedError(operation, path, reason, { cause, }); if ( @@ -482,13 +569,18 @@ export class TsharkCaptureProvider implements PacketCapturePort { "code" in cause && (cause.code === "EACCES" || cause.code === "EPERM") ) - return new AnalysisAccessDeniedError(OPERATION, path, cause.code, { + return new AnalysisAccessDeniedError(operation, path, cause.code, { cause, }); if (phase === "artifact") - return new AnalysisInputError(OPERATION, { cause }, [ + return new AnalysisInputError(operation, { cause }, [ { path: ["path"], reason: "invalid_value", message: reason }, ]); - return new AnalysisOutputError(OPERATION, reason, { cause }); + return new AnalysisOutputError(operation, reason, { + cause, + ...(observedOutput === undefined + ? {} + : { capturedOutput: observedOutput }), + }); } } diff --git a/src/captures/parseFieldDiscovery.ts b/src/captures/parseFieldDiscovery.ts new file mode 100644 index 000000000..6498e56dc --- /dev/null +++ b/src/captures/parseFieldDiscovery.ts @@ -0,0 +1,92 @@ +import type { + InspectPacketFieldsInput, + PacketFieldRecord, +} from "../domain/captures/packetFields.js"; + +export class FieldDiscoveryBudgetError extends Error {} +export const FIELD_RECORD_BUDGET = 8 * 1024 * 1024; + +/** TShark -G fields: P/name/abbreviation or F/name/abbreviation/type/parent/base/mask/blurb. */ +export const parseFieldDiscovery = ( + text: string, + input: InspectPacketFieldsInput, +) => { + const copy = (value: string) => + Buffer.from(value, "utf16le").toString("utf16le"); + const query = input.query.toLowerCase(); + const records: PacketFieldRecord[] = []; + let examined = 0, + matching = 0, + sourceLine = 0, + budget = 0; + // Walk the bounded text once; allocate record objects only for the requested page. + for (let start = 0; start < text.length;) { + const end = text.indexOf("\n", start); + let line = text.slice(start, end < 0 ? text.length : end); + start = end < 0 ? text.length : end + 1; + sourceLine++; + if (line.endsWith("\r")) line = line.slice(0, -1); + const columns = line.split("\t", 9); + const [tag, name, abbreviation] = columns; + if ( + !( + (tag === "P" && columns.length === 3) || + (tag === "F" && columns.length === 8) + ) || + name === undefined || + abbreviation === undefined || + abbreviation === "" + ) + throw new Error( + `Malformed TShark field glossary record at line ${sourceLine}`, + ); + if ((input.kind === "protocols") !== (tag === "P")) continue; + examined++; + if ( + input.protocol !== null && + (tag === "P" ? abbreviation : columns[4]) !== input.protocol + ) + continue; + if ( + ![name, abbreviation, ...(tag === "F" ? [columns[7] ?? ""] : [])].some( + (value) => value.toLowerCase().includes(query), + ) + ) + continue; + matching++; + if ( + matching <= input.offset || + records.length >= (input.limit ?? Number.MAX_SAFE_INTEGER) + ) + continue; + // Six bytes per source byte covers worst-case JSON escaping, plus record bookkeeping. + budget += Buffer.byteLength(line, "utf8") * 6 + 512; + if (budget > FIELD_RECORD_BUDGET) + throw new FieldDiscoveryBudgetError( + "Selected glossary records exceed the 8 MiB representation budget; narrow the query or page size", + ); + // Detach retained fields from the much larger producer string. + records.push({ + source_line: sourceLine, + kind: tag === "P" ? "protocol" : "field", + name: copy(name), + abbreviation: copy(abbreviation), + field_type: tag === "F" ? copy(columns[3] ?? "") : null, + parent_protocol: tag === "F" ? copy(columns[4] ?? "") : null, + display_base: tag === "F" ? copy(columns[5] ?? "") : null, + bitmask: tag === "F" ? copy(columns[6] ?? "") : null, + description: tag === "F" ? copy(columns[7] ?? "") : null, + }); + } + if (sourceLine === 0) + throw new Error("TShark returned an empty field glossary"); + return { + records, + output_window: { + examined, + matching, + returned: records.length, + has_more: Math.max(0, matching - input.offset) > records.length, + }, + }; +}; diff --git a/src/cli/captureCommands.ts b/src/cli/captureCommands.ts index 281be7457..2a3566703 100644 --- a/src/cli/captureCommands.ts +++ b/src/cli/captureCommands.ts @@ -1,3 +1,4 @@ +import { inspectPacketFieldsInputSchema } from "../domain/captures/packetFields.js"; import { z } from "incur"; import type { PacketCaptureService } from "../application/captures/PacketCaptureService.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; @@ -22,9 +23,73 @@ export const registerCaptureCommands = ( environment: Readonly, service: Pick< PacketCaptureService, - "inspect" | "close" + "inspect" | "inspectFields" | "close" > = createPacketCaptureService(environment), ): void => { + const finish = async ( + result: Awaited>, + operation: string, + ) => { + // A one-shot command must retry retained cleanup before discarding its owner. + try { + await service.close(); + } catch (cause) { + const cleanup = + cause instanceof AnalysisError + ? cause + : new ProviderCleanupError( + "tshark-capture", + [], + { reason: String(cause) }, + { cause, operation: operation }, + ); + if (result.ok) cleanup.retainPartialObservation(result.value); + return projectAnalysisError( + result.ok + ? cleanup + : analysisErrorWithCleanupFailure(result.error, cleanup, operation), + ); + } + return result.ok ? result.value : projectAnalysisError(result.error); + }; + cli.command(CLI_COMMANDS.inspectPacketFields, { + description: "Search installed TShark protocol and field definitions", + options: z + .object({ + ...inspectPacketFieldsInputSchema.shape, + limit: z + .number() + .int() + .positive() + .max(Number.MAX_SAFE_INTEGER) + .optional() + .describe("Maximum matching records to return; defaults to 50"), + all: z + .boolean() + .default(false) + .describe("Return all remaining matches within resource limits"), + }) + .superRefine((options, context) => { + if (options.all && options.limit !== undefined) + context.addIssue({ + code: "custom", + path: ["all"], + message: "all and limit are mutually exclusive", + }); + }), + run: ({ options: { all, ...options } }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectPacketFields, async () => + finish( + await service.inspectFields( + { ...options, ...(all ? { limit: null } : {}) }, + { signal }, + ), + "inspect_packet_fields", + ), + ), + ), + }); cli.command(CLI_COMMANDS.inspectPacketCapture, { description: "Inspect an existing packet capture with caller-supplied TShark", @@ -68,31 +133,7 @@ export const registerCaptureCommands = ( }, { signal }, ); - // A one-shot command must retry retained cleanup before discarding its owner. - try { - await service.close(); - } catch (cause) { - const cleanup = - cause instanceof AnalysisError - ? cause - : new ProviderCleanupError( - "tshark-capture", - [], - { reason: String(cause) }, - { cause, operation: "inspect_packet_capture" }, - ); - if (result.ok) cleanup.retainPartialObservation(result.value); - return projectAnalysisError( - result.ok - ? cleanup - : analysisErrorWithCleanupFailure( - result.error, - cleanup, - "inspect_packet_capture", - ), - ); - } - return result.ok ? result.value : projectAnalysisError(result.error); + return finish(result, "inspect_packet_capture"); }), ), }); diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index e7c9bc922..c3d373e4e 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -1,5 +1,6 @@ /** Canonical names used by every one-shot CLI command registration. */ export const CLI_COMMANDS = Object.freeze({ + inspectPacketFields: "inspect-packet-fields", inspectPacketCapture: "inspect-packet-capture", decodeSignalRecording: "decode-signal-recording", inspectWasmArtifact: "inspect-wasm-artifact", diff --git a/src/contracts/captures/captureToolContracts.ts b/src/contracts/captures/captureToolContracts.ts index 3e67b3b19..dca211511 100644 --- a/src/contracts/captures/captureToolContracts.ts +++ b/src/contracts/captures/captureToolContracts.ts @@ -1,3 +1,7 @@ +import { + inspectPacketFieldsInputSchema, + packetFieldsCatalogSchema, +} from "../../domain/captures/packetFields.js"; import { inspectPacketCaptureInputSchema, packetCaptureSchema, @@ -6,6 +10,22 @@ import type { ToolContract } from "../toolContractTypes.js"; import { toolContractMetadata } from "../toolEffects.js"; import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; export const CAPTURE_TOOL_CONTRACTS = [ + { + name: "inspect_packet_fields", + ...toolContractMetadata("inspect_packet_fields"), + kind: "artifact-provider", + description: + "Discover protocols or field names/types registered in the installed TShark 4.x. kind defaults to protocols; use fields and optional exact protocol abbreviation to select field definitions. query searches name/abbreviation/description by case-insensitive literal substring. Return 50 matches by default with offset/limit paging; null requests all remaining within resource limits. Preserve type/base/bitmask/description, duplicate registrations, source line numbers, coverage, tool identity and diagnostics. Linux, REA_TSHARK_COMMAND and prlimit required. No capture file, interface or binary target needed. Registrations describe tool capabilities, not observed traffic. Each call rereads the glossary; use selected fields with inspect_packet_capture.", + inputSchema: inspectPacketFieldsInputSchema, + outputSchema: evidenceResultOf(packetFieldsCatalogSchema), + examples: [ + { title: "Find USB protocol registrations", input: { query: "usb" } }, + { + title: "Find the installed USB vendor-ID field", + input: { kind: "fields", protocol: "usb", query: "idVendor" }, + }, + ], + }, { name: "inspect_packet_capture", ...toolContractMetadata("inspect_packet_capture"), diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 7ce3e4ded..8ea7f23e0 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -59,6 +59,11 @@ export const TOOL_EFFECTS: Readonly> = { writesFilesystem: true, launchesProcess: true, }), + inspect_packet_fields: effects({ + mutatesSession: true, + launchesProcess: true, + writesFilesystem: true, + }), inspect_packet_capture: effects({ mutatesSession: true, launchesProcess: true, diff --git a/src/domain/captures/packetFields.ts b/src/domain/captures/packetFields.ts new file mode 100644 index 000000000..23b25cfcd --- /dev/null +++ b/src/domain/captures/packetFields.ts @@ -0,0 +1,100 @@ +import { z } from "zod"; +import { packetCaptureSchema } from "./packetCapture.js"; + +export const inspectPacketFieldsInputSchema = z.strictObject({ + kind: z + .enum(["protocols", "fields"]) + .default("protocols") + .describe( + "Return protocol registrations or field definitions; defaults to protocols", + ), + protocol: z + .string() + .min(1) + .max(8192) + .nullable() + .default(null) + .describe( + "Exact protocol abbreviation; selects that protocol or its registered fields", + ), + query: z + .string() + .max(8192) + .default("") + .describe( + "Case-insensitive literal substring in name, abbreviation or field description", + ), + offset: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER) + .default(0) + .describe("Number of matching records to skip; defaults to 0"), + limit: z + .number() + .int() + .positive() + .max(Number.MAX_SAFE_INTEGER) + .nullable() + .default(50) + .describe( + "Maximum matches returned; null requests all remaining within the record budget", + ), +}); +export type InspectPacketFieldsInput = z.infer< + typeof inspectPacketFieldsInputSchema +>; +export const packetFieldRecordSchema = z.strictObject({ + source_line: z.number().int().positive(), + kind: z.enum(["protocol", "field"]), + name: z.string(), + abbreviation: z.string(), + field_type: z.string().nullable(), + parent_protocol: z.string().nullable(), + display_base: z.string().nullable(), + bitmask: z.string().nullable(), + description: z.string().nullable(), +}); +export type PacketFieldRecord = z.infer; +export const packetFieldsCatalogSchema = z + .strictObject({ + selection: inspectPacketFieldsInputSchema, + records: z.array(packetFieldRecordSchema), + output_window: z.strictObject({ + examined: z.number().int().nonnegative(), + matching: z.number().int().nonnegative(), + returned: z.number().int().nonnegative(), + has_more: z.boolean(), + }), + diagnostics: z.string(), + tool: packetCaptureSchema.shape.tool, + resource_limiter: packetCaptureSchema.shape.resource_limiter, + limits: packetCaptureSchema.shape.limits.extend({ + record_budget_bytes: z.literal(8388608), + }), + configuration: z.literal("private-empty-personal-profile"), + hardware_acquisition: z.literal("not-performed"), + }) + .superRefine((report, context) => { + const w = report.output_window; + const remaining = Math.max(0, w.matching - report.selection.offset); + const expected = Math.min(remaining, report.selection.limit ?? remaining); + if ( + w.matching > w.examined || + w.returned !== expected || + report.records.length !== expected || + w.has_more !== remaining > expected || + report.records.some( + (row) => + row.kind !== + (report.selection.kind === "protocols" ? "protocol" : "field"), + ) + ) + context.addIssue({ + code: "custom", + path: ["output_window"], + message: + "Field discovery coverage must match the selected page and record kind", + }); + }); diff --git a/src/process/ArtifactAnalysisResources.ts b/src/process/ArtifactAnalysisResources.ts index 5eb90b8e2..b0083e05b 100644 --- a/src/process/ArtifactAnalysisResources.ts +++ b/src/process/ArtifactAnalysisResources.ts @@ -36,7 +36,10 @@ export class ArtifactAnalysisResources { readonly #artifacts = new ArtifactResourceScope(); readonly #pending = new Map< ArtifactAnalysisRuntime, - ProviderProcessSupervisor | undefined + { + readonly owner: ProviderProcessSupervisor | undefined; + readonly operation: string; + } >(); #tail: Promise = Promise.resolve(); #busy = false; @@ -56,14 +59,15 @@ export class ArtifactAnalysisResources { run( work: () => Promise>, signal?: AbortSignal, + operation = this.operation, ): Promise> { if (this.#closed || signal?.aborted) - return Promise.resolve(err(new AnalysisCancelledError(this.operation))); + return Promise.resolve(err(new AnalysisCancelledError(operation))); if (this.#busy) return Promise.resolve( err( new AnalysisResourceConstraintError( - this.operation, + operation, "capacity", this.capacity.message, this.capacity.details, @@ -73,7 +77,7 @@ export class ArtifactAnalysisResources { this.#busy = true; const running = this.#tail .then(() => this.#artifacts.run(work)) - .catch((cause: unknown) => err(this.#artifactFailure(cause))) + .catch((cause: unknown) => err(this.#artifactFailure(cause, operation))) .finally(() => { this.#busy = false; }); @@ -101,8 +105,9 @@ export class ArtifactAnalysisResources { retain( root: ArtifactAnalysisRuntime, owner?: ProviderProcessSupervisor, + operation = this.operation, ): void { - this.#pending.set(root, owner); + this.#pending.set(root, { owner, operation }); } get pendingPaths(): readonly string[] { @@ -110,12 +115,13 @@ export class ArtifactAnalysisResources { } async release(): Promise { - for (const [root, owner] of this.#pending) { + for (const [root, retained] of this.#pending) { try { + const { owner } = retained; if (owner !== undefined) { const stopped = await owner.stop(); if (stopped.status === "incomplete") throw new Error(stopped.reason); - this.#pending.set(root, undefined); + this.#pending.set(root, { ...retained, owner: undefined }); } await root.close(); this.#pending.delete(root); @@ -124,7 +130,7 @@ export class ArtifactAnalysisResources { this.providerId, [root.path], { reason: String(cause) }, - { operation: this.operation, cause }, + { operation: retained.operation, cause }, ); } } @@ -134,6 +140,7 @@ export class ArtifactAnalysisResources { observation: NonNullable, cause: unknown, capturedOutput?: AnalysisCapturedOutput, + operation = this.operation, ): ProviderCleanupError { return new ProviderCleanupError( this.providerId, @@ -144,7 +151,7 @@ export class ArtifactAnalysisResources { ? {} : { captured_output: { ...capturedOutput } }), }, - { operation: this.operation, cause }, + { operation, cause }, ); } @@ -155,6 +162,7 @@ export class ArtifactAnalysisResources { readonly output_bytes: number; }, effectiveLimits: LinuxResourceLimits | undefined, + operation = this.operation, ): AnalysisError { const capturedOutput = cause.snapshot === null @@ -171,16 +179,16 @@ export class ArtifactAnalysisResources { ...(capturedOutput === undefined ? {} : { capturedOutput }), }; if (cause.reason === "cancelled") - return new AnalysisCancelledError(this.operation, detail); + return new AnalysisCancelledError(operation, detail); if (cause.reason === "timeout") return new AnalysisTimeoutError( - this.operation, + operation, limits.command_timeout_ms, detail, ); if (cause.reason === "output-limit") return new AnalysisResourceConstraintError( - this.operation, + operation, "memory", cause.message, { output_bytes: limits.output_bytes }, @@ -192,18 +200,19 @@ export class ArtifactAnalysisResources { cause.snapshot?.signal === "SIGXCPU") ) return new AnalysisResourceConstraintError( - this.operation, + operation, cause.snapshot.signal === "SIGXFSZ" ? "file-size" : "cpu", `Producer terminated with ${cause.snapshot.signal}; the signal's exact cause is not independently established.`, effectiveLimits === undefined ? null : { ...effectiveLimits }, detail, ); - return new AnalysisOutputError(this.operation, cause.message, detail); + return new AnalysisOutputError(operation, cause.message, detail); } async finish( result: Result, observation: (value: T) => AnalysisPartialObservation, + operation = this.operation, ): Promise> { try { await this.release(); @@ -223,7 +232,7 @@ export class ArtifactAnalysisResources { : projectAnalysisError(result.error), }, { - operation: this.operation, + operation, cause, ...(partialObservation === undefined ? {} : { partialObservation }), }, @@ -260,11 +269,11 @@ export class ArtifactAnalysisResources { if (descriptorFailure !== undefined) throw descriptorFailure; } - #artifactFailure(cause: unknown): AnalysisError { + #artifactFailure(cause: unknown, operation = this.operation): AnalysisError { if (cause instanceof AnalysisError) return cause; if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) - return this.cleanupFailure(cause.cleanup, cause); - return new ProviderAdapterError(this.providerId, this.operation, { + return this.cleanupFailure(cause.cleanup, cause, undefined, operation); + return new ProviderAdapterError(this.providerId, operation, { cause, diagnostics: { stage: "artifact-resources", reason: String(cause) }, }); diff --git a/src/server/registerCaptureTools.ts b/src/server/registerCaptureTools.ts index c097f89be..296faba5c 100644 --- a/src/server/registerCaptureTools.ts +++ b/src/server/registerCaptureTools.ts @@ -28,4 +28,17 @@ export const registerCaptureTools = ( toolRegistrationOptions(contract), handler(contract), ); + const discovery = toolContract("inspect_packet_fields"); + const discover = createEvidenceToolHandler( + server, + (input, options) => service.inspectFields(input, options), + logger, + recordEvidence, + withAdmittedAnalysis, + ); + server.registerTool( + discovery.name, + toolRegistrationOptions(discovery), + discover(discovery), + ); }; diff --git a/tests/boundary/captures/captureProvider.test.ts b/tests/boundary/captures/captureProvider.test.ts index 9f5fb4afc..2428b39d4 100644 --- a/tests/boundary/captures/captureProvider.test.ts +++ b/tests/boundary/captures/captureProvider.test.ts @@ -1,3 +1,4 @@ +import { packetFieldsCatalogSchema } from "../../../src/domain/captures/packetFields.js"; import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; import { it, expect, onTestFinished } from "vitest"; import { writeFile, readdir, chmod, symlink } from "node:fs/promises"; @@ -20,14 +21,15 @@ const fixture = async (mode = "ok", failCleanup = false) => { await writeFile( script, `import {readFileSync, writeFileSync, chmodSync, truncateSync} from 'node:fs'; -if (process.argv.includes('--version')) { console.log('TShark (Wireshark) 4.6.9 (fixture)'); console.error('version warning'); } +if (process.argv.includes('--version')) { console.log(${JSON.stringify(mode)} === 'bad-version' ? 'TShark (Wireshark) 3.0.0' : 'TShark (Wireshark) 4.6.9 (fixture)'); console.error('version warning'); } else if (${JSON.stringify(mode)} === 'nonzero') { console.error('invalid decoder'); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'malformed-copy') { chmodSync('capture', 0o600); writeFileSync('capture', 'changed'); } else if (${JSON.stringify(mode)} === 'oversized-copy') { chmodSync('capture', 0o600); truncateSync('capture', 32 * 1024 * 1024 + 1); console.log('[]'); } -else if (${JSON.stringify(mode)} === 'overflow') process.stdout.write('x'.repeat(5 * 1024 * 1024)); +else if (${JSON.stringify(mode)} === 'overflow') process.stdout.write('x'.repeat((process.argv.includes('-G') ? 33 : 5) * 1024 * 1024)); else if (${JSON.stringify(mode)} === 'wait') { console.log('ready'); setInterval(() => {}, 1000); } else if (${JSON.stringify(mode)} === 'invalid-json') console.log('not json'); else if (${JSON.stringify(mode)} === 'many') console.log(JSON.stringify(Array.from({length:90},(_,i)=>({_source:{layers:{"frame.number":[String(i+1)],"ip.addr":["192.0.2.1"]}}})))); +else if (process.argv.includes('-G')) { console.log('P\\tUniversal Serial Bus\\tusb\\nF\\tVendor ID\\tusb.idVendor\\tFT_UINT16\\tusb\\tBASE_HEX\\t0x0\\t\\nF\\tOther Vendor\\tusb.idVendor\\tFT_UINT32\\tusb\\tBASE_HEX\\t0x0\\tSecond registration\\nP\\tInternet Protocol\\tip\\nF\\tAddress\\tip.addr\\tFT_IPv4\\tip\\t\\t0x0\\tSource or destination'); console.error('glossary warning'); } else { if(readFileSync('capture', 'utf8') !== 'recording bytes') throw Error('wrong copy'); console.log(JSON.stringify([{_source:{layers:{"frame.number":["1"],"ip.addr":["192.0.2.1","192.0.2.2"]}}}])); console.error('producer warning'); } `, ); @@ -301,6 +303,9 @@ it("retains a completed observation when cancellation arrives at the service bou const execution = await f.service.provider.inspect(input); if (!execution.ok) throw execution.error; const service = new PacketCaptureService({ + inspectFields: async () => { + throw new Error("Unexpected field discovery call"); + }, inspect: async () => { f.controller.abort(); return execution; @@ -465,3 +470,158 @@ it("rejects oversized selector lists and UTF-8 argv expansion before provider wo } expect(f.calls()).toBe(0); }); + +it("discovers installed protocols and field definitions without a capture", async () => { + const f = await fixture(); + for (const [input, expected] of [ + [ + { query: "USB" }, + { + records: [{ abbreviation: "usb", kind: "protocol", field_type: null }], + output_window: { examined: 2, matching: 1, returned: 1 }, + }, + ], + [ + { kind: "fields", protocol: "usb", limit: 1 }, + { + records: [ + { + abbreviation: "usb.idVendor", + field_type: "FT_UINT16", + source_line: 2, + }, + ], + output_window: { + examined: 3, + matching: 2, + returned: 1, + has_more: true, + }, + }, + ], + [ + { kind: "fields", protocol: "usb", offset: 1, limit: null }, + { + records: [ + { + abbreviation: "usb.idVendor", + field_type: "FT_UINT32", + source_line: 3, + }, + ], + output_window: { matching: 2, returned: 1, has_more: false }, + }, + ], + [ + { kind: "fields", query: "destination" }, + { + records: [ + { + abbreviation: "ip.addr", + description: "Source or destination", + display_base: "", + }, + ], + }, + ], + [ + { query: "no-such-protocol" }, + { records: [], output_window: { matching: 0, returned: 0 } }, + ], + ] as const) { + const result = await f.service.inspectFields(input); + if (!result.ok) throw result.error; + const report = packetFieldsCatalogSchema.parse( + result.value.normalized_result, + ); + expect(report).toMatchObject(expected); + expect(report.tool.arguments).toEqual(["-G", "fields"]); + expect(report.tool.version_diagnostics).toBe("version warning\n"); + expect(report.diagnostics).toBe("glossary warning\n"); + expect(report.limits.output_bytes).toBe(32 * 1024 * 1024); + } + expect( + (await readdir(f.directory)).filter((name) => name.startsWith("decoder-")), + ).toEqual([]); +}); + +it("shares discovery admission with inspection and refuses work after shutdown", async () => { + const f = await fixture("gated"); + const active = f.service.inspect({ path: f.path, fields: ["ip.addr"] }); + await f.acquired; + const busy = await f.service.inspectFields({}); + f.release(); + if (!(await active).ok) throw new Error("Inspection failed"); + expect(busy).toMatchObject({ + ok: false, + error: { operation: "inspect_packet_fields", resource: "capacity" }, + }); + await f.service.close(); + expect(await f.service.inspectFields({})).toMatchObject({ + ok: false, + error: { + operation: "inspect_packet_fields", + _tag: "AnalysisCancelledError", + }, + }); +}); + +it.each([ + "bad-version", + "nonzero", + "overflow", + "wait", + "oversized-tool", + "invalid-json", +])("retains field discovery failure facts: %s", async (mode) => { + const f = await fixture(mode); + const result = await f.service.inspectFields( + {}, + { signal: f.controller.signal }, + ); + if (result.ok) throw new Error("Expected failure"); + expect(result.error).toMatchObject({ operation: "inspect_packet_fields" }); + if (mode === "bad-version") + expect(result.error.capturedOutput).toMatchObject({ + stdout: "TShark (Wireshark) 3.0.0\n", + stderr: "version warning\n", + }); + if (mode === "invalid-json") + expect(result.error.capturedOutput?.stdout).toBe("not json\n"); + if (mode === "overflow") + expect(result.error).toMatchObject({ + resource: "memory", + reportedLimits: { output_bytes: 32 * 1024 * 1024 }, + }); +}); + +it("retains completed discovery on late cancellation and cleanup failure", async () => { + for (const mode of ["post-run-abort", "cleanup"]) { + const f = await fixture(mode, mode === "cleanup"); + const result = await f.service.inspectFields( + { query: "usb" }, + { signal: f.controller.signal }, + ); + expect(result).toMatchObject({ + ok: false, + error: { + partialObservation: { + operation: "inspect_packet_fields", + normalized_result: { records: [{ abbreviation: "usb" }] }, + }, + }, + }); + if (mode === "cleanup") { + await f.service.inspectFields({}); + await f.service.inspect({ path: f.path, fields: ["ip.addr"] }); + expect(f.calls()).toBe(2); + for (let attempt = 0; attempt < 2; attempt++) + await expect(f.service.close()).rejects.toMatchObject({ + operation: "inspect_packet_fields", + cleanupIncomplete: true, + }); + } + f.allowCleanup(); + await f.service.close(); + } +}); diff --git a/tests/boundary/captures/fieldDiscovery.test.ts b/tests/boundary/captures/fieldDiscovery.test.ts new file mode 100644 index 000000000..79a27f7bf --- /dev/null +++ b/tests/boundary/captures/fieldDiscovery.test.ts @@ -0,0 +1,67 @@ +import { expect, it } from "vitest"; +import { inspectPacketFieldsInputSchema } from "../../../src/domain/captures/packetFields.js"; +import { parseFieldDiscovery } from "../../../src/captures/parseFieldDiscovery.js"; + +it("keeps duplicate registrations and source ordinals instead of overwriting abbreviations", () => { + const text = + "P\tUSB\tusb\r\nF\tVendor\tusb.idVendor\tFT_UINT16\tusb\tBASE_HEX\t0x0\t\r\nF\tVendor\tusb.idVendor\tFT_UINT32\tusb\tBASE_HEX\t0x0\t"; + const result = parseFieldDiscovery( + text, + inspectPacketFieldsInputSchema.parse({ + kind: "fields", + query: "idVendor", + offset: 1, + }), + ); + expect(result.records).toMatchObject([ + { + source_line: 3, + abbreviation: "usb.idVendor", + field_type: "FT_UINT32", + description: "", + }, + ]); + expect(result.output_window).toEqual({ + examined: 2, + matching: 2, + returned: 1, + has_more: false, + }); +}); + +it("rejects damaged records even outside the selected page", () => { + for (const text of [ + "", + "P\tmissing", + "F\tname\tabbr", + "P\tname\t", + "P\tUSB\tusb\nBAD", + ]) + expect(() => + parseFieldDiscovery( + text, + inspectPacketFieldsInputSchema.parse({ limit: 1 }), + ), + ).toThrow(); +}); + +it("bounds explicit full-result expansion while allowing a page from the same large glossary", () => { + const text = "P\t" + "Long name ".repeat(30) + "\tabbr\n"; + const glossary = text.repeat(6000); + expect(() => + parseFieldDiscovery( + glossary, + inspectPacketFieldsInputSchema.parse({ limit: null }), + ), + ).toThrow("representation budget"); + const result = parseFieldDiscovery( + glossary, + inspectPacketFieldsInputSchema.parse({ offset: 5999, limit: 1 }), + ); + expect(result.output_window).toEqual({ + examined: 6000, + matching: 6000, + returned: 1, + has_more: false, + }); +}); diff --git a/tests/boundary/cli/captureCleanup.test.ts b/tests/boundary/cli/captureCleanup.test.ts index 1ec6adedd..8f34b6839 100644 --- a/tests/boundary/cli/captureCleanup.test.ts +++ b/tests/boundary/cli/captureCleanup.test.ts @@ -32,6 +32,9 @@ it.each([false, true])( silentLogger, {}, { + inspectFields: async () => { + throw new Error("Unexpected field discovery call"); + }, inspect: async () => err(original), close: async () => { await Promise.resolve(); @@ -87,6 +90,9 @@ it("rejects an oversized selector budget before the CLI invokes its service", as silentLogger, {}, { + inspectFields: async () => { + throw new Error("Unexpected field discovery call"); + }, inspect: async () => { called = true; throw new Error("Must not inspect"); @@ -125,6 +131,9 @@ it("rejects conflicting output options before the CLI invokes its service", asyn silentLogger, {}, { + inspectFields: async () => { + throw new Error("Unexpected field discovery call"); + }, inspect: async () => { called = true; throw new Error("Must not inspect"); diff --git a/tests/boundary/mcp/packetCaptureMcp.test.ts b/tests/boundary/mcp/packetCaptureMcp.test.ts index e11b60d3e..a5acf1925 100644 --- a/tests/boundary/mcp/packetCaptureMcp.test.ts +++ b/tests/boundary/mcp/packetCaptureMcp.test.ts @@ -8,42 +8,53 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -it("advertises valid target-free packet capture schemas and actionable missing-tool diagnostics", async () => { - const harness = await createApplicationMcpHarness({ - serverOptions: { environment: {} }, - }); - onTestFinished(harness.close); - const tool = (await harness.client.listTools()).tools.find( - ({ name }) => name === "inspect_packet_capture", - ); - if (tool?.outputSchema === undefined) throw new Error("Capture tool missing"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - const inputSchema: Record = tool.inputSchema; - const outputSchema: Record = tool.outputSchema; - expect(ajv.validateSchema(inputSchema)).toBe(true); - expect(ajv.validateSchema(outputSchema)).toBe(true); - const input = { - path: - process.platform === "win32" ? "C:\\recording.pcap" : "/recording.pcap", - fields: ["ip.addr"], - }; - expect(ajv.validate(inputSchema, input)).toBe(true); - expect(ajv.validate(inputSchema, { ...input, interface: "lo0" })).toBe(false); - const result = await harness.client.callTool({ - name: tool.name, - arguments: input, - }); - expect(parseMcpToolError(result)).toMatchObject({ - error: { - code: "capability_unavailable", - details: { - operation: tool.name, - reason: expect.stringContaining("REA_TSHARK_COMMAND"), +it.each(["inspect_packet_capture", "inspect_packet_fields"])( + "advertises valid target-free schemas and missing-tool diagnostics: %s", + async (name) => { + const harness = await createApplicationMcpHarness({ + serverOptions: { environment: {} }, + }); + onTestFinished(harness.close); + const tool = (await harness.client.listTools()).tools.find( + (tool) => tool.name === name, + ); + if (tool?.outputSchema === undefined) + throw new Error("Capture tool missing"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = tool.inputSchema; + const outputSchema: Record = tool.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + const input = + name === "inspect_packet_fields" + ? { query: "usb" } + : { + path: + process.platform === "win32" + ? "C:\\recording.pcap" + : "/recording.pcap", + fields: ["ip.addr"], + }; + expect(ajv.validate(inputSchema, input)).toBe(true); + expect(ajv.validate(inputSchema, { ...input, interface: "lo0" })).toBe( + false, + ); + const result = await harness.client.callTool({ + name: tool.name, + arguments: input, + }); + expect(parseMcpToolError(result)).toMatchObject({ + error: { + code: "capability_unavailable", + details: { + operation: tool.name, + reason: expect.stringContaining("REA_TSHARK_COMMAND"), + }, }, - }, - }); - await harness.client.ping(); -}); + }); + await harness.client.ping(); + }, +); it.each(["cancelled", "cleanup"])( "retains completed Evidence after %s failure", @@ -78,6 +89,9 @@ it.each(["cancelled", "cleanup"])( }, ); const packetCapture = new PacketCaptureService({ + inspectFields: async () => { + throw new Error("Unexpected field discovery call"); + }, inspect: async () => err(failure), close: async () => {}, }); @@ -120,6 +134,9 @@ it.each(["cancelled", "cleanup"])( it("rejects oversized selector expansion through the MCP boundary before invoking the provider", async () => { let calls = 0; const packetCapture = new PacketCaptureService({ + inspectFields: async () => { + throw new Error("Unexpected field discovery call"); + }, inspect: async () => { calls++; return err(new AnalysisCancelledError("inspect_packet_capture")); From fe8d0bdd36fe12bc0836468d1a702a34fbdc219b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:56:04 +0800 Subject: [PATCH 17/27] fix(android): reject malformed UTF-8 at the JADX protocol boundary --- src/android/JadxMcpTransport.ts | 8 +- .../boundary/android/jadxMcpTransport.test.ts | 94 +++++++++++-------- 2 files changed, 61 insertions(+), 41 deletions(-) diff --git a/src/android/JadxMcpTransport.ts b/src/android/JadxMcpTransport.ts index c648696fc..a40d517f5 100644 --- a/src/android/JadxMcpTransport.ts +++ b/src/android/JadxMcpTransport.ts @@ -199,9 +199,11 @@ export class JadxMcpTransport implements Transport { throw new Error(`JADX MCP frame exceeds ${MAX_FRAME_BYTES} bytes`); this.#appendFrame(fragment, frameBytes); if (newline === -1) return; - const line = this.#frameBuffer - .subarray(0, this.#frameLength) - .toString("utf8"); + const line = new TextDecoder("utf-8", { + fatal: true, + // Preserve a leading BOM so the JSON-RPC parser can reject it. + ignoreBOM: true, + }).decode(this.#frameBuffer.subarray(0, this.#frameLength)); this.#frameBuffer = Buffer.alloc(0); this.#frameLength = 0; this.onmessage?.(deserializeMessage(line)); diff --git a/tests/boundary/android/jadxMcpTransport.test.ts b/tests/boundary/android/jadxMcpTransport.test.ts index afe79fc09..fe05f14b4 100644 --- a/tests/boundary/android/jadxMcpTransport.test.ts +++ b/tests/boundary/android/jadxMcpTransport.test.ts @@ -26,6 +26,26 @@ class FixtureProcess extends EventEmitter implements ProviderProcessHandle { } } +const fixtureLauncher = + (process: FixtureProcess): JadxLauncher => + async ({ runId }) => ({ + process, + ownership: { + runId, + leaderPid: process.pid, + processGroupId: process.pid, + expectedParentPid: process.pid, + }, + cleanup: async () => { + process.emit("exit", 0, null); + process.emit("close", 0, null); + process.stdout.end(); + process.stderr.end(); + process.stdin.end(); + return { cleaned: true, signaled: false }; + }, + }); + it("joins pending input errors before detaching the owned protocol stream", async () => { let completeWrite: ((cause: Error) => void) | undefined; const failure = new Error("owned pipe closed during cancellation"); @@ -70,26 +90,9 @@ it("joins pending input errors before detaching the owned protocol stream", asyn it("decodes split UTF-8 and newline bytes, then routes multiple complete frames", async () => { const process = new FixtureProcess(); - const launcher: JadxLauncher = async ({ runId }) => ({ - process, - ownership: { - runId, - leaderPid: process.pid, - processGroupId: process.pid, - expectedParentPid: process.pid, - }, - cleanup: async () => { - process.emit("exit", 0, null); - process.emit("close", 0, null); - process.stdout.end(); - process.stderr.end(); - process.stdin.end(); - return { cleaned: true, signaled: false }; - }, - }); const transport = new JadxMcpTransport( { command: "fixture", arguments: [] }, - launcher, + fixtureLauncher(process), ); const received: JSONRPCMessage[] = []; transport.onmessage = (message) => received.push(message); @@ -98,7 +101,9 @@ it("decodes split UTF-8 and newline bytes, then routes multiple complete frames" const first = { jsonrpc: "2.0", id: 1, - result: { content: [{ type: "text", text: "snow 雪 🦊" }] }, + result: { + content: [{ type: "text", text: "snow 雪 🦊 and literal �" }], + }, } satisfies JSONRPCMessage; const second = { jsonrpc: "2.0", @@ -120,28 +125,41 @@ it("decodes split UTF-8 and newline bytes, then routes multiple complete frames" await transport.close(); }); -it("accepts a complete JSON-RPC frame at the 8 MiB boundary", async () => { +it.each([ + ["invalid UTF-8 byte", Buffer.from([0xff])], + ["truncated UTF-8 sequence", Buffer.from([0xe2, 0x82])], +])("rejects %s in a complete JSON-RPC frame", async (_label, invalid) => { const process = new FixtureProcess(); - const launcher: JadxLauncher = async ({ runId }) => ({ - process, - ownership: { - runId, - leaderPid: process.pid, - processGroupId: process.pid, - expectedParentPid: process.pid, - }, - cleanup: async () => { - process.emit("exit", 0, null); - process.emit("close", 0, null); - process.stdout.end(); - process.stderr.end(); - process.stdin.end(); - return { cleaned: true, signaled: false }; - }, - }); const transport = new JadxMcpTransport( { command: "fixture", arguments: [] }, - launcher, + fixtureLauncher(process), + ); + const received: JSONRPCMessage[] = []; + const errors: Error[] = []; + transport.onmessage = (message) => received.push(message); + transport.onerror = (cause) => errors.push(cause); + + await transport.start(); + process.stdout.write( + Buffer.concat([ + Buffer.from('{"jsonrpc":"2.0","id":1,"result":{"text":"'), + invalid, + Buffer.from('"}}\n'), + ]), + ); + await new Promise((resolve) => setImmediate(resolve)); + await transport.close(); + + expect(received).toEqual([]); + expect(errors).toHaveLength(1); + expect(errors[0]?.message).toMatch(/encoded data was not valid/i); +}); + +it("accepts a complete JSON-RPC frame at the 8 MiB boundary", async () => { + const process = new FixtureProcess(); + const transport = new JadxMcpTransport( + { command: "fixture", arguments: [] }, + fixtureLauncher(process), ); const received: JSONRPCMessage[] = []; transport.onmessage = (message) => received.push(message); From 4276aaf18bb607cdd0182280f76cf50fcd110c3c Mon Sep 17 00:00:00 2001 From: Olorin <59321404+Olorinm@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:58:19 +0800 Subject: [PATCH 18/27] feat(signals): discover installed decoders and option help (#2487) * feat(signals): discover installed decoders and selected option help * fix(signals): bound retained discovery data and preserve resource failures * fix(signals): complete CLI help and portable decoder patterns * test(signals): verify case-insensitive decoder help queries * fix(signals): retain discovery operation across cleanup retries --- .../skills/reverse-engineer-anything/SKILL.md | 3 + .github/workflows/real-signal-recording.yml | 2 +- docs/signal-recordings.md | 34 +++ scripts/verify-signal-recording.mjs | 106 ++++++- src/application/CapabilityInventory.ts | 2 +- .../signals/SignalRecordingService.ts | 70 ++++- src/cli/signalCommands.ts | 95 ++++-- src/cliCommandNames.ts | 1 + src/contracts/signals/signalToolContracts.ts | 20 ++ src/contracts/toolEffects.ts | 5 + src/domain/signals/signalDecoders.ts | 86 ++++++ src/server/registerSignalTools.ts | 13 + src/signals/SigrokRecordingProvider.ts | 278 +++++++++++------- src/signals/parseDecoderDiscovery.ts | 79 +++++ tests/boundary/cli/signalCleanup.test.ts | 6 + tests/boundary/mcp/signalRecordingMcp.test.ts | 81 ++--- .../boundary/signals/decoderDiscovery.test.ts | 48 +++ .../signals/recordingProvider.test.ts | 189 +++++++++++- 18 files changed, 945 insertions(+), 173 deletions(-) create mode 100644 src/domain/signals/signalDecoders.ts create mode 100644 src/signals/parseDecoderDiscovery.ts create mode 100644 tests/boundary/signals/decoderDiscovery.test.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 48bc8b7df..de9f6b7c1 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -92,6 +92,9 @@ explicit path or endpoint and do not need it. [packet summaries](https://github.com/morluto/rea/blob/main/docs/pcap-analysis.md). Select an exact DNS question for address pages; page exhaustion does not prove network completeness. This reads recordings and does not acquire traffic. +- Decoder discovery: `inspect_signal_decoders` searches installed decoder IDs + or selected decoder help, including channels and options. Query and page the + relevant details before choosing settings; this does not scan hardware. - Recorded logic/signal traces: `decode_signal_recording` with explicit decoder, channel and protocol settings. On Linux with util-linux prlimit, supply sigrok-cli 0.7.2 through `REA_SIGROK_COMMAND`; see [recording guide](https://github.com/morluto/rea/blob/main/docs/signal-recordings.md). diff --git a/.github/workflows/real-signal-recording.yml b/.github/workflows/real-signal-recording.yml index 5ec413e72..cc902037e 100644 --- a/.github/workflows/real-signal-recording.yml +++ b/.github/workflows/real-signal-recording.yml @@ -44,7 +44,7 @@ jobs: - run: sudo apt-get update && sudo apt-get install -y sigrok-cli util-linux python3 - run: npm ci - run: npm run build:cached - - run: npm run test:focused -- src/process/LinuxResourceLimits.test.ts src/domain/signals/annotationTextWindow.test.ts tests/boundary/signals/recordingProvider.test.ts tests/boundary/signals/recordingDescriptorCleanup.test.ts tests/boundary/signals/resourceLimits.test.ts tests/boundary/mcp/signalRecordingMcp.test.ts tests/boundary/cli/signalCleanup.test.ts + - run: npm run test:focused -- tests/boundary/signals/decoderDiscovery.test.ts src/process/LinuxResourceLimits.test.ts src/domain/signals/annotationTextWindow.test.ts tests/boundary/signals/recordingProvider.test.ts tests/boundary/signals/recordingDescriptorCleanup.test.ts tests/boundary/signals/resourceLimits.test.ts tests/boundary/mcp/signalRecordingMcp.test.ts tests/boundary/cli/signalCleanup.test.ts - run: node scripts/verify-signal-recording.mjs env: REA_SIGROK_COMMAND: /usr/bin/sigrok-cli diff --git a/docs/signal-recordings.md b/docs/signal-recordings.md index 520d6e882..69b14d193 100644 --- a/docs/signal-recordings.md +++ b/docs/signal-recordings.md @@ -106,3 +106,37 @@ One decode is admitted at a time, including acquisition and cleanup. Excess requests receive a retryable capacity error before acquiring another file or worker. Shutdown closes admission and drains accepted work; failed cleanup remains owned and retryable. + +## Discover decoders before choosing settings + +`inspect_signal_decoders` / `inspect-signal-decoders` queries the **installed** +sigrok catalog without opening a recording or scanning devices: + +```sh +rea inspect-signal-decoders --query uart +rea inspect-signal-decoders --decoder uart --query baud +rea inspect-signal-decoders --decoder i2c --offset 0 --limit 50 +``` + +Omit `decoder` to search decoder IDs and names. Supply one exact decoder ID to +read its literal help, including required/optional channels, options, annotation +classes and documentation. `query` is a case-insensitive literal substring; +matching help retains original one-based line numbers. Empty query selects all. +`offset` skips matches; `limit` defaults to 50. MCP `limit: null` or CLI `--all` +returns all remaining matches; CLI `--all` and `--limit` conflict. +The window reports examined, matching and returned records and `has_more`. +A filtered empty page is not evidence that the installation has no decoders. + +Discovery refuses more than 65,535 line separators before constructing record +objects (65,536-line representation budget), in addition to the 4 MiB output cap. +Exceeding that budget is a resource failure with captured output. Returned text +is copied so a retained page does not keep the complete producer output alive. + +Discovery shares decoding's owned process, admission slot, resource limits, +cleanup retries and Evidence delivery. It records the executable and limiter +identities, version banner and warnings. Failed version checks and malformed +producer output preserve diagnostics. Help is producer text, not an independently +validated parameter schema. Each request probes again; it does not cache or claim +stable pages across installation changes. Libraries/decoder modules remain +trusted and are not fully fingerprinted. Catalog output includes a SHA-256 of +its executable, not a digest of the complete installation. diff --git a/scripts/verify-signal-recording.mjs b/scripts/verify-signal-recording.mjs index a095b97ff..f5baaf0b9 100644 --- a/scripts/verify-signal-recording.mjs +++ b/scripts/verify-signal-recording.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env node import assert from "node:assert/strict"; +import { estimateTokenCount } from "tokenx"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { readFile, writeFile, readdir } from "node:fs/promises"; @@ -68,6 +69,20 @@ for (const byte of Buffer.from("REA")) { lines.push(`#${tick + 1000}`, "1!"); const source = lines.join("\n") + "\n"; let cases = 0; +const discoveryMeasurements = []; +const inheritedAddressSpace = (await readFile("/proc/self/limits", "utf8")) + .split("\n") + .find((line) => line.startsWith("Max address space ")) + ?.trim() + .match(/^Max address space\s+(\S+)\s+(\S+)\s+bytes$/u); +assert.ok(inheritedAddressSpace); +const expectedAddressSpace = Math.min( + 1073741824, + ...inheritedAddressSpace + .slice(1) + .map((value) => (value === "unlimited" ? Infinity : Number(value))), +); + try { await writeFile(path, source); await client.connect(transport); @@ -104,14 +119,16 @@ try { createHash("sha256").update(source).digest("hex"), ); assert.equal(report.hardware_acquisition, "not-performed"); - assert.equal(report.limits.address_space_bytes, 1073741824); + assert.equal(report.limits.address_space_bytes, expectedAddressSpace); assert.equal(report.tool.configured_path, process.env.REA_SIGROK_COMMAND); assert.equal( report.resource_limiter.configured_path, process.env.REA_SIGROK_PRLIMIT_COMMAND ?? "/usr/bin/prlimit", ); assert.ok( - report.resource_limiter.arguments.includes("--as=1073741824:1073741824"), + report.resource_limiter.arguments.includes( + `--as=${expectedAddressSpace}:${expectedAddressSpace}`, + ), ); assert.equal(report.diagnostics, ""); const flags = Object.entries(input) @@ -245,6 +262,88 @@ try { }); assert.equal(invalid.isError, true); cases++; + const discoveryTool = (await client.listTools()).tools.find( + ({ name }) => name === "inspect_signal_decoders", + ); + assert.ok(discoveryTool?.outputSchema); + assert.ok(ajv.validateSchema(discoveryTool.inputSchema)); + assert.ok(ajv.validateSchema(discoveryTool.outputSchema)); + const nativeHelp = ( + await execute( + process.env.REA_SIGROK_COMMAND, + ["--dont-scan", "--protocol-decoders=uart", "--show"], + { env: environment, timeout: 30000, maxBuffer: 4 * 1024 * 1024 }, + ) + ).stdout.split(/\r?\n/); + if (nativeHelp.at(-1) === "") nativeHelp.pop(); + for (const input of [ + { query: "uart" }, + { limit: 1 }, + { offset: 1, limit: 2 }, + { decoder: "uart", query: "BaUd" }, + { decoder: "uart", offset: 50, limit: null }, + { query: "rea_no_such_decoder" }, + ]) { + const started = performance.now(); + const response = await client.callTool({ + name: discoveryTool.name, + arguments: input, + }); + assert.notEqual(response.isError, true, JSON.stringify(response)); + assert.ok( + ajv.validate(discoveryTool.outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ); + const evidence = parseEvidence(response.structuredContent); + const report = evidence.normalized_result; + discoveryMeasurements.push({ + input, + elapsed_ms: +(performance.now() - started).toFixed(2), + response_bytes: Buffer.byteLength(JSON.stringify(response)), + estimated_tokens_one_tool_request_response: estimateTokenCount( + JSON.stringify(discoveryTool) + + JSON.stringify(input) + + JSON.stringify(response), + ), + }); + if (input.decoder) { + const matched = nativeHelp + .map((text, i) => ({ line_number: i + 1, text })) + .filter(({ text }) => + text.toLowerCase().includes((input.query ?? "").toLowerCase()), + ); + assert.deepEqual( + report.help_lines, + matched.slice( + input.offset ?? 0, + input.limit === null + ? undefined + : (input.offset ?? 0) + (input.limit ?? 50), + ), + ); + } else if (input.query === "uart") + assert.ok(report.decoders.some(({ id }) => id === "uart")); + const flags = Object.entries(input).flatMap(([key, value]) => + value === null ? ["--all"] : [`--${key}`, String(value)], + ); + const cli = JSON.parse( + ( + await execute( + process.execPath, + [entrypoint, "inspect-signal-decoders", ...flags, "--format", "json"], + { env: environment, timeout: 60000, maxBuffer: 8 * 1024 * 1024 }, + ) + ).stdout, + ); + assert.deepEqual(cli.normalized_result, report); + cases++; + } + const unavailable = await client.callTool({ + name: discoveryTool.name, + arguments: { decoder: "rea_no_such_decoder" }, + }); + assert.equal(unavailable.isError, true); + cases++; await client.close(); assert.deepEqual((await readdir(root.path)).sort(), [ "paged.vcd", @@ -260,6 +359,9 @@ console.log( { status: "passed", cases, + discovery_measurements: discoveryMeasurements, + measurement_scope: + "Warm MCP server, fresh sigrok processes; tokenx estimates include one advertised tool/request/response, not billed model usage. No model-driven selection benchmark, CPU or peak process-family memory measurement; no cache or speedup claim.", host: `${process.platform}/${process.arch}`, scope: "real sigrok UART VCD through compiled CLI/MCP; no physical acquisition", diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 94518db85..6fefb8e20 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -405,7 +405,7 @@ const workflowAvailabilityFor = ({ remediation: "On Linux, supply TShark 4.x via absolute REA_TSHARK_COMMAND and util-linux prlimit (REA_TSHARK_PRLIMIT_COMMAND, default /usr/bin/prlimit). No active binary target or hardware is required; the tool is checked at execution.", }; - if (name === "decode_signal_recording") + if (name === "decode_signal_recording" || name === "inspect_signal_decoders") return policy.signalRecordingEnabled === true ? { reason: "available", remediation: null } : { diff --git a/src/application/signals/SignalRecordingService.ts b/src/application/signals/SignalRecordingService.ts index 921ac19c7..e5b99f1b7 100644 --- a/src/application/signals/SignalRecordingService.ts +++ b/src/application/signals/SignalRecordingService.ts @@ -1,3 +1,9 @@ +import { + inspectSignalDecodersInputSchema, + signalDecodersSchema, + type InspectSignalDecodersInput, +} from "../../domain/signals/signalDecoders.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; import { isAbsolute } from "node:path"; import type { AnalysisExecution, @@ -24,6 +30,10 @@ export interface SignalRecordingPort { input: DecodeSignalRecordingInput, options?: ExecutionOptions, ): Promise>; + inspectDecoders( + input: InspectSignalDecodersInput, + options?: ExecutionOptions, + ): Promise>; close(): Promise; } @@ -74,14 +84,58 @@ export class SignalRecordingService { "Decoder returned malformed data or changed the selected recording identity/options.", ), ); - const evidence = createEvidence(execution.subject, execution.provider, { - operation, - parameters: input.data, - result: report.data, - confidence: "observed", - locations: execution.locations, - limitations: execution.limitations, - }); + return this.#evidence(operation, input.data, execution, options); + } + async inspectDecoders( + raw: unknown, + options?: ExecutionOptions, + ): Promise> { + const operation = "inspect_signal_decoders"; + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(operation)); + const input = inspectSignalDecodersInputSchema.safeParse(raw); + if (!input.success) + return err( + analysisInputErrorFromIssues(operation, input.error.issues, raw, { + cause: input.error, + }), + ); + const decoded = await this.provider.inspectDecoders(input.data, options); + if (!decoded.ok) return decoded; + const execution = decoded.value; + const report = signalDecodersSchema.safeParse(execution.result); + if ( + !report.success || + JSON.stringify(report.data.selection) !== JSON.stringify(input.data) || + execution.subject?.path !== report.data.tool.path || + execution.subject.sha256 !== report.data.tool.sha256 + ) + return err( + new AnalysisOutputError( + operation, + "Decoder discovery returned malformed data or changed the selected installation/options.", + ), + ); + return this.#evidence(operation, input.data, execution, options); + } + #evidence( + operation: string, + parameters: Readonly>, + execution: AnalysisExecution, + options?: ExecutionOptions, + ): Result { + const evidence = createEvidence( + execution.subject ?? undefined, + execution.provider, + { + operation, + parameters, + result: execution.result, + confidence: "observed", + locations: execution.locations, + limitations: execution.limitations, + }, + ); return options?.signal?.aborted ? err( new AnalysisCancelledError(operation, { diff --git a/src/cli/signalCommands.ts b/src/cli/signalCommands.ts index 02e5555f5..e7d254507 100644 --- a/src/cli/signalCommands.ts +++ b/src/cli/signalCommands.ts @@ -1,3 +1,4 @@ +import { inspectSignalDecodersInputSchema } from "../domain/signals/signalDecoders.js"; import { z } from "incur"; import type { SignalRecordingService } from "../application/signals/SignalRecordingService.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; @@ -22,9 +23,75 @@ export const registerSignalCommands = ( environment: Readonly, service: Pick< SignalRecordingService, - "decode" | "close" + "decode" | "inspectDecoders" | "close" > = createSignalRecordingService(environment), ): void => { + const finish = async ( + result: Awaited>, + operation: string, + ) => { + // A one-shot command must retry retained cleanup before discarding its owner. + try { + await service.close(); + } catch (cause) { + const cleanup = + cause instanceof AnalysisError + ? cause + : new ProviderCleanupError( + "sigrok-recording", + [], + { reason: String(cause) }, + { cause, operation: operation }, + ); + if (result.ok) cleanup.retainPartialObservation(result.value); + return projectAnalysisError( + result.ok + ? cleanup + : analysisErrorWithCleanupFailure(result.error, cleanup, operation), + ); + } + return result.ok ? result.value : projectAnalysisError(result.error); + }; + cli.command(CLI_COMMANDS.inspectSignalDecoders, { + description: "Search installed sigrok decoders or selected decoder help", + options: z + .object({ + ...inspectSignalDecodersInputSchema.shape, + limit: z + .number() + .int() + .positive() + .max(Number.MAX_SAFE_INTEGER) + .optional() + .describe( + "Maximum matching decoders or help lines to return; defaults to 50", + ), + all: z + .boolean() + .default(false) + .describe("Return all remaining matches"), + }) + .superRefine((options, context) => { + if (options.all && options.limit !== undefined) + context.addIssue({ + code: "custom", + path: ["all"], + message: "all and limit are mutually exclusive", + }); + }), + run: ({ options: { all, ...options } }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectSignalDecoders, async () => + finish( + await service.inspectDecoders( + { ...options, ...(all ? { limit: null } : {}) }, + { signal }, + ), + "inspect_signal_decoders", + ), + ), + ), + }); cli.command(CLI_COMMANDS.decodeSignalRecording, { description: "Decode an existing signal recording with caller-supplied sigrok", @@ -68,31 +135,7 @@ export const registerSignalCommands = ( }, { signal }, ); - // A one-shot command must retry retained cleanup before discarding its owner. - try { - await service.close(); - } catch (cause) { - const cleanup = - cause instanceof AnalysisError - ? cause - : new ProviderCleanupError( - "sigrok-recording", - [], - { reason: String(cause) }, - { cause, operation: "decode_signal_recording" }, - ); - if (result.ok) cleanup.retainPartialObservation(result.value); - return projectAnalysisError( - result.ok - ? cleanup - : analysisErrorWithCleanupFailure( - result.error, - cleanup, - "decode_signal_recording", - ), - ); - } - return result.ok ? result.value : projectAnalysisError(result.error); + return finish(result, "decode_signal_recording"); }), ), }); diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index c3d373e4e..d63a9a2bc 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -2,6 +2,7 @@ export const CLI_COMMANDS = Object.freeze({ inspectPacketFields: "inspect-packet-fields", inspectPacketCapture: "inspect-packet-capture", + inspectSignalDecoders: "inspect-signal-decoders", decodeSignalRecording: "decode-signal-recording", inspectWasmArtifact: "inspect-wasm-artifact", inspectSqliteDatabase: "inspect-sqlite-database", diff --git a/src/contracts/signals/signalToolContracts.ts b/src/contracts/signals/signalToolContracts.ts index 4a055f12e..f545c452c 100644 --- a/src/contracts/signals/signalToolContracts.ts +++ b/src/contracts/signals/signalToolContracts.ts @@ -1,3 +1,7 @@ +import { + inspectSignalDecodersInputSchema, + signalDecodersSchema, +} from "../../domain/signals/signalDecoders.js"; import { decodeSignalRecordingInputSchema, signalRecordingSchema, @@ -6,6 +10,22 @@ import type { ToolContract } from "../toolContractTypes.js"; import { toolContractMetadata } from "../toolEffects.js"; import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; export const SIGNAL_TOOL_CONTRACTS = [ + { + name: "inspect_signal_decoders", + ...toolContractMetadata("inspect_signal_decoders"), + kind: "artifact-provider", + description: + "Discover installed sigrok protocol decoders or inspect one decoder's channel/option help on Linux. Omit decoder to list ID/name pairs; supply its exact ID for literal help lines with source line numbers. query is a case-insensitive literal substring. Page matches with offset/limit (default 50; null returns all remaining). Reports tool identity, version, diagnostics and examined/matching coverage. Uses configured REA_SIGROK_COMMAND and prlimit; no recording, binary target, hardware scan or acquisition. Each call probes again; decoder modules are trusted, not fully fingerprinted. Use returned IDs/options with decode_signal_recording.", + inputSchema: inspectSignalDecodersInputSchema, + outputSchema: evidenceResultOf(signalDecodersSchema), + examples: [ + { title: "Find installed UART decoders", input: { query: "uart" } }, + { + title: "Read the installed UART baud-rate option", + input: { decoder: "uart", query: "baud" }, + }, + ], + }, { name: "decode_signal_recording", ...toolContractMetadata("decode_signal_recording"), diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 8ea7f23e0..70a40c6a1 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -69,6 +69,11 @@ export const TOOL_EFFECTS: Readonly> = { launchesProcess: true, writesFilesystem: true, }), + inspect_signal_decoders: effects({ + mutatesSession: true, + launchesProcess: true, + writesFilesystem: true, + }), decode_signal_recording: effects({ mutatesSession: true, launchesProcess: true, diff --git a/src/domain/signals/signalDecoders.ts b/src/domain/signals/signalDecoders.ts new file mode 100644 index 000000000..ae6754cca --- /dev/null +++ b/src/domain/signals/signalDecoders.ts @@ -0,0 +1,86 @@ +import { z } from "zod"; +import { signalRecordingSchema } from "./signalRecording.js"; + +export const inspectSignalDecodersInputSchema = z.strictObject({ + decoder: z + .string() + .regex(/^[a-zA-Z0-9_+\x2d]+$/u) + .max(256) + .nullable() + .default(null) + .describe( + "Exact installed decoder ID for help; null searches the decoder catalog", + ), + query: z + .string() + .max(8192) + .default("") + .describe( + "Case-insensitive literal substring in decoder ID/name or help lines; empty selects all", + ), + offset: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER) + .default(0) + .describe( + "Number of matching decoders or help lines to skip; defaults to 0", + ), + limit: z + .number() + .int() + .positive() + .max(Number.MAX_SAFE_INTEGER) + .nullable() + .default(50) + .describe( + "Maximum matching decoders or help lines returned; null requests all remaining", + ), +}); +export type InspectSignalDecodersInput = z.infer< + typeof inspectSignalDecodersInputSchema +>; +export const signalDecodersSchema = z + .strictObject({ + selection: inspectSignalDecodersInputSchema, + decoders: z.array(z.strictObject({ id: z.string(), name: z.string() })), + help_lines: z.array( + z.strictObject({ + line_number: z.number().int().positive(), + text: z.string(), + }), + ), + output_window: z.strictObject({ + unit: z.enum(["decoders", "help-lines"]), + examined: z.number().int().nonnegative(), + matching: z.number().int().nonnegative(), + returned: z.number().int().nonnegative(), + has_more: z.boolean(), + }), + diagnostics: z.string(), + tool: signalRecordingSchema.shape.tool, + resource_limiter: signalRecordingSchema.shape.resource_limiter, + limits: signalRecordingSchema.shape.limits, + hardware_acquisition: z.literal("not-performed"), + }) + .superRefine((report, context) => { + const w = report.output_window; + const catalog = report.selection.decoder === null; + const remaining = Math.max(0, w.matching - report.selection.offset); + const expected = Math.min(remaining, report.selection.limit ?? remaining); + if ( + w.unit !== (catalog ? "decoders" : "help-lines") || + w.matching > w.examined || + w.returned !== expected || + (catalog ? report.decoders : report.help_lines).length !== expected || + (catalog ? report.help_lines : report.decoders).length !== 0 || + w.has_more !== remaining > expected + ) + context.addIssue({ + code: "custom", + path: ["output_window"], + message: + "Discovery coverage must match the selected page and result kind", + }); + }); diff --git a/src/server/registerSignalTools.ts b/src/server/registerSignalTools.ts index 87a67361f..f833ffbd5 100644 --- a/src/server/registerSignalTools.ts +++ b/src/server/registerSignalTools.ts @@ -28,4 +28,17 @@ export const registerSignalTools = ( toolRegistrationOptions(contract), handler(contract), ); + const discovery = toolContract("inspect_signal_decoders"); + const discover = createEvidenceToolHandler( + server, + (input, options) => service.inspectDecoders(input, options), + logger, + recordEvidence, + withAdmittedAnalysis, + ); + server.registerTool( + discovery.name, + toolRegistrationOptions(discovery), + discover(discovery), + ); }; diff --git a/src/signals/SigrokRecordingProvider.ts b/src/signals/SigrokRecordingProvider.ts index b0eb5c4ba..2e903d4c6 100644 --- a/src/signals/SigrokRecordingProvider.ts +++ b/src/signals/SigrokRecordingProvider.ts @@ -1,3 +1,11 @@ +import { + type InspectSignalDecodersInput, + signalDecodersSchema, +} from "../domain/signals/signalDecoders.js"; +import { + DecoderDiscoveryBudgetError, + parseDecoderDiscovery, +} from "./parseDecoderDiscovery.js"; import { readLinuxResourceLimits, type LinuxResourceLimits, @@ -81,7 +89,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { readLimits, { message: - "A signal decoding operation is already active; retry after it completes.", + "A sigrok operation is already active; retry after it completes.", details: { max_concurrent_decodes: 1 }, }, ); @@ -91,17 +99,28 @@ export class SigrokRecordingProvider implements SignalRecordingPort { options?: ExecutionOptions, ): Promise> { return this.#resources.run( - () => this.#decode(input, options), + () => this.#executeRequest(input, options), options?.signal, ); } + inspectDecoders( + input: InspectSignalDecodersInput, + options?: ExecutionOptions, + ): Promise> { + return this.#resources.run( + () => this.#executeRequest(input, options), + options?.signal, + "inspect_signal_decoders", + ); + } close(): Promise { return this.#resources.close(); } - async #decode( - input: DecodeSignalRecordingInput, + async #executeRequest( + input: DecodeSignalRecordingInput | InspectSignalDecodersInput, options?: ExecutionOptions, ): Promise> { + const operation = "path" in input ? OPERATION : "inspect_signal_decoders"; let phase: | "configuration" | "artifact" @@ -109,12 +128,12 @@ export class SigrokRecordingProvider implements SignalRecordingPort { | "producer" | "integrity" = "configuration"; let capturedOutput: AnalysisCapturedOutput | undefined; - let resourcePath = input.path; + let resourcePath = "path" in input ? input.path : "sigrok decoder catalog"; let effectiveLimits: LinuxResourceLimits | undefined; let root: ArtifactAnalysisRuntime | undefined; let result: Result; try { - if (options?.signal?.aborted) throw new AnalysisCancelledError(OPERATION); + if (options?.signal?.aborted) throw new AnalysisCancelledError(operation); await this.#resources.release(); if (this.platform !== "linux") throw new Error( @@ -154,7 +173,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { ]; phase = "workspace"; root = await this.createRuntime(); - this.#resources.retain(root, undefined); + this.#resources.retain(root, undefined, operation); const snapshotPath = join(root.path, "recording"); const execute = async (args: readonly string[]) => { const output = await runOwnedCommand( @@ -197,24 +216,24 @@ export class SigrokRecordingProvider implements SignalRecordingPort { ); if (currentLimiter.sha256 !== limiterIdentity.sha256) throw new AnalysisArtifactChangedError( - OPERATION, + operation, limiter, "Configured resource limiter changed during analysis.", { capturedOutput }, ); if (current.sha256 !== tool.sha256) throw new AnalysisArtifactChangedError( - OPERATION, + operation, command, "Configured decoder executable changed during analysis.", { capturedOutput }, ); phase = previousPhase; - resourcePath = input.path; + resourcePath = "path" in input ? input.path : command; if (output.exitCode !== 0) throw new ProviderAdapterError( SIGROK_PROVIDER_IDENTITY.id, - OPERATION, + operation, { diagnostics: { exit_code: output.exitCode ?? null, @@ -234,65 +253,15 @@ export class SigrokRecordingProvider implements SignalRecordingPort { throw new Error( "Expected sigrok-cli 0.7.2 with libsigrokdecode; inspect the selected installation.", ); - phase = "artifact"; - const selected = await this.#resources.read( - input.path, - LIMITS.input_bytes, - options?.signal, - ); - phase = "workspace"; - await writeFile(snapshotPath, selected.bytes, { - flag: "wx", - mode: 0o400, - }); - phase = "producer"; - const args = [ - "--dont-scan", - "--input-file=recording", - `--protocol-decoders=${input.decoders}`, - "--protocol-decoder-samplenum", - ...(input.input_format === null - ? [] - : [`--input-format=${input.input_format}`]), - ...(input.annotations === null - ? [] - : [`--protocol-decoder-annotations=${input.annotations}`]), - ]; - const output = await execute(args); - phase = "integrity"; - resourcePath = snapshotPath; - if ( - (await this.#resources.read(snapshotPath, LIMITS.input_bytes)) - .sha256 !== selected.sha256 - ) - throw new AnalysisArtifactChangedError( - OPERATION, - snapshotPath, - "Decoder changed the private recording copy.", - capturedOutput === undefined ? {} : { capturedOutput }, - ); - phase = "producer"; - resourcePath = input.path; - const report = signalRecordingSchema.parse({ - artifact: { - path: input.path, - sha256: selected.sha256, - bytes: selected.bytes.length, - }, - selection: input, - ...windowAnnotationText( - output.stdout.text, - input.line_offset, - input.line_limit, - ), - diagnostics: output.stderr.text, + const metadata = (args: readonly string[], diagnostics: string) => ({ + diagnostics, tool: { configured_path: configured, path: command, sha256: tool.sha256, version_banner: version.stdout.text, version_diagnostics: version.stderr.text, - arguments: args, + arguments: [...args], }, resource_limiter: { configured_path: configuredLimiter, @@ -301,31 +270,108 @@ export class SigrokRecordingProvider implements SignalRecordingPort { arguments: limitArguments, }, limits: { ...LIMITS, ...effectiveLimits }, - hardware_acquisition: "not-performed", + hardware_acquisition: "not-performed" as const, }); - result = ok( - createAnalysisExecution(report, SIGROK_PROVIDER_IDENTITY, { - subject: { + if (!("path" in input)) { + phase = "producer"; + const args = + input.decoder === null + ? ["--dont-scan", "--list-supported"] + : ["--dont-scan", `--protocol-decoders=${input.decoder}`, "--show"]; + const output = await execute(args); + const report = signalDecodersSchema.parse({ + selection: input, + ...parseDecoderDiscovery(output.stdout.text, input), + ...metadata(args, output.stderr.text), + }); + result = ok( + createAnalysisExecution(report, SIGROK_PROVIDER_IDENTITY, { + subject: { path: command, sha256: tool.sha256, format: "file" }, + limitations: [ + "Catalog and help are observed from the selected sigrok installation. Help lines preserve producer descriptions, not a separately validated parameter schema.", + "Each request probes the installation again; pages do not share a frozen catalog. Decoder modules and libraries are trusted and not fully fingerprinted.", + "Hardware discovery, recording reads and acquisition are not requested. Linux per-process limits and bounded output apply; this is not an OS sandbox.", + ], + }), + ); + } else { + phase = "artifact"; + const selected = await this.#resources.read( + input.path, + LIMITS.input_bytes, + options?.signal, + ); + phase = "workspace"; + await writeFile(snapshotPath, selected.bytes, { + flag: "wx", + mode: 0o400, + }); + phase = "producer"; + const args = [ + "--dont-scan", + "--input-file=recording", + `--protocol-decoders=${input.decoders}`, + "--protocol-decoder-samplenum", + ...(input.input_format === null + ? [] + : [`--input-format=${input.input_format}`]), + ...(input.annotations === null + ? [] + : [`--protocol-decoder-annotations=${input.annotations}`]), + ]; + const output = await execute(args); + phase = "integrity"; + resourcePath = snapshotPath; + if ( + (await this.#resources.read(snapshotPath, LIMITS.input_bytes)) + .sha256 !== selected.sha256 + ) + throw new AnalysisArtifactChangedError( + operation, + snapshotPath, + "Decoder changed the private recording copy.", + capturedOutput === undefined ? {} : { capturedOutput }, + ); + phase = "producer"; + resourcePath = "path" in input ? input.path : command; + const report = signalRecordingSchema.parse({ + artifact: { path: input.path, sha256: selected.sha256, - format: "file", + bytes: selected.bytes.length, }, - locations: [{ kind: "artifact-path", path: input.path }], - limitations: [ - "Annotations are the selected decoder's interpretation of the recorded samples and caller-selected settings, not independently verified protocol semantics. Empty output does not establish absence of traffic.", - "Sample-number ranges and display text are preserved verbatim, not converted to timestamps or parsed as a lossless byte stream. All producer stderr is retained, including warnings on exit zero.", - "Only a private copy of the selected file is decoded. Hardware discovery/acquisition and target execution are not requested. Caller-supplied tools, plugins and decoder modules are trusted; this is not an OS sandbox.", - "Linux prlimit enforces soft and hard ceilings before execution, preserving stricter inherited limits: at most 1 GiB address space per process, 64 MiB per file, 30 CPU seconds and no core dumps. Effective limits are reported. Resident input is capped at 32 MiB, each command at 30 wall-clock seconds and combined output at 4 MiB. Output overflow fails. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted.", - ], - }), - ); + selection: input, + ...windowAnnotationText( + output.stdout.text, + input.line_offset, + input.line_limit, + ), + ...metadata(args, output.stderr.text), + }); + result = ok( + createAnalysisExecution(report, SIGROK_PROVIDER_IDENTITY, { + subject: { + path: input.path, + sha256: selected.sha256, + format: "file", + }, + locations: [{ kind: "artifact-path", path: input.path }], + limitations: [ + "Annotations are the selected decoder's interpretation of the recorded samples and caller-selected settings, not independently verified protocol semantics. Empty output does not establish absence of traffic.", + "Sample-number ranges and display text are preserved verbatim, not converted to timestamps or parsed as a lossless byte stream. All producer stderr is retained, including warnings on exit zero.", + "Only a private copy of the selected file is decoded. Hardware discovery/acquisition and target execution are not requested. Caller-supplied tools, plugins and decoder modules are trusted; this is not an OS sandbox.", + "Linux prlimit enforces soft and hard ceilings before execution, preserving stricter inherited limits: at most 1 GiB address space per process, 64 MiB per file, 30 CPU seconds and no core dumps. Effective limits are reported. Resident input is capped at 32 MiB, each command at 30 wall-clock seconds and combined output at 4 MiB. Output overflow fails. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted.", + ], + }), + ); + } } catch (cause) { if ( root !== undefined && cause instanceof OwnedCommandFailure && cause.cleanupOwner !== undefined ) - this.#resources.retain(root, cause.cleanupOwner); + this.#resources.retain(root, cause.cleanupOwner, operation); result = err( this.#failure( cause, @@ -334,18 +380,22 @@ export class SigrokRecordingProvider implements SignalRecordingPort { root?.path ?? null, effectiveLimits, capturedOutput, + operation, ), ); } - return this.#resources.finish(result, (execution) => - createEvidence(execution.subject ?? undefined, execution.provider, { - operation: OPERATION, - parameters: input, - result: execution.result, - confidence: "observed", - locations: execution.locations, - limitations: execution.limitations, - }), + return this.#resources.finish( + result, + (execution) => + createEvidence(execution.subject ?? undefined, execution.provider, { + operation, + parameters: input, + result: execution.result, + confidence: "observed", + locations: execution.locations, + limitations: execution.limitations, + }), + operation, ); } #failure( @@ -360,6 +410,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { workspacePath: string | null, effectiveLimits: LinuxResourceLimits | undefined, observedOutput: AnalysisCapturedOutput | undefined, + operation = OPERATION, ): AnalysisError { if (cause instanceof AnalysisError) return cause; if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) @@ -367,15 +418,34 @@ export class SigrokRecordingProvider implements SignalRecordingPort { cause.cleanup, cause, observedOutput, + operation, ); if ( (cause instanceof Error && cause.name === "AbortError") || (cause instanceof ArtifactReaderFailure && cause.reason === "cancelled") ) - return new AnalysisCancelledError(OPERATION, { cause }); + return new AnalysisCancelledError(operation, { cause }); + if (cause instanceof DecoderDiscoveryBudgetError) + return new AnalysisResourceConstraintError( + operation, + "memory", + cause.message, + { line_budget: 65536 }, + { + cause, + ...(observedOutput === undefined + ? {} + : { capturedOutput: observedOutput }), + }, + ); const reason = cause instanceof Error ? cause.message : String(cause); if (cause instanceof OwnedCommandFailure) - return this.#resources.commandFailure(cause, LIMITS, effectiveLimits); + return this.#resources.commandFailure( + cause, + LIMITS, + effectiveLimits, + operation, + ); if (phase === "workspace") { const code = cause instanceof Error && @@ -385,13 +455,13 @@ export class SigrokRecordingProvider implements SignalRecordingPort { : null; if (code === "ENOSPC" || code === "EDQUOT" || code === "EFBIG") return new AnalysisResourceConstraintError( - OPERATION, + operation, "file-size", `Private analysis workspace failed (${code}): ${reason}`, { workspace_path: workspacePath, system_code: code }, { cause }, ); - return new ProviderAdapterError(SIGROK_PROVIDER_IDENTITY.id, OPERATION, { + return new ProviderAdapterError(SIGROK_PROVIDER_IDENTITY.id, operation, { cause, userMessage: `Cannot prepare the private analysis workspace: ${reason}`, diagnostics: { @@ -414,12 +484,12 @@ export class SigrokRecordingProvider implements SignalRecordingPort { ["limit", "integrity", "path"].includes(cause.reason) ) return new AnalysisArtifactChangedError( - OPERATION, + operation, path, `Post-run identity verification failed for ${path}: ${reason}`, detail, ); - return new ProviderAdapterError(SIGROK_PROVIDER_IDENTITY.id, OPERATION, { + return new ProviderAdapterError(SIGROK_PROVIDER_IDENTITY.id, operation, { ...detail, diagnostics: { stage: "integrity", resource_path: path, reason }, }); @@ -427,10 +497,13 @@ export class SigrokRecordingProvider implements SignalRecordingPort { if (phase === "configuration") return new AnalysisCapabilityUnavailableError( SIGROK_PROVIDER_IDENTITY.id, - OPERATION, + operation, reason, { cause, + ...(observedOutput === undefined + ? {} + : { capturedOutput: observedOutput }), userMessage: `${reason} On Linux, configure sigrok-cli 0.7.2 through REA_SIGROK_COMMAND and util-linux prlimit through REA_SIGROK_PRLIMIT_COMMAND (default /usr/bin/prlimit).`, }, ); @@ -440,14 +513,14 @@ export class SigrokRecordingProvider implements SignalRecordingPort { cause.reason === "limit" ) return new AnalysisResourceConstraintError( - OPERATION, + operation, "file-size", reason, { input_bytes: LIMITS.input_bytes }, { cause }, ); if (cause instanceof ArtifactReaderFailure && cause.reason === "integrity") - return new AnalysisArtifactChangedError(OPERATION, path, reason, { + return new AnalysisArtifactChangedError(operation, path, reason, { cause, }); if ( @@ -456,13 +529,18 @@ export class SigrokRecordingProvider implements SignalRecordingPort { "code" in cause && (cause.code === "EACCES" || cause.code === "EPERM") ) - return new AnalysisAccessDeniedError(OPERATION, path, cause.code, { + return new AnalysisAccessDeniedError(operation, path, cause.code, { cause, }); if (phase === "artifact") - return new AnalysisInputError(OPERATION, { cause }, [ + return new AnalysisInputError(operation, { cause }, [ { path: ["path"], reason: "invalid_value", message: reason }, ]); - return new AnalysisOutputError(OPERATION, reason, { cause }); + return new AnalysisOutputError(operation, reason, { + cause, + ...(observedOutput === undefined + ? {} + : { capturedOutput: observedOutput }), + }); } } diff --git a/src/signals/parseDecoderDiscovery.ts b/src/signals/parseDecoderDiscovery.ts new file mode 100644 index 000000000..230abe2b0 --- /dev/null +++ b/src/signals/parseDecoderDiscovery.ts @@ -0,0 +1,79 @@ +import type { InspectSignalDecodersInput } from "../domain/signals/signalDecoders.js"; + +export class DecoderDiscoveryBudgetError extends Error {} + +const copyText = (value: string): string => + Buffer.from(value, "utf16le").toString("utf16le"); + +/** Parse only the 0.7.2 catalog grammar; help remains literal producer text. */ +export const parseDecoderDiscovery = ( + text: string, + input: InspectSignalDecodersInput, +) => { + // Bound record-object expansion before splitting the already byte-bounded output. + // 65,536 records leaves an 8 MiB budget at 128 bytes of bookkeeping per record. + let separators = 0; + for (const character of text) + if (character === "\n" && ++separators >= 65536) + throw new DecoderDiscoveryBudgetError( + "Decoder discovery exceeds the 65,536-line representation budget", + ); + const lines = text.split(/\r?\n/u); + if (lines.at(-1) === "") lines.pop(); + const query = input.query.toLowerCase(); + const decoders: { id: string; name: string }[] = []; + const help: { line_number: number; text: string }[] = []; + if (input.decoder === null) { + const start = lines.indexOf("Supported protocol decoders:"); + if (start < 0) throw new Error("sigrok decoder catalog heading is missing"); + const seen = new Set(); + for (const line of lines.slice(start + 1)) { + if (line === "") continue; + const match = /^ {2}([a-zA-Z0-9_+-]+) +(.+)$/u.exec(line); + if ( + match?.[1] === undefined || + match[2] === undefined || + seen.has(match[1]) + ) + throw new Error("Malformed or duplicate sigrok decoder catalog record"); + seen.add(match[1]); + decoders.push({ id: match[1], name: match[2] }); + } + if (decoders.length === 0) + throw new Error("sigrok returned an empty decoder catalog"); + } else { + if (lines[0] !== `ID: ${input.decoder}`) + throw new Error("Decoder help does not identify the selected decoder"); + lines.forEach((line, i) => help.push({ line_number: i + 1, text: line })); + } + const catalog = input.decoder === null; + const matching = catalog + ? decoders.filter(({ id, name }) => + `${id} ${name}`.toLowerCase().includes(query), + ) + : help.filter(({ text: line }) => line.toLowerCase().includes(query)); + const start = Math.min(input.offset, matching.length); + const count = Math.min( + input.limit ?? matching.length, + matching.length - start, + ); + // Keep retained pages independent of the complete producer-output allocation. + const page = matching + .slice(start, start + count) + .map((record) => + "id" in record + ? { id: copyText(record.id), name: copyText(record.name) } + : { line_number: record.line_number, text: copyText(record.text) }, + ); + return { + decoders: catalog ? page : [], + help_lines: catalog ? [] : page, + output_window: { + unit: catalog ? "decoders" : "help-lines", + examined: catalog ? decoders.length : help.length, + matching: matching.length, + returned: page.length, + has_more: start + count < matching.length, + }, + }; +}; diff --git a/tests/boundary/cli/signalCleanup.test.ts b/tests/boundary/cli/signalCleanup.test.ts index 594a46cc0..dc706c17c 100644 --- a/tests/boundary/cli/signalCleanup.test.ts +++ b/tests/boundary/cli/signalCleanup.test.ts @@ -30,6 +30,9 @@ it.each([ silentLogger, {}, { + inspectDecoders: async () => { + throw new Error("Unexpected discovery call"); + }, decode: async () => { called = true; throw new Error("Must not decode"); @@ -84,6 +87,9 @@ it.each([false, true])( silentLogger, {}, { + inspectDecoders: async () => { + throw new Error("Unexpected discovery call"); + }, decode: async () => err(original), close: async () => { await Promise.resolve(); diff --git a/tests/boundary/mcp/signalRecordingMcp.test.ts b/tests/boundary/mcp/signalRecordingMcp.test.ts index ecabfd8e5..edd74a2bf 100644 --- a/tests/boundary/mcp/signalRecordingMcp.test.ts +++ b/tests/boundary/mcp/signalRecordingMcp.test.ts @@ -8,41 +8,51 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -it("advertises valid target-free signal decoding schemas and actionable missing-tool diagnostics", async () => { - const harness = await createApplicationMcpHarness({ - serverOptions: { environment: {} }, - }); - onTestFinished(harness.close); - const tool = (await harness.client.listTools()).tools.find( - ({ name }) => name === "decode_signal_recording", - ); - if (tool?.outputSchema === undefined) throw new Error("Signal tool missing"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - const inputSchema: Record = tool.inputSchema; - const outputSchema: Record = tool.outputSchema; - expect(ajv.validateSchema(inputSchema)).toBe(true); - expect(ajv.validateSchema(outputSchema)).toBe(true); - const input = { - path: process.platform === "win32" ? "C:\\recording.vcd" : "/recording.vcd", - decoders: "uart", - }; - expect(ajv.validate(inputSchema, input)).toBe(true); - expect(ajv.validate(inputSchema, { ...input, driver: "demo" })).toBe(false); - const result = await harness.client.callTool({ - name: tool.name, - arguments: input, - }); - expect(parseMcpToolError(result)).toMatchObject({ - error: { - code: "capability_unavailable", - details: { - operation: tool.name, - reason: expect.stringContaining("REA_SIGROK_COMMAND"), +it.each(["decode_signal_recording", "inspect_signal_decoders"])( + "advertises valid target-free schemas and missing-tool diagnostics: %s", + async (name) => { + const harness = await createApplicationMcpHarness({ + serverOptions: { environment: {} }, + }); + onTestFinished(harness.close); + const tool = (await harness.client.listTools()).tools.find( + (tool) => tool.name === name, + ); + if (tool?.outputSchema === undefined) + throw new Error("Signal tool missing"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = tool.inputSchema; + const outputSchema: Record = tool.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + const input = + name === "inspect_signal_decoders" + ? { query: "uart" } + : { + path: + process.platform === "win32" + ? "C:\\recording.vcd" + : "/recording.vcd", + decoders: "uart", + }; + expect(ajv.validate(inputSchema, input)).toBe(true); + expect(ajv.validate(inputSchema, { ...input, driver: "demo" })).toBe(false); + const result = await harness.client.callTool({ + name: tool.name, + arguments: input, + }); + expect(parseMcpToolError(result)).toMatchObject({ + error: { + code: "capability_unavailable", + details: { + operation: tool.name, + reason: expect.stringContaining("REA_SIGROK_COMMAND"), + }, }, - }, - }); - await harness.client.ping(); -}); + }); + await harness.client.ping(); + }, +); it.each(["cancelled", "cleanup"])( "retains completed Evidence after %s failure", @@ -77,6 +87,9 @@ it.each(["cancelled", "cleanup"])( }, ); const signalRecording = new SignalRecordingService({ + inspectDecoders: async () => { + throw new Error("Unexpected discovery call"); + }, decode: async () => err(failure), close: async () => {}, }); diff --git a/tests/boundary/signals/decoderDiscovery.test.ts b/tests/boundary/signals/decoderDiscovery.test.ts new file mode 100644 index 000000000..6b69d679f --- /dev/null +++ b/tests/boundary/signals/decoderDiscovery.test.ts @@ -0,0 +1,48 @@ +import { expect, it } from "vitest"; +import { inspectSignalDecodersInputSchema } from "../../../src/domain/signals/signalDecoders.js"; +import { parseDecoderDiscovery } from "../../../src/signals/parseDecoderDiscovery.js"; + +it("rejects malformed catalogs and the wrong decoder identity", () => { + for (const text of [ + "", + "Supported protocol decoders:\n", + "Supported protocol decoders:\n uart UART\n uart Duplicate", + "Supported protocol decoders:\n malformed", + ]) { + expect(() => + parseDecoderDiscovery(text, inspectSignalDecodersInputSchema.parse({})), + ).toThrow(); + } + expect(() => + parseDecoderDiscovery( + "ID: spi\nOptions:", + inspectSignalDecodersInputSchema.parse({ decoder: "uart" }), + ), + ).toThrow(); +}); + +it("keeps help line ordinals, blank lines and literal special characters through selection", () => { + const text = "ID: uart\r\nOptions:\r\n\r\n- mode: A[0] (default 'x')\r\n"; + const result = parseDecoderDiscovery( + text, + inspectSignalDecodersInputSchema.parse({ + decoder: "uart", + query: "[0]", + offset: 0, + limit: 1, + }), + ); + expect(result.help_lines).toEqual([ + { line_number: 4, text: "- mode: A[0] (default 'x')" }, + ]); + expect(result.output_window.examined).toBe(4); +}); + +it("bounds line expansion before constructing help records", () => { + expect(() => + parseDecoderDiscovery( + "ID: uart\n" + "\n".repeat(65536), + inspectSignalDecodersInputSchema.parse({ decoder: "uart" }), + ), + ).toThrow("representation budget"); +}); diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 9ddf2c21a..27bc6ff09 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -1,3 +1,4 @@ +import { signalDecodersSchema } from "../../../src/domain/signals/signalDecoders.js"; import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; import { it, expect, onTestFinished } from "vitest"; import { writeFile, readdir, chmod, symlink } from "node:fs/promises"; @@ -23,13 +24,17 @@ const fixture = async (mode = "ok", failCleanup = false) => { await writeFile( script, `import {readFileSync, writeFileSync, chmodSync, truncateSync} from 'node:fs'; -if (process.argv.includes('--version')) { console.log('sigrok-cli 0.7.2\\nlibsigrokdecode 0.5.3'); console.error('version warning'); } +if (process.argv.includes('--version')) { console.log(${JSON.stringify(mode)} === 'bad-version' ? 'sigrok-cli 0.7.1' : 'sigrok-cli 0.7.2\\nlibsigrokdecode 0.5.3'); console.error('version warning'); } +else if (${JSON.stringify(mode)} === 'bad-version') console.log('unexpected version fallback'); else if (${JSON.stringify(mode)} === 'nonzero') { console.error('invalid decoder'); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'malformed-copy') { chmodSync('recording', 0o600); writeFileSync('recording', 'changed'); } else if (${JSON.stringify(mode)} === 'oversized-copy') { chmodSync('recording', 0o600); truncateSync('recording', 32 * 1024 * 1024 + 1); } +else if (${JSON.stringify(mode)} === 'line-budget') process.stdout.write('ID: uart' + '\\n'.repeat(65536)); else if (${JSON.stringify(mode)} === 'overflow') process.stdout.write('x'.repeat(5 * 1024 * 1024)); else if (${JSON.stringify(mode)} === 'wait') { console.log('ready'); setInterval(() => {}, 1000); } else if (${JSON.stringify(mode)} === 'many') process.stdout.write(Array.from({length:90},(_,i)=>'line '+(i+1)+'\\r\\n').join('')); +else if (process.argv.includes('--list-supported')) { console.log('Supported hardware drivers:\\n ignored Device\\nSupported protocol decoders:\\n uart Universal Asynchronous Receiver/Transmitter\\n i2c Inter-Integrated Circuit\\n'); console.error('catalog warning'); } +else if (process.argv.includes('--show')) console.log('ID: uart\\nOptions:\\n- baudrate: Baud rate (default 115200)\\n- parity: Parity'); else { if(readFileSync('recording', 'utf8') !== 'recording bytes') throw Error('wrong copy'); console.log('10-20 uart-1: 42'); console.error('producer warning'); } `, ); @@ -285,6 +290,9 @@ it("retains a completed observation when cancellation arrives at the service bou const execution = await f.service.provider.decode(input); if (!execution.ok) throw execution.error; const service = new SignalRecordingService({ + inspectDecoders: async () => { + throw new Error("Unexpected discovery call"); + }, decode: async () => { f.controller.abort(); return execution; @@ -419,3 +427,182 @@ it("preserves termination signal and effective file ceiling as a resource failur capturedOutput: { signal: "SIGXFSZ", stderr: "before file limit\n" }, }); }); + +it("discovers decoders without reading a recording and retains installation provenance", async () => { + const f = await fixture(); + for (const [input, expected] of [ + [ + { query: "UART" }, + { + decoders: [ + { id: "uart", name: "Universal Asynchronous Receiver/Transmitter" }, + ], + output_window: { + examined: 2, + matching: 1, + returned: 1, + has_more: false, + }, + }, + ], + [ + { limit: 1 }, + { + decoders: [ + { id: "uart", name: "Universal Asynchronous Receiver/Transmitter" }, + ], + output_window: { matching: 2, returned: 1, has_more: true }, + }, + ], + [ + { offset: 1, limit: null }, + { + decoders: [{ id: "i2c", name: "Inter-Integrated Circuit" }], + output_window: { returned: 1, has_more: false }, + }, + ], + [ + { decoder: "uart", query: "baud" }, + { + help_lines: [ + { line_number: 3, text: "- baudrate: Baud rate (default 115200)" }, + ], + output_window: { unit: "help-lines", examined: 4, matching: 1 }, + }, + ], + [ + { query: "no-such-decoder" }, + { + decoders: [], + output_window: { examined: 2, matching: 0, returned: 0 }, + }, + ], + ] as const) { + const result = await f.service.inspectDecoders(input); + if (!result.ok) throw result.error; + const report = signalDecodersSchema.parse(result.value.normalized_result); + expect(report).toMatchObject(expected); + expect(result.value.operation).toBe("inspect_signal_decoders"); + expect(report.tool.version_diagnostics).toBe("version warning\n"); + expect(report.tool.arguments).toContain("--dont-scan"); + expect( + report.tool.arguments.some((arg) => arg.startsWith("--input-file")), + ).toBe(false); + } + expect( + (await readdir(f.directory)).filter((name) => name.startsWith("decoder-")), + ).toEqual([]); +}); + +it("attributes discovery admission failures to discovery while sharing the decoding slot", async () => { + const f = await fixture("gated"); + const active = f.service.decode({ path: f.path, decoders: "uart" }); + await f.acquired; + const busy = await f.service.inspectDecoders({}); + f.release(); + if (!(await active).ok) throw new Error("Decode failed"); + expect(busy).toMatchObject({ + ok: false, + error: { operation: "inspect_signal_decoders", resource: "capacity" }, + }); + await f.service.close(); + expect(await f.service.inspectDecoders({})).toMatchObject({ + ok: false, + error: { + operation: "inspect_signal_decoders", + _tag: "AnalysisCancelledError", + }, + }); +}); + +it.each([ + "bad-version", + "nonzero", + "overflow", + "line-budget", + "wait", + "oversized-tool", +])("preserves discovery failure diagnostics: %s", async (mode) => { + const f = await fixture(mode); + const result = await f.service.inspectDecoders( + {}, + { signal: f.controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failure"); + expect(result.error).toMatchObject({ + operation: "inspect_signal_decoders", + }); + if (mode === "line-budget") { + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + }); + expect(result.error.capturedOutput?.stdout).toBe( + "ID: uart" + "\n".repeat(65536), + ); + } + if (mode === "bad-version") + expect(result.error.capturedOutput).toMatchObject({ + stdout: "sigrok-cli 0.7.1\n", + stderr: "version warning\n", + }); +}); + +it("retains completed discovery when cleanup fails, then retries before accepting work", async () => { + const f = await fixture("ok", true); + const first = await f.service.inspectDecoders({ query: "uart" }); + expect(first).toMatchObject({ + ok: false, + error: { + operation: "inspect_signal_decoders", + partialObservation: { operation: "inspect_signal_decoders" }, + }, + }); + await f.service.inspectDecoders({}); + await f.service.decode({ path: f.path, decoders: "uart" }); + expect(f.calls()).toBe(2); + for (let attempt = 0; attempt < 2; attempt++) + await expect(f.service.close()).rejects.toMatchObject({ + operation: "inspect_signal_decoders", + cleanupIncomplete: true, + }); + f.allowCleanup(); + await f.service.close(); +}); + +it("rejects ambiguous decoder selectors before launch and retains malformed help output", async () => { + const f = await fixture(); + for (const decoder of ["uart:baudrate=1", "uart,spi", "uart\0"]) { + expect(await f.service.inspectDecoders({ decoder })).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + } + expect(f.calls()).toBe(0); + expect(await f.service.inspectDecoders({ decoder: "spi" })).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisOutputError", + capturedOutput: { stdout: expect.stringContaining("ID: uart") }, + }, + }); +}); + +it("retains completed discovery on cancellation during identity verification", async () => { + const f = await fixture("post-run-abort"); + expect( + await f.service.inspectDecoders( + { query: "uart" }, + { signal: f.controller.signal }, + ), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { + operation: "inspect_signal_decoders", + normalized_result: { decoders: [{ id: "uart" }] }, + }, + }, + }); +}); From b953f96c0aae27e1876c317a8f7db1d6f292b8ff Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:00:14 +0800 Subject: [PATCH 19/27] fix(ghidra): share Java properties semantics with PSP extension admission --- src/ghidra/GhidraInstallationPolicy.ts | 116 +----------------- src/ghidra/GhidraJavaProperties.ts | 115 +++++++++++++++++ src/ghidra/GhidraPspExtension.ts | 23 ++-- .../filesystem/ghidraPspExtension.test.ts | 37 +++++- 4 files changed, 161 insertions(+), 130 deletions(-) create mode 100644 src/ghidra/GhidraJavaProperties.ts diff --git a/src/ghidra/GhidraInstallationPolicy.ts b/src/ghidra/GhidraInstallationPolicy.ts index ead513cd8..8a35c66c3 100644 --- a/src/ghidra/GhidraInstallationPolicy.ts +++ b/src/ghidra/GhidraInstallationPolicy.ts @@ -1,4 +1,4 @@ -import { err, ok, type Result } from "../domain/result.js"; +import { parseGhidraJavaProperties } from "./GhidraJavaProperties.js"; /** Ghidra build whose bridge behavior this REA release verifies. */ export const SUPPORTED_GHIDRA_VERSION = "12.1.4"; @@ -61,7 +61,7 @@ const GHIDRA_RELEASE_PATTERN = /^(\d+)\.(\d+)(?:\.(\d+))?([-+].*)?$/u; export const ghidraApplicationProperties = ( content: string, ): GhidraApplicationProperties => { - const parsed = parseJavaProperties(content); + const parsed = parseGhidraJavaProperties(content); if (!parsed.ok) return { providerVersion: null, @@ -274,115 +274,3 @@ const unresolvedJavaRequirement = ( }); /** Parse Java Properties.load(InputStream) syntax from its ISO-8859-1 text. */ -const parseJavaProperties = ( - content: string, -): Result, string> => { - const values = new Map(); - for (const line of logicalPropertyLines(content)) { - let start = 0; - while (isPropertyWhitespace(line[start])) start += 1; - if (start === line.length || line[start] === "#" || line[start] === "!") - continue; - let end = start; - let escaped = false; - while (end < line.length) { - const character = line[end]; - if ( - !escaped && - (character === "=" || - character === ":" || - isPropertyWhitespace(character)) - ) - break; - if (character === "\\") escaped = !escaped; - else escaped = false; - end += 1; - } - let valueStart = end; - while (isPropertyWhitespace(line[valueStart])) valueStart += 1; - if (line[valueStart] === "=" || line[valueStart] === ":") valueStart += 1; - while (isPropertyWhitespace(line[valueStart])) valueStart += 1; - const key = decodePropertyEscapes(line.slice(start, end)); - if (!key.ok) return err(key.error); - const value = decodePropertyEscapes(line.slice(valueStart)); - if (!value.ok) return err(value.error); - values.set(key.value, value.value); - } - return ok(values); -}; - -const isPropertyWhitespace = (value: string | undefined): boolean => - value === " " || value === "\t" || value === "\f"; - -/** Join odd-backslash continuations while stripping leading continuation space. */ -function* logicalPropertyLines(content: string): Generator { - let physicalStart = 0; - let logical = ""; - let continuing = false; - while (physicalStart <= content.length) { - let physicalEnd = physicalStart; - while ( - physicalEnd < content.length && - content[physicalEnd] !== "\n" && - content[physicalEnd] !== "\r" - ) - physicalEnd += 1; - let line = content.slice(physicalStart, physicalEnd); - let leading = 0; - while (isPropertyWhitespace(line[leading])) leading += 1; - const comment = - !continuing && (line[leading] === "#" || line[leading] === "!"); - if (continuing) line = line.slice(leading); - let continued = false; - if (!comment) { - let slashes = 0; - for ( - let index = line.length - 1; - index >= 0 && line[index] === "\\"; - index -= 1 - ) - slashes += 1; - continued = slashes % 2 === 1; - logical += continued ? line.slice(0, -1) : line; - } - if (!continued) { - yield logical; - logical = ""; - } - continuing = continued; - if (physicalEnd === content.length) break; - physicalStart = - content[physicalEnd] === "\r" && content[physicalEnd + 1] === "\n" - ? physicalEnd + 2 - : physicalEnd + 1; - } - // Properties.load still consumes the accumulated line when EOF follows a - // continuation marker; the marker itself was already removed above. - if (continuing) yield logical; -} - -const decodePropertyEscapes = (value: string): Result => { - let result = ""; - for (let index = 0; index < value.length; index += 1) { - const character = value[index]; - if (character !== "\\") { - result += character; - continue; - } - const escaped = value[++index]; - if (escaped === undefined) - return err("Trailing escape in Java properties value"); - if (escaped === "t") result += "\t"; - else if (escaped === "n") result += "\n"; - else if (escaped === "r") result += "\r"; - else if (escaped === "f") result += "\f"; - else if (escaped === "u") { - const digits = value.slice(index + 1, index + 5); - if (digits.length !== 4 || !/^[0-9a-f]{4}$/iu.test(digits)) - return err("Malformed Unicode escape in Java properties file"); - result += String.fromCharCode(Number.parseInt(digits, 16)); - index += 4; - } else result += escaped; - } - return ok(result); -}; diff --git a/src/ghidra/GhidraJavaProperties.ts b/src/ghidra/GhidraJavaProperties.ts new file mode 100644 index 000000000..ab3b7bead --- /dev/null +++ b/src/ghidra/GhidraJavaProperties.ts @@ -0,0 +1,115 @@ +import { err, ok, type Result } from "../domain/result.js"; + +/** Parse Java Properties.load(InputStream) syntax from its ISO-8859-1 text. */ +export const parseGhidraJavaProperties = ( + content: string, +): Result, string> => { + const values = new Map(); + for (const line of logicalPropertyLines(content)) { + let start = 0; + while (isPropertyWhitespace(line[start])) start += 1; + if (start === line.length || line[start] === "#" || line[start] === "!") + continue; + let end = start; + let escaped = false; + while (end < line.length) { + const character = line[end]; + if ( + !escaped && + (character === "=" || + character === ":" || + isPropertyWhitespace(character)) + ) + break; + if (character === "\\") escaped = !escaped; + else escaped = false; + end += 1; + } + let valueStart = end; + while (isPropertyWhitespace(line[valueStart])) valueStart += 1; + if (line[valueStart] === "=" || line[valueStart] === ":") valueStart += 1; + while (isPropertyWhitespace(line[valueStart])) valueStart += 1; + const key = decodePropertyEscapes(line.slice(start, end)); + if (!key.ok) return err(key.error); + const value = decodePropertyEscapes(line.slice(valueStart)); + if (!value.ok) return err(value.error); + values.set(key.value, value.value); + } + return ok(values); +}; + +const isPropertyWhitespace = (value: string | undefined): boolean => + value === " " || value === "\t" || value === "\f"; + +/** Join odd-backslash continuations while stripping leading continuation space. */ +function* logicalPropertyLines(content: string): Generator { + let physicalStart = 0; + let logical = ""; + let continuing = false; + while (physicalStart <= content.length) { + let physicalEnd = physicalStart; + while ( + physicalEnd < content.length && + content[physicalEnd] !== "\n" && + content[physicalEnd] !== "\r" + ) + physicalEnd += 1; + let line = content.slice(physicalStart, physicalEnd); + let leading = 0; + while (isPropertyWhitespace(line[leading])) leading += 1; + const comment = + !continuing && (line[leading] === "#" || line[leading] === "!"); + if (continuing) line = line.slice(leading); + let continued = false; + if (!comment) { + let slashes = 0; + for ( + let index = line.length - 1; + index >= 0 && line[index] === "\\"; + index -= 1 + ) + slashes += 1; + continued = slashes % 2 === 1; + logical += continued ? line.slice(0, -1) : line; + } + if (!continued) { + yield logical; + logical = ""; + } + continuing = continued; + if (physicalEnd === content.length) break; + physicalStart = + content[physicalEnd] === "\r" && content[physicalEnd + 1] === "\n" + ? physicalEnd + 2 + : physicalEnd + 1; + } + // Properties.load still consumes the accumulated line when EOF follows a + // continuation marker; the marker itself was already removed above. + if (continuing) yield logical; +} + +const decodePropertyEscapes = (value: string): Result => { + let result = ""; + for (let index = 0; index < value.length; index += 1) { + const character = value[index]; + if (character !== "\\") { + result += character; + continue; + } + const escaped = value[++index]; + if (escaped === undefined) + return err("Trailing escape in Java properties value"); + if (escaped === "t") result += "\t"; + else if (escaped === "n") result += "\n"; + else if (escaped === "r") result += "\r"; + else if (escaped === "f") result += "\f"; + else if (escaped === "u") { + const digits = value.slice(index + 1, index + 5); + if (digits.length !== 4 || !/^[0-9a-f]{4}$/iu.test(digits)) + return err("Malformed Unicode escape in Java properties file"); + result += String.fromCharCode(Number.parseInt(digits, 16)); + index += 4; + } else result += escaped; + } + return ok(result); +}; diff --git a/src/ghidra/GhidraPspExtension.ts b/src/ghidra/GhidraPspExtension.ts index 1a3769378..9c017f367 100644 --- a/src/ghidra/GhidraPspExtension.ts +++ b/src/ghidra/GhidraPspExtension.ts @@ -13,6 +13,7 @@ import { import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; import type { GhidraInstallationInspection } from "./GhidraInstallation.js"; +import { parseGhidraJavaProperties } from "./GhidraJavaProperties.js"; /** Installed processor code and data, distinct from REA's analysis-extension ABI. */ export const ghidraPspExtensionSchema = z.strictObject({ @@ -88,7 +89,7 @@ export const inspectGhidraPspExtension = async ( throw new Error("The installed Allegrex extension has no loader JAR."); const hash = createHash("sha256"); let bytesRead = 0; - let properties = ""; + let properties = Buffer.alloc(0); for (const relative of files) { signal?.throwIfAborted(); const path = join(root, relative); @@ -135,22 +136,22 @@ export const inspectGhidraPspExtension = async ( ); bytesRead += before.size; if (relative === "extension.properties") - properties = Buffer.concat(chunks).toString("utf8"); + properties = Buffer.concat(chunks); } finally { await handle.close(); } } // Ghidra's extension.properties version is the compatible Ghidra version, // not the extension release (e.g. v21.4); content identity supplies the latter. - const property = (name: string): string => { - const values = properties.split(/\r?\n/u).flatMap((line) => { - const match = /^\s*([A-Za-z][A-Za-z0-9]*)\s*=\s*(.*?)\s*$/u.exec(line); - return match?.[1] === name ? [match[2] ?? ""] : []; - }); - if (values.length !== 1) - throw new Error(`Allegrex extension must declare exactly one ${name}.`); - return values[0] ?? ""; - }; + const parsedProperties = parseGhidraJavaProperties( + properties.toString("latin1"), + ); + if (!parsedProperties.ok) + throw new Error( + `Invalid Allegrex extension properties: ${parsedProperties.error}`, + ); + const property = (name: string): string | undefined => + parsedProperties.value.get(name); if ( property("name") !== "ghidra-allegrex" || property("version") !== ghidraVersion diff --git a/tests/boundary/filesystem/ghidraPspExtension.test.ts b/tests/boundary/filesystem/ghidraPspExtension.test.ts index 0e07ad629..0b90bd194 100644 --- a/tests/boundary/filesystem/ghidraPspExtension.test.ts +++ b/tests/boundary/filesystem/ghidraPspExtension.test.ts @@ -188,15 +188,42 @@ describe.skipIf(process.platform === "win32")( expect(missing.error.message).toContain("PSP_EXTENSION_UNAVAILABLE"); }); + it.each([ + { + form: "escaped key and value with colon separator", + properties: "n\\u0061me : ghidra-allegrex\nver\\u0073ion=12.1.3\n", + }, + { + form: "continued key and value lines", + properties: "name=ghidra-\\\n allegrex\nversion=12.1.\\\n 3\n", + }, + { + form: "duplicate keys use the last value", + properties: + "name=wrong\nname=ghidra-allegrex\nversion=12.1.4\nversion=12.1.3\n", + }, + ])( + "accepts Java Properties $form and hashes raw bytes", + async ({ properties }) => { + const { root } = await installation(); + const canonical = await inspectGhidraPspExtension(root, "12.1.3"); + const latin1Properties = Buffer.from( + `${properties}author=caf\u00e9\n`, + "latin1", + ); + await writeFile(join(root, "extension.properties"), latin1Properties); + const extension = await inspectGhidraPspExtension(root, "12.1.3"); + expect(extension.id).toBe("ghidra-allegrex"); + expect(extension.sha256).not.toBe(canonical.sha256); + }, + ); + it.each([ ["name=other\nversion=12.1.3\n", "does not match"], ["name=ghidra-allegrex\nversion=12.1.4\n", "does not match"], - [ - "name=ghidra-allegrex\nversion=12.1.3\nversion=12.1.3\n", - "exactly one version", - ], + ["name=ghidra-allegrex\nversion=12.1.\\uZZZZ\n", "Unicode escape"], ])( - "rejects incompatible or ambiguous extension metadata %s", + "rejects mismatched or malformed extension metadata %s", async (properties, reason) => { const { root } = await installation(); await writeFile(join(root, "extension.properties"), properties); From d9e9fb2273543caab4eb21d896c33d462ada6971 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:01:59 +0800 Subject: [PATCH 20/27] fix(process): use monotonic time for provider execution budgets --- src/firmware/FirmwareCommand.ts | 4 +-- src/javascript/recovery/RecoveryFiles.ts | 2 +- src/javascript/recovery/WakaruCommand.test.ts | 8 ++--- src/javascript/recovery/WakaruProvider.ts | 2 +- src/process/ProviderDeadline.test.ts | 35 +++++++++++++++++++ src/process/ProviderDeadline.ts | 8 ++--- 6 files changed, 47 insertions(+), 12 deletions(-) create mode 100644 src/process/ProviderDeadline.test.ts diff --git a/src/firmware/FirmwareCommand.ts b/src/firmware/FirmwareCommand.ts index b5db919f5..1604de901 100644 --- a/src/firmware/FirmwareCommand.ts +++ b/src/firmware/FirmwareCommand.ts @@ -176,11 +176,11 @@ export const runFirmwareCommand = async (context: { }; let failure: unknown; try { - const deadline = Date.now() + FIRMWARE_LIMITS.timeoutMs; + const deadline = performance.now() + FIRMWARE_LIMITS.timeoutMs; while (!(await supervisor.waitForOutputClose(100))) { checkDiagnostics(); if (signal.aborted) throw new AnalysisCancelledError(operation); - if (Date.now() >= deadline) + if (performance.now() >= deadline) throw new AnalysisTimeoutError(operation, FIRMWARE_LIMITS.timeoutMs); if (context.outputBudget !== undefined) await inventoryFirmwareOutput( diff --git a/src/javascript/recovery/RecoveryFiles.ts b/src/javascript/recovery/RecoveryFiles.ts index 0503a83e1..c64f011b4 100644 --- a/src/javascript/recovery/RecoveryFiles.ts +++ b/src/javascript/recovery/RecoveryFiles.ts @@ -279,6 +279,6 @@ export const checkRecoveryDeadline = ( signal?: AbortSignal, ): void => { checkRecoveryCancellation(signal); - if (Date.now() >= deadline) + if (performance.now() >= deadline) throw new AnalysisTimeoutError(OPERATION, RECOVERY_LIMITS.timeoutMs); }; diff --git a/src/javascript/recovery/WakaruCommand.test.ts b/src/javascript/recovery/WakaruCommand.test.ts index 08e43e44a..8dcb74be4 100644 --- a/src/javascript/recovery/WakaruCommand.test.ts +++ b/src/javascript/recovery/WakaruCommand.test.ts @@ -33,7 +33,7 @@ it("rejects an exhausted shared execution budget before launching the next comma arguments: ["--version"], cwd: "/tmp/unused", environment: {}, - deadline: Date.now() - 1, + deadline: performance.now() - 1, launcher: async () => { throw new Error("An expired budget must not launch"); }, @@ -52,7 +52,7 @@ it("preserves caller cancellation before acquiring an expired execution budget", arguments: ["--version"], cwd: "/tmp/unused", environment: {}, - deadline: Date.now() - 1, + deadline: performance.now() - 1, signal: AbortSignal.abort(), launcher: async () => { throw new Error("A cancelled request must not launch"); @@ -86,7 +86,7 @@ const commandFixture = ( arguments: ["--version"], cwd: "/tmp/unused", environment: {}, - deadline: Date.now() + 5_000, + deadline: performance.now() + 5_000, launcher, }, }; @@ -140,7 +140,7 @@ it("enforces the shared deadline while exited output streams remain open", async }); try { await expect( - runWakaruCommand({ ...context, deadline: Date.now() + 50 }), + runWakaruCommand({ ...context, deadline: performance.now() + 50 }), ).rejects.toMatchObject({ _tag: "AnalysisTimeoutError", operation: "recover_javascript_sources", diff --git a/src/javascript/recovery/WakaruProvider.ts b/src/javascript/recovery/WakaruProvider.ts index 7a51c8bd5..b1260bcd1 100644 --- a/src/javascript/recovery/WakaruProvider.ts +++ b/src/javascript/recovery/WakaruProvider.ts @@ -100,7 +100,7 @@ export class WakaruProvider implements JavaScriptRecoveryPort { let tree: SafeOutputTree | undefined; let failed: { readonly cause: unknown } | undefined; let execution: AnalysisExecution | undefined; - const deadline = Date.now() + RECOVERY_LIMITS.timeoutMs; + const deadline = performance.now() + RECOVERY_LIMITS.timeoutMs; try { checkRecoveryCancellation(options?.signal); const engine = await resolveWakaruCommand(this.environment); diff --git a/src/process/ProviderDeadline.test.ts b/src/process/ProviderDeadline.test.ts new file mode 100644 index 000000000..b43ce6654 --- /dev/null +++ b/src/process/ProviderDeadline.test.ts @@ -0,0 +1,35 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { ProviderStartupDeadline } from "./ProviderDeadline.js"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +it("keeps remaining time and cancellation classification stable across wall-clock jumps", () => { + const dateNow = vi.spyOn(Date, "now").mockReturnValue(10_000); + const caller = new AbortController(); + const deadline = new ProviderStartupDeadline(10_000, caller.signal); + try { + dateNow.mockReturnValue(9_000_000_000_000); + const remaining = deadline.remainingMs(); + expect(remaining).toBeGreaterThan(0); + expect(remaining).toBeLessThanOrEqual(10_000); + expect(Number.isInteger(remaining)).toBe(true); + expect(deadline.interruption).toBeUndefined(); + caller.abort(new Error("caller stopped startup")); + expect(deadline.interruption).toBe("cancelled"); + } finally { + deadline.dispose(); + } +}); + +it("aborts a bounded wait when its real monotonic deadline expires", async () => { + const deadline = new ProviderStartupDeadline(40); + try { + await expect(deadline.wait(500)).resolves.toBe("aborted"); + expect(deadline.interruption).toBe("timeout"); + expect(deadline.remainingMs()).toBe(0); + } finally { + deadline.dispose(); + } +}); diff --git a/src/process/ProviderDeadline.ts b/src/process/ProviderDeadline.ts index 3081f469f..33cb3a8d0 100644 --- a/src/process/ProviderDeadline.ts +++ b/src/process/ProviderDeadline.ts @@ -42,11 +42,11 @@ export class ProviderStartupDeadline { ) { if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new RangeError("Provider startup timeout must be positive"); - this.#deadlineAt = Date.now() + timeoutMs; + this.#deadlineAt = performance.now() + timeoutMs; this.#externalSignal = signal; this.#onExternalAbort = () => { if (this.#controller.signal.aborted) return; - if (Date.now() >= this.#deadlineAt) { + if (performance.now() >= this.#deadlineAt) { this.#interruption = "timeout"; this.#abort( new DOMException("Provider startup deadline elapsed", "TimeoutError"), @@ -79,13 +79,13 @@ export class ProviderStartupDeadline { get interruption(): "cancelled" | "timeout" | undefined { return ( this.#interruption ?? - (Date.now() >= this.#deadlineAt ? "timeout" : undefined) + (performance.now() >= this.#deadlineAt ? "timeout" : undefined) ); } /** Milliseconds still available to the next startup phase. */ remainingMs(): number { - return Math.max(0, this.#deadlineAt - Date.now()); + return Math.max(0, Math.ceil(this.#deadlineAt - performance.now())); } /** Wait for at most one polling interval under the shared deadline signal. */ From 6bd2b28161411f70ed5dc72e42c2736a1aab758b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:02:54 +0800 Subject: [PATCH 21/27] fix(browser): keep elapsed capture timing independent of wall-clock changes --- .../BrowserScenarioPartialObservation.ts | 2 +- .../PlaywrightBrowserScenarioProvider.test.ts | 28 +++++++++++++++++++ .../PlaywrightBrowserScenarioProvider.ts | 8 +++--- .../PlaywrightElectronActiveActions.ts | 14 +++++++--- .../PlaywrightElectronActiveProvider.ts | 8 +++--- .../PlaywrightExecutionBoundary.test.ts | 21 +++++++++++++- src/browser/PlaywrightExecutionBoundary.ts | 4 +-- src/browser/PlaywrightScenarioSession.ts | 2 +- src/browser/WebSourceMapFetcher.ts | 6 ++-- 9 files changed, 73 insertions(+), 20 deletions(-) diff --git a/src/browser/BrowserScenarioPartialObservation.ts b/src/browser/BrowserScenarioPartialObservation.ts index ca0df4273..a573f600f 100644 --- a/src/browser/BrowserScenarioPartialObservation.ts +++ b/src/browser/BrowserScenarioPartialObservation.ts @@ -91,7 +91,7 @@ export const browserScenarioCaptureData = (input: { network_content: input.scenario.capture.network, event_families: [...new Set(input.scenario.capture.events)].sort(), }, - duration_ms: Date.now() - input.startedAt, + duration_ms: Math.max(0, Math.round(performance.now() - input.startedAt)), steps: [...input.steps], events, limitations: [ diff --git a/src/browser/PlaywrightBrowserScenarioProvider.test.ts b/src/browser/PlaywrightBrowserScenarioProvider.test.ts index aacf766ad..883b0c3b4 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.test.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.test.ts @@ -303,6 +303,34 @@ describe("PlaywrightBrowserScenarioProvider", () => { }); expect(result.value.limitations).toContain(session.eventGaps[0]); }); + + it("keeps capture elapsed values valid across a backward wall-clock jump", async () => { + let wallClock = 10_000; + vi.spyOn(Date, "now").mockImplementation(() => wallClock); + const session = new FakeSession("launch"); + session.onPerform = () => { + wallClock = 1; + }; + const provider = new PlaywrightBrowserScenarioProvider(() => + Promise.resolve(session), + ); + + try { + const result = await provider.captureScenario(scenario()); + if (!result.ok) throw result.error; + expect(wallClock).toBe(1); + expect(Number.isInteger(result.value.duration_ms)).toBe(true); + expect(result.value.duration_ms).toBeGreaterThanOrEqual(0); + expect( + result.value.steps.every( + ({ elapsed_ms }) => Number.isInteger(elapsed_ms) && elapsed_ms >= 0, + ), + ).toBe(true); + } finally { + vi.restoreAllMocks(); + } + }); + it("returns an initial state and every step beyond the former action ceiling", async () => { const session = new FakeSession("launch"); const provider = new PlaywrightBrowserScenarioProvider(() => diff --git a/src/browser/PlaywrightBrowserScenarioProvider.ts b/src/browser/PlaywrightBrowserScenarioProvider.ts index a3c6d6a62..b0833e89c 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.ts @@ -172,7 +172,7 @@ const executeStep = async (input: { ), }); - const actionStartedAt = Date.now(); + const actionStartedAt = performance.now(); session.setStep(stepIndex); const requestCancelled = (): boolean => signal?.aborted === true; let outcome: BrowserScenarioStepOutcome = { @@ -200,7 +200,7 @@ const executeStep = async (input: { stepId: action.step_id, action: action.action, ...outcome, - elapsedMs: Date.now() - actionStartedAt, + elapsedMs: Math.max(0, Math.round(performance.now() - actionStartedAt)), beforeUrl, afterUrl: session.sanitizeUrl(session.currentUrl()), eventStart, @@ -233,7 +233,7 @@ const runScenario = async ( ): Promise => { if (options.signal?.aborted === true) throw new BrowserObservationError(OPERATION, "cancelled"); - const startedAt = Date.now(); + const startedAt = performance.now(); const session = await openSession(scenario, { ...(options.signal === undefined ? {} : { signal: options.signal }), retainCleanup, @@ -250,7 +250,7 @@ const runScenario = async ( await initialStep({ session, scenario, - elapsedMs: Date.now() - startedAt, + elapsedMs: Math.max(0, Math.round(performance.now() - startedAt)), signal: options.signal, }), ); diff --git a/src/browser/PlaywrightElectronActiveActions.ts b/src/browser/PlaywrightElectronActiveActions.ts index d59afb676..7bb9ec83b 100644 --- a/src/browser/PlaywrightElectronActiveActions.ts +++ b/src/browser/PlaywrightElectronActiveActions.ts @@ -72,7 +72,7 @@ export const runElectronActions = async ( ): Promise => { const actions: ElectronActions = []; for (const action of input.actions) { - const actionStartedAt = Date.now(); + const actionStartedAt = performance.now(); const windowIndex = actionWindowIndex(action); let selectedWindow: Page | undefined; try { @@ -107,7 +107,10 @@ export const runElectronActions = async ( target: selectedWindow === undefined ? null : `window:${String(windowIndex)}`, status: "completed", - elapsed_ms: Date.now() - actionStartedAt, + elapsed_ms: Math.max( + 0, + Math.round(performance.now() - actionStartedAt), + ), error: null, }); } catch (cause: unknown) { @@ -118,7 +121,10 @@ export const runElectronActions = async ( target: selectedWindow === undefined ? null : `window:${String(windowIndex)}`, status: options.signal?.aborted ? "cancelled" : "failed", - elapsed_ms: Date.now() - actionStartedAt, + elapsed_ms: Math.max( + 0, + Math.round(performance.now() - actionStartedAt), + ), error: safeActionErrorMessage(cause), }); break; @@ -366,7 +372,7 @@ export const runWithExecutionLimits = async ( new Promise((_, reject) => { timer = setTimeout( () => reject(new BrowserObservationError(OPERATION, "timeout")), - Math.max(1, deadline - Date.now()), + Math.max(1, deadline - performance.now()), ); }), ); diff --git a/src/browser/PlaywrightElectronActiveProvider.ts b/src/browser/PlaywrightElectronActiveProvider.ts index 4b6e6bb14..23379524d 100644 --- a/src/browser/PlaywrightElectronActiveProvider.ts +++ b/src/browser/PlaywrightElectronActiveProvider.ts @@ -86,7 +86,7 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati if (options.signal?.aborted === true) throw new BrowserObservationError(OPERATION, "cancelled"); await prepareProcessOwnershipInspection(options.signal); - const startupDeadline = Date.now() + STARTUP_TIMEOUT_MS; + const startupDeadline = performance.now() + STARTUP_TIMEOUT_MS; paths = { executable: input.executable_path, application: input.application_path, @@ -101,7 +101,7 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati cwd: paths.root, env: safeElectronEnvironment(this.environment, runId), args: ["-r", hookPath, paths.application, ...input.args], - timeout: Math.max(1, startupDeadline - Date.now()), + timeout: Math.max(1, startupDeadline - performance.now()), }); const leaderPid = application.process().pid; if ( @@ -128,7 +128,7 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati const state = await runWithExecutionLimits( readApplicationState(application), undefined, - options.signal?.aborted ? Date.now() + 5_000 : undefined, + options.signal?.aborted ? performance.now() + 5_000 : undefined, ); capture = projectElectronActiveCapture(paths, actions, state); outcome = options.signal?.aborted @@ -187,7 +187,7 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati await runWithExecutionLimits( application.close(), undefined, - Date.now() + 5_000, + performance.now() + 5_000, ); } catch (cause: unknown) { closeError = cause; diff --git a/src/browser/PlaywrightExecutionBoundary.test.ts b/src/browser/PlaywrightExecutionBoundary.test.ts index 323b4506c..51df81701 100644 --- a/src/browser/PlaywrightExecutionBoundary.test.ts +++ b/src/browser/PlaywrightExecutionBoundary.test.ts @@ -7,7 +7,10 @@ import { } from "../domain/analysisErrorCore.js"; describe("withPlaywrightExecutionBoundary", () => { - afterEach(() => vi.useRealTimers()); + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); it("rejects work that exceeds its absolute remaining duration", async () => { vi.useFakeTimers(); @@ -31,4 +34,20 @@ describe("withPlaywrightExecutionBoundary", () => { controller.abort(); await expect(result).rejects.toBeInstanceOf(AnalysisCancelledError); }); + + it("keeps its timeout stable across a wall-clock jump", async () => { + vi.spyOn(Date, "now") + .mockReturnValueOnce(1_000) + .mockReturnValue(10_000_000); + + await expect( + withPlaywrightExecutionBoundary( + () => + new Promise((resolve) => + setTimeout(() => resolve("done"), 5), + ), + 1_000, + ), + ).resolves.toBe("done"); + }); }); diff --git a/src/browser/PlaywrightExecutionBoundary.ts b/src/browser/PlaywrightExecutionBoundary.ts index 9fd21fcf3..759fcd484 100644 --- a/src/browser/PlaywrightExecutionBoundary.ts +++ b/src/browser/PlaywrightExecutionBoundary.ts @@ -19,10 +19,10 @@ export const withPlaywrightExecutionBoundary = ( let settled = false; let timer: ReturnType | undefined; const deadline = - timeoutMs === undefined ? undefined : Date.now() + timeoutMs; + timeoutMs === undefined ? undefined : performance.now() + timeoutMs; const scheduleTimeout = (): void => { if (deadline === undefined) return; - const remaining = deadline - Date.now(); + const remaining = deadline - performance.now(); if (remaining <= 0) { fail(new AnalysisTimeoutError(OPERATION, timeoutMs ?? 0)); return; diff --git a/src/browser/PlaywrightScenarioSession.ts b/src/browser/PlaywrightScenarioSession.ts index 486f555ea..949193b7c 100644 --- a/src/browser/PlaywrightScenarioSession.ts +++ b/src/browser/PlaywrightScenarioSession.ts @@ -133,7 +133,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort { const secrets = BrowserScenarioSecrets.resolve(scenario, environment); if (secrets === undefined) throw new BrowserObservationError(OPERATION, "secret_unavailable"); - const startedAt = Date.now(); + const startedAt = performance.now(); const opening = openPlaywrightScenarioBrowser(scenario, environment, { signal: options.signal, ...(options.retainCleanup === undefined diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index b950926ee..c4bae782a 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -86,7 +86,7 @@ export const fetchWebSourceMaps = async ( signal?.addEventListener("abort", abortFromCaller, { once: true }); const operationSignal = operationController.signal; const deadlineAt = - Date.now() + (host.timeoutMs ?? SOURCE_MAP_FETCH_TIMEOUT_MS); + performance.now() + (host.timeoutMs ?? SOURCE_MAP_FETCH_TIMEOUT_MS); const budget = { retainedBytes: 0, deadlineAt, @@ -110,7 +110,7 @@ export const fetchWebSourceMaps = async ( } const expandedBytesBeforeItem = budget.decodedRecords.expandedBytes; budget.decodedRecords.expandedBytes += reservation; - if (operationSignal.aborted || Date.now() >= deadlineAt) { + if (operationSignal.aborted || performance.now() >= deadlineAt) { items.push( emptySourceMapItem( request, @@ -789,7 +789,7 @@ const checkOperation = ( ): void => { if (callerSignal?.aborted === true) throw callerSignal.reason; if (operationSignal?.aborted === true) throw operationSignal.reason; - if (Date.now() >= deadlineAt) throw new SourceMapDeadlineError(); + if (performance.now() >= deadlineAt) throw new SourceMapDeadlineError(); }; const approvedUrl = ( From 20f51d7286b1759e42f99c0512c73230052dda86 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:10:58 -0700 Subject: [PATCH 22/27] fix(native): name the ownership helper prerequisite in command failures (#2506) * fix(native): name the ownership helper prerequisite in command failures Every supervised native command launch needs the Darwin process-ownership helper. When that helper cannot be compiled (relative REA_DARWIN_SWIFTC_PATH, missing SDKROOT, a missing compiler, or an SDK/compiler mismatch), the runner reduced the typed DarwinProcessOwnershipInspectionError to a bare `io` failure. inspect_macho then reported `execution_failure` with no hint, and inspect_artifact on any universal Mach-O said only "lipo could not enumerate universal Mach-O slices". The runner now keeps the inspection message as the failure's `detail` at the launch boundary and carries it through xcrun resolution. The native provider adds it to its diagnostics and the slice reader appends it to its message, matching what the SQLite and capture providers already report. Co-Authored-By: Claude Opus 5.5 * test(native): cover the provider's ownership helper diagnostics Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- src/artifacts/MachOSliceArtifactReader.ts | 4 ++- src/native/CommandRunner.ts | 19 +++++++++++++- src/native/NativeMacOSProvider.ts | 1 + .../providers/native/machoSliceReader.test.ts | 22 ++++++++++++++++ .../native/nativeCommandTermination.test.ts | 21 ++++++++++++++++ .../providers/native/nativeProvider.test.ts | 25 +++++++++++++++++++ 6 files changed, 90 insertions(+), 2 deletions(-) diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index 9420bef26..5928956ee 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -81,7 +81,9 @@ export class MachOSliceArtifactReader implements ArtifactReader { if (!captured.ok) throw new ArtifactReaderFailure( captured.error.reason === "cancelled" ? "cancelled" : "unavailable", - "lipo could not enumerate universal Mach-O slices", + captured.error.detail === null + ? "lipo could not enumerate universal Mach-O slices" + : `lipo could not enumerate universal Mach-O slices: ${captured.error.detail}`, { cause: captured.error, ...(captured.error.cleanupFailure === null && diff --git a/src/native/CommandRunner.ts b/src/native/CommandRunner.ts index e9775d1b7..ccd49c570 100644 --- a/src/native/CommandRunner.ts +++ b/src/native/CommandRunner.ts @@ -4,6 +4,7 @@ import { createReadStream } from "node:fs"; import { realpath } from "node:fs/promises"; import { err, ok, type Result } from "../domain/result.js"; +import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; import { OwnedCommandFailure, runOwnedCommand, @@ -39,6 +40,7 @@ export interface NativeCommandCapture { export class NativeCommandFailure extends Error { readonly cleanupOwner: ProviderProcessSupervisor | undefined; readonly cleanupFailures: readonly NativeCommandCleanupFailure[]; + readonly detail: string | null; constructor( readonly tool: string, @@ -58,6 +60,7 @@ export class NativeCommandFailure extends Error { super(`Native command ${tool} failed: ${reason}`, options); this.cleanupOwner = options?.cleanupOwner; this.cleanupFailures = options?.cleanupFailures ?? []; + this.detail = options?.detail ?? null; } } @@ -67,6 +70,8 @@ export interface NativeCommandFailureOptions extends ErrorOptions { readonly cleanupOwner?: ProviderProcessSupervisor; /** All owners still incomplete after a preflight retry blocked this command. */ readonly cleanupFailures?: readonly NativeCommandCleanupFailure[]; + /** Host-side reason the command could not be supervised, such as a missing ownership helper compiler. */ + readonly detail?: string; } /** One failed cleanup attempt; the runner retains `owner` until it is verified stopped. */ @@ -310,6 +315,9 @@ export const resolveXcrunTool = async ( ...(found.error.cleanupFailures.length === 0 ? {} : { cleanupFailures: found.error.cleanupFailures }), + ...(found.error.detail === null + ? {} + : { detail: found.error.detail }), }, found.error.capture, found.error.cleanupFailure, @@ -449,7 +457,16 @@ const captureProcess = async ( ), ); } - return err(new NativeCommandFailure(tool, "io", null, { cause })); + return err( + new NativeCommandFailure(tool, "io", null, { + cause, + // Process supervision needs the Darwin ownership helper; its typed + // message names the compiler or SDK prerequisite to fix. + ...(cause instanceof DarwinProcessOwnershipInspectionError + ? { detail: cause.message } + : {}), + }), + ); } }; diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 1d1c2a36c..a59ec90a7 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -966,6 +966,7 @@ const translateCommandFailure = ( signal: failure.capture?.signal ?? null, stdout_bytes: failure.capture?.stdoutBytes ?? 0, stderr_bytes: failure.capture?.stderrBytes ?? 0, + ...(failure.detail === null ? {} : { detail: failure.detail }), }, }); }; diff --git a/tests/boundary/providers/native/machoSliceReader.test.ts b/tests/boundary/providers/native/machoSliceReader.test.ts index f67b48388..5891eaf53 100644 --- a/tests/boundary/providers/native/machoSliceReader.test.ts +++ b/tests/boundary/providers/native/machoSliceReader.test.ts @@ -3,6 +3,8 @@ import { readFile } from "node:fs/promises"; import { describe, expect, it, onTestFinished } from "vitest"; import { MachOSliceArtifactReader } from "../../../../src/artifacts/MachOSliceArtifactReader.js"; +import { XcrunCommandRunner } from "../../../../src/native/CommandRunner.js"; +import { DarwinProcessOwnershipInspectionError } from "../../../../src/process/DarwinProcessRunTokenReader.js"; /** * Read a 32-bit FAT table directly. The host's /bin/ls changes between macOS @@ -20,6 +22,26 @@ const fatSlices = async ( }; describe("Mach-O slice reader with system lipo", () => { + it("names the process ownership helper prerequisite when lipo cannot be supervised", async () => { + const prerequisite = + "Direct Darwin helper compilation requires absolute REA_DARWIN_SWIFTC_PATH and SDKROOT paths."; + const reader = new MachOSliceArtifactReader( + process.execPath, + {}, + new XcrunCommandRunner({}, undefined, () => + Promise.reject(new DarwinProcessOwnershipInspectionError(prerequisite)), + ), + ); + onTestFinished(() => reader.close()); + const enumerate = async () => { + for await (const _entry of reader.entries()) continue; + }; + await expect(enumerate()).rejects.toMatchObject({ + reason: "unavailable", + message: `lipo could not enumerate universal Mach-O slices: ${prerequisite}`, + }); + }); + it.skipIf(process.platform !== "darwin")( "checks real /bin/ls slice ranges against its FAT table", async () => { diff --git a/tests/boundary/providers/native/nativeCommandTermination.test.ts b/tests/boundary/providers/native/nativeCommandTermination.test.ts index 233ebe85e..5d23e6b22 100644 --- a/tests/boundary/providers/native/nativeCommandTermination.test.ts +++ b/tests/boundary/providers/native/nativeCommandTermination.test.ts @@ -8,6 +8,7 @@ import { type NativeCommandLauncher, } from "../../../../src/native/CommandRunner.js"; import { ok } from "../../../../src/domain/result.js"; +import { DarwinProcessOwnershipInspectionError } from "../../../../src/process/DarwinProcessRunTokenReader.js"; afterEach(() => vi.unstubAllEnvs()); @@ -210,3 +211,23 @@ it.skipIf(process.platform === "win32")( if (!killed.ok) expect(killed.error.reason).toBe("nonzero-exit"); }, ); + +it("keeps the ownership helper prerequisite when xcrun cannot be supervised", async () => { + const inspection = new DarwinProcessOwnershipInspectionError( + "macOS process ownership inspection requires the Apple Swift compiler via xcrun", + ); + const runner = new XcrunCommandRunner({}, undefined, () => + Promise.reject(inspection), + ); + + const result = await runner.run("file", [], {}); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + tool: "file", + reason: "io", + detail: inspection.message, + cause: { tool: "xcrun", detail: inspection.message, cause: inspection }, + }); +}); diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index b4a713204..3dd2a9b2c 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { DarwinProcessOwnershipInspectionError } from "../../../../src/process/DarwinProcessRunTokenReader.js"; import { EventEmitter } from "node:events"; import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -280,6 +281,30 @@ describe("native command failure capture and limits", () => { stderr_bytes: expect.any(Number), }, }); + expect(projectAnalysisError(result.error).details).not.toHaveProperty( + "diagnostics.detail", + ); + }); + + it("projects the ownership helper prerequisite when a command cannot be supervised", async () => { + const inspection = new DarwinProcessOwnershipInspectionError( + "Direct Darwin helper compilation requires absolute REA_DARWIN_SWIFTC_PATH and SDKROOT paths.", + ); + const runner = new XcrunCommandRunner( + {}, + () => + Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), + () => Promise.reject(inspection), + ); + const result = await new NativeMacOSProvider({}, runner, "darwin") + .createClient(machoTarget("/private/fixture")) + .execute("demangle_swift", { symbols: ["fixture-symbol"] }); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(projectAnalysisError(result.error).details).toMatchObject({ + diagnostics: { reason: "io", detail: inspection.message }, + }); }); it("retains output written before cancellation and reaps the stalled command", async () => { From 6a05248d464387c0ce4ec1577b4c1b24d3d355c7 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:11:38 -0700 Subject: [PATCH 23/27] perf(frida): evict captured messages without copying the whole queue (#2628) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once a Frida session reached its 16 MiB capture budget, every new message evicted the oldest one with Array#shift, re-serialized it to recover its byte charge, and searched and spliced its per-script array. Each of those is linear in the retained messages, so with about 188k small messages retained the cost per message rose from 0.14 µs to 32 µs and capped the main thread at roughly 31k messages per second, which a hooked hot function can exceed. Keep both queues as head-offset FIFOs that compact once half their slots are evicted, and record each message's charge when it is admitted. Arrival order already makes the globally oldest message the oldest one of its script, so per-script eviction becomes a head check. Eviction is now amortized O(1) at about 0.14 µs per message; retained messages, their order and the truncation flag are unchanged. Co-authored-by: Claude Opus 5.5 --- .../frida/FridaInstrumentationManager.test.ts | 36 ++++++++ src/frida/FridaInstrumentationManager.ts | 85 ++++++++++++++----- 2 files changed, 98 insertions(+), 23 deletions(-) diff --git a/src/application/frida/FridaInstrumentationManager.test.ts b/src/application/frida/FridaInstrumentationManager.test.ts index 29ba97965..7a527a807 100644 --- a/src/application/frida/FridaInstrumentationManager.test.ts +++ b/src/application/frida/FridaInstrumentationManager.test.ts @@ -809,6 +809,42 @@ it("keeps an active script message bucket after its last message is evicted", as await manager.closeAll(); }); +it("retains the newest contiguous messages across repeated budget evictions", async () => { + const { bindings, manager, started } = await startLocalAttachedSession(); + expect(started.ok).toBe(true); + if (!started.ok) return; + bindings.device.session.nextPayload = { index: -1 }; + const loaded = await manager.loadScript(started.value.sessionId, { + sourceKind: "inline", + source: "send({index: -1});", + }); + expect(loaded.ok).toBe(true); + const script = bindings.device.session.scripts[0]; + expect(script).toBeDefined(); + if (script === undefined) return; + + // About 500 of these fit the 16 MiB budget, so 2,000 sends evict and + // compact the queue several times. + const filler = "x".repeat(32_000); + for (let index = 0; index < 2_000; index += 1) + script.message.emit({ type: "send", payload: { index, filler } }, null); + + const status = manager.status(started.value.sessionId); + expect(status?.messagesTruncated).toBe(true); + const indexes = (status?.messages ?? []).map( + (message) => (message as { payload: { index: number } }).payload.index, + ); + expect(indexes.length).toBeGreaterThan(400); + expect(indexes.at(-1)).toBe(1_999); + expect(indexes).toEqual( + Array.from( + { length: indexes.length }, + (_, offset) => 2_000 - indexes.length + offset, + ), + ); + await manager.closeAll(); +}); + it("serializes a final remote release against a new same-endpoint session", async () => { const bindings = new FakeDeviceManager(); let enteredRemoval: (() => void) | undefined; diff --git a/src/frida/FridaInstrumentationManager.ts b/src/frida/FridaInstrumentationManager.ts index 406258e5a..22b38c442 100644 --- a/src/frida/FridaInstrumentationManager.ts +++ b/src/frida/FridaInstrumentationManager.ts @@ -83,8 +83,8 @@ interface ManagedSession extends ManagedRemoteConnection { readonly pid: number; readonly mode: "attach" | "spawn"; readonly scripts: Map; - readonly messages: JsonValue[]; - readonly messagesByScript: Map; + readonly messages: MessageFifo; + readonly messagesByScript: Map>; state: "paused" | "running" | "detached"; targetResumeObserved: boolean; resumeAfterDetachPending: boolean; @@ -277,7 +277,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { pid, mode: input.mode, scripts: new Map(), - messages: [], + messages: new MessageFifo(), messagesByScript: new Map(), state: input.mode === "spawn" ? "paused" : "running", targetResumeObserved: false, @@ -373,7 +373,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { .digest("hex"); const scriptId = randomUUID(); const scriptName = `rea-${scriptId}`; - managed.messagesByScript.set(scriptId, []); + managed.messagesByScript.set(scriptId, new MessageFifo()); let nativeScript: Script | undefined; const onMessage = (message: FridaMessage, data: Buffer | null): void => { this.#recordMessage(managed, scriptId, message, data); @@ -397,7 +397,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { sourceKind: source.sourceKind, sourcePath: source.sourceKind === "file" ? source.path : null, sourceSha256, - messages: [...(managed.messagesByScript.get(scriptId) ?? [])], + messages: managed.messagesByScript.get(scriptId)?.toArray() ?? [], messagesTruncated: managed.messagesTruncated, }); } catch (cause: unknown) { @@ -517,7 +517,9 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { scriptId: id, name, })), - messages: [...managed.messages], + messages: managed.messages + .toArray() + .map(({ observation }) => observation), messagesTruncated: managed.messagesTruncated, }; } @@ -851,30 +853,21 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { return; } while ( - managed.messages.length > 0 && + managed.messages.size > 0 && managed.capturedBytes + charge > MAX_SESSION_CAPTURE_BYTES ) { const removed = managed.messages.shift(); if (removed !== undefined) { - managed.capturedBytes -= Buffer.byteLength(JSON.stringify(removed)); - if ( - typeof removed === "object" && - removed !== null && - !Array.isArray(removed) && - typeof removed.script_id === "string" - ) { - const scriptMessages = managed.messagesByScript.get( - removed.script_id, - ); - const index = scriptMessages?.indexOf(removed) ?? -1; - if (scriptMessages !== undefined && index >= 0) { - scriptMessages.splice(index, 1); - } - } + managed.capturedBytes -= removed.charge; + // Both queues are in arrival order, so the globally oldest message is + // also the oldest one retained for its script. + const scriptMessages = managed.messagesByScript.get(removed.scriptId); + if (scriptMessages?.peek() === removed.observation) + scriptMessages.shift(); } managed.messagesTruncated = true; } - managed.messages.push(observation); + managed.messages.push({ observation, scriptId, charge }); managed.messagesByScript.get(scriptId)?.push(observation); managed.capturedBytes += charge; } @@ -1240,3 +1233,49 @@ const providerError = ( const errorMessage = (cause: unknown): string => cause instanceof Error ? cause.message : String(cause); + +interface CapturedMessage { + readonly observation: JsonValue; + readonly scriptId: string; + /** Serialized bytes charged against the session capture budget. */ + readonly charge: number; +} + +/** + * Arrival-ordered queue whose eviction advances a head offset. A full capture + * budget evicts on every message, and `Array#shift` would copy every retained + * message each time. + */ +class MessageFifo { + #items: (T | undefined)[] = []; + #head = 0; + + get size(): number { + return this.#items.length - this.#head; + } + + push(item: T): void { + this.#items.push(item); + } + + peek(): T | undefined { + return this.#items[this.#head]; + } + + shift(): T | undefined { + if (this.#head >= this.#items.length) return undefined; + const item = this.#items[this.#head]; + this.#items[this.#head] = undefined; + this.#head += 1; + // Compacting once half the slots are evicted keeps eviction amortized O(1). + if (this.#head * 2 >= this.#items.length) { + this.#items = this.#items.slice(this.#head); + this.#head = 0; + } + return item; + } + + toArray(): T[] { + return this.#items.slice(this.#head) as T[]; + } +} From 7fc0b5ec9c2532bbf789d7783b7a0cab6fc054d8 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:12:08 +0800 Subject: [PATCH 24/27] fix(mcp): join owned doctor shutdown without signaling saved PIDs --- src/mcpDoctor.ts | 146 +++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/src/mcpDoctor.ts b/src/mcpDoctor.ts index 62a588280..d7bf1e366 100644 --- a/src/mcpDoctor.ts +++ b/src/mcpDoctor.ts @@ -1,6 +1,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Formatter, Help, z } from "incur"; +import { performance } from "node:perf_hooks"; import { resolve } from "node:path"; import { CATALOG_IDENTITY } from "./catalogIdentity.js"; @@ -74,14 +75,15 @@ interface McpDoctorCheck { /** Diagnose the actual production stdio adapter with one startup-bounded SDK session. */ export const runProductionMcpDoctor = async (options: McpDoctorOptions) => { const deadline = - Date.now() + (options.deadlineMs ?? MCP_STARTUP_POLICY.doctorDeadlineMs); + performance.now() + + (options.deadlineMs ?? MCP_STARTUP_POLICY.doctorDeadlineMs); const controller = new AbortController(); const timer = setTimeout( () => controller.abort(new Error("Production MCP doctor deadline expired")), - Math.max(1, deadline - Date.now()), + Math.max(1, Math.ceil(deadline - performance.now())), ); const stderr = captureStderr(); - const transport = new TrackedStdioClientTransport({ + const transport = new OwnedStdioClientTransport({ command: options.command, args: [...options.args], cwd: options.cwd, @@ -90,21 +92,20 @@ export const runProductionMcpDoctor = async (options: McpDoctorOptions) => { }); transport.stderr?.on("data", stderr.append); const client = new Client({ name: "rea-mcp-doctor", version: "1" }); + let result: ProductionMcpDoctorResult; + let cleanupFailure: { readonly cause: unknown } | undefined; try { await client.connect( transport, requestOptions(deadline, controller.signal), ); - return { - ...(await inspectProductionMcpSession( - client, - deadline, - controller.signal, - )), - diagnostics: stderr.value(), - }; + result = await inspectProductionMcpSession( + client, + deadline, + controller.signal, + ); } catch (cause: unknown) { - return { + result = { healthy: false, adapter: "production-stdio" as const, server: null, @@ -118,14 +119,71 @@ export const runProductionMcpDoctor = async (options: McpDoctorOptions) => { detail: errorMessage(cause), }, ], - diagnostics: stderr.value(), }; } finally { clearTimeout(timer); - await closeMcpChild(client, transport); + try { + await client.close(); + } catch (cause: unknown) { + cleanupFailure = { cause }; + } + try { + // An interrupted connect can clear the client's transport reference + // before its startup child is closed. The transport retains that child. + await transport.close(); + } catch (cause: unknown) { + cleanupFailure = { + cause: + cleanupFailure === undefined || cleanupFailure.cause === cause + ? cause + : new AggregateError( + [cleanupFailure.cause, cause], + `MCP client and transport cleanup failed: ${errorMessage(cleanupFailure.cause)}; ${errorMessage(cause)}`, + ), + }; + } } + if (cleanupFailure !== undefined) + result = { + ...result, + healthy: false, + checks: [ + ...result.checks, + { + name: "cleanup", + ok: false, + detail: `MCP stdio cleanup failed: ${errorMessage(cleanupFailure.cause)}`, + }, + ], + }; + return { ...result, diagnostics: stderr.value() }; }; +type ProductionMcpDoctorResult = + | Awaited> + | { + readonly healthy: false; + readonly adapter: "production-stdio"; + readonly server: null; + readonly inventory: null; + readonly identity: null; + readonly request_flow: { + readonly tool: "binary_session"; + readonly ok: false; + }; + readonly checks: readonly McpDoctorCheck[]; + }; + +/** Join SDK-triggered and caller-triggered close calls for the same child. */ +class OwnedStdioClientTransport extends StdioClientTransport { + #closePromise: Promise | undefined; + + override close(): Promise { + this.#closePromise ??= super.close(); + return this.#closePromise; + } +} + const inspectProductionMcpSession = async ( client: Client, deadline: number, @@ -385,7 +443,7 @@ const isOutputFormat = (value: string | undefined): value is OutputFormat => const requestOptions = (deadline: number, signal: AbortSignal) => ({ signal, - timeout: Math.max(1, deadline - Date.now()), + timeout: Math.max(1, Math.ceil(deadline - performance.now())), }); const targetFreeEnvironment = ( @@ -428,61 +486,3 @@ const isRecord = (value: unknown): value is Readonly> => const errorMessage = (cause: unknown): string => cause instanceof Error ? cause.message : String(cause); - -class TrackedStdioClientTransport extends StdioClientTransport { - spawnedPid: number | null = null; - - override async start(): Promise { - await super.start(); - this.spawnedPid = this.pid; - } -} - -const closeMcpChild = async ( - client: Client, - transport: TrackedStdioClientTransport, -): Promise => { - const pid = transport.spawnedPid ?? transport.pid; - await Promise.allSettled([client.close()]); - await Promise.allSettled([transport.close()]); - if (pid === null || !(await processExists(pid))) return; - await terminateProcess(pid, "SIGTERM"); - if (await waitForProcessExit(pid, 1_000)) return; - await terminateProcess(pid, "SIGKILL"); - await waitForProcessExit(pid, 1_000); -}; - -const terminateProcess = async ( - pid: number, - signal: NodeJS.Signals, -): Promise => { - try { - process.kill(pid, signal); - } catch (cause: unknown) { - if (!(cause instanceof Error && "code" in cause && cause.code === "ESRCH")) - throw cause; - } -}; - -const waitForProcessExit = async ( - pid: number, - timeoutMs: number, -): Promise => { - const deadline = Date.now() + timeoutMs; - do { - if (!(await processExists(pid))) return true; - await new Promise((resolve) => setTimeout(resolve, 20)); - } while (Date.now() < deadline); - return !(await processExists(pid)); -}; - -const processExists = async (pid: number): Promise => { - try { - process.kill(pid, 0); - return true; - } catch (cause: unknown) { - if (cause instanceof Error && "code" in cause && cause.code === "ESRCH") - return false; - throw cause; - } -}; From 9049dbaf0c3df32357419f9899965df2b611f493 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:12:08 +0800 Subject: [PATCH 25/27] fix(process): preserve monotonic capture timing and observed wall dates --- .../PlaywrightBrowserScenarioProvider.test.ts | 56 ++++++------ .../process/processCaptureValidation.ts | 4 - src/process/capture/ProcessCaptureClock.ts | 3 + src/process/capture/ProcessCaptureJournal.ts | 6 +- .../capture/ProcessCaptureLifecycle.test.ts | 5 +- .../capture/ProcessCaptureLifecycle.ts | 74 ++++++++++----- src/process/capture/ProcessCaptureProgress.ts | 3 +- src/process/capture/ProcessHarness.ts | 7 +- src/process/capture/ProcessSampling.ts | 3 +- .../filesystem/ghidraPspExtension.test.ts | 65 +++++++------- .../process/processCaptureLifecycle.test.ts | 89 ++++++++++++++++++- 11 files changed, 224 insertions(+), 91 deletions(-) create mode 100644 src/process/capture/ProcessCaptureClock.ts diff --git a/src/browser/PlaywrightBrowserScenarioProvider.test.ts b/src/browser/PlaywrightBrowserScenarioProvider.test.ts index 883b0c3b4..734e32426 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.test.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.test.ts @@ -304,33 +304,6 @@ describe("PlaywrightBrowserScenarioProvider", () => { expect(result.value.limitations).toContain(session.eventGaps[0]); }); - it("keeps capture elapsed values valid across a backward wall-clock jump", async () => { - let wallClock = 10_000; - vi.spyOn(Date, "now").mockImplementation(() => wallClock); - const session = new FakeSession("launch"); - session.onPerform = () => { - wallClock = 1; - }; - const provider = new PlaywrightBrowserScenarioProvider(() => - Promise.resolve(session), - ); - - try { - const result = await provider.captureScenario(scenario()); - if (!result.ok) throw result.error; - expect(wallClock).toBe(1); - expect(Number.isInteger(result.value.duration_ms)).toBe(true); - expect(result.value.duration_ms).toBeGreaterThanOrEqual(0); - expect( - result.value.steps.every( - ({ elapsed_ms }) => Number.isInteger(elapsed_ms) && elapsed_ms >= 0, - ), - ).toBe(true); - } finally { - vi.restoreAllMocks(); - } - }); - it("returns an initial state and every step beyond the former action ceiling", async () => { const session = new FakeSession("launch"); const provider = new PlaywrightBrowserScenarioProvider(() => @@ -404,6 +377,35 @@ describe("PlaywrightBrowserScenarioProvider", () => { }); }); +describe("PlaywrightBrowserScenarioProvider timing", () => { + it("keeps capture elapsed values valid across a backward wall-clock jump", async () => { + let wallClock = 10_000; + vi.spyOn(Date, "now").mockImplementation(() => wallClock); + const session = new FakeSession("launch"); + session.onPerform = () => { + wallClock = 1; + }; + const provider = new PlaywrightBrowserScenarioProvider(() => + Promise.resolve(session), + ); + + try { + const result = await provider.captureScenario(scenario()); + if (!result.ok) throw result.error; + expect(wallClock).toBe(1); + expect(Number.isInteger(result.value.duration_ms)).toBe(true); + expect(result.value.duration_ms).toBeGreaterThanOrEqual(0); + expect( + result.value.steps.every( + ({ elapsed_ms }) => Number.isInteger(elapsed_ms) && elapsed_ms >= 0, + ), + ).toBe(true); + } finally { + vi.restoreAllMocks(); + } + }); +}); + describe("PlaywrightBrowserScenarioProvider comparison coverage", () => { it("keeps quiet captures with different event families from comparing unchanged", async () => { const capture = async (events: readonly BrowserScenarioEventFamily[]) => { diff --git a/src/domain/process/processCaptureValidation.ts b/src/domain/process/processCaptureValidation.ts index 10d6b831b..33c6d5483 100644 --- a/src/domain/process/processCaptureValidation.ts +++ b/src/domain/process/processCaptureValidation.ts @@ -47,10 +47,6 @@ const validateCommitments = ( digestProcessCommitment( value, ), `manifest.${field}`, "commitment does not match its canonical value"); - require(Date.parse(manifest.started_at) <= - Date.parse( - manifest.completed_at, - ), "manifest.completed_at", "completion precedes start"); }; const validateOrdering = ( diff --git a/src/process/capture/ProcessCaptureClock.ts b/src/process/capture/ProcessCaptureClock.ts new file mode 100644 index 000000000..1fe9fda0a --- /dev/null +++ b/src/process/capture/ProcessCaptureClock.ts @@ -0,0 +1,3 @@ +/** Keep wall-clock timestamps separate from capture-relative elapsed time. */ +export const monotonicProcessCaptureClock = (): number => + Math.floor(performance.now()); diff --git a/src/process/capture/ProcessCaptureJournal.ts b/src/process/capture/ProcessCaptureJournal.ts index b8a0874d9..07c1f0351 100644 --- a/src/process/capture/ProcessCaptureJournal.ts +++ b/src/process/capture/ProcessCaptureJournal.ts @@ -13,6 +13,7 @@ import type { ProcessCaptureEventJournalEntry, RecordProcessCaptureEvent, } from "../../domain/process/processCapture.js"; +import { monotonicProcessCaptureClock } from "./ProcessCaptureClock.js"; /** Mutable observation-order ledger whose writer is shared by capture producers. */ export interface ProcessCaptureJournal { readonly entries: readonly ProcessCaptureEventJournalEntry[]; @@ -55,7 +56,10 @@ export const scheduleScenarioInteractions = ( const timer = scheduleProcessDelay(event.at_ms, () => { options.dispatchedEventIndexes.add(eventIndex); const terminal = getTerminal(); - const dispatchedAt = Math.max(0, Date.now() - started); + const dispatchedAt = Math.max( + 0, + monotonicProcessCaptureClock() - started, + ); let outcome: InteractionEvent["outcome"] = "dispatched"; if (terminal === undefined) outcome = "target_exited"; else { diff --git a/src/process/capture/ProcessCaptureLifecycle.test.ts b/src/process/capture/ProcessCaptureLifecycle.test.ts index 765635ccc..abad38ac2 100644 --- a/src/process/capture/ProcessCaptureLifecycle.test.ts +++ b/src/process/capture/ProcessCaptureLifecycle.test.ts @@ -5,6 +5,7 @@ import { parseProcessScenario } from "../../domain/process/processScenario.js"; import type { ProcessCaptureFinalization } from "../../domain/process/processCapture.js"; import { awaitTerminalExit } from "./ProcessCaptureLifecycle.js"; import type { ProcessTimer } from "./ProcessTimer.js"; +import { monotonicProcessCaptureClock } from "./ProcessCaptureClock.js"; type Delivery = ProcessCaptureFinalization["signals"][number]["delivery"]; @@ -79,8 +80,8 @@ const start = ( idle_timeout_ms: 10_000, ...scenario, }), - started: Date.now(), - lastOutput: () => Date.now(), + started: monotonicProcessCaptureClock(), + lastOutput: monotonicProcessCaptureClock, signal, signalTarget: fake.signalTarget, recordFinalization: (snapshot) => { diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index 2d2fcec00..ab9271dda 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -48,6 +48,7 @@ import { type ProcessCaptureTruncationDetails, } from "../../domain/process/processCaptureCoverage.js"; import { scheduleProcessInterval, type ProcessTimer } from "./ProcessTimer.js"; +import { monotonicProcessCaptureClock } from "./ProcessCaptureClock.js"; import type { FilesystemCheckpoint, @@ -266,10 +267,10 @@ export const settleProcessCaptureJournal = async ( quietMs = 25, maxWaitMs = 500, ): Promise => { - const deadline = Date.now() + maxWaitMs; + const deadline = monotonicProcessCaptureClock() + maxWaitMs; let observed = journal.length; - let lastChangeAt = Date.now(); - while (Date.now() < deadline) { + let lastChangeAt = monotonicProcessCaptureClock(); + while (monotonicProcessCaptureClock() < deadline) { await new Promise((resolve) => setTimeout(resolve, 4)); // A stalled event loop can leave timer and poll callbacks that were already // due queued behind this wake-up. Drain them before judging silence so a @@ -277,8 +278,8 @@ export const settleProcessCaptureJournal = async ( await setImmediate(); if (journal.length !== observed) { observed = journal.length; - lastChangeAt = Date.now(); - } else if (Date.now() - lastChangeAt >= quietMs) return; + lastChangeAt = monotonicProcessCaptureClock(); + } else if (monotonicProcessCaptureClock() - lastChangeAt >= quietMs) return; } }; @@ -337,6 +338,12 @@ export const buildCaptureResult = ( truncation_details: options.truncationDetails, limitations: [ "The executable digest is a prelaunch file sample; matching path metadata immediately after spawn does not prove an atomic operating-system image binding.", + ...(Date.parse(options.manifest.completed_at) < + Date.parse(options.manifest.started_at) + ? [ + "Wall-clock completion time precedes start time; elapsed capture timing is recorded separately and remains monotonic.", + ] + : []), "Process trees are sampled and may omit short-lived descendants.", ...(options.samplingPartial ? ["Process-tree sampling ended with an incomplete observation."] @@ -660,7 +667,7 @@ export const observeSettlement = async ( recordEvent("lifecycle", 1); return { state: "unverifiable", elapsed_ms: 0 }; } - const started = Date.now(); + const started = monotonicProcessCaptureClock(); let consecutiveEmpty = 0; let sawLive = false; let deadlineReached = false; @@ -678,19 +685,25 @@ export const observeSettlement = async ( [...observations.values()].some(({ state }) => state === "unverifiable") ) { recordEvent("lifecycle", 1); - return { state: "unverifiable", elapsed_ms: Date.now() - started }; + return { + state: "unverifiable", + elapsed_ms: monotonicProcessCaptureClock() - started, + }; } if ([...observations.values()].every(({ state }) => state === "empty")) { consecutiveEmpty += 1; if (consecutiveEmpty >= 2) { recordEvent("lifecycle", 1); - return { state: "quiesced", elapsed_ms: Date.now() - started }; + return { + state: "quiesced", + elapsed_ms: monotonicProcessCaptureClock() - started, + }; } } else { consecutiveEmpty = 0; sawLive = true; } - deadlineReached = Date.now() - started >= settleMs; + deadlineReached = monotonicProcessCaptureClock() - started >= settleMs; if (deadlineReached) break; await delay(50, undefined, { signal }); } @@ -698,8 +711,14 @@ export const observeSettlement = async ( // A lone empty observation proves neither liveness nor quiescence; only an // observed live member justifies alive_at_deadline. return sawLive - ? { state: "alive_at_deadline", elapsed_ms: Date.now() - started } - : { state: "unverifiable", elapsed_ms: Date.now() - started }; + ? { + state: "alive_at_deadline", + elapsed_ms: monotonicProcessCaptureClock() - started, + } + : { + state: "unverifiable", + elapsed_ms: monotonicProcessCaptureClock() - started, + }; }; /** Name why an attempt is unverified; a blank message never reaches the capture schema. */ @@ -786,7 +805,10 @@ export const awaitTerminalExit = async ({ if (finalizationStartedAt === undefined) return; const record: (typeof finalizationSignals)[number] = { signal: finalizationSignal, - sent_at_ms: Math.round(performance.now() - finalizationStartedAt), + sent_at_ms: Math.max( + 0, + monotonicProcessCaptureClock() - finalizationStartedAt, + ), delivery: "unverified" as FinalizationDelivery, reason: "delivery had not settled when the record was published", }; @@ -856,7 +878,7 @@ export const awaitTerminalExit = async ({ terminal.kill("SIGKILL"); return; } - finalizationStartedAt = performance.now(); + finalizationStartedAt = monotonicProcessCaptureClock(); attempt("SIGTERM"); }; terminal.onExit((exit) => { @@ -867,7 +889,7 @@ export const awaitTerminalExit = async ({ const finalizationElapsedMs = finalizationStartedAt === undefined ? undefined - : Math.round(performance.now() - finalizationStartedAt); + : Math.max(0, monotonicProcessCaptureClock() - finalizationStartedAt); recordEvent("lifecycle", 0); for (const [eventIndex, event] of scenario.events.entries()) { if (dispatchedEventIndexes.has(eventIndex)) continue; @@ -875,7 +897,10 @@ export const awaitTerminalExit = async ({ interactions.push({ sequence, scheduled_at_ms: event.at_ms, - dispatched_at_ms: Math.max(0, Date.now() - started), + dispatched_at_ms: Math.max( + 0, + monotonicProcessCaptureClock() - started, + ), type: event.type, data: event.type === "input" @@ -935,14 +960,20 @@ export const awaitTerminalExit = async ({ forceKill(); } else if (finalizationStartedAt !== undefined) { if ( - performance.now() - finalizationStartedAt >= + monotonicProcessCaptureClock() - finalizationStartedAt >= scenario.finalization_ms ) forceKill(); - } else if (Date.now() - started >= scenario.timeout_ms) { + } else if ( + monotonicProcessCaptureClock() - started >= + scenario.timeout_ms + ) { reason = "timeout"; terminateAtDeadline(); - } else if (Date.now() - lastOutput() >= scenario.idle_timeout_ms) { + } else if ( + monotonicProcessCaptureClock() - lastOutput() >= + scenario.idle_timeout_ms + ) { reason = "idle_timeout"; terminateAtDeadline(); } @@ -971,10 +1002,11 @@ const PROCESS_CLEANUP_VERIFICATION_INTERVAL_MS = 25; const verifyTokenOwnedProcessesUntilSettled = async ( verify: () => Promise, ): Promise => { - const deadline = Date.now() + PROCESS_CLEANUP_VERIFICATION_GRACE_MS; + const deadline = + monotonicProcessCaptureClock() + PROCESS_CLEANUP_VERIFICATION_GRACE_MS; let result = await verify(); while (!result.cleaned) { - const remaining = deadline - Date.now(); + const remaining = deadline - monotonicProcessCaptureClock(); if (remaining <= 0) return result; await new Promise((resolve) => setTimeout( @@ -1288,7 +1320,7 @@ export const captureTerminalFrames = (options: { } outputBytes += bytes; const atMs = normalizeProcessElapsedTime( - Date.now() - options.started, + monotonicProcessCaptureClock() - options.started, options.scenario.normalization.time_bucket_ms, ); const normalized = normalizeProcessText( diff --git a/src/process/capture/ProcessCaptureProgress.ts b/src/process/capture/ProcessCaptureProgress.ts index 83d5a51f5..205334db7 100644 --- a/src/process/capture/ProcessCaptureProgress.ts +++ b/src/process/capture/ProcessCaptureProgress.ts @@ -1,4 +1,5 @@ import type { ProcessCaptureCleanupReport } from "../../domain/process/processCapture.js"; +import { monotonicProcessCaptureClock } from "./ProcessCaptureClock.js"; /** * Progress port accepted by process capture. @@ -61,7 +62,7 @@ const formatProgressMessage = ( /** Serialize observed capture status through one rate-limited reporter. */ export const createProcessCaptureProgressTracker = ( progress: ProcessCaptureProgress | undefined, - now: () => number = Date.now, + now: () => number = monotonicProcessCaptureClock, ) => { const started = now(); let completed = 0; diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.ts index 1033e68cd..847e512f8 100644 --- a/src/process/capture/ProcessHarness.ts +++ b/src/process/capture/ProcessHarness.ts @@ -70,6 +70,7 @@ import { observeProcessStartIdentity, signalProcessWithStartIdentity, } from "../ProcessOwnershipObservation.js"; +import { monotonicProcessCaptureClock } from "./ProcessCaptureClock.js"; interface StartedCaptureRuntime { readonly renderer: TerminalRenderer; @@ -268,7 +269,7 @@ const startCaptureRuntime = async ( options.signal, ); const startedAt = new Date(); - const started = Date.now(); + const started = monotonicProcessCaptureClock(); let lastOutput = started; renderer = createTerminalRenderer( scenario, @@ -296,7 +297,7 @@ const startCaptureRuntime = async ( ...options, terminal, started, - onOutput: () => (lastOutput = Date.now()), + onOutput: () => (lastOutput = monotonicProcessCaptureClock()), renderer, }); scheduleScenarioInteractions({ @@ -541,7 +542,7 @@ const completeCapture = async (options: { }, { name: "after_settlement", - at_ms: Math.max(0, Date.now() - runtime.started), + at_ms: Math.max(0, monotonicProcessCaptureClock() - runtime.started), files: after.files, effects: classifyFilesystemEffects(options.before, after), truncated: after.truncated, diff --git a/src/process/capture/ProcessSampling.ts b/src/process/capture/ProcessSampling.ts index 69e7dd343..59b67f577 100644 --- a/src/process/capture/ProcessSampling.ts +++ b/src/process/capture/ProcessSampling.ts @@ -6,6 +6,7 @@ import type { } from "../../domain/process/processCapture.js"; import { execFileOutput } from "../ExecFileOutput.js"; import { readProcessRunId } from "../ProcessOwnership.js"; +import { monotonicProcessCaptureClock } from "./ProcessCaptureClock.js"; const PROC_READ_CONCURRENCY = 64; interface ProcessRow { @@ -491,7 +492,7 @@ export const startProcessSampler = (options: { } return sampleProcesses({ rootPid, - elapsedMs: Date.now() - started, + elapsedMs: monotonicProcessCaptureClock() - started, limit: limit + 1, identities, signal: controller.signal, diff --git a/tests/boundary/filesystem/ghidraPspExtension.test.ts b/tests/boundary/filesystem/ghidraPspExtension.test.ts index 0b90bd194..65a4d1512 100644 --- a/tests/boundary/filesystem/ghidraPspExtension.test.ts +++ b/tests/boundary/filesystem/ghidraPspExtension.test.ts @@ -188,36 +188,6 @@ describe.skipIf(process.platform === "win32")( expect(missing.error.message).toContain("PSP_EXTENSION_UNAVAILABLE"); }); - it.each([ - { - form: "escaped key and value with colon separator", - properties: "n\\u0061me : ghidra-allegrex\nver\\u0073ion=12.1.3\n", - }, - { - form: "continued key and value lines", - properties: "name=ghidra-\\\n allegrex\nversion=12.1.\\\n 3\n", - }, - { - form: "duplicate keys use the last value", - properties: - "name=wrong\nname=ghidra-allegrex\nversion=12.1.4\nversion=12.1.3\n", - }, - ])( - "accepts Java Properties $form and hashes raw bytes", - async ({ properties }) => { - const { root } = await installation(); - const canonical = await inspectGhidraPspExtension(root, "12.1.3"); - const latin1Properties = Buffer.from( - `${properties}author=caf\u00e9\n`, - "latin1", - ); - await writeFile(join(root, "extension.properties"), latin1Properties); - const extension = await inspectGhidraPspExtension(root, "12.1.3"); - expect(extension.id).toBe("ghidra-allegrex"); - expect(extension.sha256).not.toBe(canonical.sha256); - }, - ); - it.each([ ["name=other\nversion=12.1.3\n", "does not match"], ["name=ghidra-allegrex\nversion=12.1.4\n", "does not match"], @@ -531,3 +501,38 @@ describe.skipIf(process.platform === "win32")( }); }, ); + +describe.skipIf(process.platform === "win32")( + "PSP extension property syntax", + () => { + it.each([ + { + form: "escaped key and value with colon separator", + properties: "n\\u0061me : ghidra-allegrex\nver\\u0073ion=12.1.3\n", + }, + { + form: "continued key and value lines", + properties: "name=ghidra-\\\n allegrex\nversion=12.1.\\\n 3\n", + }, + { + form: "duplicate keys use the last value", + properties: + "name=wrong\nname=ghidra-allegrex\nversion=12.1.4\nversion=12.1.3\n", + }, + ])( + "accepts Java Properties $form and hashes raw bytes", + async ({ properties }) => { + const { root } = await installation(); + const canonical = await inspectGhidraPspExtension(root, "12.1.3"); + const latin1Properties = Buffer.from( + `${properties}author=caf\u00e9\n`, + "latin1", + ); + await writeFile(join(root, "extension.properties"), latin1Properties); + const extension = await inspectGhidraPspExtension(root, "12.1.3"); + expect(extension.id).toBe("ghidra-allegrex"); + expect(extension.sha256).not.toBe(canonical.sha256); + }, + ); + }, +); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index dcba0510f..c3dc21542 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -4,7 +4,7 @@ import { access, readFile, rm, symlink, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { expect } from "vitest"; +import { expect, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { itWithCaptureCapability } from "./processCaptureCapability.js"; @@ -123,6 +123,93 @@ const terminateFixtureProcess = async (pid: number): Promise => { await waitForProcessExit(pid); }; +itWithCaptureCapability( + "keeps the process deadline bounded when the wall clock moves backward", + async () => { + const wallStart = Date.now(); + let wallReads = 0; + const now = vi.spyOn(Date, "now").mockImplementation(() => { + wallReads += 1; + return wallReads === 1 ? wallStart : wallStart - 60_000 + wallReads; + }); + const controller = new AbortController(); + const fallback = setTimeout(() => controller.abort(), 1_500); + try { + const result = await captureProcessScenario( + parseProcessScenario({ + executable: process.execPath, + arguments: [processFixture, "hang"], + working_directory: dirname(processFixture), + timeout_ms: 100, + idle_timeout_ms: 5_000, + }), + controller.signal, + ); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(result.value.exit.reason).toBe("timeout"); + } finally { + clearTimeout(fallback); + now.mockRestore(); + } + }, + 10_000, +); + +itWithCaptureCapability( + "preserves reversed wall-clock observations and monotonic frames across capture", + async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-01-02T00:00:00.000Z")); + try { + const result = await captureProcessScenario( + parseProcessScenario({ + executable: process.execPath, + arguments: [processFixture, "partial"], + working_directory: dirname(processFixture), + settle_ms: 0, + timeout_ms: 5_000, + idle_timeout_ms: 5_000, + }), + undefined, + process.platform, + process.env, + undefined, + undefined, + { + async report(update) { + if (update.phase === "running") + vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z")); + }, + }, + ); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(result.value.frames.length).toBeGreaterThan(0); + expect(Date.parse(result.value.manifest.completed_at)).toBeLessThan( + Date.parse(result.value.manifest.started_at), + ); + expect(result.value.limitations).toContain( + "Wall-clock completion time precedes start time; elapsed capture timing is recorded separately and remains monotonic.", + ); + expect( + result.value.frames.map(({ at_ms }) => at_ms), + "capture-relative frame times stay ordered and nonnegative", + ).toEqual( + [...result.value.frames.map(({ at_ms }) => at_ms)].sort( + (left, right) => left - right, + ), + ); + expect(result.value.frames.every(({ at_ms }) => at_ms >= 0)).toBe(true); + expect(result.value.exit.reason).toBe("exited"); + expect(result.value.settlement.elapsed_ms).toBeGreaterThan(0); + } finally { + vi.useRealTimers(); + } + }, + 10_000, +); + itWithCaptureCapability( "cleans detached run-token children omitted by sampling and preserves an unowned neighbor", async () => { From e7882b34f4ac4a26f903c5f49264acc2c19bb562 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:12:17 +0800 Subject: [PATCH 26/27] fix(artifacts): honor cancellation at the output publication boundary --- src/artifacts/SafeOutputTree.ts | 33 ++++++++++------ .../extraction/ArtifactExtraction.ts | 2 +- src/browser/assets/PublishWebScripts.ts | 2 +- src/firmware/FirmwarePublication.ts | 2 +- src/javascript/recovery/WakaruProvider.ts | 2 +- .../safeOutputTreeCloseFailures.test.ts | 39 +++++++++++++++++++ ...actProvider.extractionCompleteness.test.ts | 39 ++++++++++++++++++- 7 files changed, 103 insertions(+), 16 deletions(-) diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.ts index 4bb4cfe1a..8fa40876d 100644 --- a/src/artifacts/SafeOutputTree.ts +++ b/src/artifacts/SafeOutputTree.ts @@ -280,9 +280,11 @@ export class SafeOutputTree { } /** Sync the owned output tree and prevent further writes through this instance. */ - async commit(): Promise { + async commit(signal?: AbortSignal): Promise { this.#assertWritable(); + abortIfNeeded(signal); await this.#assertOwnedDirectories(); + abortIfNeeded(signal); // Windows has no directory fsync; file contents are already synced in write(). if (this.platform === "win32") { this.#published = true; @@ -315,6 +317,7 @@ export class SafeOutputTree { ); await output.handle.sync(); await parent.handle.sync(); + abortIfNeeded(signal); this.#published = true; } catch (cause: unknown) { sync = { kind: "failed", cause }; @@ -336,23 +339,26 @@ export class SafeOutputTree { closeFailure, this.#outputRoot, ); - if (sync.kind === "failed") - throw ArtifactReaderFailure.withCleanup( - new ArtifactReaderFailure( - "path", - "Could not durably sync extraction output", - { cause: sync.cause }, - ), - cleanup, - ); + if (sync.kind === "failed") { + const primary = isCancellationFailure(sync.cause) + ? sync.cause + : new ArtifactReaderFailure( + "path", + "Could not durably sync extraction output", + { cause: sync.cause }, + ); + throw ArtifactReaderFailure.withCleanup(primary, cleanup); + } throw ArtifactReaderFailure.withCleanup(closeFailure, cleanup); } - if (sync.kind === "failed") + if (sync.kind === "failed") { + if (isCancellationFailure(sync.cause)) throw sync.cause; throw new ArtifactReaderFailure( "path", "Could not durably sync extraction output", { cause: sync.cause }, ); + } } /** Remove only this operation's unsealed tree and verify absence. */ @@ -590,6 +596,11 @@ const abortIfNeeded = (signal?: AbortSignal): void => { ); }; +const isCancellationFailure = ( + cause: unknown, +): cause is ArtifactReaderFailure => + cause instanceof ArtifactReaderFailure && cause.reason === "cancelled"; + const isAlreadyExists = (cause: unknown): boolean => cause instanceof Error && "code" in cause && cause.code === "EEXIST"; diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index ee92954eb..8cad3543e 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -249,7 +249,7 @@ const materializeSelection = async ({ compareUnicodeCodePoints(left.relative_path, right.relative_path), ); completed = createExtractionResult(input, inventory, selected, extracted); - await output.commit(); + await output.commit(signal); return completed; } catch (cause: unknown) { const cleanupFailures = await releaseExtractionOwners( diff --git a/src/browser/assets/PublishWebScripts.ts b/src/browser/assets/PublishWebScripts.ts index 2db5d7dcf..ba886f00e 100644 --- a/src/browser/assets/PublishWebScripts.ts +++ b/src/browser/assets/PublishWebScripts.ts @@ -113,7 +113,7 @@ export const publishWebScripts = async ( options.signal, ); options.signal?.throwIfAborted(); - await tree.commit(); + await tree.commit(options.signal); return result; } catch (cause: unknown) { if (cause instanceof SafeOutputTreeCreationFailure) tree = cause.tree; diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.ts index bc53084b1..b2bc4ddd5 100644 --- a/src/firmware/FirmwarePublication.ts +++ b/src/firmware/FirmwarePublication.ts @@ -96,7 +96,7 @@ export const publishFirmwareExtraction = async (context: { published_bytes: files.reduce((sum, file) => sum + file.size, 0), }); context.signal.throwIfAborted(); - await tree.commit(); + await tree.commit(context.signal); return result; } catch (cause: unknown) { if (cause instanceof SafeOutputTreeCreationFailure) tree = cause.tree; diff --git a/src/javascript/recovery/WakaruProvider.ts b/src/javascript/recovery/WakaruProvider.ts index b1260bcd1..88a51a74b 100644 --- a/src/javascript/recovery/WakaruProvider.ts +++ b/src/javascript/recovery/WakaruProvider.ts @@ -139,7 +139,7 @@ export class WakaruProvider implements JavaScriptRecoveryPort { await rm(root, { recursive: true, force: true }); root = undefined; checkRecoveryDeadline(deadline, options?.signal); - await tree.commit(); + await tree.commit(options?.signal); } catch (cause: unknown) { failed = { cause }; } diff --git a/tests/boundary/filesystem/safeOutputTreeCloseFailures.test.ts b/tests/boundary/filesystem/safeOutputTreeCloseFailures.test.ts index ee285dbd5..d835b1e46 100644 --- a/tests/boundary/filesystem/safeOutputTreeCloseFailures.test.ts +++ b/tests/boundary/filesystem/safeOutputTreeCloseFailures.test.ts @@ -10,6 +10,8 @@ const injectedFilesystemFailures = vi.hoisted(() => ({ closeNext: false, blockCloseRetries: false, chmodNext: false, + syncAbortAfter: undefined as AbortController | undefined, + syncCount: 0, rootIdentityPath: undefined as string | undefined, })); @@ -44,6 +46,15 @@ vi.mock("node:fs/promises", async (importOriginal) => { await close(); }; } + if (injectedFilesystemFailures.syncAbortAfter !== undefined) { + const sync = handle.sync.bind(handle); + handle.sync = async () => { + await sync(); + injectedFilesystemFailures.syncCount += 1; + if (injectedFilesystemFailures.syncCount === 2) + injectedFilesystemFailures.syncAbortAfter?.abort(); + }; + } return handle; }, }; @@ -82,6 +93,34 @@ describe("safe artifact output close failures", () => { }, ); + it.skipIf(process.platform === "win32")( + "keeps the tree unpublished when cancellation arrives during commit durability", + async () => { + const parent = await createTestTempDirectory("rea-safe-output-cancel-"); + const output = join(parent, "unpublished"); + const tree = await SafeOutputTree.create(output); + const bytes = Buffer.from("pending output"); + await tree.write("file.txt", Readable.from(bytes), { + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.byteLength, + }); + const controller = new AbortController(); + injectedFilesystemFailures.syncCount = 0; + injectedFilesystemFailures.syncAbortAfter = controller; + try { + await expect(tree.commit(controller.signal)).rejects.toMatchObject({ + reason: "cancelled", + }); + expect(tree.published).toBe(false); + } finally { + injectedFilesystemFailures.syncAbortAfter = undefined; + injectedFilesystemFailures.syncCount = 0; + } + expect(await tree.rollback()).toMatchObject({ status: "complete" }); + await expect(access(output)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + it.skipIf(process.platform === "win32")( "preserves setup failure when closing the staging handle also fails", async () => { diff --git a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts index d42762601..0bcf12bb2 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts @@ -3,7 +3,7 @@ import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createPackage, createPackageWithOptions } from "@electron/asar"; -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { @@ -13,6 +13,7 @@ import { import { artifactExtractionResultSchema } from "../../../../src/domain/artifactGraph.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { DirectoryArtifactReader } from "../../../../src/artifacts/DirectoryArtifactReader.js"; it.each(["file", "directory"] as const)( "rolls back when an inventoried %s disappears before extraction", @@ -61,6 +62,42 @@ it("still refuses a regular file added after inventory and rolls back", async () await expect(access(output)).rejects.toThrow(); }); +it("rolls back when cancellation arrives while closing the materialization reader", async () => { + const root = await createTestTempDirectory("rea-extract-cancel-close-"); + const source = join(root, "source"); + await mkdir(source); + await writeFile(join(source, "payload.txt"), "complete payload\n"); + const snapshot = await scanArtifactInventory(source); + const output = join(root, "output"); + const controller = new AbortController(); + const close = DirectoryArtifactReader.prototype.close; + const closeSpy = vi + .spyOn(DirectoryArtifactReader.prototype, "close") + .mockImplementation(async function (this: DirectoryArtifactReader) { + controller.abort(); + await close.call(this); + }); + try { + await expect( + materializeArtifactInventory( + { + inputPath: source, + inputFormat: "asar", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + source, + snapshot, + controller.signal, + ), + ).rejects.toMatchObject({ reason: "cancelled" }); + await expect(access(output)).rejects.toThrow(); + } finally { + closeSpy.mockRestore(); + } +}); + it("preserves every equal-content occurrence and copies a nested ASAR as its containing file", async () => { const root = await createTestTempDirectory("rea-extract-nested-"); const source = join(root, "source"); From bf34d0d2fc3cc22076f49d682e74a0b71aa1db18 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:13:12 -0700 Subject: [PATCH 27/27] perf(native): add opt-in Mach-O summary delivery and retained pages (#2634) * perf(native): add opt-in Mach-O summary delivery and retained pages inspect_macho returns every import and export inline. For VS Code's Electron Framework one MCP call carries 2.5 MB of text (about 828k o200k tokens), and the same again as structuredContent; the MCP response budget admits it. Agents asking for slice identity, UUID or dependency counts paid that cost on every first call. Add the established opt-in detail selection: omitted or complete keeps the existing complete result and Evidence identity, while summary retains the complete Evidence and returns slice identity with per-collection returned/total/exhaustive counts (956 tokens). inspect_analysis_view kind macho pages the original rows of any collection or the command provenance, repeating the producer's own total and exhaustive flag so page exhaustion is never read as absence. Delivery detail never reaches the native tools or the source Evidence ID: registerEvidenceTools gains a parametersForEvidence hook, so the summary's parent is the same Evidence as a complete call. Co-Authored-By: Claude Opus 5.5 * fix(mcp): refuse unretainable summary delivery before analysis A server without Evidence retention (a fixed analysis source) ran the complete inspect_macho or inspect_artifact analysis for detail summary and only then refused it in summarizeRetainedAnalysis, paying the very cost summary delivery exists to avoid. registerEvidenceTools gains a preflight hook; both registrations check retention there, as the WASM, EVM, managed, pcap and capture handlers already do before dispatch. Co-Authored-By: Claude Opus 5.5 * fix(native): keep whole rows in Mach-O architecture and segment pages The page item union listed plain-object row arrays, and zod returns the first union member that parses while dropping unknown keys. Entrypoint rows declare only file_offset, so every architecture and segment row (which also carry a numeric file_offset) matched it and lost its name, CPU fields, permissions and sections. Rows now use exact key sets, and imports/exports share one member. The MCP test pages every facet of a real /bin/ls and compares it with the unparsed complete rows. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../references/native-and-artifacts.md | 8 + docs/mcp-contracts.md | 17 +- src/contracts/analysisViewToolContracts.ts | 4 +- src/contracts/native/nativeToolContracts.ts | 12 +- src/contracts/toolOutputSchemaGroups.ts | 5 +- src/domain/analysisView/analysisView.ts | 29 +- src/domain/analysisView/machoView.test.ts | 246 +++++++++++++++++ src/domain/analysisView/machoView.ts | 247 ++++++++++++++++++ src/server/createServer.ts | 5 +- src/server/registerArtifactTools.ts | 18 +- src/server/registerEvidenceTools.ts | 20 +- src/server/registerNativeTools.ts | 54 ++++ .../mcp/nativeMachoSummaryMcp.test.ts | 216 +++++++++++++++ 13 files changed, 864 insertions(+), 17 deletions(-) create mode 100644 src/domain/analysisView/machoView.test.ts create mode 100644 src/domain/analysisView/machoView.ts create mode 100644 src/server/registerNativeTools.ts create mode 100644 tests/boundary/mcp/nativeMachoSummaryMcp.test.ts diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 58c1671b6..71fe3e950 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -83,6 +83,14 @@ names, the native API record, and value-flow summaries. Pseudocode offsets and l code units; use the returned `next_offset` for continuation. Provider limitations and unavailable facts remain explicit; the complete parent Evidence is retained. +`inspect_macho` on a large framework returns megabytes of imports and exports. +Select `detail: "summary"` for slice identity, UUID and per-collection counts, +then page rows with `view: {kind: "macho", facet: "exports", offset: 0, limit: 64}` +and `normalized_result.parent_evidence_id`. Facets are entrypoints, +architectures, build_metadata, load_commands, dependencies, imports, exports, +segments and provenance. `source_exhaustive: false` means unreturned symbols +may exist. + To see which functions or Objective-C methods a Mach-O actually calls in one run, use `observe_native_calls` with explicit breakpoints and a bounded `duration_ms`/`max_events`. It launches a new process under LLDB, so the target diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index c4169dd73..f3f7892f6 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -147,8 +147,8 @@ after target content was read. ## Retained application Evidence inputs -`analyze_javascript_application`, `inspect_binary_layout`, `inspect_artifact`, and -`inspect_wasm_artifact` accept `detail`: +`analyze_javascript_application`, `inspect_binary_layout`, `inspect_artifact`, +`inspect_wasm_artifact`, and `inspect_macho` accept `detail`: `complete` (default) returns the complete analysis Evidence; `summary` records that complete Evidence in the session and returns the derived `inspect_analysis_view` summary Evidence instead. Its @@ -218,6 +218,19 @@ attempting oversized delivery, and returns the same summary with an exact `parent_evidence_id` for later export or facet reads. Delivery detail does not reach WABT or change the original source Evidence ID. +Completed `inspect_macho` Evidence supports a summary and +`view: {kind: "macho", facet: "exports", offset: 0, limit: 64}`. Facets are +`entrypoints`, `architectures`, `build_metadata`, `load_commands`, +`dependencies`, `imports`, `exports`, `segments`, and `provenance`; pages return +the original rows. The summary keeps the selected slice, header fields and UUID, +and replaces each collection with its returned count, reported total and +`exhaustive` flag. Each page repeats the paged collection's `source_total` and +`source_exhaustive`; exhausting a non-exhaustive collection does not establish +that unreturned symbols are absent. Large frameworks return megabytes of +symbols (Electron Framework: 2.5 MB of imports and exports, 3.3 KB as a +summary), so select `detail: "summary"` on the first call. Delivery detail +never reaches the native tools or changes the source Evidence ID. + `trace_application_feature`, `trace_javascript_semantics`, `compare_application_versions`, `compare_source_to_bundle`, and `compare_javascript_export_shapes` accept complete inline application Evidence diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index aba5b9a93..7d023752d 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -9,14 +9,14 @@ import { evidenceResultOf } from "./toolOutputSchemaPrimitives.js"; const RETAINED_EXAMPLE_ID = `ev_${"a".repeat(64)}`; -/** Selected views of completed layout, JavaScript, native, inventory, WASM and web network Evidence. */ +/** Selected views of completed layout, JavaScript, native, inventory, WASM, Mach-O and web network Evidence. */ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ { name: "inspect_analysis_view", ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_evm_interface, inspect_managed_artifact, inspect_android_package, inspect_dart_aot or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. For layout, JavaScript, inventory, WASM, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions; occurrence logical_paths matches exact recorded strings (omitted means all available, [] means none). source_complete distinguishes complete inventories from partial source records; page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, inspect_evm_interface, inspect_managed_artifact, inspect_android_package, inspect_dart_aot or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. For layout, JavaScript, inventory, WASM, Mach-O, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions; occurrence logical_paths matches exact recorded strings (omitted means all available, [] means none). source_complete distinguishes complete inventories from partial source records; page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/contracts/native/nativeToolContracts.ts b/src/contracts/native/nativeToolContracts.ts index 093add61e..16101368b 100644 --- a/src/contracts/native/nativeToolContracts.ts +++ b/src/contracts/native/nativeToolContracts.ts @@ -11,6 +11,7 @@ import { jsonValueSchema } from "../../domain/jsonValue.js"; import { localPathStringSchema } from "../../domain/localPath.js"; import { toolContractMetadata } from "../toolEffects.js"; import { requireOutputSchema } from "../toolOutputSchemaPrimitives.js"; +import { analysisDetailSchema } from "../analysisDetail.js"; const examples: Readonly>>> = { inspect_macho: {}, @@ -94,8 +95,15 @@ export const NATIVE_TOOL_CONTRACTS = [ ), native( "inspect_macho", - "Inspect Mach-O slices, load commands, imports, exports, dependencies, build metadata, segments, sections, permissions, and exact command provenance without launching Hopper.", - z.object({}), + "Inspect Mach-O slices, load commands, imports, exports, dependencies, build metadata, segments, sections, permissions, and exact command provenance without launching Hopper. Large frameworks return megabytes of symbols; detail summary retains the complete Evidence and returns slice identity with per-collection counts, then page rows with inspect_analysis_view kind macho.", + z.object({ + detail: analysisDetailSchema + .unwrap() + .optional() + .describe( + "Omitted or complete returns the complete inspection Evidence. summary retains that complete Evidence in this session and returns its inspect_analysis_view summary; use normalized_result.parent_evidence_id for macho pages.", + ), + }), ), native( "inspect_signature", diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index e828486ba..174339bdf 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { artifactInventorySummaryViewResultSchema, + machoSummaryViewResultSchema, managedArtifactSummaryViewResultSchema, } from "../domain/analysisView/analysisView.js"; import { evidenceBundleSummarySchema } from "../domain/evidenceRecordSummary.js"; @@ -381,7 +382,9 @@ export const nativeOutputSchemas: Readonly> = { observe_native_ui: evidenceResultOf(nativeUiResultSchema), capture_native_ui_scenario: evidenceResultOf(nativeUiResultSchema), observe_native_calls: evidenceResultOf(nativeCallObservationResultSchema), - inspect_macho: evidenceResultOf(inspectMachoSchema), + inspect_macho: evidenceResultOf( + z.union([inspectMachoSchema, machoSummaryViewResultSchema]), + ), inspect_signature: evidenceResultOf(inspectSignatureSchema), inspect_plist: evidenceResultOf(inspectPlistSchema), list_architectures: evidenceResultOf(listArchitecturesSchema), diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 6cc5ae8a3..7f985f349 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -55,6 +55,13 @@ import { wasmViewItemSchema, wasmSummarySchema, } from "./wasmArtifactView.js"; +import { + machoSummarySchema, + machoViewContextSchema, + machoViewItemSchema, + machoViewRequestSchema, + projectMachoView, +} from "./machoView.js"; import { projectWebNetworkCaptureView, captureViewRequestSchema, @@ -196,6 +203,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ nativeViewSchema, inventoryViewSchema, wasmViewRequestSchema, + machoViewRequestSchema, evmViewRequestSchema, managedViewRequestSchema, androidViewRequestSchema, @@ -315,6 +323,7 @@ const viewResultBase = { "inspect_artifact", "inventory_artifact", "inspect_wasm_artifact", + "inspect_macho", "inspect_evm_interface", "inspect_managed_artifact", "inspect_web_network_capture", @@ -364,6 +373,13 @@ export const wasmArtifactSummaryViewResultSchema = z.strictObject({ summary: wasmSummarySchema, }); +/** Exact slice identity and collection counts for Mach-O's selected delivery. */ +export const machoSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + parent_operation: z.literal("inspect_macho"), + summary: machoSummarySchema, +}); + /** Exact inferred-interface summary for EVM's caller-selected delivery. */ export const evmInterfaceSummaryViewResultSchema = z.strictObject({ ...summaryViewResultBase, @@ -425,6 +441,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ javascriptSummarySchema, inventorySummarySchema, wasmSummarySchema, + machoSummarySchema, evmSummarySchema, managedSummarySchema, captureSummarySchema, @@ -487,6 +504,14 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ item: wasmViewItemSchema, ...viewResultBase, }), + z.strictObject({ + kind: z.literal("macho"), + view: machoViewRequestSchema, + item: machoViewItemSchema, + ...viewResultBase, + ...machoViewContextSchema.shape, + parent_operation: z.literal("inspect_macho"), + }), z.strictObject({ kind: z.literal("evm"), view: evmViewRequestSchema, @@ -583,6 +608,7 @@ export type UnsignedAnalysisView = "view_digest" > | Omit, "view_digest"> + | Omit, "view_digest"> | Omit, "view_digest"> | Omit, "view_digest"> | Omit, "view_digest"> @@ -646,7 +672,7 @@ const unsupportedParent = ( new AnalysisUnsupportedTargetError( INSPECT_ANALYSIS_VIEW_OPERATION, parentArtifactPath(parent), - `Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function, artifact inventory/inspection, WASM, EVM, managed artifact, Android package, Dart AOT, or web network capture Evidence.`, + `Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function, artifact inventory/inspection, WASM, Mach-O, EVM, managed artifact, Android package, Dart AOT, or web network capture Evidence.`, ); const parentArtifactPath = (parent: AnalysisViewParent): string => { @@ -673,6 +699,7 @@ const parentArtifactPath = (parent: AnalysisViewParent): string => { /** Named source owners for views that need no extra decoder or accumulator. */ const retainedViewProjectors = new Map([ ["inspect_wasm_artifact", projectWasmArtifactView], + ["inspect_macho", projectMachoView], ["inspect_evm_interface", projectEvmInterfaceView], ["inspect_managed_artifact", projectManagedArtifactView], ["inspect_android_package", projectAndroidPackageView], diff --git a/src/domain/analysisView/machoView.test.ts b/src/domain/analysisView/machoView.test.ts new file mode 100644 index 000000000..2018438be --- /dev/null +++ b/src/domain/analysisView/machoView.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it } from "vitest"; + +import { + projectAnalysisView, + type AnalysisViewParent, +} from "./analysisView.js"; + +const symbol = (name: string) => ({ + name, + address: null, + weak: null, + reexport: null, + source: "libSystem", +}); +const coverage = (items: T[], exhaustive = true) => ({ + items, + total: exhaustive ? items.length : null, + exhaustive, + limitations: exhaustive ? [] : ["nm output may omit stripped symbols."], +}); +const report = { + format: "mach-o", + selected_architecture: { name: "arm64", container_file_offset: 0 }, + endian: null, + word_size: 64, + file_type: "2", + flags: ["0x00200085"], + uuid: "4C4C44E9-5555-3144-A1E9-D858E24A1CCE", + entrypoints: coverage([{ file_offset: 16384 }]), + architectures: coverage([ + { + name: "arm64", + cpu_type: "16777228", + cpu_subtype: "0", + file_offset: 0, + size: 4096, + alignment: 14, + }, + ]), + build_metadata: coverage([ + { platform: "1", minimum_os: "14.0", sdk: "15.0", tools: [] }, + ]), + load_commands: coverage([{ index: 0, kind: "LC_MAIN", fields: {} }]), + dependencies: coverage([ + { + path: "/usr/lib/libSystem.B.dylib", + kind: "LC_LOAD_DYLIB", + current_version: "1351.0.0", + compatibility_version: "1.0.0", + }, + ]), + imports: coverage( + ["_malloc", "_free", "_open"].map((name) => symbol(name)), + false, + ), + exports: coverage([symbol("__mh_execute_header")]), + segments: coverage([ + { + name: "__TEXT", + vm_address: "0x100000000", + vm_size: 16384, + file_offset: 0, + file_size: 16384, + maximum_permissions: { + read: true, + write: false, + execute: true, + raw: "5", + }, + initial_permissions: { + read: true, + write: false, + execute: true, + raw: "5", + }, + sections: coverage([]), + }, + ]), + provenance: [ + { + tool: "nm", + command: ["/usr/bin/nm", "$ARTIFACT"], + tool_version: null, + version_reason: "fixture", + executable_sha256: "a".repeat(64), + exit: { code: 0, signal: null }, + stdout_bytes: 10, + stderr_bytes: 0, + }, + ], + limitations: ["Imports and exports combine dyld_info and nm."], +}; +const parent = ( + normalizedResult: unknown = report, + subject: AnalysisViewParent["subject"] = { + name: "tool", + digest: { sha256: "b".repeat(64) }, + format: "mach-o", + architecture: "arm64", + local_path: "/bin/tool", + }, +): AnalysisViewParent => ({ + evidenceId: `ev_${"c".repeat(64)}`, + operation: "inspect_macho", + normalizedResult, + subject, + limitations: ["Parent limitation."], +}); + +describe("Mach-O analysis views", () => { + it("summarizes slice identity and collection counts without rows", () => { + const projected = projectAnalysisView(parent(), { kind: "summary" }); + if (!projected.ok) throw projected.error; + expect(projected.value).toMatchObject({ + kind: "summary", + parent_operation: "inspect_macho", + artifact: { path: "/bin/tool", sha256: "b".repeat(64) }, + summary: { + selected_architecture: { name: "arm64", container_file_offset: 0 }, + uuid: report.uuid, + collections: { + imports: { returned: 3, total: null, exhaustive: false }, + exports: { returned: 1, total: 1, exhaustive: true }, + segments: { returned: 1, total: 1, exhaustive: true }, + }, + provenance_count: 1, + limitation_count: 3, + }, + coverage: { status: "complete-within-view", examined: 10, total: 10 }, + limitations: [ + "Imports and exports combine dyld_info and nm.", + "Parent limitation.", + "nm output may omit stripped symbols.", + ], + unknowns: [expect.stringContaining("exhaustive imports;")], + }); + expect(JSON.stringify(projected.value)).not.toContain("_malloc"); + }); + + it("pages original rows with the producer's own coverage", () => { + const first = projectAnalysisView(parent(), { + kind: "macho", + facet: "imports", + offset: 1, + limit: 1, + }); + if (!first.ok) throw first.error; + expect(first.value).toMatchObject({ + kind: "macho", + item: [symbol("_free")], + source_total: null, + source_exhaustive: false, + coverage: { status: "page", examined: 1, total: 3, next_offset: 2 }, + }); + expect(first.value.limitations).toContain( + "nm output may omit stripped symbols.", + ); + // Rows whose keys include an earlier row shape's keys stay whole. + for (const facet of ["architectures", "segments"] as const) { + const page = projectAnalysisView(parent(), { + kind: "macho", + facet, + offset: 0, + limit: 64, + }); + if (!page.ok) throw page.error; + expect(page.value.kind === "macho" && page.value.item).toEqual( + report[facet].items, + ); + } + const provenance = projectAnalysisView(parent(), { + kind: "macho", + facet: "provenance", + offset: 0, + limit: 64, + }); + if (!provenance.ok) throw provenance.error; + expect(provenance.value).toMatchObject({ + item: report.provenance, + source_total: 1, + source_exhaustive: true, + coverage: { exhausted: true, next_offset: null }, + }); + const past = projectAnalysisView(parent(), { + kind: "macho", + facet: "exports", + offset: 5, + limit: 64, + }); + if (!past.ok) throw past.error; + expect(past.value.coverage).toMatchObject({ + status: "empty", + exhausted: true, + }); + }); + + it("keeps a missing subject unknown instead of inventing identity", () => { + const projected = projectAnalysisView(parent(report, null), { + kind: "summary", + }); + if (!projected.ok) throw projected.error; + expect(projected.value.artifact).toEqual({ path: "", sha256: null }); + expect(projected.value.unknowns).toContain( + "The parent has no recorded subject, so the inspected artifact path and SHA-256 are unknown.", + ); + }); + + it("rejects other view kinds and malformed retained results", () => { + const wrongKind = projectAnalysisView(parent(), { + kind: "wasm", + facet: "exports", + offset: 0, + limit: 64, + }); + if (wrongKind.ok) throw new Error("expected a view-kind refusal"); + expect(wrongKind.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [ + { + path: ["view"], + message: "Mach-O inspection supports summary and macho views.", + }, + ], + }); + const contradictory = projectAnalysisView( + parent(report, { + name: "tool", + digest: { sha256: "b".repeat(64) }, + format: "elf", + architecture: null, + local_path: "/bin/tool", + }), + { kind: "summary" }, + ); + if (contradictory.ok) throw new Error("expected a subject refusal"); + expect(contradictory.error).toMatchObject({ + issues: [{ path: ["source", "subject"] }], + }); + const malformed = projectAnalysisView( + parent({ ...report, imports: { items: "x" } }), + { kind: "summary" }, + ); + expect(malformed.ok).toBe(false); + if (!malformed.ok) expect(malformed.error._tag).toBe("AnalysisInputError"); + }); +}); diff --git a/src/domain/analysisView/machoView.ts b/src/domain/analysisView/machoView.ts new file mode 100644 index 000000000..1a64d23aa --- /dev/null +++ b/src/domain/analysisView/machoView.ts @@ -0,0 +1,247 @@ +import { z } from "zod"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { uniqueSorted } from "../canonicalOrdering.js"; +import { isMachOFormat } from "../binaryTargetTypes.js"; +import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; +import { inspectMachoSchema } from "../native/nativeInspection.js"; +import { err, ok, type Result } from "../result.js"; +import { + analysisViewInputError, + type AnalysisViewParent, + type AnalysisViewRequest, + type UnsignedAnalysisView, +} from "./analysisView.js"; +import { + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; + +const MACHO_COLLECTIONS = [ + "entrypoints", + "architectures", + "build_metadata", + "load_commands", + "dependencies", + "imports", + "exports", + "segments", +] as const; +type MachoCollection = (typeof MACHO_COLLECTIONS)[number]; + +/** Original inspect_macho rows; offsets count rows in the retained collection. */ +export const machoViewRequestSchema = z.strictObject({ + kind: z.literal("macho"), + facet: z.enum([...MACHO_COLLECTIONS, "provenance"]), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}); + +const { shape } = inspectMachoSchema; +/** + * Union members are tried in order and plain objects drop unknown keys, so a + * segment or architecture row would otherwise match the entrypoint row and + * lose every field but file_offset. Exact key sets keep each page's rows whole. + */ +const exactRows = ( + rows: z.ZodArray>, +) => z.array(z.strictObject(rows.element.shape)); +export const machoViewItemSchema = z.union([ + exactRows(shape.entrypoints.shape.items), + exactRows(shape.architectures.shape.items), + exactRows(shape.build_metadata.shape.items), + exactRows(shape.load_commands.shape.items), + exactRows(shape.dependencies.shape.items), + // Imports and exports share one row shape. + exactRows(shape.imports.shape.items), + exactRows(shape.segments.shape.items), + exactRows(shape.provenance), +]); + +/** The producer's own coverage of the paged collection, not of this page. */ +export const machoViewContextSchema = z.strictObject({ + source_total: z.number().int().nonnegative().nullable(), + source_exhaustive: z.boolean(), +}); + +const collectionCountSchema = z.strictObject({ + returned: z.number().int().nonnegative(), + total: z.number().int().nonnegative().nullable(), + exhaustive: z.boolean(), +}); + +/** Slice identity and per-collection counts without the collections themselves. */ +export const machoSummarySchema = z.strictObject({ + format: shape.format, + selected_architecture: shape.selected_architecture, + endian: shape.endian, + word_size: shape.word_size, + file_type: shape.file_type, + flags: shape.flags, + uuid: shape.uuid, + collections: z.strictObject( + Object.fromEntries( + MACHO_COLLECTIONS.map((name) => [name, collectionCountSchema]), + ) as Record, + ), + provenance_count: z.number().int().nonnegative(), + limitation_count: z.number().int().nonnegative(), +}); + +type MachoReport = z.output; + +/** Admit a well-formed report whose recorded subject, if any, is a Mach-O. */ +const parseMachoReport = ( + parent: AnalysisViewParent, +): Result => { + const parsed = inspectMachoSchema.safeParse(parent.normalizedResult); + if (!parsed.success) + return err( + analysisInputErrorFromIssues( + "inspect_analysis_view", + parsed.error.issues.map((issue) => ({ + ...issue, + path: ["source", "normalized_result", ...issue.path], + })), + { source: { normalized_result: parent.normalizedResult } }, + { cause: parsed.error }, + ), + ); + if (parent.subject != null && !isMachOFormat(parent.subject.format)) + return err( + analysisViewInputError([ + { + path: ["source", "subject"], + reason: "invalid_value", + message: + "Mach-O parent Evidence subject must identify a thin or universal Mach-O.", + }, + ]), + ); + return ok(parsed.data); +}; + +/** Counts replace rows; every collection caveat stays in the limitations. */ +const summarizeMacho = ( + report: MachoReport, + parentLimitations: readonly string[], +) => { + const limitations = uniqueSorted([ + ...parentLimitations, + ...report.limitations, + ...MACHO_COLLECTIONS.flatMap((name) => report[name].limitations), + ]); + const collections = Object.fromEntries( + MACHO_COLLECTIONS.map((name) => [ + name, + { + returned: report[name].items.length, + total: report[name].total, + exhaustive: report[name].exhaustive, + }, + ]), + ) as Record>; + return { + limitations, + returned: MACHO_COLLECTIONS.reduce( + (sum, name) => sum + report[name].items.length, + 0, + ), + summary: { + format: report.format, + selected_architecture: report.selected_architecture, + endian: report.endian, + word_size: report.word_size, + file_type: report.file_type, + flags: report.flags, + uuid: report.uuid, + collections, + provenance_count: report.provenance.length, + limitation_count: limitations.length, + }, + }; +}; + +/** Project retained inspect_macho Evidence without rerunning native tools. */ +export const projectMachoView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if (view.kind !== "summary" && view.kind !== "macho") + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: "Mach-O inspection supports summary and macho views.", + }, + ]), + ); + const parsed = parseMachoReport(parent); + if (!parsed.ok) return parsed; + const report = parsed.value; + const incomplete = MACHO_COLLECTIONS.filter( + (name) => !report[name].exhaustive, + ); + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: "inspect_macho" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: + parent.subject == null + ? { path: "", sha256: null } + : { + path: parent.subject.local_path, + sha256: parent.subject.digest.sha256, + }, + unknowns: [ + ...(incomplete.length === 0 + ? [] + : [ + `Native tools did not report exhaustive ${incomplete.join(", ")}; page exhaustion does not establish absence of unreturned rows.`, + ]), + ...(parent.subject == null + ? [ + "The parent has no recorded subject, so the inspected artifact path and SHA-256 are unknown.", + ] + : []), + ], + }; + if (view.kind === "summary") { + const { summary, limitations, returned } = summarizeMacho( + report, + parent.limitations, + ); + return ok({ + kind: "summary", + view, + ...shared, + limitations, + summary, + coverage: completeWithinViewCoverage(returned, returned), + }); + } + const source = + view.facet === "provenance" + ? { + items: report.provenance, + total: report.provenance.length, + exhaustive: true, + limitations: [], + } + : report[view.facet]; + const item = source.items.slice(view.offset, view.offset + view.limit); + return ok({ + kind: "macho", + view, + item, + ...shared, + limitations: uniqueSorted([ + ...parent.limitations, + ...report.limitations, + ...source.limitations, + ]), + source_total: source.total, + source_exhaustive: source.exhaustive, + coverage: pageViewCoverage(view.offset, item.length, source.items.length), + }); +}; diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 294984339..777028cf8 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -54,6 +54,7 @@ import { silentLogger } from "../logger.js"; import type { Logger } from "pino"; import { registerApplicationTools } from "./registerApplicationTools.js"; import { registerArtifactTools } from "./registerArtifactTools.js"; +import { registerNativeTools } from "./registerNativeTools.js"; import { registerBrowserScenarioTool } from "./registerBrowserScenarioTool.js"; import { registerBrowserTools } from "./registerBrowserTools.js"; import { registerWebScriptTool } from "./registerWebScriptTool.js"; @@ -107,8 +108,6 @@ import type { AdbDeviceAnalysisPort } from "../application/adb/AdbDeviceAnalysis import { AdbDeviceAnalysisService } from "../application/adb/AdbDeviceAnalysisService.js"; import { createAdbDeviceAnalysisProvider } from "../composition/adb.js"; import { registerManagedWorkflowTools } from "./registerManagedWorkflowTools.js"; -import { NATIVE_TOOL_CONTRACTS } from "../contracts/native/nativeToolContracts.js"; -import { registerEvidenceTools } from "./registerEvidenceTools.js"; import { registerOfficialTools } from "./registerOfficialTools.js"; import { registerGuidedPrompts } from "./registerPrompts.js"; import { registerSessionTools } from "./registerSessionTools.js"; @@ -725,7 +724,7 @@ const registerBinaryAnalysisTools = ({ recordEvidence, withAdmittedAnalysis, }; - registerEvidenceTools(server, NATIVE_TOOL_CONTRACTS, evidenceOptions); + registerNativeTools(server, { ...evidenceOptions, evidenceById }); registerArtifactTools(server, { ...evidenceOptions, evidenceById }); registerManagedTools(server, { ...evidenceOptions, evidenceById }); if (session !== undefined) diff --git a/src/server/registerArtifactTools.ts b/src/server/registerArtifactTools.ts index cf95fea41..1217664e6 100644 --- a/src/server/registerArtifactTools.ts +++ b/src/server/registerArtifactTools.ts @@ -6,7 +6,10 @@ import { artifactInspectionInputSchema, } from "../contracts/artifactToolContracts.js"; import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; -import { summarizeRetainedAnalysis } from "../application/analysisView/AnalysisViewService.js"; +import { + requireAnalysisViewRetention, + summarizeRetainedAnalysis, +} from "../application/analysisView/AnalysisViewService.js"; import { createArtifactExtractionDestination } from "../application/artifacts/ArtifactExtractionDestination.js"; import { ok } from "../domain/result.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; @@ -25,6 +28,10 @@ export const registerArtifactTools = ( readonly withAdmittedAnalysis: import("./analysisAdmission.js").WithAdmittedAnalysis; }, ): void => { + const retention = { + recordEvidence: options.recordEvidence, + evidenceById: options.evidenceById, + }; registerEvidenceTools(server, ARTIFACT_TOOL_CONTRACTS, { ...options, parametersForExecution: (operation, parameters) => { @@ -37,12 +44,13 @@ export const registerArtifactTools = ( ? { ...parameters, output_root: createArtifactExtractionDestination() } : parameters; }, + preflight: (operation, parameters) => + operation === "inspect_artifact" && parameters.detail === "summary" + ? requireAnalysisViewRetention(operation, retention) + : ok(null), projectEvidence: (operation, parameters, evidence) => operation === "inspect_artifact" && parameters.detail === "summary" - ? summarizeRetainedAnalysis(evidence, { - recordEvidence: options.recordEvidence, - evidenceById: options.evidenceById, - }) + ? summarizeRetainedAnalysis(evidence, retention) : ok(evidence), sourceEvidence: (operation, result) => operation === "inspect_artifact" diff --git a/src/server/registerEvidenceTools.ts b/src/server/registerEvidenceTools.ts index a8efac6de..b3b2358d1 100644 --- a/src/server/registerEvidenceTools.ts +++ b/src/server/registerEvidenceTools.ts @@ -33,6 +33,16 @@ interface EvidenceToolRegistration { operation: Exclude, parameters: Readonly>, ) => Readonly>; + /** Exclude adapter-only selections, such as delivery detail, from Evidence identity. */ + readonly parametersForEvidence?: ( + operation: Exclude, + parameters: Readonly>, + ) => Readonly>; + /** Refuse an unsatisfiable delivery selection before any analysis runs. */ + readonly preflight?: ( + operation: Exclude, + parameters: Readonly>, + ) => Result; readonly projectEvidence?: ( operation: Exclude, parameters: Readonly>, @@ -51,6 +61,12 @@ export const registerEvidenceTools = ( contract.name, toolRegistrationOptions(contract), async (input, context) => { + const ready = options.preflight?.( + contract.name, + jsonObjectSchema.parse(input), + ); + if (ready !== undefined && !ready.ok) + return server.delivery.toCallToolResult(ready, contract); const admitted = await options.withAdmittedAnalysis( contract.name, context.mcpReq.signal, @@ -95,7 +111,9 @@ export const registerEvidenceTools = ( execution.value.provider, { operation: contract.name, - parameters, + parameters: + options.parametersForEvidence?.(contract.name, parameters) ?? + parameters, result: execution.value.result, ...(execution.value.analysisProfile === undefined ? {} diff --git a/src/server/registerNativeTools.ts b/src/server/registerNativeTools.ts new file mode 100644 index 000000000..036389d16 --- /dev/null +++ b/src/server/registerNativeTools.ts @@ -0,0 +1,54 @@ +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; + +import { NATIVE_TOOL_CONTRACTS } from "../contracts/native/nativeToolContracts.js"; +import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; +import { + requireAnalysisViewRetention, + summarizeRetainedAnalysis, +} from "../application/analysisView/AnalysisViewService.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { ok } from "../domain/result.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import type { Logger } from "pino"; +import { registerEvidenceTools } from "./registerEvidenceTools.js"; + +/** Delivery detail selects a response shape; it never reaches the tools or Evidence identity. */ +const withoutDelivery = ( + operation: string, + parameters: Readonly>, +): Readonly> => { + if (operation !== "inspect_macho") return parameters; + const { detail: _detail, ...selected } = parameters; + return selected; +}; + +/** Register macOS native contracts; Mach-O inspection can deliver a retained summary. */ +export const registerNativeTools = ( + server: EvidenceMcpServer, + options: { + readonly logger: Logger; + readonly activeTarget: (() => BinaryTarget | undefined) | undefined; + readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; + readonly evidenceById?: EvidenceLookup | undefined; + readonly withAdmittedAnalysis: import("./analysisAdmission.js").WithAdmittedAnalysis; + }, +): void => { + const retention = { + recordEvidence: options.recordEvidence, + evidenceById: options.evidenceById, + }; + registerEvidenceTools(server, NATIVE_TOOL_CONTRACTS, { + ...options, + parametersForExecution: withoutDelivery, + parametersForEvidence: withoutDelivery, + preflight: (operation, parameters) => + operation === "inspect_macho" && parameters.detail === "summary" + ? requireAnalysisViewRetention(operation, retention) + : ok(null), + projectEvidence: (operation, parameters, evidence) => + operation === "inspect_macho" && parameters.detail === "summary" + ? summarizeRetainedAnalysis(evidence, retention) + : ok(evidence), + }); +}; diff --git a/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts new file mode 100644 index 000000000..e492263fa --- /dev/null +++ b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts @@ -0,0 +1,216 @@ +import { silentLogger } from "../../../src/logger.js"; +import { copyFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect } from "vitest"; +import { z } from "zod"; + +import { createBinarySession } from "../../../src/composition/binary.js"; +import { parseConfig } from "../../../src/config/parseConfig.js"; +import { inspectMachoSchema } from "../../../src/domain/native/nativeInspection.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { observed } from "../../fixtures/analysisExecution.js"; +import { ok } from "../../../src/domain/result.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; + +const evidenceOf = (schema: Schema) => + z.object({ + evidence_id: z.string(), + parameters: z.record(z.string(), z.unknown()), + normalized_result: schema, + }); +const countSchema = z.object({ + returned: z.number(), + total: z.number().nullable(), + exhaustive: z.boolean(), +}); +const summarySchema = z.object({ + kind: z.literal("summary"), + parent_evidence_id: z.string(), + summary: z.object({ + uuid: z.string().nullable(), + collections: z.record(z.string(), countSchema), + provenance_count: z.number(), + }), +}); +const pageSchema = z.object({ + kind: z.literal("macho"), + item: z.array(z.unknown()), + coverage: z.object({ next_offset: z.number().nullable() }), +}); + +describe.skipIf(process.platform !== "darwin")( + "Mach-O summary delivery through MCP", + () => { + mcpTest( + "retains complete inspect_macho Evidence and pages its exact rows", + async ({ mcp, onTestFinished }) => { + const target = join( + await createTestTempDirectory("rea-macho-summary-mcp-"), + "ls", + ); + await copyFile("/bin/ls", target); + const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); + if (!configured.ok) throw configured.error; + const session = createBinarySession( + configured.value, + silentLogger, + process.env, + ); + onTestFinished(async () => { + await session.close(); + }); + const client = await mcp.connect( + createServer({ kind: "session", session }), + ); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: target }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + + const complete = await client.callTool({ + name: "inspect_macho", + arguments: {}, + }); + expect(complete.isError, JSON.stringify(complete.content)).not.toBe( + true, + ); + const full = evidenceOf(inspectMachoSchema).parse( + complete.structuredContent, + ); + // Omitted detail keeps the default Evidence parameters unchanged. + expect(full.parameters).toEqual({}); + + const summarized = await client.callTool({ + name: "inspect_macho", + arguments: { detail: "summary" }, + }); + expect(summarized.isError, JSON.stringify(summarized.content)).not.toBe( + true, + ); + const summary = evidenceOf(summarySchema).parse( + summarized.structuredContent, + ).normalized_result; + // Delivery detail never reaches the tools or the source Evidence ID. + expect(summary.parent_evidence_id).toBe(full.evidence_id); + const report = full.normalized_result; + expect(summary.summary.uuid).toBe(report.uuid); + expect(summary.summary.provenance_count).toBe(report.provenance.length); + for (const name of ["imports", "exports", "dependencies"] as const) + expect(summary.summary.collections[name]).toEqual({ + returned: report[name].items.length, + total: report[name].total, + exhaustive: report[name].exhaustive, + }); + expect(JSON.stringify(summarized.structuredContent)).not.toContain( + report.imports.items[0]?.name ?? "_", + ); + + // Every facet must reassemble the unparsed complete rows exactly. + const raw = z + .object({ normalized_result: z.record(z.string(), z.unknown()) }) + .parse(complete.structuredContent).normalized_result; + const facets = [ + "entrypoints", + "architectures", + "build_metadata", + "load_commands", + "dependencies", + "imports", + "exports", + "segments", + "provenance", + ] as const; + for (const facet of facets) { + const rows: unknown[] = []; + let offset: number | null = 0; + while (offset !== null) { + const paged = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }, + view: { kind: "macho", facet, offset, limit: 16 }, + }, + }); + expect(paged.isError, JSON.stringify(paged.content)).not.toBe(true); + const page = evidenceOf(pageSchema).parse( + paged.structuredContent, + ).normalized_result; + rows.push(...page.item); + offset = page.coverage.next_offset; + } + expect(rows, facet).toEqual( + facet === "provenance" + ? raw.provenance + : z.object({ items: z.array(z.unknown()) }).parse(raw[facet]) + .items, + ); + } + + const wrongKind = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }, + view: { kind: "wasm", facet: "exports" }, + }, + }); + expect(wrongKind.isError).toBe(true); + expect(JSON.stringify(wrongKind.content)).toContain( + "Mach-O inspection supports summary and macho views.", + ); + }, + ); + }, +); + +mcpTest( + "refuses summary delivery before analysis when the server retains no Evidence", + async ({ mcp, onTestFinished }) => { + const target = join( + await createTestTempDirectory("rea-summary-preflight-"), + "tool", + ); + await writeFile(target, thinMach(0xfeedfacf, 0x0100000c)); + const executed: string[] = []; + const session = createTestBinarySession(() => ({ + execute: (operation) => { + executed.push(operation); + return Promise.resolve(observed(null)); + }, + close: () => Promise.resolve(ok(null)), + })); + onTestFinished(async () => { + await session.close(); + }); + const opened = await session.open(target); + if (!opened.ok) throw opened.error; + const client = await mcp.connect( + createServer({ kind: "fixed", analysis: session }), + ); + const refused = []; + for (const name of ["inspect_macho", "inspect_artifact"]) + refused.push( + await client.callTool({ name, arguments: { detail: "summary" } }), + ); + // Neither analysis ran only to be discarded by the delivery refusal. + expect(executed.filter((operation) => operation !== "health")).toEqual([]); + for (const result of refused) + expect(parseMcpToolError(result)).toMatchObject({ + error: { + code: "capability_unavailable", + message: expect.stringContaining("does not retain Evidence"), + }, + }); + }, +);