diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b35b6a6e1..f55667dce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -634,6 +634,7 @@ jobs: tests/boundary/cli/nativeSignatureCli.test.ts tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts src/process/capture/FilesystemSnapshot.test.ts + tests/process-global/filesystemSnapshotFallbackCancellation.test.ts tests/boundary/process/darwinProcessRunTokenReader.test.ts tests/boundary/process/directAnalysisSignals.test.ts tests/boundary/process/processCaptureLifecycle.test.ts @@ -714,6 +715,8 @@ jobs: - name: Run curated Windows capability tests run: >- npx vitest run --coverage.enabled=false + src/process/capture/FilesystemSnapshot.test.ts + tests/process-global/filesystemSnapshotFallbackCancellation.test.ts tests/boundary/filesystem/binaryTarget.test.ts tests/boundary/filesystem/safeOutputTree.test.ts tests/boundary/providers/native/artifactProvider.archiveSafety.test.ts diff --git a/docs/testing.md b/docs/testing.md index aa8d2163f..042248a26 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -19,7 +19,7 @@ behavior, and complete CI evidence before merging. | `npm run test:boundary` | Boundary, CLI boundary, MCP boundary, process boundary, and process-global projects | | `npm run test:mcp` | MCP boundary project | | `npm run test:acceptance` | Complete compiled CLI and MCP acceptance workflows | -| `npm run test:python` | LLDB producer retention and launch contracts with a fake LLDB API; no native debugger required | +| `npm run test:python` | LLDB and mitmproxy producer contracts plus analysis-efficiency checks; no real debugger or proxy required | | `npm run test:watch` | Dirty source tests in watch mode, without build | | `npm run test:watch:all` | Changed tests from every project; builds at startup, so rebuild after production edits before relying on compiled tests | | `npm run check:changed` | Cached typecheck/lint and branch-related source feedback | @@ -449,8 +449,9 @@ adds in-place reparse, breakaway, environment, and token observations. The separate `scripts/verify-windows-native-inspection.mjs` lane uses the `.cache/windows-native-inspection-fixture` artifact built on Linux with `node scripts/build-windows-native.mjs --inspection-fixture`. Its controlled -first deletion failure checks retained NTFS ownership, later-query recovery, -successful-admission caching, and environment-teardown cleanup. It neither +first deletion failure checks retained NTFS ownership across a same-volume +rename, later-query handle-based recovery, successful-admission caching, and +environment-teardown cleanup. It neither establishes the cause of the intermittent #1701 host failure nor replaces verification with the shipped production artifact. `npm run verify:ghidra:windows:package` additionally packs and installs REA into diff --git a/knip.json b/knip.json index 5ed82edf0..97a323b6c 100644 --- a/knip.json +++ b/knip.json @@ -9,8 +9,8 @@ "src/application/RealHopperAssertions.ts", "scripts/verify/fridaBindingsCompatibility.ts", "scripts/verify/termux/browser.mjs", - "scripts/verify-real-cutter.mjs", "scripts/verify-real-ghidra-noreturn.mjs", + "scripts/run-exact-tests.mjs", "tests/processOwnershipRunner.ts" ], "project": ["src/**/*.ts", "tests/**/*.ts", "scripts/**/*.mjs"], diff --git a/package.json b/package.json index 5ccf53c99..45e73afc0 100644 --- a/package.json +++ b/package.json @@ -138,7 +138,7 @@ "test": "npm run test:with-build", "test:with-build": "npm run test:prepare && vitest run", "test:run": "vitest run", - "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", + "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", "test:local": "node scripts/run-development-tests.mjs local", "test:fast": "node scripts/ci/test-group.mjs source", "test:built": "node scripts/ci/test-group.mjs built", diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 6e3f950b8..cb942bfd9 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -202,6 +202,9 @@ if (tier === "reused") { if (GUARDED_WORKFLOWS.has(file)) selected.code = true; if (file !== ".github/workflows/ci.yml") continue; } + // The Notes example is also the source-owned agent evaluation fixture. + if (file.startsWith("website/public/examples/notes-electron/")) + selected.code = true; if (owners.length === 1 && owners[0] === "website") continue; if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true; if (file === "package.json") { diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index b13057b33..01cb8e329 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -79,6 +79,7 @@ export const SCOPES = { "tests/boundary/cli/nativeSignatureCli.test.ts", "tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts", "src/process/capture/FilesystemSnapshot.test.ts", + "tests/process-global/filesystemSnapshotFallbackCancellation.test.ts", "tests/boundary/process/darwinProcessRunTokenReader.test.ts", "tests/boundary/process/directAnalysisSignals.test.ts", "tests/boundary/process/processCaptureLifecycle.test.ts", @@ -89,6 +90,9 @@ export const SCOPES = { package: [ "src/windows/**", "native/windows/**", + "src/process/capture/FilesystemSnapshot.ts", + "src/process/capture/FilesystemSnapshot.test.ts", + "tests/process-global/filesystemSnapshotFallbackCancellation.test.ts", "src/inspector/**", "scripts/verify/inspector/**", "scripts/verify-package.mjs", diff --git a/scripts/create-ghidra-windows-fixture.mjs b/scripts/create-ghidra-windows-fixture.mjs index 214d7dadc..aa22c8165 100644 --- a/scripts/create-ghidra-windows-fixture.mjs +++ b/scripts/create-ghidra-windows-fixture.mjs @@ -1,6 +1,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { writeWindowsGhidraByteSearchFixture } from "./lib/ghidra-windows-byte-search.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); // Source-owned static fixtures: no compiler, target loading or execution. @@ -61,7 +62,7 @@ for (const architecture of ["x86_64", "x86"]) { } bytes.write(".text", sectionTable, "ascii"); - bytes.writeUInt32LE(dll ? 0x180 : 0x50, sectionTable + 8); + bytes.writeUInt32LE(0x200, sectionTable + 8); bytes.writeUInt32LE(0x1000, sectionTable + 12); bytes.writeUInt32LE(0x200, sectionTable + 16); bytes.writeUInt32LE(0x200, sectionTable + 20); @@ -136,6 +137,8 @@ for (const architecture of ["x86_64", "x86"]) { bytes.write("fixture_return42\0", 0x340, "binary"); } + writeWindowsGhidraByteSearchFixture(bytes); + await mkdir(dirname(output), { recursive: true }); await writeFile(output, bytes); if (!x86 && !dll) process.stdout.write(`${output}\n`); diff --git a/scripts/lib/ghidra-windows-byte-search.mjs b/scripts/lib/ghidra-windows-byte-search.mjs new file mode 100644 index 000000000..b432b9fc7 --- /dev/null +++ b/scripts/lib/ghidra-windows-byte-search.mjs @@ -0,0 +1,149 @@ +import assert from "node:assert/strict"; + +export const WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE = Object.freeze({ + pattern: "a0 ?? c0", + mask: "f0 ff f0", + startFileOffset: 0x380, + endFileOffset: 0x392, + markerOffsets: Object.freeze([0x380, 0x390]), +}); + +/** Put two masked, wildcarded signatures in loaded, source-owned PE bytes. */ +export function writeWindowsGhidraByteSearchFixture(bytes) { + bytes.set([0xa1, 0xb2, 0xc3], 0x380); + bytes.set([0xa4, 0xb5, 0xc6], 0x390); +} + +/** Verify real Ghidra search pages against the immutable source fixture. */ +export async function verifyWindowsGhidraByteSearch({ + invoke, + sourceBytes, + architecture, + sourceOwnedFixture = true, + fallbackAddress, + fallbackFileOffset, + runCliPage, +}) { + const imageBase = architecture === "x86" ? 0x400000n : 0x140000000n; + const query = sourceOwnedFixture + ? { + pattern: WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.pattern, + mask: WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.mask, + start: addressAtFileOffset( + imageBase, + WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.startFileOffset, + ), + end: addressAtFileOffset( + imageBase, + WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.endFileOffset, + ), + limit: 1, + } + : { + pattern: sourceBytes + .subarray(fallbackFileOffset, fallbackFileOffset + 1) + .toString("hex"), + start: fallbackAddress, + end: fallbackAddress, + limit: 1, + }; + + if (!sourceOwnedFixture) { + const result = await invoke("search_bytes", query); + assert.equal(result.complete, true); + assert.equal(result.next_cursor, null); + assert.deepEqual( + result.matches.map(({ address, file_offset }) => ({ + address, + file_offset, + })), + [{ address: fallbackAddress, file_offset: fallbackFileOffset }], + ); + assert.ok(result.searched.bytes > 0); + return { bounded_source_match: true }; + } + + const { startFileOffset, endFileOffset } = WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE; + const expected = []; + for ( + let fileOffset = startFileOffset; + fileOffset + 2 <= endFileOffset; + fileOffset += 1 + ) { + const first = sourceBytes[fileOffset]; + const last = sourceBytes[fileOffset + 2]; + if (first === undefined || last === undefined) break; + if ((first & 0xf0) === 0xa0 && (last & 0xf0) === 0xc0) + expected.push({ + address: addressAtFileOffset(imageBase, fileOffset), + file_offset: fileOffset, + }); + } + assert.deepEqual( + expected.map(({ file_offset }) => file_offset), + [...WINDOWS_GHIDRA_BYTE_SEARCH_FIXTURE.markerOffsets], + "The source-owned byte oracle must retain both distinct signatures", + ); + + const first = await invoke("search_bytes", query); + assert.deepEqual( + first.matches.map(({ address, file_offset }) => ({ address, file_offset })), + [expected[0]], + ); + assert.equal(first.complete, false); + assert.equal(typeof first.next_cursor, "string"); + assert.ok(first.searched.bytes > 0); + if (runCliPage !== undefined) { + const cli = await runCliPage(query); + assert.deepEqual(cli.matches, first.matches); + assert.deepEqual(cli.searched, first.searched); + assert.equal(cli.complete, first.complete); + } + + const second = await invoke("search_bytes", { + ...query, + cursor: first.next_cursor, + }); + assert.deepEqual( + second.matches.map(({ address, file_offset }) => ({ + address, + file_offset, + })), + [expected[1]], + ); + assert.equal(second.complete, true); + assert.equal(second.next_cursor, null); + assert.ok(second.searched.bytes > 0); + + const covered = new Set(); + for (const page of [first, second]) { + assert.equal( + page.searched.bytes, + page.searched.blocks.reduce((sum, block) => sum + block.bytes, 0), + ); + for (const block of page.searched.blocks) { + const start = BigInt(block.start); + const end = BigInt(block.end); + assert.ok(start >= BigInt(query.start) && end <= BigInt(query.end)); + for (let offset = start; offset <= end; offset += 1n) + covered.add(offset.toString()); + } + } + assert.equal( + covered.size, + endFileOffset - startFileOffset + 1, + "Paged byte-search coverage must account for the complete bounded source range", + ); + return { + bounded: true, + wildcardAndMask: true, + matches: expected, + continuation: true, + examinedBytes: covered.size, + cliMcpParity: runCliPage !== undefined, + }; +} + +function addressAtFileOffset(imageBase, fileOffset) { + return `0x${(imageBase + 0x1000n + BigInt(fileOffset - 0x200)).toString(16)}`; +} diff --git a/scripts/lib/real-ghidra-address-boundaries.mjs b/scripts/lib/real-ghidra-address-boundaries.mjs index 0835ecec3..a2211a1ba 100644 --- a/scripts/lib/real-ghidra-address-boundaries.mjs +++ b/scripts/lib/real-ghidra-address-boundaries.mjs @@ -62,36 +62,68 @@ export async function verifyGhidraAddressBoundaries(client, entry, cli) { await query("read_bytes", { address: padded, length: 1 }), await query("read_bytes", { address: entry, length: 1 }), ); - const aliasKeys = [entry, padded]; - assert.deepEqual( - await query("set_addresses_names", { - names: { - [entry]: "rea_same_address_alias", - [padded]: "rea_same_address_alias", - }, - }), - Object.fromEntries(aliasKeys.map((address) => [address, true])), - "Equivalent address spellings with one requested name share one edit", - ); - const nameBeforeAliasedBatch = await query("address_name", { - address: entry, - }); - assert.equal(nameBeforeAliasedBatch, "rea_same_address_alias"); - assert.deepEqual( - await query("set_addresses_names", { - names: { - [entry]: "rea_first_address_alias", - [padded]: "rea_second_address_alias", - }, - }), - Object.fromEntries(aliasKeys.map((address) => [address, false])), - "Conflicting spellings of one Ghidra address must be rejected together", - ); + const originalName = await query("address_name", { address: entry }); assert.equal( - await query("address_name", { address: entry }), - nameBeforeAliasedBatch, - "A rejected alias collision must leave the preexisting target name unchanged", + typeof originalName, + "string", + "The function entry needs an observed name that can be restored", ); + const aliasKeys = [entry, padded]; + let primaryFailure; + try { + assert.deepEqual( + await query("set_addresses_names", { + names: { + [entry]: "rea_same_address_alias", + [padded]: "rea_same_address_alias", + }, + }), + Object.fromEntries(aliasKeys.map((address) => [address, true])), + "Equivalent address spellings with one requested name share one edit", + ); + const nameBeforeAliasedBatch = await query("address_name", { + address: entry, + }); + assert.equal(nameBeforeAliasedBatch, "rea_same_address_alias"); + assert.deepEqual( + await query("set_addresses_names", { + names: { + [entry]: "rea_first_address_alias", + [padded]: "rea_second_address_alias", + }, + }), + Object.fromEntries(aliasKeys.map((address) => [address, false])), + "Conflicting spellings of one Ghidra address must be rejected together", + ); + assert.equal( + await query("address_name", { address: entry }), + nameBeforeAliasedBatch, + "A rejected alias collision must leave the preexisting target name unchanged", + ); + } catch (cause) { + primaryFailure = cause; + throw cause; + } finally { + try { + assert.equal( + await query("set_address_name", { address: entry, name: originalName }), + true, + "The temporary alias edit must be restored", + ); + assert.equal( + await query("address_name", { address: entry }), + originalName, + "The original function-entry name must survive restoration", + ); + } catch (cleanupCause) { + if (primaryFailure !== undefined) + throw new AggregateError( + [primaryFailure, cleanupCause], + "Ghidra address-boundary verification and alias restoration failed", + ); + throw cleanupCause; + } + } for (const [command, flags, operation] of [ ["read-bytes", ["--length", "1"], "read_bytes"], [ diff --git a/scripts/lib/real-hopper-search.d.mts b/scripts/lib/real-hopper-search.d.mts index 0a7260227..d3bb4c0d3 100644 --- a/scripts/lib/real-hopper-search.d.mts +++ b/scripts/lib/real-hopper-search.d.mts @@ -1,3 +1,7 @@ +import type { z } from "zod"; + +import type { analysisStringSchema } from "../../src/domain/hopperValues.js"; + type VerifierCall = ( name: string, arguments_?: Record, @@ -44,6 +48,3 @@ export function verifyHopperStringObjects( }[]; readonly displayEscapesExcludedFromMatching: true; }>; -import type { z } from "zod"; - -import type { analysisStringSchema } from "../../src/domain/hopperValues.js"; diff --git a/scripts/run-development-tests.mjs b/scripts/run-development-tests.mjs index bd3c65b28..d53b59c84 100644 --- a/scripts/run-development-tests.mjs +++ b/scripts/run-development-tests.mjs @@ -68,7 +68,11 @@ try { } if (!process.exitCode && !shutdownSignal) { const tested = await run(process.execPath, [ - resolve("node_modules/vitest/vitest.mjs"), + resolve( + request.paths.length > 0 + ? "scripts/run-exact-tests.mjs" + : "node_modules/vitest/vitest.mjs", + ), ...plan.vitestArguments, ]); process.exitCode = tested.code; diff --git a/scripts/run-exact-tests.mjs b/scripts/run-exact-tests.mjs new file mode 100644 index 000000000..98e723b3b --- /dev/null +++ b/scripts/run-exact-tests.mjs @@ -0,0 +1,134 @@ +import { lstat, realpath, stat } from "node:fs/promises"; +import { parse, relative, resolve, sep } from "node:path"; + +import { createVitest, parseCLI } from "vitest/node"; + +// CLI file filters are substring matches. Select discovered specifications by +// identity instead, retaining each project's include/exclude and runner settings. +const { filter, options } = parseCLI(["vitest", ...process.argv.slice(2)]); +const vitest = await createVitest({ ...options, watch: false }); +const signalExitCodes = { SIGINT: 130, SIGTERM: 143 }; +let shutdownSignal; +let closePromise; +const close = () => (closePromise ??= vitest.close()); +for (const signal of Object.keys(signalExitCodes)) + process.on(signal, () => { + if (shutdownSignal !== undefined) return; + shutdownSignal = signal; + void close().catch((cause) => { + console.error(cause instanceof Error ? cause.message : String(cause)); + process.exitCode = 1; + }); + }); +try { + if (shutdownSignal === undefined) await vitest.standalone(); + if (shutdownSignal === undefined) { + const discovered = new Map(); + for (const specification of await vitest.globTestSpecifications()) { + const path = resolve(specification.moduleId); + const specifications = discovered.get(path) ?? []; + specifications.push(specification); + discovered.set(path, specifications); + } + const selected = new Set(); + const missing = []; + let discoveredByIdentity; + for (const path of filter) { + const absolute = resolve(path); + let matches = discovered.get(absolute) ?? []; + if (matches.length === 0) { + try { + await lstat(absolute); + } catch (cause) { + if (cause?.code === "ENOENT" || cause?.code === "ENOTDIR") { + missing.push(path); + continue; + } + throw cause; + } + const [canonical, selectedIdentity] = await Promise.all([ + realpath(absolute), + stat(absolute, { bigint: true }), + ]); + if (discoveredByIdentity === undefined) { + discoveredByIdentity = new Map(); + for (const [candidate, specifications] of discovered) { + try { + const entry = await stat(candidate, { bigint: true }); + const identity = `${entry.dev}:${entry.ino}`; + const entries = discoveredByIdentity.get(identity) ?? []; + entries.push([candidate, specifications]); + discoveredByIdentity.set(identity, entries); + } catch (cause) { + if (cause?.code === "ENOENT" || cause?.code === "ENOTDIR") + continue; + throw cause; + } + } + } + const identity = `${selectedIdentity.dev}:${selectedIdentity.ino}`; + for (const [candidate, specifications] of discoveredByIdentity.get( + identity, + ) ?? []) { + try { + await lstat(candidate); + } catch (cause) { + if (cause?.code === "ENOENT" || cause?.code === "ENOTDIR") continue; + throw cause; + } + if ( + !(await hasCaseMismatchedSymlinkComponent(absolute, candidate)) && + (await realpath(candidate)) === canonical + ) + matches = [...matches, ...specifications]; + } + } + if (matches.length === 0) missing.push(path); + for (const specification of matches) selected.add(specification); + } + if (missing.length > 0 || selected.size === 0) + throw new Error( + `Selected test files were not discovered: ${missing.join(", ")}`, + ); + if (shutdownSignal === undefined) + await vitest.runTestSpecifications([...selected]); + } +} finally { + await close(); +} +if (shutdownSignal !== undefined) + process.exitCode = signalExitCodes[shutdownSignal]; + +async function hasCaseMismatchedSymlinkComponent(selected, candidate) { + const selectedAbsolute = resolve(selected); + const candidateAbsolute = resolve(candidate); + const root = parse(selectedAbsolute).root; + if (root !== parse(candidateAbsolute).root) return true; + const selectedParts = relative(root, selectedAbsolute).split(sep); + const candidateParts = relative(root, candidateAbsolute).split(sep); + let selectedPath = root; + let candidatePath = root; + for (let index = 0; index < selectedParts.length; index += 1) { + const selectedPart = selectedParts[index]; + const candidatePart = candidateParts[index]; + if (selectedPart === undefined || candidatePart === undefined) return true; + selectedPath = selectedPath + ? resolve(selectedPath, selectedPart) + : selectedPart; + candidatePath = candidatePath + ? resolve(candidatePath, candidatePart) + : candidatePart; + if ( + selectedPart !== candidatePart && + selectedPart.toLowerCase() === candidatePart.toLowerCase() + ) { + const [selectedEntry, candidateEntry] = await Promise.all([ + lstat(selectedPath), + lstat(candidatePath), + ]); + if (selectedEntry.isSymbolicLink() || candidateEntry.isSymbolicLink()) + return true; + } + } + return false; +} diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 71cd6383c..349877075 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -155,6 +155,18 @@ try { requiredAnswerTermGroups: [["profileapi"], ["profile:read"]], prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`, }, + { + id: "notes-export", + expectedFirstTool: "analyze_javascript_application", + requiresEvidence: true, + requiredAnswerTermGroups: [], + requiredToolSubsequence: [ + "analyze_javascript_application", + "trace_application_feature", + "trace_javascript_semantics", + ], + prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`, + }, { id: "javascript-module-view", expectedFirstTool: "analyze_javascript_application", diff --git a/scripts/verify-go-binary.mjs b/scripts/verify-go-binary.mjs index 0724c5f4d..4af896656 100644 --- a/scripts/verify-go-binary.mjs +++ b/scripts/verify-go-binary.mjs @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { isUtf8 } from "node:buffer"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { lstat, mkdir, readFile, writeFile } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; @@ -24,6 +24,8 @@ if (process.env.REA_VERIFY_GO_COMMAND !== undefined && !isAbsolute(compiler)) let compilerVersion = null; const run = createVerifierRun(); const root = await PrivateRuntimeRoot.create({ prefix: "rea-go-verifier-" }); +// Keep Go's directory enumeration below the private root's Windows ownership lease. +const sourceRoot = join(root.path, "source"); const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url)); const environment = Object.fromEntries( Object.entries(process.env).filter(([, value]) => typeof value === "string"), @@ -115,14 +117,54 @@ try { ], { GOOS: os, GOARCH: arch }, ); - await verify(path, target, oracle, null, "stripped"); + const strippedInfo = await verify(path, target, oracle, null, "stripped"); if (os === "linux" && arch === "amd64") { const sectionless = join(root.path, "linux-amd64-sectionless"); const bytes = Buffer.from(await readFile(path)); bytes.fill(0, 40, 48); // ELF64 e_shoff. bytes.fill(0, 58, 64); // e_shentsize, e_shnum and e_shstrndx. await writeFile(sectionless, bytes); - await verify(sectionless, target, oracle, null, "sectionless"); + const sectionlessInfo = await verify( + sectionless, + target, + oracle, + null, + "sectionless", + path, + ); + assert.deepEqual(sectionlessInfo, strippedInfo); + + const pie = join(root.path, "linux-amd64-pie"); + await go( + [ + "build", + "-trimpath", + "-buildvcs=false", + "-ldflags=-s -w", + "-buildmode=pie", + "-o", + pie, + ".", + ], + { GOOS: os, GOARCH: arch }, + ); + const originalInfo = await verify(pie, target, oracle, null, "pie"); + const pieBytes = Buffer.from(await readFile(pie)); + pieBytes.fill(0, 40, 48); + pieBytes.fill(0, 58, 64); + const pieSectionless = `${pie}-sectionless`; + await writeFile(pieSectionless, pieBytes); + // Keep the producer image as the independent oracle. Removing section + // headers must preserve every metadata field and original source span. + const retainedInfo = await verify( + pieSectionless, + target, + oracle, + null, + "sectionless-pie", + pie, + ); + assert.deepEqual(retainedInfo, originalInfo); } } for (const target of targets.filter(([os]) => os === "darwin")) { @@ -143,7 +185,7 @@ try { const expected = JSON.parse( ( await execute(oracle, ["--build-byte-fixture", compiler, path], { - cwd: root.path, + cwd: sourceRoot, env: { ...goEnvironment, GOOS: "linux", GOARCH: "amd64" }, timeout: 120_000, maxBuffer: 1024 * 1024, @@ -169,7 +211,7 @@ try { } const verifier = await completeVerifierRun(run); try { - await assert.rejects(readFile(join(root.path, "go.mod")), { code: "ENOENT" }); + await assert.rejects(lstat(root.path), { code: "ENOENT" }); if (verifier.process_lineage.status === "verified") assert.deepEqual(verifier.process_lineage.descendants, []); } catch (cause) { @@ -214,7 +256,7 @@ console.log( async function go(args, overrides = {}) { return execute(compiler, args, { - cwd: root.path, + cwd: sourceRoot, env: { ...goEnvironment, ...overrides }, timeout: 120_000, maxBuffer: 1024 * 1024, @@ -222,29 +264,30 @@ async function go(args, overrides = {}) { } async function prepareFixture() { + await mkdir(sourceRoot); await mkdir(join(root.path, "tmp")); await mkdir(join(root.path, "telemetry")); await writeFile(join(root.path, "telemetry", "mode"), "off\n"); - await mkdir(join(root.path, "dependency")); - await mkdir(join(root.path, "oracle")); + await mkdir(join(sourceRoot, "dependency")); + await mkdir(join(sourceRoot, "oracle")); await writeFile( - join(root.path, "go.mod"), + join(sourceRoot, "go.mod"), "module example.test/rea-go-fixture\n\ngo 1.20\n\nrequire example.test/dependency v1.2.3\nreplace example.test/dependency => ./dependency\n", ); await writeFile( - join(root.path, "dependency", "go.mod"), + join(sourceRoot, "dependency", "go.mod"), "module example.test/dependency\n\ngo 1.20\n", ); await writeFile( - join(root.path, "dependency", "dependency.go"), + join(sourceRoot, "dependency", "dependency.go"), 'package dependency\nfunc Value() string { return "REA fixture" }\n', ); await writeFile( - join(root.path, "main.go"), + join(sourceRoot, "main.go"), 'package main\nimport ("fmt"; "example.test/dependency")\nfunc main() { fmt.Println(dependency.Value()) }\n', ); await writeFile( - join(root.path, "oracle", "main.go"), + join(sourceRoot, "oracle", "main.go"), String.raw`package main import ( @@ -332,13 +375,14 @@ async function verify( oracle, byteExpected = null, fileKind = "stripped", + oraclePath = path, ) { const bytes = await readFile(path); const expected = byteExpected ?? JSON.parse( ( - await execute(oracle, [path], { + await execute(oracle, [oraclePath], { timeout: 10_000, maxBuffer: 1024 * 1024, }) @@ -408,6 +452,7 @@ async function verify( ? {} : { go_version_bytes_base64: expected.GoVersionBytesBase64 }), }); + return reports[0].build_info; } function verifyByteStrings(info, expected) { diff --git a/scripts/verify-module-boundaries.mjs b/scripts/verify-module-boundaries.mjs index 1064d2ae4..2069c96db 100644 --- a/scripts/verify-module-boundaries.mjs +++ b/scripts/verify-module-boundaries.mjs @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; -import { readFile } from "node:fs/promises"; -import { resolve } from "node:path"; +import { readdir, readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; import { promisify } from "node:util"; import { inspectModuleBoundaries } from "./lib/module-boundaries.mjs"; @@ -11,24 +11,44 @@ const selected = process.argv.slice(2); for (const file of selected) if (!file.endsWith(".ts")) throw new TypeError(`Expected an exact TypeScript source file: ${file}`); -const files = - selected.length > 0 - ? selected - : ( - await execute( - "git", - [ - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-standard", - "--", - "src", - ], - { cwd: root, maxBuffer: 8 * 1024 * 1024 }, - ) - ).stdout.split("\0"); + +const enumerateSourceFiles = async () => { + if (selected.length > 0) return selected; + let isGitRoot = false; + try { + const { stdout } = await execute("git", ["rev-parse", "--show-cdup"], { + cwd: root, + maxBuffer: 8 * 1024 * 1024, + }); + isGitRoot = stdout.trim() === ""; + } catch { + // Archives and source snapshots may not be in a Git worktree. + } + if (isGitRoot) { + try { + const { stdout } = await execute( + "git", + [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + "src", + ], + { cwd: root, maxBuffer: 8 * 1024 * 1024 }, + ); + return stdout.split("\0"); + } catch { + // Fall back to disk when Git's index is unavailable or unreadable. + } + } + const entries = await readdir(resolve(root, "src"), { recursive: true }); + return entries.map((entry) => join("src", entry)); +}; + +const files = await enumerateSourceFiles(); let checked = 0; for (const file of new Set(files.filter((path) => path.endsWith(".ts")))) { @@ -36,7 +56,11 @@ for (const file of new Set(files.filter((path) => path.endsWith(".ts")))) { try { source = await readFile(resolve(root, file), "utf8"); } catch (cause) { - if (selected.length === 0 && cause.code === "ENOENT") continue; + if ( + selected.length === 0 && + (cause.code === "ENOENT" || cause.code === "EISDIR") + ) + continue; throw cause; } checked += 1; diff --git a/scripts/verify-opencode-client.mjs b/scripts/verify-opencode-client.mjs index a29610ab3..b1006f8ba 100644 --- a/scripts/verify-opencode-client.mjs +++ b/scripts/verify-opencode-client.mjs @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { isAbsolute, join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; import { @@ -42,6 +42,21 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); +const isManagedSkillTarget = (target) => { + const value = relative(skillDirectory, target); + return ( + value === "" || + (value !== ".." && !value.startsWith(`..${sep}`) && !isAbsolute(value)) + ); +}; +assert.equal( + isManagedSkillTarget(`${skillDirectory}-sibling/AGENTS.md`), + false, +); +assert.equal( + isManagedSkillTarget(join(skillDirectory, "..", "outside", "AGENTS.md")), + false, +); await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); const environment = { @@ -264,8 +279,7 @@ try { assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); assert( parsed.plannedActions.every( - (a) => - a.target === configPath || a.target.startsWith(skillDirectory), + (a) => a.target === configPath || isManagedSkillTarget(a.target), ), ); } diff --git a/scripts/verify-package-setup.d.mts b/scripts/verify-package-setup.d.mts new file mode 100644 index 000000000..2c7244291 --- /dev/null +++ b/scripts/verify-package-setup.d.mts @@ -0,0 +1,6 @@ +/** Validate a live registration and its byte-identical pre-setup backup. */ +export function assertClientConfig( + cli: string, + configPath: string, + original: Uint8Array, +): Promise; diff --git a/scripts/verify-package-setup.mjs b/scripts/verify-package-setup.mjs index 9a87a1695..44f28c142 100644 --- a/scripts/verify-package-setup.mjs +++ b/scripts/verify-package-setup.mjs @@ -414,7 +414,8 @@ const verifySetupApply = async ({ }; }; -const assertClientConfig = async (cli, configPath) => { +/** Verify the live registration and exact pre-setup backup bytes. */ +export const assertClientConfig = async (cli, configPath, original) => { const config = json(await readFile(configPath, "utf8")); if ( config.existing !== true || @@ -422,9 +423,7 @@ const assertClientConfig = async (cli, configPath) => { JSON.stringify(config.mcpServers?.rea?.args) !== JSON.stringify(["mcp"]) ) throw new Error("packaged client readback failed"); - if ( - !(await readFile(`${configPath}.rea.backup`, "utf8")).includes("existing") - ) + if (!(await readFile(`${configPath}.rea.backup`)).equals(original)) throw new Error("packaged client backup failed"); }; @@ -467,9 +466,7 @@ const assertCodexSymlink = async ({ !codex.includes(`command = "${cli}"`) || !codex.includes("startup_timeout_sec = 30") || !codex.startsWith(CODEX_ORIGINAL) || - (await readFile(`${codexConfig}.rea.backup`, "utf8")) !== CODEX_ORIGINAL || - (await readFile(`${cursorConfig}.rea.backup`, "utf8")) !== - '{"existing":true}\n' + (await readFile(`${codexConfig}.rea.backup`, "utf8")) !== CODEX_ORIGINAL ) throw new Error("packaged setup did not preserve config symlinks"); }; @@ -534,9 +531,11 @@ const verifyConfigReadback = async ({ siblingSkillPath, opencodeConfig, opencodeOriginal, + claudeOriginal, + cursorOriginal, }) => { - await assertClientConfig(cli, claudeConfig); - await assertClientConfig(cli, cursorConfig); + await assertClientConfig(cli, claudeConfig, claudeOriginal); + await assertClientConfig(cli, cursorConfig, cursorOriginal); await assertCodexSymlink({ cli, codexConfig, @@ -729,6 +728,9 @@ export async function verifyPackageSetup({ opencodeConfig, skillPath, ]; + const [claudeOriginal, cursorOriginal] = supportedSetupHost + ? await Promise.all([readFile(claudeConfig), readFile(cursorConfig)]) + : []; await verifyInteractiveSetup({ command: packageRunnerCli, environment, @@ -772,6 +774,8 @@ export async function verifyPackageSetup({ siblingSkillPath, opencodeConfig, opencodeOriginal: OPENCODE_ORIGINAL, + claudeOriginal, + cursorOriginal, }); await verifySetupFailureRecovery({ cli, diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index 2c5f15e62..4d85e6782 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -6,6 +6,7 @@ import { requireMcpOperationResult, } from "./lib/mcp-verifier-results.mjs"; import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs"; +import { verifyWindowsGhidraByteSearch } from "./lib/ghidra-windows-byte-search.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -358,11 +359,12 @@ try { const mapping = await call("address_to_file_offset", { address: procedure.address, }); + const sourceBytes = await readFile(target); assert.equal(memory.complete, true); assert.equal(memory.returned_bytes, 16); assert.equal( memory.bytes_hex, - (await readFile(target)) + sourceBytes .subarray(mapping.file_offset, mapping.file_offset + 16) .toString("hex"), ); @@ -374,6 +376,35 @@ try { await call("resolve_containing_procedure", { address: procedure.address }); await call("search_procedures", { pattern: procedure.value }); await call("search_strings", { pattern: "rea" }); + report.byteSearch = await verifyWindowsGhidraByteSearch({ + invoke: call, + sourceBytes, + architecture: parsedTarget.value.architecture, + sourceOwnedFixture: targetArgument === undefined, + fallbackAddress: procedure.address, + fallbackFileOffset: mapping.file_offset, + runCliPage: async (query) => { + const cliArguments = [entry, "search-bytes", target, query.pattern]; + for (const [option, value] of [ + ["--mask", query.mask], + ["--start", query.start], + ["--end", query.end], + ["--limit", String(query.limit)], + ]) { + if (value !== undefined) cliArguments.push(option, value); + } + cliArguments.push("--provider", "ghidra", "--json"); + const result = await exec(process.execPath, cliArguments, { + env: environment, + cwd: callerDirectory, + timeout: 360_000, + maxBuffer: 8 * 1024 * 1024, + }); + const evidence = JSON.parse(result.stdout); + assert.ok(evidence.error === undefined, "CLI byte search failed"); + return evidence.normalized_result; + }, + }); for (const name of [ "procedure_assembly", "procedure_callees", @@ -429,9 +460,24 @@ try { name !== "shutdown" && !GHIDRA_MUTATING_OPERATIONS.has(name), ); - assert.ok( - expected.every((name) => report.mcpOperations.includes(name)), - "Packaged verification omitted an admitted operation.", + const expectedOperations = new Set([ + ...expected, + "open_binary", + "binary_overview", + "binary_session", + "close_binary", + ]); + const observedOperations = new Set(report.mcpOperations); + const missingOperations = expected.filter( + (name) => !observedOperations.has(name), + ); + const unexpectedOperations = [...observedOperations].filter( + (name) => !expectedOperations.has(name), + ); + assert.deepEqual( + { missing: missingOperations, unexpected: unexpectedOperations }, + { missing: [], unexpected: [] }, + "Packaged verification operation coverage drifted.", ); const session = await call("binary_session", {}); assert.equal( diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 0aa58e2f0..5cdf04427 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -1,7 +1,14 @@ #!/usr/bin/env node import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { copyFile, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + readdir, + rm, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,6 +16,11 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js"; +import { cleanupOwnedProcessGroup } from "../dist/process/ProcessOwnership.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../dist/process/ProviderProcess.js"; import { toolContract } from "../dist/contracts/toolContracts.js"; import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -48,10 +60,21 @@ const source = fileURLToPath( const literalSource = fileURLToPath( new URL("../tests/conformance/swift/string-literals.swift", import.meta.url), ); +const unresolvedJumpSource = fileURLToPath( + new URL( + "../tests/conformance/swift/unresolved-string-jump.s", + import.meta.url, + ), +); const binary = join(root, "swift-inventory"); const literalBinary = join(root, "swift-literals"); +const literalAssembly = fileURLToPath( + new URL("../tests/conformance/swift/decoder-safeguards.S", import.meta.url), +); +const literalObject = join(root, "decoder-safeguards.o"); const reports = []; let literalReport; +let decoderCleaned = true; try { await execute("swiftc", ["-O", "-g", source, "-o", binary], { timeout: 60_000, @@ -59,6 +82,19 @@ try { await mkdir(join(root, "copied")); const copied = join(root, "copied", "swift-inventory"); await copyFile(binary, copied); + await execute( + "xcrun", + [ + "clang", + "-target", + "arm64-apple-macosx13.0", + "-c", + literalAssembly, + "-o", + literalObject, + ], + { timeout: 60_000 }, + ); // Decoding covers AArch64 register pairs, so build that slice on any Mac host. await execute( "swiftc", @@ -67,11 +103,14 @@ try { "-target", "arm64-apple-macosx13.0", literalSource, + literalObject, + unresolvedJumpSource, "-o", literalBinary, ], { timeout: 60_000 }, ); + await verifyDecoderSafeguards(); await client.connect(transport); for (const [variant, path] of [ ["debug", binary], @@ -154,7 +193,7 @@ try { } finally { await client.close(); await transport.close(); - await rm(root, { recursive: true, force: true }); + if (decoderCleaned) await rm(root, { recursive: true, force: true }); } console.log( JSON.stringify( @@ -222,8 +261,144 @@ async function verifyStringLiterals() { } decoded[name] = observed.length; } + const unresolvedJump = procedures.find(({ value }) => + value.includes("rea_unresolved_string_jump"), + ); + assert( + unresolvedJump, + "unresolved jump fixture is missing from the procedure inventory", + ); + const unresolvedDossier = await call("analyze_function", { + procedure: unresolvedJump.address, + }); + assert.deepEqual(unresolvedDossier.swift_string_literals, []); + assert.deepEqual( + unresolvedDossier.swift_string_literal_omissions.map( + ({ reason }) => reason, + ), + ["unresolved_jump"], + ); + decoded.unresolvedStringJump = + unresolvedDossier.swift_string_literal_omissions.length; return decoded; } finally { await call("close_binary", {}); } } + +// Exercise the Java producer inside an ephemeral Ghidra database. The explicit +// budget seam makes exhaustion deterministic without exposing a runtime option. +async function verifyDecoderSafeguards() { + const scripts = join(root, "decoder-scripts"); + await mkdir(scripts); + const bridgeSources = fileURLToPath( + new URL("../bridge/ghidra/", import.meta.url), + ); + for (const name of await readdir(bridgeSources)) { + if (name.endsWith(".java")) + await copyFile(join(bridgeSources, name), join(scripts, name)); + } + await copyFile( + fileURLToPath( + new URL( + "../tests/conformance/swift/ReaSwiftDecoderRegressions.java", + import.meta.url, + ), + ), + join(scripts, "ReaSwiftDecoderRegressions.java"), + ); + const report = join(root, "decoder-safeguards.txt"); + await runDecoderProbe(scripts, report); + assert.equal( + await readFile(report, "utf8"), + "REA_SWIFT_DECODER_SAFEGUARDS_OK\n", + ); +} + +async function runDecoderProbe(scripts, report) { + const controller = new AbortController(); + let interrupt; + const interrupted = new Promise((resolve) => { + interrupt = () => { + controller.abort(new Error("Swift decoder probe interrupted")); + resolve(false); + }; + }); + process.on("SIGINT", interrupt); + process.on("SIGTERM", interrupt); + let supervisor; + let launch; + try { + launch = await spawnOwnedProviderProcess({ + command: installation.analyzeHeadlessPath, + arguments: [ + root, + "decoder-regressions", + "-import", + literalBinary, + "-scriptPath", + scripts, + "-postScript", + "ReaSwiftDecoderRegressions.java", + report, + "-deleteProject", + "-max-cpu", + "2", + ], + runId: run.run_id, + expectedCommand: null, + env: { JAVA_HOME: installation.javaHome }, + signal: controller.signal, + }); + decoderCleaned = false; + supervisor = new ProviderProcessSupervisor( + { + ...launch, + ownsProcessLifetime: true, + cleanup: () => cleanupOwnedProcessGroup(launch.ownership), + }, + { maxDiagnosticBytes: 8 * 1024 * 1024 }, + ); + assert.ok( + await Promise.race([supervisor.waitForExit(120_000), interrupted]), + controller.signal.aborted + ? "Swift decoder probe interrupted" + : "Swift decoder probe timed out", + ); + assert.ok( + await supervisor.waitForOutputClose(1_000), + "Swift decoder probe output did not close", + ); + const snapshot = supervisor.snapshot(); + assert.equal( + snapshot.exitCode, + 0, + `${snapshot.stdout.text}\n${snapshot.stderr.text}`, + ); + assert.ok( + !snapshot.diagnosticTruncated, + "Swift decoder probe exceeded its diagnostic capacity", + ); + } finally { + try { + if (supervisor !== undefined) { + const stopped = await supervisor.stop(); + decoderCleaned = stopped.status === "verified-cleanup"; + assert.ok( + decoderCleaned, + `Owned Ghidra cleanup incomplete; retained ${root}: ${JSON.stringify(stopped)}`, + ); + } else if (launch !== undefined) { + const stopped = await cleanupOwnedProcessGroup(launch.ownership); + decoderCleaned = stopped.cleaned; + assert.ok( + decoderCleaned, + `Owned Ghidra cleanup incomplete; retained ${root}: ${JSON.stringify(stopped)}`, + ); + } + } finally { + process.off("SIGINT", interrupt); + process.off("SIGTERM", interrupt); + } + } +} diff --git a/scripts/verify-real-ghidra-windows.mjs b/scripts/verify-real-ghidra-windows.mjs index bc822211f..2e6e57d22 100644 --- a/scripts/verify-real-ghidra-windows.mjs +++ b/scripts/verify-real-ghidra-windows.mjs @@ -29,6 +29,7 @@ import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs"; import { verifyGhidraFileOffsets } from "./lib/real-ghidra-file-offsets.mjs"; +import { verifyWindowsGhidraByteSearch } from "./lib/ghidra-windows-byte-search.mjs"; const verifierRun = createVerifierRun(); const SESSION_CAPABILITIES = GHIDRA_SESSION_CAPABILITIES.filter( @@ -215,6 +216,11 @@ try { }); await inventory("search_procedures", { pattern: procedure.value }); await inventory("search_strings", { pattern: "rea" }); + const byteSearch = await verifyWindowsGhidraByteSearch({ + invoke: inventory, + sourceBytes, + architecture, + }); for (const operation of [ "procedure_assembly", @@ -242,12 +248,18 @@ try { ...GHIDRA_INVENTORY_OPERATIONS, ...GHIDRA_FUNCTION_OPERATIONS, ].filter((name) => !GHIDRA_MUTATING_OPERATIONS.has(name)); + const expectedOperations = new Set(expected); + const missingOperations = expected.filter((name) => !observed.has(name)); + const unexpectedOperations = [...observed].filter( + (name) => !expectedOperations.has(name), + ); if ( - observed.size !== expected.length || - expected.some((operation) => !observed.has(operation)) + missingOperations.length > 0 || + unexpectedOperations.length > 0 || + observed.size !== expectedOperations.size ) throw new Error( - `Windows Ghidra operation proof was incomplete: ${JSON.stringify([...observed])}`, + `Windows Ghidra operation proof was incomplete: ${JSON.stringify({ missing: missingOperations, unexpected: unexpectedOperations })}`, ); // The reference controls occupy the EXE fixture's additional function. @@ -271,6 +283,7 @@ try { }, transport: "authenticated-loopback-tcp", operations: [...observed].sort((left, right) => left.localeCompare(right)), + byte_search: byteSearch, cleanup: "complete", compact_references: compactReferences, native_controls: { diff --git a/scripts/verify/clients/model-fixture.d.mts b/scripts/verify/clients/model-fixture.d.mts new file mode 100644 index 000000000..b162fe315 --- /dev/null +++ b/scripts/verify/clients/model-fixture.d.mts @@ -0,0 +1,21 @@ +export interface OpenAiModelFixture { + readonly baseUrl: string; + readonly requests: readonly { readonly path: string }[]; + readonly probes: readonly { + readonly path: string | undefined; + readonly status: 404; + }[]; + readonly failure: string | undefined; + close(): Promise; +} + +export function createOpenAiModelFixture(options: { + readonly directory: string; + readonly prefix: string; + readonly onRequest: () => + | { + readonly role: "assistant"; + readonly content: string; + } + | Promise<{ readonly role: "assistant"; readonly content: string }>; +}): Promise; diff --git a/scripts/verify/clients/model-fixture.mjs b/scripts/verify/clients/model-fixture.mjs index cdc99a31c..b218a0985 100644 --- a/scripts/verify/clients/model-fixture.mjs +++ b/scripts/verify/clients/model-fixture.mjs @@ -15,7 +15,8 @@ export const createOpenAiModelFixture = async ({ let failure; const server = createServer(async (request, response) => { try { - if (request.method === "GET" && request.url?.endsWith("/models")) { + const path = new URL(request.url ?? "/", "http://localhost").pathname; + if (request.method === "GET" && path.endsWith("/models")) { response.setHeader("Content-Type", "application/json"); response.end( JSON.stringify({ @@ -47,7 +48,7 @@ export const createOpenAiModelFixture = async ({ join(directory, `${prefix}-incoming-${++incoming}.json`), JSON.stringify({ method: request.method, path: request.url, body }), ); - if (request.url !== "/v1/chat/completions") { + if (path !== "/v1/chat/completions") { // Hermes probes custom endpoints for Ollama/llama.cpp metadata. // Unsupported HTTP endpoints return 404, as on an OpenAI-only server. probes.push({ diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index 804e8fffa..b90dba14b 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -201,6 +201,16 @@ async function stageCli() { ), ); + assert.ok( + files.extraction_gaps.some( + (gap) => + gap.kind === "unknown-chunk-handler" && + gap.range !== null && + gap.range.offset === 0 && + gap.range.length === 64, + ), + "The unknown chunk must carry its explicit recovery row", + ); return { regions, files }; } @@ -247,6 +257,20 @@ async function stageMcp(cli) { ); assert.equal(depthParsed.coverage, "partial"); assert.ok(depthParsed.depth_limited_paths.length > 0); + assert.ok( + depthParsed.extraction_gaps.some( + (gap) => + gap.kind === "depth-limited" && + gap.dependencies.length === 0 && + depthParsed.extraction_continuations.some( + ({ id, instruction }) => + id === gap.continuation_id && + instruction.includes("max_depth") && + instruction.includes("fresh output_directory"), + ), + ), + "Depth-limited paths must carry the explicit depth continuation", + ); console.log( "PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits", @@ -292,8 +316,17 @@ async function stageExt4() { ); assert.equal(degradedParsed.coverage, "partial"); assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs")); + assert.ok( + degradedParsed.extraction_gaps.some( + (gap) => + gap.kind === "missing-extractor-dependency" && + gap.dependencies.includes("debugfs") && + gap.handler === "extfs", + ), + "The missing extractor must be named in its explicit recovery row", + ); console.log( - "PASS real missing-extractor diagnostics retained despite Unblob exit 1", + "PASS real missing-extractor diagnostics and recovery row retained despite Unblob exit 1", ); } diff --git a/src/evaluation/AgentEvaluationFixtures.ts b/src/evaluation/AgentEvaluationFixtures.ts index b0535a96e..83b22db41 100644 --- a/src/evaluation/AgentEvaluationFixtures.ts +++ b/src/evaluation/AgentEvaluationFixtures.ts @@ -8,6 +8,11 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js"; import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js"; import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { + applicationFeatureTraceResultSchema, + featureSeedMatchMode, +} from "../domain/javascript/javascriptFeatureTraceSchemas.js"; +import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js"; import { managedArtifactInspectionSchema, managedMemberInspectionSchema, @@ -28,6 +33,8 @@ export interface AgentEvaluationFixtureTargets { readonly javascriptSha256: string; readonly javascriptShapeLeft: string; readonly javascriptShapeRight: string; + readonly notesElectron: string; + readonly notesElectronSha256: string; readonly managed?: string; readonly managedSha256?: string; } @@ -50,6 +57,8 @@ export const createAgentEvaluationFixtures = async ( const javascript = join(root, "desktop-app.asar"); const javascriptShapeLeft = join(root, "parser-v1"); const javascriptShapeRight = join(root, "parser-v2"); + const notesDirectory = join(root, "notes-electron"); + const notesElectron = join(root, "notes-electron.asar"); await Promise.all([ mkdir(join(desktop, "renderer"), { recursive: true }), mkdir(javascriptShapeLeft, { recursive: true }), @@ -67,8 +76,14 @@ export const createAgentEvaluationFixtures = async ( join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"), join(javascriptShapeRight, "parser.mjs"), ), + cp( + join(repositoryRoot, "website/public/examples/notes-electron"), + notesDirectory, + { recursive: true }, + ), ]); await createPackage(desktop, javascript); + await createPackage(notesDirectory, notesElectron); return { javascript, javascriptSha256: createHash("sha256") @@ -76,6 +91,10 @@ export const createAgentEvaluationFixtures = async ( .digest("hex"), javascriptShapeLeft, javascriptShapeRight, + notesElectron, + notesElectronSha256: createHash("sha256") + .update(await readFile(notesElectron)) + .digest("hex"), }; }; @@ -189,21 +208,116 @@ const runtimeSemantics = (value: JsonValue): JsonValue | undefined => { : undefined; }; -const asarClaims = ( +const featureModules = (value: JsonValue): JsonValue | undefined => { + const result = applicationFeatureTraceResultSchema.safeParse(value); + if (!result.success || result.data.graph === null) return undefined; + if ( + result.data.seed.kind !== "channel" || + result.data.seed.value !== "notes:export" || + featureSeedMatchMode(result.data.seed) !== "exact" + ) + return undefined; + const paths = result.data.graph.nodes + .filter((node) => node.kind === "javascript-module") + .flatMap((node) => node.observations) + .filter(({ properties }) => properties.semantic_role === "source-module") + .map(({ properties }) => properties.path); + if (paths.length === 0 || !paths.every((path) => typeof path === "string")) + return undefined; + return { + paths: [...new Set(paths)].sort(compareUnicodeCodePoints), + coverage: result.data.coverage.status, + }; +}; + +const semanticBoundary = (value: JsonValue): JsonValue | undefined => { + const result = javaScriptSemanticTraceResultSchema.safeParse(value); + if (!result.success) return undefined; + return { + status: result.data.status, + coverage: result.data.coverage.status, + relevant_unknowns: result.data.summary.relevant_unknowns, + }; +}; + +const notesClaims = ( targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => { + const subject = { + path: targets.notesElectron, + sha256: targets.notesElectronSha256, + }; + const derived = { + authority: "analyst-inference", + confidence: "inferred", + linkedSubjects: [ + { operation: "analyze_javascript_application", ...subject }, + ], + } satisfies Pick< + FixtureClaimExpectation["source"], + "authority" | "confidence" | "linkedSubjects" + >; + // Fixture source establishes bridge/module facts; the semantic claim checks + // the bounded static-query contract, not execution or CSV formatter behavior. + return [ + ...bridgeClaims(subject, "notes", ["exportCsv"], "notes:export"), + { + id: "feature_modules", + expectedValue: { + paths: ["csv.js", "main.js", "preload.js", "renderer.js"], + coverage: "complete-within-source", + }, + source: { + ...derived, + operation: "trace_application_feature", + parameters: { + direction: "both", + }, + select: featureModules, + }, + }, + { + id: "semantic_boundary", + expectedValue: { + status: "ambiguous", + coverage: "partial", + relevant_unknowns: 1, + }, + source: { + ...derived, + operation: "trace_javascript_semantics", + parameters: { + query: { + seed: { kind: "property", name: "invoke" }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: false, + expected: null, + }, + }, + select: semanticBoundary, + }, + }, + ]; +}; + +const bridgeClaims = ( + subject: { readonly path: string; readonly sha256: string }, + key: string, + members: readonly string[], + channel: string, ): readonly FixtureClaimExpectation[] => { const source = { operation: "analyze_javascript_application", authority: "shipped-artifact", confidence: "derived", - subject: { path: targets.javascript, sha256: targets.javascriptSha256 }, + subject, } satisfies Omit; return [ { id: "renderer_api", expectedValue: { - key: "profileApi", - members: ["read"], + key, + members: [...members], world: "main", source: "preload.js", }, @@ -212,7 +326,7 @@ const asarClaims = ( { id: "renderer_ipc", expectedValue: { - channel: "profile:read", + channel, operation: "invoke", source: "preload.js", }, @@ -221,7 +335,7 @@ const asarClaims = ( { id: "main_ipc", expectedValue: { - channel: "profile:read", + channel, operation: "handle", source: "main.js", }, @@ -235,6 +349,16 @@ const asarClaims = ( ]; }; +const asarClaims = ( + targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => + bridgeClaims( + { path: targets.javascript, sha256: targets.javascriptSha256 }, + "profileApi", + ["read"], + "profile:read", + ); + const parserClaims = ( targets: AgentEvaluationFixtureTargets, ): readonly FixtureClaimExpectation[] => { @@ -399,25 +523,33 @@ export const agentFixtureClaims = ( if (scenarioId === "asar") return asarClaims(targets); if (scenarioId === "managed") return managedClaims(targets); if (scenarioId === "javascript-export-shape") return parserClaims(targets); + if (scenarioId === "notes-export") return notesClaims(targets); return undefined; }; +const bridgeAnswerDescriptions = [ + 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', + 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', + 'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.', + 'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.', +]; + const answerValueDescriptions: Readonly> = { managed: [ "public_types: string[], listing the exact public top-level type full names in Unicode code-point order from unfiltered member Evidence (omit type_names).", 'entry_point: {"declaring_type": string, "name": string}, resolving the CLI metadata entry-point token against the member inventory from the same artifact SHA-256 and module MVID. Cite member Evidence; both artifact and member Evidence must be delivered.', ], - asar: [ - 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', - 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', - 'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.', - 'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.', - ], + asar: bridgeAnswerDescriptions, "javascript-export-shape": [ 'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.', 'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.', 'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.', ], + "notes-export": [ + ...bridgeAnswerDescriptions, + 'feature_modules: {"paths": string[], "coverage": string}, listing the unique source-module observation paths in the feature trace in Unicode code-point order, and its coverage status.', + 'semantic_boundary: {"status": string, "coverage": string, "relevant_unknowns": number}, copying the semantic trace status, coverage status, and summary.relevant_unknowns. Static traces do not establish runtime execution.', + ], }; /** Describe the strict answer contract without revealing any golden fixture values. */ diff --git a/tests/acceptance/applications/applicationWorkflowCli.test.ts b/tests/acceptance/applications/applicationWorkflowCli.test.ts index 6548d210b..ed1ca8d16 100644 --- a/tests/acceptance/applications/applicationWorkflowCli.test.ts +++ b/tests/acceptance/applications/applicationWorkflowCli.test.ts @@ -873,8 +873,9 @@ describe("rest parameter semantic trace CLI", () => { kind === "expression" && label === `argument ${String(index)}`, ); if (argument === undefined) throw new Error("Missing argument node"); - const traced = await runCli([ - "trace-javascript-semantics", + const requestPath = join(root, `trace-request-${index}.json`); + await writeFile( + requestPath, JSON.stringify({ application, query: { @@ -884,6 +885,10 @@ describe("rest parameter semantic trace CLI", () => { expected: { role: "sink", classes: ["parameter"] }, }, }), + ); + const traced = await runCli([ + "trace-javascript-semantics", + requestPath, "--json", ]); expect(traced).toMatchObject({ diff --git a/tests/acceptance/webRuntimeCli.test.ts b/tests/acceptance/webRuntimeCli.test.ts index 084687dc2..2b37fecdd 100644 --- a/tests/acceptance/webRuntimeCli.test.ts +++ b/tests/acceptance/webRuntimeCli.test.ts @@ -111,17 +111,124 @@ const cancellationSignals = process.platform === "win32" ? (["SIGINT"] as const) : (["SIGINT", "SIGTERM"] as const); -const webRuntimeCancellationCases = ( - ["observe-web-execution", "inspect-web-event-listeners"] as const -).flatMap((command) => - cancellationSignals.map( - (signal) => [command, signal, signal === "SIGINT" ? 130 : 143] as const, - ), + +interface CliCancellationCommand { + readonly command: string; + readonly operation: string; + readonly buildArgs: (endpoint: string) => readonly string[]; +} + +const cancellationCommands: readonly CliCancellationCommand[] = [ + { + command: "observe-web-execution", + operation: "observe_web_execution", + buildArgs: (endpoint) => [ + "observe-web-execution", + endpoint, + "page-fixture", + "--observation-ms", + "30000", + "--json", + ], + }, + { + command: "inspect-web-event-listeners", + operation: "inspect_web_event_listeners", + buildArgs: (endpoint) => [ + "inspect-web-event-listeners", + endpoint, + "page-fixture", + "body", + "--json", + ], + }, + { + command: "list-browser-targets", + operation: "list_browser_targets", + buildArgs: (endpoint) => ["list-browser-targets", endpoint, "--json"], + }, + { + command: "inspect-web-page", + operation: "inspect_web_page", + buildArgs: (endpoint) => [ + "inspect-web-page", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "analyze-web-bundle", + operation: "analyze_web_bundle", + buildArgs: (endpoint) => [ + "analyze-web-bundle", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "observe-web-session", + operation: "observe_web_session", + buildArgs: (endpoint) => [ + "observe-web-session", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "discover-webmcp-tools", + operation: "discover_webmcp_tools", + buildArgs: (endpoint) => [ + "discover-webmcp-tools", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "capture-web-screenshot", + operation: "capture_web_screenshot", + buildArgs: (endpoint) => [ + "capture-web-screenshot", + endpoint, + "page-fixture", + "--json", + ], + }, + { + command: "list-javascript-runtime-targets", + operation: "list_javascript_runtime_targets", + buildArgs: (endpoint) => [ + "list-javascript-runtime-targets", + endpoint, + "--json", + ], + }, + { + command: "observe-javascript-runtime", + operation: "observe_javascript_runtime", + buildArgs: (endpoint) => [ + "observe-javascript-runtime", + endpoint, + "target-fixture", + "--json", + ], + }, +]; + +const cancellationCases = cancellationCommands.flatMap((command) => + cancellationSignals.map((signal) => ({ + ...command, + signal, + expectedExitCode: signal === "SIGINT" ? 130 : 143, + })), ); -it.each(webRuntimeCancellationCases)( - "%s preserves %s status when cancellation interrupts CDP discovery", - async (command, signal, expectedExitCode) => { +it.each(cancellationCases)( + "$command preserves $signal status and awaited cleanup during CDP discovery", + async ({ command, operation, buildArgs, signal, expectedExitCode }) => { const sockets = new Set(); const server = createServer(() => undefined); server.on("connection", (socket) => { @@ -136,21 +243,14 @@ it.each(webRuntimeCancellationCases)( const endpoint = `http://127.0.0.1:${String(address.port)}`; const contacted = once(server, "request"); - const args = - command === "observe-web-execution" - ? [ - command, - endpoint, - "page-fixture", - "--observation-ms", - "30000", - "--json", - ] - : [command, endpoint, "page-fixture", "body", "--json"]; - const child = spawn(process.execPath, [entrypoint, ...args], { - env: environment(), - stdio: ["ignore", "pipe", "pipe"], - }); + const child = spawn( + process.execPath, + [entrypoint, ...buildArgs(endpoint)], + { + env: environment(), + stdio: ["ignore", "pipe", "pipe"], + }, + ); const closed = once(child, "close"); let stdout = ""; child.stdout.setEncoding("utf8"); @@ -180,7 +280,7 @@ it.each(webRuntimeCancellationCases)( expect(JSON.parse(stdout)).toMatchObject({ code: "cancelled", category: "cancelled", - details: { cleanup: "complete" }, + details: { cleanup: "complete", operation }, }); } finally { clearTimeout(timeout); diff --git a/tests/boundary/captures/captureProvider.test.ts b/tests/boundary/captures/captureProvider.test.ts index 881b43e63..30281bed3 100644 --- a/tests/boundary/captures/captureProvider.test.ts +++ b/tests/boundary/captures/captureProvider.test.ts @@ -53,6 +53,8 @@ else { if(readFileSync('capture', 'utf8') !== 'recording bytes') throw Error('wr }, async (input) => { calls++; + if (mode === "launch-failure" && !input.arguments.includes("--version")) + throw Object.assign(new Error("spawn E2BIG"), { code: "E2BIG" }); if (mode === "file-limit" && !input.arguments.includes("--version")) { const stdout = { text: "", bytes: 0, observedBytes: 0 }; const stderr = { @@ -255,6 +257,21 @@ it.each([ }, ); +it.each(["capture", "fields"] as const)( + "does not attribute the version probe's output to a %s command that fails to launch", + async (operation) => { + const f = await fixture("launch-failure"); + const result = + operation === "capture" + ? await f.service.inspect({ path: f.path, fields: ["ip.addr"] }) + : await f.service.inspectFields({ kind: "fields" }); + if (result.ok) throw new Error("Expected launch failure"); + expect(f.calls()).toBe(2); + expect(result.error.message).toContain("spawn E2BIG"); + expect(result.error.capturedOutput).toBeUndefined(); + }, +); + it("retains a completed observation and cleanup ownership for retry", async () => { const f = await fixture("ok", true); const result = await f.service.inspect({ path: f.path, fields: ["ip.addr"] }); diff --git a/tests/boundary/cli/developmentTests.test.ts b/tests/boundary/cli/developmentTests.test.ts index 7becdd4e7..d9b9fa7f1 100644 --- a/tests/boundary/cli/developmentTests.test.ts +++ b/tests/boundary/cli/developmentTests.test.ts @@ -1,5 +1,6 @@ import { copyFile, + lstat, mkdir, readFile, symlink, @@ -44,6 +45,7 @@ const fixture = async () => { await mkdir(join(root, "src")); for (const path of [ "scripts/run-development-tests.mjs", + "scripts/run-exact-tests.mjs", "scripts/lib/development-tests.mjs", "scripts/lib/test-projects.mjs", ]) @@ -76,6 +78,135 @@ const testCommand = (root: string, ...arguments_: string[]) => ...arguments_, ]); +describe("development test runner exact selection", () => { + it("matches selected path casing to the host filesystem", async () => { + const root = await fixture(); + try { + await lstat(join(root, "src/VALUE.test.ts")); + } catch (cause) { + if ( + !(cause instanceof Error) || + !("code" in cause) || + cause.code !== "ENOENT" + ) + throw cause; + await expect( + testCommand(root, "focused", "src/VALUE.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("ENOENT"), + }); + return; + } + await testCommand(root, "focused", "src/VALUE.test.ts"); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }); + + it("matches Unicode-normalized aliases by filesystem identity", async () => { + const root = await fixture(); + const composed = "caf\u00e9.test.ts"; + const decomposed = composed.normalize("NFD"); + await writeFile( + join(root, "src", composed), + 'import { it } from "vitest"; it("checks the selected Unicode path", () => {});\n', + ); + const selected = join(root, "src", decomposed); + try { + await lstat(selected); + } catch (cause) { + if ( + !(cause instanceof Error) || + !("code" in cause) || + cause.code !== "ENOENT" + ) + throw cause; + await expect( + testCommand(root, "focused", `src/${decomposed}`), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("ENOENT"), + }); + return; + } + const result = await testCommand(root, "focused", `src/${decomposed}`); + expect(result.stdout).toContain("1 passed"); + }); + + it.each(["local", "focused"])( + "runs only the exact selected file in %s mode", + async (mode) => { + const root = await fixture(); + await mkdir(join(root, "src", "nested", "src"), { recursive: true }); + await writeFile( + join(root, "src", "nested", "src", "value.test.ts"), + 'import { it } from "vitest"; it("must not run", () => { throw new Error("unselected nested test ran"); });\n', + ); + const result = await testCommand(root, mode, "src/value.test.ts"); + expect(result.stdout).toContain("1 passed"); + expect(result.stdout).not.toContain("nested/src/value.test.ts"); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }, + ); + + it("fails when any selected existing test file is excluded from discovery", async () => { + const root = await fixture(); + await writeFile( + join(root, "src", "excluded.test.ts"), + 'import { it } from "vitest"; it("excluded", () => {});\n', + ); + await writeFile( + join(root, "vitest.config.mjs"), + 'export default { test: { projects: [{ test: { name: "domain", include: ["src/value.test.ts"] } }] } };\n', + ); + await expect( + testCommand(root, "focused", "src/value.test.ts", "src/excluded.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining( + "Selected test files were not discovered: src/excluded.test.ts", + ), + }); + }); + + it("runs every selected file and preserves selected test failures", async () => { + const root = await fixture(); + await writeFile( + join(root, "src", "failure.test.ts"), + 'import { it } from "vitest"; it("selected failure", () => { throw new Error("selected regression"); });\n', + ); + await expect( + testCommand(root, "focused", "src/value.test.ts", "src/failure.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stdout: expect.stringContaining("1 failed | 1 passed"), + stderr: expect.stringContaining("selected regression"), + }); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }); + + it("preserves all configured projects for an exact file", async () => { + const root = await fixture(); + await writeFile( + join(root, "vitest.config.mjs"), + 'export default { test: { projects: ["domain", "evaluation"].map(name => ({ test: { name, include: ["src/value.test.ts"] } })) } };\n', + ); + const result = await testCommand(root, "focused", "src/value.test.ts"); + expect(result.stdout).toMatch(/Test Files\s+2 passed/u); + }); + + it("fails an exact file with no tests even when another selected file passes", async () => { + const root = await fixture(); + await writeFile(join(root, "src", "empty.test.ts"), "export {};\n"); + await expect( + testCommand(root, "focused", "src/value.test.ts", "src/empty.test.ts"), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("No test suite found"), + }); + expect(await readFile(join(root, "selected.txt"), "utf8")).toBe("1"); + }); +}); + describe("development test runner public CLI", () => { it("runs an explicitly selected source test on a clean tree without build artifacts", async () => { const root = await fixture(); diff --git a/tests/boundary/cli/keyedArchiveXmlExtensionCli.test.ts b/tests/boundary/cli/keyedArchiveXmlExtensionCli.test.ts new file mode 100644 index 000000000..6ed029228 --- /dev/null +++ b/tests/boundary/cli/keyedArchiveXmlExtensionCli.test.ts @@ -0,0 +1,38 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { build } from "plist"; +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "compiled CLI inspects an XML keyed archive with a non-plist suffix", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-keyed-xml-extension-cli-"); + const path = join(root, "Model.archive"); + await writeFile( + path, + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "leaf"], + $top: { root: { UID: 1 } }, + }), + ); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", path, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expect(output.exitCode, output.stderr).toBe(0); + expect(parseEvidence(output.json).normalized_result).toMatchObject({ + archive_path: "Model.archive", + archive_format: "xml-plist", + objects: expect.arrayContaining([ + expect.objectContaining({ value: "leaf" }), + ]), + }); + }, +); diff --git a/tests/boundary/filesystem/android/application.test.ts b/tests/boundary/filesystem/android/application.test.ts index 01a5901c8..a29844641 100644 --- a/tests/boundary/filesystem/android/application.test.ts +++ b/tests/boundary/filesystem/android/application.test.ts @@ -205,6 +205,42 @@ describe("APK family without a root manifest", () => { }); }); +it("retains encrypted native library paths as unknown component evidence", async () => { + const root = await createTestTempDirectory("rea-android-encrypted-"); + const path = join(root, "Encrypted.apk"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add( + "lib/arm64-v8a/libentry.so", + new TextReader("encrypted native library"), + { password: "fixture-only" }, + ); + await writeFile(path, await writer.close()); + const inventory = parseEvidence( + await runProviderAnalysis(path, "inventory_artifact", {}), + ); + const projection = androidApplicationProjectionResultSchema.parse( + requireSuccessfulProjection( + projectAndroidApplicationEvidence({ inventory_evidence: [inventory] }), + ).normalized_result, + ); + + expect(projection.unavailable_components).toContainEqual( + expect.objectContaining({ + path: "lib/arm64-v8a/libentry.so", + artifact_id: null, + sha256: null, + encrypted: true, + }), + ); + expect(projection.coverage).toMatchObject({ + status: "partial", + inventory_complete: true, + }); + expect(projection.limitations).toContainEqual( + expect.stringContaining("component absence is unknown"), + ); +}); + describe("case-distinct APK resource names", () => { it("inventories and projects an APK whose resource names differ only in case", async () => { const root = await createTestTempDirectory("rea-android-case-"); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 9265e2e9b..30597523e 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -616,6 +616,10 @@ it.each([ { code: true, runtime: false, browser: false, package: false }, ], ["website/src/index.ts", { code: false, website: true, package: false }], + [ + "website/public/examples/notes-electron/preload.js", + { code: true, website: true, package: false }, + ], [ "website/locales/zh-CN/messages.po", { code: false, website: true, package: false }, diff --git a/tests/conformance/swift/ReaSwiftDecoderRegressions.java b/tests/conformance/swift/ReaSwiftDecoderRegressions.java new file mode 100644 index 000000000..751c0d3ab --- /dev/null +++ b/tests/conformance/swift/ReaSwiftDecoderRegressions.java @@ -0,0 +1,152 @@ +// Producer regressions run inside a real, ephemeral Ghidra analysis of the +// source-owned AArch64 fixture. No target code is executed and no decoded JSON +// fixture substitutes for the Java decoder under test. +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import ghidra.app.util.headless.HeadlessScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionIterator; +import ghidra.program.model.listing.Instruction; +import ghidra.program.model.listing.InstructionIterator; +import ghidra.program.model.lang.Register; +import ghidra.program.model.symbol.Symbol; +import ghidra.program.model.symbol.SymbolIterator; + +public class ReaSwiftDecoderRegressions extends HeadlessScript { + private ReaGhidraBridge decoder; + private Class budgetClass; + + @Override + public void run() throws Exception { + decoder = new ReaGhidraBridge(); + setInherited(decoder, "currentProgram", currentProgram); + setInherited(decoder, "monitor", monitor); + setInherited(decoder, "sessionDefaultAddressSpace", currentProgram.getAddressFactory().getDefaultAddressSpace()); + budgetClass = Class.forName("ReaGhidraBridge$SwiftLiteralBudget", true, decoder.getClass().getClassLoader()); + Function copied = function("rea_swift_decoder_copied"); + JsonArray copiedValues = literals(decode(copied, 65536)); + require(copiedValues.size() > 0, "copied constant pair was not decoded"); + for (var element : copiedValues) { + require(element.getAsJsonObject().get("value").getAsString().equals("ok"), "copied words invented text"); + } + for (String name : new String[] {"select", "selected_copy", "padding", "utf8"}) { + Object result = decode(function("rea_swift_decoder_" + name), 65536); + require(literals(result).isEmpty(), name + " produced an invented Swift literal"); + require(omissions(result).isEmpty(), name + " was confused with capacity exhaustion"); + } + for (String name : new String[] {"select", "selected_copy"}) { + Function selected = function("rea_swift_decoder_" + name); + require((Boolean) invoke("selectedValue", new Class[] {Function.class, Instruction.class, Register.class}, selected, returnInstruction(selected), currentProgram.getRegister("x0")), + name + " did not retain conditional-select provenance"); + } + require(invoke("swiftLiteralLayout", new Class[] {long.class, long.class}, 0x6261L, 0xe100000000000000L) == null, + "nonzero small-string padding was accepted"); + require(invoke("swiftLiteralLayout", new Class[] {long.class, long.class}, 0xc0L, 0xa100000000000000L) == null, + "malformed small-string UTF-8 was accepted"); + rejectLarge("rea_swift_decoder_bad_terminator", 0xd000000000000003L); + rejectLarge("rea_swift_decoder_bad_large_utf8", 0x1000000000000001L); + // Zero covers provenance-state exhaustion; 8192 admits both register + // cache entries and then rejects the retained literal/text charge. + assertCapacityOmission(copied, 0); + assertCapacityOmission(copied, 8192); + require(getScriptArgs().length == 1, "Expected owned result path"); + Files.writeString(Path.of(getScriptArgs()[0]), "REA_SWIFT_DECODER_SAFEGUARDS_OK\n"); + } + + private void assertCapacityOmission(Function function, long remaining) throws Exception { + Object exhausted = decode(function, remaining); + require(literals(exhausted).isEmpty(), "capacity failure published decoded small text at budget " + remaining); + JsonArray omitted = omissions(exhausted); + require(omitted.size() == 1, "capacity failure did not stop subsequent use sites at budget " + remaining); + JsonObject omission = omitted.get(0).getAsJsonObject(); + require(omission.get("reason").getAsString().equals("capacity"), "capacity omission lost its reason"); + require(!omission.has("value"), "omission claimed a retained literal value"); + } + + private Object decode(Function function, long remaining) throws Exception { + Constructor constructor = budgetClass.getDeclaredConstructor(long.class); + constructor.setAccessible(true); + return invoke("swiftStringLiterals", new Class[] {Function.class, budgetClass}, function, constructor.newInstance(remaining)); + } + + private void rejectLarge(String name, long word) throws Exception { + Address address = symbol(name).getAddress(); + long object = 0x8000000000000000L | ((address.getOffset() - 0x20) & 0x00ffffffffffffffL); + Object layout = invoke("swiftLiteralLayout", new Class[] {long.class, long.class}, word, object); + require(layout != null, "large negative fixture did not reach the memory-text decoder"); + require(invoke("swiftLargeLiteralText", new Class[] {layout.getClass()}, layout) == null, + name + " was accepted as literal text"); + } + + private Instruction returnInstruction(Function function) { + InstructionIterator instructions = currentProgram.getListing().getInstructions(function.getBody(), true); + while (instructions.hasNext()) { + Instruction instruction = instructions.next(); + if (instruction.getMnemonicString().equalsIgnoreCase("ret")) return instruction; + } + throw new AssertionError("Fixture has no return instruction"); + } + + private Function function(String name) { + FunctionIterator functions = currentProgram.getFunctionManager().getFunctions(true); + Function found = null; + while (functions.hasNext()) { + Function candidate = functions.next(); + if (!candidate.getName().equals(name) && !candidate.getName().equals("_" + name)) continue; + require(found == null, "ambiguous fixture function " + name); + found = candidate; + } + require(found != null, "missing fixture function " + name); + return found; + } + + private Symbol symbol(String name) { + SymbolIterator symbols = currentProgram.getSymbolTable().getAllSymbols(true); + Symbol found = null; + while (symbols.hasNext()) { + Symbol candidate = symbols.next(); + if (!candidate.getName().equals(name) && !candidate.getName().equals("_" + name)) continue; + require(found == null, "ambiguous fixture symbol " + name); + found = candidate; + } + require(found != null, "missing fixture symbol " + name); + return found; + } + + private Object invoke(String name, Class[] types, Object... arguments) throws Exception { + Method method = ReaGhidraBridge.class.getDeclaredMethod(name, types); + method.setAccessible(true); + return method.invoke(decoder, arguments); + } + + private JsonArray literals(Object result) throws Exception { return array(result, "literals"); } + private JsonArray omissions(Object result) throws Exception { return array(result, "omissions"); } + private JsonArray array(Object result, String name) throws Exception { + Method method = result.getClass().getDeclaredMethod(name); + method.setAccessible(true); + return (JsonArray) method.invoke(result); + } + + private void setInherited(Object target, String name, Object value) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException nextOwner) { } + } + throw new AssertionError("Missing Ghidra state field " + name); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/tests/conformance/swift/decoder-safeguards.S b/tests/conformance/swift/decoder-safeguards.S new file mode 100644 index 000000000..c426cfac7 --- /dev/null +++ b/tests/conformance/swift/decoder-safeguards.S @@ -0,0 +1,58 @@ +/* Static-analysis fixture only. Never executed. Deliberate malformed ABI words + * cannot be produced by a well-formed Swift source literal. */ +.text +.p2align 2 +.globl _rea_swift_decoder_copied +_rea_swift_decoder_copied: + mov x5, #0x6b6f + movz x6, #0xe200, lsl #48 + mov x0, x5 + mov x1, x6 + stp x0, x1, [sp, #-16]! + ldp x0, x1, [sp], #16 + ret + +.p2align 2 +.globl _rea_swift_decoder_select +_rea_swift_decoder_select: + mov x2, #0x61 + mov x3, #0x6262 + movz x6, #0xe100, lsl #48 + movz x7, #0xe200, lsl #48 + cmp w4, #0 + csel x0, x2, x3, eq + csel x1, x6, x7, eq + ret + +.p2align 2 +.globl _rea_swift_decoder_selected_copy +_rea_swift_decoder_selected_copy: + mov x2, #0x61 + mov x3, #0x62 + cmp w4, #0 + csel x5, x2, x3, eq + mov x0, x5 + movz x1, #0xe100, lsl #48 + ret + +.p2align 2 +.globl _rea_swift_decoder_padding +_rea_swift_decoder_padding: + mov x0, #0x6261 + movz x1, #0xe100, lsl #48 + ret + +.p2align 2 +.globl _rea_swift_decoder_utf8 +_rea_swift_decoder_utf8: + mov x0, #0xc0 + movz x1, #0xa100, lsl #48 + ret + +.section __DATA,__const +.globl _rea_swift_decoder_bad_terminator +_rea_swift_decoder_bad_terminator: + .byte 0x61, 0x62, 0x63, 0x78 +.globl _rea_swift_decoder_bad_large_utf8 +_rea_swift_decoder_bad_large_utf8: + .byte 0xc0, 0 diff --git a/tests/conformance/swift/unresolved-string-jump.s b/tests/conformance/swift/unresolved-string-jump.s new file mode 100644 index 000000000..2c3e71b85 --- /dev/null +++ b/tests/conformance/swift/unresolved-string-jump.s @@ -0,0 +1,8 @@ +.text +.globl _rea_unresolved_string_jump +.p2align 2 +_rea_unresolved_string_jump: + movz x0, #0x4220 + movk x0, #0x732f, lsl #16 + movz x1, #0xe400, lsl #48 + br x16 diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts index daed23cac..a9a4cc32b 100644 --- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts +++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts @@ -140,6 +140,8 @@ const managedTranscript = () => { javascriptSha256: "", javascriptShapeLeft: "", javascriptShapeRight: "", + notesElectron: "", + notesElectronSha256: "", managed: target.path, managedSha256: target.sha256, }); diff --git a/tests/evaluation/model-evals/openAiModelFixture.test.ts b/tests/evaluation/model-evals/openAiModelFixture.test.ts new file mode 100644 index 000000000..5fba1a94d --- /dev/null +++ b/tests/evaluation/model-evals/openAiModelFixture.test.ts @@ -0,0 +1,52 @@ +import { expect, it } from "vitest"; + +import { createOpenAiModelFixture } from "../../../scripts/verify/clients/model-fixture.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("routes OpenAI fixture requests by pathname, not query", async () => { + const directory = await createTestTempDirectory("rea-model-fixture-"); + let calls = 0; + const fixture = await createOpenAiModelFixture({ + directory, + prefix: "probe", + onRequest: () => { + calls += 1; + return { role: "assistant", content: "ok" }; + }, + }); + try { + const statusOf = async (path: string, method = "POST") => + ( + await fetch(`${fixture.baseUrl}${path}`, { + method, + ...(method === "POST" + ? { + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "rea-client-fixture", + messages: [{ role: "user", content: "hi" }], + }), + } + : {}), + }) + ).status; + + expect([ + await statusOf("/chat/completions"), + await statusOf("/chat/completions?api-version=1"), + await statusOf("/completions"), + await statusOf("/models?limit=1", "GET"), + ]).toEqual([200, 200, 404, 200]); + expect(calls).toBe(2); + expect(fixture.requests.map(({ path }) => path)).toEqual([ + "/v1/chat/completions", + "/v1/chat/completions?api-version=1", + ]); + expect(fixture.probes).toEqual([ + expect.objectContaining({ path: "/v1/completions", status: 404 }), + ]); + expect(fixture.failure).toBeUndefined(); + } finally { + await fixture.close(); + } +}); diff --git a/tests/evaluation/moduleBoundaries.test.ts b/tests/evaluation/moduleBoundaries.test.ts index 09f906963..786f1032a 100644 --- a/tests/evaluation/moduleBoundaries.test.ts +++ b/tests/evaluation/moduleBoundaries.test.ts @@ -275,6 +275,47 @@ describe("module import syntax and source discovery", () => { expect(result.stderr).toContain("src/domain/probe.ts:1: pure-layer"); expect(result.stderr).not.toContain("old.ts"); }); + + it("discovers forbidden nested sources without a Git worktree", async () => { + const temporary = await createTestTempDirectory("rea-module-no-git-"); + await mkdir(join(temporary, "src", "domain", "nested"), { + recursive: true, + }); + await writeFile( + join(temporary, "src/domain/nested/probe.ts"), + 'import { createServer } from "../../server/createServer.js";\n', + ); + const result = spawnSync(process.execPath, [verifier], { + cwd: temporary, + encoding: "utf8", + }); + if (result.error !== undefined) throw result.error; + expect(result.status).toBe(1); + expect(result.stderr).toContain("src/domain/nested/probe.ts"); + expect(result.stderr).toContain("src/server/createServer.js"); + }); + + it("discovers sources in an ignored archive nested under an ancestor Git worktree", async () => { + const outer = await createTestTempDirectory("rea-module-ancestor-git-"); + const archive = join(outer, "archive"); + await writeFile(join(outer, ".gitignore"), "archive/\n"); + await execute("git", ["init", "--quiet"], { cwd: outer }); + await execute("git", ["add", "--", ".gitignore"], { cwd: outer }); + await mkdir(join(archive, "src", "domain"), { recursive: true }); + await writeFile( + join(archive, "src/domain/probe.ts"), + 'import { createServer } from "../server/createServer.js";\n', + ); + + const result = spawnSync(process.execPath, [verifier], { + cwd: archive, + encoding: "utf8", + }); + if (result.error !== undefined) throw result.error; + expect(result.status).toBe(1); + expect(result.stderr).toContain("src/domain/probe.ts:1: pure-layer"); + expect(result.stderr).toContain("src/server/createServer.js"); + }); }); describe("test lane import boundaries", () => { diff --git a/tests/process-global/updateMaintenance.test.ts b/tests/process-global/updateMaintenance.test.ts index 171761478..141cf23ec 100644 --- a/tests/process-global/updateMaintenance.test.ts +++ b/tests/process-global/updateMaintenance.test.ts @@ -319,6 +319,8 @@ describe("updated executable maintenance planning", () => { it("runs real setup as a noninteractive dry run and creates no new integrations or backups", async () => { const configPath = await writeClient("codex", staleCodex); const untouched = await writeClient("claude_code", '{"mcpServers":{}}'); + const skillContent = `---\nname: ${PRODUCT_IDENTITY.skillName}\nmetadata:\n version: "1"\n---\n# REA\n`; + const skillPath = await writeSkill(skillContent); vi.stubEnv("npm_command", "exec"); const result = await planIntegrationMaintenance( home, @@ -328,18 +330,35 @@ describe("updated executable maintenance planning", () => { ); expect(result).toMatchObject({ status: "planned", - scope: { clients: ["codex"], skill: false }, - plannedActions: [ - { id: "configure_client:codex", kind: "configure_client" }, - ], + scope: { + clients: ["codex"], + skill: true, + skillDestinations: [{ client: "shared", path: dirname(skillPath) }], + }, }); + if (result.status !== "planned") throw new Error(JSON.stringify(result)); + expect(result.plannedActions).toEqual([ + expect.objectContaining({ + id: "configure_client:codex", + kind: "configure_client", + }), + expect.objectContaining({ + id: "install_skill", + kind: "install_skill", + target: dirname(skillPath), + }), + ]); expect(await readFile(configPath, "utf8")).toBe(staleCodex); expect(await readFile(untouched, "utf8")).toBe('{"mcpServers":{}}'); + expect(await readFile(skillPath, "utf8")).toBe(skillContent); await expect(access(`${configPath}.rea.backup`)).rejects.toMatchObject({ code: "ENOENT", }); + await expect(access(`${skillPath}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); await expect( - access(join(home, ".agents", "skills", PRODUCT_IDENTITY.skillName)), + access(join(home, ".claude", "skills", PRODUCT_IDENTITY.skillName)), ).rejects.toMatchObject({ code: "ENOENT" }); }); }); diff --git a/tests/support/mcp/productionSession.ts b/tests/support/mcp/productionSession.ts new file mode 100644 index 000000000..38ce87e09 --- /dev/null +++ b/tests/support/mcp/productionSession.ts @@ -0,0 +1,33 @@ +import type { Client } from "@modelcontextprotocol/client"; +import { expect, type OnTestFinishedHandler } from "vitest"; + +import { createBinarySession } from "../../../src/composition/binary.js"; +import { parseConfig } from "../../../src/config/parseConfig.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createServer } from "../../../src/server/createServer.js"; +import type { TestMcp } from "./mcpFixture.js"; + +/** Open one real target in a production session served over MCP. */ +export const openProductionTarget = async ( + mcp: TestMcp, + onTestFinished: (handler: OnTestFinishedHandler) => void, + path: string, +): Promise => { + const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); + if (!configured.ok) throw configured.error; + const session = createBinarySession( + configured.value, + silentLogger, + process.env, + ); + onTestFinished(async () => { + await session.close(); + }); + const client = await mcp.connect(createServer({ kind: "session", session })); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + return client; +};