diff --git a/.github/workflows/real-hopper.yml b/.github/workflows/real-hopper.yml index 9308a6a30..b666f1e9c 100644 --- a/.github/workflows/real-hopper.yml +++ b/.github/workflows/real-hopper.yml @@ -2,15 +2,6 @@ name: Real Hopper E2E on: workflow_dispatch: - inputs: - first_target: - description: Absolute path to first test binary on the runner - required: true - type: string - second_target: - description: Absolute path to distinct second test binary on the runner - required: true - type: string permissions: contents: read @@ -27,6 +18,8 @@ jobs: node-version-file: .nvmrc cache: npm - run: npm ci + - name: Build source-owned Hopper conformance fixtures + run: npm run verify:fixtures - name: Install packed CLI in isolation shell: bash run: | @@ -35,8 +28,7 @@ jobs: npm install --global --ignore-scripts --prefix "${prefix}" "./${tarball}" echo "REA_VERIFY_SERVER_COMMAND=${prefix}/bin/rea" >> "${GITHUB_ENV}" echo 'REA_VERIFY_SERVER_ARGS_JSON=["mcp"]' >> "${GITHUB_ENV}" - - name: Verify two real binaries + - name: Verify source-owned fixtures with real Hopper env: - HOPPER_TARGET_PATH: ${{ inputs.first_target }} - HOPPER_SECOND_TARGET_PATH: ${{ inputs.second_target }} + REA_HOPPER_CONFORMANCE_MANIFEST_PATH: ${{ github.workspace }}/build/conformance/manifest.json run: npm run verify:hopper diff --git a/scripts/build-conformance-fixtures.mjs b/scripts/build-conformance-fixtures.mjs index 11b4f0889..37d8a87ec 100644 --- a/scripts/build-conformance-fixtures.mjs +++ b/scripts/build-conformance-fixtures.mjs @@ -6,6 +6,7 @@ import { promisify } from "node:util"; import { generateLargeFixture, + HOPPER_C_ORACLE, LARGE_FIXTURE_COUNT, sha256, sourceDigest, @@ -47,6 +48,7 @@ await build( expectations: { symbols: ["_rea_entry", "_rea_branch", "_rea_leaf", "_rea_c_global"], strings: ["REA_C_ENTRY", "REA_C_LEAF"], + hopperOracle: HOPPER_C_ORACLE, }, }, ); diff --git a/scripts/lib/conformance-fixtures.d.mts b/scripts/lib/conformance-fixtures.d.mts index 949e300ce..06b5ae514 100644 --- a/scripts/lib/conformance-fixtures.d.mts +++ b/scripts/lib/conformance-fixtures.d.mts @@ -1,4 +1,13 @@ export const LARGE_FIXTURE_COUNT: number; +export const HOPPER_C_ORACLE: Readonly<{ + mainProcedure: "main"; + entryProcedure: "rea_entry"; + branchProcedure: "rea_branch"; + leafProcedure: "rea_leaf"; + entryString: "REA_C_ENTRY"; + leafString: "REA_C_LEAF"; + globalName: "rea_c_global"; +}>; export function sha256(value: string | Uint8Array): string; export function generateLargeFixture(count?: number): string; export function sourceDigest( diff --git a/scripts/lib/conformance-fixtures.mjs b/scripts/lib/conformance-fixtures.mjs index 64952dadb..66e80b7c2 100644 --- a/scripts/lib/conformance-fixtures.mjs +++ b/scripts/lib/conformance-fixtures.mjs @@ -2,6 +2,16 @@ import { createHash } from "node:crypto"; export const LARGE_FIXTURE_COUNT = 1_205; +export const HOPPER_C_ORACLE = Object.freeze({ + mainProcedure: "main", + entryProcedure: "rea_entry", + branchProcedure: "rea_branch", + leafProcedure: "rea_leaf", + entryString: "REA_C_ENTRY", + leafString: "REA_C_LEAF", + globalName: "rea_c_global", +}); + export const sha256 = (value) => createHash("sha256").update(value).digest("hex"); diff --git a/scripts/lib/real-hopper-fixture.d.mts b/scripts/lib/real-hopper-fixture.d.mts new file mode 100644 index 000000000..a9b99837b --- /dev/null +++ b/scripts/lib/real-hopper-fixture.d.mts @@ -0,0 +1,34 @@ +export interface RealHopperFixtureOracle { + readonly mainProcedure: string; + readonly entryProcedure: string; + readonly branchProcedure: string; + readonly leafProcedure: string; + readonly entryString: string; + readonly leafString: string; + readonly globalName: string; +} + +export interface RealHopperFixtureTarget { + readonly path: string; + readonly sha256: string; +} + +export interface RealHopperLargeFixtureOracle { + readonly symbolPrefix: string; + readonly symbolCount: number; + readonly stringPrefix: string; + readonly stringCount: number; +} + +export interface RealHopperFixtureTargets { + readonly manifestPath: string; + readonly primary: RealHopperFixtureTarget; + readonly secondary: RealHopperFixtureTarget; + readonly large: RealHopperFixtureTarget; + readonly oracle: RealHopperFixtureOracle; + readonly largeOracle: RealHopperLargeFixtureOracle; +} + +export function loadRealHopperFixtureTargets( + manifestPath: string, +): Promise; diff --git a/scripts/lib/real-hopper-fixture.mjs b/scripts/lib/real-hopper-fixture.mjs new file mode 100644 index 000000000..1c1d955d9 --- /dev/null +++ b/scripts/lib/real-hopper-fixture.mjs @@ -0,0 +1,113 @@ +import { createHash } from "node:crypto"; +import { readFile, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +/** Bind the real-Hopper semantic verifier to source-owned manifest artifacts. */ +export async function loadRealHopperFixtureTargets(manifestPath) { + const canonicalManifest = await realpath(manifestPath); + const manifest = JSON.parse(await readFile(canonicalManifest, "utf8")); + if (manifest?.schemaVersion !== 1 || !Array.isArray(manifest.fixtures)) + throw new Error("Invalid conformance fixture manifest"); + const primary = fixtureEntry(manifest.fixtures, "c"); + const secondary = fixtureEntry(manifest.fixtures, "version-v2"); + const large = fixtureEntry(manifest.fixtures, "large"); + const oracle = requireCOracle(primary.expectations?.hopperOracle); + const largeOracle = requireLargeOracle(large.expectations); + const root = dirname(canonicalManifest); + const [primaryArtifact, secondaryArtifact, largeArtifact] = await Promise.all( + [ + bindFixtureArtifact(root, primary), + bindFixtureArtifact(root, secondary), + bindFixtureArtifact(root, large), + ], + ); + requireDistinctArtifacts([primaryArtifact, secondaryArtifact, largeArtifact]); + return { + manifestPath: canonicalManifest, + primary: primaryArtifact, + secondary: secondaryArtifact, + large: largeArtifact, + oracle, + largeOracle, + }; +} + +function requireCOracle(oracle) { + if ( + oracle === null || + typeof oracle !== "object" || + typeof oracle.mainProcedure !== "string" || + typeof oracle.entryProcedure !== "string" || + typeof oracle.branchProcedure !== "string" || + typeof oracle.leafProcedure !== "string" || + typeof oracle.entryString !== "string" || + typeof oracle.leafString !== "string" || + typeof oracle.globalName !== "string" || + [ + oracle.mainProcedure, + oracle.entryProcedure, + oracle.branchProcedure, + oracle.leafProcedure, + oracle.entryString, + oracle.leafString, + oracle.globalName, + ].some((value) => value.length === 0) + ) + throw new Error("C fixture omitted its Hopper semantic oracle"); + return oracle; +} + +function requireLargeOracle(largeOracle) { + if ( + largeOracle === null || + typeof largeOracle !== "object" || + typeof largeOracle.symbolPrefix !== "string" || + largeOracle.symbolPrefix.length === 0 || + typeof largeOracle.stringPrefix !== "string" || + largeOracle.stringPrefix.length === 0 || + !Number.isInteger(largeOracle.symbolCount) || + largeOracle.symbolCount !== largeOracle.stringCount || + largeOracle.symbolCount <= 0 + ) + throw new Error("Large fixture omitted its exact pagination oracle"); + return largeOracle; +} + +function requireDistinctArtifacts(artifacts) { + if ( + new Set(artifacts.map(({ path }) => path)).size !== artifacts.length || + new Set(artifacts.map(({ sha256 }) => sha256)).size !== artifacts.length + ) + throw new Error("Hopper conformance fixtures must be distinct artifacts"); +} + +function fixtureEntry(fixtures, name) { + const matches = fixtures.filter((fixture) => fixture?.name === name); + if (matches.length !== 1) throw new Error(`Missing fixture ${name}`); + return matches[0]; +} + +async function bindFixtureArtifact(root, fixture) { + if ( + typeof fixture.artifact !== "string" || + typeof fixture.artifactSha256 !== "string" || + !/^[0-9a-f]{64}$/u.test(fixture.artifactSha256) + ) + throw new Error(`Invalid artifact declaration for ${fixture.name}`); + const path = await realpath(resolve(root, fixture.artifact)); + const relation = relative(root, path); + if ( + relation === ".." || + relation.startsWith(`..${sep}`) || + isAbsolute(relation) + ) + throw new Error(`Fixture ${fixture.name} escapes its manifest root`); + const actual = sha256(await readFile(path)); + if (actual !== fixture.artifactSha256) + throw new Error( + `Fixture ${fixture.name} digest did not match its manifest`, + ); + return { path, sha256: actual }; +} + +const sha256 = (value) => createHash("sha256").update(value).digest("hex"); diff --git a/scripts/lib/real-hopper-pagination.d.mts b/scripts/lib/real-hopper-pagination.d.mts new file mode 100644 index 000000000..784a77e9d --- /dev/null +++ b/scripts/lib/real-hopper-pagination.d.mts @@ -0,0 +1,33 @@ +interface ToolClient { + callTool(request: unknown, options: unknown): Promise; +} + +interface PaginationSummary { + readonly count: number; + readonly pages: number; +} + +export function verifyLargeFixturePagination(input: { + readonly client: ToolClient; + readonly options: unknown; + readonly normalizedResult: (value: unknown, operation: string) => unknown; + readonly expectedCount: number; + readonly symbolPrefix: string; + readonly stringPrefix: string; +}): Promise<{ + readonly procedures: PaginationSummary; + readonly strings: PaginationSummary; +}>; + +export function openAndVerifyLargeFixture(input: { + readonly client: ToolClient; + readonly options: unknown; + readonly normalizedResult: (value: unknown, operation: string) => unknown; + readonly path: string; + readonly expectedCount: number; + readonly symbolPrefix: string; + readonly stringPrefix: string; +}): Promise<{ + readonly procedures: PaginationSummary; + readonly strings: PaginationSummary; +}>; diff --git a/scripts/lib/real-hopper-pagination.mjs b/scripts/lib/real-hopper-pagination.mjs new file mode 100644 index 000000000..f9408292a --- /dev/null +++ b/scripts/lib/real-hopper-pagination.mjs @@ -0,0 +1,138 @@ +const PAGE_LIMIT = 100; + +/** Open the large fixture and exhaustively prove its public search pages. */ +export async function openAndVerifyLargeFixture({ + client, + options, + normalizedResult, + path, + expectedCount, + symbolPrefix, + stringPrefix, +}) { + const opened = await client.callTool( + { name: "open_binary", arguments: { path } }, + options, + ); + if (opened.isError === true) + throw new Error("Hopper rejected the large fixture"); + return verifyLargeFixturePagination({ + client, + options, + normalizedResult, + expectedCount, + symbolPrefix, + stringPrefix, + }); +} + +/** Exhaustively prove the source-owned large fixture through public search pages. */ +export async function verifyLargeFixturePagination({ + client, + options, + normalizedResult, + expectedCount, + symbolPrefix, + stringPrefix, +}) { + const procedures = await collectExactPages({ + client, + options, + normalizedResult, + tool: "search_procedures", + pattern: symbolPrefix.replace(/^_+/u, ""), + expectedPrefix: symbolPrefix.replace(/^_+/u, ""), + expectedCount, + normalize: (value) => value.replace(/^_+/u, ""), + }); + const strings = await collectExactPages({ + client, + options, + normalizedResult, + tool: "search_strings", + pattern: stringPrefix, + expectedPrefix: stringPrefix, + expectedCount, + normalize: (value) => value, + }); + return { procedures, strings }; +} + +async function collectExactPages({ + client, + options, + normalizedResult, + tool, + pattern, + expectedPrefix, + expectedCount, + normalize, +}) { + const seenAddresses = new Set(); + const seenValues = new Set(); + let offset = 0; + let pages = 0; + while (true) { + const page = normalizedResult( + await client.callTool( + { + name: tool, + arguments: { + pattern, + mode: "literal", + case_sensitive: true, + offset, + limit: PAGE_LIMIT, + }, + }, + options, + ), + `${tool} offset ${offset}`, + ); + validatePage(page, { offset, expectedCount }); + for (const item of page.items) { + const value = normalize(item.value); + if (item.value_truncated || !/^0x[0-9a-f]+$/iu.test(item.address)) + throw new Error(`${tool} returned truncated or malformed evidence`); + if (seenAddresses.has(item.address) || seenValues.has(value)) + throw new Error(`${tool} returned duplicate evidence`); + seenAddresses.add(item.address); + seenValues.add(value); + } + pages += 1; + if (page.next_offset === null) break; + offset = page.next_offset; + } + const expectedValues = Array.from( + { length: expectedCount }, + (_, index) => `${expectedPrefix}${String(index).padStart(4, "0")}`, + ); + if ( + seenValues.size !== expectedCount || + expectedValues.some((value) => !seenValues.has(value)) + ) + throw new Error(`${tool} did not exhaustively prove the fixture values`); + return { count: seenValues.size, pages }; +} + +function validatePage(page, { offset, expectedCount }) { + if ( + page === null || + typeof page !== "object" || + !Array.isArray(page.items) || + page.offset !== offset || + page.limit !== PAGE_LIMIT || + page.total !== expectedCount || + page.items.length !== Math.min(PAGE_LIMIT, expectedCount - offset) + ) + throw new Error("Hopper returned an inconsistent fixture search page"); + const expectedNext = + offset + page.items.length < expectedCount + ? offset + page.items.length + : null; + if ( + page.next_offset !== expectedNext || + page.has_more !== (expectedNext !== null) + ) + throw new Error("Hopper returned inconsistent pagination metadata"); +} diff --git a/scripts/lib/real-hopper-semantic.d.mts b/scripts/lib/real-hopper-semantic.d.mts new file mode 100644 index 000000000..6adca7fd2 --- /dev/null +++ b/scripts/lib/real-hopper-semantic.d.mts @@ -0,0 +1,34 @@ +export interface FixtureProcedure { + readonly address: string; + readonly name: string; +} + +interface ToolClient { + callTool( + request: { readonly name: string; readonly arguments: unknown }, + options: unknown, + ): Promise; +} + +export function resolveFixtureProcedure( + client: ToolClient, + options: unknown, + expectedName: string, + normalizedResult: (value: unknown, operation: string) => unknown, +): Promise; + +export function requireSafeDiagnostics(chunks: readonly string[]): number; + +export function requireCurrentDocument( + client: ToolClient, + options: unknown, + documents: readonly string[], + normalizedResult: (value: unknown, operation: string) => unknown, +): Promise; + +export function analyzeFixtureProcedure( + client: ToolClient, + options: unknown, + procedure: FixtureProcedure, + normalizedResult: (value: unknown, operation: string) => unknown, +): Promise; diff --git a/scripts/lib/real-hopper-semantic.mjs b/scripts/lib/real-hopper-semantic.mjs new file mode 100644 index 000000000..9a2ae0c7e --- /dev/null +++ b/scripts/lib/real-hopper-semantic.mjs @@ -0,0 +1,194 @@ +import { requireFunctionDossierOracle } from "../../dist/application/RealHopperAssertions.js"; + +/** Verify named positive semantics from the source-owned C fixture. */ +export async function verifyRealHopperFixture({ + client, + options, + document, + oracle, + normalizedResult, +}) { + const main = await resolveFixtureProcedure( + client, + options, + oracle.mainProcedure, + normalizedResult, + ); + const entry = await resolveFixtureProcedure( + client, + options, + oracle.entryProcedure, + normalizedResult, + ); + const branch = await resolveFixtureProcedure( + client, + options, + oracle.branchProcedure, + normalizedResult, + ); + const leaf = await resolveFixtureProcedure( + client, + options, + oracle.leafProcedure, + normalizedResult, + ); + const comment = "REA real-Hopper conformance verifier"; + const commentSet = normalizedResult( + await client.callTool( + { + name: "set_comment", + arguments: { address: entry.address, comment, document }, + }, + options, + ), + "set_comment fixture marker", + ); + if (commentSet !== true) + throw new Error("Hopper did not read back the fixture comment marker"); + + const entryDossier = requireFunctionDossierOracle( + await analyzeFixtureProcedure(client, options, entry, normalizedResult), + { + procedure_address: entry.address, + callee_address: branch.address, + caller_address: main.address, + referenced_string: oracle.entryString, + comment, + require_assembly: true, + }, + ); + const branchDossier = requireFunctionDossierOracle( + await analyzeFixtureProcedure(client, options, branch, normalizedResult), + { + procedure_address: branch.address, + callee_address: leaf.address, + caller_address: entry.address, + require_cfg_successor: true, + require_assembly: true, + }, + ); + const leafDossier = requireFunctionDossierOracle( + await analyzeFixtureProcedure(client, options, leaf, normalizedResult), + { + procedure_address: leaf.address, + caller_address: branch.address, + referenced_string: oracle.leafString, + referenced_name: oracle.globalName, + require_assembly: true, + }, + ); + return { + procedures: { main, entry, branch, leaf }, + entryComments: entryDossier.comments.returned, + entryStrings: entryDossier.referenced_strings.returned, + entryCallees: entryDossier.callees.returned, + branchCallees: branchDossier.callees.returned, + branchBlocks: branchDossier.basic_blocks.returned, + leafStrings: leafDossier.referenced_strings.returned, + leafNames: leafDossier.referenced_names.returned, + }; +} + +/** Reject malformed, unsafe, or error-level diagnostics from the MCP runtime. */ +export function requireSafeDiagnostics(chunks) { + const lines = chunks + .join("") + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + for (const line of lines) { + let diagnostic; + try { + diagnostic = JSON.parse(line); + } catch { + throw new Error(`The MCP runtime emitted malformed stderr: ${line}`); + } + if ( + diagnostic === null || + typeof diagnostic !== "object" || + diagnostic.application !== "rea" || + typeof diagnostic.level !== "number" || + diagnostic.level >= 40 + ) + throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`); + } + return lines.length; +} + +/** Bind mutations to the active document rather than list ordering. */ +export async function requireCurrentDocument( + client, + options, + documents, + normalizedResult, +) { + const current = normalizedResult( + await client.callTool({ name: "current_document", arguments: {} }, options), + "current_document", + ); + if ( + typeof current !== "string" || + current.length === 0 || + !documents.includes(current) + ) + throw new Error("current_document was not present in list_documents"); + return current; +} + +export async function resolveFixtureProcedure( + client, + options, + expectedName, + normalizedResult, +) { + const page = normalizedResult( + await client.callTool( + { + name: "search_procedures", + arguments: { + pattern: expectedName, + mode: "literal", + case_sensitive: true, + limit: 100, + }, + }, + options, + ), + `search_procedures ${expectedName}`, + ); + const matches = Array.isArray(page?.items) + ? page.items.filter( + (item) => + typeof item?.value === "string" && + item.value_truncated === false && + item.value.replace(/^_+/u, "") === expectedName, + ) + : []; + if ( + matches.length !== 1 || + typeof matches[0]?.address !== "string" || + !/^0x[0-9a-f]+$/iu.test(matches[0].address) + ) + throw new Error( + `Expected exactly one Hopper procedure named ${expectedName}`, + ); + return { address: matches[0].address, name: matches[0].value }; +} + +/** Request the assembly evidence required by the fixture oracle. */ +export const analyzeFixtureProcedure = async ( + client, + options, + procedure, + normalizedResult, +) => + normalizedResult( + await client.callTool( + { + name: "analyze_function", + arguments: { procedure: procedure.address, include_assembly: true }, + }, + options, + ), + `analyze_function ${procedure.name}`, + ); diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index de009ba2f..7cc324e1c 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -1,7 +1,5 @@ import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { createReadStream } from "node:fs"; -import { readdir, realpath } from "node:fs/promises"; +import { readdir } from "node:fs/promises"; import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -10,17 +8,21 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { firstProcedureAddress, - requireDistinctTargetHashes, requireAddressArray, requireFunctionDossier, requirePseudocode, } from "../dist/application/RealHopperAssertions.js"; import { HOPPER_PROVIDER_IDENTITY } from "../dist/hopper/HopperProvider.js"; import { REA_WORKFLOW_PROVIDER } from "../dist/application/InvestigationProviders.js"; - +import { loadRealHopperFixtureTargets } from "./lib/real-hopper-fixture.mjs"; +import { + requireCurrentDocument, + requireSafeDiagnostics, + verifyRealHopperFixture, +} from "./lib/real-hopper-semantic.mjs"; +import { openAndVerifyLargeFixture } from "./lib/real-hopper-pagination.mjs"; const execFileAsync = promisify(execFile); const timeout = 180_000; - const parseServerArgs = (encoded) => { const parsed = JSON.parse(encoded); if ( @@ -34,7 +36,6 @@ const parseServerArgs = (encoded) => { const sessionsBefore = new Set( (await readdir("/tmp")).filter((name) => name.startsWith("rea-")), ); - const textValue = (result) => { const text = result.content?.find((item) => item.type === "text")?.text; if (typeof text !== "string") throw new Error("Tool result omitted text"); @@ -285,49 +286,15 @@ const verifyCurrentTarget = async (client, options) => { }; }; -const requireSafeDiagnostics = (chunks) => { - const lines = chunks - .join("") - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0); - for (const line of lines) { - let diagnostic; - try { - diagnostic = JSON.parse(line); - } catch { - throw new Error(`The MCP runtime emitted malformed stderr: ${line}`); - } - if ( - diagnostic === null || - typeof diagnostic !== "object" || - diagnostic.application !== "rea" || - typeof diagnostic.level !== "number" || - diagnostic.level >= 40 - ) { - throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`); - } - } - return lines.length; -}; - -const targetPath = process.env.HOPPER_TARGET_PATH; -const secondTargetPath = process.env.HOPPER_SECOND_TARGET_PATH; -if (!targetPath || !secondTargetPath) - throw new Error( - "HOPPER_TARGET_PATH and HOPPER_SECOND_TARGET_PATH are required and must name distinct binaries", - ); -const [targetA, targetB] = await Promise.all([ - realpath(targetPath), - realpath(secondTargetPath), -]); -if (targetA === targetB) - throw new Error("Real-Hopper verification requires two distinct targets"); -const [targetHashA, targetHashB] = await Promise.all([ - sha256File(targetA), - sha256File(targetB), -]); -requireDistinctTargetHashes(targetHashA, targetHashB); +const fixtureManifestPath = process.env.REA_HOPPER_CONFORMANCE_MANIFEST_PATH; +if (!fixtureManifestPath) + throw new Error("REA_HOPPER_CONFORMANCE_MANIFEST_PATH is required"); +const fixtureTargets = await loadRealHopperFixtureTargets(fixtureManifestPath); +const targetA = fixtureTargets.primary.path; +const targetB = fixtureTargets.secondary.path; +const largeTarget = fixtureTargets.large.path; +const targetHashA = fixtureTargets.primary.sha256; +const targetHashB = fixtureTargets.secondary.sha256; const serverEnvironment = { ...process.env }; serverEnvironment.REA_ANALYSIS_PROVIDER = "auto"; delete serverEnvironment.HOPPER_TARGET_PATH; @@ -409,13 +376,30 @@ try { requireSuccessfulTool(segments, "list_segments"), ); const firstDocuments = requireSuccessfulTool(documents, "list_documents"); - if (!Array.isArray(firstDocuments) || firstDocuments.length === 0) + if ( + !Array.isArray(firstDocuments) || + firstDocuments.length === 0 || + typeof firstDocuments[0] !== "string" + ) throw new Error("list_documents returned no real Hopper document"); + const currentDocument = await requireCurrentDocument( + client, + options, + firstDocuments, + requireSuccessfulTool, + ); const firstOverview = requireOverview( requireSuccessfulTool(overview, "binary_overview"), "binary_overview", ); const firstAnalysis = await verifyCurrentTarget(client, options); + const fixtureAnalysis = await verifyRealHopperFixture({ + client, + options, + document: currentDocument, + oracle: fixtureTargets.oracle, + normalizedResult: requireSuccessfulTool, + }); const switched = await client.callTool( { name: "open_binary", arguments: { path: targetB } }, options, @@ -439,6 +423,15 @@ try { "binary_overview after target switch", ); const secondAnalysis = await verifyCurrentTarget(client, options); + const largePagination = await openAndVerifyLargeFixture({ + client, + options, + normalizedResult: requireSuccessfulTool, + path: largeTarget, + expectedCount: fixtureTargets.largeOracle.symbolCount, + symbolPrefix: fixtureTargets.largeOracle.symbolPrefix, + stringPrefix: fixtureTargets.largeOracle.stringPrefix, + }); const processList = await execFileAsync("ps", ["-ax", "-o", "command="]); const bundledMcpRunning = processList.stdout @@ -468,13 +461,16 @@ try { overview: firstOverview, segmentCount: firstOverview.segment_count, analyses: [firstAnalysis, secondAnalysis], + fixtureAnalysis, + largePagination, + fixtureManifest: fixtureTargets.manifestPath, bundledMcpRunning, stderrBytes, diagnosticCount, dynamicSession: true, providerBinding, - targets: [targetA, targetB], - targetHashes: [targetHashA, targetHashB], + targets: [targetA, targetB, largeTarget], + targetHashes: [targetHashA, targetHashB, fixtureTargets.large.sha256], switched: true, secondOverview: verifiedSecondOverview, }; @@ -520,9 +516,3 @@ await new Promise((resolve, reject) => { }, ); }); - -async function sha256File(path) { - const hash = createHash("sha256"); - for await (const chunk of createReadStream(path)) hash.update(chunk); - return hash.digest("hex"); -} diff --git a/src/application/RealHopperAssertions.ts b/src/application/RealHopperAssertions.ts index 45e12d22b..56386e118 100644 --- a/src/application/RealHopperAssertions.ts +++ b/src/application/RealHopperAssertions.ts @@ -80,6 +80,151 @@ export const requireFunctionDossier = ( return value; }; +/** Positive semantic facts required from one source-owned Hopper fixture. */ +export interface FunctionDossierOracle { + readonly procedure_address: string; + readonly callee_address?: string; + readonly caller_address?: string; + readonly referenced_string?: string; + readonly referenced_name?: string; + readonly comment?: string; + readonly require_cfg_successor?: boolean; + readonly require_assembly?: boolean; +} + +/** Reject a structurally valid dossier that omits fixture-proven semantics. */ +export const requireFunctionDossierOracle = ( + input: unknown, + oracle: FunctionDossierOracle, +): Record => { + const value = requireFunctionDossier(input, oracle.procedure_address); + if (oracle.callee_address !== undefined) { + const callees = z + .array(identitySchema) + .parse(collectionSchema.parse(value.callees).items); + if (!callees.some(({ address }) => address === oracle.callee_address)) + throw new TypeError( + "analyze_function omitted the expected fixture callee", + ); + + const references = z + .array( + z.object({ + source_address: addressSchema, + target_address: addressSchema, + }), + ) + .parse(collectionSchema.parse(value.outgoing_references).items); + if ( + !references.some( + ({ target_address }) => target_address === oracle.callee_address, + ) + ) + throw new TypeError( + "analyze_function omitted the expected fixture reference", + ); + } + + if (oracle.caller_address !== undefined) { + const callers = z + .array(identitySchema) + .parse(collectionSchema.parse(value.callers).items); + if (!callers.some(({ address }) => address === oracle.caller_address)) + throw new TypeError( + "analyze_function omitted the expected fixture caller", + ); + } + + if (oracle.referenced_string !== undefined) + requireReferencedValue( + value, + "referenced_strings", + oracle.referenced_string, + (candidate) => candidate, + ); + + if (oracle.referenced_name !== undefined) + requireReferencedValue( + value, + "referenced_names", + oracle.referenced_name, + (candidate) => candidate.replace(/^_+/u, ""), + ); + + if (oracle.comment !== undefined) { + const comments = z + .array( + z.object({ + address: addressSchema, + kind: z.enum(["comment", "inline"]), + text: z.string(), + }), + ) + .parse(collectionSchema.parse(value.comments).items); + if ( + !comments.some( + ({ address, kind, text }) => + address === oracle.procedure_address && + kind === "comment" && + text === oracle.comment, + ) + ) + throw new TypeError("analyze_function omitted the verifier comment"); + } + + if ( + oracle.require_assembly === true && + collectionSchema.parse(value.assembly).items.length === 0 + ) + throw new TypeError("analyze_function omitted fixture assembly"); + + if (oracle.require_cfg_successor) { + const blocks = z + .array(z.object({ successors: z.array(addressSchema) })) + .parse(collectionSchema.parse(value.basic_blocks).items); + if (!blocks.some(({ successors }) => successors.length > 0)) + throw new TypeError("analyze_function omitted a real CFG successor"); + } + return value; +}; + +const referencedValueSchema = z.object({ + address: addressSchema, + value: z.string(), + source_address: addressSchema, +}); + +const referenceEdgeSchema = z.object({ + source_address: addressSchema, + target_address: addressSchema, +}); + +const requireReferencedValue = ( + dossier: Record, + field: "referenced_strings" | "referenced_names", + expected: string, + normalize: (value: string) => string, +): void => { + const values = z + .array(referencedValueSchema) + .parse(collectionSchema.parse(dossier[field]).items); + const references = z + .array(referenceEdgeSchema) + .parse(collectionSchema.parse(dossier.outgoing_references).items); + const found = values.some( + ({ address, value, source_address: sourceAddress }) => + normalize(value) === expected && + references.some( + ({ source_address, target_address }) => + source_address === sourceAddress && target_address === address, + ), + ); + if (!found) + throw new TypeError( + `analyze_function omitted the expected fixture ${field === "referenced_strings" ? "string" : "name"}`, + ); +}; + /** Require every relationship to be an actual hexadecimal address. */ export const requireAddressArray = ( input: unknown, @@ -92,6 +237,7 @@ export const requireAddressArray = ( }; const COLLECTION_FIELDS = [ + "assembly", "comments", "callers", "callees", diff --git a/tests/conformanceFixtures.test.ts b/tests/conformanceFixtures.test.ts index 6b02b9ec0..b181f1919 100644 --- a/tests/conformanceFixtures.test.ts +++ b/tests/conformanceFixtures.test.ts @@ -1,13 +1,30 @@ +import { readFile } from "node:fs/promises"; + import { describe, expect, it } from "vitest"; import { generateLargeFixture, + HOPPER_C_ORACLE, LARGE_FIXTURE_COUNT, sha256, sourceDigest, } from "../scripts/lib/conformance-fixtures.mjs"; describe("source-built conformance fixtures", () => { + it("grounds the Hopper semantic oracle in the source-owned C fixture", async () => { + const source = await readFile("tests/conformance/c/fixture.c", "utf8"); + for (const procedure of [ + HOPPER_C_ORACLE.mainProcedure, + HOPPER_C_ORACLE.entryProcedure, + HOPPER_C_ORACLE.branchProcedure, + HOPPER_C_ORACLE.leafProcedure, + ]) + expect(source).toContain(`int ${procedure}`); + expect(source).toContain(HOPPER_C_ORACLE.entryString); + expect(source).toContain(HOPPER_C_ORACLE.leafString); + expect(source).toContain(HOPPER_C_ORACLE.globalName); + }); + it("generates a deterministic large pagination fixture", () => { const first = generateLargeFixture(); const second = generateLargeFixture(); diff --git a/tests/realHopperAssertions.test.ts b/tests/realHopperAssertions.test.ts index fe6d0424e..f50a185f5 100644 --- a/tests/realHopperAssertions.test.ts +++ b/tests/realHopperAssertions.test.ts @@ -4,6 +4,7 @@ import { firstProcedureAddress, requireDistinctTargetHashes, requireAddressArray, + requireFunctionDossierOracle, requireFunctionDossier, requirePseudocode, } from "../src/application/RealHopperAssertions.js"; @@ -86,8 +87,161 @@ describe("real Hopper semantic assertions", () => { }; expect(requireFunctionDossier(scanLimited, "0x1000")).toEqual(scanLimited); }); + + it("requires fixture-specific positive dossier evidence", () => { + const entry = fixtureDossier({ + address: "0x1000", + name: "rea_entry", + callee: { address: "0x2000", name: "rea_branch" }, + caller: { address: "0x0900", name: "main" }, + referencedString: "REA_C_ENTRY", + comment: "REA real-Hopper verifier", + successor: "0x1010", + }); + expect( + requireFunctionDossierOracle(entry, { + procedure_address: "0x1000", + callee_address: "0x2000", + caller_address: "0x0900", + referenced_string: "REA_C_ENTRY", + referenced_name: "rea_c_global", + comment: "REA real-Hopper verifier", + require_cfg_successor: true, + require_assembly: true, + }), + ).toEqual(entry); + + for (const field of [ + "callees", + "callers", + "outgoing_references", + "referenced_strings", + "referenced_names", + "comments", + "assembly", + ] as const) { + expect(() => + requireFunctionDossierOracle( + { ...entry, [field]: emptyCollection() }, + { + procedure_address: "0x1000", + callee_address: "0x2000", + caller_address: "0x0900", + referenced_string: "REA_C_ENTRY", + referenced_name: "rea_c_global", + comment: "REA real-Hopper verifier", + require_cfg_successor: true, + require_assembly: true, + }, + ), + ).toThrow(); + } + expect(() => + requireFunctionDossierOracle( + { + ...entry, + referenced_names: bounded([ + { + address: "0x4000", + value: "_rea_c_global", + source_address: "0x2000", + }, + ]), + }, + { + procedure_address: "0x1000", + referenced_name: "rea_c_global", + }, + ), + ).toThrow(); + expect(() => + requireFunctionDossierOracle( + { + ...entry, + basic_blocks: bounded([{ start: "0x1000", successors: [] }]), + }, + { + procedure_address: "0x1000", + callee_address: "0x2000", + referenced_string: "REA_C_ENTRY", + comment: "REA real-Hopper verifier", + require_cfg_successor: true, + }, + ), + ).toThrow(/CFG successor/u); + }); }); +const fixtureDossier = (input: { + readonly address: string; + readonly name: string; + readonly callee: { readonly address: string; readonly name: string }; + readonly caller: { readonly address: string; readonly name: string }; + readonly referencedString: string; + readonly comment: string; + readonly successor: string; +}) => ({ + ...validDossier(), + procedure: { address: input.address, name: input.name }, + assembly: bounded(["mov eax, eax"]), + comments: bounded([ + { address: input.address, kind: "comment", text: input.comment }, + ]), + callees: bounded([input.callee]), + callers: bounded([input.caller]), + outgoing_references: bounded([ + { + source_address: input.address, + target_address: input.callee.address, + source_procedure: { address: input.address, name: input.name }, + target_procedure: input.callee, + kind: { available: false, reason: "public API did not classify kind" }, + }, + { + source_address: "0x1004", + target_address: "0x3000", + source_procedure: { address: input.address, name: input.name }, + target_procedure: null, + kind: { available: false, reason: "public API did not classify kind" }, + }, + { + source_address: "0x1008", + target_address: "0x4000", + source_procedure: { address: input.address, name: input.name }, + target_procedure: null, + kind: { available: false, reason: "public API did not classify kind" }, + }, + ]), + referenced_strings: bounded([ + { + address: "0x3000", + value: input.referencedString, + source_address: "0x1004", + }, + ]), + referenced_names: bounded([ + { + address: "0x4000", + value: "_rea_c_global", + source_address: "0x1008", + }, + ]), + basic_blocks: bounded([ + { start: input.address, successors: [input.successor] }, + { start: input.successor, successors: [] }, + ]), +}); + +const bounded = (items: readonly unknown[]) => ({ + items, + total: items.length, + returned: items.length, + truncated: false, + next_offset: null, +}); + +const emptyCollection = () => bounded([]); + const validDossier = () => { const collection = { items: [], @@ -99,6 +253,7 @@ const validDossier = () => { return { procedure: { address: "0x1000", name: "main" }, pseudocode: { text: "return 0;", returned_chars: 9, total_chars: 9 }, + assembly: collection, comments: collection, callers: collection, callees: collection, diff --git a/tests/realHopperFixture.test.ts b/tests/realHopperFixture.test.ts new file mode 100644 index 000000000..d7e0e9f79 --- /dev/null +++ b/tests/realHopperFixture.test.ts @@ -0,0 +1,135 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { loadRealHopperFixtureTargets } from "../scripts/lib/real-hopper-fixture.mjs"; + +let root: string | undefined; +afterEach(async () => { + if (root !== undefined) await rm(root, { recursive: true, force: true }); + root = undefined; +}); + +describe("real Hopper fixture binding", () => { + it("binds exact source-owned artifacts and their semantic oracle", async () => { + const manifest = await fixtureManifest(); + await expect(loadRealHopperFixtureTargets(manifest)).resolves.toMatchObject( + { + primary: { path: join(root ?? "", "c") }, + secondary: { path: join(root ?? "", "version-v2") }, + large: { path: join(root ?? "", "large") }, + largeOracle: { + symbolPrefix: "_rea_page_", + symbolCount: 1_205, + stringPrefix: "REA_PAGE_", + stringCount: 1_205, + }, + oracle: { + mainProcedure: "main", + entryProcedure: "rea_entry", + branchProcedure: "rea_branch", + leafProcedure: "rea_leaf", + entryString: "REA_C_ENTRY", + leafString: "REA_C_LEAF", + globalName: "rea_c_global", + }, + }, + ); + }); + + it("rejects digest drift and artifacts escaping the manifest root", async () => { + const manifest = await fixtureManifest(); + await writeFile(join(root ?? "", "c"), "changed"); + await expect(loadRealHopperFixtureTargets(manifest)).rejects.toThrow( + /digest/u, + ); + + const outside = await mkdtemp(join(tmpdir(), "rea-hopper-outside-")); + try { + await writeFile(join(outside, "c"), "primary"); + await rm(join(root ?? "", "c")); + await symlink(join(outside, "c"), join(root ?? "", "c")); + await expect(loadRealHopperFixtureTargets(manifest)).rejects.toThrow( + /escapes/u, + ); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it("rejects manifest entries that alias the same artifact", async () => { + const manifest = await fixtureManifest(); + const parsed = JSON.parse(await readFile(manifest, "utf8")) as { + fixtures: { name: string; artifact: string; artifactSha256: string }[]; + }; + const primary = parsed.fixtures.find(({ name }) => name === "c"); + const secondary = parsed.fixtures.find(({ name }) => name === "version-v2"); + if (primary === undefined || secondary === undefined) + throw new Error("Fixture manifest setup failed"); + secondary.artifact = primary.artifact; + secondary.artifactSha256 = primary.artifactSha256; + await writeFile(manifest, JSON.stringify(parsed)); + + await expect(loadRealHopperFixtureTargets(manifest)).rejects.toThrow( + /distinct artifacts/u, + ); + }); +}); + +const fixtureManifest = async (): Promise => { + root = await mkdtemp(join(tmpdir(), "rea-hopper-fixture-")); + const primary = Buffer.from("primary"); + const secondary = Buffer.from("secondary"); + await Promise.all([ + writeFile(join(root, "c"), primary), + writeFile(join(root, "version-v2"), secondary), + writeFile(join(root, "large"), Buffer.from("large")), + ]); + const manifest = join(root, "manifest.json"); + await writeFile( + manifest, + JSON.stringify({ + schemaVersion: 1, + fixtures: [ + { + name: "c", + artifact: "c", + artifactSha256: digest(primary), + expectations: { + hopperOracle: { + mainProcedure: "main", + entryProcedure: "rea_entry", + branchProcedure: "rea_branch", + leafProcedure: "rea_leaf", + entryString: "REA_C_ENTRY", + leafString: "REA_C_LEAF", + globalName: "rea_c_global", + }, + }, + }, + { + name: "version-v2", + artifact: "version-v2", + artifactSha256: digest(secondary), + }, + { + name: "large", + artifact: "large", + artifactSha256: digest(Buffer.from("large")), + expectations: { + symbolPrefix: "_rea_page_", + symbolCount: 1_205, + stringPrefix: "REA_PAGE_", + stringCount: 1_205, + }, + }, + ], + }), + ); + return manifest; +}; + +const digest = (value: Buffer): string => + createHash("sha256").update(value).digest("hex"); diff --git a/tests/realHopperPagination.test.ts b/tests/realHopperPagination.test.ts new file mode 100644 index 000000000..834a8e9a8 --- /dev/null +++ b/tests/realHopperPagination.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; + +import { verifyLargeFixturePagination } from "../scripts/lib/real-hopper-pagination.mjs"; + +const normalize = (value: unknown): unknown => value; + +describe("real Hopper large-fixture pagination", () => { + it("proves every procedure and string across exact search pages", async () => { + await expect( + verifyLargeFixturePagination({ + client: fixtureClient(205), + options: {}, + normalizedResult: normalize, + expectedCount: 205, + symbolPrefix: "_rea_page_", + stringPrefix: "REA_PAGE_", + }), + ).resolves.toEqual({ + procedures: { count: 205, pages: 3 }, + strings: { count: 205, pages: 3 }, + }); + }); + + it.each(["truncated", "duplicate", "wrong-total"] as const)( + "rejects %s positive evidence", + async (fault) => { + await expect( + verifyLargeFixturePagination({ + client: fixtureClient(205, fault), + options: {}, + normalizedResult: normalize, + expectedCount: 205, + symbolPrefix: "_rea_page_", + stringPrefix: "REA_PAGE_", + }), + ).rejects.toThrow(); + }, + ); +}); + +const fixtureClient = ( + count: number, + fault?: "truncated" | "duplicate" | "wrong-total", +) => ({ + callTool: async (request: unknown) => { + const parsed = request as { + name: string; + arguments: { offset: number; limit: number }; + }; + const { offset, limit } = parsed.arguments; + const returned = Math.min(limit, count - offset); + const procedure = parsed.name === "search_procedures"; + const items = Array.from({ length: returned }, (_, pageIndex) => { + const index = offset + pageIndex; + const evidenceIndex = fault === "duplicate" && index === 1 ? 0 : index; + return { + address: `0x${(0x1000 + index).toString(16)}`, + value: `${procedure ? "_rea_page_" : "REA_PAGE_"}${String(evidenceIndex).padStart(4, "0")}`, + value_truncated: fault === "truncated" && index === 0, + }; + }); + const next = offset + returned < count ? offset + returned : null; + return { + items, + offset, + limit, + total: fault === "wrong-total" ? count + 1 : count, + next_offset: next, + has_more: next !== null, + }; + }, +}); diff --git a/tests/realHopperSemantic.test.ts b/tests/realHopperSemantic.test.ts new file mode 100644 index 000000000..b97981734 --- /dev/null +++ b/tests/realHopperSemantic.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; + +import { + analyzeFixtureProcedure, + requireCurrentDocument, + resolveFixtureProcedure, +} from "../scripts/lib/real-hopper-semantic.mjs"; + +const normalize = (value: unknown): unknown => value; + +describe("real Hopper semantic fixture resolution", () => { + it("requests assembly when analyzing fixture procedures", async () => { + const requests: unknown[] = []; + const result = await analyzeFixtureProcedure( + { + callTool: async (request: unknown) => { + requests.push(request); + return { dossier: true }; + }, + }, + {}, + { address: "0x1000", name: "rea_entry" }, + normalize, + ); + + expect(result).toEqual({ dossier: true }); + expect(requests).toEqual([ + { + name: "analyze_function", + arguments: { procedure: "0x1000", include_assembly: true }, + }, + ]); + }); + + it("binds mutations to the active document, not the first listed one", async () => { + await expect( + requireCurrentDocument( + clientReturning("active"), + {}, + ["unrelated", "active"], + normalize, + ), + ).resolves.toBe("active"); + await expect( + requireCurrentDocument( + clientReturning("unknown"), + {}, + ["unrelated", "active"], + normalize, + ), + ).rejects.toThrow(/current_document/u); + }); + + it("resolves the public search value and preserves its reported name", async () => { + const procedure = await resolveFixtureProcedure( + clientWithItems([ + { + address: "0x1000", + value: "_rea_entry", + value_truncated: false, + }, + ]), + {}, + "rea_entry", + normalize, + ); + + expect(procedure).toEqual({ address: "0x1000", name: "_rea_entry" }); + }); + + it.each([ + ["empty", []], + [ + "truncated", + [{ address: "0x1000", value: "rea_entry", value_truncated: true }], + ], + [ + "duplicate", + [ + { address: "0x1000", value: "rea_entry", value_truncated: false }, + { address: "0x2000", value: "_rea_entry", value_truncated: false }, + ], + ], + [ + "wrong name", + [{ address: "0x1000", value: "rea_other", value_truncated: false }], + ], + ])("rejects %s search evidence", async (_label, items) => { + await expect( + resolveFixtureProcedure( + clientWithItems(items), + {}, + "rea_entry", + normalize, + ), + ).rejects.toThrow(/exactly one Hopper procedure/u); + }); +}); + +const clientWithItems = (items: readonly unknown[]) => ({ + callTool: async () => ({ items }), +}); + +const clientReturning = (value: unknown) => ({ + callTool: async () => value, +});