diff --git a/docs/reconstruction-readiness.md b/docs/reconstruction-readiness.md index 4a7bcea96..b21454b0e 100644 --- a/docs/reconstruction-readiness.md +++ b/docs/reconstruction-readiness.md @@ -1,7 +1,7 @@ # Reconstruction readiness conformance -Run the complete source-owned native, JavaScript CLI, Electron, Evidence, -comparison, obligation, replay, CLI, and MCP journey from a clean checkout: +Run the source-owned native, JavaScript CLI, Electron, Evidence, comparison, +obligation, replay, and cleanup conformance verifier from a clean checkout: ```sh npm ci @@ -10,10 +10,10 @@ npm run verify:readiness The command builds the redistributable native conformance fixtures, inventories the native executable through the public CLI, analyzes the generated JavaScript -CLI and Electron fixture through both CLI and MCP, evaluates the same nine-stage -report through both surfaces, verifies that MCP returns the complete report -inline, and proves stale/tampered replay rejection. It prints one compact JSON -summary. +CLI and Electron fixtures through both CLI and MCP, evaluates an internal +nine-stage conformance report, and proves stale/tampered replay rejection. The +report evaluator is part of the repository verifier, not a public REA tool. It +prints one compact JSON summary. Set `REA_READINESS_REPORT_PATH` to retain the complete machine-readable report: @@ -24,11 +24,11 @@ REA_READINESS_REPORT_PATH=readiness-report.json npm run verify:readiness CI runs this command on Linux, publishes the compact summary, and retains the full report as the `reconstruction-readiness` artifact. -## Public contracts +## Project conformance contract -- CLI: `rea evaluate-reconstruction-readiness --json` -- MCP tool: `evaluate_reconstruction_readiness` - The MCP tool returns the complete report inline, including the Evidence bundle, stage records, comparisons, contradictions, obligation ledger, and replay inputs. +The verifier runs as `npm run verify:readiness`; it is not exposed as a public +CLI command or MCP tool. The synthetic report input and evaluator exist to +check project-owned conformance fixtures and fail-closed report behavior. ## Required stages diff --git a/docs/verification/managed-conformance-manifest.json b/docs/verification/managed-conformance-manifest.json index b6f9402af..fa217881e 100644 --- a/docs/verification/managed-conformance-manifest.json +++ b/docs/verification/managed-conformance-manifest.json @@ -1,5 +1,5 @@ { - "manifest_id": "ecm_0fb5d4adf7665b6541b81de155576b250471d46ed881563f0bf3dbcf839dbc8d", + "manifest_id": "ecm_529002491b8fb1e1b2402a9d6355676d7630be5e7ff919caaa0b8c9456a0a7a4", "verifier": { "id": "rea-managed-conformance", "version": "1" @@ -13,7 +13,7 @@ "skill_digests": [ { "skill_id": "reverse-engineer-anything", - "sha256": "513748a1cfcc40d5d353e694a7029ee4d118d4c25ef3f3631d1fe9017016cc84" + "sha256": "03bd2dbc07964720ac317d4fe8ad429b3697ce905dbee48e363fe314e812064c" } ], "claims": [ diff --git a/scripts/verify-reconstruction-readiness.mjs b/scripts/verify-reconstruction-readiness.mjs index 775d328a6..1de268ae8 100644 --- a/scripts/verify-reconstruction-readiness.mjs +++ b/scripts/verify-reconstruction-readiness.mjs @@ -12,6 +12,7 @@ import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; import { RECONSTRUCTION_READINESS_EXAMPLE } from "../dist/contracts/reconstructionReadinessExample.js"; import { parseEvidence } from "../dist/domain/evidence.js"; import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js"; +import { createReconstructionReadinessReport } from "../dist/domain/reconstructionReadiness.js"; import { reconstructionReadinessReportSchema } from "../dist/domain/reconstructionReadinessSchemas.js"; import { PACKAGE_METADATA } from "../dist/generatedPackageMetadata.js"; @@ -54,17 +55,6 @@ const client = new Client({ try { await client.connect(transport); - const catalog = await client.listTools(); - const readinessTool = requireTool( - catalog.tools, - "evaluate_reconstruction_readiness", - ); - assert.equal( - readinessTool.annotations?.readOnlyHint, - false, - "readiness Evidence retention side effect must stay visible", - ); - const javascriptMcp = await analyzeMcp(client, javascriptPath); const electronMcp = await analyzeMcp(client, electronPath); assert.equal( @@ -107,46 +97,21 @@ try { electronCliEvidence, ); - const inputPath = join(temporaryRoot, "input.json"); - await writeFile(inputPath, JSON.stringify(input)); - const cliEvidence = await runCli([ - "evaluate-reconstruction-readiness", - inputPath, - "--json", - ]); - const cliReport = reconstructionReadinessReportSchema.parse( - cliEvidence.normalized_result, + const readinessReport = reconstructionReadinessReportSchema.parse( + createReconstructionReadinessReport(input), ); - assert.equal(cliReport.status, "pass"); + assert.equal(readinessReport.status, "pass"); if (process.env.REA_READINESS_REPORT_PATH !== undefined) await writeFile( resolve(root, process.env.REA_READINESS_REPORT_PATH), - `${JSON.stringify(cliReport, null, 2)}\n`, + `${JSON.stringify(readinessReport, null, 2)}\n`, ); - const mcpResult = await client.callTool({ - name: "evaluate_reconstruction_readiness", - arguments: input, - }); - assert.notEqual(mcpResult.isError, true); - const projection = requireObject(mcpResult.structuredContent); - const projectedReport = requireObject(projection.result); - assert.equal(projectedReport.report_digest, cliReport.report_digest); - assert.equal(projectedReport.status, "pass"); - assert.equal(projectedReport.report_digest, cliReport.report_digest); - const tampered = structuredClone(input); - tampered.replay.expected_source_digest = cliReport.source_digest; + tampered.replay.expected_source_digest = readinessReport.source_digest; tampered.identity.server_version = "tampered"; - const tamperedPath = join(temporaryRoot, "tampered.json"); - await writeFile(tamperedPath, JSON.stringify(tampered)); - const tamperedEvidence = await runCli([ - "evaluate-reconstruction-readiness", - tamperedPath, - "--json", - ]); const tamperedReport = reconstructionReadinessReportSchema.parse( - tamperedEvidence.normalized_result, + createReconstructionReadinessReport(tampered), ); assert.equal(tamperedReport.status, "fail"); assert.ok( @@ -157,11 +122,11 @@ try { process.stdout.write( `${JSON.stringify({ - status: cliReport.status, - report_id: cliReport.report_id, - report_digest: cliReport.report_digest, - stages: cliReport.summary.stages, - findings: cliReport.summary.findings, + status: readinessReport.status, + report_id: readinessReport.report_id, + report_digest: readinessReport.report_digest, + stages: readinessReport.summary.stages, + findings: readinessReport.summary.findings, native_evidence_id: nativeEvidence.evidence_id, javascript_evidence_id: javascriptCliEvidence.evidence_id, electron_evidence_id: electronCliEvidence.evidence_id, @@ -196,12 +161,6 @@ async function analyzeMcp(client_, inputPath) { return { evidence_id: projected.evidence_id }; } -function requireTool(tools, name) { - const tool = tools.find(({ name: toolName }) => toolName === name); - if (tool === undefined) throw new Error(`Missing public MCP tool: ${name}`); - return tool; -} - function requireObject(value) { if (typeof value !== "object" || value === null || Array.isArray(value)) throw new TypeError("Expected an object"); diff --git a/src/application/BrowserEvidence.ts b/src/application/BrowserEvidence.ts index 433442104..81ec19efc 100644 --- a/src/application/BrowserEvidence.ts +++ b/src/application/BrowserEvidence.ts @@ -93,7 +93,11 @@ const browserParameters = ( input: BrowserEvidenceInput, ): EvidenceObservation["parameters"] => { if (!("cdp_endpoint" in input)) { - if ("before_scenario" in input) + if ( + "before_scenario" in input && + input.before_scenario !== undefined && + input.after_scenario !== undefined + ) return { comparison_kind: "browser_scenario", before_browser: input.before_scenario.browser, @@ -102,10 +106,14 @@ const browserParameters = ( after_start_origin: input.after_scenario.scenario.start_origin, max_changes: input.max_changes, normalization_sha256: commitBrowserScenarioNormalization( - input.normalization, + input.normalization ?? { rules: [] }, ).sha256, }; - if ("max_changes" in input) + if ( + "max_changes" in input && + input.before !== undefined && + input.after !== undefined + ) return { before_target_id: input.before.inspection.target.target_id, before_capture_ended_at: @@ -114,12 +122,14 @@ const browserParameters = ( after_capture_ended_at: input.after.inspection.capture_window.ended_at, max_changes: input.max_changes, }; - return { - before_artifact_uri: input.before.uri, - after_artifact_uri: input.after.uri, - channel_threshold: input.channel_threshold, - maximum_pixels: input.maximum_pixels, - }; + if (isScreenshotComparison(input)) + return { + before_artifact_uri: input.before.uri, + after_artifact_uri: input.after.uri, + channel_threshold: input.channel_threshold, + maximum_pixels: input.maximum_pixels, + }; + throw new Error("Browser comparison input did not match a comparison type"); } const scope = { cdp_endpoint: input.cdp_endpoint, @@ -172,6 +182,10 @@ const browserParameters = ( }; }; +const isScreenshotComparison = ( + input: BrowserEvidenceInput, +): input is CompareWebScreenshotsInput => "channel_threshold" in input; + const browserPredicate = (operation: BrowserEvidenceOperation): string => { switch (operation) { case "list_browser_targets": diff --git a/src/application/ReconstructionReadinessService.ts b/src/application/ReconstructionReadinessService.ts deleted file mode 100644 index 3f38abfb1..000000000 --- a/src/application/ReconstructionReadinessService.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { z } from "zod"; - -import { - AnalysisInputError, - AnalysisProtocolError, - type AnalysisError, -} from "../domain/errors.js"; -import { createEvidence, type Evidence } from "../domain/evidence.js"; -import { jsonObjectSchema, jsonValueSchema } from "../domain/jsonValue.js"; -import { createReconstructionReadinessReport } from "../domain/reconstructionReadiness.js"; -import { - reconstructionReadinessInputSchema, - type ReconstructionReadinessInput, -} from "../domain/reconstructionReadinessSchemas.js"; -import { err, ok, type Result } from "../domain/result.js"; -import { JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER } from "./InvestigationProviders.js"; - -const OPERATION = "evaluate_reconstruction_readiness" as const; - -/** Parse one portable readiness request for both CLI and MCP adapters. */ -export const resolveReconstructionReadinessRequest = ( - input: unknown, -): Result => { - const parsed = reconstructionReadinessInputSchema.safeParse(input); - return parsed.success - ? ok(parsed.data) - : err(new AnalysisInputError(OPERATION, { cause: parsed.error })); -}; - -/** Evaluate and wrap one deterministic readiness report in Evidence. */ -export const evaluateReconstructionReadinessValidated = ( - input: ReconstructionReadinessInput, -): Result => { - try { - const report = createReconstructionReadinessReport(input); - return ok( - createEvidence(undefined, JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER, { - predicateType: "rea.reconstruction-readiness-report/v1", - operation: OPERATION, - parameters: jsonObjectSchema.parse({ - source_digest: report.source_digest, - expected_source_digest: input.replay.expected_source_digest, - }), - result: jsonValueSchema.parse(report), - rawResult: null, - confidence: "inferred", - authority: "analyst-inference", - environment: null, - limitations: - report.status === "pass" - ? [] - : [ - "Aggregate pass is withheld until every required conformance stage passes.", - ], - evidenceLinks: report.evidence_links, - }), - ); - } catch (cause: unknown) { - if (cause instanceof z.ZodError) - return err(new AnalysisInputError(OPERATION, { cause })); - return err( - new AnalysisProtocolError("Reconstruction readiness evaluation failed", { - cause, - }), - ); - } -}; diff --git a/src/cliApplicationCommands.ts b/src/cliApplicationCommands.ts index d4d99d173..ce6ed8cc4 100644 --- a/src/cliApplicationCommands.ts +++ b/src/cliApplicationCommands.ts @@ -29,10 +29,6 @@ import { buildReconstructionObligationLedgerEvidenceValidated, resolveReconstructionObligationLedgerRequest, } from "./application/ReconstructionObligationLedgerService.js"; -import { - evaluateReconstructionReadinessValidated, - resolveReconstructionReadinessRequest, -} from "./application/ReconstructionReadinessService.js"; import { authorizeRootPermission } from "./application/DeferredFileAuthorization.js"; import { loadConfiguredPermissionAuthority } from "./application/PermissionConfiguration.js"; import { CLI_COMMANDS } from "./cliCommandNames.js"; @@ -151,7 +147,6 @@ export const registerApplicationCommands = ( executeNodeCharacterization(replayDependencies(config, authority), input), }); registerObligationLedgerCommand(cli, logger); - registerReadinessCommand(cli, logger); registerCoverageCommand(cli, logger); }; @@ -169,17 +164,6 @@ const registerObligationLedgerCommand = ( workflow: buildReconstructionObligationLedgerEvidenceValidated, }); -const registerReadinessCommand = (cli: CliInstance, logger: Logger): void => - registerJsonCommand({ - cli, - logger, - name: CLI_COMMANDS.evaluateReconstructionReadiness, - description: - "Evaluate a complete Evidence-backed reconstruction readiness journey", - resolveInput: resolveReconstructionReadinessRequest, - workflow: evaluateReconstructionReadinessValidated, - }); - interface AuthorizedJsonCommandOptions { readonly cli: CliInstance; readonly logger: Logger; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index f2b0f09f3..7fda25fa9 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -16,7 +16,6 @@ export const CLI_COMMANDS = Object.freeze({ instructions: "instructions", search: "search", importReferenceSource: "import-reference-source", - inventoryArtifact: "inventory-artifact", inspectArtifact: "inspect-artifact", extractArtifact: "extract-artifact", inspectManagedArtifact: "inspect-managed-artifact", @@ -64,7 +63,6 @@ export const CLI_COMMANDS = Object.freeze({ prepareNodeCharacterization: "prepare-node-characterization", executeNodeCharacterization: "execute-node-characterization", buildReconstructionObligationLedger: "build-reconstruction-obligation-ledger", - evaluateReconstructionReadiness: "evaluate-reconstruction-readiness", evaluateReconstructionCoverage: "evaluate-reconstruction-coverage", }); diff --git a/src/contracts/applicationToolContracts.ts b/src/contracts/applicationToolContracts.ts index 5554a587b..ea05275f7 100644 --- a/src/contracts/applicationToolContracts.ts +++ b/src/contracts/applicationToolContracts.ts @@ -26,10 +26,6 @@ import { reconstructionObligationLedgerInputSchema, reconstructionObligationLedgerSchema, } from "../domain/reconstructionObligationLedgerSchemas.js"; -import { - reconstructionReadinessInputSchema, - reconstructionReadinessReportSchema, -} from "../domain/reconstructionReadinessSchemas.js"; import { jsonObjectSchema } from "../domain/jsonValue.js"; import type { ToolContract } from "./toolContracts.js"; import { toolContractMetadata } from "./toolEffects.js"; @@ -40,7 +36,6 @@ import { JAVASCRIPT_FEATURE_TRACE_EXAMPLE, SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, } from "./javascriptApplicationWorkflowExamples.js"; -import { RECONSTRUCTION_READINESS_EXAMPLE } from "./reconstructionReadinessExample.js"; const traceOutputSchema = evidenceResultOf(applicationFeatureTraceResultSchema); const semanticTraceOutputSchema = evidenceResultOf( @@ -58,9 +53,6 @@ const exportShapeComparisonOutputSchema = evidenceResultOf( const reconstructionObligationLedgerOutputSchema = evidenceResultOf( reconstructionObligationLedgerSchema, ); -const reconstructionReadinessOutputSchema = evidenceResultOf( - reconstructionReadinessReportSchema, -); const HASH = "0".repeat(64); const NODE_PREPARATION_EXAMPLE = jsonObjectSchema.parse({ preparation_approved: true, @@ -116,7 +108,7 @@ export const APPLICATION_TOOL_CONTRACTS = [ name: "trace_javascript_semantics", ...toolContractMetadata("trace_javascript_semantics"), description: - "Trace static JavaScript data-flow, direct call/return, and closure relations from the inline Evidence returned by analyze_javascript_application. Dynamic or unsupported semantics remain explicit unknowns; static reachability never claims runtime execution.", + "Trace static JavaScript data-flow, direct call/return, and closure relations from application Evidence returned inline by analyze_javascript_application or referenced by its session Evidence ID. Dynamic or unsupported semantics remain explicit unknowns; static reachability never claims runtime execution.", kind: "application", inputSchema: traceJavaScriptSemanticsRequestSchema, outputSchema: semanticTraceOutputSchema, @@ -296,21 +288,6 @@ export const APPLICATION_TOOL_CONTRACTS = [ }, ], }, - { - name: "evaluate_reconstruction_readiness", - ...toolContractMetadata("evaluate_reconstruction_readiness"), - description: - "Evaluate the fixed nine-stage public reconstruction journey into a deterministic ReconstructionReadinessReport. Exact capability limits, provider routing, CLI/MCP status parity, bounded authority, partial-order comparison, contradictions, obligation closure, cleanup, and replay digests fail closed. Aggregate pass is emitted only when every required stage and check passes with attributable Evidence.", - kind: "application", - inputSchema: reconstructionReadinessInputSchema, - outputSchema: reconstructionReadinessOutputSchema, - examples: [ - { - title: "Evaluate the complete synthetic reconstruction journey", - input: RECONSTRUCTION_READINESS_EXAMPLE, - }, - ], - }, { name: "evaluate_reconstruction_coverage", ...toolContractMetadata("evaluate_reconstruction_coverage"), @@ -392,12 +369,9 @@ export function applicationToolContract( export function applicationToolContract( name: "build_reconstruction_obligation_ledger", ): (typeof APPLICATION_TOOL_CONTRACTS)[8]; -export function applicationToolContract( - name: "evaluate_reconstruction_readiness", -): (typeof APPLICATION_TOOL_CONTRACTS)[9]; export function applicationToolContract( name: "evaluate_reconstruction_coverage", -): (typeof APPLICATION_TOOL_CONTRACTS)[10]; +): (typeof APPLICATION_TOOL_CONTRACTS)[9]; export function applicationToolContract( name: (typeof APPLICATION_TOOL_CONTRACTS)[number]["name"], ): (typeof APPLICATION_TOOL_CONTRACTS)[number] { diff --git a/src/contracts/applicationWorkflowInputContracts.ts b/src/contracts/applicationWorkflowInputContracts.ts index e15a5a22b..cdc074ced 100644 --- a/src/contracts/applicationWorkflowInputContracts.ts +++ b/src/contracts/applicationWorkflowInputContracts.ts @@ -56,30 +56,44 @@ const compareApplicationVersionsFacts = { compareApplicationVersionsInputSchema.shape.unknown_registry_approved, } as const; -/** MCP/CLI comparison request accepting full Evidence or ledger references. */ -export const compareApplicationVersionsRequestSchema = z.union([ - z.strictObject({ - ...compareApplicationVersionsFacts, - left: evidenceSchema, - right: evidenceSchema, - }), - z.strictObject({ - ...compareApplicationVersionsFacts, - left: evidenceSchema, - right_evidence_id: evidenceIdSchema, - }), - z.strictObject({ - ...compareApplicationVersionsFacts, - left_evidence_id: evidenceIdSchema, - right: evidenceSchema, - }), - z.strictObject({ - ...compareApplicationVersionsFacts, - left_evidence_id: evidenceIdSchema, - right_evidence_id: evidenceIdSchema, - }), +const applicationEvidenceReferenceSchema = z.union([ + evidenceSchema, + evidenceIdSchema, ]); +/** MCP/CLI comparison request accepting inline Evidence or session Evidence IDs. */ +export const compareApplicationVersionsRequestSchema = z + .strictObject({ + ...compareApplicationVersionsFacts, + left: applicationEvidenceReferenceSchema + .optional() + .describe( + "Full application Evidence or an Evidence ID returned this session", + ), + right: applicationEvidenceReferenceSchema + .optional() + .describe( + "Full application Evidence or an Evidence ID returned this session", + ), + // Retain the original ID property names for existing callers. + left_evidence_id: evidenceIdSchema.optional(), + right_evidence_id: evidenceIdSchema.optional(), + }) + .superRefine((input, context) => { + for (const side of ["left", "right"] as const) { + const alias = `${side}_evidence_id` as const; + const hasValue = input[side] !== undefined; + const hasAlias = input[alias] !== undefined; + if (hasValue === hasAlias) { + context.addIssue({ + code: "custom", + path: [side], + message: `Provide exactly one of ${side} or ${alias}`, + }); + } + } + }); + const compareSourceToBundleFacts = { reference: compareSourceToBundleInputSchema.shape.reference, limits: compareSourceToBundleInputSchema.shape.limits, diff --git a/src/contracts/browserToolContracts.ts b/src/contracts/browserToolContracts.ts index a77bac1ab..164f562d4 100644 --- a/src/contracts/browserToolContracts.ts +++ b/src/contracts/browserToolContracts.ts @@ -277,7 +277,7 @@ export const BROWSER_TOOL_CONTRACTS = [ name: "compare_web_captures", ...toolContractMetadata("compare_web_captures"), description: - "Compare either passive web captures or step-indexed browser scenarios. Scenario comparison aligns exact step IDs, records deterministic literal normalization, and exposes bounded artifact-level changes plus alignment failures. Missing or truncated evidence never proves equality.", + "Compare passive web captures by providing before and after, or compare recorded scenarios by providing before_scenario and after_scenario with an optional normalization policy. Use exactly one input group. Scenario comparison aligns exact step IDs, records deterministic literal normalization, and exposes bounded artifact-level changes plus alignment failures. Missing or truncated evidence never proves equality.", kind: "browser-provider", inputSchema: browserCaptureComparisonInputSchema, outputSchema: captureDiffOutputSchema, diff --git a/src/contracts/contractSnapshot.test.ts b/src/contracts/contractSnapshot.test.ts index 0ad00737a..5467472a2 100644 --- a/src/contracts/contractSnapshot.test.ts +++ b/src/contracts/contractSnapshot.test.ts @@ -59,6 +59,46 @@ const describesObject = (schema: ReturnType) => ); describe("tool contract surface", () => { + it("advertises capture comparison alternatives on an object root", () => { + const contract = BROWSER_TOOL_CONTRACTS.find( + ({ name }) => name === "compare_web_captures", + ); + if (contract === undefined) + throw new Error("compare_web_captures contract is missing"); + + const schema = contractJsonSchema(contract.inputSchema); + expect(schema.type).toBe("object"); + expect(schema.oneOf).toBeUndefined(); + expect(Object.keys(schema.properties ?? {})).toEqual( + expect.arrayContaining([ + "before", + "after", + "before_scenario", + "after_scenario", + "normalization", + "max_changes", + ]), + ); + const example = contract.examples[0]; + if (example === undefined) + throw new Error("compare_web_captures example is missing"); + if ( + !("before_scenario" in example.input) || + !("after_scenario" in example.input) + ) + throw new Error("compare_web_captures example is not a scenario pair"); + const withoutNormalization = Object.fromEntries( + Object.entries(example.input).filter(([key]) => key !== "normalization"), + ); + expect(contract.inputSchema.safeParse(withoutNormalization).success).toBe( + true, + ); + const incomplete = Object.fromEntries( + Object.entries(example.input).filter(([key]) => key !== "after_scenario"), + ); + expect(contract.inputSchema.safeParse(incomplete).success).toBe(false); + }); + it("advertises complete typed schemas and annotations for every public tool", () => { const contracts = TOOL_CONTRACTS; for (const contract of contracts) { diff --git a/src/contracts/javascriptApplicationWorkflowExamples.ts b/src/contracts/javascriptApplicationWorkflowExamples.ts index e533219b3..14e036909 100644 --- a/src/contracts/javascriptApplicationWorkflowExamples.ts +++ b/src/contracts/javascriptApplicationWorkflowExamples.ts @@ -63,8 +63,8 @@ export const JAVASCRIPT_FEATURE_TRACE_EXAMPLE = { /** Natural version comparison using two producer-returned Evidence IDs. */ export const JAVASCRIPT_APPLICATION_VERSION_COMPARISON_EXAMPLE = { - left_evidence_id: EXAMPLE_EVIDENCE_ID, - right_evidence_id: SECOND_EXAMPLE_EVIDENCE_ID, + left: EXAMPLE_EVIDENCE_ID, + right: SECOND_EXAMPLE_EVIDENCE_ID, }; /** Historical source inventory compared with one producer-returned graph ID. */ diff --git a/src/contracts/promptContracts.test.ts b/src/contracts/promptContracts.test.ts index 022ccacb0..089cafb88 100644 --- a/src/contracts/promptContracts.test.ts +++ b/src/contracts/promptContracts.test.ts @@ -91,7 +91,7 @@ describe("guided prompt contracts", () => { expect( rendered .get("compare_application_versions") - ?.indexOf("`inventory_artifact`"), + ?.indexOf("`inspect_artifact`"), ).toBeLessThan( rendered .get("compare_application_versions") diff --git a/src/contracts/promptContracts.ts b/src/contracts/promptContracts.ts index 86f3cc30c..c011c5afc 100644 --- a/src/contracts/promptContracts.ts +++ b/src/contracts/promptContracts.ts @@ -184,9 +184,9 @@ export const PROMPT_CONTRACTS = [ }, steps: [ { - tools: ["open_binary", "inventory_artifact"], + tools: ["open_binary", "inspect_artifact"], instruction: - "For each supplied target path, open that target before inventory because inventory_artifact uses the active target and accepts no path. Follow nodes, occurrences, and edges to completion and retain every Evidence page for each manifest.", + "For each supplied target path, open that target before inspection because inspect_artifact uses the active target and accepts no path. Retain the returned graph, substep Evidence, integrity findings, and next probes for each manifest.", }, { tools: [ diff --git a/src/contracts/toolContracts.test.ts b/src/contracts/toolContracts.test.ts index db8b626bc..6370e187a 100644 --- a/src/contracts/toolContracts.test.ts +++ b/src/contracts/toolContracts.test.ts @@ -19,7 +19,6 @@ describe("tool contract inventory", () => { expect(ENHANCED_TOOL_CONTRACTS).toHaveLength(13); expect(NATIVE_TOOL_CONTRACTS).toHaveLength(5); expect(ARTIFACT_TOOL_CONTRACTS.map(({ name }) => name)).toEqual([ - "inventory_artifact", "inspect_artifact", "extract_artifact", ]); @@ -62,7 +61,6 @@ describe("tool contract inventory", () => { "prepare_node_characterization", "execute_node_characterization", "build_reconstruction_obligation_ledger", - "evaluate_reconstruction_readiness", "evaluate_reconstruction_coverage", ]); expect(new Set(TOOL_CONTRACTS.map(({ name }) => name)).size).toBe( diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index 5a6d00067..f52b3b373 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -407,7 +407,7 @@ export const SESSION_TOOL_CONTRACTS = [ ), session( "compare_artifacts", - "Compare complete inventory_artifact Evidence sets by logical occurrence path, content identity, metadata, and graph relations. Every delta cites both sets, and gaps yield truncated or unknown, never equivalence. Returns every change inline.", + "Compare complete artifact inventories by logical occurrence path, content identity, metadata, and graph relations. Use the inventory Evidence nested in each inspect_artifact result. Every delta cites both inspections, and gaps yield truncated or unknown, never equivalence. Returns every change inline.", artifactComparisonInputSchema, ), session( diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 8849ec195..c4a6737dc 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -98,7 +98,6 @@ export const TOOL_EFFECTS: Readonly> = { inspect_plist: nativeEvidence, list_architectures: nativeEvidence, demangle_swift: nativeEvidence, - inventory_artifact: evidence, inspect_artifact: evidence, extract_artifact: effects({ mutatesSession: true, writesFilesystem: true }), inspect_managed_artifact: evidence, @@ -149,7 +148,6 @@ export const TOOL_EFFECTS: Readonly> = { compare_source_to_bundle: evidence, compare_javascript_export_shapes: evidence, build_reconstruction_obligation_ledger: evidence, - evaluate_reconstruction_readiness: evidence, run_replay_machine: effects(), run_controlled_replay: effects({ mutatesSession: true, diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index a4a53be69..ed85b7dc4 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -19,10 +19,7 @@ import { inspectSignatureSchema, listArchitecturesSchema, } from "../domain/nativeInspection.js"; -import { - artifactExtractionResultSchema, - artifactInventoryResultSchema, -} from "../domain/artifactGraph.js"; +import { artifactExtractionResultSchema } from "../domain/artifactGraph.js"; import { artifactInspectionResultSchema } from "../domain/artifactInspection.js"; import { managedArtifactInspectionSchema, @@ -296,7 +293,6 @@ export const nativeOutputSchemas: Readonly> = { /** Exact Evidence v2 schemas for provider-neutral artifact graph operations. */ export const artifactOutputSchemas: Readonly> = { - inventory_artifact: resultOf(artifactInventoryResultSchema), inspect_artifact: resultOf(artifactInspectionResultSchema), extract_artifact: resultOf(artifactExtractionResultSchema), }; diff --git a/src/contracts/workflowInputContracts.test.ts b/src/contracts/workflowInputContracts.test.ts index 13861db88..83b469443 100644 --- a/src/contracts/workflowInputContracts.test.ts +++ b/src/contracts/workflowInputContracts.test.ts @@ -1,10 +1,7 @@ import { describe, expect, it } from "vitest"; import { traceJavaScriptSemanticsRequestSchema } from "./applicationWorkflowInputContracts.js"; -import { - artifactInspectionInputSchema, - artifactInventoryInputSchema, -} from "./artifactToolContracts.js"; +import { ARTIFACT_TOOL_CONTRACTS } from "./artifactToolContracts.js"; import { JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE } from "./javascriptApplicationWorkflowExamples.js"; import { MANAGED_WORKFLOW_TOOL_CONTRACTS, @@ -89,7 +86,10 @@ describe("workflow input contracts", () => { integrity_policy: "record-and-continue" as const, integrity_continue_approved: false, }; - expect(artifactInventoryInputSchema.safeParse(input).success).toBe(false); - expect(artifactInspectionInputSchema.safeParse(input).success).toBe(false); + const inspect = ARTIFACT_TOOL_CONTRACTS.find( + ({ name }) => name === "inspect_artifact", + ); + if (inspect === undefined) throw new Error("Missing inspect_artifact"); + expect(inspect.inputSchema.safeParse(input).success).toBe(false); }); }); diff --git a/src/domain/browserCaptureComparison.ts b/src/domain/browserCaptureComparison.ts index 52affa89b..ca2a79745 100644 --- a/src/domain/browserCaptureComparison.ts +++ b/src/domain/browserCaptureComparison.ts @@ -5,12 +5,12 @@ import { compareBrowserScenarios, compareBrowserScenariosInputSchema, type BrowserScenarioDiff, - type CompareBrowserScenariosInput, } from "./browserScenarioDiff.js"; import { compareWebCaptures, captureSnapshotSchema, webCaptureDiffSchema, + type CompareWebCapturesInput, type WebCaptureDiff, } from "./webCaptureDiff.js"; @@ -22,34 +22,61 @@ const maxChangesSchema = z .default(2_000) .describe("Maximum normalized change records to retain."); -/** Top-level MCP shape for exactly one complete passive or scenario pair. */ -export const browserCaptureComparisonInputSchema = z.union([ - z.strictObject({ - before: captureSnapshotSchema.describe("Earlier passive web-page capture."), - after: captureSnapshotSchema.describe("Later passive web-page capture."), +/** Caller-visible object schema for passive and scenario capture comparisons. */ +export const browserCaptureComparisonInputSchema = z + .strictObject({ + before: captureSnapshotSchema + .optional() + .describe("Earlier passive web-page capture."), + after: captureSnapshotSchema + .optional() + .describe("Later passive web-page capture."), + before_scenario: compareBrowserScenariosInputSchema.shape.before_scenario + .optional() + .describe("Earlier browser scenario capture."), + after_scenario: compareBrowserScenariosInputSchema.shape.after_scenario + .optional() + .describe("Later browser scenario capture."), + normalization: compareBrowserScenariosInputSchema.shape.normalization + .unwrap() + .optional() + .describe("Exact-literal normalization policy."), max_changes: maxChangesSchema, - }), - z.strictObject({ - before_scenario: - compareBrowserScenariosInputSchema.shape.before_scenario.describe( - "Earlier browser scenario capture.", - ), - after_scenario: - compareBrowserScenariosInputSchema.shape.after_scenario.describe( - "Later browser scenario capture.", - ), - normalization: - compareBrowserScenariosInputSchema.shape.normalization.describe( - "Exact-literal normalization policy.", - ), - max_changes: maxChangesSchema, - }), -]); + }) + .superRefine((input, context) => { + const passive = input.before !== undefined && input.after !== undefined; + const scenario = + input.before_scenario !== undefined && input.after_scenario !== undefined; + const passiveFieldsPresent = + input.before !== undefined || input.after !== undefined; + const scenarioFieldsPresent = + input.before_scenario !== undefined || + input.after_scenario !== undefined || + input.normalization !== undefined; + + if (passive && !scenarioFieldsPresent) return; + if (scenario && !passiveFieldsPresent) return; + + context.addIssue({ + code: "custom", + message: + "Provide before and after passive captures, or before_scenario, after_scenario, and normalization; do not mix comparison types.", + }); + }); /** Parsed browser capture comparison input. */ export type BrowserCaptureComparisonInput = z.output< typeof browserCaptureComparisonInputSchema >; +type ScenarioCapturePair = Pick< + BrowserCaptureComparisonInput, + "before_scenario" | "after_scenario" | "normalization" | "max_changes" +> & { + before_scenario: NonNullable< + BrowserCaptureComparisonInput["before_scenario"] + >; + after_scenario: NonNullable; +}; /** Result from passive page or browser scenario comparison. */ export const browserCaptureComparisonSchema = z.union([ @@ -64,9 +91,25 @@ export const compareBrowserCaptures = ( input: BrowserCaptureComparisonInput, ): BrowserCaptureComparison => isScenarioComparison(input) - ? compareBrowserScenarios(input) - : compareWebCaptures(input); + ? compareBrowserScenarios({ + ...input, + normalization: input.normalization ?? { rules: [] }, + }) + : compareWebCaptures(toPassiveComparison(input)); const isScenarioComparison = ( input: BrowserCaptureComparisonInput, -): input is CompareBrowserScenariosInput => "before_scenario" in input; +): input is ScenarioCapturePair => + input.before_scenario !== undefined && input.after_scenario !== undefined; + +const toPassiveComparison = ( + input: BrowserCaptureComparisonInput, +): CompareWebCapturesInput => { + if (input.before === undefined || input.after === undefined) + throw new Error("Validated passive comparison is missing a capture"); + return { + before: input.before, + after: input.after, + max_changes: input.max_changes, + }; +}; diff --git a/src/domain/reconstructionReadiness.test.ts b/src/domain/reconstructionReadiness.test.ts index 644fde8db..d6a7d18e7 100644 --- a/src/domain/reconstructionReadiness.test.ts +++ b/src/domain/reconstructionReadiness.test.ts @@ -12,7 +12,7 @@ const input = ( }); describe("reconstruction readiness report", () => { - it("passes the complete public-contract journey deterministically", () => { + it("passes the complete repository conformance journey deterministically", () => { const first = createReconstructionReadinessReport(input()); const second = createReconstructionReadinessReport(input()); diff --git a/src/domain/reconstructionUnknowns.ts b/src/domain/reconstructionUnknowns.ts index 99215f18c..2cfeb9b4a 100644 --- a/src/domain/reconstructionUnknowns.ts +++ b/src/domain/reconstructionUnknowns.ts @@ -125,7 +125,7 @@ const defaultProbe = ( rationale: "Capture complete function dossiers under equal limits.", } : { - operation: "inventory_artifact", + operation: "inspect_artifact", rationale: "Capture complete artifact inventories under equal limits.", }; diff --git a/src/server/createServer.ts b/src/server/createServer.ts index f91e43440..cab2eaded 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -55,7 +55,7 @@ import { } from "./sessionToolPolicies.js"; const TARGET_FREE_INSTRUCTIONS = - "ASAR/JavaScript -> analyze_javascript_application; archive/package -> open_binary(path), then inspect_artifact/inventory_artifact; managed PE/CLI -> inspect_managed_artifact; browser/Electron -> list_browser_targets/list_electron_targets; capture_browser_scenario/capture_electron_scenario; Node/Electron Inspector -> list_javascript_runtime_targets; native binary/database -> open_binary, then binary_overview. Start with binary_session; use tools/list; capabilities via binary_session."; + "Start with binary_session to see the active target and available providers. For an archive or application package, call open_binary(path), then inspect_artifact for the complete graph and findings; use extract_artifact only for selected files. For standalone JavaScript or ASAR analysis, use analyze_javascript_application. For managed PE/CLI files, use inspect_managed_artifact. For native binaries, open the target and start with binary_overview. For a live page or runtime, discover targets with list_browser_targets, list_electron_targets, or list_javascript_runtime_targets before inspection; scenario capture tools are for controlled interactions."; const ACTIVE_TARGET_INSTRUCTIONS = "REA analyzes the active reverse-engineering target. Start native analysis with binary_overview, then use analyze_function, literal search, callers, callees, and xrefs as needed."; diff --git a/src/server/registerApplicationTools.ts b/src/server/registerApplicationTools.ts index 62b8b14c6..75e2b9935 100644 --- a/src/server/registerApplicationTools.ts +++ b/src/server/registerApplicationTools.ts @@ -9,7 +9,6 @@ import { registerControlledReplayTool } from "./registerApplicationTools/control import { registerCharacterizationTools } from "./registerApplicationTools/characterization.js"; import { registerCoverageTools } from "./registerApplicationTools/coverage.js"; import { registerReconstructionObligationLedgerTool } from "./registerApplicationTools/obligationLedger.js"; -import { registerReconstructionReadinessTool } from "./registerApplicationTools/readiness.js"; import type { ApplicationToolRegistration } from "./registerApplicationTools/types.js"; export type { ApplicationToolRegistration }; @@ -27,6 +26,5 @@ export const registerApplicationTools = ( registerControlledReplayTool(server, options); registerCharacterizationTools(server, options); registerReconstructionObligationLedgerTool(server, options); - registerReconstructionReadinessTool(server, options); registerCoverageTools(server, options); }; diff --git a/src/server/registerApplicationTools/readiness.ts b/src/server/registerApplicationTools/readiness.ts deleted file mode 100644 index 88674a72d..000000000 --- a/src/server/registerApplicationTools/readiness.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; -import type { McpServer } from "@modelcontextprotocol/server"; - -import { - evaluateReconstructionReadinessValidated, - resolveReconstructionReadinessRequest, -} from "../../application/ReconstructionReadinessService.js"; -import { applicationToolContract } from "../../contracts/applicationToolContracts.js"; -import { logToolExecution } from "../toolLogging.js"; -import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; -import { toCallToolResult } from "../toolResult.js"; - -import type { ApplicationToolRegistration } from "./types.js"; - -const contract = applicationToolContract("evaluate_reconstruction_readiness"); - -/** Register deterministic public-contract reconstruction conformance. */ -export const registerReconstructionReadinessTool = ( - server: McpServer, - options: ApplicationToolRegistration, -): void => { - server.registerTool( - contract.name, - toolRegistrationOptions(contract), - async (input) => { - const resolved = resolveReconstructionReadinessRequest(input); - if (!resolved.ok) return toCallToolResult(resolved, contract); - const result = await logToolExecution(options.logger, contract.name, () => - Promise.resolve( - evaluateReconstructionReadinessValidated(resolved.value), - ), - ); - if (!result.ok) return toCallToolResult(result, contract); - const recorded = recordSessionEvidenceSources( - options.recordEvidence, - resolved.value.evidence_bundle.records, - ); - if (!recorded.ok) return toCallToolResult(recorded, contract); - const recordedResult = options.recordEvidence?.(result.value); - if (recordedResult !== undefined && !recordedResult.ok) - return toCallToolResult(recordedResult, contract); - return toCallToolResult({ ok: true, value: result.value }, contract); - }, - ); -}; diff --git a/src/server/registerEvidenceTools.ts b/src/server/registerEvidenceTools.ts index b7eb49a5a..aba370c3f 100644 --- a/src/server/registerEvidenceTools.ts +++ b/src/server/registerEvidenceTools.ts @@ -137,7 +137,7 @@ const permissionRequest = ( operation_identity: `extract_artifact:${parameters.output_root}`, }; if ( - ["inventory_artifact", "inspect_artifact"].includes(operation) && + operation === "inspect_artifact" && parameters.native_mount_approved === true ) return { diff --git a/tests/acceptance/setup/packageInstallWorkflow.test.ts b/tests/acceptance/setup/packageInstallWorkflow.test.ts index 81c92eb50..9800251f1 100644 --- a/tests/acceptance/setup/packageInstallWorkflow.test.ts +++ b/tests/acceptance/setup/packageInstallWorkflow.test.ts @@ -1,97 +1,156 @@ import { readFile } from "node:fs/promises"; import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; import { packageHopperEnvironment } from "../../../scripts/verify-package-environment.mjs"; +const workflowStepSchema = z + .object({ + run: z.unknown().optional(), + name: z.unknown().optional(), + env: z.record(z.string(), z.unknown()).optional(), + }) + .passthrough(); +const workflowSchema = z + .object({ + jobs: z.record( + z.string(), + z + .object({ + steps: z.array(workflowStepSchema).default([]), + needs: z.union([z.array(z.string()), z.string()]).optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +type Workflow = z.output; + describe("package installation workflows", () => { it("runs package E2E without the retired native rebuild script", async () => { - const continuousIntegration = await readFile( - new URL("../../../.github/workflows/ci.yml", import.meta.url), - "utf8", - ); - const realHopperLinux = await readFile( - new URL( - "../../../.github/workflows/real-hopper-linux.yml", - import.meta.url, + const continuousIntegration = await readWorkflow("ci.yml"); + const realHopperLinux = await readWorkflow("real-hopper-linux.yml"); + const realHopperMac = await readWorkflow("real-hopper.yml"); + + expect(workflowJob(continuousIntegration, "package-e2e")).toMatchObject({ + strategy: { matrix: { os: ["ubuntu-latest", "macos-14"] } }, + steps: expect.arrayContaining([{ run: "npm run verify:package" }]), + }); + expect( + workflowJob(continuousIntegration, "static").steps.map( + ({ run }: { readonly run?: unknown }) => run, ), - "utf8", - ); - const realHopperMac = await readFile( - new URL("../../../.github/workflows/real-hopper.yml", import.meta.url), - "utf8", + ).toContain("npm run check:ci"); + expect(workflowJob(continuousIntegration, "test-shard")).toMatchObject({ + strategy: { matrix: { shard: ["1/4", "2/4", "3/4", "4/4"] } }, + steps: expect.arrayContaining([ + { run: "npm run test:ci:shard:run -- --shard=${{ matrix.shard }}" }, + ]), + }); + const testJob = workflowJob(continuousIntegration, "test"); + expect(testJob.needs).toContain("test-shard"); + expect(testJob.steps).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "Require successful test shards", + env: expect.objectContaining({ + CODE_REQUIRED: "${{ needs.changes.outputs.code }}", + }), + }), + expect.objectContaining({ + run: expect.stringContaining("npm run test:ci:merge"), + }), + ]), ); - expect(continuousIntegration).toContain("os: [ubuntu-latest, macos-14]"); - expect(continuousIntegration).toContain("npm run verify:package"); - expect(continuousIntegration).toContain("name: Static checks"); - expect(continuousIntegration).toContain("npm run check:ci"); - expect(continuousIntegration).toContain("shard: [1/4, 2/4, 3/4, 4/4]"); - expect(continuousIntegration).toContain("name: Build package"); - expect(continuousIntegration).toContain("npm run build:cached"); - expect(continuousIntegration).toContain("name: rea-dist"); - expect(continuousIntegration).toContain("npm run test:ci:shard:run"); - expect(continuousIntegration).toContain("needs: [changes, test-shard]"); - expect(continuousIntegration).toContain("if-no-files-found: error"); - expect(continuousIntegration).toContain("overwrite: true"); - expect(continuousIntegration).toContain( - "code: ${{ steps.package.outputs.required }}", - ); - expect(continuousIntegration).toContain("CODE_REQUIRED:"); - expect(continuousIntegration).toContain( - 'test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success"', - ); - expect(continuousIntegration).toContain("npm run test:ci:merge"); - expect(continuousIntegration).toContain("needs: changes"); - expect(continuousIntegration).toContain( - "cancel-in-progress: ${{ github.event_name == 'pull_request' }}", - ); + for (const workflow of [realHopperLinux, realHopperMac]) { + const verifyJob = workflowJob(workflow, "verify"); + const verificationStep = verifyJob.steps.find( + ({ run }: { readonly run?: unknown }) => + typeof run === "string" && run.includes("npm run verify:hopper"), + ); + expect( + verifyJob.steps.some( + ({ run }: { readonly run?: unknown }) => + typeof run === "string" && + run.includes("npm install --global --ignore-scripts"), + ), + ).toBe(true); + expect(verificationStep).toBeDefined(); + expect(verificationStep?.env).toMatchObject({ + REA_HOPPER_CONFORMANCE_MANIFEST_PATH: + "${{ github.workspace }}/build/conformance/manifest.json", + }); + } + for (const workflow of [ continuousIntegration, realHopperLinux, realHopperMac, - ]) - expect(workflow).not.toContain("npm run rebuild:native"); - for (const workflow of [realHopperLinux, realHopperMac]) - expect(workflow).toContain("npm install --global --ignore-scripts"); - for (const workflow of [realHopperLinux, realHopperMac]) - expect(workflow).toContain( - "REA_HOPPER_CONFORMANCE_MANIFEST_PATH: ${{ github.workspace }}/build/conformance/manifest.json", + ]) { + const commands = Object.values(workflow.jobs).flatMap((job) => + job.steps.flatMap(({ run }: { readonly run?: unknown }) => + typeof run === "string" ? [run] : [], + ), ); - expect(realHopperLinux).not.toContain("HOPPER_TARGET_PATH"); - expect(realHopperLinux).not.toContain("HOPPER_SECOND_TARGET_PATH"); - expect(realHopperLinux).not.toContain("first_target"); - expect(realHopperLinux).not.toContain("second_target"); + expect(commands.join("\n")).not.toContain("npm run rebuild:native"); + } }); it("verifies the package before publish and runs published canaries outside the checkout", async () => { - const release = await readFile( - new URL("../../../.github/workflows/release.yml", import.meta.url), - "utf8", - ); + const release = await readWorkflow("release.yml"); const canary = await readFile( new URL("../../../scripts/verify-published-package.mjs", import.meta.url), "utf8", ); - expect(release.indexOf("npm run verify:package")).toBeLessThan( - release.indexOf("npm publish --access public"), + const publishJob = workflowJob(release, "publish"); + const publishSteps = publishJob.steps; + const verifyPackageIndex = publishSteps.findIndex( + ({ run }: { readonly run?: unknown }) => run === "npm run verify:package", + ); + const npmPublishIndex = publishSteps.findIndex( + ({ run }: { readonly run?: unknown }) => + run === "npm publish --access public", + ); + expect(verifyPackageIndex).toBeGreaterThanOrEqual(0); + expect(npmPublishIndex).toBeGreaterThan(verifyPackageIndex); + + const publishedCanary = publishSteps.find( + ({ name }: { readonly name?: unknown }) => + name === "Verify published CLI from npm", + ); + expect(publishedCanary?.run).toContain('verification_root="$(mktemp -d)"'); + expect(publishedCanary?.run).toContain('cd "${verification_root}"'); + expect(publishedCanary?.run).not.toContain( + 'npm run verify:published -- "${version}"', ); - expect(release).toContain('verification_root="$(mktemp -d)"'); - expect(release).toContain('cd "${verification_root}"'); - expect(release).not.toContain('npm run verify:published -- "${version}"'); expect(canary).toContain('mkdtemp(join(tmpdir(), "rea-published-canary-")'); expect(canary).toContain("cwd: canaryRoot"); - expect(release).toContain("publish-mcp:"); - expect(release).toContain("needs: [release-please, publish]"); - expect(release).toContain("MCP_PUBLISHER_VERSION: v1.8.0"); - expect(release).toContain("mcp-publisher validate"); - expect(release).toContain("mcp-publisher login github-oidc"); - expect(release).toContain("mcp-publisher publish"); - expect(release).toContain("id-token: write"); - expect(release).toContain("contents: read"); - expect(release).not.toContain("MCP_GITHUB_TOKEN"); - expect(release.indexOf("publish-mcp:")).toBeGreaterThan( - release.indexOf("npm publish --access public"), + const publisherJob = workflowJob(release, "publish-mcp"); + const publisherSteps = publisherJob.steps; + expect(publisherJob).toMatchObject({ + needs: ["release-please", "publish"], + permissions: { contents: "read", "id-token": "write" }, + }); + expect( + publisherSteps.map(({ run }: { readonly run?: unknown }) => run), + ).toEqual( + expect.arrayContaining([ + "./mcp-publisher validate", + "./mcp-publisher login github-oidc", + "./mcp-publisher publish", + ]), ); + expect( + publisherSteps.find( + ({ name }: { readonly name?: unknown }) => + name === "Install MCP Registry publisher", + )?.env, + ).toMatchObject({ + MCP_PUBLISHER_VERSION: "v1.8.0", + MCP_PUBLISHER_SHA256_LINUX_AMD64: expect.any(String), + }); }); it("uses direct Node ownership on macOS and Windows", () => { @@ -117,3 +176,17 @@ describe("package installation workflows", () => { }); }); }); + +async function readWorkflow(path: string) { + const yaml = await readFile( + new URL(`../../../.github/workflows/${path}`, import.meta.url), + "utf8", + ); + return workflowSchema.parse(parse(yaml)); +} + +const workflowJob = (workflow: Workflow, name: string) => { + const value = workflow.jobs[name]; + if (value === undefined) throw new Error(`Missing workflow job: ${name}`); + return value; +}; diff --git a/tests/boundary/cli/productCatalog.test.ts b/tests/boundary/cli/productCatalog.test.ts index eba4351e4..7ea16363b 100644 --- a/tests/boundary/cli/productCatalog.test.ts +++ b/tests/boundary/cli/productCatalog.test.ts @@ -126,7 +126,6 @@ describe("canonical product catalog", () => { "compare_javascript_export_shapes", "compare_source_to_bundle", "evaluate_reconstruction_coverage", - "evaluate_reconstruction_readiness", "execute_node_characterization", "prepare_node_characterization", "run_controlled_replay", diff --git a/tests/boundary/cli/reconstructionReadinessCli.test.ts b/tests/boundary/cli/reconstructionReadinessCli.test.ts deleted file mode 100644 index 6654eaf7b..000000000 --- a/tests/boundary/cli/reconstructionReadinessCli.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; - -import { describe, expect, it } from "vitest"; - -import { RECONSTRUCTION_READINESS_EXAMPLE } from "../../../src/contracts/reconstructionReadinessExample.js"; - -const execute = promisify(execFile); - -describe("reconstruction readiness CLI parity", () => { - it("emits the same passing report through the public CLI", async () => { - const { stdout } = await execute( - process.execPath, - [ - "scripts/rea.mjs", - "evaluate-reconstruction-readiness", - JSON.stringify(RECONSTRUCTION_READINESS_EXAMPLE), - "--json", - ], - { - cwd: process.cwd(), - env: process.env, - maxBuffer: 32 * 1_024 * 1_024, - }, - ); - expect(JSON.parse(stdout)).toMatchObject({ - operation: "evaluate_reconstruction_readiness", - predicate_type: "rea.reconstruction-readiness-report/v1", - normalized_result: { - schema: "ReconstructionReadinessReport", - status: "pass", - summary: { passed_required_stages: 9 }, - }, - }); - }, 20_000); -}); diff --git a/tests/boundary/mcp/browserMcp.test.ts b/tests/boundary/mcp/browserMcp.test.ts index afb352c86..38f502da9 100644 --- a/tests/boundary/mcp/browserMcp.test.ts +++ b/tests/boundary/mcp/browserMcp.test.ts @@ -27,7 +27,7 @@ afterEach(async () => { }); it( - "exposes CLI-equivalent Evidence v2 and session resources", + "exposes CLI-equivalent Evidence v2 results and session tools", async () => { const browser = await startFakeCdpBrowser({ sessionTimeline: "same_origin", diff --git a/tests/boundary/mcp/promptCompletion.test.ts b/tests/boundary/mcp/promptCompletion.test.ts index 3fadadc4b..3d511f520 100644 --- a/tests/boundary/mcp/promptCompletion.test.ts +++ b/tests/boundary/mcp/promptCompletion.test.ts @@ -9,6 +9,8 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js"; import { PROCESS_CAPTURE_REFERENCE } from "../../../src/contracts/investigationExamples.js"; import { createEvidence } from "../../../src/domain/evidence.js"; +import { createArtifactInspection } from "../../../src/domain/artifactInspection.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { createPromptCompletionSource } from "../../../src/server/promptCompletion.js"; import { observed } from "../../fixtures/analysisExecution.js"; @@ -71,8 +73,20 @@ describe("guided prompt completion from investigation records", () => { confidence: "observed", authority: "shipped-artifact", }); + const inventoryInspection = createArtifactInspection( + ARTIFACT_COMPARISON_EXAMPLE.left, + ); + const inspectionEvidence = createEvidence(undefined, fixtureProvider, { + operation: "inspect_artifact", + parameters: {}, + result: jsonValueSchema.parse(inventoryInspection), + confidence: "observed", + authority: "shipped-artifact", + evidenceLinks: inventoryInspection.evidence_links, + }); for (const evidence of [ ARTIFACT_COMPARISON_EXAMPLE.left, + inspectionEvidence, PROCESS_CAPTURE_REFERENCE, invalidCapture, invalidInventory, diff --git a/tests/boundary/mcp/reconstructionReadinessMcp.test.ts b/tests/boundary/mcp/reconstructionReadinessMcp.test.ts deleted file mode 100644 index e319aee49..000000000 --- a/tests/boundary/mcp/reconstructionReadinessMcp.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { describe, expect, it } from "vitest"; - -import { createTestBinarySession } from "../../fixtures/binarySession.js"; -import { RECONSTRUCTION_READINESS_EXAMPLE } from "../../../src/contracts/reconstructionReadinessExample.js"; -import { createServer } from "../../../src/server/createServer.js"; -import { observed } from "../../fixtures/analysisExecution.js"; - -describe("reconstruction readiness MCP parity", () => { - it("returns the complete passing report inline", async () => { - const session = createTestBinarySession(() => ({ - execute: () => Promise.resolve(observed(null)), - close: () => Promise.resolve(), - })); - const server = createServer(session, session); - const client = new Client({ name: "readiness-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const result = await client.callTool({ - name: "evaluate_reconstruction_readiness", - arguments: RECONSTRUCTION_READINESS_EXAMPLE, - }); - expect(result.isError).not.toBe(true); - expect(result.structuredContent).toMatchObject({ - evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u), - result: { - schema: "ReconstructionReadinessReport", - status: "pass", - summary: { passed_required_stages: 9 }, - stages: expect.arrayContaining([expect.any(Object)]), - }, - }); - expect(result.structuredContent).toHaveProperty("result.snapshot"); - expect(result.structuredContent).toHaveProperty("result.stages"); - expect(result.structuredContent).not.toHaveProperty( - "result.report_resource_uri", - ); - } finally { - await client.close(); - await server.close(); - await session.close(); - } - }); -}); diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index e89b9ab90..1b782e6bf 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -241,12 +241,11 @@ describe("live server identity over MCP", () => { const assertLiveIdentity = async (client: Client): Promise => { const instructions = client.getInstructions(); - expect(instructions?.length).toBeLessThanOrEqual(640); expect(instructions).toContain( - "ASAR/JavaScript -> analyze_javascript_application", + "For standalone JavaScript or ASAR analysis, use analyze_javascript_application", ); expect(instructions).toContain( - "native binary/database -> open_binary, then binary_overview", + "For an archive or application package, call open_binary(path), then inspect_artifact", ); const identity = await client.callTool({ name: "binary_session", diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index db6d5e52d..ae53dd615 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -116,14 +116,12 @@ describe("target-free MCP workflow", () => { const beforeTools = (await mcp.listTools()).tools; const beforeNames = beforeTools.map(({ name }) => name); expect(mcp.getInstructions()).toContain( - "archive/package -> open_binary(path), then inspect_artifact/inventory_artifact", + "For an archive or application package, call open_binary(path), then inspect_artifact", ); expect( - beforeTools.find(({ name }) => name === "inventory_artifact") - ?.description, - ).toContain( - "This tool accepts no path; in a target-free session open the target first.", - ); + beforeTools.find(({ name }) => name === "inspect_artifact")?.description, + ).toContain("Returns the complete content-addressed artifact graph"); + expect(beforeNames).not.toContain("inventory_artifact"); expect(beforeNames).toContain("open_binary"); expect(beforeNames).toContain("binary_session"); expect(beforeNames).toContain("current_document"); @@ -132,14 +130,22 @@ describe("target-free MCP workflow", () => { (await mcp.callTool({ name: "open_binary", arguments: { path: first } })) .isError, ).not.toBe(true); + const openedStatus = structured( + await mcp.callTool({ name: "binary_session", arguments: {} }), + ); + expect(openedStatus).toMatchObject({ + result: { path: await realpath(first) }, + }); expect( - structured( - await mcp.callTool({ - name: "binary_session", - arguments: {}, - }), - ), - ).toMatchObject({ result: { path: await realpath(first) } }); + z + .object({ + result: z.object({ + capabilities: z.array(z.object({ operation: z.string() })), + }), + }) + .parse(openedStatus) + .result.capabilities.map(({ operation }) => operation), + ).not.toContain("inventory_artifact"); expect( text(await mcp.callTool({ name: "current_document", arguments: {} })), ).toContain("first.hop"); diff --git a/tests/conformance/README.md b/tests/conformance/README.md index 308731336..d8296e6ea 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -29,7 +29,7 @@ fixtures provide metadata-oriented inputs without package or network dependencies. `readiness/` adds source-only generated JavaScript CLI and Electron fixtures for -the public reconstruction-readiness journey. `npm run verify:readiness` combines -them with the compiled native fixtures, exercises CLI and MCP parity, and emits -the versioned machine-readable readiness report documented in -`docs/reconstruction-readiness.md`. +the repository's reconstruction-readiness verifier. `npm run verify:readiness` +combines them with the compiled native fixtures, exercises CLI and MCP parity +for real analysis operations, and checks the internal machine-readable report +documented in `docs/reconstruction-readiness.md`. diff --git a/tests/conformance/providers/conformanceFixtures.test.ts b/tests/conformance/providers/conformanceFixtures.test.ts index ab27398f0..bfe584354 100644 --- a/tests/conformance/providers/conformanceFixtures.test.ts +++ b/tests/conformance/providers/conformanceFixtures.test.ts @@ -1,41 +1,25 @@ -import { readFile } from "node:fs/promises"; - import { describe, expect, it } from "vitest"; import { generateLargeFixture, - HOPPER_C_ORACLE, LARGE_FIXTURE_COUNT, sha256, sourceDigest, } from "../../../scripts/lib/conformance-fixtures.mjs"; describe("source-built conformance fixtures", () => { - it("grounds the Hopper semantic oracle in the source-owned C fixture", async () => { - const source = await readFile("tests/conformance/c/fixture.c", "utf8"); - for (const procedure of [ - HOPPER_C_ORACLE.mainProcedure, - HOPPER_C_ORACLE.entryProcedure, - HOPPER_C_ORACLE.branchProcedure, - HOPPER_C_ORACLE.leafProcedure, - ]) - expect(source).toContain(`int ${procedure}`); - expect(source).toContain(HOPPER_C_ORACLE.entryString); - expect(source).toContain(HOPPER_C_ORACLE.leafString); - expect(source).toContain(HOPPER_C_ORACLE.globalName); - }); - - it("generates a deterministic large complete-inventory fixture", () => { + it("generates deterministic large-fixture source", () => { const first = generateLargeFixture(); const second = generateLargeFixture(); expect(first).toBe(second); - expect(first.match(/int rea_inventory_/gu)).toHaveLength( - LARGE_FIXTURE_COUNT, - ); expect(first).toContain("rea_inventory_0000"); - expect(first).toContain("rea_inventory_1204"); - expect(() => generateLargeFixture(0)).toThrow(/positive integer/u); + expect(first).toContain( + `rea_inventory_${String(LARGE_FIXTURE_COUNT - 1).padStart(4, "0")}`, + ); + expect(() => generateLargeFixture(0)).toThrow( + "Fixture count must be a positive integer", + ); }); it("hashes source manifests independently of input ordering", () => { diff --git a/tests/process-global/cliLogging.test.ts b/tests/process-global/cliLogging.test.ts index d703c7fcb..20c2209eb 100644 --- a/tests/process-global/cliLogging.test.ts +++ b/tests/process-global/cliLogging.test.ts @@ -19,7 +19,7 @@ describe("CLI operation status", () => { }; await expect( - logCliCommand(silentLogger, "inventory-artifact", () => + logCliCommand(silentLogger, "inspect-artifact", () => Promise.resolve(output), ), ).resolves.toBe(output);