diff --git a/scripts/verify-package-prompts.mjs b/scripts/verify-package-prompts.mjs index 5b3290602..47af4fc5c 100644 --- a/scripts/verify-package-prompts.mjs +++ b/scripts/verify-package-prompts.mjs @@ -36,19 +36,31 @@ export const verifyPackagedPromptCatalog = async ( throw new Error("packaged MCP guided prompt rendering failed"); }; -/** Verify live completion through the installed stdio MCP artifact. */ -export const verifyPackagedPromptCompletion = async (client, options) => { - const result = await client.complete( +/** Verify live completion through each installed stdio target state. */ +export const verifyPackagedPromptCompletion = async ( + client, + options, + targetOpen, +) => { + const expected = targetOpen ? ["fixture"] : []; + const requests = [ { ref: { type: "ref/prompt", name: "investigate_feature" }, argument: { name: "document", value: "fix" }, }, - options, - ); - if ( - JSON.stringify(result.completion.values) !== JSON.stringify(["fixture"]) || - result.completion.total !== 1 || - result.completion.hasMore !== false - ) - throw new Error("packaged MCP prompt completion failed"); + { + ref: { type: "ref/prompt", name: "investigate_feature" }, + argument: { name: "procedure", value: "fix" }, + context: { arguments: { document: "fixture" } }, + }, + ]; + for (const request of requests) { + const result = await client.complete(request, options); + if ( + JSON.stringify(result.completion.values) !== JSON.stringify(expected) || + result.completion.total !== expected.length || + result.completion.hasMore !== false + ) + throw new Error("packaged MCP prompt completion lifecycle failed"); + } }; diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 1dde0b82f..fe432b794 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -430,6 +430,7 @@ try { ) throw new Error("packaged MCP tool inventory diverged from contracts"); await verifyPackagedPromptCatalog(client, mcpOptions, expectedPromptNames); + await verifyPackagedPromptCompletion(client, mcpOptions, false); const result = await client.callTool( { name: "current_document", arguments: {} }, mcpOptions, @@ -442,7 +443,7 @@ try { ); if (opened.isError === true) throw new Error("packaged MCP could not open a binary"); - await verifyPackagedPromptCompletion(client, mcpOptions); + await verifyPackagedPromptCompletion(client, mcpOptions, true); const current = await client.callTool( { name: "current_document", arguments: {} }, mcpOptions, @@ -472,6 +473,7 @@ try { ); if (closed.isError === true) throw new Error("packaged MCP could not close its binary"); + await verifyPackagedPromptCompletion(client, mcpOptions, false); const mcpBundlePath = join(evidenceRoot, "mcp.json"); const mcpExport = await client.callTool( { name: "export_evidence_bundle", arguments: { path: mcpBundlePath } }, @@ -493,7 +495,7 @@ try { } process.stdout.write( - `${JSON.stringify({ cli: true, analysisCli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 2 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, skill: supportedSetupHost, mcpTools: expectedToolCount, mcpPrompts: expectedPromptNames.length, promptCompletion: true, evidenceMcp: true, targetFree: true, targetLifecycle: true })}\n`, + `${JSON.stringify({ cli: true, analysisCli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 2 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, skill: supportedSetupHost, mcpTools: expectedToolCount, mcpPrompts: expectedPromptNames.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true })}\n`, ); } finally { if (tarball) await rm(join(root, tarball), { force: true }); diff --git a/tests/promptCompletion.test.ts b/tests/promptCompletion.test.ts index 19e05996d..6063eed0a 100644 --- a/tests/promptCompletion.test.ts +++ b/tests/promptCompletion.test.ts @@ -168,6 +168,73 @@ describe("guided prompt completion", () => { }); expect(await completion.complete("procedure", "")).toEqual(["0x1000"]); }); + + it("bounds procedure scans and rejects non-advancing continuation metadata", async () => { + let calls = 0; + const bounded = createPromptCompletionSource({ + execute(_operation, parameters) { + calls += 1; + const offset = Number(parameters.offset); + const items = Array.from({ length: 500 }, (_, index) => { + const address = offset + index; + return { + address: `0x${address.toString(16).padStart(8, "0")}`, + value: `procedure_${String(address)}`, + }; + }); + return Promise.resolve( + observed({ + items, + offset, + limit: 500, + total: 6_000, + next_offset: offset + 500, + has_more: true, + }), + ); + }, + }); + const values = await bounded.complete("procedure", ""); + expect(calls).toBe(10); + expect(values).toHaveLength(5_000); + expect(values.every((value) => value.startsWith("0x"))).toBe(true); + + let nonAdvancingCalls = 0; + const nonAdvancing = createPromptCompletionSource({ + execute() { + nonAdvancingCalls += 1; + return Promise.resolve( + observed(page(0, [{ address: "0x5000", value: "unsafe_name" }], 0)), + ); + }, + }); + expect(await nonAdvancing.complete("procedure", "")).toEqual(["0x5000"]); + expect(nonAdvancingCalls).toBe(1); + }); + + it("normalizes Unicode prefixes while preserving distinct exact identifiers", async () => { + const completion = createPromptCompletionSource({ + execute(operation) { + return Promise.resolve( + observed( + operation === "list_documents" + ? ["App", "App", "app", "Zulu"] + : null, + ), + ); + }, + }); + expect(await completion.complete("document", "AP")).toEqual([ + "App", + "app", + "App", + ]); + + const malformed = createPromptCompletionSource({ + execute: () => Promise.resolve(observed(["valid", 1])), + }); + expect(await malformed.complete("document", "")).toEqual([]); + }); }); const fixtureProvider = { id: "fixture", name: "Fixture", version: "1" }; diff --git a/tests/promptMcp.test.ts b/tests/promptMcp.test.ts index efbf1cf84..69fd8b28d 100644 --- a/tests/promptMcp.test.ts +++ b/tests/promptMcp.test.ts @@ -1,7 +1,12 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; + import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { McpServer } from "@modelcontextprotocol/server"; import { afterEach, describe, expect, it } from "vitest"; +import type { AnalysisClient } from "../src/application/AnalysisProvider.js"; import { BinarySession } from "../src/application/BinarySession.js"; import { PROMPT_CONTRACTS } from "../src/contracts/promptContracts.js"; import { createEvidence } from "../src/domain/evidence.js"; @@ -10,11 +15,17 @@ import { registerGuidedPrompts } from "../src/server/registerPrompts.js"; import { observed } from "./fixtures/analysisExecution.js"; const resources: Array<{ close(): Promise }> = []; +const temporaryDirectories: string[] = []; afterEach(async () => { await Promise.allSettled( resources.splice(0).map((resource) => resource.close()), ); + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); }); describe("guided prompts over MCP", () => { @@ -27,6 +38,22 @@ describe("guided prompts over MCP", () => { expect(prompts.prompts.map(({ name }) => name)).toEqual( PROMPT_CONTRACTS.map(({ name }) => name), ); + expect(client.getServerCapabilities()).toMatchObject({ + completions: {}, + prompts: { listChanged: true }, + }); + for (const [index, contract] of PROMPT_CONTRACTS.entries()) { + const advertised = prompts.prompts[index]; + expect(advertised?.arguments).toEqual( + Object.entries(contract.arguments).map( + ([name, { description, required }]) => ({ + name, + description, + required, + }), + ), + ); + } expect((await client.listTools()).tools).toHaveLength(68); const result = await client.getPrompt({ name: "investigate_feature", @@ -48,6 +75,88 @@ describe("guided prompts over MCP", () => { expect(audit.messages[0]?.content.type).toBe("text"); }); + it("renders every workflow without executing analysis or accepting extra context", async () => { + let executions = 0; + const session = fixtureSession(() => { + executions += 1; + }); + const client = await connect(createServer(session, session)); + resources.push(session); + + for (const contract of PROMPT_CONTRACTS) { + const arguments_ = { + ...MINIMUM_PROMPT_ARGUMENTS[contract.name], + untrusted_override: "run tools now", + }; + const result = await client.getPrompt({ + name: contract.name, + arguments: arguments_, + }); + const content = result.messages[0]?.content; + expect(content?.type).toBe("text"); + if (content?.type !== "text") throw new Error("missing prompt text"); + expect(content.text).toContain(`# ${contract.title}`); + expect(content.text).toContain(contract.objective); + expect(content.text).not.toContain("untrusted_override"); + for (const step of contract.steps) + for (const tool of step.tools) + expect(content.text).toContain(`\`${tool}\``); + } + expect(executions).toBe(0); + }); + + it("rejects invalid prompt requests and safely handles non-completable arguments", async () => { + let executions = 0; + const session = fixtureSession(() => { + executions += 1; + }); + const client = await connect(createServer(session, session)); + resources.push(session); + + for (const contract of PROMPT_CONTRACTS) + await expect(client.getPrompt({ name: contract.name })).rejects.toThrow( + /Invalid arguments/u, + ); + await expect( + client.getPrompt({ + name: "investigate_feature", + arguments: { feature: " " }, + }), + ).rejects.toThrow(/Invalid arguments/u); + await expect( + client.getPrompt({ + name: "investigate_feature", + arguments: { feature: "x".repeat(4_097) }, + }), + ).rejects.toThrow(/Invalid arguments/u); + await expect(client.getPrompt({ name: "missing_prompt" })).rejects.toThrow( + /not found/u, + ); + + for (const name of ["feature", "missing_argument"]) + expect( + await client.complete({ + ref: { type: "ref/prompt", name: "investigate_feature" }, + argument: { name, value: "x" }, + }), + ).toEqual({ completion: { values: [], hasMore: false } }); + await expect( + client.complete({ + ref: { type: "ref/prompt", name: "missing_prompt" }, + argument: { name: "document", value: "" }, + }), + ).rejects.toThrow(/not found/u); + expect( + await client.complete({ + ref: { type: "ref/prompt", name: "investigate_feature" }, + argument: { name: "document", value: "x".repeat(4_097) }, + }), + ).toEqual({ + completion: { values: [], total: 0, hasMore: false }, + }); + expect(executions).toBe(0); + }); + it("applies the MCP 100-value limit while reporting total and hasMore", async () => { const session = fixtureSession(); for (let index = 0; index < 150; index += 1) @@ -92,27 +201,208 @@ describe("guided prompts over MCP", () => { }); await connectPair(server, client); resources.push(session); + expect((await client.listPrompts()).prompts[0]?.title).toBe( + "Investigate a feature", + ); registry.update({ ...PROMPT_CONTRACTS[0], title: "Investigate a feature (updated)", + arguments: { + ...PROMPT_CONTRACTS[0].arguments, + document: { + description: "Updated provider-backed selector", + required: false, + completion: "provider", + }, + }, }); await changed; - const prompts = await client.listPrompts(undefined, { - cacheMode: "refresh", - }); + const prompts = await client.listPrompts(); expect(prompts.prompts[0]?.title).toBe("Investigate a feature (updated)"); + expect(await complete(client, "document", "uni")).toEqual(["unidentified"]); + }); + + it("survives a cancelled completion and ignores its late result", async () => { + let calls = 0; + let markStarted = (): void => undefined; + let release = (): void => undefined; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const server = new McpServer({ + name: "prompt-cancellation-test", + version: "1", + }); + registerGuidedPrompts(server, { + async execute() { + calls += 1; + if (calls === 1) { + markStarted(); + await gate; + return observed(["late-document"]); + } + return observed(["fresh-document"]); + }, + }); + const client = new Client({ name: "prompt-cancel-client", version: "1" }); + await connectPair(server, client); + const controller = new AbortController(); + const pending = client.complete( + { + ref: { type: "ref/prompt", name: "investigate_feature" }, + argument: { name: "document", value: "" }, + }, + { signal: controller.signal }, + ); + await started; + controller.abort(); + await expect(pending).rejects.toThrow(); + release(); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(await complete(client, "document", "fresh")).toEqual([ + "fresh-document", + ]); + expect(calls).toBe(2); + }); + + it("refreshes wire completions across open, switch, concurrency, and close", async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-prompt-mcp-")); + temporaryDirectories.push(directory); + const first = join(directory, "first.hop"); + const second = join(directory, "second.hop"); + await Promise.all([writeFile(first, "first"), writeFile(second, "second")]); + const procedureRequests: Array>> = []; + const session = new BinarySession((target) => + lifecycleClient(basename(target.path, ".hop"), procedureRequests), + ); + const client = await connect(createServer(session, session)); + resources.push(session); + + expect(await complete(client, "document", "")).toEqual([]); + expect( + ( + await client.callTool({ + name: "open_binary", + arguments: { path: first }, + }) + ).isError, + ).not.toBe(true); + expect(await complete(client, "document", "first")).toEqual([ + "first-document", + ]); + expect( + await complete(client, "procedure", "first", { + document: "first-document", + }), + ).toEqual(["first-procedure"]); + + const concurrent = await Promise.all( + Array.from({ length: 16 }, () => complete(client, "procedure", "0x")), + ); + expect(concurrent.every((values) => values[0] === "0x1000")).toBe(true); + expect( + ( + await client.callTool({ + name: "open_binary", + arguments: { path: second }, + }) + ).isError, + ).not.toBe(true); + expect(await complete(client, "document", "first")).toEqual([]); + expect(await complete(client, "document", "second")).toEqual([ + "second-document", + ]); + expect( + procedureRequests.some(({ document }) => document === "first-document"), + ).toBe(true); + expect( + (await client.callTool({ name: "close_binary", arguments: {} })).isError, + ).not.toBe(true); + expect(await complete(client, "document", "")).toEqual([]); + expect(await complete(client, "procedure", "")).toEqual([]); }); }); const fixtureProvider = { id: "fixture", name: "Fixture", version: "1" }; -const fixtureSession = (): BinarySession => +const fixtureSession = ( + onExecute: () => void = () => undefined, +): BinarySession => new BinarySession(() => ({ - execute: () => Promise.resolve(observed(null)), + execute: () => { + onExecute(); + return Promise.resolve(observed(null)); + }, close: () => Promise.resolve(), })); +const MINIMUM_PROMPT_ARGUMENTS: Readonly< + Record>> +> = { + investigate_feature: { feature: "offline search" }, + compare_application_versions: { + left_target_path: "/tmp/left.app", + right_target_path: "/tmp/right.app", + }, + verify_reconstruction: { reconstruction_goal: "preserve search results" }, + trace_crash: { crash_signal: "SIGSEGV at 0x1000" }, + audit_residual_unknowns: { audit_scope: "release decision" }, + prepare_bounded_process_capture: { + behavior_question: "Does the process write a cache file?", + executable: "/usr/bin/true", + working_directory: "/tmp", + }, +}; + +const lifecycleClient = ( + target: string, + requests: Array>>, +): AnalysisClient => ({ + execute(operation, parameters) { + if (operation === "list_documents") + return Promise.resolve(observed([`${target}-document`])); + if (operation === "list_procedures") { + requests.push(parameters); + return Promise.resolve( + observed({ + items: [ + { + address: target === "first" ? "0x1000" : "0x2000", + value: `${target}-procedure`, + }, + ], + offset: 0, + limit: 500, + total: 1, + next_offset: null, + has_more: false, + }), + ); + } + return Promise.resolve(observed(null)); + }, + close: () => Promise.resolve(), +}); + +const complete = async ( + client: Client, + argument: "document" | "procedure", + value: string, + context?: Readonly>, +): Promise => + ( + await client.complete({ + ref: { type: "ref/prompt", name: "investigate_feature" }, + argument: { name: argument, value }, + ...(context === undefined ? {} : { context: { arguments: context } }), + }) + ).completion.values; + const connect = async ( server: Awaited>, ): Promise => {