import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; import { exec } from "./lib/verify-package-core.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; const client = process.argv[2]; const mode = process.argv[3] ?? "call"; assert(["pi", "hermes"].includes(client), "Select pi or hermes."); assert(["chat", "call"].includes(mode), "Select chat or call."); assert( process.platform !== "win32", "This real-client lane requires POSIX; native Windows is unverified.", ); const repo = fileURLToPath(new URL("..", import.meta.url)); const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; const command = process.env[`REA_VERIFY_${client.toUpperCase()}_COMMAND`] ?? client; const lab = await mkdtemp(join(tmpdir(), `rea-${client}-client-`)); const account = join(lab, "account"); const profile = join(lab, "selected profile"); const stickyHermes = client === "hermes" && process.env.REA_VERIFY_HERMES_STICKY_PROFILE === "1"; const selectedProfile = stickyHermes ? join(profile, "profiles", "coder") : profile; const workspace = join(lab, "workspace"); const target = join(workspace, "fixture α测试"); const artifactPath = join(workspace, "evidence.json"); const skillPath = join( client === "pi" ? join(account, ".agents") : selectedProfile, "skills", "reverse-engineer-anything", "SKILL.md", ); const verifier = createVerifierRun(); await Promise.all([ mkdir(account), mkdir(profile, { recursive: true }), ...(stickyHermes ? [mkdir(selectedProfile, { recursive: true })] : []), mkdir(target, { recursive: true }), ]); await writeFile( join(target, "app.js"), 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); const environment = { PATH: process.env.PATH ?? "", HOME: account, USERPROFILE: account, [client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile, PI_OFFLINE: "1", REA_PROCESS_RUN_ID: verifier.run_id, }; const configPath = join( selectedProfile, client === "pi" ? "mcp.json" : "config.yaml", ); if (stickyHermes) await writeFile(join(profile, "active_profile"), "coder\n"); const original = client === "pi" ? JSON.stringify({ mcpServers: { other: { command: "unrelated-server", enabled: false } }, }) : "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n"; await writeFile(configPath, original); const clientVersion = ( await exec(command, ["--version"], { env: environment, timeout: 30_000 }) ).stdout.trim(); for (const args of [ ["setup", "--client", client, "--dry-run"], ["setup", "--client", client, "--yes"], ["doctor", "--client", client, "--skill"], ]) { const result = await exec( process.execPath, [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], { env: environment, cwd: workspace, timeout: 60_000 }, ); const parsed = JSON.parse(result.stdout); if (args[0] === "doctor") assert.equal(parsed.healthy, true); else assert.equal( parsed.status, args.includes("--dry-run") ? "planned" : "ready", ); await writeFile( join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), result.stdout, ); } assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); const installedConfiguration = await readFile(configPath, "utf8"); assert(installedConfiguration.includes("unrelated-server")); if (client === "hermes") assert(installedConfiguration.includes("# Keep caller settings.")); const repeated = await exec( process.execPath, [ join(runtimeRoot, "scripts/rea.mjs"), "setup", "--client", client, "--yes", "--json", ], { env: environment, cwd: workspace, timeout: 60_000 }, ); assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); assert.equal(await readFile(configPath, "utf8"), installedConfiguration); assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); assert( (await readFile(skillPath, "utf8")).includes( "name: reverse-engineer-anything", ), ); let evidenceSeen = false; let skillLoaded = false; let skillRequested = false; let callRequested = false; let searchRequested = false; let describeRequested = false; let finalSeen = false; let catalog; let artifactDigest; const extractEvidence = (value) => { if (typeof value === "string") { if ( value.startsWith("") ) { // Hermes wraps external MCP content with its own instruction banner. // Decode only that observed envelope, retaining the original request artifact. const start = value.indexOf("\n\n"); if (start >= 0) return extractEvidence( value .slice(start + 2, value.lastIndexOf("")) .trim(), ); } try { return extractEvidence(JSON.parse(value)); } catch { return undefined; } } if (value && typeof value === "object") { if ( typeof value.evidence_id === "string" && value.normalized_result?.graph?.nodes ) return value; for (const child of Object.values(value)) { const found = extractEvidence(child); if (found) return found; } } }; const validateEvidence = async (value) => { const evidence = toolContract( "analyze_javascript_application", ).outputSchema.parse(value); assert.equal(evidence.subject.local_path, target); const shape = evidence.normalized_result.graph.nodes .flatMap((n) => n.observations) .map((o) => o.properties) .find( (p) => p.semantic_role === "export-return-shapes" && p.exported_name === "result", ); assert(shape?.return_shape_coverage?.projection_complete); const fields = shape.static_return_shapes.flatMap((s) => s.fields); assert( fields.some( (f) => f.path === "/clientCompatibility" && f.state === "literal" && f.value === "REA_CLIENT_FIXTURE", ), ); assert( fields.some( (f) => f.path === "/count" && f.state === "literal" && f.value === 7, ), ); const serialized = JSON.stringify(evidence); await writeFile(join(lab, "evidence.json"), serialized); artifactDigest = createHash("sha256").update(serialized).digest("hex"); evidenceSeen = true; }; const fixture = await createOpenAiModelFixture({ directory: lab, prefix: `${client}-${mode}`, onRequest: async ({ body, tools, requestIndex }) => { assert( requestIndex <= 6, "Client did not complete the bounded fixture workflow.", ); const messages = body.messages ?? []; const toolMessages = messages.filter((m) => m.role === "tool"); const serializedTools = JSON.stringify(toolMessages); const call = (name, args) => ({ role: "assistant", content: null, tool_calls: [ { id: `call_rea_${requestIndex}`, type: "function", function: { name, arguments: JSON.stringify(args) }, }, ], }); if ( client === "pi" && callRequested && serializedTools.includes("REA_CLIENT_FIXTURE") ) { const fact = toolMessages .flatMap((m) => { const text = typeof m.content === "string" ? m.content : JSON.stringify(m.content); const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true'); if (marker < 0) return []; const end = text.indexOf("}\n", marker); try { return [ JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)), ]; } catch { return []; } }) .find((v) => Array.isArray(v.catalog)); assert( fact, "Pi codemode must return its actual catalog and skill-read receipt.", ); catalog = fact.catalog; skillLoaded = fact.skill_loaded === true; await validateEvidence(JSON.parse(await readFile(artifactPath, "utf8"))); } else if (client === "hermes") { const search = tools.find((t) => t.name === "tool_search"); const exposed = tools .filter((t) => t.name.startsWith("mcp__rea__")) .map((t) => t.name.slice(10)); const deferred = [ ...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu), ].map((m) => m[1]); const native = [...new Set([...exposed, ...deferred])]; // Hermes advertises MCP prompt utilities beside server tools. if (native.length) catalog = native.filter( (name) => !["get_prompt", "list_prompts"].includes(name), ); skillLoaded ||= serializedTools.includes(skillPath) && serializedTools.includes("reverse-engineer-anything"); for (const message of toolMessages) { const evidence = extractEvidence(message.content); if (evidence) await validateEvidence(evidence); } } if (mode === "chat" || evidenceSeen) { finalSeen = true; return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; } if (client === "pi") { const tool = tools.find((t) => t.name === "codemode"); assert( tool, `Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`, ); assert( JSON.stringify(messages).includes(skillPath), "Pi must advertise the installed REA skill.", ); callRequested = true; return call(tool.name, { code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`, }); } if (!skillRequested) { const tool = tools.find((t) => t.name === "skill_view"); assert(tool, "Hermes skill_view must be available."); skillRequested = true; return call(tool.name, { name: "reverse-engineer-anything" }); } if (!searchRequested) { const tool = tools.find((t) => t.name === "tool_search"); assert(tool, "Hermes native deferred tool_search must be available."); searchRequested = true; return call(tool.name, { queries: ["analyze_javascript_application"] }); } assert( serializedTools.includes("mcp__rea__analyze_javascript_application"), "Hermes must discover the analysis tool.", ); if (!describeRequested) { const tool = tools.find((t) => t.name === "tool_describe"); assert(tool, "Hermes must load the actual parameter schema."); describeRequested = true; return call(tool.name, { names: ["mcp__rea__analyze_javascript_application"], }); } const tool = tools.find((t) => t.name === "tool_call"); assert(tool, "Hermes native deferred tool_call must be available."); callRequested = true; return call(tool.name, { calls: [ { name: "mcp__rea__analyze_javascript_application", arguments: { input_path: target, format: "directory" }, }, ], }); }, }); if (client === "pi") { await writeFile( join(profile, "models.json"), JSON.stringify({ providers: { "rea-fixture": { baseUrl: fixture.baseUrl, api: "openai-completions", apiKey: "local-fixture-key", models: [ { id: "rea-client-fixture", reasoning: false, contextWindow: 200000, maxTokens: 4096, }, ], }, }, }), ); } else { const configured = await readFile(configPath, "utf8"); await writeFile( configPath, configured + `\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`, ); } const args = client === "pi" ? [ "--offline", "--provider", "rea-fixture", "--model", "rea-client-fixture", "--no-session", "--no-context-files", "--mode", "json", "-p", mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`, ] : [ "--oneshot", mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`, "--provider", "custom", "--model", "rea-client-fixture", "--reasoning", "none", ]; let failure; try { const execution = exec(command, args, { env: { ...environment, OPENAI_API_KEY: "local-fixture-key", OPENAI_BASE_URL: fixture.baseUrl, }, cwd: workspace, timeout: 120_000, maxBuffer: 10 * 1024 * 1024, }); // Pi consumes piped input before its first prompt; an unused open pipe is not EOF. execution.child.stdin.end(); const result = await execution; await writeFile(join(lab, "client.stdout"), result.stdout); await writeFile(join(lab, "client.stderr"), result.stderr); assert.equal(fixture.failure, undefined); assert(finalSeen); assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); if (mode === "call") { assert(evidenceSeen); assert( skillLoaded, "Native client must actually read the installed skill.", ); assert.deepEqual( catalog?.sort(), TOOL_CONTRACTS.map((t) => t.name).sort(), "The complete REA catalog must survive native client discovery.", ); } } catch (cause) { failure = cause; if (typeof cause?.stdout === "string") await writeFile(join(lab, "client.stdout"), cause.stdout); if (typeof cause?.stderr === "string") await writeFile(join(lab, "client.stderr"), cause.stderr); } finally { await fixture.close(); const receipt = { client, clientVersion, mode, passed: failure === undefined, artifacts: lab, runtimeRoot, host: { platform: process.platform, architecture: process.arch, node: process.versions.node, }, modelFixture: "loopback deterministic OpenAI-compatible endpoint; no live provider claim", skillDiscovery: client === "pi" ? "native HOME shared personal directory; no explicit skill setting" : "native HERMES_HOME personal directory", stickyHermes, catalogSize: catalog?.length, evidenceSeen, skillLoaded, artifactDigest, finalSeen, requests: fixture.requests, probes: fixture.probes, fixtureFailure: fixture.failure, verifier: await completeVerifierRun(verifier), }; await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); console.log(JSON.stringify(receipt)); } if (failure !== undefined) throw failure;