From b25922e4e8fe389cf17c151ccb7b4be4a42824d9 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:59:32 +0800 Subject: [PATCH] test(clients): verify native Qwen Pi and Hermes workflows (#1511) --- docs/testing.md | 15 + package.json | 3 + scripts/verify-native-agent-client.mjs | 456 ++++++++++++++++++ scripts/verify-qwen-client.mjs | 403 ++++++++++++++++ scripts/verify/clients/model-fixture.mjs | 171 +++++++ .../setup}/hermesClientSetup.test.ts | 0 .../setup}/piClientSetup.test.ts | 0 .../acceptance/setup/qwenProfileSetup.test.ts | 96 ++++ 8 files changed, 1144 insertions(+) create mode 100644 scripts/verify-native-agent-client.mjs create mode 100644 scripts/verify-qwen-client.mjs create mode 100644 scripts/verify/clients/model-fixture.mjs rename tests/{boundary/cli => acceptance/setup}/hermesClientSetup.test.ts (100%) rename tests/{boundary/cli => acceptance/setup}/piClientSetup.test.ts (100%) create mode 100644 tests/acceptance/setup/qwenProfileSetup.test.ts diff --git a/docs/testing.md b/docs/testing.md index 9730f027b..82b529929 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -88,6 +88,21 @@ entrypoint and include its verifier/fixture files in their cache inputs. ## End-to-end, integration and golden evidence +The optional `verify:qwen-client`, `verify:pi-client`, and `verify:hermes-client` +lanes require an installed native client. Select its executable with +`REA_VERIFY_QWEN_COMMAND`, `REA_VERIFY_PI_COMMAND`, or `REA_VERIFY_HERMES_COMMAND`. +They configure disposable profiles through the public REA CLI, preserve caller +settings and backups, and exercise real stdio MCP calls and skill loading. The +`call` mode checks complete catalog discovery and actual JavaScript Evidence; +pass `-- chat` to check plain chat with REA enabled. Qwen also reads the full +result that its client offloads, and Pi exercises default codemode execution. +Use `REA_VERIFY_RUNTIME_ROOT` to select a production-only installed REA package. +Receipts retain client versions, result digests, host coverage and owned-process +cleanup. These POSIX lanes use a deterministic loopback model, so they do not +prove live model-provider or native Windows compatibility. Qwen and Pi use +caller-configured skill directories to isolate the fixture account from the OS +home; this does not prove their default home-directory discovery. + Full E2E tests invoke the production command dispatcher and real providers, without fake launchers, runners or responses. `verify:keyed-archive` writes an actual Foundation binary archive, runs the CLI and a separate stdio MCP diff --git a/package.json b/package.json index 439746c70..0c47e5901 100644 --- a/package.json +++ b/package.json @@ -179,6 +179,9 @@ "verify:readiness": "npm run build:cached && npm run verify:fixtures && node scripts/verify-reconstruction-readiness.mjs", "verify:package": "npm run build:cached && node scripts/verify-package.mjs", "verify:package:windows": "npm run build:cached && node scripts/verify-windows-package.mjs", + "verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs", + "verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi", + "verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes", "probe:windows-capabilities": "npm run build:cached && node scripts/windows-capabilities.mjs", "verify:published": "node scripts/verify-published-package.mjs", "config:print": "node scripts/print-mcp-config.mjs", diff --git a/scripts/verify-native-agent-client.mjs b/scripts/verify-native-agent-client.mjs new file mode 100644 index 000000000..f561e353e --- /dev/null +++ b/scripts/verify-native-agent-client.mjs @@ -0,0 +1,456 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../dist/contracts/toolContracts.js"; +import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; + +const client = process.argv[2]; +const mode = process.argv[3] ?? "call"; +assert(["pi", "hermes"].includes(client), "Select pi or hermes."); +assert(["chat", "call"].includes(mode), "Select chat or call."); +assert( + process.platform !== "win32", + "This real-client lane requires POSIX; native Windows is unverified.", +); +const repo = fileURLToPath(new URL("..", import.meta.url)); +const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; +const command = + process.env[`REA_VERIFY_${client.toUpperCase()}_COMMAND`] ?? client; +const lab = await mkdtemp(join(tmpdir(), `rea-${client}-client-`)); +const account = join(lab, "account"); +const profile = join(lab, "selected profile"); +const workspace = join(lab, "workspace"); +const target = join(workspace, "fixture α测试"); +const artifactPath = join(workspace, "evidence.json"); +const skillPath = join( + client === "pi" ? join(account, ".agents") : profile, + "skills", + "reverse-engineer-anything", + "SKILL.md", +); +const verifier = createVerifierRun(); +await Promise.all([ + mkdir(account), + mkdir(profile), + mkdir(target, { recursive: true }), +]); +await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', +); +const environment = { + PATH: process.env.PATH ?? "", + USERPROFILE: account, + [client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile, + PI_OFFLINE: "1", + REA_PROCESS_RUN_ID: verifier.run_id, +}; +const configPath = join(profile, client === "pi" ? "mcp.json" : "config.yaml"); +const original = + client === "pi" + ? JSON.stringify({ + mcpServers: { other: { command: "unrelated-server", enabled: false } }, + }) + : "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n"; +await writeFile(configPath, original); +if (client === "pi") { + // The CLI fixture account and native os.homedir() differ; use Pi's caller-owned skill setting. + await writeFile( + join(profile, "settings.json"), + JSON.stringify({ skills: [join(account, ".agents", "skills")] }), + ); +} +const clientVersion = ( + await exec(command, ["--version"], { env: environment, timeout: 30_000 }) +).stdout.trim(); +for (const args of [ + ["setup", "--client", client, "--dry-run"], + ["setup", "--client", client, "--yes"], + ["doctor", "--client", client, "--skill"], +]) { + const result = await exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000 }, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + await writeFile( + join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), + result.stdout, + ); +} +assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); +const installedConfiguration = await readFile(configPath, "utf8"); +assert(installedConfiguration.includes("unrelated-server")); +if (client === "hermes") + assert(installedConfiguration.includes("# Keep caller settings.")); +const repeated = await exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + client, + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000 }, +); +assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); +assert.equal(await readFile(configPath, "utf8"), installedConfiguration); +assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); +assert( + (await readFile(skillPath, "utf8")).includes( + "name: reverse-engineer-anything", + ), +); +let evidenceSeen = false; +let skillLoaded = false; +let skillRequested = false; +let callRequested = false; +let searchRequested = false; +let describeRequested = false; +let finalSeen = false; +let catalog; +let artifactDigest; +const extractEvidence = (value) => { + if (typeof value === "string") { + if ( + value.startsWith("") + ) { + // Hermes wraps external MCP content with its own instruction banner. + // Decode only that observed envelope, retaining the original request artifact. + const start = value.indexOf("\n\n"); + if (start >= 0) + return extractEvidence( + value + .slice(start + 2, value.lastIndexOf("")) + .trim(), + ); + } + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes + ) + return value; + for (const child of Object.values(value)) { + const found = extractEvidence(child); + if (found) return found; + } + } +}; +const validateEvidence = async (value) => { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(value); + assert.equal(evidence.subject.local_path, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const serialized = JSON.stringify(evidence); + await writeFile(join(lab, "evidence.json"), serialized); + artifactDigest = createHash("sha256").update(serialized).digest("hex"); + evidenceSeen = true; +}; +const fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: `${client}-${mode}`, + onRequest: async ({ body, tools, requestIndex }) => { + assert( + requestIndex <= 6, + "Client did not complete the bounded fixture workflow.", + ); + const messages = body.messages ?? []; + const toolMessages = messages.filter((m) => m.role === "tool"); + const serializedTools = JSON.stringify(toolMessages); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if ( + client === "pi" && + callRequested && + serializedTools.includes("REA_CLIENT_FIXTURE") + ) { + const fact = toolMessages + .flatMap((m) => { + const text = + typeof m.content === "string" + ? m.content + : JSON.stringify(m.content); + const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true'); + if (marker < 0) return []; + const end = text.indexOf("}\n", marker); + try { + return [ + JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)), + ]; + } catch { + return []; + } + }) + .find((v) => Array.isArray(v.catalog)); + assert( + fact, + "Pi codemode must return its actual catalog and skill-read receipt.", + ); + catalog = fact.catalog; + skillLoaded = fact.skill_loaded === true; + await validateEvidence(JSON.parse(await readFile(artifactPath, "utf8"))); + } else if (client === "hermes") { + const search = tools.find((t) => t.name === "tool_search"); + const exposed = tools + .filter((t) => t.name.startsWith("mcp__rea__")) + .map((t) => t.name.slice(10)); + const deferred = [ + ...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu), + ].map((m) => m[1]); + const native = [...new Set([...exposed, ...deferred])]; + // Hermes advertises MCP prompt utilities beside server tools. + if (native.length) + catalog = native.filter( + (name) => !["get_prompt", "list_prompts"].includes(name), + ); + skillLoaded ||= + serializedTools.includes(skillPath) && + serializedTools.includes("reverse-engineer-anything"); + for (const message of toolMessages) { + const evidence = extractEvidence(message.content); + if (evidence) await validateEvidence(evidence); + } + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + if (client === "pi") { + const tool = tools.find((t) => t.name === "codemode"); + assert( + tool, + `Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`, + ); + assert( + JSON.stringify(messages).includes(skillPath), + "Pi must advertise the installed REA skill.", + ); + callRequested = true; + return call(tool.name, { + code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`, + }); + } + if (!skillRequested) { + const tool = tools.find((t) => t.name === "skill_view"); + assert(tool, "Hermes skill_view must be available."); + skillRequested = true; + return call(tool.name, { name: "reverse-engineer-anything" }); + } + if (!searchRequested) { + const tool = tools.find((t) => t.name === "tool_search"); + assert(tool, "Hermes native deferred tool_search must be available."); + searchRequested = true; + return call(tool.name, { queries: ["analyze_javascript_application"] }); + } + assert( + serializedTools.includes("mcp__rea__analyze_javascript_application"), + "Hermes must discover the analysis tool.", + ); + if (!describeRequested) { + const tool = tools.find((t) => t.name === "tool_describe"); + assert(tool, "Hermes must load the actual parameter schema."); + describeRequested = true; + return call(tool.name, { + names: ["mcp__rea__analyze_javascript_application"], + }); + } + const tool = tools.find((t) => t.name === "tool_call"); + assert(tool, "Hermes native deferred tool_call must be available."); + callRequested = true; + return call(tool.name, { + calls: [ + { + name: "mcp__rea__analyze_javascript_application", + arguments: { input_path: target, format: "directory" }, + }, + ], + }); + }, +}); +if (client === "pi") { + await writeFile( + join(profile, "models.json"), + JSON.stringify({ + providers: { + "rea-fixture": { + baseUrl: fixture.baseUrl, + api: "openai-completions", + apiKey: "local-fixture-key", + models: [ + { + id: "rea-client-fixture", + reasoning: false, + contextWindow: 200000, + maxTokens: 4096, + }, + ], + }, + }, + }), + ); +} else { + const configured = await readFile(configPath, "utf8"); + await writeFile( + configPath, + configured + + `\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`, + ); +} +const args = + client === "pi" + ? [ + "--offline", + "--provider", + "rea-fixture", + "--model", + "rea-client-fixture", + "--no-session", + "--no-context-files", + "--mode", + "json", + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze ${target}.`, + ] + : [ + "--oneshot", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze ${target}.`, + "--provider", + "custom", + "--model", + "rea-client-fixture", + "--reasoning", + "none", + ]; +let failure; +try { + const execution = exec(command, args, { + env: { + ...environment, + OPENAI_API_KEY: "local-fixture-key", + OPENAI_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + }); + // Pi consumes piped input before its first prompt; an unused open pipe is not EOF. + execution.child.stdin.end(); + const result = await execution; + await writeFile(join(lab, "client.stdout"), result.stdout); + await writeFile(join(lab, "client.stderr"), result.stderr); + assert.equal(fixture.failure, undefined); + assert(finalSeen); + assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); + if (mode === "call") { + assert(evidenceSeen); + assert( + skillLoaded, + "Native client must actually read the installed skill.", + ); + assert.deepEqual( + catalog?.sort(), + TOOL_CONTRACTS.map((t) => t.name).sort(), + "The complete REA catalog must survive native client discovery.", + ); + } +} catch (cause) { + failure = cause; + if (typeof cause?.stdout === "string") + await writeFile(join(lab, "client.stdout"), cause.stdout); + if (typeof cause?.stderr === "string") + await writeFile(join(lab, "client.stderr"), cause.stderr); +} finally { + await fixture.close(); + const receipt = { + client, + clientVersion, + mode, + passed: failure === undefined, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback deterministic OpenAI-compatible endpoint; no live provider claim", + skillDiscovery: + client === "pi" + ? "caller-configured isolated shared directory; default OS home discovery unverified" + : "native HERMES_HOME personal directory", + catalogSize: catalog?.length, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture.requests, + probes: fixture.probes, + fixtureFailure: fixture.failure, + verifier: await completeVerifierRun(verifier), + }; + await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); + console.log(JSON.stringify(receipt)); +} +if (failure !== undefined) throw failure; diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs new file mode 100644 index 000000000..4b85470d6 --- /dev/null +++ b/scripts/verify-qwen-client.mjs @@ -0,0 +1,403 @@ +import assert from "node:assert/strict"; +import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; +import { + mkdir, + writeFile, + readFile, + mkdtemp, + realpath, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; +import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../dist/contracts/toolContracts.js"; + +assert( + process.platform !== "win32", + "The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.", +); +const repo = fileURLToPath(new URL("..", import.meta.url)); +const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; +const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-")); +const client = "qwen"; +const mode = process.argv[2] ?? "call"; +assert( + ["chat", "call"].includes(mode), + "Usage: node scripts/verify-qwen-client.mjs [chat|call]", +); +const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen"; +const verifier = createVerifierRun(); +const profile = join(lab, "selected profile"); +const account = join(lab, "account"); +const runtime = join(profile, "runtime"); +const workspace = join(lab, "workspace"); +const sharedSkills = join(account, ".agents", "skills"); +await Promise.all([ + mkdir(profile, { recursive: true }), + mkdir(account), + mkdir(workspace), +]); +const baseEnvironment = { + PATH: process.env.PATH ?? "", + USERPROFILE: account, + QWEN_HOME: profile, + QWEN_RUNTIME_DIR: runtime, + REA_PROCESS_RUN_ID: verifier.run_id, +}; +let clientVersion; +try { + clientVersion = ( + await exec(command, ["--version"], { + env: baseEnvironment, + timeout: 30_000, + }) + ).stdout.trim(); +} catch (cause) { + throw new Error( + `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`, + ); +} +const original = JSON.stringify( + { + security: { + auth: { selectedType: "openai" }, + folderTrust: { enabled: false }, + }, + telemetry: { enabled: false }, + model: { name: "rea-client-fixture" }, + // Qwen uses os.homedir(), while REA's isolated CLI account uses USERPROFILE. + // This caller-owned setting lets the real client load the REA-installed bundle. + skills: { directories: [sharedSkills] }, + mcpServers: { other: { command: "unrelated-server", disabled: true } }, + }, + null, + 2, +); +await writeFile(join(profile, "settings.json"), original); +for (const step of [ + ["setup", "--client", "qwen_code", "--dry-run"], + ["setup", "--client", "qwen_code", "--yes"], + ["doctor", "--client", "qwen_code", "--skill"], +]) { + const response = await exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"], + { env: baseEnvironment, cwd: repo, timeout: 60_000 }, + ); + const result = JSON.parse(response.stdout); + if (step.includes("--dry-run")) + assert( + result.plannedActions.every( + (a) => + a.target.startsWith(profile + "/") || + a.target.startsWith(sharedSkills + "/"), + ), + ); + await writeFile( + join(lab, `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`), + response.stdout, + ); +} +assert.equal( + await readFile(join(profile, "settings.json.rea.backup"), "utf8"), + original, +); +const configured = JSON.parse( + await readFile(join(profile, "settings.json"), "utf8"), +); +assert.deepEqual(configured.model, { name: "rea-client-fixture" }); +assert.deepEqual(configured.skills, { directories: [sharedSkills] }); +assert.deepEqual(configured.mcpServers.other, { + command: "unrelated-server", + disabled: true, +}); +const target = join(lab, "fixture α测试"); +await mkdir(target, { recursive: true }); +await writeFile( + `${target}/app.js`, + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', +); +let evidenceSeen = false; +let skillSeen = false; +let finalSeen = false; +let skillRequested = false; +let searchRequested = false; +let artifactReadByAgent = false; +let artifactPath; +let selectedSkillLoaded = false; +let artifactDigest; +const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return null; + } + } + if (Array.isArray(value)) { + for (const item of value) { + const result = extractEvidence(item); + if (result) return result; + } + } else if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const result = extractEvidence(nested); + if (result) return result; + } + } + return null; +}; +const fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: `${client}-${mode}`, + onRequest: async ({ body, tools, requestIndex }) => { + skillSeen ||= JSON.stringify(body.messages ?? []).includes( + "reverse-engineer-anything", + ); + selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes( + join(sharedSkills, "reverse-engineer-anything"), + ); + const toolText = JSON.stringify( + (body.messages ?? []).filter((m) => m.role === "tool"), + ); + if ( + toolText.includes("REA_CLIENT_FIXTURE") && + /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText) + ) { + artifactReadByAgent = true; + assert(artifactPath); + const artifact = await readFile(artifactPath); + artifactDigest = createHash("sha256").update(artifact).digest("hex"); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(artifact.toString("utf8"))); + assert(extractEvidence(evidence)); + assert.equal(evidence.subject.local_path, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + evidenceSeen = true; + await writeFile( + `${lab}/${client}-${mode}-evidence.json`, + JSON.stringify(evidence), + ); + } + for (const message of body.messages ?? []) { + if (message.role !== "tool") continue; + const evidence = extractEvidence(message.content); + if (evidence) { + assert( + JSON.stringify(evidence.normalized_result).includes( + "REA_CLIENT_FIXTURE", + ), + ); + evidenceSeen = true; + await writeFile( + `${lab}/${client}-${mode}-evidence.json`, + JSON.stringify(evidence), + ); + } + } + const selected = tools.find((t) => + t.name.includes("analyze_javascript_application"), + ); + const reviewedNames = [ + ...JSON.stringify( + (body.messages ?? []).filter((m) => m.role === "tool"), + ).matchAll( + /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g, + ), + ] + .map((m) => m[1]) + .sort((a, b) => b.length - a.length); + const search = tools.find((t) => t.name === "tool_search"); + const invoke = tools.find((t) => t.name === "tool_call"); + const skill = tools.find((t) => t.name === "skill"); + const shell = tools.find((t) => t.name === "run_shell_command"); + const persisted = toolText.match( + /Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u, + ); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_fixture_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + let message; + if (mode === "chat" || evidenceSeen) { + message = { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + finalSeen = true; + } else if (skill && !skillRequested) { + skillRequested = true; + message = call(skill.name, { skill: "reverse-engineer-anything" }); + } else if (persisted && shell) { + artifactPath = persisted[1]; + assert( + (await realpath(artifactPath)).startsWith( + (await realpath(runtime)) + "/", + ), + ); + const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`; + const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'"; + message = call(shell.name, { + command: `node -e ${quote(script)}`, + description: + "Read the complete saved REA Evidence and verify the fixture facts.", + }); + } else if (selected) + message = call(selected.name, { + input_path: target, + format: "directory", + }); + else if (reviewedNames.length && invoke) + message = call(invoke.name, { + name: reviewedNames[0], + arguments: { input_path: target, format: "directory" }, + }); + else if (search && !searchRequested) { + searchRequested = true; + message = call(search.name, { query: "analyze_javascript_application" }); + } else + throw new Error( + `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`, + ); + return message; + }, +}); +const endpoint = fixture.baseUrl; +const prompt = + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze the JavaScript directory ${target}. Report the result.`; +const env = { + ...baseEnvironment, + OPENAI_API_KEY: "local-fixture-key", + OPENAI_BASE_URL: endpoint, +}; +const args = [ + "--auth-type", + "openai", + "--openai-base-url", + endpoint, + "--model", + "rea-client-fixture", + "--approval-mode", + mode === "chat" ? "default" : "yolo", + "--allowed-mcp-server-names", + "rea", + "--output-format", + "stream-json", + "--max-session-turns", + "8", + "-p", + prompt, +]; +let response; +let failure; +try { + response = await exec(command, args, { + env, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + }); + await writeFile(join(lab, `${mode}.stdout`), response.stdout); + await writeFile(join(lab, `${mode}.stderr`), response.stderr); + const initialized = response.stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + .find((v) => v.type === "system" && v.subtype === "init"); + assert(initialized); + const observed = initialized.tools + .filter((n) => n.startsWith("mcp__rea__")) + .sort(); + assert.deepEqual( + observed, + TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(), + "Qwen must discover the complete REA catalog", + ); + assert.equal(fixture.failure, undefined); + assert(finalSeen); + if (mode === "call") { + assert(evidenceSeen); + assert(selectedSkillLoaded); + assert(artifactReadByAgent); + } +} catch (cause) { + failure = cause; + if (typeof cause?.stdout === "string") + await writeFile(join(lab, `${mode}.stdout`), cause.stdout); + if (typeof cause?.stderr === "string") + await writeFile(join(lab, `${mode}.stderr`), cause.stderr); +} finally { + await fixture.close(); + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Qwen Code", + clientVersion, + mode, + passed: failure === undefined, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim", + skillDiscovery: + "caller-configured isolated shared directory; default OS home discovery unverified", + catalogSize: TOOL_CONTRACTS.length, + requests: fixture.requests, + probes: fixture.probes, + evidenceSeen, + selectedSkillLoaded, + artifactReadByAgent, + artifactPath, + artifactDigest, + finalSeen, + fixtureFailure: fixture.failure, + verifier: lineage, + }; + await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); + console.log(JSON.stringify(receipt)); +} +if (failure !== undefined) throw failure; diff --git a/scripts/verify/clients/model-fixture.mjs b/scripts/verify/clients/model-fixture.mjs new file mode 100644 index 000000000..cdc99a31c --- /dev/null +++ b/scripts/verify/clients/model-fixture.mjs @@ -0,0 +1,171 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +/** A loopback model fixture exercises real clients without claiming live-provider compatibility. */ +export const createOpenAiModelFixture = async ({ + directory, + prefix, + onRequest, +}) => { + const requests = []; + const probes = []; + let incoming = 0; + let failure; + const server = createServer(async (request, response) => { + try { + if (request.method === "GET" && request.url?.endsWith("/models")) { + response.setHeader("Content-Type", "application/json"); + response.end( + JSON.stringify({ + object: "list", + data: [ + { + id: "rea-client-fixture", + object: "model", + owned_by: "local-fixture", + }, + ], + }), + ); + return; + } + const chunks = []; + let bytes = 0; + for await (const chunk of request) { + bytes += chunk.length; + assert( + bytes <= 10 * 1024 * 1024, + "Local model fixture exceeded its 10 MiB request resource budget", + ); + chunks.push(chunk); + } + const rawBody = Buffer.concat(chunks).toString("utf8"); + const body = rawBody.length === 0 ? null : JSON.parse(rawBody); + await writeFile( + join(directory, `${prefix}-incoming-${++incoming}.json`), + JSON.stringify({ method: request.method, path: request.url, body }), + ); + if (request.url !== "/v1/chat/completions") { + // Hermes probes custom endpoints for Ollama/llama.cpp metadata. + // Unsupported HTTP endpoints return 404, as on an OpenAI-only server. + probes.push({ + method: request.method, + path: request.url, + body, + status: 404, + }); + response.statusCode = 404; + response.setHeader("Content-Type", "application/json"); + response.end( + JSON.stringify({ + error: + "This OpenAI fixture exposes models and chat completions only.", + }), + ); + return; + } + assert.equal(request.method, "POST"); + assert(body !== null && typeof body === "object" && !Array.isArray(body)); + assert(Array.isArray(body.messages)); + const tools = (body.tools ?? []) + .filter((t) => t.type === "function") + .map((t) => t.function ?? t); + requests.push({ + path: request.url, + tools: tools.map((t) => t.name), + stream: body.stream, + messageRoles: body.messages.map((m) => m.role), + }); + await writeFile( + join(directory, `${prefix}-request-${requests.length}.json`), + JSON.stringify(body), + ); + const message = await onRequest({ + body, + tools, + requestIndex: requests.length, + }); + const base = { + id: `chatcmpl-rea-${requests.length}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: body.model, + }; + const finish = message.tool_calls ? "tool_calls" : "stop"; + const usage = { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }; + if (body.stream) { + response.setHeader("Content-Type", "text/event-stream"); + const emit = (delta, finishReason = null) => + response.write( + `data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta, finish_reason: finishReason }] })}\n\n`, + ); + if (message.tool_calls) { + const fragments = message.tool_calls.map((call, index) => { + const split = Math.floor(call.function.arguments.length / 2); + return { + first: { + index, + ...call, + function: { + name: call.function.name, + arguments: call.function.arguments.slice(0, split), + }, + }, + second: { + index, + function: { arguments: call.function.arguments.slice(split) }, + }, + }; + }); + emit({ + role: "assistant", + content: null, + tool_calls: fragments.map((f) => f.first), + }); + emit({ tool_calls: fragments.map((f) => f.second) }); + } else emit(message); + emit({}, finish); + response.write( + `data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [], usage })}\n\n`, + ); + response.end("data: [DONE]\n\n"); + } else { + response.setHeader("Content-Type", "application/json"); + response.end( + JSON.stringify({ + ...base, + choices: [{ index: 0, message, finish_reason: finish }], + usage, + }), + ); + } + } catch (cause) { + failure = String(cause); + response.statusCode = 400; + response.setHeader("Content-Type", "application/json"); + response.end( + JSON.stringify({ + error: { message: failure, type: "invalid_request_error" }, + }), + ); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + assert(address !== null && typeof address !== "string"); + return { + baseUrl: `http://127.0.0.1:${address.port}/v1`, + requests, + probes, + get failure() { + return failure; + }, + close: () => new Promise((resolve) => server.close(resolve)), + }; +}; diff --git a/tests/boundary/cli/hermesClientSetup.test.ts b/tests/acceptance/setup/hermesClientSetup.test.ts similarity index 100% rename from tests/boundary/cli/hermesClientSetup.test.ts rename to tests/acceptance/setup/hermesClientSetup.test.ts diff --git a/tests/boundary/cli/piClientSetup.test.ts b/tests/acceptance/setup/piClientSetup.test.ts similarity index 100% rename from tests/boundary/cli/piClientSetup.test.ts rename to tests/acceptance/setup/piClientSetup.test.ts diff --git a/tests/acceptance/setup/qwenProfileSetup.test.ts b/tests/acceptance/setup/qwenProfileSetup.test.ts new file mode 100644 index 000000000..9f52ba08f --- /dev/null +++ b/tests/acceptance/setup/qwenProfileSetup.test.ts @@ -0,0 +1,96 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { parse } from "jsonc-parser"; +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "keeps Qwen profile registration and shared skill consistent through setup, doctor and uninstall", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-qwen-profile-"); + const home = join(root, "account"); + const profile = join(root, "selected profile"); + const settings = join(profile, "settings.json"); + const skill = join( + home, + ".agents", + "skills", + "reverse-engineer-anything", + "SKILL.md", + ); + const environment = { USERPROFILE: home, QWEN_HOME: profile }; + const original = + '{\n // Keep my configuration.\n "model": {"name": "existing"},\n "mcpServers": {"other": {"command": "other-server"}}\n}\n'; + await mkdir(profile, { recursive: true }); + await writeFile(settings, original); + const run = (arguments_: readonly string[]) => + cli.run({ arguments: [...arguments_, "--json"], environment }); + + const plan = await run(["setup", "--client", "qwen_code", "--dry-run"]); + expect(plan.exitCode, plan.stderr).toBe(0); + expect(plan.json).toMatchObject({ + status: "planned", + plannedActions: [ + { kind: "configure_client", target: settings }, + { + kind: "install_skill", + target: join(home, ".agents", "skills", "reverse-engineer-anything"), + }, + ], + }); + expect(await readFile(settings, "utf8")).toBe(original); + await expect(access(skill)).rejects.toMatchObject({ code: "ENOENT" }); + + const installed = await run(["setup", "--client", "qwen_code", "--yes"]); + expect(installed.exitCode, installed.stderr).toBe(0); + expect(installed.json).toMatchObject({ status: "ready" }); + expect(await readFile(`${settings}.rea.backup`, "utf8")).toBe(original); + const configured = await readFile(settings, "utf8"); + expect(configured).toContain("// Keep my configuration."); + expect(parse(configured)).toMatchObject({ + model: { name: "existing" }, + mcpServers: { + other: { command: "other-server" }, + rea: { + command: + process.platform === "win32" + ? process.execPath + : resolve("scripts/rea.mjs"), + args: + process.platform === "win32" + ? [resolve("scripts/rea.mjs"), "mcp"] + : ["mcp"], + }, + }, + }); + expect(await readFile(skill, "utf8")).toContain( + "name: reverse-engineer-anything", + ); + + const doctor = await run(["doctor", "--client", "qwen_code", "--skill"]); + expect(doctor.exitCode, doctor.stderr).toBe(0); + expect(doctor.json).toMatchObject({ healthy: true }); + const repeated = await run(["setup", "--client", "qwen_code", "--yes"]); + expect(repeated.exitCode, repeated.stderr).toBe(0); + expect(repeated.json).toMatchObject({ + status: "ready", + appliedActions: [], + }); + expect(await readFile(settings, "utf8")).toBe(configured); + + const removed = await run(["uninstall"]); + expect(removed.exitCode, removed.stderr).toBe(0); + expect(removed.json).toMatchObject({ status: "complete" }); + expect(parse(await readFile(settings, "utf8"))).toEqual({ + model: { name: "existing" }, + mcpServers: { other: { command: "other-server" } }, + }); + await expect(access(skill)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(access(join(home, ".qwen"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }, + 120_000, +);