From 8c7569aca79349eec9def0fa98cbf04fdbcf0d02 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:16:41 +0800 Subject: [PATCH] test(clients): verify native Claude Code integration (#1549) --- docs/testing.md | 14 + package.json | 1 + scripts/verify-claude-client.mjs | 368 ++++++++++++++++++ .../clients/anthropic-model-fixture.mjs | 158 ++++++++ 4 files changed, 541 insertions(+) create mode 100644 scripts/verify-claude-client.mjs create mode 100644 scripts/verify/clients/anthropic-model-fixture.mjs diff --git a/docs/testing.md b/docs/testing.md index 52271a140..26da7f15c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -132,6 +132,20 @@ context and synthetic usage. Skills use an explicit isolated `skills.paths` directory, so default OS-home discovery remains unverified. Live model APIs and native Windows are also unverified. +`verify:claude-client` requires installed Claude Code (verified with +`@anthropic-ai/claude-code@2.1.296`); select it with `REA_VERIFY_CLAUDE_COMMAND`. +The optional POSIX lane uses native `CLAUDE_CONFIG_DIR` discovery, an independent +Git workspace, user settings and default built-in tools. It preserves caller +preferences and backups, checks idempotent setup, waits through the native +`WaitForMcpServers` tool when discovery is pending, and validates the complete +catalog and forwarded input schemas. Call mode loads the personal skill through +native `Skill`, checks its full body and verifies named-schema JavaScript Evidence +for a Unicode path. Use `-- chat` for ordinary chat with REA enabled, or +`REA_VERIFY_RUNTIME_ROOT` for a production-only installed package. The local +Anthropic Messages/SSE fixture preserves native resource-hint envelopes in its +request artifacts. Usage is synthetic; live Anthropic API, native Windows and +bare-mode personal-skill activation remain unverified by this lane. + `verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`; verified with `@deepseek-ai/dsh@0.2.0-rc.2`) and Git. Set `REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane diff --git a/package.json b/package.json index 080f901f6..75cd2eb1c 100644 --- a/package.json +++ b/package.json @@ -182,6 +182,7 @@ "verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs", "verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs", "verify:opencode-client": "npm run build:cached && node scripts/verify-opencode-client.mjs", + "verify:claude-client": "npm run build:cached && node scripts/verify-claude-client.mjs", "verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi", "verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes", "verify:deepseek-client": "npm run build:cached && node scripts/verify-deepseek-client.mjs", diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs new file mode 100644 index 000000000..d1b7aa933 --- /dev/null +++ b/scripts/verify-claude-client.mjs @@ -0,0 +1,368 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import Ajv2020 from "ajv/dist/2020.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../dist/contracts/toolContracts.js"; +import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { createAnthropicModelFixture } from "./verify/clients/anthropic-model-fixture.mjs"; + +assert( + process.platform !== "win32", + "Native Claude Code verifier requires POSIX; Windows is unverified", +); +const repo = fileURLToPath(new URL("..", import.meta.url)); +const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; +const command = process.env.REA_VERIFY_CLAUDE_COMMAND ?? "claude"; +const mode = process.argv[2] ?? "call"; +assert( + ["chat", "call"].includes(mode), + "Usage: verify-claude-client.mjs [chat|call]", +); +const verifier = createVerifierRun(); +const lab = await mkdtemp(join(tmpdir(), "rea-claude-client-")); +const profile = join(lab, "selected profile"); +const workspace = join(lab, "workspace"); +const configPath = join(profile, ".claude.json"); +const skillDirectory = join(profile, "skills", "reverse-engineer-anything"); +await Promise.all([mkdir(profile), mkdir(workspace)]); +await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); +const environment = { + ...process.env, + CLAUDE_CONFIG_DIR: profile, + DISABLE_AUTOUPDATER: "1", + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + REA_PROCESS_RUN_ID: verifier.run_id, +}; +let clientVersion; +try { + clientVersion = ( + await exec(command, ["--version"], { env: environment, timeout: 30_000 }) + ).stdout.trim(); +} catch (cause) { + throw new Error( + `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND. ${String(cause)}`, + ); +} +const target = join(lab, "fixture α测试"); +await mkdir(target); +await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', +); +let skillRequested = false; +let skillLoaded = false; +let expectedSkillBody; +let evidenceSeen = false; +let finalSeen = false; +let catalogSize = 0; +let schemasChecked = 0; +let pendingServerWaits = 0; +let artifactDigest; +const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); +const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + if (value && typeof value === "object") + return Object.values(value).some((nested) => containsText(nested, text)); + return false; +}; +const extractEvidence = (value) => { + if (typeof value === "string") { + try { + // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. + const json = value.replace( + /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, + "", + ); + return extractEvidence(JSON.parse(json)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const evidence = extractEvidence(nested); + if (evidence) return evidence; + } + } + return undefined; +}; +const fixture = await createAnthropicModelFixture({ + directory: lab, + onRequest: async ({ body, tools, requestIndex }) => { + if (tools.length === 0) + return [{ type: "text", text: "REA fixture title" }]; + assert(requestIndex <= 8, "Local model workflow exceeded its turn budget"); + const call = (name, input) => [ + { type: "tool_use", id: `toolu_rea_${requestIndex}`, name, input }, + ]; + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + if (!analysis) { + // Native automatic discovery can begin the conversation while MCP is still connecting. + const wait = tools.find((t) => t.name === "WaitForMcpServers"); + assert( + wait, + "Claude must expose REA tools or its native pending-server wait", + ); + pendingServerWaits++; + return call(wait.name, { servers: ["rea"] }); + } + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const reaTools = tools.filter((t) => t.name.startsWith(prefix)); + assert.deepEqual( + reaTools.map((t) => t.name).sort(), + expected, + "Claude must forward the complete REA catalog after connection", + ); + catalogSize = expected.length; + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.input_schema, + `${tool.name}: missing advertised input schema`, + ); + assert( + schemaValidator.validateSchema(tool.input_schema), + `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + } + skillLoaded ||= + expectedSkillBody !== undefined && + containsText(body.messages, expectedSkillBody) && + containsText(body.messages, skillDirectory); + const observed = extractEvidence(body.messages); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + assert.equal(evidence.subject.local_path, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "claude-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }]; + } + if (!skillRequested) { + const skill = tools.find((t) => t.name === "Skill"); + assert(skill, "Claude must expose its native Skill tool"); + skillRequested = true; + return call(skill.name, { skill: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Claude must load the complete installed personal REA skill", + ); + return call(analysis.name, { input_path: target, format: "directory" }); + }, +}); +const original = JSON.stringify({ theme: "light", mcpServers: {} }, null, 2); +await writeFile(configPath, original); +let failure; +try { + for (const args of [ + ["setup", "--client", "claude_code", "--dry-run"], + ["setup", "--client", "claude_code", "--yes"], + ["doctor", "--client", "claude_code", "--skill"], + ]) { + const result = await exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000 }, + ); + await writeFile( + join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); + assert( + parsed.plannedActions.every( + (a) => + a.target === configPath || a.target.startsWith(skillDirectory), + ), + ); + } + } + } + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.equal(JSON.parse(configured).theme, "light"); + const repeated = await exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "claude_code", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000 }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const installedSkill = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + assert(installedSkill.includes("name: reverse-engineer-anything")); + const lines = installedSkill.split("\n"); + const headerEnd = lines.indexOf("---", 1); + assert( + lines[0] === "---" && headerEnd > 0, + "Installed skill must have its authored frontmatter", + ); + expectedSkillBody = lines + .slice(headerEnd + 1) + .join("\n") + .trim(); + assert(expectedSkillBody.length > 0); + const execution = exec( + command, + [ + "--setting-sources", + "user", + "--print", + "--output-format", + "json", + "--no-session-persistence", + "--model", + "claude-sonnet-4-6", + "--tools", + "default", + "--permission-mode", + "dontAsk", + "--settings", + JSON.stringify({ + permissions: { + allow: [ + "Skill", + "WaitForMcpServers", + "mcp__rea__analyze_javascript_application", + ], + }, + }), + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, + ], + { + env: { + ...environment, + ANTHROPIC_API_KEY: "local-fixture-key", + ANTHROPIC_AUTH_TOKEN: "", + ANTHROPIC_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + }, + ); + execution.child.stdin.end(); + const response = await execution; + await writeFile(join(lab, `${mode}.stdout`), response.stdout); + await writeFile(join(lab, `${mode}.stderr`), response.stderr); + assert.equal(fixture.failure, undefined); + const parsed = JSON.parse(response.stdout); + assert.equal(parsed.is_error, false); + assert.equal(parsed.subtype, "success"); + assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK"); + assert.deepEqual(parsed.permission_denials, []); + assert(finalSeen); + if (mode === "call") assert(evidenceSeen && skillLoaded); +} catch (cause) { + failure = cause; + if (typeof cause?.stdout === "string") + await writeFile(join(lab, `${mode}.stdout`), cause.stdout); + if (typeof cause?.stderr === "string") + await writeFile(join(lab, `${mode}.stderr`), cause.stderr); +} finally { + await fixture.close(); + const receipt = { + client: "Claude Code", + clientVersion, + mode, + passed: failure === undefined, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback Anthropic Messages/SSE; synthetic usage and native resource hints; no live API claim", + discovery: + "native CLAUDE_CONFIG_DIR user configuration and skills; separate Git root, user settings only, default built-in tools", + bareMode: + "not used; native bare print mode did not load the personal skill", + catalogSize, + schemasChecked, + pendingServerWaits, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture.requests, + probes: fixture.probes, + fixtureFailure: fixture.failure, + verifier: await completeVerifierRun(verifier), + }; + await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); + console.log(JSON.stringify(receipt)); +} +if (failure !== undefined) throw failure; diff --git a/scripts/verify/clients/anthropic-model-fixture.mjs b/scripts/verify/clients/anthropic-model-fixture.mjs new file mode 100644 index 000000000..8f8e448c9 --- /dev/null +++ b/scripts/verify/clients/anthropic-model-fixture.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +/** Exercise native Anthropic adapters locally, without live-provider claims. */ +export const createAnthropicModelFixture = async ({ directory, onRequest }) => { + const requests = []; + const probes = []; + let incoming = 0; + let failure; + const server = createServer(async (request, response) => { + try { + const chunks = []; + let bytes = 0; + for await (const chunk of request) { + bytes += chunk.length; + assert( + bytes <= 10 * 1024 * 1024, + "Model fixture request exceeds 10 MiB budget", + ); + chunks.push(chunk); + } + const raw = Buffer.concat(chunks).toString("utf8"); + const body = raw.length === 0 ? null : JSON.parse(raw); + const path = new URL(request.url, "http://localhost").pathname; + // Retain producer bodies and URLs, without transport authentication headers. + await writeFile( + join(directory, `claude-incoming-${++incoming}.json`), + JSON.stringify({ method: request.method, path: request.url, body }), + ); + if (path === "/v1/messages/count_tokens") { + probes.push({ path, kind: "synthetic-token-count" }); + response.setHeader("Content-Type", "application/json"); + response.end(JSON.stringify({ input_tokens: 100 })); + return; + } + if (path !== "/v1/messages") { + probes.push({ path, status: 404 }); + response.statusCode = 404; + response.end( + JSON.stringify({ + error: { message: "Unsupported fixture endpoint" }, + }), + ); + return; + } + assert.equal(request.method, "POST"); + assert(body && typeof body === "object" && !Array.isArray(body)); + assert(Array.isArray(body.messages)); + const tools = body.tools ?? []; + requests.push({ + path: request.url, + stream: body.stream, + tools: tools.map((t) => t.name), + messageRoles: body.messages.map((m) => m.role), + }); + const content = await onRequest({ + body, + tools, + requestIndex: requests.length, + }); + const stopReason = content.some((c) => c.type === "tool_use") + ? "tool_use" + : "end_turn"; + const usage = { + input_tokens: 100, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }; + const result = { + id: `msg_rea_${requests.length}`, + type: "message", + role: "assistant", + model: body.model, + content, + stop_reason: stopReason, + stop_sequence: null, + usage, + }; + if (body.stream) { + response.setHeader("Content-Type", "text/event-stream"); + const emit = (event, data) => + response.write( + `event: ${event}\ndata: ${JSON.stringify({ type: event, ...data })}\n\n`, + ); + emit("message_start", { + message: { + ...result, + content: [], + stop_reason: null, + usage: { ...usage, output_tokens: 0 }, + }, + }); + for (const [index, block] of content.entries()) { + emit("content_block_start", { + index, + content_block: + block.type === "tool_use" + ? { ...block, input: {} } + : { type: "text", text: "" }, + }); + if (block.type === "tool_use") { + const json = JSON.stringify(block.input); + const split = Math.floor(json.length / 2); + for (const partialJson of [ + json.slice(0, split), + json.slice(split), + ]) { + emit("content_block_delta", { + index, + delta: { type: "input_json_delta", partial_json: partialJson }, + }); + } + } else { + emit("content_block_delta", { + index, + delta: { type: "text_delta", text: block.text }, + }); + } + emit("content_block_stop", { index }); + } + emit("message_delta", { + delta: { stop_reason: stopReason, stop_sequence: null }, + usage: { output_tokens: 20 }, + }); + emit("message_stop", {}); + response.end(); + } else { + response.setHeader("Content-Type", "application/json"); + response.end(JSON.stringify(result)); + } + } catch (cause) { + failure = String(cause); + response.statusCode = 400; + response.setHeader("Content-Type", "application/json"); + response.end( + JSON.stringify({ + type: "error", + error: { type: "invalid_request_error", message: failure }, + }), + ); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + assert(address !== null && typeof address !== "string"); + return { + baseUrl: `http://127.0.0.1:${address.port}`, + requests, + probes, + get failure() { + return failure; + }, + close: () => new Promise((resolve) => server.close(resolve)), + }; +};