import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { Ajv2020 } from "ajv/dist/2020.js"; import { parse as parseYaml } from "yaml"; import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; import { exec, pathExists } from "./lib/verify-package-core.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; const mode = process.argv[2] ?? "call"; assert(["chat", "call"].includes(mode), "Select chat or call."); assert(process.platform !== "win32", "Native Windows coverage is unverified."); const repo = fileURLToPath(new URL("..", import.meta.url)); const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; const command = process.env.REA_VERIFY_DEEPSEEK_COMMAND ?? "dsh"; const lab = await mkdtemp(join(tmpdir(), "rea-deepseek-client-")); const account = join(lab, "account"); const profile = join(lab, "selected profile"); const workspace = join(lab, "workspace"); const target = join(workspace, "fixture α测试"); const skillPath = join( account, ".agents", "skills", "reverse-engineer-anything", "SKILL.md", ); const cli = join(runtimeRoot, "scripts", "rea.mjs"); const configPath = join(profile, "cordis.patch.yml"); const verifier = createVerifierRun(); await Promise.all([ mkdir(account), mkdir(profile), mkdir(target, { recursive: true }), ]); const environment = { PATH: process.env.PATH ?? "", HOME: account, USERPROFILE: account, DSH_HOME: profile, OPENAI_API_KEY: "local-fixture-key", REA_PROCESS_RUN_ID: verifier.run_id, }; // A temporary directory under the checkout must not inherit its project skills or AGENTS.md. await exec( "git", ["-c", "init.defaultBranch=main", "init", "--quiet", workspace], { env: environment, timeout: 30_000, }, ); await writeFile( join(target, "app.js"), 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); const clientVersion = ( await exec(command, ["--version"], { env: environment, cwd: workspace, timeout: 30_000, }) ).stdout.trim(); const lifecycle = []; const runRea = async (args, artifact) => { const result = await exec(process.execPath, [cli, ...args, "--json"], { env: environment, cwd: workspace, timeout: 60_000, }); await writeFile(join(lab, `${artifact}.json`), result.stdout); return JSON.parse(result.stdout); }; const patches = (text) => parseYaml(text, { customTags: [ { tag: "tag:yaml.org,2002:js", resolve: (source) => ({ __jsExpr: source }), }, ], }); const withoutRea = (text) => patches(text).flatMap((op) => { if (op.id === "rea-mcp") return []; if (!Array.isArray(op.insert)) return [op]; const insert = op.insert.filter((entry) => entry.id !== "rea-mcp"); return insert.length === 0 ? [] : [{ ...op, insert }]; }); let catalog; let patternCount = 0; let skillRequested = false; let skillLoaded = false; let evidenceSeen = false; let finalSeen = false; let artifactDigest; let mainRequests = 0; const ajv = new Ajv2020({ strict: false, validateFormats: false }); const patterns = new Set(); const validatePatterns = (value) => { if (!value || typeof value !== "object") return; for (const [key, child] of Object.entries(value)) { if (key === "pattern" && typeof child === "string") { // Check the representation the native client actually forwards to its model adapter. for (const flags of ["", "u", "v"]) new RegExp(child, flags); patterns.add(child); patternCount++; } else validatePatterns(child); } }; const extractEvidence = (value) => { if (typeof value === "string") { try { return extractEvidence(JSON.parse(value)); } catch { return undefined; } } if (value && typeof value === "object") { if ( typeof value.evidence_id === "string" && value.normalized_result?.graph?.nodes ) return value; for (const child of Object.values(value)) { const found = extractEvidence(child); if (found) return found; } } }; const fixture = await createOpenAiModelFixture({ directory: lab, prefix: `deepseek-${mode}`, onRequest: async ({ body, tools, requestIndex }) => { // Harness generates a session title in a separate, tool-free model request. if (tools.length === 0) return { role: "assistant", content: "REA validation" }; assert( ++mainRequests <= 4, "Harness did not complete the bounded fixture workflow.", ); const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__")); const observedCatalog = rea .map((tool) => tool.name.slice("mcp__rea__".length)) .sort(); assert.deepEqual( observedCatalog, TOOL_CONTRACTS.map((tool) => tool.name).sort(), ); if (catalog === undefined) { catalog = observedCatalog; for (const tool of rea) { assert( ajv.validateSchema(tool.parameters), `${tool.name}: ${ajv.errorsText(ajv.errors)}`, ); validatePatterns(tool.parameters); } assert( patternCount > 0, "The advertised regex constraints must reach the model.", ); } const toolMessages = (body.messages ?? []).filter( (message) => message.role === "tool", ); const serialized = JSON.stringify(toolMessages); skillLoaded ||= serialized.includes( '', ) && serialized.includes(dirname(skillPath)); for (const message of toolMessages) { const found = extractEvidence(message.content); if (found === undefined) continue; const evidence = toolContract( "analyze_javascript_application", ).outputSchema.parse(found); assert.equal(evidence.subject.local_path, target); const shape = evidence.normalized_result.graph.nodes .flatMap((node) => node.observations) .map((observation) => observation.properties) .find( (properties) => properties.semantic_role === "export-return-shapes" && properties.exported_name === "result", ); assert(shape?.return_shape_coverage?.projection_complete); const fields = shape.static_return_shapes.flatMap( (entry) => entry.fields, ); assert( fields.some( (field) => field.path === "/clientCompatibility" && field.state === "literal" && field.value === "REA_CLIENT_FIXTURE", ), ); assert( fields.some( (field) => field.path === "/count" && field.state === "literal" && field.value === 7, ), ); const text = JSON.stringify(evidence); await writeFile(join(lab, "evidence.json"), text); artifactDigest = createHash("sha256").update(text).digest("hex"); evidenceSeen = true; } if (mode === "chat" || evidenceSeen) { finalSeen = true; return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; } const call = (name, args) => ({ role: "assistant", content: null, tool_calls: [ { id: `call_rea_${requestIndex}`, type: "function", function: { name, arguments: JSON.stringify(args) }, }, ], }); if (!skillRequested) { assert(tools.some((tool) => tool.name === "skill")); skillRequested = true; return call("skill", { name: "reverse-engineer-anything" }); } assert( skillLoaded, "Harness must load the installed skill before analysis.", ); return call("mcp__rea__analyze_javascript_application", { input_path: target, format: "directory", }); }, }); let failure; try { // Seed model settings and a stale registration; the public setup workflow // must produce the registration this native client actually boots. const original = ` - id: llm-pi-ai config: providers: rea-fixture: apiKeyEnv: OPENAI_API_KEY api: openai-completions baseURL: ${fixture.baseUrl} defaultMaxTokens: 4096 retryPolicy: mode: normal maxRetries: 0 models: - id: rea-client-fixture contextWindow: 1000000 - id: agent-default-model config: provider: rea-fixture model: rea-client-fixture - insert: - id: rea-mcp name: '@deepseek-ai/dsh-mcp-client' config: serverName: rea transport: stdio command: stale-launcher args: [mcp] cwd: !!js process.cwd() env: REA_PROCESS_RUN_ID: ${JSON.stringify(verifier.run_id)} `; await writeFile(configPath, original); const planned = await runRea( ["setup", "--client", "dsh", "--dry-run"], "setup-plan", ); assert.equal(planned.status, "planned"); assert(planned.plannedActions.some((action) => action.target === configPath)); assert( planned.plannedActions.some( (action) => action.target === dirname(skillPath), ), ); assert( planned.plannedActions.every( (action) => action.target === configPath || action.target === dirname(skillPath), ), ); assert.equal(await readFile(configPath, "utf8"), original); assert.equal(await pathExists(`${configPath}.rea.backup`), false); lifecycle.push("dry-run"); const setup = await runRea(["setup", "--client", "dsh", "--yes"], "setup"); assert.equal(setup.status, "ready"); assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); const configured = await readFile(configPath, "utf8"); assert.deepEqual(withoutRea(configured), withoutRea(original)); assert(configured.includes("!!js process.cwd()")); assert( (await readFile(skillPath, "utf8")).includes( "name: reverse-engineer-anything", ), ); lifecycle.push("setup"); const repeated = await runRea( ["setup", "--client", "dsh", "--yes"], "setup-repeat", ); assert.deepEqual(repeated.appliedActions, []); assert.equal(await readFile(configPath, "utf8"), configured); assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); lifecycle.push("idempotent-setup"); const doctor = await runRea( ["doctor", "--client", "dsh", "--skill"], "doctor", ); assert.equal(doctor.healthy, true); lifecycle.push("doctor"); const execution = exec( command, [ "--profile", "headless", "--json", mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`, ], { env: environment, cwd: workspace, timeout: 120_000, maxBuffer: 10 * 1024 * 1024, }, ); execution.child.stdin.end(); const result = await execution; await writeFile(join(lab, "client.stdout"), result.stdout); await writeFile(join(lab, "client.stderr"), result.stderr); assert.equal(fixture.failure, undefined); assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); if (mode === "call") assert(evidenceSeen && skillLoaded); lifecycle.push(mode === "call" ? "native-tool-invocation" : "native-chat"); const uninstall = await runRea(["uninstall"], "uninstall"); assert.equal(uninstall.status, "complete"); assert( uninstall.items.some( (item) => item.name === "dsh" && item.status === "removed", ), ); assert.deepEqual( patches(await readFile(configPath, "utf8")), withoutRea(original), ); assert.equal(await pathExists(skillPath), false); lifecycle.push("uninstall"); } catch (cause) { failure = cause; if (typeof cause?.stdout === "string") await writeFile(join(lab, "client.stdout"), cause.stdout); if (typeof cause?.stderr === "string") await writeFile(join(lab, "client.stderr"), cause.stderr); } finally { await fixture.close(); const receipt = { client: "deepseek-harness", clientVersion, mode, lifecycle, passed: failure === undefined, artifacts: lab, runtimeRoot, host: { platform: process.platform, architecture: process.arch, node: process.versions.node, }, modelFixture: "loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim", modelContextWindow: 1000000, skillDiscovery: "native HOME shared personal directory; isolated project root", catalogSize: catalog?.length, patternCount, uniquePatterns: patterns.size, skillLoaded, evidenceSeen, artifactDigest, finalSeen, requests: fixture.requests, fixtureFailure: fixture.failure, verifier: await completeVerifierRun(verifier), }; await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); console.log(JSON.stringify(receipt)); } if (failure !== undefined) throw failure;