import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import { isAbsolute, join, relative } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; import { exec } from "./lib/verify-package-core.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; assert( process.platform !== "win32", "Native Grok verifier requires POSIX; Windows is unverified", ); const repo = fileURLToPath(new URL("..", import.meta.url)); const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; const command = process.env.REA_VERIFY_GROK_COMMAND ?? "grok"; const mode = process.argv[2] ?? "call"; assert( ["chat", "call"].includes(mode), "Usage: verify-grok-client.mjs [chat|call]", ); const verifier = createVerifierRun(); const lab = await mkdtemp(join(tmpdir(), "rea-grok-client-")); const account = join(lab, "account"); const profile = join(lab, "selected profile"); const workspace = join(lab, "workspace"); const configPath = join(profile, "config.toml"); const skillDirectory = join( account, ".agents", "skills", "reverse-engineer-anything", ); await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); // Native compatibility discovery is independent of GROK_HOME. Disable those // sources and ignore the real shared skill root instead of changing HOME. const environment = { PATH: process.env.PATH, NODE_OPTIONS: process.env.NODE_OPTIONS, USERPROFILE: account, GROK_HOME: profile, GROK_DISABLE_AUTOUPDATER: "1", ...Object.fromEntries( ["CLAUDE", "CURSOR"].flatMap((vendor) => ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map((cell) => [ `GROK_${vendor}_${cell}_ENABLED`, "false", ]), ), ), XDG_DATA_HOME: join(lab, "data"), XDG_CONFIG_HOME: join(lab, "config"), XDG_CACHE_HOME: join(lab, "cache"), REA_PROCESS_RUN_ID: verifier.run_id, }; const target = join(lab, "fixture α测试"); await mkdir(target); await writeFile( join(target, "app.js"), 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' + // A real analysis result large enough to exercise native MCP offloading. Array.from( { length: 60 }, (_, index) => `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`, ).join("\n"), ); const expectedNames = TOOL_CONTRACTS.map((tool) => "rea__" + tool.name).sort(); assert( expectedNames.length <= 255, "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools", ); const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; const containsText = (value, text) => { if (typeof value === "string") return value.includes(text); return ( value !== null && typeof value === "object" && Object.values(value).some((item) => containsText(item, text)) ); }; let clientVersion; let expectedSkill; let turn = 0; let skillRequested = false; let skillLoaded = false; let diagnosticsRequested = false; let diagnosticsVerified = false; let lastTool; let catalogChecked = false; let schemasChecked = 0; let analysisRequested = false; let evidenceSeen = false; let spillQueryRequested = false; let projectionSeen = false; let artifactDigest; let artifactBytes; let retainedArtifact; let expectedProjection; let finalSeen = false; let failure; const call = (name, args) => { lastTool = name; return { role: "assistant", content: null, tool_calls: [ { id: `call_grok_${turn}`, type: "function", function: { name, arguments: JSON.stringify(args) }, }, ], }; }; const verifyEvidence = (bytes) => { const evidence = toolContract( "analyze_javascript_application", ).outputSchema.parse(JSON.parse(bytes.toString("utf8"))); assert.equal(evidence.subject.local_path, target); const shapes = evidence.normalized_result.graph.nodes .flatMap((node) => node.observations) .map((observation) => observation.properties) .filter( (properties) => properties.semantic_role === "export-return-shapes" && properties.exported_name === "result", ); assert( shapes.length > 0 && shapes.every((shape) => shape.return_shape_coverage?.projection_complete), ); const fields = shapes.flatMap((shape) => shape.static_return_shapes.flatMap((item) => item.fields), ); assert( fields.some( (field) => field.path === "/clientCompatibility" && field.state === "literal" && field.value === "REA_CLIENT_FIXTURE", ), ); assert( fields.some( (field) => field.path === "/count" && field.state === "literal" && field.value === 7, ), ); artifactDigest = createHash("sha256").update(bytes).digest("hex"); artifactBytes = bytes.length; evidenceSeen = true; return { evidence_id: evidence.evidence_id, subject: evidence.subject, operation: evidence.operation, artifact_sha256: artifactDigest, byte_length: artifactBytes, shapes, }; }; const diagnosticFile = join(lab, "native-doctor.json"); const diagnosticCode = "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));"; // Query the saved result through a native agent command. Only the facts needed // for this analyst question enter context; the complete named-schema Evidence // and its digest remain available in the native artifact. const recoveryCode = "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));"; const fixture = await createOpenAiModelFixture({ directory: lab, prefix: "grok", onRequest: async ({ body, tools }) => { // Native title/summary requests cannot execute the analyst workflow. if (!tools.some((tool) => tool.name === "search_tool")) return { role: "assistant", content: "REA fixture auxiliary response" }; turn++; assert( containsText(body.messages, skillDirectory), "Grok must discover the isolated installed skill", ); assert( !containsText(body.messages, join(homedir(), ".agents", "skills")), "Do not inherit real personal skills", ); if (mode === "chat") { finalSeen = true; return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; } const replies = body.messages.filter((message) => message.role === "tool"); if (!skillRequested) { skillRequested = true; return call("read_file", { target_file: join(skillDirectory, "SKILL.md"), }); } assert( expectedSkill !== undefined && containsText(replies, expectedSkill), "Native read_file must return the full skill with its documented line-number rendering", ); skillLoaded = true; const last = replies.at(-1)?.content; assert.equal(typeof last, "string"); if (!catalogChecked) { const discovery = lastTool === "search_tool" ? JSON.parse(last) : undefined; if (discovery?.status === "partial" && !diagnosticsRequested) { diagnosticsRequested = true; const diagnostic = [ process.execPath, join(runtimeRoot, "scripts/rea.mjs"), "doctor", "--client", "grok_build", "--skill", "--json", ] .map(shellQuote) .join(" "); const projection = [ process.execPath, "-e", diagnosticCode, diagnosticFile, ] .map(shellQuote) .join(" "); return call("run_terminal_command", { command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`, description: "Diagnose REA registration while native MCP discovery is pending", }); } if (lastTool === "run_terminal_command") { const line = last .split("\n") .find((item) => item.startsWith('{"rea_client_diagnostic":')); assert( line, "Native REA diagnostics failed or did not return the selected status", ); const diagnostic = JSON.parse(line).rea_client_diagnostic; assert.equal(diagnostic.healthy, true); assert.equal(diagnostic.skill, "aligned"); assert( diagnostic.registrations.some( (registration) => registration.state === "aligned" && registration.config_path === configPath, ), ); diagnosticsVerified = true; } if (discovery?.status !== "ready") return call("search_tool", { query: "rea", limit: 255 }); const inventory = discovery.results .filter((result) => result.server === "rea") .flatMap((result) => result.tools); assert.deepEqual( inventory.map((tool) => tool.tool_name).sort(), expectedNames, "Native search_tool must discover the complete REA catalog", ); for (const tool of inventory) { assert(tool.input_schema, `${tool.tool_name}: missing input schema`); assert( schemaValidator.validateSchema(tool.input_schema), `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`, ); schemasChecked++; } await writeFile(join(lab, "native-discovery.json"), last); catalogChecked = true; analysisRequested = true; return call("use_tool", { tool_name: "rea__analyze_javascript_application", tool_input: { input_path: target, format: "directory" }, }); } if (!spillQueryRequested) { const marker = "Full output written to: "; const start = last.indexOf(marker); if (start < 0) { verifyEvidence(Buffer.from(last, "utf8")); finalSeen = true; return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; } // The native hint explicitly calls for a query rather than read_file on // the saved JSON's very long line. Preserve the original hint in requests. const file = last .slice(start + marker.length) .split(". The full output ")[0]; assert(isAbsolute(file)); const childPath = relative(join(profile, "sessions"), file); assert( childPath !== "" && !childPath.startsWith("..") && !isAbsolute(childPath) && file.endsWith(".json"), "Unexpected native MCP spill path", ); retainedArtifact = file; expectedProjection = verifyEvidence(await readFile(file)); spillQueryRequested = true; return call("run_terminal_command", { command: [process.execPath, "-e", recoveryCode, file] .map(shellQuote) .join(" "), description: "Query full saved REA Evidence while preserving artifact identity", }); } const line = last .split("\n") .find((item) => item.startsWith('{"rea_recovered":')); assert( line, "The next model request must contain native artifact query output", ); const projection = JSON.parse(line).rea_recovered; assert.deepEqual( projection, expectedProjection, "Native query must recover the observed facts and complete artifact digest", ); projectionSeen = true; finalSeen = true; return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; }, }); const q = JSON.stringify; const original = `# Keep unrelated settings and server state. disabled_mcp_servers = ["rea", "other"] [cli] auto_update = false [features] telemetry = "off" [skills] paths = [${q(join(account, ".agents", "skills"))}] ignore = [${q(join(homedir(), ".agents", "skills"))}] [model.rea-client-fixture] model = "rea-client-fixture" base_url = ${q(fixture.baseUrl)} api_key = "local-fixture-key" api_backend = "chat_completions" context_window = 1000000 max_completion_tokens = 4096 [mcp_servers.other] command = "unrelated-server" enabled = false `; await writeFile(configPath, original); const rea = async (args) => exec( process.execPath, [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], { env: environment, cwd: workspace, timeout: 60_000 }, ); try { clientVersion = ( await exec(command, ["--version"], { env: environment, timeout: 30_000 }) ).stdout.trim(); for (const args of [ ["setup", "--client", "grok_build", "--dry-run"], ["setup", "--client", "grok_build", "--yes"], ["doctor", "--client", "grok_build", "--skill"], ]) { const result = await rea(args); await writeFile( join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), result.stdout, ); const parsed = JSON.parse(result.stdout); if (args[0] === "doctor") assert.equal(parsed.healthy, true); else { assert.equal( parsed.status, args.includes("--dry-run") ? "planned" : "ready", ); if (args.includes("--dry-run")) { assert( parsed.plannedActions.some((action) => action.target === configPath), ); assert( parsed.plannedActions.some( (action) => action.target === skillDirectory, ), ); assert( parsed.plannedActions.every( (action) => action.target === configPath || action.target === skillDirectory || action.target.startsWith(skillDirectory + "/"), ), "Refuse writes outside the owned profile/skill", ); } } } assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); const configured = await readFile(configPath, "utf8"); assert( configured.includes("unrelated-server") && configured.includes("# Keep unrelated settings"), ); const repeated = JSON.parse( (await rea(["setup", "--client", "grok_build", "--yes"])).stdout, ); assert.deepEqual(repeated.appliedActions, []); assert.equal(await readFile(configPath, "utf8"), configured); assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); expectedSkill = skill .trimEnd() .split("\n") .map((line, index) => index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line, ) .join("\n"); const execution = exec( command, [ "-p", mode === "chat" ? "Say hello briefly." : `Use the REA skill to analyze ${target}.`, "-m", "rea-client-fixture", "--output-format", "streaming-messages-json", "--always-approve", "--no-subagents", ], { env: environment, cwd: workspace, timeout: 120_000, maxBuffer: 10 * 1024 * 1024, }, ); execution.child.stdin.end(); const response = await execution; await writeFile(join(lab, `${mode}.stdout`), response.stdout); await writeFile(join(lab, `${mode}.stderr`), response.stderr); assert.equal(fixture.failure, undefined); const events = response.stdout .trim() .split("\n") .map((line) => JSON.parse(line)); const init = events.find( (event) => event.type === "system" && event.subtype === "init", ); assert(init?.mcp_servers?.some((server) => server.name === "rea")); assert( init.mcp_servers.every((server) => ["rea", "other"].includes(server.name)), "Do not inherit foreign MCP configuration", ); assert( events.some( (event) => event.type === "result" && event.subtype === "success" && event.is_error === false && event.result === "REA_CLIENT_COMPATIBILITY_OK", ), ); assert(finalSeen); if (mode === "call") assert( skillLoaded && catalogChecked && analysisRequested && evidenceSeen && (!spillQueryRequested || projectionSeen) && (!diagnosticsRequested || diagnosticsVerified), ); } catch (cause) { failure = cause; if (typeof cause?.stdout === "string") await writeFile(join(lab, `${mode}.stdout`), cause.stdout); if (typeof cause?.stderr === "string") await writeFile(join(lab, `${mode}.stderr`), cause.stderr); } finally { await fixture.close(); const lineage = await completeVerifierRun(verifier); const receipt = { client: "Grok Build", clientVersion, mode, passed: failure === undefined, artifacts: lab, runtimeRoot, host: { platform: process.platform, architecture: process.arch, node: process.versions.node, }, modelFixture: "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim", discovery: "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified", interface: "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal", catalogChecked, catalogSize: catalogChecked ? expectedNames.length : 0, schemasChecked, skillLoaded, diagnosticsRequested, diagnosticsVerified, analysisRequested, evidenceSeen, spillQueryRequested, projectionSeen, retainedArtifact, artifactBytes, artifactDigest, finalSeen, evidenceScope: "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact", requests: fixture.requests, probes: fixture.probes, fixtureFailure: fixture.failure, verifier: lineage, }; await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); console.log(JSON.stringify(receipt)); } if (failure !== undefined) throw failure;