From efd81f977604a7b0ed55c2ee11f8b0da356d6c0f Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:59:43 +0800 Subject: [PATCH] test(clients): verify native Grok Build workflows (#1571) --- docs/testing.md | 20 ++ package.json | 1 + scripts/verify-grok-client.mjs | 539 +++++++++++++++++++++++++++++++++ 3 files changed, 560 insertions(+) create mode 100644 scripts/verify-grok-client.mjs diff --git a/docs/testing.md b/docs/testing.md index 22e409afa..61440dfb1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -167,6 +167,26 @@ fix for that client/model limit. It preserves the complete catalog and schemas. Live model APIs, native Windows and default OS-home skill discovery remain unverified. +`verify:grok-client` requires installed Grok Build (verified with the official +Linux x64 1.0.50 binary); select it with `REA_VERIFY_GROK_COMMAND`. This optional +POSIX lane isolates `GROK_HOME`, an independent Git workspace and additional +skill roots, disables foreign configuration discovery, guards setup writes, +preserves unrelated registrations/backups and checks idempotence. Call mode +loads the full skill through native `read_file`, queries all REA names and input +schemas through native `search_tool`, and calls analysis through `use_tool`. +If discovery reports `partial`, the native agent diagnoses REA registration and +retries discovery. No startup-timeout override or fixed readiness delay is used. +A larger Unicode-path fixture exercises native result offloading: the full +retained Evidence is validated against its named schema, then native terminal +queries recover the selected export facts, subject and full artifact digest in +the next model request. This verifies useful artifact recovery without claiming +that every offloaded graph fact enters model context. Native line-number and +truncation envelopes remain in the request artifacts. Use `-- chat` for ordinary +chat, or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed package. +The loopback OpenAI completions/SSE custom model declares a one-million-token +context and synthetic usage. Live xAI APIs, native Windows and default OS-home +skill discovery remain unverified. + `verify:omp-client` requires installed OMP (verified with the official Linux x64 18.8.7 binary); select it with `REA_VERIFY_OMP_COMMAND`. This optional POSIX lane isolates the default agent, global config and XDG roots, guards setup targets, diff --git a/package.json b/package.json index 9e59d0c6e..ed36813a8 100644 --- a/package.json +++ b/package.json @@ -182,6 +182,7 @@ "verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs", "verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs", "verify:opencode-client": "npm run build:cached && node scripts/verify-opencode-client.mjs", + "verify:grok-client": "npm run build:cached && node scripts/verify-grok-client.mjs", "verify:omp-client": "npm run build:cached && node scripts/verify-omp-client.mjs", "verify:copilot-client": "npm run build:cached && node scripts/verify-copilot-client.mjs", "verify:claude-client": "npm run build:cached && node scripts/verify-claude-client.mjs", diff --git a/scripts/verify-grok-client.mjs b/scripts/verify-grok-client.mjs new file mode 100644 index 000000000..e1c066770 --- /dev/null +++ b/scripts/verify-grok-client.mjs @@ -0,0 +1,539 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import { isAbsolute, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import Ajv2020 from "ajv/dist/2020.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../dist/contracts/toolContracts.js"; +import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; + +assert( + process.platform !== "win32", + "Native Grok verifier requires POSIX; Windows is unverified", +); +const repo = fileURLToPath(new URL("..", import.meta.url)); +const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; +const command = process.env.REA_VERIFY_GROK_COMMAND ?? "grok"; +const mode = process.argv[2] ?? "call"; +assert( + ["chat", "call"].includes(mode), + "Usage: verify-grok-client.mjs [chat|call]", +); +const verifier = createVerifierRun(); +const lab = await mkdtemp(join(tmpdir(), "rea-grok-client-")); +const account = join(lab, "account"); +const profile = join(lab, "selected profile"); +const workspace = join(lab, "workspace"); +const configPath = join(profile, "config.toml"); +const skillDirectory = join( + account, + ".agents", + "skills", + "reverse-engineer-anything", +); +await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); +await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); +// Native compatibility discovery is independent of GROK_HOME. Disable those +// sources and ignore the real shared skill root instead of changing HOME. +const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + GROK_HOME: profile, + GROK_DISABLE_AUTOUPDATER: "1", + ...Object.fromEntries( + ["CLAUDE", "CURSOR"].flatMap((vendor) => + ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map((cell) => [ + `GROK_${vendor}_${cell}_ENABLED`, + "false", + ]), + ), + ), + XDG_DATA_HOME: join(lab, "data"), + XDG_CONFIG_HOME: join(lab, "config"), + XDG_CACHE_HOME: join(lab, "cache"), + REA_PROCESS_RUN_ID: verifier.run_id, +}; +const target = join(lab, "fixture α测试"); +await mkdir(target); +await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' + + // A real analysis result large enough to exercise native MCP offloading. + Array.from( + { length: 60 }, + (_, index) => + `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`, + ).join("\n"), +); +const expectedNames = TOOL_CONTRACTS.map((tool) => "rea__" + tool.name).sort(); +assert( + expectedNames.length <= 255, + "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools", +); +const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); +const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; +const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + return ( + value !== null && + typeof value === "object" && + Object.values(value).some((item) => containsText(item, text)) + ); +}; +let clientVersion; +let expectedSkill; +let turn = 0; +let skillRequested = false; +let skillLoaded = false; +let diagnosticsRequested = false; +let diagnosticsVerified = false; +let lastTool; +let catalogChecked = false; +let schemasChecked = 0; +let analysisRequested = false; +let evidenceSeen = false; +let spillQueryRequested = false; +let projectionSeen = false; +let artifactDigest; +let artifactBytes; +let retainedArtifact; +let expectedProjection; +let finalSeen = false; +let failure; +const call = (name, args) => { + lastTool = name; + return { + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_grok_${turn}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }; +}; +const verifyEvidence = (bytes) => { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(bytes.toString("utf8"))); + assert.equal(evidence.subject.local_path, target); + const shapes = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .filter( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert( + shapes.length > 0 && + shapes.every((shape) => shape.return_shape_coverage?.projection_complete), + ); + const fields = shapes.flatMap((shape) => + shape.static_return_shapes.flatMap((item) => item.fields), + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + artifactBytes = bytes.length; + evidenceSeen = true; + return { + evidence_id: evidence.evidence_id, + subject: evidence.subject, + operation: evidence.operation, + artifact_sha256: artifactDigest, + byte_length: artifactBytes, + shapes, + }; +}; +const diagnosticFile = join(lab, "native-doctor.json"); +const diagnosticCode = + "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));"; +// Query the saved result through a native agent command. Only the facts needed +// for this analyst question enter context; the complete named-schema Evidence +// and its digest remain available in the native artifact. +const recoveryCode = + "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));"; +const fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "grok", + onRequest: async ({ body, tools }) => { + // Native title/summary requests cannot execute the analyst workflow. + if (!tools.some((tool) => tool.name === "search_tool")) + return { role: "assistant", content: "REA fixture auxiliary response" }; + turn++; + assert( + containsText(body.messages, skillDirectory), + "Grok must discover the isolated installed skill", + ); + assert( + !containsText(body.messages, join(homedir(), ".agents", "skills")), + "Do not inherit real personal skills", + ); + if (mode === "chat") { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const replies = body.messages.filter((message) => message.role === "tool"); + if (!skillRequested) { + skillRequested = true; + return call("read_file", { + target_file: join(skillDirectory, "SKILL.md"), + }); + } + assert( + expectedSkill !== undefined && containsText(replies, expectedSkill), + "Native read_file must return the full skill with its documented line-number rendering", + ); + skillLoaded = true; + const last = replies.at(-1)?.content; + assert.equal(typeof last, "string"); + if (!catalogChecked) { + const discovery = + lastTool === "search_tool" ? JSON.parse(last) : undefined; + if (discovery?.status === "partial" && !diagnosticsRequested) { + diagnosticsRequested = true; + const diagnostic = [ + process.execPath, + join(runtimeRoot, "scripts/rea.mjs"), + "doctor", + "--client", + "grok_build", + "--skill", + "--json", + ] + .map(shellQuote) + .join(" "); + const projection = [ + process.execPath, + "-e", + diagnosticCode, + diagnosticFile, + ] + .map(shellQuote) + .join(" "); + return call("run_terminal_command", { + command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`, + description: + "Diagnose REA registration while native MCP discovery is pending", + }); + } + if (lastTool === "run_terminal_command") { + const line = last + .split("\n") + .find((item) => item.startsWith('{"rea_client_diagnostic":')); + assert( + line, + "Native REA diagnostics failed or did not return the selected status", + ); + const diagnostic = JSON.parse(line).rea_client_diagnostic; + assert.equal(diagnostic.healthy, true); + assert.equal(diagnostic.skill, "aligned"); + assert( + diagnostic.registrations.some( + (registration) => + registration.state === "aligned" && + registration.config_path === configPath, + ), + ); + diagnosticsVerified = true; + } + if (discovery?.status !== "ready") + return call("search_tool", { query: "rea", limit: 255 }); + const inventory = discovery.results + .filter((result) => result.server === "rea") + .flatMap((result) => result.tools); + assert.deepEqual( + inventory.map((tool) => tool.tool_name).sort(), + expectedNames, + "Native search_tool must discover the complete REA catalog", + ); + for (const tool of inventory) { + assert(tool.input_schema, `${tool.tool_name}: missing input schema`); + assert( + schemaValidator.validateSchema(tool.input_schema), + `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + await writeFile(join(lab, "native-discovery.json"), last); + catalogChecked = true; + analysisRequested = true; + return call("use_tool", { + tool_name: "rea__analyze_javascript_application", + tool_input: { input_path: target, format: "directory" }, + }); + } + if (!spillQueryRequested) { + const marker = "Full output written to: "; + const start = last.indexOf(marker); + if (start < 0) { + verifyEvidence(Buffer.from(last, "utf8")); + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + // The native hint explicitly calls for a query rather than read_file on + // the saved JSON's very long line. Preserve the original hint in requests. + const file = last + .slice(start + marker.length) + .split(". The full output ")[0]; + assert(isAbsolute(file)); + const childPath = relative(join(profile, "sessions"), file); + assert( + childPath !== "" && + !childPath.startsWith("..") && + !isAbsolute(childPath) && + file.endsWith(".json"), + "Unexpected native MCP spill path", + ); + retainedArtifact = file; + expectedProjection = verifyEvidence(await readFile(file)); + spillQueryRequested = true; + return call("run_terminal_command", { + command: [process.execPath, "-e", recoveryCode, file] + .map(shellQuote) + .join(" "), + description: + "Query full saved REA Evidence while preserving artifact identity", + }); + } + const line = last + .split("\n") + .find((item) => item.startsWith('{"rea_recovered":')); + assert( + line, + "The next model request must contain native artifact query output", + ); + const projection = JSON.parse(line).rea_recovered; + assert.deepEqual( + projection, + expectedProjection, + "Native query must recover the observed facts and complete artifact digest", + ); + projectionSeen = true; + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + }, +}); +const q = JSON.stringify; +const original = `# Keep unrelated settings and server state. +disabled_mcp_servers = ["rea", "other"] +[cli] +auto_update = false +[features] +telemetry = "off" +[skills] +paths = [${q(join(account, ".agents", "skills"))}] +ignore = [${q(join(homedir(), ".agents", "skills"))}] +[model.rea-client-fixture] +model = "rea-client-fixture" +base_url = ${q(fixture.baseUrl)} +api_key = "local-fixture-key" +api_backend = "chat_completions" +context_window = 1000000 +max_completion_tokens = 4096 +[mcp_servers.other] +command = "unrelated-server" +enabled = false +`; +await writeFile(configPath, original); +const rea = async (args) => + exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000 }, + ); +try { + clientVersion = ( + await exec(command, ["--version"], { env: environment, timeout: 30_000 }) + ).stdout.trim(); + for (const args of [ + ["setup", "--client", "grok_build", "--dry-run"], + ["setup", "--client", "grok_build", "--yes"], + ["doctor", "--client", "grok_build", "--skill"], + ]) { + const result = await rea(args); + await writeFile( + join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert( + parsed.plannedActions.some((action) => action.target === configPath), + ); + assert( + parsed.plannedActions.some( + (action) => action.target === skillDirectory, + ), + ); + assert( + parsed.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse writes outside the owned profile/skill", + ); + } + } + } + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert( + configured.includes("unrelated-server") && + configured.includes("# Keep unrelated settings"), + ); + const repeated = JSON.parse( + (await rea(["setup", "--client", "grok_build", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); + expectedSkill = skill + .trimEnd() + .split("\n") + .map((line, index) => + index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line, + ) + .join("\n"); + const execution = exec( + command, + [ + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + "-m", + "rea-client-fixture", + "--output-format", + "streaming-messages-json", + "--always-approve", + "--no-subagents", + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + }, + ); + execution.child.stdin.end(); + const response = await execution; + await writeFile(join(lab, `${mode}.stdout`), response.stdout); + await writeFile(join(lab, `${mode}.stderr`), response.stderr); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const init = events.find( + (event) => event.type === "system" && event.subtype === "init", + ); + assert(init?.mcp_servers?.some((server) => server.name === "rea")); + assert( + init.mcp_servers.every((server) => ["rea", "other"].includes(server.name)), + "Do not inherit foreign MCP configuration", + ); + assert( + events.some( + (event) => + event.type === "result" && + event.subtype === "success" && + event.is_error === false && + event.result === "REA_CLIENT_COMPATIBILITY_OK", + ), + ); + assert(finalSeen); + if (mode === "call") + assert( + skillLoaded && + catalogChecked && + analysisRequested && + evidenceSeen && + (!spillQueryRequested || projectionSeen) && + (!diagnosticsRequested || diagnosticsVerified), + ); +} catch (cause) { + failure = cause; + if (typeof cause?.stdout === "string") + await writeFile(join(lab, `${mode}.stdout`), cause.stdout); + if (typeof cause?.stderr === "string") + await writeFile(join(lab, `${mode}.stderr`), cause.stderr); +} finally { + await fixture.close(); + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Grok Build", + clientVersion, + mode, + passed: failure === undefined, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim", + discovery: + "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified", + interface: + "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal", + catalogChecked, + catalogSize: catalogChecked ? expectedNames.length : 0, + schemasChecked, + skillLoaded, + diagnosticsRequested, + diagnosticsVerified, + analysisRequested, + evidenceSeen, + spillQueryRequested, + projectionSeen, + retainedArtifact, + artifactBytes, + artifactDigest, + finalSeen, + evidenceScope: + "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact", + requests: fixture.requests, + probes: fixture.probes, + fixtureFailure: fixture.failure, + verifier: lineage, + }; + await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); + console.log(JSON.stringify(receipt)); +} +if (failure !== undefined) throw failure;