diff --git a/docs/testing.md b/docs/testing.md index c38805fb0..ecb2600b6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -187,6 +187,30 @@ The loopback OpenAI completions/SSE custom model declares a one-million-token context and synthetic usage. Live xAI APIs, native Windows and default OS-home skill discovery remain unverified. +`verify:commandcode-client` requires installed Command Code (verified with npm +1.79.2); select it with `REA_VERIFY_COMMANDCODE_COMMAND`. Auth, MCP tokens and +sessions use the actual OS home, so provision a disposable POSIX account rather +than overriding `HOME`. Set `REA_VERIFY_COMMANDCODE_ACCOUNT_HOME` to that account's +actual home and create `.rea-client-verification` there with exactly +`Disposable REA client verification account` followed by a newline. The lane +owns that account's `.commandcode` configuration and shared REA skill. Build the +runtime as the checkout owner, then run `node scripts/verify-commandcode-client.mjs` +as the disposable account; the npm shortcut also needs a writable checkout. +The verifier does not create accounts or install clients. It guards setup +targets, preserves an unrelated registration and backups, checks idempotence, +and verifies native default shared-skill discovery and complete activation. +Default deferred-schema delivery stays enabled: the native prompt advertises +every REA tool, exact-name `search_tools` returns every input schema, and a native +shell query recovers the saved catalog's count and digest when the client spills +it. Actual analysis of a Unicode-path fixture delivers full named-schema Evidence +to the next model request. Use `-- chat` for ordinary chat or +`REA_VERIFY_RUNTIME_ROOT` for a production-only installed package. The loopback +OpenAI completions/SSE model is a keyless BYOK endpoint declaring a million-token +context with synthetic usage. A synthetic account-key value satisfies the +client's print-mode gate; native local-only mode refuses hosted API calls. +Updates, telemetry and cron are disabled. Hosted authentication, live models +and Windows remain unverified. + `verify:omp-client` requires installed OMP (verified with the official Linux x64 18.8.7 binary); select it with `REA_VERIFY_OMP_COMMAND`. This optional POSIX lane isolates the default agent, global config and XDG roots, guards setup targets, diff --git a/package.json b/package.json index 01a42e359..13900238f 100644 --- a/package.json +++ b/package.json @@ -188,6 +188,7 @@ "verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs", "verify:opencode-client": "npm run build:cached && node scripts/verify-opencode-client.mjs", "verify:grok-client": "npm run build:cached && node scripts/verify-grok-client.mjs", + "verify:commandcode-client": "npm run build:cached && node scripts/verify-commandcode-client.mjs", "verify:omp-client": "npm run build:cached && node scripts/verify-omp-client.mjs", "verify:copilot-client": "npm run build:cached && node scripts/verify-copilot-client.mjs", "verify:claude-client": "npm run build:cached && node scripts/verify-claude-client.mjs", diff --git a/scripts/verify-commandcode-client.mjs b/scripts/verify-commandcode-client.mjs new file mode 100644 index 000000000..71e97deab --- /dev/null +++ b/scripts/verify-commandcode-client.mjs @@ -0,0 +1,452 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { isAbsolute, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import Ajv2020 from "ajv/dist/2020.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../dist/contracts/toolContracts.js"; +import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; + +assert( + process.platform !== "win32", + "This verifier requires a disposable POSIX account; Windows is unverified", +); +const account = process.env.REA_VERIFY_COMMANDCODE_ACCOUNT_HOME; +assert( + account && isAbsolute(account) && account === homedir(), + "Run as a disposable account whose actual OS home matches REA_VERIFY_COMMANDCODE_ACCOUNT_HOME; do not override HOME", +); +assert.equal( + await readFile(join(account, ".rea-client-verification"), "utf8"), + "Disposable REA client verification account\n", +); +const repo = fileURLToPath(new URL("..", import.meta.url)); +const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; +const command = process.env.REA_VERIFY_COMMANDCODE_COMMAND ?? "cmd"; +const mode = process.argv[2] ?? "call"; +assert( + ["chat", "call"].includes(mode), + "Usage: verify-commandcode-client.mjs [chat|call]", +); +const verifier = createVerifierRun(); +const lab = await mkdtemp(join(account, "rea-commandcode-client-")); +const workspace = join(lab, "workspace"); +const target = join(workspace, "fixture α测试"); +const profile = join(account, ".commandcode"); +const configPath = join(profile, "mcp.json"); +const skillDirectory = join( + account, + ".agents", + "skills", + "reverse-engineer-anything", +); +await Promise.all([ + mkdir(target, { recursive: true }), + mkdir(profile, { recursive: true }), + mkdir(join(lab, "tmp")), +]); +await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); +await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', +); +const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + TMPDIR: join(lab, "tmp"), + CMD_LOCAL_ONLY: "1", + // Native print mode checks for an account key even with a keyless BYOK model. + // This synthetic value is never a real credential; local-only refuses hosted API calls. + COMMAND_CODE_API_KEY: "local-fixture-only", + COMMANDCODE_SKIP_UPDATES: "1", + COMMANDCODE_DISABLE_CRON: "1", + OTEL_SDK_DISABLED: "true", + REA_PROCESS_RUN_ID: verifier.run_id, +}; +const expectedNames = TOOL_CONTRACTS.map( + (tool) => "mcp__rea__" + tool.name, +).sort(); +const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); +const textValues = (value) => + typeof value === "string" + ? [value] + : value !== null && typeof value === "object" + ? Object.values(value).flatMap(textValues) + : []; +let clientVersion; +let expectedSkill; +let inventoryChecked = false; +let skillLoaded = false; +let schemasChecked = 0; +let catalogArtifact; +let catalogDigest; +let catalogProjection; +let artifactProjectionSeen = false; +let evidenceSeen = false; +let evidenceBytes; +let finalSeen = false; +let stage = 0; +let failure; +const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_commandcode_${++stage}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], +}); +const validateCatalog = async (text) => { + const entries = [ + ...text.matchAll( + /^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm, + ), + ]; + assert.deepEqual( + entries.map((entry) => entry[1]).sort(), + expectedNames, + "Native exact-name search must return every REA input schema", + ); + for (const entry of entries) { + const schema = JSON.parse(entry[2]); + assert( + schemaValidator.validateSchema(schema), + `${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + } + schemasChecked = entries.length; + catalogDigest = createHash("sha256").update(text).digest("hex"); + await writeFile(join(lab, "native-catalog.md"), text); +}; +const fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "commandcode", + onRequest: async ({ body, tools }) => { + assert(tools.some((tool) => tool.name === "search_tools")); + const strings = textValues(body.messages); + if (!inventoryChecked) { + const names = [ + ...new Set( + strings.flatMap( + (text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [], + ), + ), + ].sort(); + assert.deepEqual( + names, + expectedNames, + "Default native deferred-tool prompt must advertise the complete REA inventory", + ); + assert.equal( + tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length, + 0, + "Keep native lazy schema delivery enabled", + ); + assert( + tools + .find((tool) => tool.name === "activate_skill") + ?.parameters.properties.name.enum.includes( + "reverse-engineer-anything", + ), + ); + inventoryChecked = true; + } + if (mode === "chat") { + assert.equal( + body.messages.filter((message) => message.role === "tool").length, + 0, + ); + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + if (stage === 0) + return call("activate_skill", { name: "reverse-engineer-anything" }); + assert( + strings.some((text) => text.includes(expectedSkill)), + "The model must receive the complete activated skill body", + ); + skillLoaded = true; + const last = body.messages + .filter((message) => message.role === "tool") + .at(-1)?.content; + assert.equal(typeof last, "string"); + if (stage === 1) + return call("search_tools", { + query: "select:" + expectedNames.join(","), + }); + if (stage === 2) { + const spill = last.match( + /\[full output saved to: (.+?) — read it with read_file/, + ); + if (spill) { + catalogArtifact = spill[1]; + const child = relative(environment.TMPDIR, catalogArtifact); + assert( + isAbsolute(catalogArtifact) && + child && + !child.startsWith("..") && + !isAbsolute(child), + "Native spill must remain inside this run's temporary directory", + ); + const text = await readFile(catalogArtifact, "utf8"); + await validateCatalog(text); + catalogProjection = { schemas: schemasChecked, sha256: catalogDigest }; + const query = + "const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));"; + return call("shell_command", { + command: process.execPath, + args: ["-e", query, catalogArtifact], + }); + } + await validateCatalog(last); + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + } + if (catalogArtifact && !artifactProjectionSeen) { + const line = last + .split("\n") + .find((item) => item.startsWith('{"rea_catalog":')); + assert(line, "Native shell must return the selected saved-catalog facts"); + assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection); + artifactProjectionSeen = true; + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + } + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(last)); + assert.equal(evidence.subject.local_path, target); + const shapes = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .filter( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert( + shapes.length > 0 && + shapes.every( + (shape) => shape.return_shape_coverage?.projection_complete, + ), + ); + const fields = shapes.flatMap((shape) => + shape.static_return_shapes.flatMap((item) => item.fields), + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + evidenceBytes = Buffer.byteLength(last); + await writeFile(join(lab, "native-evidence.json"), last); + evidenceSeen = true; + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + }, +}); +await writeFile( + join(profile, "providers.json"), + JSON.stringify({ + provider: { + "rea-local": { + baseURL: fixture.baseUrl, + apiKey: false, + models: { + "rea-client-fixture": { contextWindow: 1_000_000, maxOutput: 4096 }, + }, + }, + }, + }), +); +const original = + JSON.stringify( + { + mcpServers: { other: { command: "unrelated-server", enabled: false } }, + preserved: "keep unrelated settings", + }, + null, + 2, + ) + "\n"; +await writeFile(configPath, original); +const rea = (args) => + exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000 }, + ); +try { + clientVersion = ( + await exec(command, ["--version"], { + env: environment, + cwd: workspace, + timeout: 30_000, + }) + ).stdout.trim(); + for (const args of [ + ["setup", "--client", "commandcode", "--dry-run"], + ["setup", "--client", "commandcode", "--yes"], + ["doctor", "--client", "commandcode", "--skill"], + ]) { + const response = await rea(args); + await writeFile( + join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), + response.stdout, + ); + const result = JSON.parse(response.stdout); + if (args[0] === "doctor") assert.equal(result.healthy, true); + else + assert.equal( + result.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert( + result.plannedActions.some((action) => action.target === configPath), + ); + assert( + result.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse setup writes outside the disposable account's configuration and skill", + ); + } + } + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.equal(JSON.parse(configured).preserved, "keep unrelated settings"); + assert.equal( + JSON.parse(configured).mcpServers.other.command, + "unrelated-server", + ); + const repeated = JSON.parse( + (await rea(["setup", "--client", "commandcode", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8")) + .replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "") + .trim(); + const execution = exec( + command, + [ + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + "-m", + "rea-local/rea-client-fixture", + "--local-only", + "--skip-onboarding", + "--trust", + "--output-format", + "json", + "--max-turns", + "8", + "--yolo", + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + }, + ); + execution.child.stdin.end(); + const response = await execution; + await writeFile(join(lab, `${mode}.stdout`), response.stdout); + await writeFile(join(lab, `${mode}.stderr`), response.stderr); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert( + events.some( + (event) => + event.type === "result" && + event.subtype === "success" && + event.finalText === "REA_CLIENT_COMPATIBILITY_OK", + ), + ); + assert(inventoryChecked && finalSeen); + if (mode === "call") + assert( + skillLoaded && + schemasChecked === expectedNames.length && + evidenceSeen && + (!catalogArtifact || artifactProjectionSeen), + ); +} catch (cause) { + failure = cause; + if (typeof cause?.stdout === "string") + await writeFile(join(lab, `${mode}.stdout`), cause.stdout); + if (typeof cause?.stderr === "string") + await writeFile(join(lab, `${mode}.stderr`), cause.stderr); +} finally { + await fixture.close(); + const receipt = { + client: "Command Code", + clientVersion, + mode, + passed: failure === undefined, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim", + discovery: + "actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified", + interface: + "default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call", + inventoryChecked, + catalogSize: inventoryChecked ? expectedNames.length : 0, + schemasChecked, + skillLoaded, + catalogArtifact, + catalogDigest, + artifactProjectionSeen, + evidenceSeen, + evidenceBytes, + finalSeen, + requests: fixture.requests, + probes: fixture.probes, + fixtureFailure: fixture.failure, + verifier: await completeVerifierRun(verifier), + }; + await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); + console.log(JSON.stringify(receipt)); +} +if (failure !== undefined) throw failure; diff --git a/src/domain/adb/adbDeviceAnalysis.test.ts b/src/domain/adb/adbDeviceAnalysis.test.ts new file mode 100644 index 000000000..822292945 --- /dev/null +++ b/src/domain/adb/adbDeviceAnalysis.test.ts @@ -0,0 +1,82 @@ +import { expect, it } from "vitest"; + +import { adbInputSchemas } from "./adbDeviceAnalysis.js"; + +it("preserves ADB key, action and component admission when making advertised patterns portable", () => { + const serial = "emulator-5554"; + const variants = [ + { + previous: /^[A-Za-z0-9._-]+$/u, + max: 256, + accepts: (key: string) => + adbInputSchemas.read_adb_setting.safeParse({ + serial, + namespace: "global", + key, + }).success, + }, + { + previous: /^[A-Za-z0-9._-]+$/u, + max: 256, + accepts: (key: string) => + adbInputSchemas.start_adb_activity.safeParse({ + serial, + action: "android.intent.action.VIEW", + extras: [{ key, type: "string", value: "payload" }], + }).success, + }, + { + previous: /^[A-Za-z][A-Za-z0-9._-]*$/u, + max: 256, + accepts: (action: string) => + adbInputSchemas.start_adb_activity.safeParse({ serial, action }) + .success, + }, + { + previous: /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]*)?$/u, + max: 512, + accepts: (component: string) => + adbInputSchemas.start_adb_activity.safeParse({ + serial, + action: "android.intent.action.VIEW", + component, + }).success, + }, + ]; + const samples = [ + "", + "screen_brightness", + "0", + "_", + ".", + "-", + "android.intent.action.VIEW", + "com.example/.MainActivity", + "com.example/", + "com.example/a/b", + "a\n", + "a\r\n", + "é", + "aé", + "a b", + "a\\b", + "a".repeat(256), + "a".repeat(257), + "a".repeat(512), + "a".repeat(513), + ]; + for (let code = 0; code < 128; code += 1) { + const character = String.fromCharCode(code); + samples.push(character, `a${character}`, `a${character}b`); + } + for (const variant of variants) + for (const sample of samples) + expect( + variant.accepts(sample), + JSON.stringify({ pattern: variant.previous.source, sample }), + ).toBe( + sample.length > 0 && + sample.length <= variant.max && + variant.previous.test(sample), + ); +}); diff --git a/src/domain/adb/adbDeviceAnalysis.ts b/src/domain/adb/adbDeviceAnalysis.ts index ef66952ce..5966f508b 100644 --- a/src/domain/adb/adbDeviceAnalysis.ts +++ b/src/domain/adb/adbDeviceAnalysis.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { localPathStringSchema } from "../localPath.js"; +import { IDENTIFIER_PATTERN } from "../stringPatterns.js"; /** * Device serials as adb prints them: emulator-5554, USB identifiers, and @@ -29,6 +30,9 @@ const packageNameSchema = z "Package name must be dot-separated Java identifiers", ); +const keyStringSchema = z.string().min(1).max(256); +const keyPattern = /^[A-Za-z0-9._\x2d]+$/u; + /** * Caller intent for ADB-backed device inspection and acquisition. * @@ -174,14 +178,8 @@ export const adbInputSchemas = { namespace: z .enum(["system", "secure", "global"]) .describe("Settings namespace to read"), - key: z - .string() - .min(1) - .max(256) - .regex( - /^[A-Za-z0-9._\-]+$/u, - "Setting keys are restricted to identifiers", - ) + key: keyStringSchema + .regex(keyPattern, "Setting keys are restricted to identifiers") .describe("Setting key to read"), }), collect_adb_bugreport: z.strictObject({ @@ -241,7 +239,7 @@ export const adbInputSchemas = { .min(1) .max(256) .regex( - /^[A-Za-z][A-Za-z0-9._\-]*$/u, + IDENTIFIER_PATTERN, "Intent action must be a dotted identifier such as android.intent.action.VIEW", ) .describe("Intent action to start"), @@ -257,7 +255,7 @@ export const adbInputSchemas = { .min(1) .max(512) .regex( - /^[A-Za-z0-9._\-]+(\/[A-Za-z0-9._\-]*)?$/u, + /^[A-Za-z0-9._\x2d]+(\/[A-Za-z0-9._\x2d]*)?$/u, "Component must be package/activity form", ) .optional() @@ -265,11 +263,8 @@ export const adbInputSchemas = { extras: z .array( z.strictObject({ - key: z - .string() - .min(1) - .max(256) - .regex(/^[A-Za-z0-9._\-]+$/u, "Extra keys are identifiers") + key: keyStringSchema + .regex(keyPattern, "Extra keys are identifiers") .describe("Extra key"), type: z .enum(["string", "boolean", "int", "long", "float"])