593 lines
22 KiB
JavaScript
593 lines
22 KiB
JavaScript
import { spawn } from "node:child_process";
|
||
import {
|
||
chmod,
|
||
copyFile,
|
||
mkdir,
|
||
mkdtemp,
|
||
readFile,
|
||
rm,
|
||
writeFile,
|
||
} from "node:fs/promises";
|
||
import { homedir } from "node:os";
|
||
import { delimiter, dirname, join, resolve } from "node:path";
|
||
import { createInterface } from "node:readline";
|
||
import { fileURLToPath } from "node:url";
|
||
import { z } from "zod";
|
||
|
||
import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js";
|
||
import {
|
||
createAgentEvaluationFixtures,
|
||
agentFixtureClaims,
|
||
agentFixtureAnswerInstructions,
|
||
} from "../dist/evaluation/AgentEvaluationFixtures.js";
|
||
import {
|
||
agentEvaluationPassed,
|
||
summarizeFactualCorrectness,
|
||
} from "../dist/evaluation/AgentEvaluationReport.js";
|
||
import { PRODUCT_IDENTITY } from "../dist/identity.js";
|
||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||
|
||
const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||
const verifierRun = createVerifierRun();
|
||
const timeoutMs = Number(process.env.REA_AGENT_EVAL_TIMEOUT_MS ?? 480_000);
|
||
const codex = process.env.REA_CODEX_CLI ?? "codex";
|
||
const optionalModel = process.env.REA_AGENT_EVAL_MODEL;
|
||
|
||
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10_000)
|
||
throw new Error("REA_AGENT_EVAL_TIMEOUT_MS must be an integer >= 10000");
|
||
// Codex refuses helper aliases when CODEX_HOME is under the OS temp directory.
|
||
// Keep the disposable account in a private cache, then remove the owned run.
|
||
const evaluationParent = resolve(
|
||
process.env.REA_AGENT_EVAL_ROOT ??
|
||
join(homedir(), ".cache", "rea-agent-evaluations"),
|
||
);
|
||
await mkdir(evaluationParent, { recursive: true, mode: 0o700 });
|
||
const evaluationRoot = await mkdtemp(join(evaluationParent, "run-"));
|
||
const fixtureRoot = join(evaluationRoot, "targets");
|
||
const account = join(evaluationRoot, "account");
|
||
const codexHome = join(evaluationRoot, "codex");
|
||
const authPath = join(codexHome, "auth.json");
|
||
const skillDestination = join(
|
||
account,
|
||
".agents/skills/reverse-engineer-anything",
|
||
);
|
||
const evaluationEnvironment = {
|
||
...Object.fromEntries(
|
||
Object.entries(process.env).filter(
|
||
([name]) => !/^(?:REA_|CODEX_|GHIDRA_|HOPPER_|IDA_)/u.test(name),
|
||
),
|
||
),
|
||
HOME: account,
|
||
USERPROFILE: account,
|
||
CODEX_HOME: codexHome,
|
||
XDG_CONFIG_HOME: join(account, ".config"),
|
||
XDG_DATA_HOME: join(account, ".local/share"),
|
||
XDG_CACHE_HOME: join(account, ".cache"),
|
||
REA_PROCESS_RUN_ID: verifierRun.run_id,
|
||
PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH ?? ""}`,
|
||
};
|
||
|
||
try {
|
||
const codexVersion = await readCommandVersion(codex);
|
||
await Promise.all([
|
||
mkdir(fixtureRoot, { recursive: true }),
|
||
mkdir(account),
|
||
mkdir(codexHome, { mode: 0o700 }),
|
||
]);
|
||
// Reuse authentication only. User config, plugins, rules, history and skills
|
||
// must not make a fresh installed-skill trial pass accidentally.
|
||
try {
|
||
await copyFile(
|
||
join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"),
|
||
authPath,
|
||
);
|
||
await chmod(authPath, 0o600);
|
||
} catch (cause) {
|
||
if (cause?.code !== "ENOENT") throw cause;
|
||
// Environment/API-key authentication can work without a local auth file.
|
||
}
|
||
await writeFile(
|
||
join(codexHome, "config.toml"),
|
||
'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n',
|
||
{ mode: 0o600 },
|
||
);
|
||
for (const approval of ["--dry-run", "--yes"]) {
|
||
await runProcess(
|
||
process.execPath,
|
||
[
|
||
join(repositoryRoot, "scripts/rea.mjs"),
|
||
"setup",
|
||
"--client",
|
||
"codex",
|
||
approval,
|
||
"--json",
|
||
],
|
||
evaluationRoot,
|
||
120_000,
|
||
evaluationEnvironment,
|
||
);
|
||
}
|
||
const installedSkill = await readFile(
|
||
join(skillDestination, "SKILL.md"),
|
||
"utf8",
|
||
);
|
||
if (!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`))
|
||
throw new Error("Setup did not install the current packaged REA skill");
|
||
const requestedScenarioIds = new Set(
|
||
(process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
|
||
.split(",")
|
||
.map((value) => value.trim())
|
||
.filter((value) => value.length > 0),
|
||
);
|
||
const targets = await createTargets(
|
||
fixtureRoot,
|
||
requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
|
||
);
|
||
const allScenarios = [
|
||
{
|
||
id: "native",
|
||
expectedFirstTool: "open_binary",
|
||
requiresEvidence: false,
|
||
requiredAnswerTermGroups: [["unavailable", "could not", "provider"]],
|
||
// The verifier authorizes these named session operations, not arbitrary
|
||
// writes or runtime execution. Missing-engine behavior stays observable.
|
||
approvedTools: ["open_binary", "close_binary"],
|
||
requiredToolSubsequence: ["open_binary", "close_binary"],
|
||
prompt: `Explain what static artifact evidence establishes about the shipped native program at ${targets.native}. Do not execute the target or install tools. Clearly state unavailable provider evidence, and close any analysis session you opened before finishing.`,
|
||
},
|
||
{
|
||
id: "asar",
|
||
expectedFirstTool: "analyze_javascript_application",
|
||
requiresEvidence: true,
|
||
requiredAnswerTermGroups: [
|
||
["profileapi"],
|
||
["profile:read"],
|
||
["preload", "contextbridge"],
|
||
],
|
||
prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`,
|
||
},
|
||
{
|
||
id: "asar-zh",
|
||
fixtureScenarioId: "asar",
|
||
expectedFirstTool: "analyze_javascript_application",
|
||
requiresEvidence: true,
|
||
requiredAnswerTermGroups: [["profileapi"], ["profile:read"]],
|
||
prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`,
|
||
},
|
||
{
|
||
id: "javascript-module-view",
|
||
expectedFirstTool: "analyze_javascript_application",
|
||
requiresEvidence: true,
|
||
requiredToolSubsequence: [
|
||
"analyze_javascript_application",
|
||
"inspect_analysis_view",
|
||
"inspect_analysis_view",
|
||
],
|
||
requiredAnswerTermGroups: [
|
||
["main.js"],
|
||
["node", "identity"],
|
||
["observations", "recorded"],
|
||
],
|
||
prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`,
|
||
},
|
||
{
|
||
id: "missing-target",
|
||
expectedFirstTool: null,
|
||
requiresEvidence: false,
|
||
requiredAnswerTermGroups: [],
|
||
prompt:
|
||
'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. End with one JSON object containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).',
|
||
},
|
||
{
|
||
id: "javascript-export-shape",
|
||
expectedFirstTool: "analyze_javascript_application",
|
||
requiresEvidence: true,
|
||
requiredToolSubsequence: [
|
||
"analyze_javascript_application",
|
||
"analyze_javascript_application",
|
||
"compare_javascript_export_shapes",
|
||
],
|
||
requiredAnswerTermGroups: [
|
||
["/depth", "depth"],
|
||
[
|
||
"literal value `1`",
|
||
"literal `1`",
|
||
"depth: 1",
|
||
'"depth": 1',
|
||
"depth = 1",
|
||
],
|
||
["static", "inferred"],
|
||
["runtime", "behavioral probes"],
|
||
],
|
||
prompt: `Without directly reading target files, compare the default export in parser.mjs between ${targets.javascriptShapeLeft} and ${targets.javascriptShapeRight}. Use REA to analyze each shipped artifact and then compare its exact static export return shapes. State the exact heading-shape change, cite produced Evidence, and distinguish static inference from runtime semantics.`,
|
||
},
|
||
{
|
||
id: "managed",
|
||
expectedFirstTool: "inspect_managed_artifact",
|
||
requiresEvidence: true,
|
||
requiredAnswerTermGroups: [["profile"], ["main", "entry point"]],
|
||
requiredToolSubsequence: [
|
||
"inspect_managed_artifact",
|
||
"inspect_managed_members",
|
||
],
|
||
prompt: `Explain the public managed types and entry point in ${targets.managed}. Use shipped-artifact evidence and distinguish unavailable behavior from observed metadata.`,
|
||
},
|
||
{
|
||
id: "browser",
|
||
expectedFirstTool: "list_browser_targets",
|
||
requiresEvidence: false,
|
||
requiredAnswerTermGroups: [
|
||
["unavailable", "could not", "connection", "endpoint"],
|
||
],
|
||
prompt:
|
||
"A user-owned page is already open at http://127.0.0.1:3000 through the approved local debugging endpoint http://127.0.0.1:9222. Inspect what is available without navigating or executing page code, and report any authority or availability limits.",
|
||
},
|
||
{
|
||
id: "navigation-context",
|
||
expectedFirstTool: "get_navigation_context",
|
||
requiresEvidence: false,
|
||
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
|
||
prompt:
|
||
"Report the selected document, current address, and containing procedure from the current native-analysis session in one coherent request. If no native session is available, report that limitation plainly.",
|
||
},
|
||
{
|
||
id: "address-context",
|
||
expectedFirstTool: "inspect_address_context",
|
||
requiresEvidence: false,
|
||
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
|
||
prompt:
|
||
"Inspect explicit address 0x401000 in the current native-analysis session. Report its analyzed name, containing procedure, regular comment, inline comment, and matching bookmarks. If no native session is available, report that limitation plainly.",
|
||
},
|
||
];
|
||
const scenarios =
|
||
requestedScenarioIds.size === 0
|
||
? allScenarios
|
||
: allScenarios.filter(({ id }) => requestedScenarioIds.has(id));
|
||
if (
|
||
requestedScenarioIds.size > 0 &&
|
||
scenarios.length !== requestedScenarioIds.size
|
||
)
|
||
throw new Error(
|
||
`Unknown or duplicate REA_AGENT_EVAL_SCENARIOS value: ${[...requestedScenarioIds].join(",")}`,
|
||
);
|
||
const results = [];
|
||
for (const scenario of scenarios) {
|
||
process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`);
|
||
const fixtureId = scenario.fixtureScenarioId ?? scenario.id;
|
||
const fixtureClaims = agentFixtureClaims(fixtureId, targets);
|
||
const answerInstructions = agentFixtureAnswerInstructions(fixtureId);
|
||
const execution = await runCodex(
|
||
answerInstructions.length === 0
|
||
? scenario.prompt
|
||
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
|
||
scenario.approvedTools ?? [],
|
||
);
|
||
const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
|
||
if (transcriptDirectory !== undefined) {
|
||
const transcriptPath = resolve(
|
||
transcriptDirectory,
|
||
`${scenario.id}.jsonl`,
|
||
);
|
||
await mkdir(dirname(transcriptPath), { recursive: true });
|
||
await writeFile(
|
||
transcriptPath,
|
||
`${execution.events.map((event) => JSON.stringify(event)).join("\n")}\n`,
|
||
);
|
||
}
|
||
const metrics = evaluateCodexEvents(
|
||
execution.events,
|
||
scenario.expectedFirstTool,
|
||
{
|
||
requireEvidence: scenario.requiresEvidence,
|
||
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
|
||
requiredToolSubsequence: scenario.requiredToolSubsequence,
|
||
forbidInputValidationFailures: true,
|
||
...(fixtureClaims === undefined ? {} : { fixtureClaims }),
|
||
},
|
||
);
|
||
// Fixtures are deliberately visible beside prior investigation artifacts.
|
||
// Their existence does not identify the unnamed app in this request.
|
||
const targetClarificationPassed =
|
||
scenario.expectedFirstTool === null
|
||
? validTargetClarification(metrics.finalMessage)
|
||
: undefined;
|
||
const result = {
|
||
id: scenario.id,
|
||
expectedFirstTool: scenario.expectedFirstTool,
|
||
requiresEvidence: scenario.requiresEvidence,
|
||
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
|
||
requiredToolSubsequence: scenario.requiredToolSubsequence ?? [],
|
||
configuredClaimIds: fixtureClaims?.map(({ id }) => id) ?? [],
|
||
exitCode: execution.exitCode,
|
||
stderr: execution.stderr,
|
||
...metrics,
|
||
...(targetClarificationPassed === undefined
|
||
? {}
|
||
: { targetClarificationPassed }),
|
||
};
|
||
results.push(result);
|
||
process.stderr.write(
|
||
`${scenario.id}: first=${metrics.firstTool ?? "none"}, calls=${String(metrics.reaCalls.length)}, repeats=${String(metrics.repeatedCallCount)}, validation_failures=${String(metrics.inputValidationFailureCount)}, input_tokens=${String(metrics.inputTokens)}\n`,
|
||
);
|
||
}
|
||
|
||
const factualCorrectness = summarizeFactualCorrectness(results);
|
||
const summary = {
|
||
schemaVersion: 4,
|
||
evaluationScope: "routing_workflow_and_configured_fixture_claims",
|
||
factualCorrectness,
|
||
verifier_run: await completeVerifierRun(verifierRun),
|
||
codex,
|
||
codexVersion,
|
||
model: optionalModel ?? null,
|
||
installation: {
|
||
method: "rea_setup_codex",
|
||
isolatedHome: true,
|
||
isolatedCodexHome: true,
|
||
skillVersion: PRODUCT_IDENTITY.skillVersion,
|
||
authentication: "caller_auth_only_removed_after_run",
|
||
},
|
||
scenarios: results,
|
||
totals: {
|
||
scenarios: results.length,
|
||
naturalUse: results.filter(({ naturalUse }) => naturalUse).length,
|
||
correctFirstTool: results.filter(
|
||
({ correctFirstTool }) => correctFirstTool,
|
||
).length,
|
||
repeatedCallCount: results.reduce(
|
||
(total, { repeatedCallCount }) => total + repeatedCallCount,
|
||
0,
|
||
),
|
||
inputValidationFailureCount: results.reduce(
|
||
(total, { inputValidationFailureCount }) =>
|
||
total + inputValidationFailureCount,
|
||
0,
|
||
),
|
||
requiredToolSubsequence: results.filter(
|
||
({ requiredToolSubsequenceMet }) => requiredToolSubsequenceMet,
|
||
).length,
|
||
inputTokens: results.reduce(
|
||
(total, { inputTokens }) => total + inputTokens,
|
||
0,
|
||
),
|
||
cachedInputTokens: results.reduce(
|
||
(total, { cachedInputTokens }) => total + cachedInputTokens,
|
||
0,
|
||
),
|
||
answerHeuristicsMet: results.filter(({ answerHeuristicsMet }) =>
|
||
Boolean(answerHeuristicsMet),
|
||
).length,
|
||
epistemicCuePresent: results.filter(({ epistemicCuePresent }) =>
|
||
Boolean(epistemicCuePresent),
|
||
).length,
|
||
factualScenariosPassed: factualCorrectness.counts.passed,
|
||
factualScenariosFailed: factualCorrectness.counts.failed,
|
||
factualScenariosNotAssessed: factualCorrectness.counts.notAssessed,
|
||
},
|
||
};
|
||
const encoded = `${JSON.stringify(summary, null, 2)}\n`;
|
||
process.stdout.write(encoded);
|
||
const reportPath = process.env.REA_AGENT_EVAL_REPORT_PATH;
|
||
if (reportPath !== undefined) {
|
||
await mkdir(dirname(resolve(reportPath)), { recursive: true });
|
||
await writeFile(resolve(reportPath), encoded);
|
||
}
|
||
|
||
const failed = results.filter((result) => !agentEvaluationPassed(result));
|
||
if (failed.length > 0)
|
||
throw new Error(
|
||
`Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
|
||
);
|
||
} finally {
|
||
// Even a deliberately retained fixture directory must never retain auth.
|
||
await rm(authPath, { force: true });
|
||
if (process.env.REA_AGENT_EVAL_KEEP_FIXTURES !== "true")
|
||
await rm(evaluationRoot, { recursive: true, force: true });
|
||
}
|
||
|
||
async function createTargets(root, includeManaged) {
|
||
const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot);
|
||
const largeJavascript = join(root, "large-app");
|
||
await mkdir(join(largeJavascript, "lib"), { recursive: true });
|
||
await Promise.all([
|
||
writeFile(
|
||
join(largeJavascript, "package.json"),
|
||
'{"name":"large-app","main":"main.js"}\n',
|
||
),
|
||
writeFile(
|
||
join(largeJavascript, "main.js"),
|
||
'const { ipcMain } = require("electron");\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n',
|
||
),
|
||
...Array.from({ length: 180 }, (_, index) =>
|
||
writeFile(
|
||
join(largeJavascript, "lib", `formatter-${String(index)}.js`),
|
||
`export const format = (value) => String(value).trim();\n`,
|
||
),
|
||
),
|
||
]);
|
||
|
||
const managedOutput = join(root, "managed-output");
|
||
if (includeManaged) {
|
||
const managedProject = join(root, "managed-project");
|
||
await mkdir(managedProject, { recursive: true });
|
||
await Promise.all([
|
||
writeFile(
|
||
join(managedProject, "AgentEval.csproj"),
|
||
'<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><OutputType>Exe</OutputType><TargetFramework>net8.0</TargetFramework><ImplicitUsings>enable</ImplicitUsings><Nullable>enable</Nullable></PropertyGroup></Project>\n',
|
||
),
|
||
writeFile(
|
||
join(managedProject, "Program.cs"),
|
||
'namespace AgentEval; public sealed record Profile(string Id); public static class Program { public static void Main() => Console.WriteLine(new Profile("fixture")); }\n',
|
||
),
|
||
]);
|
||
await runProcess(
|
||
"dotnet",
|
||
[
|
||
"build",
|
||
join(managedProject, "AgentEval.csproj"),
|
||
"--configuration",
|
||
"Release",
|
||
"--output",
|
||
managedOutput,
|
||
"--nologo",
|
||
],
|
||
root,
|
||
120_000,
|
||
);
|
||
}
|
||
return {
|
||
native: "/bin/true",
|
||
...fixtures,
|
||
largeJavascript,
|
||
managed: join(managedOutput, "AgentEval.dll"),
|
||
};
|
||
}
|
||
|
||
function validTargetClarification(message) {
|
||
const schema = z.strictObject({
|
||
status: z.literal("needs_target"),
|
||
requested_input: z.literal("app_name_or_artifact_path"),
|
||
question: z.string().refine((value) => value.trim().length > 0),
|
||
});
|
||
try {
|
||
return schema.safeParse(JSON.parse(message)).success;
|
||
} catch (cause) {
|
||
if (cause instanceof SyntaxError) return false;
|
||
throw cause;
|
||
}
|
||
}
|
||
|
||
async function runCodex(prompt, approvedTools) {
|
||
const arguments_ = [
|
||
"--no-daemon",
|
||
"exec",
|
||
"--ephemeral",
|
||
"--skip-git-repo-check",
|
||
"--sandbox",
|
||
"read-only",
|
||
"--json",
|
||
"--color",
|
||
"never",
|
||
"-C",
|
||
evaluationRoot,
|
||
"-c",
|
||
'approval_policy="never"',
|
||
...approvedTools.flatMap((name) => [
|
||
"-c",
|
||
`mcp_servers.rea.tools.${name}.approval_mode="approve"`,
|
||
]),
|
||
...(optionalModel === undefined ? [] : ["--model", optionalModel]),
|
||
prompt,
|
||
];
|
||
const child = spawn(codex, arguments_, {
|
||
cwd: evaluationRoot,
|
||
env: evaluationEnvironment,
|
||
stdio: ["ignore", "pipe", "pipe"],
|
||
});
|
||
const events = [];
|
||
const stderr = [];
|
||
const lines = createInterface({ input: child.stdout });
|
||
const linesClosed = new Promise((resolveClose) =>
|
||
lines.once("close", resolveClose),
|
||
);
|
||
lines.on("line", (line) => {
|
||
try {
|
||
events.push(JSON.parse(line));
|
||
} catch {
|
||
stderr.push(`non-json stdout: ${line}`);
|
||
}
|
||
});
|
||
child.stderr.on("data", (chunk) => {
|
||
if (stderr.join("").length < 32_768) stderr.push(chunk.toString("utf8"));
|
||
});
|
||
const exitCode = await new Promise((resolveExit, reject) => {
|
||
const timeout = setTimeout(() => {
|
||
child.kill("SIGTERM");
|
||
reject(
|
||
new Error(`Codex evaluation timed out after ${String(timeoutMs)}ms`),
|
||
);
|
||
}, timeoutMs);
|
||
child.once("error", (cause) => {
|
||
clearTimeout(timeout);
|
||
reject(cause);
|
||
});
|
||
child.once("exit", (code) => {
|
||
clearTimeout(timeout);
|
||
resolveExit(code ?? 1);
|
||
});
|
||
});
|
||
await linesClosed;
|
||
return { events, exitCode, stderr: stderr.join("").slice(0, 32_768) };
|
||
}
|
||
|
||
async function runProcess(
|
||
command,
|
||
arguments_,
|
||
cwd,
|
||
timeout,
|
||
env = process.env,
|
||
) {
|
||
const child = spawn(command, arguments_, {
|
||
cwd,
|
||
env,
|
||
stdio: ["ignore", "ignore", "pipe"],
|
||
});
|
||
let stderr = "";
|
||
child.stderr.on("data", (chunk) => {
|
||
if (stderr.length < 32_768) stderr += chunk.toString("utf8");
|
||
});
|
||
const code = await new Promise((resolveExit, reject) => {
|
||
const timer = setTimeout(() => {
|
||
child.kill("SIGTERM");
|
||
reject(new Error(`${command} timed out`));
|
||
}, timeout);
|
||
child.once("error", (cause) => {
|
||
clearTimeout(timer);
|
||
reject(cause);
|
||
});
|
||
child.once("exit", (value) => {
|
||
clearTimeout(timer);
|
||
resolveExit(value ?? 1);
|
||
});
|
||
});
|
||
if (code !== 0)
|
||
throw new Error(`${command} failed with ${String(code)}: ${stderr}`);
|
||
}
|
||
|
||
async function readCommandVersion(command) {
|
||
const child = spawn(command, ["--version"], {
|
||
cwd: repositoryRoot,
|
||
stdio: ["ignore", "pipe", "pipe"],
|
||
});
|
||
let stdout = "";
|
||
let stderr = "";
|
||
child.stdout.on("data", (chunk) => {
|
||
if (stdout.length < 4096) stdout += chunk.toString("utf8");
|
||
});
|
||
child.stderr.on("data", (chunk) => {
|
||
if (stderr.length < 4096) stderr += chunk.toString("utf8");
|
||
});
|
||
const code = await new Promise((resolveExit, reject) => {
|
||
const timer = setTimeout(() => {
|
||
child.kill("SIGTERM");
|
||
reject(new Error(`${command} --version timed out`));
|
||
}, 10_000);
|
||
child.once("error", (cause) => {
|
||
clearTimeout(timer);
|
||
reject(cause);
|
||
});
|
||
child.once("exit", (value) => {
|
||
clearTimeout(timer);
|
||
resolveExit(value ?? 1);
|
||
});
|
||
});
|
||
if (code !== 0)
|
||
throw new Error(
|
||
`${command} --version failed with ${String(code)}: ${stderr}`,
|
||
);
|
||
const version = stdout.trim();
|
||
if (version.length === 0)
|
||
throw new Error(`${command} --version returned empty output`);
|
||
return version;
|
||
}
|