Merge pull request #232 from morluto/codex/test-suite-speed

fix(process): stabilize and speed up the test suite
This commit is contained in:
morluto 2026-07-15 06:17:36 +08:00 committed by GitHub
commit a3b7de9fa5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 465 additions and 75 deletions

View File

@ -51,6 +51,7 @@
"scripts": {
"prebuild": "node scripts/generate-package-metadata.mjs",
"build": "tsc -p tsconfig.build.json",
"postbuild": "node scripts/record-build-cache.mjs",
"check": "npm run typecheck && npm run lint && npm run format:check && npm test",
"format": "prettier --write .",
"format:check": "prettier --check .",

103
scripts/build-cache.mjs Normal file
View File

@ -0,0 +1,103 @@
import { createHash } from "node:crypto";
import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
const CACHE_DIRECTORY = ["node_modules", ".cache"];
const BUILD_INFO_NAME = "rea-tsconfig-build.tsbuildinfo";
const MANIFEST_NAME = "rea-tsconfig-build-outputs.json";
const isMissing = (cause) =>
typeof cause === "object" &&
cause !== null &&
"code" in cause &&
cause.code === "ENOENT";
const outputFiles = async (root) => {
const sourceRoot = join(root, "src");
const entries = await readdir(sourceRoot, { recursive: true });
return entries
.filter((entry) => entry.endsWith(".ts") && !entry.endsWith(".d.ts"))
.sort()
.map((entry) => {
const output = `${entry.slice(0, -3)}.js`;
return { output, path: join(root, "dist", output) };
});
};
const digest = async (path) =>
createHash("sha256")
.update(await readFile(path))
.digest("hex");
const cachePath = (root, name) => join(root, ...CACHE_DIRECTORY, name);
const invalidate = async (root) => {
await Promise.all([
rm(cachePath(root, BUILD_INFO_NAME), { force: true }),
rm(cachePath(root, MANIFEST_NAME), { force: true }),
]);
};
const readManifest = async (root) => {
try {
const parsed = JSON.parse(
await readFile(cachePath(root, MANIFEST_NAME), "utf8"),
);
if (
typeof parsed !== "object" ||
parsed === null ||
parsed.version !== 1 ||
typeof parsed.outputs !== "object" ||
parsed.outputs === null ||
Array.isArray(parsed.outputs)
)
return undefined;
return parsed.outputs;
} catch (cause) {
if (isMissing(cause) || cause instanceof SyntaxError) return undefined;
throw cause;
}
};
/** Invalidates incremental state when generated JavaScript was removed or changed. */
export const validateBuildCache = async (root) => {
const [files, manifest] = await Promise.all([
outputFiles(root),
readManifest(root),
]);
if (manifest === undefined || Object.keys(manifest).length !== files.length) {
await invalidate(root);
return;
}
const matches = await Promise.all(
files.map(async ({ output, path }) => {
if (typeof manifest[output] !== "string") return false;
try {
return (await digest(path)) === manifest[output];
} catch (cause) {
if (isMissing(cause)) return false;
throw cause;
}
}),
);
if (!matches.every(Boolean)) await invalidate(root);
};
/** Records generated JavaScript so a later incremental build can verify its cache. */
export const recordBuildCache = async (root) => {
const files = await outputFiles(root);
const entries = await Promise.all(
files.map(async ({ output, path }) => [output, await digest(path)]),
);
const source = `${JSON.stringify({ version: 1, outputs: Object.fromEntries(entries) })}\n`;
const directory = join(root, ...CACHE_DIRECTORY);
await mkdir(directory, { recursive: true });
const path = cachePath(root, MANIFEST_NAME);
let existing;
try {
existing = await readFile(path, "utf8");
} catch (cause) {
if (!isMissing(cause)) throw cause;
}
if (existing !== source) await writeFile(path, source, "utf8");
};

View File

@ -2,6 +2,8 @@ import { readFile, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { validateBuildCache } from "./build-cache.mjs";
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
const packageJson = JSON.parse(
await readFile(join(root, "package.json"), "utf8"),
@ -32,4 +34,18 @@ export const PACKAGE_METADATA = {
skillVersion: ${JSON.stringify(skillVersion)},
} as const;
`;
await writeFile(join(root, "src/generatedPackageMetadata.ts"), source, "utf8");
const outputPath = join(root, "src/generatedPackageMetadata.ts");
let existingSource;
try {
existingSource = await readFile(outputPath, "utf8");
} catch (cause) {
if (
typeof cause !== "object" ||
cause === null ||
!("code" in cause) ||
cause.code !== "ENOENT"
)
throw cause;
}
if (existingSource !== source) await writeFile(outputPath, source, "utf8");
await validateBuildCache(root);

View File

@ -0,0 +1,7 @@
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { recordBuildCache } from "./build-cache.mjs";
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
await recordBuildCache(root);

View File

@ -285,7 +285,7 @@ export const releaseProcessResources = async (options: {
readonly checkpoints: ProcessCheckpoints | undefined;
readonly runId: string;
readonly temporaryRoot: string;
readonly sampledProcessGroupIds: readonly number[];
readonly capturedProcessGroupIds: readonly number[];
}): Promise<string | undefined> => {
for (const timer of options.timers) clearTimeout(timer);
let failure: string | undefined;
@ -310,11 +310,7 @@ export const releaseProcessResources = async (options: {
failure ??= "filesystem checkpoint cleanup failed";
}
if (options.terminal !== undefined && process.platform !== "win32") {
const processGroupIds = new Set([
options.terminal.pid,
...options.sampledProcessGroupIds,
]);
for (const processGroupId of processGroupIds) {
for (const processGroupId of new Set(options.capturedProcessGroupIds)) {
const cleaned = await cleanupOwnedProcessGroup({
runId: options.runId,
leaderPid: processGroupId,

View File

@ -25,6 +25,7 @@ import {
type CommandShimReplay,
} from "./CommandShimReplay.js";
import { normalizeProcessText } from "./ProcessNormalization.js";
import { selectCapturedProcessGroupIds } from "./ProcessOwnership.js";
import {
awaitTerminalExit,
buildCaptureResult,
@ -157,7 +158,8 @@ const cleanupFailedStartup = async (options: {
options.terminal?.kill("SIGKILL");
const cleanupFailure = await releaseProcessResources({
...options,
sampledProcessGroupIds: [],
capturedProcessGroupIds:
options.terminal === undefined ? [] : [options.terminal.pid],
});
if (cleanupFailure !== undefined)
throw new ProcessCaptureError(cleanupFailure, {
@ -319,9 +321,13 @@ const finishProcessRun = async (options: {
checkpoints: options.runtime?.checkpoints,
runId: options.runId,
temporaryRoot: options.temporaryRoot,
sampledProcessGroupIds: options.samples.flatMap(({ process_group_id }) =>
process_group_id === null ? [] : [process_group_id],
),
capturedProcessGroupIds:
options.runtime === undefined
? []
: selectCapturedProcessGroupIds(
options.runtime.terminal.pid,
options.samples,
),
});
let { capture, executionFailure } = options;
if (capture !== undefined) {
@ -365,12 +371,7 @@ const completeCapture = async (options: {
if (reason === "cancelled") throw processCaptureCancelled();
const settlement = await observeSettlement(
options.runId,
[
runtime.terminal.pid,
...options.samples.flatMap(({ process_group_id }) =>
process_group_id === null ? [] : [process_group_id],
),
],
selectCapturedProcessGroupIds(runtime.terminal.pid, options.samples),
scenario.settle_ms,
);
runtime.checkpoints.trigger("settled");

View File

@ -1,6 +1,7 @@
import { execFile } from "node:child_process";
import { readFile } from "node:fs/promises";
import { promisify } from "node:util";
import type { ProcessSample } from "../domain/processCapture.js";
const execFileAsync = promisify(execFile);
@ -51,6 +52,22 @@ export type ProcessGroupObservation =
| { readonly state: "alive" }
| { readonly state: "unverifiable"; readonly reason: string };
/** Select the PTY root group and groups led by an observed captured process. */
export const selectCapturedProcessGroupIds = (
rootPid: number,
samples: readonly Pick<ProcessSample, "pid" | "process_group_id">[],
): readonly number[] => {
const processGroupIds = new Set<number>([rootPid]);
for (const sample of samples) {
// A sampled member may transiently inherit an unrelated group. POSIX group
// leaders have pid === pgid, so only that observation establishes that the
// group leader itself belonged to the captured tree. Token checks below
// remain the final authority immediately before observation or signaling.
if (sample.pid === sample.process_group_id) processGroupIds.add(sample.pid);
}
return [...processGroupIds];
};
/** Parse the NUL-delimited Linux process environment without nameless keys. */
export const parseProcessEnvironment = (
value: string,

View File

@ -314,17 +314,19 @@ describe("artifact graph provider", () => {
const client = new ArtifactProvider().createClient(
target(root, "directory"),
);
const first = await client.execute(
"inventory_artifact",
artifactInventoryInputSchema.parse({ occurrence_limit: 500 }),
);
const secondPage = await client.execute(
"inventory_artifact",
artifactInventoryInputSchema.parse({
occurrence_offset: 500,
occurrence_limit: 500,
}),
);
const [first, secondPage] = await Promise.all([
client.execute(
"inventory_artifact",
artifactInventoryInputSchema.parse({ occurrence_limit: 500 }),
),
client.execute(
"inventory_artifact",
artifactInventoryInputSchema.parse({
occurrence_offset: 500,
occurrence_limit: 500,
}),
),
]);
expect(first.ok && secondPage.ok).toBe(true);
if (!first.ok || !secondPage.ok) return;
const left = artifactInventoryResultSchema.parse(first.value.result);

View File

@ -1,11 +1,9 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { z } from "zod";
const execFileAsync = promisify(execFile);
const bridgePath = fileURLToPath(
new URL("../bridge/hopper_bridge.py", import.meta.url),
);
@ -49,6 +47,7 @@ const probeResultSchema = z.union([
matchResultSchema,
searchResultSchema,
]);
type ProbeResult = z.infer<typeof probeResultSchema>;
type ProbeInput =
| {
@ -63,15 +62,194 @@ type ProbeInput =
readonly params: Readonly<Record<string, string | number | boolean>>;
};
const probe = async (input: ProbeInput) => {
const result = await execFileAsync(
"python3",
[probePath, bridgePath, JSON.stringify(input)],
{ timeout: 3_000, maxBuffer: 1024 * 1024 },
);
return probeResultSchema.parse(JSON.parse(result.stdout));
interface PendingProbe {
readonly resolve: (result: ProbeResult) => void;
readonly reject: (error: Error) => void;
timeout: NodeJS.Timeout | undefined;
}
const PROBE_TIMEOUT_MS = 3_000;
const PROBE_BUFFER_LIMIT = 1_024 * 1_024;
const pendingProbes: PendingProbe[] = [];
let probeProcess: ChildProcessWithoutNullStreams | undefined;
let probeStdout = "";
let probeStderr = "";
let probeFailure: Error | undefined;
let stoppingProbe = false;
const failureWithStderr = (message: string): Error => {
const stderr = probeStderr.trim();
return new Error(stderr.length === 0 ? message : `${message}: ${stderr}`);
};
const rejectPendingProbes = (error: Error): void => {
for (const pending of pendingProbes.splice(0)) {
if (pending.timeout !== undefined) clearTimeout(pending.timeout);
pending.reject(error);
}
};
const failProbeProcess = (error: Error): void => {
probeFailure ??= error;
rejectPendingProbes(probeFailure);
if (
!stoppingProbe &&
probeProcess?.exitCode === null &&
probeProcess.signalCode === null
)
probeProcess.kill("SIGKILL");
};
const acceptProbeLine = (line: string): void => {
const pending = pendingProbes.shift();
if (pending === undefined) {
failProbeProcess(
failureWithStderr("Python probe returned an extra result"),
);
return;
}
if (pending.timeout !== undefined) clearTimeout(pending.timeout);
try {
const decoded: unknown = JSON.parse(line);
pending.resolve(probeResultSchema.parse(decoded));
} catch (cause: unknown) {
const error =
cause instanceof Error
? cause
: failureWithStderr("Python probe returned an invalid result");
pending.reject(error);
failProbeProcess(error);
}
};
const acceptProbeOutput = (chunk: string): void => {
probeStdout += chunk;
if (Buffer.byteLength(probeStdout) > PROBE_BUFFER_LIMIT) {
failProbeProcess(failureWithStderr("Python probe output exceeded 1 MiB"));
return;
}
let newline = probeStdout.indexOf("\n");
while (newline >= 0) {
const line = probeStdout.slice(0, newline).replace(/\r$/u, "");
probeStdout = probeStdout.slice(newline + 1);
acceptProbeLine(line);
newline = probeStdout.indexOf("\n");
}
};
const startProbeProcess = async (): Promise<void> => {
stoppingProbe = false;
probeFailure = undefined;
probeStdout = "";
probeStderr = "";
const child = spawn("python3", [probePath, bridgePath], {
stdio: ["pipe", "pipe", "pipe"],
});
probeProcess = child;
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", acceptProbeOutput);
child.stderr.on("data", (chunk: string) => {
probeStderr += chunk;
if (Buffer.byteLength(probeStderr) > PROBE_BUFFER_LIMIT)
failProbeProcess(new Error("Python probe stderr exceeded 1 MiB"));
});
child.stdin.on("error", (error) => {
if (!stoppingProbe) failProbeProcess(error);
});
child.on("error", (error) => {
if (!stoppingProbe) failProbeProcess(error);
});
child.on("exit", (code, signal) => {
if (!stoppingProbe)
failProbeProcess(
failureWithStderr(
`Python probe exited before responding (code ${String(code)}, signal ${String(signal)})`,
),
);
});
await new Promise<void>((resolve, reject) => {
const onSpawn = (): void => {
child.off("error", onError);
resolve();
};
const onError = (error: Error): void => {
child.off("spawn", onSpawn);
reject(error);
};
child.once("spawn", onSpawn);
child.once("error", onError);
});
};
const hasExited = (child: ChildProcessWithoutNullStreams): boolean =>
child.exitCode !== null || child.signalCode !== null;
const waitForExit = (
child: ChildProcessWithoutNullStreams,
timeoutMs: number,
): Promise<boolean> => {
if (hasExited(child)) return Promise.resolve(true);
return new Promise((resolve) => {
const onExit = (): void => {
clearTimeout(timeout);
resolve(true);
};
const timeout = setTimeout(() => {
child.off("exit", onExit);
resolve(false);
}, timeoutMs);
child.once("exit", onExit);
});
};
const stopProbeProcess = async (): Promise<void> => {
const child = probeProcess;
if (child === undefined) return;
stoppingProbe = true;
rejectPendingProbes(new Error("Python probe stopped before responding"));
if (!hasExited(child)) child.stdin.end();
if (!(await waitForExit(child, 500))) child.kill("SIGTERM");
if (!(await waitForExit(child, 500))) child.kill("SIGKILL");
if (!(await waitForExit(child, 500)))
throw new Error("Python probe did not stop after SIGKILL");
probeProcess = undefined;
};
const probe = (input: ProbeInput): Promise<ProbeResult> => {
const child = probeProcess;
if (probeFailure !== undefined) return Promise.reject(probeFailure);
if (child === undefined || hasExited(child))
return Promise.reject(new Error("Python probe is not running"));
return new Promise((resolve, reject) => {
const pending: PendingProbe = { resolve, reject, timeout: undefined };
pending.timeout = setTimeout(() => {
failProbeProcess(
new Error(
`Python probe request timed out after ${String(PROBE_TIMEOUT_MS)} ms`,
),
);
}, PROBE_TIMEOUT_MS);
pendingProbes.push(pending);
child.stdin.write(`${JSON.stringify(input)}\n`, (error) => {
if (error !== null && error !== undefined) failProbeProcess(error);
});
});
};
beforeAll(startProbeProcess);
beforeEach(async () => {
if (
probeFailure === undefined &&
probeProcess !== undefined &&
!hasExited(probeProcess)
)
return;
await stopProbeProcess();
await startProbeProcess();
});
afterAll(stopProbeProcess);
const invalidRequest = (action: "match" | "search", message: string) => ({
action,
ok: false,

View File

@ -16,6 +16,25 @@ import {
sessionOutputSchemas,
} from "../src/contracts/toolOutputSchemas.js";
const convertContractJsonSchema = (schema: z.ZodType) =>
z.toJSONSchema(schema, {
target: "draft-07",
unrepresentable: "any",
});
const jsonSchemaCache = new WeakMap<
z.ZodType,
ReturnType<typeof convertContractJsonSchema>
>();
const contractJsonSchema = (schema: z.ZodType) => {
const cached = jsonSchemaCache.get(schema);
if (cached !== undefined) return cached;
const converted = convertContractJsonSchema(schema);
jsonSchemaCache.set(schema, converted);
return converted;
};
const emptySchemaPaths = (value: unknown, path = "$"): string[] => {
if (Array.isArray(value))
return value.flatMap((item, index) =>
@ -41,14 +60,8 @@ describe("tool contract surface", () => {
];
expect(contracts).toHaveLength(60);
for (const contract of contracts) {
const inputSchema = z.toJSONSchema(contract.inputSchema, {
target: "draft-07",
unrepresentable: "any",
});
const outputSchema = z.toJSONSchema(contract.outputSchema, {
target: "draft-07",
unrepresentable: "any",
});
const inputSchema = contractJsonSchema(contract.inputSchema);
const outputSchema = contractJsonSchema(contract.outputSchema);
expect(inputSchema.type).toBe("object");
expect(outputSchema.type).toBe("object");
expect(typeof contract.annotations.idempotentHint).toBe("boolean");
@ -128,10 +141,7 @@ describe("tool contract surface", () => {
...ENHANCED_TOOL_CONTRACTS,
...SESSION_TOOL_CONTRACTS,
]) {
const schema = z.toJSONSchema(contract.outputSchema, {
target: "draft-07",
unrepresentable: "any",
});
const schema = contractJsonSchema(contract.outputSchema);
expect(JSON.stringify(schema)).not.toContain('"result":{}');
expect(contract.description.length).toBeGreaterThan(100);
expect(contract.description).toMatch(/[.;]/u);
@ -150,10 +160,7 @@ describe("tool contract surface", () => {
];
expect(contracts).toHaveLength(78);
for (const contract of contracts) {
const schema = z.toJSONSchema(contract.outputSchema, {
target: "draft-07",
unrepresentable: "any",
});
const schema = contractJsonSchema(contract.outputSchema);
expect(emptySchemaPaths(schema), contract.name).toEqual([]);
}
});

View File

@ -214,12 +214,11 @@ describe("enhanced MCP tools", () => {
["analyze_function", { procedure: "0x1" }],
["trace_feature", { query: "hello", max_operations: 10 }],
] as const;
const results = [];
for (const [name, arguments_] of calls) {
results.push(
const results = await Promise.all(
calls.map(async ([name, arguments_]) =>
jsonResult(await client.callTool({ name, arguments: arguments_ })),
);
}
),
);
expect(results[0]).toMatchObject({ count: 1 });
expect(results[1]).toMatchObject({ count: 1 });
expect(results[2]).toMatchObject({ count: 1 });

View File

@ -58,5 +58,11 @@ def _probe(namespace, payload):
if __name__ == "__main__":
bridge = _load_bridge(sys.argv[1])
request = json.loads(sys.argv[2])
print(json.dumps(_probe(bridge, request), sort_keys=True))
if len(sys.argv) >= 3:
print(json.dumps(_probe(bridge, json.loads(sys.argv[2])), sort_keys=True))
else:
for line in sys.stdin:
if not line.strip():
continue
request = json.loads(line)
print(json.dumps(_probe(bridge, request), sort_keys=True), flush=True)

View File

@ -3,6 +3,7 @@ import {
cleanupOwnedProcessGroup,
observeOwnedProcessGroup,
parseProcessEnvironment,
selectCapturedProcessGroupIds,
type ProcessOwnershipHost,
} from "../src/application/ProcessOwnership.js";
@ -39,6 +40,18 @@ const host = (
};
describe("owned process-group cleanup", () => {
it("excludes sampled groups whose leader was outside the captured tree", () => {
expect(
selectCapturedProcessGroupIds(100, [
{ pid: 100, process_group_id: 42 },
{ pid: 100, process_group_id: 100 },
{ pid: 101, process_group_id: 42 },
{ pid: 102, process_group_id: 102 },
{ pid: 103, process_group_id: 102 },
]),
).toEqual([100, 102]);
});
it("drops nameless Linux environment entries", () => {
expect(
parseProcessEnvironment("=ignored\0REA_PROCESS_RUN_ID=owned\0EMPTY=\0"),

View File

@ -13,18 +13,26 @@ const sourceRoot = new URL("../src/", import.meta.url);
describe("provider-neutral architecture", () => {
it("keeps Hopper imports behind its adapter and composition root", async () => {
const forbiddenRoots = ["domain", "contracts", "server", "application"];
const violations: string[] = [];
for (const root of forbiddenRoots) {
for (const file of await typescriptFiles(
fileURLToPath(new URL(`${root}/`, sourceRoot)),
)) {
if (root === "application" && file.endsWith("/runtime.ts")) continue;
const source = await readFile(file, "utf8");
if (/from\s+["'][^"']*\/hopper\//u.test(source)) {
violations.push(relative(fileURLToPath(sourceRoot), file));
}
}
}
const inspected = await Promise.all(
forbiddenRoots.map(async (root) => {
const files = await typescriptFiles(
fileURLToPath(new URL(`${root}/`, sourceRoot)),
);
return Promise.all(
files.map(async (file) => {
if (root === "application" && file.endsWith("/runtime.ts"))
return undefined;
const source = await readFile(file, "utf8");
return /from\s+["'][^"']*\/hopper\//u.test(source)
? relative(fileURLToPath(sourceRoot), file)
: undefined;
}),
);
}),
);
const violations = inspected
.flat()
.filter((violation) => violation !== undefined);
expect(violations).toEqual([]);
});

View File

@ -1,7 +1,9 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"incremental": true,
"rootDir": "src",
"tsBuildInfoFile": "node_modules/.cache/rea-tsconfig-build.tsbuildinfo",
"types": ["node"]
},
"include": ["src/**/*.ts"],

View File

@ -1,8 +1,42 @@
import { availableParallelism } from "node:os";
import { defineConfig } from "vitest/config";
const TEST_FILES = ["tests/**/*.test.ts"];
const PROCESS_CAPTURE_TEST = "tests/processCapture.test.ts";
const SERIAL_INTEGRATION_TESTS = [
"tests/browserCli.test.ts",
"tests/cliOutput.test.ts",
PROCESS_CAPTURE_TEST,
"tests/runtime.test.ts",
];
const dualCoreProjects =
availableParallelism() === 2
? {
maxWorkers: 2,
projects: [
{
test: {
name: "parallel",
include: TEST_FILES,
exclude: SERIAL_INTEGRATION_TESTS,
},
},
{
test: {
name: "serial-integration",
include: SERIAL_INTEGRATION_TESTS,
fileParallelism: false,
},
},
],
}
: {};
export default defineConfig({
test: {
include: ["tests/**/*.test.ts"],
include: TEST_FILES,
...dualCoreProjects,
retry: 2,
reporters: ["default", "verbose"],
coverage: {