ci: keep validation fresh and preserve failed-shard reports (#2210)

Run the existing README consistency and Python producer guards from their
CI owners. Execute full local validation fresh while preserving cached
preparation, and permit scoped reuse of earlier portable build entries.

Collect failed-shard diagnostics before enforcing success and publish
compact, portable timing history only after a complete successful merge.
Preserve installer truncation failure across supported Bash versions.

Keep cancellation tests focused on the called MCP tool, observe server
progress before timing target readiness, and retain delayed-startup,
cancellation, cleanup, and live follow-up request assertions.

Validation: all 32 CI jobs and the separate Go lane passed; 935 files,
9,630 cases, 9,544 passed, 86 skipped, zero failed. Controlled regressions
and fault/restoration probes are recorded in PR #2210.

Refs #2190.

Assisted-by: Kogasa (gensokyo-skills:kogasa-surprise-testing)
This commit is contained in:
Ysopwnable 2026-10-12 23:42:47 +08:00 committed by GitHub
parent 94153f1c75
commit 428eee00fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 511 additions and 92 deletions

View File

@ -11,7 +11,7 @@ runs:
id: plan
shell: bash
env:
BUILD_CACHE_PREFIX: rea-portable-build-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc') }}
BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }}
run: |
npm run --silent test:prepare -- --dry=json |
node --input-type=module -e '
@ -25,6 +25,7 @@ runs:
}).sort();
const hash = createHash("sha256").update(hashes.join("\n")).digest("hex");
console.log(`key=${process.env.BUILD_CACHE_PREFIX}-${hash}`);
console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`);
' >> "$GITHUB_OUTPUT"
- name: Restore portable build outputs
if: inputs.save != 'true'
@ -32,9 +33,11 @@ runs:
with:
path: .turbo/cache/
key: ${{ steps.plan.outputs.key }}
restore-keys: ${{ steps.plan.outputs.prefix }}
- name: Restore and save portable build outputs
if: inputs.save == 'true'
uses: actions/cache@v4
with:
path: .turbo/cache/
key: ${{ steps.plan.outputs.key }}
restore-keys: ${{ steps.plan.outputs.prefix }}

View File

@ -94,6 +94,9 @@ jobs:
run: npm run check:ci
- if: needs.changes.outputs.code == 'true'
run: npm run verify:test-discovery
- name: Check LLDB producer retention and launch contracts
if: needs.changes.outputs.code == 'true'
run: npm run test:python
- if: needs.changes.outputs.code != 'true'
run: npm run format:check
@ -168,7 +171,7 @@ jobs:
continue-on-error: true
uses: actions/download-artifact@v4
with:
name: test-results
name: test-timings
run-id: ${{ steps.history.outputs.run-id }}
github-token: ${{ github.token }}
path: .cache/ci-test-history
@ -203,14 +206,10 @@ jobs:
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
- name: Require successful test shards
- name: Require successful change classification
env:
CHANGES_RESULT: ${{ needs.changes.result }}
CODE_REQUIRED: ${{ needs.changes.outputs.code }}
SHARD_RESULT: ${{ needs.test-shard.result }}
run: |
test "${CHANGES_RESULT}" = "success"
test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success"
run: test "${CHANGES_RESULT}" = "success"
- uses: actions/checkout@v4
if: needs.changes.outputs.code == 'true' && success()
- uses: actions/setup-node@v4
@ -227,6 +226,7 @@ jobs:
pattern: vitest-report-*
merge-multiple: true
- name: Merge coverage, JUnit, and timing reports
id: merge
if: needs.changes.outputs.code == 'true' && success()
run: >-
npm run test:ci:merge --
@ -240,27 +240,13 @@ jobs:
--outputFile.junit=test-results.xml
--outputFile.json=test-results.json
- name: Summarize slowest test files
if: needs.changes.outputs.code == 'true' && success()
shell: bash
run: |
node <<'NODE' >> "${GITHUB_STEP_SUMMARY}"
const report = JSON.parse(require("node:fs").readFileSync("test-results.json", "utf8"));
const files = (report.testResults ?? [])
.map((result) => ({
name: result.name,
duration: Math.max(0, (result.endTime ?? result.startTime) - result.startTime),
}))
.sort((left, right) => right.duration - left.duration)
.slice(0, 20);
console.log("## Slowest deterministic test files\n");
console.log("| File | Duration (ms) |");
console.log("| --- | ---: |");
for (const file of files) {
console.log(`| \`${file.name}\` | ${file.duration} |`);
}
NODE
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
env:
SHARD_RESULT: ${{ needs.test-shard.result }}
MERGE_RESULT: ${{ steps.merge.outcome }}
run: node scripts/ci/test-report.mjs
- uses: actions/upload-artifact@v4
if: needs.changes.outputs.code == 'true' && always()
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
with:
name: test-results
path: |
@ -269,6 +255,21 @@ jobs:
coverage/
if-no-files-found: error
retention-days: 30
- name: Publish successful timing history
if: needs.changes.outputs.code == 'true' && success() && needs.test-shard.result == 'success'
uses: actions/upload-artifact@v4
with:
name: test-timings
path: .cache/ci-test-history/test-results.json
include-hidden-files: true
if-no-files-found: error
retention-days: 30
- name: Require successful test shards
if: ${{ !cancelled() }}
env:
CODE_REQUIRED: ${{ needs.changes.outputs.code }}
SHARD_RESULT: ${{ needs.test-shard.result }}
run: test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success"
conformance-fixtures:
name: Source-built conformance fixtures

View File

@ -182,6 +182,10 @@ CI diagnosis. Packaging/setup/distribution changes additionally require
`npm run verify:package` and `npm pack --dry-run`; provider changes require the
matching real-provider lane. State which host/provider workflows ran.
`check:pr` reuses build and static-check outputs while executing the test suite
fresh, including the fast Python producer checks. `docs:check` also runs the
onboarding example consistency suite for README-only changes.
Pre-commit formats and lints staged files; pre-push runs `check:fast`.
### Generated files and command ownership

View File

@ -19,11 +19,12 @@ behavior, and complete CI evidence before merging.
| `npm run test:boundary` | Boundary, CLI boundary, MCP boundary, process boundary, and process-global projects |
| `npm run test:mcp` | MCP boundary project |
| `npm run test:acceptance` | Complete compiled CLI and MCP acceptance workflows |
| `npm run test:python` | LLDB producer retention and launch contracts with a fake LLDB API; no native debugger required |
| `npm run test:watch` | Dirty source tests in watch mode, without build |
| `npm run test:watch:all` | Changed tests from every project; builds at startup, so rebuild after production edits before relying on compiled tests |
| `npm run check:changed` | Cached typecheck/lint and branch-related source feedback |
| `npm run check:pr` | Opt-in complete local deterministic gate and generated-file checks |
| `npm run docs:check` | Generated-document validation from current source and build outputs |
| `npm run docs:check` | Generated-document validation and fresh onboarding example consistency checks |
For example:
@ -97,6 +98,12 @@ runner. Preserve simultaneous launches where concurrency is itself under test.
Build/documentation writers hold checkout-local output locks. Tests do not
hold a broad command lock; `check:pr` finishes tests before document validation.
Turbo caches deterministic preparation and static checks, but `test:run` always
executes. Repository files do not fully describe installed tool contents or host
state. A previous test pass therefore cannot replace current validation, even
when the source tree is unchanged. `check:pr` also runs the fast Python producer
suite; CI runs it in the static job for code changes. These producer tests do not
establish real LLDB engine coverage.
Vitest's persistent compile cache remains disabled. The `cliTest` fixture shares
a temporary Node compile cache among its fresh CLI subprocesses. The worker
@ -247,17 +254,25 @@ Coverage does not replace named boundary or real-provider scenarios.
CI runs six native Vitest shards without retries. Each shard emits a blob
report; the merge job produces aggregate coverage plus JUnit and JSON timing
reports, uploads them together, and writes the slowest files to the workflow
summary. Static checks, documentation, build, package verification, and
reports even when a shard fails, uploads available diagnostics, and writes the
slowest files to the workflow summary. Partial reports are marked diagnostic
only. Failed, cancelled, or missing selected shards still fail the required gate,
and aggregate coverage thresholds remain enforced by the merge command.
Only successful complete merges publish the small `test-timings` artifact, with
checkout-relative paths; the build job downloads that artifact for the sequencer
instead of the coverage/JUnit bundle. Missing history falls back to native
sharding. Static checks, documentation, build, package verification, and
real-system lanes remain separate jobs so one kind of evidence cannot stand in
for another.
The build job saves portable Turbo outputs across runs. Eight Linux real-system
lanes restore this cache without waiting for the build job. The shared action
derives its key from `test:prepare --dry=json` task hashes, runner OS and
architecture, and `.npmrc`; test-only changes can reuse compiled outputs. Every
architecture, `.npmrc`, `.nvmrc`, and the lockfile; test-only changes can reuse compiled outputs. Every
build and verification command still runs, so Turbo checks task inputs and
rebuilds misses. There is no prefix fallback to accumulate older build outputs.
rebuilds misses. A restore prefix with the same platform and toolchain permits
partial task reuse when one generator's inputs change; Turbo still invalidates
that generator's entry. Native outputs are not stored in this cache.
Native package jobs, the Windows curated lane, and the Apple artifact lane reuse
the same workflow's Turbo cache for portable compiled JavaScript, skills, and
@ -266,9 +281,10 @@ commands, which restore matching outputs or rebuild on a cache miss. Native
artifacts, dependency installs, package archives, and runtime verification remain
host-local. Keep platform-dependent outputs out of this shared cache.
The PR acceptance target is a median `npm run check:pr` wall time below three
minutes across three warm-build runs on the benchmark host. Keep Vitest caches
cold unless separately identified. A PR that touches packaging or real-system
Record cold and warm preparation separately from fresh test execution when
benchmarking the optional `npm run check:pr` gate. Compare multiple runs on the
same host, including subprocess and queue costs; a cached pass is not a test
runtime measurement. A PR that touches packaging or real-system
behavior requires the applicable `verify:*` lanes; packaging and installation
changes also require `npm run verify:package`. The full-gate benchmark measures
that explicit lane, not the routine iteration requirement.
@ -682,7 +698,8 @@ not make native host acceptance unavailable.
Pull requests that change only root `README*.md` files, `docs/`, `AGENTS.md`,
`CONTRIBUTING.md`, `CHANGELOG.md`, `SECURITY.md`, `native/windows/README.md`,
or the GitHub issue and pull-request templates run formatting and
generated-document validation without the source-test shards or native package
generated-document validation, including the small onboarding example consistency
suite, without the source-test shards or native package
lanes. Classification compares the PR head with its merge base, so later
base-branch changes do not expand that scope. Static and coverage aggregate jobs
remain present and fail if classification fails. Every code change retains the

View File

@ -25,6 +25,14 @@ fail() {
exit 1
}
# Bash can return status 0 while parsing an incomplete [[ condition. Keep that
# grammar in a constant quoted expression so truncated downloads fail while
# parsing the outer brace group. eval parses only this literal; the arguments
# are expanded as data by the conditional, not interpolated into shell code.
matches() {
eval '[[ "$1" =~ $2 ]]'
}
# Resolve the current host's Node.js tarball label pair; a label we cannot map
# is kept verbatim so the operator can still read it. Falls back on the raw
# `uname` outputs rather than inventing a label the Node.js release index does
@ -95,41 +103,41 @@ main() {
esac
done
if [[ "$verbose" == true ]]; then
if [ "$verbose" = true ]; then
set -x
fi
platform="$(uname -s)"
[[ "$platform" == "Darwin" || "$platform" == "Linux" ]] || fail "REA supports macOS and selected 64-bit Linux distributions."
[ "$platform" = "Darwin" ] || [ "$platform" = "Linux" ] || fail "REA supports macOS and selected 64-bit Linux distributions."
command -v curl >/dev/null 2>&1 || fail "curl is required. Install curl, then rerun this installer."
command -v node >/dev/null 2>&1 || fail "Node.js 22.x (>=22.19), 24.x (>=24.11), or 26+ is required; install it from https://nodejs.org and retry."
command -v npm >/dev/null 2>&1 || fail "npm is required; install it with Node.js and retry."
node_version="$(node -p 'process.versions.node' 2>/dev/null || true)"
IFS=. read -r node_major node_minor _node_patch <<<"$node_version"
[[ "$node_major" =~ ^[0-9]+$ && "$node_minor" =~ ^[0-9]+$ ]] || fail "the active Node.js version could not be read. Check that node works and is on PATH, then retry."
if ! [[ "$_node_patch" =~ ^[0-9]+(\+[0-9A-Za-z.-]+)?$ ]] ||
matches "$node_major" '^[0-9]+$' && matches "$node_minor" '^[0-9]+$' || fail "the active Node.js version could not be read. Check that node works and is on PATH, then retry."
if ! matches "$_node_patch" '^[0-9]+(\+[0-9A-Za-z.-]+)?$' ||
! ((node_major == 22 && node_minor >= 19 || node_major == 24 && node_minor >= 11 || node_major >= 26)); then
fail_node_version "$node_version"
fi
if [[ -n "$version" ]]; then
[[ "$version" =~ ^${version_pattern}$ ]] || fail "version must be an exact semantic version."
if [ -n "$version" ]; then
matches "$version" "^${version_pattern}$" || fail "version must be an exact semantic version."
else
release_json="$(curl -fsSL --proto '=https' --tlsv1.2 --retry 3 -H 'Accept: application/vnd.github+json' "https://api.github.com/repos/$REPOSITORY/releases/latest")" || fail "the latest REA release could not be resolved. Check network access or pass --version VERSION, then retry."
tag="$(printf '%s' "$release_json" | node -e 'let s="";process.stdin.on("data",c=>s+=c).on("end",()=>{const v=JSON.parse(s).tag_name;if(typeof v!=="string")process.exit(1);process.stdout.write(v)})' 2>/dev/null)" || fail "the release response was invalid. Retry later or pass --version VERSION."
[[ "$tag" =~ ^rea-agents-(${version_pattern})$ ]] || fail "the latest release tag was invalid. Retry later or pass --version VERSION."
matches "$tag" "^rea-agents-(${version_pattern})$" || fail "the latest release tag was invalid. Retry later or pass --version VERSION."
version="${BASH_REMATCH[1]}"
fi
# npm's exact-version resolver cleans build metadata from the requested version,
# so it may install the base release and make the post-install identity check fail.
if [[ "$version" =~ \+ ]]; then
if matches "$version" '\+'; then
fail "npm cannot install an exact version with build metadata. Pass a version without build metadata, then retry."
fi
install_args=(install --global)
if [[ "$platform" == "Linux" ]]; then
if [ "$platform" = "Linux" ]; then
install_args+=(--prefix "$HOME/.local")
install_bin="$HOME/.local/bin/rea"
else
@ -145,21 +153,21 @@ main() {
printf ' %q' npm "${install_args[@]}"
printf '\n'
printf ' Binary: %s\n' "$install_bin"
printf ' Setup: %s\n' "$([[ "$start_setup" == true ]] && printf 'start when a terminal is available' || printf 'skipped')"
printf ' Setup: %s\n' "$([ "$start_setup" = true ] && printf 'start when a terminal is available' || printf 'skipped')"
if [[ "$dry_run" == true ]]; then
if [ "$dry_run" = true ]; then
printf 'Dry run complete; no changes made.\n'
exit 0
fi
printf 'Installing %s@%s...\n' "$PACKAGE" "$version"
npm "${install_args[@]}" || fail "npm could not install REA. Check registry access and npm permissions, then retry."
[[ -x "$install_bin" ]] || fail "npm completed without installing the rea command. Check the npm global bin directory and PATH, then retry."
[ -x "$install_bin" ] || fail "npm completed without installing the rea command. Check the npm global bin directory and PATH, then retry."
installed_version="$("$install_bin" --version 2>/dev/null | tr -d '[:space:]')" || fail "the installed REA version could not be read. Reinstall the requested version, then retry."
[[ "$installed_version" == "$version" ]] || fail "installed version $installed_version does not match $version. Reinstall the requested version, then retry."
[ "$installed_version" = "$version" ] || fail "installed version $installed_version does not match $version. Reinstall the requested version, then retry."
printf 'REA %s is installed.\n' "$version"
if [[ "$start_setup" == true && -r /dev/tty && -w /dev/tty && -t 1 ]]; then
if [ "$start_setup" = true ] && [ -r /dev/tty ] && [ -w /dev/tty ] && [ -t 1 ]; then
printf 'Starting REA setup...\n'
"$install_bin" setup </dev/tty >/dev/tty
else

View File

@ -110,7 +110,7 @@
"precheck:test": "npm run deps:check",
"check:test": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run",
"precheck:pr": "npm run deps:check",
"check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run docs:check && npm run jscpd && npm run scan:todos",
"check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos",
"precheck:ci": "npm run deps:check",
"check:ci": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos && npm run verify:architecture-guards",
"build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills",
@ -128,6 +128,7 @@
"test": "npm run test:with-build",
"test:with-build": "npm run test:prepare && vitest run",
"test:run": "vitest run",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v",
"test:local": "node scripts/run-development-tests.mjs local",
"test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
"pretest:boundary": "npm run test:prepare",
@ -231,7 +232,7 @@
"docs:build": "vitepress build docs",
"docs:preview": "vitepress preview docs",
"docs:generate": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts npm run docs:prepare",
"docs:check": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run docs:check:built",
"docs:check": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run docs:check:built && npm run test:local -- tests/conformance/readmeConsistency.test.ts",
"docs:check:built": "node scripts/generate-package-metadata.mjs --check && node scripts/generate-skill-metadata.mjs --check && node scripts/generate-product-catalog.mjs --check && node scripts/generate-completion-ledger.mjs --check && node scripts/check-doc-facts.mjs && node scripts/generate-error-schema.mjs --check",
"docs:error-schema": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:error-schema",
"prepack": "node scripts/prepack.mjs",

View File

@ -0,0 +1,73 @@
import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { isAbsolute, join, relative, resolve, sep } from "node:path";
import { z } from "zod";
const root = process.cwd();
const historyDirectory = join(root, ".cache/ci-test-history");
const historyPath = join(historyDirectory, "test-results.json");
// Never let a failed or partial rerun leave a previous successful history file.
await rm(historyPath, { force: true });
const report = z
.object({
success: z.boolean(),
numFailedTests: z.number().int().nonnegative(),
testResults: z.array(
z.object({
name: z.string().min(1),
startTime: z.number().finite(),
endTime: z.number().finite(),
}),
),
})
.parse(JSON.parse(await readFile("test-results.json", "utf8")));
const files = report.testResults.map(({ name, startTime, endTime }) => {
const path = relative(root, resolve(root, name));
if (
path === "" ||
path === ".." ||
path.startsWith(".." + sep) ||
isAbsolute(path)
)
throw new Error(`Test report path is outside the checkout: ${name}`);
const duration = endTime - startTime;
if (!Number.isFinite(duration) || duration < 0)
throw new Error(`Test report has an invalid duration: ${name}`);
return { name: path.split(sep).join("/"), startTime: 0, endTime: duration };
});
const complete = process.env.SHARD_RESULT === "success";
const passed =
complete &&
process.env.MERGE_RESULT === "success" &&
report.success &&
report.numFailedTests === 0;
const summary = [
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
"",
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
passed
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
"",
"| Slowest file | Duration (ms) |",
"| --- | ---: |",
...files
.toSorted((left, right) => right.endTime - left.endTime)
.slice(0, 20)
.map(
({ name, endTime }) => `| ${name.replaceAll("|", "\\|")} | ${endTime} |`,
),
"",
].join("\n");
if (process.env.GITHUB_STEP_SUMMARY)
await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
else console.log(summary);
if (passed) {
await mkdir(historyDirectory, { recursive: true });
await writeFile(
historyPath,
JSON.stringify({ success: true, numFailedTests: 0, testResults: files }),
);
}

View File

@ -30,7 +30,7 @@ type CaptureOutcome =
| { readonly kind: "partial"; readonly partial: unknown };
const captureScenario = (
mode: "cooperative" | "ignoring" | "descendant",
mode: "cooperative" | "ignoring" | "delayed-ignoring" | "descendant",
root: string,
additions: CaptureScenario = {},
finalExitCode?: number,
@ -453,11 +453,13 @@ captureTest(
"rea-finalization-cli-cancel-",
);
const cliScenarioPath = join(cliRoot, "scenario.json");
// Match the other finalization cases' observation window. The old 300 ms
// deadline could signal the fixture before it installed its handlers.
await writeFile(
cliScenarioPath,
JSON.stringify(
captureScenario("ignoring", cliRoot, {
timeout_ms: 300,
captureScenario("delayed-ignoring", cliRoot, {
timeout_ms: 1_500,
finalization_ms: 20_000,
}),
),
@ -527,18 +529,36 @@ captureTest(
const mcpRoot = await createTestTempDirectory(
"rea-finalization-mcp-cancel-",
);
const { client } = await connectLocalToolsMcp();
const { client, call: callTool } = await connectLocalToolsMcp();
const controller = new AbortController();
const call = client.callTool(
{
name: "capture_process_scenario",
arguments: captureScenario("ignoring", mcpRoot, {
timeout_ms: 300,
finalization_ms: 20_000,
}),
},
{ signal: controller.signal },
let reportStarted: (() => void) | undefined;
const captureStarted = new Promise<void>((resolve) => {
reportStarted = resolve;
});
const call = callTool(
"capture_process_scenario",
captureScenario("delayed-ignoring", mcpRoot, {
timeout_ms: 1_500,
finalization_ms: 20_000,
}),
{ signal: controller.signal, onprogress: () => reportStarted?.() },
);
// Observe rejection immediately: a failed readiness wait must not leave a
// pending request to reject unhandled when the fixture closes its client.
// The original promise remains available for the cancellation assertion.
const observedCall = call.catch(() => undefined);
onTestFinished(async () => {
controller.abort();
await observedCall;
});
// Start the fixture's readiness wait after the server begins this capture;
// SDK schema preparation before dispatch is not target startup time.
await Promise.race([
captureStarted,
call.then(() => {
throw new Error("MCP capture completed before reporting startup");
}),
]);
const mcpFixturePid = await waitForFinalization(mcpRoot);
const mcpStarted = Date.now();
controller.abort(new Error("cancel finalization acceptance fixture"));
@ -548,7 +568,7 @@ captureTest(
).rejects.toThrow("cancel finalization acceptance fixture");
await waitForProcessExit(mcpFixturePid);
await expect(
client.listTools(),
client.listTools(undefined, { cacheMode: "refresh" }),
"the server accepts a follow-up request after cancelling the capture",
).resolves.toMatchObject({ tools: expect.any(Array) });
expect(

View File

@ -312,6 +312,14 @@ it.each([
"tests/boundary/process/darwinProcessRunTokenReader.test.ts",
{ code: true, apple: true, package: false, runtime: false },
],
[
"tests/boundary/cli/nativePlistCli.test.ts",
{ code: true, apple: true, package: false, runtime: false },
],
[
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
{ code: true, package: true, runtime: true, apple: false },
],
[
"src/process/ProcessOwnership.test.ts",
{ code: true, package: true, apple: false, evm: false, browser: false },

View File

@ -0,0 +1,174 @@
import { execFile } from "node:child_process";
import { readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import { parseTestDurations } from "../../ciTestSequencer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const execute = promisify(execFile);
const script = new URL("../../../scripts/ci/test-report.mjs", import.meta.url);
const stepsSchema = z.array(
z.object({
name: z.string().optional(),
run: z.string().optional(),
uses: z.string().optional(),
if: z.string().optional(),
env: z.record(z.string(), z.string()).optional(),
with: z.record(z.string(), z.unknown()).optional(),
}),
);
const workflow = async () =>
z
.object({
jobs: z.object({
test: z.object({ steps: stepsSchema }),
build: z.object({ steps: stepsSchema }),
docs: z.object({ steps: stepsSchema }),
static: z.object({ steps: stepsSchema }),
}),
})
.parse(
parse(
await readFile(
new URL("../../../.github/workflows/ci.yml", import.meta.url),
"utf8",
),
),
).jobs;
it("collects failed-shard diagnostics before enforcing the shard result", async () => {
const steps = (await workflow()).test!.steps;
const merge = steps.findIndex((step) => step.run?.includes("test:ci:merge"));
const upload = steps.findIndex((step) => step.with?.name === "test-results");
const gate = steps.findIndex(
(step) => step.env?.SHARD_RESULT && step.run?.includes("test "),
);
expect(merge).toBeGreaterThan(0);
expect(upload).toBeGreaterThan(merge);
expect(gate).toBeGreaterThan(upload);
expect(steps[merge]?.if).not.toContain("needs.test-shard.result");
const command = steps[gate]?.run;
if (command === undefined) throw new Error("Missing shard result gate");
for (const result of ["failure", "cancelled", "skipped", ""]) {
await expect(
execute("bash", ["-c", command], {
env: { ...process.env, CODE_REQUIRED: "true", SHARD_RESULT: result },
}),
).rejects.toMatchObject({ code: 1 });
}
await expect(
execute("bash", ["-c", command], {
env: { ...process.env, CODE_REQUIRED: "false", SHARD_RESULT: "skipped" },
}),
).resolves.toMatchObject({ stderr: "" });
});
it("keeps diagnosis available after merge failure and publishes only complete successful history", async () => {
const steps = (await workflow()).test!.steps;
const diagnosis = steps.find((step) => step.with?.name === "test-results");
expect(diagnosis?.if).toContain("!cancelled()");
expect(diagnosis?.if).not.toContain("success()");
const history = steps.find((step) => step.with?.name === "test-timings");
expect(history?.if).toContain("success()");
expect(history?.if).toContain("needs.test-shard.result == 'success'");
expect(history?.with?.path).toBe(".cache/ci-test-history/test-results.json");
const download = (await workflow()).build!.steps.find(
(step) => step.name === "Download previous timing report",
);
expect(download?.with?.name).toBe(history?.with?.name);
});
it("runs the existing onboarding and Python producer guards from their selected CI owners", async () => {
const jobs = await workflow();
const scripts = z
.object({ scripts: z.record(z.string(), z.string()) })
.parse(
JSON.parse(
await readFile(
new URL("../../../package.json", import.meta.url),
"utf8",
),
),
).scripts;
expect(
jobs.docs.steps.some((step) => step.run === "npm run docs:check"),
).toBe(true);
expect(scripts["docs:check"]).toContain(
"tests/conformance/readmeConsistency.test.ts",
);
const producer = jobs.static.steps.find(
(step) => step.run === "npm run test:python",
);
expect(producer?.if).toBe("needs.changes.outputs.code == 'true'");
expect(scripts["test:python"]).toContain(
"bridge/native/test_rea_lldb_tracer.py",
);
});
it("exports portable timing history and removes it on a failed, partial, or invalid rerun", async () => {
const root = await createTestTempDirectory("rea-ci-report-");
const summary = join(root, "summary.md");
const history = join(root, ".cache/ci-test-history/test-results.json");
const report = {
success: true,
numFailedTests: 0,
testResults: [
{
name: join(root, "tests/slow.test.ts"),
startTime: 10,
endTime: 110,
assertionResults: [{ large: "not timing data" }],
},
{ name: join(root, "tests/fast.test.ts"), startTime: 20, endTime: 30 },
],
};
const run = async (value: unknown, shards = "success", merge = "success") => {
await writeFile(join(root, "test-results.json"), JSON.stringify(value));
await writeFile(summary, "");
return execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: {
...process.env,
GITHUB_STEP_SUMMARY: summary,
SHARD_RESULT: shards,
MERGE_RESULT: merge,
},
});
};
await run(report);
const contents = await readFile(history, "utf8");
expect(contents).not.toContain("assertionResults");
expect(contents).not.toContain(root);
expect(
parseTestDurations(contents, resolve(root, "different-checkout")),
).toEqual(
new Map([
["tests/slow.test.ts", 100],
["tests/fast.test.ts", 10],
]),
);
for (const [value, shards, merge] of [
[{ ...report, success: false, numFailedTests: 1 }, "failure", "failure"],
[report, "cancelled", "success"],
[report, "success", "failure"],
] as const) {
await run(report);
await run(value, shards, merge);
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
expect(await readFile(summary, "utf8")).toContain("diagnostic only");
}
await run(report);
await expect(
run({
...report,
testResults: [{ name: "../outside.test.ts", startTime: 1, endTime: 3 }],
}),
).rejects.toMatchObject({ code: 1 });
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
});

View File

@ -334,13 +334,18 @@ describe("native macOS installer", { timeout: 20_000 }, () => {
});
describe("installer semantic version parsing", { timeout: 20_000 }, () => {
it.each(["1.2.3-01"])(
it.each([
"1.2.3-01",
"1.2.3-$(printf unexpected)",
'1.2.3-"; printf unexpected; #',
])(
"rejects malformed semantic versions before invoking npm: %s",
async (version) => {
const fixture = await createFixture();
await expect(
runInstaller(fixture, ["--version", version, "--dry-run"]),
).rejects.toMatchObject({
stdout: "",
stderr:
"REA installation failed: version must be an exact semantic version.\n",
});

View File

@ -0,0 +1,108 @@
import { execFile } from "node:child_process";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { z } from "zod";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { workflowGit } from "../../support/workflowGit.js";
const execute = promisify(execFile);
it.skipIf(process.platform === "android")(
"reruns test validation after repository or external input changes while reusing preparation",
async () => {
const config = z
.object({
tasks: z.record(z.string(), z.record(z.string(), z.unknown())),
})
.parse(
JSON.parse(
await readFile(
new URL("../../../turbo.json", import.meta.url),
"utf8",
),
),
);
const testTask = config.tasks["test:run"];
const dependencies = z.array(z.string()).parse(testTask?.dependsOn);
const root = await createTestTempDirectory("rea-test-cache-");
const external = await createTestTempDirectory("rea-test-dependency-");
const dependency = join(external, "provider.json");
const scripts: Record<string, string> = { "test:run": "node check.mjs" };
const tasks: Record<string, unknown> = { "test:run": testTask };
for (const [index, task] of dependencies.entries()) {
scripts[task] = `node prepare.mjs ${index}`;
tasks[task] = {
inputs: ["artifact-input.txt"],
outputs: [`prepared-${index}.txt`],
};
}
await writeFile(
join(root, "package.json"),
JSON.stringify({
name: "rea-cache-fixture",
version: "1.0.0",
private: true,
type: "module",
packageManager: "npm@11.16.0",
scripts,
}),
);
await writeFile(
join(root, "package-lock.json"),
JSON.stringify({
name: "rea-cache-fixture",
version: "1.0.0",
lockfileVersion: 3,
packages: { "": { name: "rea-cache-fixture", version: "1.0.0" } },
}),
);
await writeFile(join(root, "turbo.json"), JSON.stringify({ tasks }));
await writeFile(
join(root, "artifact-input.txt"),
"unchanged preparation input",
);
await writeFile(join(root, "README_zh.md"), "aligned");
await writeFile(dependency, "healthy");
await writeFile(
join(root, "prepare.mjs"),
'import { appendFileSync } from "node:fs"; appendFileSync(`prepared-${process.argv[2]}.txt`, "prepared\\n");',
);
await writeFile(
join(root, "check.mjs"),
`import { readFileSync } from "node:fs"; if(readFileSync("README_zh.md","utf8") !== "aligned" || readFileSync(${JSON.stringify(dependency)},"utf8") !== "healthy") throw new Error("current input is invalid");`,
);
await workflowGit(root, ["init", "--initial-branch=main"]);
await workflowGit(root, ["add", "."]);
await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]);
const run = () =>
execute(
process.execPath,
[
join(process.cwd(), "node_modules/turbo/bin/turbo"),
"run",
"test:run",
"--output-logs=errors-only",
],
{ cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } },
);
await run();
await writeFile(join(root, "README_zh.md"), "wrong command");
await expect(run()).rejects.toMatchObject({ code: 1 });
await writeFile(join(root, "README_zh.md"), "aligned");
await run();
await writeFile(dependency, "faulty");
await expect(run()).rejects.toMatchObject({ code: 1 });
await expect(run()).rejects.toMatchObject({ code: 1 });
await writeFile(dependency, "healthy");
await run();
for (const index of dependencies.keys()) {
expect(await readFile(join(root, `prepared-${index}.txt`), "utf8")).toBe(
"prepared\n",
);
}
},
);

View File

@ -1,4 +1,7 @@
import { Client } from "@modelcontextprotocol/client";
import {
Client,
type CallToolRequestOptions,
} from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import type { ValidateFunction } from "ajv";
@ -69,12 +72,19 @@ export async function connectLocalToolsMcp() {
validatedSchemas.add(serialized);
}
const definitions = new Map(tools.map((tool) => [tool.name, tool]));
const call = async (name: string, arguments_: Record<string, unknown>) => {
const call = async (
name: string,
arguments_: Record<string, unknown>,
options?: Pick<CallToolRequestOptions, "signal" | "onprogress">,
) => {
const definition = definitions.get(name);
if (definition !== undefined) compileValidator(definition.inputSchema);
const response = await client.callTool(
{ name, arguments: arguments_ },
definition === undefined ? undefined : { toolDefinition: definition },
{
...options,
...(definition === undefined ? {} : { toolDefinition: definition }),
},
);
if (
response.structuredContent !== undefined &&

View File

@ -1,6 +1,7 @@
import { spawn } from "node:child_process";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { fileURLToPath } from "node:url";
// Writes a periodic report, then behaves by mode:
@ -8,11 +9,16 @@ import { fileURLToPath } from "node:url";
// and exits cleanly.
// - "ignoring": stays alive and swallows SIGTERM and SIGINT, so only SIGKILL
// ends the process.
// - "delayed-ignoring": installs those handlers after a slow startup.
// - "exits": exits on its own at once.
// - "descendant": cooperates while leaving a SIGTERM-ignoring child for owned cleanup.
// An optional third argument is the exit code a cooperative finalizer uses.
const [mode, directory, finalExitCode = "0"] = process.argv.slice(2);
// Make the cancellation case exercise startup beyond its old 300 ms capture
// window deterministically, rather than relying on a busy CI runner.
if (mode === "delayed-ignoring") await delay(500);
writeFileSync(join(directory, "pid.txt"), String(process.pid));
writeFileSync(
join(directory, "periodic.json"),
@ -31,7 +37,7 @@ if (mode === "descendant") {
);
}
if (mode === "ignoring") {
if (mode === "ignoring" || mode === "delayed-ignoring") {
process.on("SIGTERM", () => {
writeFileSync(join(directory, "sigterm-received"), "received");
});

View File

@ -104,31 +104,12 @@
]
},
"test:run": {
"cache": false,
"dependsOn": [
"artifacts:mcp-catalog",
"artifacts:product-catalog",
"artifacts:managed-evidence",
"artifacts:skills"
],
"inputs": [
".nvmrc",
"package.json",
"package-lock.json",
"src/**",
"tests/**",
"scripts/**",
"bridge/**",
"vitest.config.ts",
{
"mode": "dependencyOutputs",
"globs": [
".cache/mcp-tool-catalog.json",
"skills/**",
"docs/public/product-catalog.json",
"docs/verification/managed-conformance-manifest.json",
"docs/verification/managed-conformance-ledger.json"
]
}
]
},
"docs:check:built": {